From de2914179918300429fb3634e3edc364af56b94c Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 7 Mar 2024 14:17:52 -0600 Subject: [PATCH 001/128] Initial set of archiving yamls. --- parm/archive/enkfgdas.yaml | 54 +++++++++++++ parm/archive/enkfgdas_hybvar.yaml | 9 +++ parm/archive/enkfgdas_hybvar_lobsdiag.yaml | 5 ++ parm/archive/enkfgdas_hybvar_nolobsdiag.yaml | 19 +++++ parm/archive/enkfgfs.yaml | 55 +++++++++++++ parm/archive/enkfgfs_hybvar.yaml | 9 +++ parm/archive/enkfgfs_hybvar_lobsdiag.yaml | 5 ++ parm/archive/enkfgfs_hybvar_nolobsdiag.yaml | 19 +++++ parm/archive/gdas.yaml | 84 ++++++++++++++++++++ parm/archive/gdas_aero.yaml | 4 + parm/archive/gdas_dopost.yaml | 4 + parm/archive/gdas_ice.yaml | 10 +++ parm/archive/gdas_ocn.yaml | 21 +++++ parm/archive/gdas_verfozn.yaml | 24 ++++++ parm/archive/gdas_verfrad.yaml | 11 +++ parm/archive/gdas_vminmon.yaml | 8 ++ parm/archive/gdas_wave.yaml | 10 +++ parm/archive/gfs.yaml | 40 ++++++++++ parm/archive/gfs_aero.yaml | 4 + parm/archive/gfs_archgaussian.yaml | 34 ++++++++ parm/archive/gfs_archgaussian_cycled.yaml | 10 +++ parm/archive/gfs_bufrsnd.yaml | 10 +++ parm/archive/gfs_cycled.yaml | 22 +++++ parm/archive/gfs_forecastonly.yaml | 16 ++++ parm/archive/gfs_ice.yaml | 21 +++++ parm/archive/gfs_ocn.yaml | 21 +++++ parm/archive/gfs_vminmon.yaml | 8 ++ parm/archive/gfs_wave.yaml | 8 ++ 28 files changed, 545 insertions(+) create mode 100644 parm/archive/enkfgdas.yaml create mode 100644 parm/archive/enkfgdas_hybvar.yaml create mode 100644 parm/archive/enkfgdas_hybvar_lobsdiag.yaml create mode 100644 parm/archive/enkfgdas_hybvar_nolobsdiag.yaml create mode 100644 parm/archive/enkfgfs.yaml create mode 100644 parm/archive/enkfgfs_hybvar.yaml create mode 100644 parm/archive/enkfgfs_hybvar_lobsdiag.yaml create mode 100644 parm/archive/enkfgfs_hybvar_nolobsdiag.yaml create mode 100644 parm/archive/gdas.yaml create mode 100644 parm/archive/gdas_aero.yaml create mode 100644 parm/archive/gdas_dopost.yaml create mode 100644 parm/archive/gdas_ice.yaml create mode 100644 parm/archive/gdas_ocn.yaml create mode 100644 parm/archive/gdas_verfozn.yaml create mode 100644 parm/archive/gdas_verfrad.yaml create mode 100644 parm/archive/gdas_vminmon.yaml create mode 100644 parm/archive/gdas_wave.yaml create mode 100644 parm/archive/gfs.yaml create mode 100644 parm/archive/gfs_aero.yaml create mode 100644 parm/archive/gfs_archgaussian.yaml create mode 100644 parm/archive/gfs_archgaussian_cycled.yaml create mode 100644 parm/archive/gfs_bufrsnd.yaml create mode 100644 parm/archive/gfs_cycled.yaml create mode 100644 parm/archive/gfs_forecastonly.yaml create mode 100644 parm/archive/gfs_ice.yaml create mode 100644 parm/archive/gfs_ocn.yaml create mode 100644 parm/archive/gfs_vminmon.yaml create mode 100644 parm/archive/gfs_wave.yaml diff --git a/parm/archive/enkfgdas.yaml b/parm/archive/enkfgdas.yaml new file mode 100644 index 0000000000..6616a51a1e --- /dev/null +++ b/parm/archive/enkfgdas.yaml @@ -0,0 +1,54 @@ +enkfgdas: + mandatory: + rotdir: + - 'enkfgdasfcst.log' + - 'enkfgdasepos.log' + com_atmos_history_ensstat: + forecast_hour: + - 'enkfgdas.t${cyc}z.atmf${fhr}.ensmean.nc' + - 'enkfgdas.t${cyc}z.sfcf${fhr}.ensmean.nc' + optional: + com_atmos_analysis_ensstat: + - 'enkfgdas.t${cyc}z.gsistat.ensmean' + - 'enkfgdas.t${cyc}z.cnvstat.ensmean' + - 'enkfgdas.t${cyc}z.oznstat.ensmean' + - 'enkfgdas.t${cyc}z.radstat.ensmean' + - 'enkfgdas.t${cyc}z.atmens.yaml' + - 'enkfgdas.t${cyc}z.atmanl.ensmean.nc' + - 'enkfgdas.t${cyc}z.atminc.ensmean.nc' + forecast_hour: + - 'enkfgdas.t${cyc}z.atma00${fhr}.ensmean.nc' + - 'enkfgdas.t${cyc}z.atmi00${fhr}.ensmean.nc' + com_atmos_history_ensstat: + forecast_hour: + - 'enkfgdas.t${cyc}z.atmf${fhr}.ensspread.nc' + +enkfgdas_grpN: + mandatory: + com_atmos_history_mem: + - 'enkfgdas.t${cyc}z.sfcf006.nc' + forecast_hour: + - 'enkfgdas.t${cyc}z.atmf00${fhr}.nc' + optional: + com_atmos_analysis_mem: + - 'enkfgdas.t${cyc}z.ratminc.nc' + - 'enkfgdas.t${cyc}z.atmanl.nc' + +enkfgdas_restarta_grpN: + mandatory: + com_atmos_restart_mem: + - '*0000.sfcanl_data.tile1.nc' + - '*0000.sfcanl_data.tile2.nc' + - '*0000.sfcanl_data.tile3.nc' + - '*0000.sfcanl_data.tile4.nc' + - '*0000.sfcanl_data.tile5.nc' + - '*0000.sfcanl_data.tile6.nc' + optional: + com_atmos_analysis_mem: + - 'enkfgdas.t${cyc}z.ratminc.nc' + - 'enkfgdas.t${cyc}z.ratmi00${fhr}.nc' + +enkfgdas_restartb_grpN: + mandatory: + com_atmos_restart_mem: + - '.' diff --git a/parm/archive/enkfgdas_hybvar.yaml b/parm/archive/enkfgdas_hybvar.yaml new file mode 100644 index 0000000000..c543d0fad4 --- /dev/null +++ b/parm/archive/enkfgdas_hybvar.yaml @@ -0,0 +1,9 @@ +enkfgdas: + mandatory: + rotdir: + - 'logs/${PDY}${cyc}/enkfgdasechgres.log' + full_cycle: + - 'logs/${PDY}${cyc}/enkfgdaseobs.log' + - 'logs/${PDY}${cyc}/enkfgdasecen*.log' + - 'logs/${PDY}${cyc}/enkfgdasesfc.log' + - 'logs/${PDY}${cyc}/enkfgdaseupd.log' diff --git a/parm/archive/enkfgdas_hybvar_lobsdiag.yaml b/parm/archive/enkfgdas_hybvar_lobsdiag.yaml new file mode 100644 index 0000000000..41ca274db5 --- /dev/null +++ b/parm/archive/enkfgdas_hybvar_lobsdiag.yaml @@ -0,0 +1,5 @@ +enkfgdas: + mandatory: + rotdir: + full_cycle: + - 'logs/${PDY}${cyc}/enkfgdaseomg*.log' diff --git a/parm/archive/enkfgdas_hybvar_nolobsdiag.yaml b/parm/archive/enkfgdas_hybvar_nolobsdiag.yaml new file mode 100644 index 0000000000..9f7a8d861a --- /dev/null +++ b/parm/archive/enkfgdas_hybvar_nolobsdiag.yaml @@ -0,0 +1,19 @@ +enkfgdas_grpN: + mandatory: + com_atmos_restart_mem: + - 'enkfgdas.t${cyc}z.gsistat' + optional: + com_atmos_restart_mem: + - 'enkfgdas.t${cyc}z.cnvstat' + +enkfgdas_restarta_grpN: + mandatory: + com_atmos_restart_mem: + - 'enkfgdas.t${cyc}z.abias' + - 'enkfgdas.t${cyc}z.abias_air' + - 'enkfgdas.t${cyc}z.abias_int' + - 'enkfgdas.t${cyc}z.abias_pc' + optional: + com_atmos_restart_mem: + - 'enkfgdas.t${cyc}z.radstat' + - 'enkfgdas.t${cyc}z.cnvstat' diff --git a/parm/archive/enkfgfs.yaml b/parm/archive/enkfgfs.yaml new file mode 100644 index 0000000000..f95a1991b0 --- /dev/null +++ b/parm/archive/enkfgfs.yaml @@ -0,0 +1,55 @@ +enkfgfs: + mandatory: + rotdir: + - 'enkfgfsfcst.log' + - 'enkfgfsepos.log' + com_atmos_history_ensstat: + forecast_hour: + - 'enkfgfs.t${cyc}z.atmf${fhr}.ensmean.nc' + - 'enkfgfs.t${cyc}z.sfcf${fhr}.ensmean.nc' + optional: + com_atmos_analysis_ensstat: + - 'enkfgfs.t${cyc}z.gsistat.ensmean' + - 'enkfgfs.t${cyc}z.cnvstat.ensmean' + - 'enkfgfs.t${cyc}z.oznstat.ensmean' + - 'enkfgfs.t${cyc}z.radstat.ensmean' + - 'enkfgfs.t${cyc}z.atmens.yaml' + - 'enkfgfs.t${cyc}z.atmanl.ensmean.nc' + - 'enkfgfs.t${cyc}z.atminc.ensmean.nc' + forecast_hour: + - 'enkfgfs.t${cyc}z.atma00${fhr}.ensmean.nc' + - 'enkfgfs.t${cyc}z.atmi00${fhr}.ensmean.nc' + com_atmos_history_ensstat: + forecast_hour: + - 'enkfgfs.t${cyc}z.atmf${fhr}.ensspread.nc' + +enkfgfs_grpN: + mandatory: + com_atmos_history_mem: + - 'enkfgfs.t${cyc}z.sfcf006.nc' + forecast_hour: + - 'enkfgfs.t${cyc}z.atmf00${fhr}.nc' + optional: + com_atmos_analysis_mem: + - 'enkfgfs.t${cyc}z.ratminc.nc' + - 'enkfgfs.t${cyc}z.atmanl.nc' + +enkfgfs_restarta_grpN: + mandatory: + com_atmos_restart_mem: + - '*0000.sfcanl_data.tile1.nc' + - '*0000.sfcanl_data.tile2.nc' + - '*0000.sfcanl_data.tile3.nc' + - '*0000.sfcanl_data.tile4.nc' + - '*0000.sfcanl_data.tile5.nc' + - '*0000.sfcanl_data.tile6.nc' + optional: + com_atmos_analysis_mem: + - 'enkfgfs.t${cyc}z.ratminc.nc' + forecast_hour: + - 'enkfgfs.t${cyc}z.ratmi00${fhr}.nc' + +enkfgfs_restartb_grpN: + mandatory: + com_atmos_restart: + - '.' diff --git a/parm/archive/enkfgfs_hybvar.yaml b/parm/archive/enkfgfs_hybvar.yaml new file mode 100644 index 0000000000..6e78c31790 --- /dev/null +++ b/parm/archive/enkfgfs_hybvar.yaml @@ -0,0 +1,9 @@ +enkfgfs: + mandatory: + rotdir: + - 'logs/${PDY}${cyc}/enkfgfsechgres.log' + full_cycle: + - 'logs/${PDY}${cyc}/enkfgfseobs.log' + - 'logs/${PDY}${cyc}/enkfgfsecen*.log' + - 'logs/${PDY}${cyc}/enkfgfsesfc.log' + - 'logs/${PDY}${cyc}/enkfgfseupd.log' diff --git a/parm/archive/enkfgfs_hybvar_lobsdiag.yaml b/parm/archive/enkfgfs_hybvar_lobsdiag.yaml new file mode 100644 index 0000000000..db4a69b7bb --- /dev/null +++ b/parm/archive/enkfgfs_hybvar_lobsdiag.yaml @@ -0,0 +1,5 @@ +enkfgfs: + mandatory: + rotdir: + full_cycle: + - 'logs/${PDY}${cyc}/enkfgfseomg*.log' diff --git a/parm/archive/enkfgfs_hybvar_nolobsdiag.yaml b/parm/archive/enkfgfs_hybvar_nolobsdiag.yaml new file mode 100644 index 0000000000..e5d8658ac5 --- /dev/null +++ b/parm/archive/enkfgfs_hybvar_nolobsdiag.yaml @@ -0,0 +1,19 @@ +enkfgfs_grpN: + mandatory: + com_atmos_restart_mem: + - 'enkfgfs.t${cyc}z.gsistat' + optional: + com_atmos_restart_mem: + - 'enkfgfs.t${cyc}z.cnvstat' + +enkfgfs_restarta_grpN: + mandatory: + com_atmos_restart_mem: + - 'enkfgfs.t${cyc}z.abias' + - 'enkfgfs.t${cyc}z.abias_air' + - 'enkfgfs.t${cyc}z.abias_int' + - 'enkfgfs.t${cyc}z.abias_pc' + optional: + com_atmos_restart_mem: + - 'enkfgfs.t${cyc}z.radstat' + - 'enkfgfs.t${cyc}z.cnvstat' diff --git a/parm/archive/gdas.yaml b/parm/archive/gdas.yaml new file mode 100644 index 0000000000..04fa452947 --- /dev/null +++ b/parm/archive/gdas.yaml @@ -0,0 +1,84 @@ +gdas: + mandatory: + com_atmos_grib_0p25: + - 'gdas.t${cyc}z.pgrb2.0p25.anl' + - 'gdas.t${cyc}z.pgrb2.0p25.anl.idx' + forecast_hour: + - 'gdas.t${cyc}z.pgrb2.0p25.f${fhr}' + - 'gdas.t${cyc}z.pgrb2.0p25.f${fhr}.idx' + com_atmos_grib_1p00: + - 'gdas.t${cyc}z.pgrb2.1p00.anl' + - 'gdas.t${cyc}z.pgrb2.1p00.anl.idx' + forecast_hour: + - 'gdas.t${cyc}z.pgrb2.1p00.f${fhr}' + - 'gdas.t${cyc}z.pgrb2.1p00.f${fhr}.idx' + com_atmos_analysis: + - 'gdas.t${cyc}z.atmanl.nc' + - 'gdas.t${cyc}z.sfcanl.nc' + com_atmos_master: + forecast_hour: + - 'gdas.t${cyc}z.sfluxgrbf${fhr}.grib2' + - 'gdas.t${cyc}z.sfluxgrbf${fhr}.grib2.idx' + rotdir: + - './logs/gdas*prod*.log' + com_atmos_history: + forecast_hour: + - 'gdas.t${cyc}z.atm.logf${fhr}.txt' + - 'gdas.t${cyc}z.atmf${fhr}.nc' + - 'gdas.t${cyc}z.sfcf${fhr}.nc' + optional: + com_atmos_analysis: + - 'gdas.t${cyc}z.atmvar.yaml' + - 'gdas.t${cyc}z.atmstat' + - 'gdas.t${cyc}z.gsistat' + - 'gdas.t${cyc}z.atmanl.ensres.nc' + - 'gdas.t${cyc}z.atma003.ensres.nc' + - 'gdas.t${cyc}z.atma009.ensres.nc' + - 'gdas.t${cyc}z.cnvstat' + - 'gdas.t${cyc}z.oznstat' + - 'gdas.t${cyc}z.radstat' + rotdir: + - './logs/gdasprep.log' + - './logs/gdasanal.log' + - './logs/gdasfcst.log' + - './logs/gdasverfozn.log' + - './logs/gdasverfrad.log' + - './logs/gdasvminmon.log' + com_atmos_master: + forecast_hour: + - 'gdas.t${cyc}z.sfluxgrbf${fhr}.grib2' + - 'gdas.t${cyc}z.sfluxgrbf${fhr}.grib2.idx' + +gdas_restarta: + mandatory: + com_atmos_analysis: + - 'gdas.t${cyc}z.atmi*nc' + com_atmos_restart: + - '*0000.sfcanl_data.tile1.nc' + - '*0000.sfcanl_data.tile2.nc' + - '*0000.sfcanl_data.tile3.nc' + - '*0000.sfcanl_data.tile4.nc' + - '*0000.sfcanl_data.tile5.nc' + - '*0000.sfcanl_data.tile6.nc' + optional: + com_atmos_analysis: + - 'gdas.t${cyc}z.cnvstat' + - 'gdas.t${cyc}z.radstat' + - 'gdas.t${cyc}z.abias' + - 'gdas.t${cyc}z.abias_air' + - 'gdas.t${cyc}z.abias_int' + - 'gdas.t${cyc}z.abias_pc' + - 'gdas.t${cyc}z.dtfanl.nc' + - 'gdas.t${cyc}z.loginc.txt' + - 'gdas.t${cyc}z.amsua_n19.satbias.nc4' + - 'gdas.t${cyc}z.amsua_n19.satbias_cov.nc4' + - 'gdas.t${cyc}z.amsua_n19.tlapse.txt' + com_obs: + - 'gdas.t${cyc}z.nsstbufr' + - 'gdas.t${cyc}z.prepbur' + - 'gdas.t${cyc}z.prepbufr.acft_profiles' + +gdas_restartb: + mandatory: + com_atmos_restart: + - '.' diff --git a/parm/archive/gdas_aero.yaml b/parm/archive/gdas_aero.yaml new file mode 100644 index 0000000000..fc16ab87fd --- /dev/null +++ b/parm/archive/gdas_aero.yaml @@ -0,0 +1,4 @@ +gdas: + optional: + com_chem_analysis: + - 'gdas.t${cyc}z.aerostat' diff --git a/parm/archive/gdas_dopost.yaml b/parm/archive/gdas_dopost.yaml new file mode 100644 index 0000000000..fae09dcfd9 --- /dev/null +++ b/parm/archive/gdas_dopost.yaml @@ -0,0 +1,4 @@ +gdas: + optional: + rotdir: + - './logs/${PDY}${cyc}/gdas*upp*.log' diff --git a/parm/archive/gdas_ice.yaml b/parm/archive/gdas_ice.yaml new file mode 100644 index 0000000000..09a1039100 --- /dev/null +++ b/parm/archive/gdas_ice.yaml @@ -0,0 +1,10 @@ +gdasice: + mandatory: + com_ice_history: + - 'gdas.t${cyc}z.*' + - 'ice_in' + +gdasice_restart: + mandatory: + com_ice_restart: + - '*' diff --git a/parm/archive/gdas_ocn.yaml b/parm/archive/gdas_ocn.yaml new file mode 100644 index 0000000000..6d0257cf7e --- /dev/null +++ b/parm/archive/gdas_ocn.yaml @@ -0,0 +1,21 @@ +gdasocean: + mandatory: + com_ocean_history: + - 'gdas.t${cyc}z.*' + com_ocean_input: + - '.' + +gdasocean_restart: + mandatory: + com_ocean_restart: + - '*' + com_med_restart: + - '*' + +gdasocean_analysis: + mandatory: + com_ocean_analysis: + - 'gdas.t${cyc}z.*' + - 'gdas.t??z.ocngrid.nc' + - 'diags' + - 'yaml' diff --git a/parm/archive/gdas_verfozn.yaml b/parm/archive/gdas_verfozn.yaml new file mode 100644 index 0000000000..ea46841d54 --- /dev/null +++ b/parm/archive/gdas_verfozn.yaml @@ -0,0 +1,24 @@ +gdas: + mandatory: + com_atmos_oznmon: + - 'bad_cnt.${PDY}${cyc}' + - 'bad_diag.${PDY}${cyc}' + - 'bad_pen.${PDY}${cyc}' + - 'stdout.${type}.tar.gz' + optional: + com_atmos_oznmon: + subtypes: + - 'gome_metop-b' + - 'omi_aura' + - 'ompslp_npp' + - 'ompsnp_n20' + - 'ompsnp_npp' + - 'ompstc8_n20' + - 'ompstc8_npp' + - 'sbuv2_n19' + - '${subtype}.anl.ctl' + - '${subtype}.anl.${PDY}${cyc}.ieee_d' + - '${subtype}.anl.${PDY}${cyc}.ieee_d.gz' + - '${subtype}.ges.ctl' + - '${subtype}.ges.${PDY}${cyc}.ieee_d' + - '${subtype}.ges.${PDY}${cyc}.ieee_d.gz' diff --git a/parm/archive/gdas_verfrad.yaml b/parm/archive/gdas_verfrad.yaml new file mode 100644 index 0000000000..8456b6ca2a --- /dev/null +++ b/parm/archive/gdas_verfrad.yaml @@ -0,0 +1,11 @@ +gdas: + mandatory: + com_atmos_radmon: + - 'bad_diag.${PDY}${cyc}' + - 'bad_pen.${PDY}${cyc}' + - 'low_count.${PDY}${cyc}' + - 'radmon_angle.tar.gz' + - 'radmon_bcoef.tar.gz' + - 'radmon_bcor.tar.gz' + - 'radmon_time.tar.gz' + - 'warning.${PDY}${cyc}' diff --git a/parm/archive/gdas_vminmon.yaml b/parm/archive/gdas_vminmon.yaml new file mode 100644 index 0000000000..f6644b0712 --- /dev/null +++ b/parm/archive/gdas_vminmon.yaml @@ -0,0 +1,8 @@ +gdas: + mandatory: + com_atmos_minmon: + - '${PDY}${cyc}.costs.txt' + - '${PDY}${cyc}.cost_terms.txt' + - '${PDY}${cyc}.gnorms.ieee_d' + - '${PDY}${cyc}.reduction.ieee_d' + - 'gnorm_data.txt' diff --git a/parm/archive/gdas_wave.yaml b/parm/archive/gdas_wave.yaml new file mode 100644 index 0000000000..aa10e6d0c6 --- /dev/null +++ b/parm/archive/gdas_wave.yaml @@ -0,0 +1,10 @@ +gdaswave: + mandatory: + com_wave_grid: + - 'gdaswave.t${cyc}z.*' + com_wave_station: + - 'gdaswave.t${cyc}z.*' +gdaswave_restart: + mandatory: + com_wave_restart: + - '*' diff --git a/parm/archive/gfs.yaml b/parm/archive/gfs.yaml new file mode 100644 index 0000000000..8e06a30652 --- /dev/null +++ b/parm/archive/gfs.yaml @@ -0,0 +1,40 @@ +gfsa: + mandatory: + rotdir: + - './logs/${PDY}${cyc}/gfs!(arch).log' + com_conf: + - 'ufs.input.nml' + com_atmos_grib_0p25: + - 'gfs.t${cyc}z.pgrb2.0p25.anl' + - 'gfs.t${cyc}z.pgrb2.0p25.anl.idx' + - 'gfs.t${cyc}z.pgrb2.0p25.f${fhr}' + - 'gfs.t${cyc}z.pgrb2.0p25.f${fhr}.idx' + com_atmos_history: + - 'gfs.t${cyc}z.atm.logf${fhr}.txt' + optional: + com_atmos_track: + - 'avno.t${cyc}z.cyclone.trackatcfunix' + - 'avnop.t${cyc}z.cyclone.trackatcfunix' + - 'trak.gfso.atcfunix.${PDY}${cyc}' + - 'trak.gfso.atcfunix.altg.${PDY}${cyc}' + - 'storms.gfso.atcf_gen.${PDY}${cyc}' + - 'storms.gfso.atcf_gen.altg.${PDY}${cyc}' + +gfsb: + mandatory: + com_atmos_grib_0p50: + - 'gfs.t${cyc}z.pgrb2.0p50.anl' + - 'gfs.t${cyc}z.pgrb2.0p50.anl.idx' + com_atmos_grib_1p00: + - 'gfs.t${cyc}z.pgrb2.1p00.anl' + - 'gfs.t${cyc}z.pgrb2.1p00.anl.idx' + optional: + com_atmos_grib_0p50: + forecast_hour: + - 'gfs.t${cyc}z.pgrb2.0p50.f${fhr}' + - 'gfs.t${cyc}z.pgrb2.0p50.f${fhr}.idx' + com_atmos_grib_1p00: + forecast_hour: + - 'gfs.t${cyc}z.pgrb2.1p00.f${fhr}' + - 'gfs.t${cyc}z.pgrb2.1p00.f${fhr}.idx' + diff --git a/parm/archive/gfs_aero.yaml b/parm/archive/gfs_aero.yaml new file mode 100644 index 0000000000..780d1f4548 --- /dev/null +++ b/parm/archive/gfs_aero.yaml @@ -0,0 +1,4 @@ +chem: + mandatory: + com_checm_history: + - 'gocart*' diff --git a/parm/archive/gfs_archgaussian.yaml b/parm/archive/gfs_archgaussian.yaml new file mode 100644 index 0000000000..baf5a5248e --- /dev/null +++ b/parm/archive/gfs_archgaussian.yaml @@ -0,0 +1,34 @@ +gfs_pgrb2b: + mandatory: + com_atmos_grib_0p25: + - 'gfs.t${cyc}z.pgrb2b.0p25.anl' + - 'gfs.t${cyc}z.pgrb2b.0p25.anl.idx' + com_atmos_grib_1p00: + - 'gfs.t${cyc}z.pgrb2b.1p00.anl' + - 'gfs.t${cyc}z.pgrb2b.1p00.anl.idx' + +gfs_netcdfb: + mandatory: + com_atmos_history: + forecast_hour: + - 'gfs.t${cyc}z.atmf${fhr}.nc' + - 'gfs.t${cyc}z.sfcf${fhr}.nc' + +gfs_flux: + mandatory: + com_atmos_master: + forecast_hour: + - 'gfs.t${cyc}z.sfluxgrbf${fhr}.grib2' + - 'gfs.t${cyc}z.sfluxgrbf${fhr}.grib2.idx' + +gfs_pgrb2b: + mandatory: + com_atmos_grib_0p25: + forecast_hour: + - 'gfs.t${cyc}z.pgrb2b.0p25.f${fhr}' + - 'gfs.t${cyc}z.pgrb2b.0p25.f${fhr}.idx' + optional: + com_atmos_grib_1p00: + forecast_hour: + - 'gfs.t${cyc}z.pgrb2b.1p00.f${fhr}' + - 'gfs.t${cyc}z.pgrb2b.1p00.f${fhr}.idx' diff --git a/parm/archive/gfs_archgaussian_cycled.yaml b/parm/archive/gfs_archgaussian_cycled.yaml new file mode 100644 index 0000000000..5b921da680 --- /dev/null +++ b/parm/archive/gfs_archgaussian_cycled.yaml @@ -0,0 +1,10 @@ +gfs_netcdfa: + mandatory: + com_atmos_analysis: + - 'gfs.t${cyc}z.atmanl.nc' + - 'gfs.t${cyc}z.sfcanl.nc' + - 'gfs.t${cyc}z.atmi*.nc' + optional: + com_atmos_analysis: + - 'gfs.t${cyc}z.dtfanl.nc' + - 'gfs.t${cyc}z.loginc.txt' diff --git a/parm/archive/gfs_bufrsnd.yaml b/parm/archive/gfs_bufrsnd.yaml new file mode 100644 index 0000000000..52faf7cb78 --- /dev/null +++ b/parm/archive/gfs_bufrsnd.yaml @@ -0,0 +1,10 @@ +gfs_downstream: + mandatory: + com_atmos_gempak: + - 'gfs_${PDY}${cyc}.sfc' + - 'gfs_${PDY}${cyc}.snd' + com_atmos_wmo: + - 'gfs_collective*.postsnd_${cyc}' + com_atmos_bufr: + - 'bufr.t${cyc}z' + - 'gfs.t${cyc}z.bufrsnd.tar.gz' diff --git a/parm/archive/gfs_cycled.yaml b/parm/archive/gfs_cycled.yaml new file mode 100644 index 0000000000..242650f27e --- /dev/null +++ b/parm/archive/gfs_cycled.yaml @@ -0,0 +1,22 @@ +gfsa: + mandatory: + + optional: + com_atmos_analysis: + - 'gfs.t${cyc}z.gsistat' + - 'gfs.t${cyc}z.atmvar.yaml' + - 'gfs.t${cyc}z.atmstat' + com_obs: + - 'gfs.t${cyc}z.nsstbufr' + - 'gfs.t${cyc}z.prepbufr' + - 'gfs.t${cyc}z.prepbufr.acft_profiles' + +gfs_restarta: + mandatory: + com_atmos_restart: + - '*0000.sfcanl_data.tile1.nc' + - '*0000.sfcanl_data.tile2.nc' + - '*0000.sfcanl_data.tile3.nc' + - '*0000.sfcanl_data.tile4.nc' + - '*0000.sfcanl_data.tile5.nc' + - '*0000.sfcanl_data.tile6.nc' diff --git a/parm/archive/gfs_forecastonly.yaml b/parm/archive/gfs_forecastonly.yaml new file mode 100644 index 0000000000..599144d4a5 --- /dev/null +++ b/parm/archive/gfs_forecastonly.yaml @@ -0,0 +1,16 @@ +gfs_restarta: + mandatory: + com_atmos_input: + - 'gfs_ctrl.nc' + - 'gfs_data.tile1.nc' + - 'gfs_data.tile2.nc' + - 'gfs_data.tile3.nc' + - 'gfs_data.tile4.nc' + - 'gfs_data.tile5.nc' + - 'gfs_data.tile6.nc' + - 'sfc_data.tile1.nc' + - 'sfc_data.tile2.nc' + - 'sfc_data.tile3.nc' + - 'sfc_data.tile4.nc' + - 'sfc_data.tile5.nc' + - 'sfc_data.tile6.nc' diff --git a/parm/archive/gfs_ice.yaml b/parm/archive/gfs_ice.yaml new file mode 100644 index 0000000000..ab847d5d68 --- /dev/null +++ b/parm/archive/gfs_ice.yaml @@ -0,0 +1,21 @@ +ice_grib2: + optional: + com_ice_grib: + - '5p00/gfs.ice.t${cyc}z.5p00.f*.grib2' + - '5p00/gfs.ice.t${cyc}z.5p00.f*.grib2.idx' + - '1p00/gfs.ice.t${cyc}z.1p00.f*.grib2' + - '1p00/gfs.ice.t${cyc}z.1p00.f*.grib2.idx' + - '0p25/gfs.ice.t${cyc}z.0p25.f*.grib2' + - '0p25/gfs.ice.t${cyc}z.0p25.f*.grib2.idx' + +ice_6hravg: + mandatory: + com_ice_history: + forecast_hour: + - 'gfs.ice.t${cyc}z.6hr_avg.f*.nc' + +ice_daily: + mandatory: + com_ice_history: + forecast_hour: + - 'gfs.ice.t${cyc}z.daily.f*.nc' diff --git a/parm/archive/gfs_ocn.yaml b/parm/archive/gfs_ocn.yaml new file mode 100644 index 0000000000..1517358e10 --- /dev/null +++ b/parm/archive/gfs_ocn.yaml @@ -0,0 +1,21 @@ +ocean_grib2: + optional: + com_ocean_grib: + - '5p00/gfs.ocean.t${cyc}z.5p00.f*.grib2' + - '5p00/gfs.ocean.t${cyc}z.5p00.f*.grib2.idx' + - '1p00/gfs.ocean.t${cyc}z.1p00.f*.grib2' + - '1p00/gfs.ocean.t${cyc}z.1p00.f*.grib2.idx' + - '0p25/gfs.ocean.t${cyc}z.0p25.f*.grib2' + - '0p25/gfs.ocean.t${cyc}z.0p25.f*.grib2.idx' + +ocean_6hravg: + mandatory: + com_ocean_history: + forecast_hour: + - 'gfs.ocean.t${cyc}z.6hr_avg.f*.nc' + +ocean_daily: + mandatory: + com_ocean_history: + forecast_hour: + - 'gfs.ocean.t${cyc}z.daily.f*.nc' diff --git a/parm/archive/gfs_vminmon.yaml b/parm/archive/gfs_vminmon.yaml new file mode 100644 index 0000000000..5a72af48d0 --- /dev/null +++ b/parm/archive/gfs_vminmon.yaml @@ -0,0 +1,8 @@ +gfsa: + mandatory: + com_atmos_minmon: + - '${PDY}${cyc}.costs.txt' + - '${PDY}${cyc}.cost_terms.txt' + - '${PDY}${cyc}.gnorms.ieee_d' + - '${PDY}${cyc}.reduction.ieee_d' + - 'gnorm_data.txt' diff --git a/parm/archive/gfs_wave.yaml b/parm/archive/gfs_wave.yaml new file mode 100644 index 0000000000..a0a529a5cc --- /dev/null +++ b/parm/archive/gfs_wave.yaml @@ -0,0 +1,8 @@ +gfswave: + mandatory: + com_wave_history: + - 'ww3_multi*' + com_wave_grid: + - 'gfswave.t${cyc}z.*' + com_wave_station: + - 'gfswave.t${cyc}z.*' From 404a46b8acaaad263cb8dd22e7e06ca9ae5b79e3 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 12 Mar 2024 08:58:04 -0500 Subject: [PATCH 002/128] rework gdas.yaml to Jinja template --- parm/archive/gdas.yaml | 145 ++++++++++++++++++----------------------- 1 file changed, 64 insertions(+), 81 deletions(-) diff --git a/parm/archive/gdas.yaml b/parm/archive/gdas.yaml index 04fa452947..1be0a56873 100644 --- a/parm/archive/gdas.yaml +++ b/parm/archive/gdas.yaml @@ -1,84 +1,67 @@ gdas: mandatory: - com_atmos_grib_0p25: - - 'gdas.t${cyc}z.pgrb2.0p25.anl' - - 'gdas.t${cyc}z.pgrb2.0p25.anl.idx' - forecast_hour: - - 'gdas.t${cyc}z.pgrb2.0p25.f${fhr}' - - 'gdas.t${cyc}z.pgrb2.0p25.f${fhr}.idx' - com_atmos_grib_1p00: - - 'gdas.t${cyc}z.pgrb2.1p00.anl' - - 'gdas.t${cyc}z.pgrb2.1p00.anl.idx' - forecast_hour: - - 'gdas.t${cyc}z.pgrb2.1p00.f${fhr}' - - 'gdas.t${cyc}z.pgrb2.1p00.f${fhr}.idx' - com_atmos_analysis: - - 'gdas.t${cyc}z.atmanl.nc' - - 'gdas.t${cyc}z.sfcanl.nc' - com_atmos_master: - forecast_hour: - - 'gdas.t${cyc}z.sfluxgrbf${fhr}.grib2' - - 'gdas.t${cyc}z.sfluxgrbf${fhr}.grib2.idx' - rotdir: - - './logs/gdas*prod*.log' - com_atmos_history: - forecast_hour: - - 'gdas.t${cyc}z.atm.logf${fhr}.txt' - - 'gdas.t${cyc}z.atmf${fhr}.nc' - - 'gdas.t${cyc}z.sfcf${fhr}.nc' + - '{{ COM_ATMOS_GRIB_0P25 }}/gdas.t{{ cyc }}z.pgrb2.0p25.anl' + - '{{ COM_ATMOS_GRIB_0P25 }}/gdas.t{{ cyc }}z.pgrb2.0p25.anl.idx' + - '{{ COM_ATMOS_GRIB_0P25 }}/gdas.t{{ cyc }}z.pgrb2.0p25.f{{ fhr }}' + - '{{ COM_ATMOS_GRIB_0P25 }}/gdas.t{{ cyc }}z.pgrb2.0p25.f{{ fhr }}.idx' + - '{{ COM_ATMOS_GRIB_1P00 }}/gdas.t{{ cyc }}z.pgrb2.1p00.anl' + - '{{ COM_ATMOS_GRIB_1P00 }}/gdas.t{{ cyc }}z.pgrb2.1p00.anl.idx' + - '{{ COM_ATMOS_GRIB_1P00 }}/gdas.t{{ cyc }}z.pgrb2.1p00.f{{ fhr }}' + - '{{ COM_ATMOS_GRIB_1P00 }}/gdas.t{{ cyc }}z.pgrb2.1p00.f{{ fhr }}.idx' + - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.atmanl.nc' + - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.sfcanl.nc' + - '{{ COM_ATMOS_MASTER }}/gdas.t{{ cyc }}z.sfluxgrbf{{ fhr }}.grib2' + - '{{ COM_ATMOS_MASTER }}/gdas.t{{ cyc }}z.sfluxgrbf{{ fhr }}.grib2.idx' + - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdas*prod*.log' + - '{{ COM_ATMOS_HISTORY }}/gdas.t{{ cyc }}z.atm.logf{{ fhr }}.txt' + - '{{ COM_ATMOS_HISTORY }}/gdas.t{{ cyc }}z.atmf{{ fhr }}.nc' + - '{{ COM_ATMOS_HISTORY }}/gdas.t{{ cyc }}z.sfcf{{ fhr }}.nc' + {% if DO_VERFOZN = "YES" %} + - '{{ COM_ATMOS_OZNMON }}/bad_cnt.{{ PDY }}{{ cyc }}' + - '{{ COM_ATMOS_OZNMON }}/bad_diag.{{ PDY }}{{ cyc }}' + - '{{ COM_ATMOS_OZNMON }}/bad_pen.{{ PDY }}{{ cyc }}' + {% set oznmon_types = [ + 'gome_metop-b', 'omi_aura', 'ompslp_npp', 'ompsnp_n20', + 'ompsnp_npp', 'ompstc8_n20', 'ompstc8_npp', 'sbuv2_n19' + ] %} + {% for type in oznmon_types %} + - '{{ COM_ATMOS_OZNMON }}/stdout.{{ type }}.tar.gz' + {% endfor %} + {% endif %} optional: - com_atmos_analysis: - - 'gdas.t${cyc}z.atmvar.yaml' - - 'gdas.t${cyc}z.atmstat' - - 'gdas.t${cyc}z.gsistat' - - 'gdas.t${cyc}z.atmanl.ensres.nc' - - 'gdas.t${cyc}z.atma003.ensres.nc' - - 'gdas.t${cyc}z.atma009.ensres.nc' - - 'gdas.t${cyc}z.cnvstat' - - 'gdas.t${cyc}z.oznstat' - - 'gdas.t${cyc}z.radstat' - rotdir: - - './logs/gdasprep.log' - - './logs/gdasanal.log' - - './logs/gdasfcst.log' - - './logs/gdasverfozn.log' - - './logs/gdasverfrad.log' - - './logs/gdasvminmon.log' - com_atmos_master: - forecast_hour: - - 'gdas.t${cyc}z.sfluxgrbf${fhr}.grib2' - - 'gdas.t${cyc}z.sfluxgrbf${fhr}.grib2.idx' - -gdas_restarta: - mandatory: - com_atmos_analysis: - - 'gdas.t${cyc}z.atmi*nc' - com_atmos_restart: - - '*0000.sfcanl_data.tile1.nc' - - '*0000.sfcanl_data.tile2.nc' - - '*0000.sfcanl_data.tile3.nc' - - '*0000.sfcanl_data.tile4.nc' - - '*0000.sfcanl_data.tile5.nc' - - '*0000.sfcanl_data.tile6.nc' - optional: - com_atmos_analysis: - - 'gdas.t${cyc}z.cnvstat' - - 'gdas.t${cyc}z.radstat' - - 'gdas.t${cyc}z.abias' - - 'gdas.t${cyc}z.abias_air' - - 'gdas.t${cyc}z.abias_int' - - 'gdas.t${cyc}z.abias_pc' - - 'gdas.t${cyc}z.dtfanl.nc' - - 'gdas.t${cyc}z.loginc.txt' - - 'gdas.t${cyc}z.amsua_n19.satbias.nc4' - - 'gdas.t${cyc}z.amsua_n19.satbias_cov.nc4' - - 'gdas.t${cyc}z.amsua_n19.tlapse.txt' - com_obs: - - 'gdas.t${cyc}z.nsstbufr' - - 'gdas.t${cyc}z.prepbur' - - 'gdas.t${cyc}z.prepbufr.acft_profiles' - -gdas_restartb: - mandatory: - com_atmos_restart: - - '.' + - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.atmvar.yaml' + - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.atmstat' + - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.gsistat' + - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.atmanl.ensres.nc' + - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.atma003.ensres.nc' + - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.atma009.ensres.nc' + - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.cnvstat' + - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.oznstat' + - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.radstat' + - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdasprep.log' + - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdasanal.log' + - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdasfcst.log' + - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdasverfozn.log' + - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdasverfrad.log' + - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdasvminmon.log' + - '{{ COM_ATMOS_MASTER }}/gdas.t{{ cyc }}z.sfluxgrbf{{ fhr }}.grib2' + - '{{ COM_ATMOS_MASTER }}/gdas.t{{ cyc }}z.sfluxgrbf{{ fhr }}.grib2.idx' + {% if DO_AERO = "YES" %} + - '{{ COM_CHEM_ANALYSIS }}/gdas.t{{ cyc }}z.aerostat' + {% endif } + {% if WRITE_DOPOST = ".true." %} + - '{{ ROTDIR }}/{{ PDY }}{{ cyc }}/gdas*upp*.log' + {% endif } + {% if DO_VERFOZN = "YES" %} + {% set oznmon_types = [ + 'gome_metop-b', 'omi_aura', 'ompslp_npp', 'ompsnp_n20', + 'ompsnp_npp', 'ompstc8_n20', 'ompstc8_npp', 'sbuv2_n19' + ] %} + {% for type in oznmon_types %} + - '{{ type }}.anl.ctl' + - '{{ type }}.anl.{{ PDY }}{{ cyc }}.ieee_d' + - '{{ type }}.anl.{{ PDY }}{{ cyc }}.ieee_d.gz' + - '{{ type }}.ges.ctl' + - '{{ type }}.ges.{{ PDY }}{{ cyc }}.ieee_d' + - '{{ type }}.ges.{{ PDY }}{{ cyc }}.ieee_d.gz' + {% endfor %} From 172dc6808da848d3fd5495c9399d46ab298089c2 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 12 Mar 2024 12:39:54 -0500 Subject: [PATCH 003/128] Restart with a blank slate; rename gdas.yaml. --- parm/archive/enkfgdas.yaml | 54 ------------------- parm/archive/enkfgdas_hybvar.yaml | 9 ---- parm/archive/enkfgdas_hybvar_lobsdiag.yaml | 5 -- parm/archive/enkfgdas_hybvar_nolobsdiag.yaml | 19 ------- parm/archive/enkfgfs.yaml | 55 -------------------- parm/archive/enkfgfs_hybvar.yaml | 9 ---- parm/archive/enkfgfs_hybvar_lobsdiag.yaml | 5 -- parm/archive/enkfgfs_hybvar_nolobsdiag.yaml | 19 ------- parm/archive/{gdas.yaml => gdas.yaml.j2} | 0 parm/archive/gdas_aero.yaml | 4 -- parm/archive/gdas_dopost.yaml | 4 -- parm/archive/gdas_ice.yaml | 10 ---- parm/archive/gdas_ocn.yaml | 21 -------- parm/archive/gdas_verfozn.yaml | 24 --------- parm/archive/gdas_verfrad.yaml | 11 ---- parm/archive/gdas_vminmon.yaml | 8 --- parm/archive/gdas_wave.yaml | 10 ---- parm/archive/gfs.yaml | 40 -------------- parm/archive/gfs_aero.yaml | 4 -- parm/archive/gfs_archgaussian.yaml | 34 ------------ parm/archive/gfs_archgaussian_cycled.yaml | 10 ---- parm/archive/gfs_bufrsnd.yaml | 10 ---- parm/archive/gfs_cycled.yaml | 22 -------- parm/archive/gfs_forecastonly.yaml | 16 ------ parm/archive/gfs_ice.yaml | 21 -------- parm/archive/gfs_ocn.yaml | 21 -------- parm/archive/gfs_vminmon.yaml | 8 --- parm/archive/gfs_wave.yaml | 8 --- 28 files changed, 461 deletions(-) delete mode 100644 parm/archive/enkfgdas.yaml delete mode 100644 parm/archive/enkfgdas_hybvar.yaml delete mode 100644 parm/archive/enkfgdas_hybvar_lobsdiag.yaml delete mode 100644 parm/archive/enkfgdas_hybvar_nolobsdiag.yaml delete mode 100644 parm/archive/enkfgfs.yaml delete mode 100644 parm/archive/enkfgfs_hybvar.yaml delete mode 100644 parm/archive/enkfgfs_hybvar_lobsdiag.yaml delete mode 100644 parm/archive/enkfgfs_hybvar_nolobsdiag.yaml rename parm/archive/{gdas.yaml => gdas.yaml.j2} (100%) delete mode 100644 parm/archive/gdas_aero.yaml delete mode 100644 parm/archive/gdas_dopost.yaml delete mode 100644 parm/archive/gdas_ice.yaml delete mode 100644 parm/archive/gdas_ocn.yaml delete mode 100644 parm/archive/gdas_verfozn.yaml delete mode 100644 parm/archive/gdas_verfrad.yaml delete mode 100644 parm/archive/gdas_vminmon.yaml delete mode 100644 parm/archive/gdas_wave.yaml delete mode 100644 parm/archive/gfs.yaml delete mode 100644 parm/archive/gfs_aero.yaml delete mode 100644 parm/archive/gfs_archgaussian.yaml delete mode 100644 parm/archive/gfs_archgaussian_cycled.yaml delete mode 100644 parm/archive/gfs_bufrsnd.yaml delete mode 100644 parm/archive/gfs_cycled.yaml delete mode 100644 parm/archive/gfs_forecastonly.yaml delete mode 100644 parm/archive/gfs_ice.yaml delete mode 100644 parm/archive/gfs_ocn.yaml delete mode 100644 parm/archive/gfs_vminmon.yaml delete mode 100644 parm/archive/gfs_wave.yaml diff --git a/parm/archive/enkfgdas.yaml b/parm/archive/enkfgdas.yaml deleted file mode 100644 index 6616a51a1e..0000000000 --- a/parm/archive/enkfgdas.yaml +++ /dev/null @@ -1,54 +0,0 @@ -enkfgdas: - mandatory: - rotdir: - - 'enkfgdasfcst.log' - - 'enkfgdasepos.log' - com_atmos_history_ensstat: - forecast_hour: - - 'enkfgdas.t${cyc}z.atmf${fhr}.ensmean.nc' - - 'enkfgdas.t${cyc}z.sfcf${fhr}.ensmean.nc' - optional: - com_atmos_analysis_ensstat: - - 'enkfgdas.t${cyc}z.gsistat.ensmean' - - 'enkfgdas.t${cyc}z.cnvstat.ensmean' - - 'enkfgdas.t${cyc}z.oznstat.ensmean' - - 'enkfgdas.t${cyc}z.radstat.ensmean' - - 'enkfgdas.t${cyc}z.atmens.yaml' - - 'enkfgdas.t${cyc}z.atmanl.ensmean.nc' - - 'enkfgdas.t${cyc}z.atminc.ensmean.nc' - forecast_hour: - - 'enkfgdas.t${cyc}z.atma00${fhr}.ensmean.nc' - - 'enkfgdas.t${cyc}z.atmi00${fhr}.ensmean.nc' - com_atmos_history_ensstat: - forecast_hour: - - 'enkfgdas.t${cyc}z.atmf${fhr}.ensspread.nc' - -enkfgdas_grpN: - mandatory: - com_atmos_history_mem: - - 'enkfgdas.t${cyc}z.sfcf006.nc' - forecast_hour: - - 'enkfgdas.t${cyc}z.atmf00${fhr}.nc' - optional: - com_atmos_analysis_mem: - - 'enkfgdas.t${cyc}z.ratminc.nc' - - 'enkfgdas.t${cyc}z.atmanl.nc' - -enkfgdas_restarta_grpN: - mandatory: - com_atmos_restart_mem: - - '*0000.sfcanl_data.tile1.nc' - - '*0000.sfcanl_data.tile2.nc' - - '*0000.sfcanl_data.tile3.nc' - - '*0000.sfcanl_data.tile4.nc' - - '*0000.sfcanl_data.tile5.nc' - - '*0000.sfcanl_data.tile6.nc' - optional: - com_atmos_analysis_mem: - - 'enkfgdas.t${cyc}z.ratminc.nc' - - 'enkfgdas.t${cyc}z.ratmi00${fhr}.nc' - -enkfgdas_restartb_grpN: - mandatory: - com_atmos_restart_mem: - - '.' diff --git a/parm/archive/enkfgdas_hybvar.yaml b/parm/archive/enkfgdas_hybvar.yaml deleted file mode 100644 index c543d0fad4..0000000000 --- a/parm/archive/enkfgdas_hybvar.yaml +++ /dev/null @@ -1,9 +0,0 @@ -enkfgdas: - mandatory: - rotdir: - - 'logs/${PDY}${cyc}/enkfgdasechgres.log' - full_cycle: - - 'logs/${PDY}${cyc}/enkfgdaseobs.log' - - 'logs/${PDY}${cyc}/enkfgdasecen*.log' - - 'logs/${PDY}${cyc}/enkfgdasesfc.log' - - 'logs/${PDY}${cyc}/enkfgdaseupd.log' diff --git a/parm/archive/enkfgdas_hybvar_lobsdiag.yaml b/parm/archive/enkfgdas_hybvar_lobsdiag.yaml deleted file mode 100644 index 41ca274db5..0000000000 --- a/parm/archive/enkfgdas_hybvar_lobsdiag.yaml +++ /dev/null @@ -1,5 +0,0 @@ -enkfgdas: - mandatory: - rotdir: - full_cycle: - - 'logs/${PDY}${cyc}/enkfgdaseomg*.log' diff --git a/parm/archive/enkfgdas_hybvar_nolobsdiag.yaml b/parm/archive/enkfgdas_hybvar_nolobsdiag.yaml deleted file mode 100644 index 9f7a8d861a..0000000000 --- a/parm/archive/enkfgdas_hybvar_nolobsdiag.yaml +++ /dev/null @@ -1,19 +0,0 @@ -enkfgdas_grpN: - mandatory: - com_atmos_restart_mem: - - 'enkfgdas.t${cyc}z.gsistat' - optional: - com_atmos_restart_mem: - - 'enkfgdas.t${cyc}z.cnvstat' - -enkfgdas_restarta_grpN: - mandatory: - com_atmos_restart_mem: - - 'enkfgdas.t${cyc}z.abias' - - 'enkfgdas.t${cyc}z.abias_air' - - 'enkfgdas.t${cyc}z.abias_int' - - 'enkfgdas.t${cyc}z.abias_pc' - optional: - com_atmos_restart_mem: - - 'enkfgdas.t${cyc}z.radstat' - - 'enkfgdas.t${cyc}z.cnvstat' diff --git a/parm/archive/enkfgfs.yaml b/parm/archive/enkfgfs.yaml deleted file mode 100644 index f95a1991b0..0000000000 --- a/parm/archive/enkfgfs.yaml +++ /dev/null @@ -1,55 +0,0 @@ -enkfgfs: - mandatory: - rotdir: - - 'enkfgfsfcst.log' - - 'enkfgfsepos.log' - com_atmos_history_ensstat: - forecast_hour: - - 'enkfgfs.t${cyc}z.atmf${fhr}.ensmean.nc' - - 'enkfgfs.t${cyc}z.sfcf${fhr}.ensmean.nc' - optional: - com_atmos_analysis_ensstat: - - 'enkfgfs.t${cyc}z.gsistat.ensmean' - - 'enkfgfs.t${cyc}z.cnvstat.ensmean' - - 'enkfgfs.t${cyc}z.oznstat.ensmean' - - 'enkfgfs.t${cyc}z.radstat.ensmean' - - 'enkfgfs.t${cyc}z.atmens.yaml' - - 'enkfgfs.t${cyc}z.atmanl.ensmean.nc' - - 'enkfgfs.t${cyc}z.atminc.ensmean.nc' - forecast_hour: - - 'enkfgfs.t${cyc}z.atma00${fhr}.ensmean.nc' - - 'enkfgfs.t${cyc}z.atmi00${fhr}.ensmean.nc' - com_atmos_history_ensstat: - forecast_hour: - - 'enkfgfs.t${cyc}z.atmf${fhr}.ensspread.nc' - -enkfgfs_grpN: - mandatory: - com_atmos_history_mem: - - 'enkfgfs.t${cyc}z.sfcf006.nc' - forecast_hour: - - 'enkfgfs.t${cyc}z.atmf00${fhr}.nc' - optional: - com_atmos_analysis_mem: - - 'enkfgfs.t${cyc}z.ratminc.nc' - - 'enkfgfs.t${cyc}z.atmanl.nc' - -enkfgfs_restarta_grpN: - mandatory: - com_atmos_restart_mem: - - '*0000.sfcanl_data.tile1.nc' - - '*0000.sfcanl_data.tile2.nc' - - '*0000.sfcanl_data.tile3.nc' - - '*0000.sfcanl_data.tile4.nc' - - '*0000.sfcanl_data.tile5.nc' - - '*0000.sfcanl_data.tile6.nc' - optional: - com_atmos_analysis_mem: - - 'enkfgfs.t${cyc}z.ratminc.nc' - forecast_hour: - - 'enkfgfs.t${cyc}z.ratmi00${fhr}.nc' - -enkfgfs_restartb_grpN: - mandatory: - com_atmos_restart: - - '.' diff --git a/parm/archive/enkfgfs_hybvar.yaml b/parm/archive/enkfgfs_hybvar.yaml deleted file mode 100644 index 6e78c31790..0000000000 --- a/parm/archive/enkfgfs_hybvar.yaml +++ /dev/null @@ -1,9 +0,0 @@ -enkfgfs: - mandatory: - rotdir: - - 'logs/${PDY}${cyc}/enkfgfsechgres.log' - full_cycle: - - 'logs/${PDY}${cyc}/enkfgfseobs.log' - - 'logs/${PDY}${cyc}/enkfgfsecen*.log' - - 'logs/${PDY}${cyc}/enkfgfsesfc.log' - - 'logs/${PDY}${cyc}/enkfgfseupd.log' diff --git a/parm/archive/enkfgfs_hybvar_lobsdiag.yaml b/parm/archive/enkfgfs_hybvar_lobsdiag.yaml deleted file mode 100644 index db4a69b7bb..0000000000 --- a/parm/archive/enkfgfs_hybvar_lobsdiag.yaml +++ /dev/null @@ -1,5 +0,0 @@ -enkfgfs: - mandatory: - rotdir: - full_cycle: - - 'logs/${PDY}${cyc}/enkfgfseomg*.log' diff --git a/parm/archive/enkfgfs_hybvar_nolobsdiag.yaml b/parm/archive/enkfgfs_hybvar_nolobsdiag.yaml deleted file mode 100644 index e5d8658ac5..0000000000 --- a/parm/archive/enkfgfs_hybvar_nolobsdiag.yaml +++ /dev/null @@ -1,19 +0,0 @@ -enkfgfs_grpN: - mandatory: - com_atmos_restart_mem: - - 'enkfgfs.t${cyc}z.gsistat' - optional: - com_atmos_restart_mem: - - 'enkfgfs.t${cyc}z.cnvstat' - -enkfgfs_restarta_grpN: - mandatory: - com_atmos_restart_mem: - - 'enkfgfs.t${cyc}z.abias' - - 'enkfgfs.t${cyc}z.abias_air' - - 'enkfgfs.t${cyc}z.abias_int' - - 'enkfgfs.t${cyc}z.abias_pc' - optional: - com_atmos_restart_mem: - - 'enkfgfs.t${cyc}z.radstat' - - 'enkfgfs.t${cyc}z.cnvstat' diff --git a/parm/archive/gdas.yaml b/parm/archive/gdas.yaml.j2 similarity index 100% rename from parm/archive/gdas.yaml rename to parm/archive/gdas.yaml.j2 diff --git a/parm/archive/gdas_aero.yaml b/parm/archive/gdas_aero.yaml deleted file mode 100644 index fc16ab87fd..0000000000 --- a/parm/archive/gdas_aero.yaml +++ /dev/null @@ -1,4 +0,0 @@ -gdas: - optional: - com_chem_analysis: - - 'gdas.t${cyc}z.aerostat' diff --git a/parm/archive/gdas_dopost.yaml b/parm/archive/gdas_dopost.yaml deleted file mode 100644 index fae09dcfd9..0000000000 --- a/parm/archive/gdas_dopost.yaml +++ /dev/null @@ -1,4 +0,0 @@ -gdas: - optional: - rotdir: - - './logs/${PDY}${cyc}/gdas*upp*.log' diff --git a/parm/archive/gdas_ice.yaml b/parm/archive/gdas_ice.yaml deleted file mode 100644 index 09a1039100..0000000000 --- a/parm/archive/gdas_ice.yaml +++ /dev/null @@ -1,10 +0,0 @@ -gdasice: - mandatory: - com_ice_history: - - 'gdas.t${cyc}z.*' - - 'ice_in' - -gdasice_restart: - mandatory: - com_ice_restart: - - '*' diff --git a/parm/archive/gdas_ocn.yaml b/parm/archive/gdas_ocn.yaml deleted file mode 100644 index 6d0257cf7e..0000000000 --- a/parm/archive/gdas_ocn.yaml +++ /dev/null @@ -1,21 +0,0 @@ -gdasocean: - mandatory: - com_ocean_history: - - 'gdas.t${cyc}z.*' - com_ocean_input: - - '.' - -gdasocean_restart: - mandatory: - com_ocean_restart: - - '*' - com_med_restart: - - '*' - -gdasocean_analysis: - mandatory: - com_ocean_analysis: - - 'gdas.t${cyc}z.*' - - 'gdas.t??z.ocngrid.nc' - - 'diags' - - 'yaml' diff --git a/parm/archive/gdas_verfozn.yaml b/parm/archive/gdas_verfozn.yaml deleted file mode 100644 index ea46841d54..0000000000 --- a/parm/archive/gdas_verfozn.yaml +++ /dev/null @@ -1,24 +0,0 @@ -gdas: - mandatory: - com_atmos_oznmon: - - 'bad_cnt.${PDY}${cyc}' - - 'bad_diag.${PDY}${cyc}' - - 'bad_pen.${PDY}${cyc}' - - 'stdout.${type}.tar.gz' - optional: - com_atmos_oznmon: - subtypes: - - 'gome_metop-b' - - 'omi_aura' - - 'ompslp_npp' - - 'ompsnp_n20' - - 'ompsnp_npp' - - 'ompstc8_n20' - - 'ompstc8_npp' - - 'sbuv2_n19' - - '${subtype}.anl.ctl' - - '${subtype}.anl.${PDY}${cyc}.ieee_d' - - '${subtype}.anl.${PDY}${cyc}.ieee_d.gz' - - '${subtype}.ges.ctl' - - '${subtype}.ges.${PDY}${cyc}.ieee_d' - - '${subtype}.ges.${PDY}${cyc}.ieee_d.gz' diff --git a/parm/archive/gdas_verfrad.yaml b/parm/archive/gdas_verfrad.yaml deleted file mode 100644 index 8456b6ca2a..0000000000 --- a/parm/archive/gdas_verfrad.yaml +++ /dev/null @@ -1,11 +0,0 @@ -gdas: - mandatory: - com_atmos_radmon: - - 'bad_diag.${PDY}${cyc}' - - 'bad_pen.${PDY}${cyc}' - - 'low_count.${PDY}${cyc}' - - 'radmon_angle.tar.gz' - - 'radmon_bcoef.tar.gz' - - 'radmon_bcor.tar.gz' - - 'radmon_time.tar.gz' - - 'warning.${PDY}${cyc}' diff --git a/parm/archive/gdas_vminmon.yaml b/parm/archive/gdas_vminmon.yaml deleted file mode 100644 index f6644b0712..0000000000 --- a/parm/archive/gdas_vminmon.yaml +++ /dev/null @@ -1,8 +0,0 @@ -gdas: - mandatory: - com_atmos_minmon: - - '${PDY}${cyc}.costs.txt' - - '${PDY}${cyc}.cost_terms.txt' - - '${PDY}${cyc}.gnorms.ieee_d' - - '${PDY}${cyc}.reduction.ieee_d' - - 'gnorm_data.txt' diff --git a/parm/archive/gdas_wave.yaml b/parm/archive/gdas_wave.yaml deleted file mode 100644 index aa10e6d0c6..0000000000 --- a/parm/archive/gdas_wave.yaml +++ /dev/null @@ -1,10 +0,0 @@ -gdaswave: - mandatory: - com_wave_grid: - - 'gdaswave.t${cyc}z.*' - com_wave_station: - - 'gdaswave.t${cyc}z.*' -gdaswave_restart: - mandatory: - com_wave_restart: - - '*' diff --git a/parm/archive/gfs.yaml b/parm/archive/gfs.yaml deleted file mode 100644 index 8e06a30652..0000000000 --- a/parm/archive/gfs.yaml +++ /dev/null @@ -1,40 +0,0 @@ -gfsa: - mandatory: - rotdir: - - './logs/${PDY}${cyc}/gfs!(arch).log' - com_conf: - - 'ufs.input.nml' - com_atmos_grib_0p25: - - 'gfs.t${cyc}z.pgrb2.0p25.anl' - - 'gfs.t${cyc}z.pgrb2.0p25.anl.idx' - - 'gfs.t${cyc}z.pgrb2.0p25.f${fhr}' - - 'gfs.t${cyc}z.pgrb2.0p25.f${fhr}.idx' - com_atmos_history: - - 'gfs.t${cyc}z.atm.logf${fhr}.txt' - optional: - com_atmos_track: - - 'avno.t${cyc}z.cyclone.trackatcfunix' - - 'avnop.t${cyc}z.cyclone.trackatcfunix' - - 'trak.gfso.atcfunix.${PDY}${cyc}' - - 'trak.gfso.atcfunix.altg.${PDY}${cyc}' - - 'storms.gfso.atcf_gen.${PDY}${cyc}' - - 'storms.gfso.atcf_gen.altg.${PDY}${cyc}' - -gfsb: - mandatory: - com_atmos_grib_0p50: - - 'gfs.t${cyc}z.pgrb2.0p50.anl' - - 'gfs.t${cyc}z.pgrb2.0p50.anl.idx' - com_atmos_grib_1p00: - - 'gfs.t${cyc}z.pgrb2.1p00.anl' - - 'gfs.t${cyc}z.pgrb2.1p00.anl.idx' - optional: - com_atmos_grib_0p50: - forecast_hour: - - 'gfs.t${cyc}z.pgrb2.0p50.f${fhr}' - - 'gfs.t${cyc}z.pgrb2.0p50.f${fhr}.idx' - com_atmos_grib_1p00: - forecast_hour: - - 'gfs.t${cyc}z.pgrb2.1p00.f${fhr}' - - 'gfs.t${cyc}z.pgrb2.1p00.f${fhr}.idx' - diff --git a/parm/archive/gfs_aero.yaml b/parm/archive/gfs_aero.yaml deleted file mode 100644 index 780d1f4548..0000000000 --- a/parm/archive/gfs_aero.yaml +++ /dev/null @@ -1,4 +0,0 @@ -chem: - mandatory: - com_checm_history: - - 'gocart*' diff --git a/parm/archive/gfs_archgaussian.yaml b/parm/archive/gfs_archgaussian.yaml deleted file mode 100644 index baf5a5248e..0000000000 --- a/parm/archive/gfs_archgaussian.yaml +++ /dev/null @@ -1,34 +0,0 @@ -gfs_pgrb2b: - mandatory: - com_atmos_grib_0p25: - - 'gfs.t${cyc}z.pgrb2b.0p25.anl' - - 'gfs.t${cyc}z.pgrb2b.0p25.anl.idx' - com_atmos_grib_1p00: - - 'gfs.t${cyc}z.pgrb2b.1p00.anl' - - 'gfs.t${cyc}z.pgrb2b.1p00.anl.idx' - -gfs_netcdfb: - mandatory: - com_atmos_history: - forecast_hour: - - 'gfs.t${cyc}z.atmf${fhr}.nc' - - 'gfs.t${cyc}z.sfcf${fhr}.nc' - -gfs_flux: - mandatory: - com_atmos_master: - forecast_hour: - - 'gfs.t${cyc}z.sfluxgrbf${fhr}.grib2' - - 'gfs.t${cyc}z.sfluxgrbf${fhr}.grib2.idx' - -gfs_pgrb2b: - mandatory: - com_atmos_grib_0p25: - forecast_hour: - - 'gfs.t${cyc}z.pgrb2b.0p25.f${fhr}' - - 'gfs.t${cyc}z.pgrb2b.0p25.f${fhr}.idx' - optional: - com_atmos_grib_1p00: - forecast_hour: - - 'gfs.t${cyc}z.pgrb2b.1p00.f${fhr}' - - 'gfs.t${cyc}z.pgrb2b.1p00.f${fhr}.idx' diff --git a/parm/archive/gfs_archgaussian_cycled.yaml b/parm/archive/gfs_archgaussian_cycled.yaml deleted file mode 100644 index 5b921da680..0000000000 --- a/parm/archive/gfs_archgaussian_cycled.yaml +++ /dev/null @@ -1,10 +0,0 @@ -gfs_netcdfa: - mandatory: - com_atmos_analysis: - - 'gfs.t${cyc}z.atmanl.nc' - - 'gfs.t${cyc}z.sfcanl.nc' - - 'gfs.t${cyc}z.atmi*.nc' - optional: - com_atmos_analysis: - - 'gfs.t${cyc}z.dtfanl.nc' - - 'gfs.t${cyc}z.loginc.txt' diff --git a/parm/archive/gfs_bufrsnd.yaml b/parm/archive/gfs_bufrsnd.yaml deleted file mode 100644 index 52faf7cb78..0000000000 --- a/parm/archive/gfs_bufrsnd.yaml +++ /dev/null @@ -1,10 +0,0 @@ -gfs_downstream: - mandatory: - com_atmos_gempak: - - 'gfs_${PDY}${cyc}.sfc' - - 'gfs_${PDY}${cyc}.snd' - com_atmos_wmo: - - 'gfs_collective*.postsnd_${cyc}' - com_atmos_bufr: - - 'bufr.t${cyc}z' - - 'gfs.t${cyc}z.bufrsnd.tar.gz' diff --git a/parm/archive/gfs_cycled.yaml b/parm/archive/gfs_cycled.yaml deleted file mode 100644 index 242650f27e..0000000000 --- a/parm/archive/gfs_cycled.yaml +++ /dev/null @@ -1,22 +0,0 @@ -gfsa: - mandatory: - - optional: - com_atmos_analysis: - - 'gfs.t${cyc}z.gsistat' - - 'gfs.t${cyc}z.atmvar.yaml' - - 'gfs.t${cyc}z.atmstat' - com_obs: - - 'gfs.t${cyc}z.nsstbufr' - - 'gfs.t${cyc}z.prepbufr' - - 'gfs.t${cyc}z.prepbufr.acft_profiles' - -gfs_restarta: - mandatory: - com_atmos_restart: - - '*0000.sfcanl_data.tile1.nc' - - '*0000.sfcanl_data.tile2.nc' - - '*0000.sfcanl_data.tile3.nc' - - '*0000.sfcanl_data.tile4.nc' - - '*0000.sfcanl_data.tile5.nc' - - '*0000.sfcanl_data.tile6.nc' diff --git a/parm/archive/gfs_forecastonly.yaml b/parm/archive/gfs_forecastonly.yaml deleted file mode 100644 index 599144d4a5..0000000000 --- a/parm/archive/gfs_forecastonly.yaml +++ /dev/null @@ -1,16 +0,0 @@ -gfs_restarta: - mandatory: - com_atmos_input: - - 'gfs_ctrl.nc' - - 'gfs_data.tile1.nc' - - 'gfs_data.tile2.nc' - - 'gfs_data.tile3.nc' - - 'gfs_data.tile4.nc' - - 'gfs_data.tile5.nc' - - 'gfs_data.tile6.nc' - - 'sfc_data.tile1.nc' - - 'sfc_data.tile2.nc' - - 'sfc_data.tile3.nc' - - 'sfc_data.tile4.nc' - - 'sfc_data.tile5.nc' - - 'sfc_data.tile6.nc' diff --git a/parm/archive/gfs_ice.yaml b/parm/archive/gfs_ice.yaml deleted file mode 100644 index ab847d5d68..0000000000 --- a/parm/archive/gfs_ice.yaml +++ /dev/null @@ -1,21 +0,0 @@ -ice_grib2: - optional: - com_ice_grib: - - '5p00/gfs.ice.t${cyc}z.5p00.f*.grib2' - - '5p00/gfs.ice.t${cyc}z.5p00.f*.grib2.idx' - - '1p00/gfs.ice.t${cyc}z.1p00.f*.grib2' - - '1p00/gfs.ice.t${cyc}z.1p00.f*.grib2.idx' - - '0p25/gfs.ice.t${cyc}z.0p25.f*.grib2' - - '0p25/gfs.ice.t${cyc}z.0p25.f*.grib2.idx' - -ice_6hravg: - mandatory: - com_ice_history: - forecast_hour: - - 'gfs.ice.t${cyc}z.6hr_avg.f*.nc' - -ice_daily: - mandatory: - com_ice_history: - forecast_hour: - - 'gfs.ice.t${cyc}z.daily.f*.nc' diff --git a/parm/archive/gfs_ocn.yaml b/parm/archive/gfs_ocn.yaml deleted file mode 100644 index 1517358e10..0000000000 --- a/parm/archive/gfs_ocn.yaml +++ /dev/null @@ -1,21 +0,0 @@ -ocean_grib2: - optional: - com_ocean_grib: - - '5p00/gfs.ocean.t${cyc}z.5p00.f*.grib2' - - '5p00/gfs.ocean.t${cyc}z.5p00.f*.grib2.idx' - - '1p00/gfs.ocean.t${cyc}z.1p00.f*.grib2' - - '1p00/gfs.ocean.t${cyc}z.1p00.f*.grib2.idx' - - '0p25/gfs.ocean.t${cyc}z.0p25.f*.grib2' - - '0p25/gfs.ocean.t${cyc}z.0p25.f*.grib2.idx' - -ocean_6hravg: - mandatory: - com_ocean_history: - forecast_hour: - - 'gfs.ocean.t${cyc}z.6hr_avg.f*.nc' - -ocean_daily: - mandatory: - com_ocean_history: - forecast_hour: - - 'gfs.ocean.t${cyc}z.daily.f*.nc' diff --git a/parm/archive/gfs_vminmon.yaml b/parm/archive/gfs_vminmon.yaml deleted file mode 100644 index 5a72af48d0..0000000000 --- a/parm/archive/gfs_vminmon.yaml +++ /dev/null @@ -1,8 +0,0 @@ -gfsa: - mandatory: - com_atmos_minmon: - - '${PDY}${cyc}.costs.txt' - - '${PDY}${cyc}.cost_terms.txt' - - '${PDY}${cyc}.gnorms.ieee_d' - - '${PDY}${cyc}.reduction.ieee_d' - - 'gnorm_data.txt' diff --git a/parm/archive/gfs_wave.yaml b/parm/archive/gfs_wave.yaml deleted file mode 100644 index a0a529a5cc..0000000000 --- a/parm/archive/gfs_wave.yaml +++ /dev/null @@ -1,8 +0,0 @@ -gfswave: - mandatory: - com_wave_history: - - 'ww3_multi*' - com_wave_grid: - - 'gfswave.t${cyc}z.*' - com_wave_station: - - 'gfswave.t${cyc}z.*' From 60b9cd415db6d2b02ac65b50b200dca39af7423c Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 15 Mar 2024 13:52:14 -0500 Subject: [PATCH 004/128] Create archive task class with initial gdas.yaml.j2 This also - adds a missing COM variable to JGLOBAL_ARCHIVE - Creates a test driver script and python program --- jobs/JGLOBAL_ARCHIVE | 2 +- jobs/run_archive.sh | 72 ++++++++++ parm/archive/gdas.yaml.j2 | 163 ++++++++++++--------- scripts/call_archive.py | 44 ++++++ ush/python/pygfs/task/archive.py | 235 +++++++++++++++++++++++++++++++ 5 files changed, 448 insertions(+), 68 deletions(-) create mode 100755 jobs/run_archive.sh create mode 100755 scripts/call_archive.py create mode 100755 ush/python/pygfs/task/archive.py diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index 235084e631..699b95ed44 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -18,7 +18,7 @@ YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS COM_ATMOS_BUFR COM_ATMO COM_OBS COM_TOP \ COM_OCEAN_HISTORY COM_OCEAN_INPUT COM_OCEAN_RESTART COM_OCEAN_GRIB COM_OCEAN_NETCDF \ COM_OCEAN_ANALYSIS \ - COM_WAVE_GRID COM_WAVE_HISTORY COM_WAVE_STATION \ + COM_WAVE_GRID COM_WAVE_HISTORY COM_WAVE_STATION COM_WAVE_RESTART \ COM_ATMOS_OZNMON COM_ATMOS_RADMON COM_ATMOS_MINMON COM_CONF for grid in "0p25" "0p50" "1p00"; do diff --git a/jobs/run_archive.sh b/jobs/run_archive.sh new file mode 100755 index 0000000000..56daca7ee4 --- /dev/null +++ b/jobs/run_archive.sh @@ -0,0 +1,72 @@ +#!/usr/bin/bash +# Change to a directory that you can write to +export ATARDIR=/work/noaa/global/dhuber/para/stmp/test_arch + +set -e +module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core +module load stack-intel stack-python py-jinja2 py-pyyaml +export HOMEgfs=/work/noaa/global/dhuber/GW/gw_arch +export wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/sorc/wxflow/src" +export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" + +export ROTDIR=/work2/noaa/global/dhuber/para/com/herc_mon +export RUN=gdas +export PDY=20211111 +export cyc=00 +export COM_ATMOS_ANALYSIS=$ROTDIR/$RUN.$PDY/$cyc/analysis/atmos +export COM_ATMOS_BUFR=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/bufr +export COM_ATMOS_GEMPAK=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/gempak/ +export COM_ATMOS_GENESIS=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/cyclone/genesis_vital +export COM_ATMOS_HISTORY=$ROTDIR/$RUN.$PDY/$cyc/model_data/atmos/history +export COM_ATMOS_INPUT=$ROTDIR/$RUN.$PDY/$cyc/model_data/atmos/input +export COM_ATMOS_MASTER=$ROTDIR/$RUN.$PDY/$cyc/model_data/atmos/master +export COM_ATMOS_RESTART=$ROTDIR/$RUN.$PDY/$cyc/model_data/atmos/restart +export COM_ATMOS_TRACK=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/cyclone/tracks +export COM_ATMOS_WMO=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/wmo +export COM_CHEM_HISTORY=$ROTDIR/$RUN.$PDY/$cyc/model_data/chem/history +export COM_CHEM_ANALYSIS=$ROTDIR/$RUN.$PDY/$cyc/analysis/chem +export COM_MED_RESTART=$ROTDIR/$RUN.$PDY/$cyc/model_data/med/restart +export COM_ICE_HISTORY=$ROTDIR/$RUN.$PDY/$cyc/model_data/ice/history +export COM_ICE_INPUT=$ROTDIR/$RUN.$PDY/$cyc/model_data/ice/input +export COM_ICE_RESTART=$ROTDIR/$RUN.$PDY/$cyc/model_data/ice/restart +export COM_ICE_GRIB=$ROTDIR/$RUN.$PDY/$cyc/products/ice/grib2 +export COM_OBS=$ROTDIR/$RUN.$PDY/$cyc/obs +export COM_TOP=$ROTDIR/$RUN.$PDY/$cyc +export COM_OCEAN_HISTORY=$ROTDIR/$RUN.$PDY/$cyc/model_data/ocean/history +export COM_OCEAN_INPUT=$ROTDIR/$RUN.$PDY/$cyc/model_data/ocean/input +export COM_OCEAN_RESTART=$ROTDIR/$RUN.$PDY/$cyc/model_data/ocean/restart +export COM_OCEAN_GRIB=$ROTDIR/$RUN.$PDY/$cyc/products/ocean/grib2 +export COM_OCEAN_NETCDF=$ROTDIR/$RUN.$PDY/$cyc/products/ocean/netcdf +export COM_OCEAN_ANALYSIS=$ROTDIR/$RUN.$PDY/$cyc/analysis/ocean +export COM_WAVE_GRID=$ROTDIR/$RUN.$PDY/$cyc/products/wave/gridded +export COM_WAVE_HISTORY=$ROTDIR/$RUN.$PDY/$cyc/model_data/wave/history +export COM_WAVE_STATION=$ROTDIR/$RUN.$PDY/$cyc/products/wave/station +export COM_WAVE_RESTART=$ROTDIR/$RUN.$PDY/$cyc/products/wave/restart +export COM_ATMOS_OZNMON=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/oznmon +export COM_ATMOS_RADMON=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/radmon +export COM_ATMOS_MINMON=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/minmon +export COM_CONF=$ROTDIR/$RUN.$PDY/$cyc/conf +export COM_ATMOS_GRIB_0p25=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/grib2/0p25 +export COM_ATMOS_GRIB_0p50=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/grib2/0p50 +export COM_ATMOS_GRIB_1p00=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/grib2/1p00 + +export FHMIN=0 +export FHMAX=9 +export FHOUT=1 +export DO_VERFRAD=YES +export DO_VMINMON=YES +export DO_VERFOZN=YES +export DO_ICE=NO +export WRITE_DOPOST=YES +export ARCHIVE_RUN=$RUN +export PARMgfs=/work/noaa/global/dhuber/GW/gw_arch/parm +# Not needed for task, but required by wxflow +export DATA="" +export CDUMP="" + +rm -rf $ATARDIR +mkdir $ATARDIR + +cd ${ROTDIR} + +/work/noaa/global/dhuber/GW/gw_arch/scripts/call_archive.py diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 1be0a56873..d5dd3360b5 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -1,67 +1,96 @@ -gdas: - mandatory: - - '{{ COM_ATMOS_GRIB_0P25 }}/gdas.t{{ cyc }}z.pgrb2.0p25.anl' - - '{{ COM_ATMOS_GRIB_0P25 }}/gdas.t{{ cyc }}z.pgrb2.0p25.anl.idx' - - '{{ COM_ATMOS_GRIB_0P25 }}/gdas.t{{ cyc }}z.pgrb2.0p25.f{{ fhr }}' - - '{{ COM_ATMOS_GRIB_0P25 }}/gdas.t{{ cyc }}z.pgrb2.0p25.f{{ fhr }}.idx' - - '{{ COM_ATMOS_GRIB_1P00 }}/gdas.t{{ cyc }}z.pgrb2.1p00.anl' - - '{{ COM_ATMOS_GRIB_1P00 }}/gdas.t{{ cyc }}z.pgrb2.1p00.anl.idx' - - '{{ COM_ATMOS_GRIB_1P00 }}/gdas.t{{ cyc }}z.pgrb2.1p00.f{{ fhr }}' - - '{{ COM_ATMOS_GRIB_1P00 }}/gdas.t{{ cyc }}z.pgrb2.1p00.f{{ fhr }}.idx' - - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.atmanl.nc' - - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.sfcanl.nc' - - '{{ COM_ATMOS_MASTER }}/gdas.t{{ cyc }}z.sfluxgrbf{{ fhr }}.grib2' - - '{{ COM_ATMOS_MASTER }}/gdas.t{{ cyc }}z.sfluxgrbf{{ fhr }}.grib2.idx' - - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdas*prod*.log' - - '{{ COM_ATMOS_HISTORY }}/gdas.t{{ cyc }}z.atm.logf{{ fhr }}.txt' - - '{{ COM_ATMOS_HISTORY }}/gdas.t{{ cyc }}z.atmf{{ fhr }}.nc' - - '{{ COM_ATMOS_HISTORY }}/gdas.t{{ cyc }}z.sfcf{{ fhr }}.nc' - {% if DO_VERFOZN = "YES" %} - - '{{ COM_ATMOS_OZNMON }}/bad_cnt.{{ PDY }}{{ cyc }}' - - '{{ COM_ATMOS_OZNMON }}/bad_diag.{{ PDY }}{{ cyc }}' - - '{{ COM_ATMOS_OZNMON }}/bad_pen.{{ PDY }}{{ cyc }}' - {% set oznmon_types = [ - 'gome_metop-b', 'omi_aura', 'ompslp_npp', 'ompsnp_n20', - 'ompsnp_npp', 'ompstc8_n20', 'ompstc8_npp', 'sbuv2_n19' - ] %} - {% for type in oznmon_types %} - - '{{ COM_ATMOS_OZNMON }}/stdout.{{ type }}.tar.gz' - {% endfor %} - {% endif %} - optional: - - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.atmvar.yaml' - - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.atmstat' - - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.gsistat' - - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.atmanl.ensres.nc' - - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.atma003.ensres.nc' - - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.atma009.ensres.nc' - - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.cnvstat' - - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.oznstat' - - '{{ COM_ATMOS_ANALYSIS }}/gdas.t{{ cyc }}z.radstat' - - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdasprep.log' - - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdasanal.log' - - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdasfcst.log' - - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdasverfozn.log' - - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdasverfrad.log' - - '{{ ROTDIR }}/logs/{{ PDY }}{{ cyc }}/gdasvminmon.log' - - '{{ COM_ATMOS_MASTER }}/gdas.t{{ cyc }}z.sfluxgrbf{{ fhr }}.grib2' - - '{{ COM_ATMOS_MASTER }}/gdas.t{{ cyc }}z.sfluxgrbf{{ fhr }}.grib2.idx' - {% if DO_AERO = "YES" %} - - '{{ COM_CHEM_ANALYSIS }}/gdas.t{{ cyc }}z.aerostat' - {% endif } - {% if WRITE_DOPOST = ".true." %} - - '{{ ROTDIR }}/{{ PDY }}{{ cyc }}/gdas*upp*.log' - {% endif } - {% if DO_VERFOZN = "YES" %} - {% set oznmon_types = [ - 'gome_metop-b', 'omi_aura', 'ompslp_npp', 'ompsnp_n20', - 'ompsnp_npp', 'ompstc8_n20', 'ompstc8_npp', 'sbuv2_n19' - ] %} - {% for type in oznmon_types %} - - '{{ type }}.anl.ctl' - - '{{ type }}.anl.{{ PDY }}{{ cyc }}.ieee_d' - - '{{ type }}.anl.{{ PDY }}{{ cyc }}.ieee_d.gz' - - '{{ type }}.ges.ctl' - - '{{ type }}.ges.{{ PDY }}{{ cyc }}.ieee_d' - - '{{ type }}.ges.{{ PDY }}{{ cyc }}.ieee_d.gz' - {% endfor %} +name: "GDAS" +description: "Basic GDAS analysis and forecast produts and data" +{% set cycle_HH = current_cycle | strftime('%H') %} +{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} +archive_name: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas.tar" +mandatory: + - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.anl" + - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.anl.idx" + - "{{ atmos_grib_1p00_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.1p00.anl" + - "{{ atmos_grib_1p00_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.1p00.anl.idx" + - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmanl.nc" + - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfcanl.nc" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}*prod*.log" + {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} + - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.f{{ '%03d' % fhr }}" + - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ atmos_grib_1p00_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.1p00.f{{ '%03d' % fhr }}" + - "{{ atmos_grib_1p00_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.1p00.f{{ '%03d' % fhr }}.idx" + - "{{ atmos_master_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ atmos_master_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + - "{{ atmos_history_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atm.logf{{ '%03d' % fhr }}.txt" + - "{{ atmos_history_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmf{{ '%03d' % fhr }}.nc" + - "{{ atmos_history_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfcf{{ '%03d' % fhr }}.nc" + {% endfor %} + {% if DO_VERFOZN %} + - "{{ atmos_oznmon_dir }}/time/bad_cnt.{{ cycle_YYYYMMDDHH }}" + - "{{ atmos_oznmon_dir }}/time/bad_diag.{{ cycle_YYYYMMDDHH }}" + - "{{ atmos_oznmon_dir }}/time/bad_pen.{{ cycle_YYYYMMDDHH }}" + {% set oznmon_types = [ + "gome_metop-b", "omi_aura", "ompslp_npp", "ompsnp_n20", + "ompsnp_npp", "ompstc8_n20", "ompstc8_npp", "sbuv2_n19" + ] %} + {% for type in oznmon_types %} + - "{{ atmos_oznmon_dir }}/time/stdout.time.tar.gz" + - "{{ atmos_oznmon_dir }}/horiz/stdout.horiz.tar.gz" + {% endfor %} + {% endif %} + {% if DO_VERFRAD %} + - "{{ atmos_radmon_dir }}/bad_diag.{{ cycle_YYYYMMDDHH }}" + - "{{ atmos_radmon_dir }}/bad_pen.{{ cycle_YYYYMMDDHH }}" + - "{{ atmos_radmon_dir }}/low_count.{{ cycle_YYYYMMDDHH }}" + - "{{ atmos_radmon_dir }}/radmon_angle.tar.gz" + - "{{ atmos_radmon_dir }}/radmon_bcoef.tar.gz" + - "{{ atmos_radmon_dir }}/radmon_bcor.tar.gz" + - "{{ atmos_radmon_dir }}/radmon_time.tar.gz" + - "{{ atmos_radmon_dir }}/warning.{{ cycle_YYYYMMDDHH }}" + {% endif %} + {% if DO_VMINMON %} + - "{{ atmos_minmon_dir }}/{{ cycle_YYYYMMDDHH }}.costs.txt" + - "{{ atmos_minmon_dir }}/{{ cycle_YYYYMMDDHH }}.cost_terms.txt" + - "{{ atmos_minmon_dir }}/{{ cycle_YYYYMMDDHH }}.gnorms.ieee_d" + - "{{ atmos_minmon_dir }}/{{ cycle_YYYYMMDDHH }}.reduction.ieee_d" + - "{{ atmos_minmon_dir }}/gnorm_data.txt" + {% endif %} +optional: + - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmvar.yaml" + - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmstat" + - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.gsistat" + - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmanl.ensres.nc" + - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atma003.ensres.nc" + - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atma009.ensres.nc" + - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.cnvstat" + - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.oznstat" + - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.radstat" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}prep.log" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}anal.log" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}fcst.log" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}verfozn.log" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}verfrad.log" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}vminmon.log" + {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} + - "{{ atmos_master_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ atmos_master_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + {% endfor %} + {% if DO_AERO %} + - "{{ chem_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.aerostat" + {% endif %} + {% if WRITE_DOPOST %} + - "{{ cycle_YYYYMMDDHH }}/{{ RUN }}*upp*.log" + {% endif %} + {% if DO_VERFOZN %} + {% set oznmon_types = [ + "gome_metop-b", "omi_aura", "ompslp_npp", "ompsnp_n20", + "ompsnp_npp", "ompstc8_n20", "ompstc8_npp", "sbuv2_n19" + ] %} + {% for group in [ "horiz", "time" ] %} + {% for type in oznmon_types %} + - "{{ atmos_oznmon_dir }}/{{ group }}/{{ type }}.anl.ctl" + - "{{ atmos_oznmon_dir }}/{{ group }}/{{ type }}.anl.{{ cycle_YYYYMMDDHH }}.ieee_d" + - "{{ atmos_oznmon_dir }}/{{ group }}/{{ type }}.anl.{{ cycle_YYYYMMDDHH }}.ieee_d.gz" + - "{{ atmos_oznmon_dir }}/{{ group }}/{{ type }}.ges.ctl" + - "{{ atmos_oznmon_dir }}/{{ group }}/{{ type }}.ges.{{ cycle_YYYYMMDDHH }}.ieee_d" + - "{{ atmos_oznmon_dir }}/{{ group }}/{{ type }}.ges.{{ cycle_YYYYMMDDHH }}.ieee_d.gz" + {% endfor %} + {% endfor %} + {% endif %} diff --git a/scripts/call_archive.py b/scripts/call_archive.py new file mode 100755 index 0000000000..48d36051fe --- /dev/null +++ b/scripts/call_archive.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +import os + +from wxflow import AttrDict, Logger, logit, cast_strdict_as_dtypedict +from pygfs.task.archive import Archive + +# initialize root logger +logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) + + +@logit(logger) +def main(): + + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the UPP object + archive = Archive(config) + + # Pull out all the configuration keys needed to run the rest of UPP steps + keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', + 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'PARMgfs', + 'DO_OCN', 'WRITE_DOPOST', 'cyc', 'atmos_analysis_dir', + 'atmos_analysis_ensstat_dir', 'atmos_analysis_mem_dir', 'atmos_bufr_dir', + 'atmos_gempak_dir', 'atmos_grib_0p25_dir', 'atmos_grib_0p50_dir', + 'atmos_grib_1p00_dir', 'atmos_history_dir', 'atmos_history_ensstat_dir', + 'atmos_history_mem_dir', 'atmos_input_dir', 'atmos_master_dir', + 'atmos_minmon_dir', 'atmos_oznmon_dir', 'atmos_radmon_dir', + 'atmos_restart_dir', 'atmos_restart_mem_dir', 'atmos_track_dir', + 'atmos_wmo_dir', 'checm_history_dir', 'chem_analysis_dir', 'conf_dir', + 'ice_grib_dir', 'ice_history_dir', 'med_restart_dir', 'obs_dir', + 'ocean_analysis_dir', 'ocean_grib_dir', 'ocean_history_dir', + 'ocean_input_dir', 'ocean_restart_dir', 'wave_grid_dir', + 'wave_history_dir', 'wave_restart_dir', 'wave_station_dir'] + + archive_dict = AttrDict() + for key in keys: + archive_dict[key] = archive.task_config[key] + + # Determine which archives to create and create them + archive.execute(archive_dict) + +if __name__ == '__main__': + main() diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py new file mode 100755 index 0000000000..1f96c21f86 --- /dev/null +++ b/ush/python/pygfs/task/archive.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 + +from yaml import load +from yaml import CLoader as Loader +from typing import Dict, Any +from wxflow import logit, cast_strdict_as_dtypedict, AttrDict, Task, parse_j2yaml +from logging import getLogger +import os + +logger = getLogger(__name__.split('.')[-1]) + +class Archive(Task): + """Task to archive ROTDIR data to HPSS (or locally) + """ + + @logit(logger, name="Archive") + def __init__(self, config: Dict[str, Any]) -> None: + """Constructor for the Archive task + The constructor is responsible for collecting necessary yamls based on + the runtime options and RUN. + + Parameters + ---------- + config : Dict[str, Any] + Incoming configuration for the task from the environment + + Returns + ------- + None + """ + super().__init__(config) + + rotdir=self.config.ROTDIR + os.sep + + local_dict = AttrDict( + {'atmos_analysis_dir': self.config.COM_ATMOS_ANALYSIS.replace(rotdir,''), + 'atmos_bufr_dir': self.config.COM_ATMOS_BUFR.replace(rotdir,''), + 'atmos_gempak_dir': self.config.COM_ATMOS_GEMPAK.replace(rotdir,''), + 'atmos_grib_0p25_dir': self.config.COM_ATMOS_GRIB_0p25.replace(rotdir,''), + 'atmos_grib_0p50_dir': self.config.COM_ATMOS_GRIB_0p50.replace(rotdir,''), + 'atmos_grib_1p00_dir': self.config.COM_ATMOS_GRIB_1p00.replace(rotdir,''), + 'atmos_history_dir': self.config.COM_ATMOS_HISTORY.replace(rotdir,''), + 'atmos_input_dir': self.config.COM_ATMOS_INPUT.replace(rotdir,''), + 'atmos_master_dir': self.config.COM_ATMOS_MASTER.replace(rotdir,''), + 'atmos_minmon_dir': self.config.COM_ATMOS_MINMON.replace(rotdir,''), + 'atmos_oznmon_dir': self.config.COM_ATMOS_OZNMON.replace(rotdir,''), + 'atmos_radmon_dir': self.config.COM_ATMOS_RADMON.replace(rotdir,''), + 'atmos_restart_dir': self.config.COM_ATMOS_RESTART.replace(rotdir,''), + 'atmos_track_dir': self.config.COM_ATMOS_TRACK.replace(rotdir,''), + 'atmos_wmo_dir': self.config.COM_ATMOS_WMO.replace(rotdir,''), + 'chem_history_dir': self.config.COM_CHEM_HISTORY.replace(rotdir,''), + 'chem_analysis_dir': self.config.COM_CHEM_ANALYSIS.replace(rotdir,''), + 'conf_dir': self.config.COM_CONF.replace(rotdir,''), + 'ice_grib_dir': self.config.COM_ICE_GRIB.replace(rotdir,''), + 'ice_history_dir': self.config.COM_ICE_HISTORY.replace(rotdir,''), + 'med_restart_dir': self.config.COM_MED_RESTART.replace(rotdir,''), + 'obs_dir': self.config.COM_OBS.replace(rotdir,''), + 'ocean_analysis_dir': self.config.COM_OCEAN_ANALYSIS.replace(rotdir,''), + 'ocean_grib_dir': self.config.COM_OCEAN_GRIB.replace(rotdir,''), + 'ocean_history_dir': self.config.COM_OCEAN_HISTORY.replace(rotdir,''), + 'ocean_input_dir': self.config.COM_OCEAN_INPUT.replace(rotdir,''), + 'ocean_restart_dir': self.config.COM_OCEAN_RESTART.replace(rotdir,''), + 'wave_grid_dir': self.config.COM_WAVE_GRID.replace(rotdir,''), + 'wave_history_dir': self.config.COM_WAVE_HISTORY.replace(rotdir,''), + 'wave_restart_dir': self.config.COM_WAVE_RESTART.replace(rotdir,''), + 'wave_station_dir': self.config.COM_WAVE_STATION.replace(rotdir,'') + } + ) + + self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) + + @staticmethod + @logit(logger) + def execute(arch_dict: Dict[str, Any]) -> None: + """Run the UPP executable and index the output master and flux files + + Parameters + ---------- + workdir : str | os.PathLike + work directory with the staged data, parm files, namelists, etc. + aprun_cmd : str + launcher command for UPP.x + forecast_hour : int + default: 0 + forecast hour being processed + + Returns + ------- + None + """ + + archive_sets = Archive._get_archive_sets(arch_dict) + + Archive._create_archives(archive_sets) + + @logit(logger) + def _get_archive_sets(arch_dict: Dict[str, Any]) -> None: + """Determine which tarballs will need to be created. + + Parameters + ---------- + arch_dict : Dict + Task specific keys, e.g. runtime options (DO_AERO, DO_ICE, etc) + """ + + # Collect datasets that need to be archived + # Each dataset represents one tarball + + if(arch_dict.RUN == "gdas"): + + datasets = ['gdas'] # , 'gdas_restarta', 'gdas_restartb'] + + #if(arch_dict.DO_ICE == "YES"): + # datasets.append('gdas_ice') + # datasets.append('gdas_ice_restart') + + #if(arch_dict.DO_OCN == "YES"): + # datasets.append('gdas_ocean_6hravg') + # datasets.append('gdas_ocean_daily') + # datasets.append('gdas_ocean_grib2') + + #if(arch_dict.DO_WAVE == "YES"): + # datasets.append('gdas_wave') + + elif (arch_dict.RUN == "gfs"): + print("Set me up!") + + else: + raise NotImplementedError(f'Archiving is not enabled for {arch_dict.RUN} runs') + + archive_sets = [] + + archive_parm = os.path.join(arch_dict.PARMgfs, "archive") + + for dataset in datasets: + + archive_filename = os.path.join(archive_parm, dataset + ".yaml.j2") + + print(archive_filename) + archive_set = parse_j2yaml(archive_filename, arch_dict) + + archive_set = Archive._create_fileset(archive_set) + + archive_sets.append(archive_set) + + return archive_sets + + @logit(logger) + def _create_fileset(archive_set: Dict[str, Any]) -> None: + """ + Collect the list of all available files from the parsed yaml dict. + Globs are expanded and if mandatory files are missing, an error is + raised. + + TODO: expand all globs in the jinja yaml files instead of expanding + them here and issue errors here if globbing patterns (*, ?, []) + are found. + + Parameters + ---------- + archive_set: Dict + Contains full paths for mandatory and optional files to be archived. + """ + + import glob + + archive_set.fileset = [] + if 'mandatory' in archive_set: + for item in archive_set.mandatory: + glob_set = glob.glob(item) + if len(glob_set) == 0: + raise FileNotFoundError(f'Mandatory file or glob {item} not found!') + for file in glob_set: + Archive._check_isfile(file) + archive_set.fileset.append(file) + + if 'optional' in archive_set: + for item in archive_set.optional: + glob_set = glob.glob(item) + if len(glob_set) == 0: + print (f'WARNING: optional file/glob {item} not found!') + else: + for file in glob_set: + Archive._check_isfile(file,False) + archive_set.fileset.append(file) + + if len(archive_set.fileset) == 0: + print (f'WARNING: the fileset for the {archive_set.name} archive is empty!') + + return archive_set + + @logit(logger) + def _check_isfile(filename: str, error_on_noexist = True) -> None: + """ + Checks that the input filename is not a directory. + TODO: Expand this to check that 'filename' is not a glob and that + the file actually exists. + + Parameters + ---------- + filename : str + The name of the file to check including the path. + + error_on_noexist : boolean + (not yet implemented) + If the file does not exist, this flag determines if a warning or + error should be generated. + Errors will always be issued if filename points to a directory. + """ + + if os.path.isdir(filename): + raise IsADirectoryError(f'{file} is a directory\n' + + f'only files are allowed to be archived.') + + @logit(logger) + def _create_archives(archive_sets: list[Dict[str, Any]]) -> None: + """Create the tarballs from the list of yaml dicts. + + Parameters + ---------- + arch_dict : Dict + Task specific keys, e.g. runtime options (DO_AERO, DO_ICE, etc) + """ + + import tarfile + + for archive_set in archive_sets: + archive_name = archive_set.archive_name + + if not os.path.exists(os.path.dirname(archive_name)): + os.mkdir(os.path.dirname(archive_name)) + + with tarfile.open(archive_name, "w") as tar: + for file in archive_set.fileset: + tar.add(file) From 754c1962619f3f06d75586e1519d31e070166f59 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 15 Mar 2024 15:04:09 -0500 Subject: [PATCH 005/128] Create configure method --- jobs/run_archive.sh | 7 ++-- scripts/call_archive.py | 11 +++-- ush/python/pygfs/task/archive.py | 72 +++++++++++--------------------- 3 files changed, 36 insertions(+), 54 deletions(-) diff --git a/jobs/run_archive.sh b/jobs/run_archive.sh index 56daca7ee4..dc266723f5 100755 --- a/jobs/run_archive.sh +++ b/jobs/run_archive.sh @@ -1,16 +1,18 @@ #!/usr/bin/bash # Change to a directory that you can write to export ATARDIR=/work/noaa/global/dhuber/para/stmp/test_arch +# Change to the head of your clone +export HOMEgfs=/work/noaa/global/dhuber/GW/gw_arch set -e module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel stack-python py-jinja2 py-pyyaml -export HOMEgfs=/work/noaa/global/dhuber/GW/gw_arch +export PARMgfs=$HOMEgfs/parm export wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/sorc/wxflow/src" export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export ROTDIR=/work2/noaa/global/dhuber/para/com/herc_mon export RUN=gdas +export ROTDIR=/work2/noaa/global/dhuber/para/com/herc_mon export PDY=20211111 export cyc=00 export COM_ATMOS_ANALYSIS=$ROTDIR/$RUN.$PDY/$cyc/analysis/atmos @@ -59,7 +61,6 @@ export DO_VERFOZN=YES export DO_ICE=NO export WRITE_DOPOST=YES export ARCHIVE_RUN=$RUN -export PARMgfs=/work/noaa/global/dhuber/GW/gw_arch/parm # Not needed for task, but required by wxflow export DATA="" export CDUMP="" diff --git a/scripts/call_archive.py b/scripts/call_archive.py index 48d36051fe..d68aa7670b 100755 --- a/scripts/call_archive.py +++ b/scripts/call_archive.py @@ -14,10 +14,10 @@ def main(): config = cast_strdict_as_dtypedict(os.environ) - # Instantiate the UPP object + # Instantiate the Archive object archive = Archive(config) - # Pull out all the configuration keys needed to run the rest of UPP steps + # Pull out all the configuration keys needed to run the rest of archive steps keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'PARMgfs', 'DO_OCN', 'WRITE_DOPOST', 'cyc', 'atmos_analysis_dir', @@ -37,8 +37,11 @@ def main(): for key in keys: archive_dict[key] = archive.task_config[key] - # Determine which archives to create and create them - archive.execute(archive_dict) + # Determine which archives to create + archive_sets = archive.configure(archive_dict) + + # Create the archives + archive.execute(archive_sets) if __name__ == '__main__': main() diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 1f96c21f86..703165d299 100755 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -71,30 +71,7 @@ def __init__(self, config: Dict[str, Any]) -> None: @staticmethod @logit(logger) - def execute(arch_dict: Dict[str, Any]) -> None: - """Run the UPP executable and index the output master and flux files - - Parameters - ---------- - workdir : str | os.PathLike - work directory with the staged data, parm files, namelists, etc. - aprun_cmd : str - launcher command for UPP.x - forecast_hour : int - default: 0 - forecast hour being processed - - Returns - ------- - None - """ - - archive_sets = Archive._get_archive_sets(arch_dict) - - Archive._create_archives(archive_sets) - - @logit(logger) - def _get_archive_sets(arch_dict: Dict[str, Any]) -> None: + def configure(arch_dict: Dict[str, Any]) -> None: """Determine which tarballs will need to be created. Parameters @@ -136,7 +113,6 @@ def _get_archive_sets(arch_dict: Dict[str, Any]) -> None: archive_filename = os.path.join(archive_parm, dataset + ".yaml.j2") - print(archive_filename) archive_set = parse_j2yaml(archive_filename, arch_dict) archive_set = Archive._create_fileset(archive_set) @@ -145,6 +121,30 @@ def _get_archive_sets(arch_dict: Dict[str, Any]) -> None: return archive_sets + @staticmethod + @logit(logger) + def execute(archive_sets: list[Dict[str, Any]]) -> None: + """Create the tarballs from the list of yaml dicts. + + Parameters + ---------- + arch_dict : Dict + Task specific keys, e.g. runtime options (DO_AERO, DO_ICE, etc) + """ + + import tarfile + + for archive_set in archive_sets: + archive_name = archive_set.archive_name + + if not os.path.exists(os.path.dirname(archive_name)): + os.mkdir(os.path.dirname(archive_name)) + + with tarfile.open(archive_name, "w") as tar: + for file in archive_set.fileset: + tar.add(file) + + @logit(logger) def _create_fileset(archive_set: Dict[str, Any]) -> None: """ @@ -211,25 +211,3 @@ def _check_isfile(filename: str, error_on_noexist = True) -> None: if os.path.isdir(filename): raise IsADirectoryError(f'{file} is a directory\n' + f'only files are allowed to be archived.') - - @logit(logger) - def _create_archives(archive_sets: list[Dict[str, Any]]) -> None: - """Create the tarballs from the list of yaml dicts. - - Parameters - ---------- - arch_dict : Dict - Task specific keys, e.g. runtime options (DO_AERO, DO_ICE, etc) - """ - - import tarfile - - for archive_set in archive_sets: - archive_name = archive_set.archive_name - - if not os.path.exists(os.path.dirname(archive_name)): - os.mkdir(os.path.dirname(archive_name)) - - with tarfile.open(archive_name, "w") as tar: - for file in archive_set.fileset: - tar.add(file) From 24dddf526448f774ba7b30e4e500025a45ef2ba5 Mon Sep 17 00:00:00 2001 From: David Huber Date: Sat, 23 Mar 2024 09:43:43 -0500 Subject: [PATCH 006/128] Updates to archive task. --- parm/archive/gdas.yaml.j2 | 2 +- ush/python/pygfs/task/archive.py | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index d5dd3360b5..51ca06714d 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -2,7 +2,7 @@ name: "GDAS" description: "Basic GDAS analysis and forecast produts and data" {% set cycle_HH = current_cycle | strftime('%H') %} {% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} -archive_name: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas.tar" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas.tar" mandatory: - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.anl" - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.anl.idx" diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 703165d299..a74536d817 100755 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -3,7 +3,13 @@ from yaml import load from yaml import CLoader as Loader from typing import Dict, Any -from wxflow import logit, cast_strdict_as_dtypedict, AttrDict, Task, parse_j2yaml +from wxflow import ( + logit, + cast_strdict_as_dtypedict, + AttrDict, + Task, + parse_j2yaml, + archive_utils) from logging import getLogger import os @@ -105,6 +111,10 @@ def configure(arch_dict: Dict[str, Any]) -> None: else: raise NotImplementedError(f'Archiving is not enabled for {arch_dict.RUN} runs') + tar_cmd = 'tar' + if arch_dict.HPSSARCH: + tar_cmd = 'htar' + archive_sets = [] archive_parm = os.path.join(arch_dict.PARMgfs, "archive") @@ -112,10 +122,9 @@ def configure(arch_dict: Dict[str, Any]) -> None: for dataset in datasets: archive_filename = os.path.join(archive_parm, dataset + ".yaml.j2") - archive_set = parse_j2yaml(archive_filename, arch_dict) - archive_set = Archive._create_fileset(archive_set) + archive_set['protocol'] = tar_cmd archive_sets.append(archive_set) @@ -135,15 +144,7 @@ def execute(archive_sets: list[Dict[str, Any]]) -> None: import tarfile for archive_set in archive_sets: - archive_name = archive_set.archive_name - - if not os.path.exists(os.path.dirname(archive_name)): - os.mkdir(os.path.dirname(archive_name)) - - with tarfile.open(archive_name, "w") as tar: - for file in archive_set.fileset: - tar.add(file) - + archive_utils.ArchiveHandler(archive_set).create() @logit(logger) def _create_fileset(archive_set: Dict[str, Any]) -> None: From bdf66df9660b6eeefa83edd971dcb08f65396544 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 1 Apr 2024 11:12:49 -0400 Subject: [PATCH 007/128] Add htar features to archive task, update gdas template. (untested) --- parm/archive/gdas.yaml.j2 | 37 ++++++++++++- ush/python/pygfs/task/archive.py | 89 ++++++++++++++++++++++++++++---- 2 files changed, 114 insertions(+), 12 deletions(-) diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 51ca06714d..461b9533db 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -2,6 +2,13 @@ name: "GDAS" description: "Basic GDAS analysis and forecast produts and data" {% set cycle_HH = current_cycle | strftime('%H') %} {% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} +{% set first_cycle = cycle_YYYYMMDDHH == SDATE %} +{% set n_fhrs = (FHMAX - FHMIN + 1) / FHOUT %} +{% if n_fhrs % NFHRS_PER_GROUP == 0 %} +{% set n_post_groups = n_fhrs // NFHRS_PER_GROUP %} +{% else %} +{% set n_post_groups = n_fhrs // NFHRS_PER_GROUP + 1 %} +{% endif %} target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas.tar" mandatory: - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.anl" @@ -10,7 +17,21 @@ mandatory: - "{{ atmos_grib_1p00_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.1p00.anl.idx" - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmanl.nc" - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfcanl.nc" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}*prod*.log" + {% if not first_cycle } + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmanlprod.log" + {% endif %} + {% for post_group in range(n_post_groups) %} + {% set grp_st_fhr = post_group * NFHRS_PER_GROUP * FHOUT + FHMIN %} + {% set grp_en_fhr = (post_group + 1) * NFHRS_PER_GROUP * FHOUT + FHMIN - 1 %} + {% if grp_en_fhr > FHMAX %} + {% set grp_en_fhr = FHMAX %} + {% endif %} + {% if grp_en_fhr != grp_st_fhr %} + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_prod_f{{ '%03d' % grp_st_fhr }}-f{{ '%03d' %grp_en_fhr}}.log" + {% else %} + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_prod_f{{ '%03d' % grp_st_fhr }}.log" + {% endif %} + {% endfor %} {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.f{{ '%03d' % fhr }}" - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.f{{ '%03d' % fhr }}.idx" @@ -75,9 +96,21 @@ optional: {% if DO_AERO %} - "{{ chem_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.aerostat" {% endif %} + - "{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmanlupp.log" {% if WRITE_DOPOST %} - - "{{ cycle_YYYYMMDDHH }}/{{ RUN }}*upp*.log" + {% for post_group in range(n_post_groups) %} + {% set grp_st_fhr = post_group * NFHRS_PER_GROUP * FHOUT + FHMIN %} + {% set grp_en_fhr = (post_group + 1) * NFHRS_PER_GROUP * FHOUT + FHMIN - 1 %} + {% if grp_en_fhr > FHMAX %} + {% set grp_en_fhr = FHMAX %} + {% endif %} + {% if grp_en_fhr != grp_st_fhr %} + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_upp_f{{ '%03d' % grp_st_fhr }}-f{{ '%03d' %grp_en_fhr}}.log" + {% else %} + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_upp_f{{ '%03d' % grp_st_fhr }}.log" {% endif %} + {% endfor %} + {% endif %} # WRITE_DOPOST {% if DO_VERFOZN %} {% set oznmon_types = [ "gome_metop-b", "omi_aura", "ompslp_npp", "ompsnp_n20", diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index a74536d817..2364675446 100755 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -7,7 +7,10 @@ logit, cast_strdict_as_dtypedict, AttrDict, + get_gid, Task, + htar, + hsi, parse_j2yaml, archive_utils) from logging import getLogger @@ -75,9 +78,9 @@ def __init__(self, config: Dict[str, Any]) -> None: self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) - @staticmethod + @classmethod @logit(logger) - def configure(arch_dict: Dict[str, Any]) -> None: + def configure(cls, arch_dict: Dict[str, Any]) -> None: """Determine which tarballs will need to be created. Parameters @@ -123,7 +126,8 @@ def configure(arch_dict: Dict[str, Any]) -> None: archive_filename = os.path.join(archive_parm, dataset + ".yaml.j2") archive_set = parse_j2yaml(archive_filename, arch_dict) - archive_set = Archive._create_fileset(archive_set) + archive_set = cls._create_fileset(archive_set) + archive_set['has_rstprod'] = cls._has_rstprod(archive_set.fileset) archive_set['protocol'] = tar_cmd archive_sets.append(archive_set) @@ -141,13 +145,21 @@ def execute(archive_sets: list[Dict[str, Any]]) -> None: Task specific keys, e.g. runtime options (DO_AERO, DO_ICE, etc) """ - import tarfile - for archive_set in archive_sets: - archive_utils.ArchiveHandler(archive_set).create() + + if archive_set.has_rstprod: + + try: + archive_utils.ArchiveHandler(archive_set).create() + success = True + except: + success = False + + cls._protect_rstprod(archive_set) @logit(logger) - def _create_fileset(archive_set: Dict[str, Any]) -> None: + @classmethod + def _create_fileset(cls, archive_set: Dict[str, Any]) -> None: """ Collect the list of all available files from the parsed yaml dict. Globs are expanded and if mandatory files are missing, an error is @@ -172,7 +184,7 @@ def _create_fileset(archive_set: Dict[str, Any]) -> None: if len(glob_set) == 0: raise FileNotFoundError(f'Mandatory file or glob {item} not found!') for file in glob_set: - Archive._check_isfile(file) + cls._check_isfile(file) archive_set.fileset.append(file) if 'optional' in archive_set: @@ -182,7 +194,7 @@ def _create_fileset(archive_set: Dict[str, Any]) -> None: print (f'WARNING: optional file/glob {item} not found!') else: for file in glob_set: - Archive._check_isfile(file,False) + cls._check_isfile(file,False) archive_set.fileset.append(file) if len(archive_set.fileset) == 0: @@ -210,5 +222,62 @@ def _check_isfile(filename: str, error_on_noexist = True) -> None: """ if os.path.isdir(filename): - raise IsADirectoryError(f'{file} is a directory\n' + + raise IsADirectoryError(f'{file} is a directory\n' + f'only files are allowed to be archived.') + + + @logit(logger) + def _has_rstprod(fileset: list) -> bool: + """ + Checks if any files in the input fileset belongs to rstprod. + + Parameters + ---------- + fileset : list + List of filenames to check. + """ + + try: + rstprod_gid = get_gid("rstprod") + except KeyError: + # rstprod does not exist on this machine + return False + + # Expand globs and check each file for group ownership + for file in fileset: + if os.stat(filename).st_gid == rstprod_gid: + return True + + return False + + + + @logit(logger) + def _protect_rstprod(self, archive_set) -> None: + """ + Changes the group of the target tarball to rstprod and the permissions to + 640. If this fails for any reason, attempt to delete the file before exiting. + + """ + + if archive_set.protocol = "htar": + chgrp = hsi.chgrp + chmod = hsi.chmod + rm = hsi.rm + else: + # TODO verify these are correct + chgrp = f_chgrp + chmod = os.chmod + rm = os.rm + + try: + chgrp("rstprod", archive_set.target) + chmod("640", archive_set.target) + except + try: + rm(archive_set.target) + except: + pass + + raise OSError(f"Failed to protect {archive_set.target}!\n" + f"Please verify that it has been deleted!!") From e8cb2c11141122d624a5d2bde0eef47573de8d5d Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 8 Apr 2024 17:50:46 +0000 Subject: [PATCH 008/128] Update archive protocol function, add templates. --- jobs/run_archive.sh | 24 ++- parm/archive/gdas.yaml.j2 | 42 ++--- parm/archive/gdas_ice.yaml.j2 | 6 + parm/archive/gdas_ice_restart.yaml.j2 | 5 + parm/archive/gdas_ocean_6hravg.yaml.j2 | 5 + parm/archive/gdas_ocean_daily.yaml.j2 | 5 + parm/archive/gdas_ocean_grib2.yaml.j2 | 10 ++ parm/archive/gdas_restarta.yaml.j2 | 27 ++++ parm/archive/gdas_restartb.yaml.j2 | 5 + parm/archive/gdas_wave_restart.yaml.j2 | 5 + scripts/call_archive.py | 12 +- sorc/wxflow | 2 +- ush/python/pygfs/task/archive.py | 210 ++++++++++++++----------- 13 files changed, 222 insertions(+), 136 deletions(-) create mode 100644 parm/archive/gdas_ice.yaml.j2 create mode 100644 parm/archive/gdas_ice_restart.yaml.j2 create mode 100644 parm/archive/gdas_ocean_6hravg.yaml.j2 create mode 100644 parm/archive/gdas_ocean_daily.yaml.j2 create mode 100644 parm/archive/gdas_ocean_grib2.yaml.j2 create mode 100644 parm/archive/gdas_restarta.yaml.j2 create mode 100644 parm/archive/gdas_restartb.yaml.j2 create mode 100644 parm/archive/gdas_wave_restart.yaml.j2 diff --git a/jobs/run_archive.sh b/jobs/run_archive.sh index dc266723f5..bc70daac3e 100755 --- a/jobs/run_archive.sh +++ b/jobs/run_archive.sh @@ -1,20 +1,23 @@ #!/usr/bin/bash # Change to a directory that you can write to -export ATARDIR=/work/noaa/global/dhuber/para/stmp/test_arch +export ATARDIR=/scratch1/NCEPDEV/global/David.Huber/archive_rotdir/minmon # Change to the head of your clone -export HOMEgfs=/work/noaa/global/dhuber/GW/gw_arch +export HOMEgfs=/scratch1/NCEPDEV/global/David.Huber/GW/gw_archive set -e -module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core +module reset +module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core module load stack-intel stack-python py-jinja2 py-pyyaml export PARMgfs=$HOMEgfs/parm export wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/sorc/wxflow/src" export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export RUN=gdas -export ROTDIR=/work2/noaa/global/dhuber/para/com/herc_mon -export PDY=20211111 +export ROTDIR=/scratch1/NCEPDEV/global/David.Huber/para/comrot/minmon +export PDY=20240302 export cyc=00 +export assim_freq=6 +export NFHRS_PER_GROUP=3 export COM_ATMOS_ANALYSIS=$ROTDIR/$RUN.$PDY/$cyc/analysis/atmos export COM_ATMOS_BUFR=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/bufr export COM_ATMOS_GEMPAK=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/gempak/ @@ -59,15 +62,20 @@ export DO_VERFRAD=YES export DO_VMINMON=YES export DO_VERFOZN=YES export DO_ICE=NO -export WRITE_DOPOST=YES +export DO_AERO=NO +export DO_OCN=NO +export DO_WAVE=NO +export WRITE_DOPOST=.true. export ARCHIVE_RUN=$RUN +export HPSSARCH="NO" +export LOCALARCH="YES" # Not needed for task, but required by wxflow export DATA="" export CDUMP="" rm -rf $ATARDIR -mkdir $ATARDIR +#mkdir -p $ATARDIR cd ${ROTDIR} -/work/noaa/global/dhuber/GW/gw_arch/scripts/call_archive.py +${HOMEgfs}/scripts/call_archive.py diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 461b9533db..2546def59d 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -1,14 +1,7 @@ -name: "GDAS" -description: "Basic GDAS analysis and forecast produts and data" {% set cycle_HH = current_cycle | strftime('%H') %} {% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} {% set first_cycle = cycle_YYYYMMDDHH == SDATE %} -{% set n_fhrs = (FHMAX - FHMIN + 1) / FHOUT %} -{% if n_fhrs % NFHRS_PER_GROUP == 0 %} -{% set n_post_groups = n_fhrs // NFHRS_PER_GROUP %} -{% else %} -{% set n_post_groups = n_fhrs // NFHRS_PER_GROUP + 1 %} -{% endif %} +name: "GDAS" target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas.tar" mandatory: - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.anl" @@ -17,21 +10,10 @@ mandatory: - "{{ atmos_grib_1p00_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.1p00.anl.idx" - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmanl.nc" - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfcanl.nc" - {% if not first_cycle } + {% if not first_cycle %} - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmanlprod.log" {% endif %} - {% for post_group in range(n_post_groups) %} - {% set grp_st_fhr = post_group * NFHRS_PER_GROUP * FHOUT + FHMIN %} - {% set grp_en_fhr = (post_group + 1) * NFHRS_PER_GROUP * FHOUT + FHMIN - 1 %} - {% if grp_en_fhr > FHMAX %} - {% set grp_en_fhr = FHMAX %} - {% endif %} - {% if grp_en_fhr != grp_st_fhr %} - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_prod_f{{ '%03d' % grp_st_fhr }}-f{{ '%03d' %grp_en_fhr}}.log" - {% else %} - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_prod_f{{ '%03d' % grp_st_fhr }}.log" - {% endif %} - {% endfor %} + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_prod_f*.log" {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.f{{ '%03d' % fhr }}" - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.f{{ '%03d' % fhr }}.idx" @@ -73,6 +55,10 @@ mandatory: - "{{ atmos_minmon_dir }}/{{ cycle_YYYYMMDDHH }}.reduction.ieee_d" - "{{ atmos_minmon_dir }}/gnorm_data.txt" {% endif %} + {% if DO_WAVE %} + - "{{ wave_grid_dir }}/gdaswave.t{{ cycle_HH }}z.*" + - "{{ wave_station_dir }}/gdaswave.t{{ cycle_HH }}z.*" + {% endif %} optional: - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmvar.yaml" - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmstat" @@ -98,18 +84,8 @@ optional: {% endif %} - "{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmanlupp.log" {% if WRITE_DOPOST %} - {% for post_group in range(n_post_groups) %} - {% set grp_st_fhr = post_group * NFHRS_PER_GROUP * FHOUT + FHMIN %} - {% set grp_en_fhr = (post_group + 1) * NFHRS_PER_GROUP * FHOUT + FHMIN - 1 %} - {% if grp_en_fhr > FHMAX %} - {% set grp_en_fhr = FHMAX %} - {% endif %} - {% if grp_en_fhr != grp_st_fhr %} - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_upp_f{{ '%03d' % grp_st_fhr }}-f{{ '%03d' %grp_en_fhr}}.log" - {% else %} - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_upp_f{{ '%03d' % grp_st_fhr }}.log" - {% endif %} - {% endfor %} + # TODO set the forecast hours explicitly. This will require emulating numpy.array_split + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_upp_f*.log" {% endif %} # WRITE_DOPOST {% if DO_VERFOZN %} {% set oznmon_types = [ diff --git a/parm/archive/gdas_ice.yaml.j2 b/parm/archive/gdas_ice.yaml.j2 new file mode 100644 index 0000000000..defc9acd03 --- /dev/null +++ b/parm/archive/gdas_ice.yaml.j2 @@ -0,0 +1,6 @@ +{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} +name: "GDASICE" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdasice.tar" +mandatory: + - '{{ ice_history_dir }}/gdas.t{{ cyc }}z.*' + - '{{ ice_history_dir }}/ice_in' diff --git a/parm/archive/gdas_ice_restart.yaml.j2 b/parm/archive/gdas_ice_restart.yaml.j2 new file mode 100644 index 0000000000..a33c643cbc --- /dev/null +++ b/parm/archive/gdas_ice_restart.yaml.j2 @@ -0,0 +1,5 @@ +{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} +name: "GDAS_ICE_RESTART" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdasice_restart.tar" +mandatory: + - '{{ ice_history_dir }}/*' diff --git a/parm/archive/gdas_ocean_6hravg.yaml.j2 b/parm/archive/gdas_ocean_6hravg.yaml.j2 new file mode 100644 index 0000000000..a63b09c712 --- /dev/null +++ b/parm/archive/gdas_ocean_6hravg.yaml.j2 @@ -0,0 +1,5 @@ +{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} +name: "GDAS_OCEAN_6HR_AVG" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/ocean_6hravg.tar" +mandatory: + - '{{ ocean_history_dir }}/gfs.ocean.t${cyc}z.6hr_avg.f*.nc' diff --git a/parm/archive/gdas_ocean_daily.yaml.j2 b/parm/archive/gdas_ocean_daily.yaml.j2 new file mode 100644 index 0000000000..4dd9310c6c --- /dev/null +++ b/parm/archive/gdas_ocean_daily.yaml.j2 @@ -0,0 +1,5 @@ +{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} +name: "GDAS_OCEAN_DAILY" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/ocean_daily.tar" +mandatory: + - '{{ ocean_history_dir }}/gfs.ocean.t${cyc}z.daily.f*.nc' diff --git a/parm/archive/gdas_ocean_grib2.yaml.j2 b/parm/archive/gdas_ocean_grib2.yaml.j2 new file mode 100644 index 0000000000..56c6cb4455 --- /dev/null +++ b/parm/archive/gdas_ocean_grib2.yaml.j2 @@ -0,0 +1,10 @@ +{% set cycle_HH = current_cycle | strftime('%H') %} +{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} +name: "GDAS_OCEAN_GRIB2" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/ocean_grib2.tar" +optional: + {% set grid_resolutions = ["5p00", "1p00", "0p25"] %} + {% for grid_res in grid_resolutions %} + - "{{ ocean_grib_dir }}{{ grid_res }}/gfs.ocean.t{{ cycle_HH }}z.{{ grid_res }}.f*.grib2" + - "{{ ocean_grib_dir }}{{ grid_res }}/gfs.ocean.t{{ cycle_HH }}z.{{ grid_res }}.f*.grib2.idx" + {% endfor %} diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 new file mode 100644 index 0000000000..0cd68568f4 --- /dev/null +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -0,0 +1,27 @@ +{% set cycle_HH = current_cycle | strftime('%H') %} +{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} +name: "GDAS_RESTARTA" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas_restarta.tar" +mandatory: + - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.atmi*nc" + - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile1.nc" + - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile2.nc" + - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile3.nc" + - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile4.nc" + - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile5.nc" + - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile6.nc" +optional: + - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.cnvstat" + - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.radstat" + - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.abias" + - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.abias_air" + - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.abias_int" + - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.abias_pc" + - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.dtfanl.nc" + - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.loginc.txt" + - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.amsua_n19.satbias.nc4" + - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.amsua_n19.satbias_cov.nc4" + - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.amsua_n19.tlapse.txt" + - "{{ obs_dir }}/gdas.t{{ cycle_HH }}z.nsstbufr" + - "{{ obs_dir }}/gdas.t{{ cycle_HH }}z.prepbur" + - "{{ obs_dir }}/gdas.t{{ cycle_HH }}z.prepbufr.acft_profiles" diff --git a/parm/archive/gdas_restartb.yaml.j2 b/parm/archive/gdas_restartb.yaml.j2 new file mode 100644 index 0000000000..a99c6cf482 --- /dev/null +++ b/parm/archive/gdas_restartb.yaml.j2 @@ -0,0 +1,5 @@ +{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} +name: "GDAS_RESTARTB" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas_restartb.tar" +mandatory: + - '{{ atmos_restart_dir }}/.' diff --git a/parm/archive/gdas_wave_restart.yaml.j2 b/parm/archive/gdas_wave_restart.yaml.j2 new file mode 100644 index 0000000000..c18ebafce1 --- /dev/null +++ b/parm/archive/gdas_wave_restart.yaml.j2 @@ -0,0 +1,5 @@ +{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} +name: "GDAS_WAVE_RESTART" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas_wave_restart.tar" +mandatory: + - "{{ wave_restart_dir }}/*" diff --git a/scripts/call_archive.py b/scripts/call_archive.py index d68aa7670b..f77276d86b 100755 --- a/scripts/call_archive.py +++ b/scripts/call_archive.py @@ -20,14 +20,14 @@ def main(): # Pull out all the configuration keys needed to run the rest of archive steps keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'PARMgfs', - 'DO_OCN', 'WRITE_DOPOST', 'cyc', 'atmos_analysis_dir', - 'atmos_analysis_ensstat_dir', 'atmos_analysis_mem_dir', 'atmos_bufr_dir', + 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'cyc', 'atmos_analysis_dir', + 'atmos_bufr_dir', 'NFHRS_PER_GROUP', 'HPSSARCH', 'LOCALARCH', 'atmos_gempak_dir', 'atmos_grib_0p25_dir', 'atmos_grib_0p50_dir', - 'atmos_grib_1p00_dir', 'atmos_history_dir', 'atmos_history_ensstat_dir', - 'atmos_history_mem_dir', 'atmos_input_dir', 'atmos_master_dir', + 'atmos_grib_1p00_dir', 'atmos_history_dir', + 'atmos_input_dir', 'atmos_master_dir', 'atmos_minmon_dir', 'atmos_oznmon_dir', 'atmos_radmon_dir', - 'atmos_restart_dir', 'atmos_restart_mem_dir', 'atmos_track_dir', - 'atmos_wmo_dir', 'checm_history_dir', 'chem_analysis_dir', 'conf_dir', + 'atmos_restart_dir', 'atmos_track_dir', + 'atmos_wmo_dir', 'chem_history_dir', 'chem_analysis_dir', 'conf_dir', 'ice_grib_dir', 'ice_history_dir', 'med_restart_dir', 'obs_dir', 'ocean_analysis_dir', 'ocean_grib_dir', 'ocean_history_dir', 'ocean_input_dir', 'ocean_restart_dir', 'wave_grid_dir', diff --git a/sorc/wxflow b/sorc/wxflow index 942b90bfaa..2961846ccb 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit 942b90bfaa14f6b6d7374310dbdfd421ddb30548 +Subproject commit 2961846ccb7f4e277e9f26bce0d6e98029e6985c diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 2364675446..8721545fc0 100755 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -1,18 +1,21 @@ #!/usr/bin/env python3 +import glob from yaml import load from yaml import CLoader as Loader -from typing import Dict, Any +from typing import Dict, Any, List from wxflow import ( logit, cast_strdict_as_dtypedict, AttrDict, get_gid, Task, - htar, - hsi, - parse_j2yaml, - archive_utils) + Htar, + Hsi, + rm_p, + mkdir_p, + chgrp, + parse_j2yaml) from logging import getLogger import os @@ -41,6 +44,7 @@ def __init__(self, config: Dict[str, Any]) -> None: rotdir=self.config.ROTDIR + os.sep + # Collect COM locations relative to ${ROTDIR} local_dict = AttrDict( {'atmos_analysis_dir': self.config.COM_ATMOS_ANALYSIS.replace(rotdir,''), 'atmos_bufr_dir': self.config.COM_ATMOS_BUFR.replace(rotdir,''), @@ -80,7 +84,7 @@ def __init__(self, config: Dict[str, Any]) -> None: @classmethod @logit(logger) - def configure(cls, arch_dict: Dict[str, Any]) -> None: + def configure(self, arch_dict: Dict[str, Any]) -> list[Dict[str, Any]]: """Determine which tarballs will need to be created. Parameters @@ -92,31 +96,46 @@ def configure(cls, arch_dict: Dict[str, Any]) -> None: # Collect datasets that need to be archived # Each dataset represents one tarball + if arch_dict.HPSSARCH: + self.tar_cmd = "htar" + self.hsi = staticmethod(Hsi()) + self.htar = staticmethod(Htar()) + elif arch_dict.LOCALARCH: + self.tar_cmd = "tar" + else: #Nothing to do + raise ValueError(f"Neither HPSSARCH nor LOCALARCH are set to YES.\n" + f"Unable to determine archiving method!") + if(arch_dict.RUN == "gdas"): - datasets = ['gdas'] # , 'gdas_restarta', 'gdas_restartb'] + datasets = ['gdas', 'gdas_restarta', 'gdas_restartb'] - #if(arch_dict.DO_ICE == "YES"): - # datasets.append('gdas_ice') - # datasets.append('gdas_ice_restart') + if(arch_dict.DO_ICE == "YES"): + datasets.append('gdas_ice') + datasets.append('gdas_ice_restart') - #if(arch_dict.DO_OCN == "YES"): - # datasets.append('gdas_ocean_6hravg') - # datasets.append('gdas_ocean_daily') - # datasets.append('gdas_ocean_grib2') + if(arch_dict.DO_OCN == "YES"): + datasets.append('gdas_ocean_6hravg') + datasets.append('gdas_ocean_daily') + datasets.append('gdas_ocean_grib2') - #if(arch_dict.DO_WAVE == "YES"): - # datasets.append('gdas_wave') + if(arch_dict.DO_WAVE == "YES"): + datasets.append('gdas_wave_restart') elif (arch_dict.RUN == "gfs"): - print("Set me up!") + raise NotImplementedError("Archiving is not yet set up for GFS runs") - else: - raise NotImplementedError(f'Archiving is not enabled for {arch_dict.RUN} runs') + elif (arch_dict.RUN == "enkfgdas"): + raise NotImplementedError("Archiving is not yet set up for ENKF GDAS runs") + + elif (arch_dict.RUN == "enkfgfs"): + raise NotImplementedError("Archiving is not yet set up for ENKF GFS runs") - tar_cmd = 'tar' - if arch_dict.HPSSARCH: - tar_cmd = 'htar' + elif (arch_dict.RUN == "gefs"): + raise NotImplementedError("Archiving is not yet set up for GEFS runs") + + else: + raise ValueError(f'Archiving is not enabled for {arch_dict.RUN} runs') archive_sets = [] @@ -126,17 +145,15 @@ def configure(cls, arch_dict: Dict[str, Any]) -> None: archive_filename = os.path.join(archive_parm, dataset + ".yaml.j2") archive_set = parse_j2yaml(archive_filename, arch_dict) - archive_set = cls._create_fileset(archive_set) - archive_set['has_rstprod'] = cls._has_rstprod(archive_set.fileset) - archive_set['protocol'] = tar_cmd + archive_set['fileset'] = Archive._create_fileset(archive_set) + archive_set['has_rstprod'] = Archive._has_rstprod(archive_set.fileset) archive_sets.append(archive_set) return archive_sets - @staticmethod @logit(logger) - def execute(archive_sets: list[Dict[str, Any]]) -> None: + def execute(self, archive_sets: list[Dict[str, Any]]) -> None: """Create the tarballs from the list of yaml dicts. Parameters @@ -147,19 +164,29 @@ def execute(archive_sets: list[Dict[str, Any]]) -> None: for archive_set in archive_sets: + if self.tar_cmd == "htar": + cvf = self.htar.cvf + rm_cmd = self.hsi.rm + else: + cvf = Archive._create_tarball + rm_cmd = rm_p + if archive_set.has_rstprod: try: - archive_utils.ArchiveHandler(archive_set).create() - success = True + cvf(archive_set.target, archive_set.fileset) except: - success = False + rm_cmd(archive_set.target) + raise RuntimeError(f"Failed to create restricted archive {archive_set.target}, deleting!") - cls._protect_rstprod(archive_set) + self._protect_rstprod(archive_set) + + else: + cvf(archive_set.target, archive_set.fileset) @logit(logger) - @classmethod - def _create_fileset(cls, archive_set: Dict[str, Any]) -> None: + @staticmethod + def _create_fileset(archive_set: Dict[str, Any]) -> list: """ Collect the list of all available files from the parsed yaml dict. Globs are expanded and if mandatory files are missing, an error is @@ -175,17 +202,14 @@ def _create_fileset(cls, archive_set: Dict[str, Any]) -> None: Contains full paths for mandatory and optional files to be archived. """ - import glob - - archive_set.fileset = [] + fileset = [] if 'mandatory' in archive_set: for item in archive_set.mandatory: glob_set = glob.glob(item) if len(glob_set) == 0: - raise FileNotFoundError(f'Mandatory file or glob {item} not found!') - for file in glob_set: - cls._check_isfile(file) - archive_set.fileset.append(file) + raise FileNotFoundError(f'Mandatory file, directory, or glob {item} not found!') + for entry in glob_set: + fileset.append(entry) if 'optional' in archive_set: for item in archive_set.optional: @@ -193,40 +217,16 @@ def _create_fileset(cls, archive_set: Dict[str, Any]) -> None: if len(glob_set) == 0: print (f'WARNING: optional file/glob {item} not found!') else: - for file in glob_set: - cls._check_isfile(file,False) - archive_set.fileset.append(file) + for entry in glob_set: + fileset.append(entry) - if len(archive_set.fileset) == 0: + if len(fileset) == 0: print (f'WARNING: the fileset for the {archive_set.name} archive is empty!') - return archive_set - - @logit(logger) - def _check_isfile(filename: str, error_on_noexist = True) -> None: - """ - Checks that the input filename is not a directory. - TODO: Expand this to check that 'filename' is not a glob and that - the file actually exists. - - Parameters - ---------- - filename : str - The name of the file to check including the path. - - error_on_noexist : boolean - (not yet implemented) - If the file does not exist, this flag determines if a warning or - error should be generated. - Errors will always be issued if filename points to a directory. - """ - - if os.path.isdir(filename): - raise IsADirectoryError(f'{file} is a directory\n' + - f'only files are allowed to be archived.') - + return fileset @logit(logger) + @staticmethod def _has_rstprod(fileset: list) -> bool: """ Checks if any files in the input fileset belongs to rstprod. @@ -244,40 +244,74 @@ def _has_rstprod(fileset: list) -> bool: return False # Expand globs and check each file for group ownership - for file in fileset: - if os.stat(filename).st_gid == rstprod_gid: - return True + for file_or_glob in fileset: + glob_set = glob.glob(file_or_glob) + for filename in glob_set: + if os.stat(filename).st_gid == rstprod_gid: + return True return False - - @logit(logger) - def _protect_rstprod(self, archive_set) -> None: + def _protect_rstprod(self, archive_set: Dict[str, any]) -> None: """ Changes the group of the target tarball to rstprod and the permissions to 640. If this fails for any reason, attempt to delete the file before exiting. """ - if archive_set.protocol = "htar": - chgrp = hsi.chgrp - chmod = hsi.chmod - rm = hsi.rm + if self.tar_cmd == "htar": + chgrp_cmd = self.hsi.chgrp + chmod_cmd = self.hsi.chmod + rm_cmd = self.hsi.rm + elif self.tar_cmd == "tar": + chgrp_cmd = chgrp + chmod_cmd = os.fchmod + rm_cmd = rm_p else: - # TODO verify these are correct - chgrp = f_chgrp - chmod = os.chmod - rm = os.rm + raise KeyError(f"Invalid archiving command given: {self.tar_cmd}") try: - chgrp("rstprod", archive_set.target) - chmod("640", archive_set.target) - except + if self.tar_cmd == "htar": + self.hsi.chgrp("rstprod", archive_set.target) + self.hsi.chmod("640", archive_set.target) + else: + chgrp("rstprod", archive_set.target) + os.chmod(archive_set.target, 0o640) + except: try: - rm(archive_set.target) + if self.tar_cmd == "htar": + self.hsi.rm(archive_set.target) + else: + rm_p(archive_set.target) except: pass - raise OSError(f"Failed to protect {archive_set.target}!\n" + raise RuntimeError(f"Failed to protect {archive_set.target}!\n" f"Please verify that it has been deleted!!") + + + @logit(logger) + @staticmethod + def _create_tarball(target: str, fileset: list) -> None: + """Method to create a local tarball. + + Parameters + ---------- + target : str + Tarball to create + + file_list : list + List of files to add to an archive + """ + import tarfile + + # TODO create a set of tar helper functions in wxflow + + # Attempt to create the parent directory if it does not exist + mkdir_p(os.path.dirname(os.path.realpath(target))) + + # Create the archive + with tarfile.open(target, "w") as tarball: + for filename in fileset: + tarball.add(filename) From 7b5a9cc0f212ea8246ffc1add8a143d623944591 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 9 Apr 2024 18:38:36 +0000 Subject: [PATCH 009/128] Archive protocol overhaul. - Use new hsi/htar classes in wxflow - Assign cycle dates/hours in the job to avoid assigning in each template - Rename templates to match current/EE2 naming conventions - Write function to relative-ize COM paths --- jobs/run_archive.sh | 9 +-- parm/archive/gdas.yaml.j2 | 72 +++++++++---------- parm/archive/gdas_ocean_6hravg.yaml.j2 | 5 -- parm/archive/gdas_ocean_daily.yaml.j2 | 5 -- parm/archive/gdas_ocean_grib2.yaml.j2 | 10 --- parm/archive/gdas_restarta.yaml.j2 | 40 ++++++----- parm/archive/gdas_restartb.yaml.j2 | 1 - parm/archive/gdas_wave_restart.yaml.j2 | 5 -- .../{gdas_ice.yaml.j2 => gdasice.yaml.j2} | 4 +- ...estart.yaml.j2 => gdasice_restart.yaml.j2} | 3 +- parm/archive/gdasocean.yaml.j2 | 6 ++ parm/archive/gdasocean_restart.yaml.j2 | 5 ++ parm/archive/gdaswave.yaml.j2 | 6 ++ parm/archive/gdaswave_restart.yaml.j2 | 4 ++ scripts/call_archive.py | 20 +++--- sorc/wxflow | 2 +- ush/python/pygfs/task/archive.py | 68 +++++++++--------- 17 files changed, 128 insertions(+), 137 deletions(-) delete mode 100644 parm/archive/gdas_ocean_6hravg.yaml.j2 delete mode 100644 parm/archive/gdas_ocean_daily.yaml.j2 delete mode 100644 parm/archive/gdas_ocean_grib2.yaml.j2 delete mode 100644 parm/archive/gdas_wave_restart.yaml.j2 rename parm/archive/{gdas_ice.yaml.j2 => gdasice.yaml.j2} (51%) rename parm/archive/{gdas_ice_restart.yaml.j2 => gdasice_restart.yaml.j2} (54%) create mode 100644 parm/archive/gdasocean.yaml.j2 create mode 100644 parm/archive/gdasocean_restart.yaml.j2 create mode 100644 parm/archive/gdaswave.yaml.j2 create mode 100644 parm/archive/gdaswave_restart.yaml.j2 mode change 100755 => 100644 ush/python/pygfs/task/archive.py diff --git a/jobs/run_archive.sh b/jobs/run_archive.sh index bc70daac3e..5210e3ceb7 100755 --- a/jobs/run_archive.sh +++ b/jobs/run_archive.sh @@ -1,6 +1,6 @@ #!/usr/bin/bash # Change to a directory that you can write to -export ATARDIR=/scratch1/NCEPDEV/global/David.Huber/archive_rotdir/minmon +export ATARDIR=/NCEPDEV/emc-global/1year/David.Huber/HERA/test_archive # Change to the head of your clone export HOMEgfs=/scratch1/NCEPDEV/global/David.Huber/GW/gw_archive @@ -16,6 +16,7 @@ export RUN=gdas export ROTDIR=/scratch1/NCEPDEV/global/David.Huber/para/comrot/minmon export PDY=20240302 export cyc=00 +export SDATE=2024030118 export assim_freq=6 export NFHRS_PER_GROUP=3 export COM_ATMOS_ANALYSIS=$ROTDIR/$RUN.$PDY/$cyc/analysis/atmos @@ -67,13 +68,13 @@ export DO_OCN=NO export DO_WAVE=NO export WRITE_DOPOST=.true. export ARCHIVE_RUN=$RUN -export HPSSARCH="NO" -export LOCALARCH="YES" +export HPSSARCH="YES" +export LOCALARCH="NO" # Not needed for task, but required by wxflow export DATA="" export CDUMP="" -rm -rf $ATARDIR +#rm -rf $ATARDIR #mkdir -p $ATARDIR cd ${ROTDIR} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 2546def59d..3c1975b1bc 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -1,42 +1,36 @@ -{% set cycle_HH = current_cycle | strftime('%H') %} -{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} -{% set first_cycle = cycle_YYYYMMDDHH == SDATE %} +{% set head = "gdas.t" + cycle_HH + "z." %} name: "GDAS" target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas.tar" mandatory: - - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.anl" - - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.anl.idx" - - "{{ atmos_grib_1p00_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.1p00.anl" - - "{{ atmos_grib_1p00_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.1p00.anl.idx" - - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmanl.nc" - - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfcanl.nc" + - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.anl" + - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.anl.idx" + - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.anl" + - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.anl.idx" + - "{{ atmos_analysis_dir }}/{{ head }}atmanl.nc" + - "{{ atmos_analysis_dir }}/{{ head }}sfcanl.nc" {% if not first_cycle %} - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmanlprod.log" {% endif %} - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_prod_f*.log" {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} - - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.f{{ '%03d' % fhr }}" - - "{{ atmos_grib_0p25_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - - "{{ atmos_grib_1p00_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.1p00.f{{ '%03d' % fhr }}" - - "{{ atmos_grib_1p00_dir }}/{{ RUN }}.t{{ cycle_HH }}z.pgrb2.1p00.f{{ '%03d' % fhr }}.idx" - - "{{ atmos_master_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ atmos_master_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" - - "{{ atmos_history_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atm.logf{{ '%03d' % fhr }}.txt" - - "{{ atmos_history_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmf{{ '%03d' % fhr }}.nc" - - "{{ atmos_history_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfcf{{ '%03d' % fhr }}.nc" + - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" + - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" + - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" + - "{{ atmos_history_dir }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" + - "{{ atmos_history_dir }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + - "{{ atmos_history_dir }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" + {% endfor %} + {% for fhr in range(FHMIN, FHMAX + FHOUT) %} + - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" {% endfor %} {% if DO_VERFOZN %} - "{{ atmos_oznmon_dir }}/time/bad_cnt.{{ cycle_YYYYMMDDHH }}" - "{{ atmos_oznmon_dir }}/time/bad_diag.{{ cycle_YYYYMMDDHH }}" - "{{ atmos_oznmon_dir }}/time/bad_pen.{{ cycle_YYYYMMDDHH }}" - {% set oznmon_types = [ - "gome_metop-b", "omi_aura", "ompslp_npp", "ompsnp_n20", - "ompsnp_npp", "ompstc8_n20", "ompstc8_npp", "sbuv2_n19" - ] %} - {% for type in oznmon_types %} - "{{ atmos_oznmon_dir }}/time/stdout.time.tar.gz" - "{{ atmos_oznmon_dir }}/horiz/stdout.horiz.tar.gz" - {% endfor %} {% endif %} {% if DO_VERFRAD %} - "{{ atmos_radmon_dir }}/bad_diag.{{ cycle_YYYYMMDDHH }}" @@ -55,20 +49,18 @@ mandatory: - "{{ atmos_minmon_dir }}/{{ cycle_YYYYMMDDHH }}.reduction.ieee_d" - "{{ atmos_minmon_dir }}/gnorm_data.txt" {% endif %} - {% if DO_WAVE %} - - "{{ wave_grid_dir }}/gdaswave.t{{ cycle_HH }}z.*" - - "{{ wave_station_dir }}/gdaswave.t{{ cycle_HH }}z.*" - {% endif %} optional: - - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmvar.yaml" - - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmstat" - - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.gsistat" - - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atmanl.ensres.nc" - - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atma003.ensres.nc" - - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.atma009.ensres.nc" - - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.cnvstat" - - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.oznstat" - - "{{ atmos_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.radstat" + - "{{ atmos_analysis_dir }}/{{ head }}atmvar.yaml" + - "{{ atmos_analysis_dir }}/{{ head }}atmstat" + - "{{ atmos_analysis_dir }}/{{ head }}gsistat" + - "{{ atmos_analysis_dir }}/{{ head }}atmanl.ensres.nc" + - "{{ atmos_analysis_dir }}/{{ head }}atma003.ensres.nc" + - "{{ atmos_analysis_dir }}/{{ head }}atma009.ensres.nc" + - "{{ atmos_analysis_dir }}/{{ head }}cnvstat" + - "{{ atmos_analysis_dir }}/{{ head }}oznstat" + - "{{ chem_analysis_dir }}/{{ head }}aerostat" + - "{{ snow_analysis_dir }}/{{ head }}snowstat.tgz" + - "{{ atmos_analysis_dir }}/{{ head }}radstat" - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}prep.log" - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}anal.log" - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}fcst.log" @@ -76,11 +68,11 @@ optional: - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}verfrad.log" - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}vminmon.log" {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} - - "{{ atmos_master_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ atmos_master_dir }}/{{ RUN }}.t{{ cycle_HH }}z.sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" {% endfor %} {% if DO_AERO %} - - "{{ chem_analysis_dir }}/{{ RUN }}.t{{ cycle_HH }}z.aerostat" + - "{{ chem_analysis_dir }}/{{ head }}aerostat" {% endif %} - "{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmanlupp.log" {% if WRITE_DOPOST %} diff --git a/parm/archive/gdas_ocean_6hravg.yaml.j2 b/parm/archive/gdas_ocean_6hravg.yaml.j2 deleted file mode 100644 index a63b09c712..0000000000 --- a/parm/archive/gdas_ocean_6hravg.yaml.j2 +++ /dev/null @@ -1,5 +0,0 @@ -{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} -name: "GDAS_OCEAN_6HR_AVG" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/ocean_6hravg.tar" -mandatory: - - '{{ ocean_history_dir }}/gfs.ocean.t${cyc}z.6hr_avg.f*.nc' diff --git a/parm/archive/gdas_ocean_daily.yaml.j2 b/parm/archive/gdas_ocean_daily.yaml.j2 deleted file mode 100644 index 4dd9310c6c..0000000000 --- a/parm/archive/gdas_ocean_daily.yaml.j2 +++ /dev/null @@ -1,5 +0,0 @@ -{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} -name: "GDAS_OCEAN_DAILY" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/ocean_daily.tar" -mandatory: - - '{{ ocean_history_dir }}/gfs.ocean.t${cyc}z.daily.f*.nc' diff --git a/parm/archive/gdas_ocean_grib2.yaml.j2 b/parm/archive/gdas_ocean_grib2.yaml.j2 deleted file mode 100644 index 56c6cb4455..0000000000 --- a/parm/archive/gdas_ocean_grib2.yaml.j2 +++ /dev/null @@ -1,10 +0,0 @@ -{% set cycle_HH = current_cycle | strftime('%H') %} -{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} -name: "GDAS_OCEAN_GRIB2" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/ocean_grib2.tar" -optional: - {% set grid_resolutions = ["5p00", "1p00", "0p25"] %} - {% for grid_res in grid_resolutions %} - - "{{ ocean_grib_dir }}{{ grid_res }}/gfs.ocean.t{{ cycle_HH }}z.{{ grid_res }}.f*.grib2" - - "{{ ocean_grib_dir }}{{ grid_res }}/gfs.ocean.t{{ cycle_HH }}z.{{ grid_res }}.f*.grib2.idx" - {% endfor %} diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 0cd68568f4..6010bcedbe 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -1,9 +1,8 @@ -{% set cycle_HH = current_cycle | strftime('%H') %} -{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} +{% set head = "gdas.t" + cycle_HH + "z." %} name: "GDAS_RESTARTA" target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas_restarta.tar" mandatory: - - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.atmi*nc" + - "{{ atmos_analysis_dir }}/{{ head }}atmi*nc" - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile1.nc" - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile2.nc" - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile3.nc" @@ -11,17 +10,24 @@ mandatory: - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile5.nc" - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile6.nc" optional: - - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.cnvstat" - - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.radstat" - - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.abias" - - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.abias_air" - - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.abias_int" - - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.abias_pc" - - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.dtfanl.nc" - - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.loginc.txt" - - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.amsua_n19.satbias.nc4" - - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.amsua_n19.satbias_cov.nc4" - - "{{ atmos_analysis_dir }}/gdas.t{{ cycle_HH }}z.amsua_n19.tlapse.txt" - - "{{ obs_dir }}/gdas.t{{ cycle_HH }}z.nsstbufr" - - "{{ obs_dir }}/gdas.t{{ cycle_HH }}z.prepbur" - - "{{ obs_dir }}/gdas.t{{ cycle_HH }}z.prepbufr.acft_profiles" + - "{{ atmos_analysis_dir }}/{{ head }}cnvstat" + - "{{ atmos_analysis_dir }}/{{ head }}radstat" + - "{{ obs_dir }}/{{ head }}nsstbufr" + - "{{ obs_dir }}/{{ head }}prepbur" + - "{{ obs_dir }}/{{ head }}prepbufr.acft_profiles" + - "{{ atmos_analysis_dir }}/{{ head }}abias" + - "{{ atmos_analysis_dir }}/{{ head }}abias_air" + - "{{ atmos_analysis_dir }}/{{ head }}abias_int" + - "{{ atmos_analysis_dir }}/{{ head }}abias_pc" + - "{{ atmos_analysis_dir }}/{{ head }}dtfanl.nc" + - "{{ atmos_analysis_dir }}/{{ head }}loginc.txt" + - "{{ atmos_analysis_dir }}/{{ head }}amsua_n19.satbias.nc4" + - "{{ atmos_analysis_dir }}/{{ head }}amsua_n19.satbias_cov.nc4" + - "{{ atmos_analysis_dir }}/{{ head }}amsua_n19.tlapse.txt" + - "{{ conf_dir }}/{{ head }}letkfoi.yaml" + - "{{ snow_analysis_dir }}/*0000.sfc_data.tile1.nc" + - "{{ snow_analysis_dir }}/*0000.sfc_data.tile2.nc" + - "{{ snow_analysis_dir }}/*0000.sfc_data.tile3.nc" + - "{{ snow_analysis_dir }}/*0000.sfc_data.tile4.nc" + - "{{ snow_analysis_dir }}/*0000.sfc_data.tile5.nc" + - "{{ snow_analysis_dir }}/*0000.sfc_data.tile6.nc" diff --git a/parm/archive/gdas_restartb.yaml.j2 b/parm/archive/gdas_restartb.yaml.j2 index a99c6cf482..d274a9cd16 100644 --- a/parm/archive/gdas_restartb.yaml.j2 +++ b/parm/archive/gdas_restartb.yaml.j2 @@ -1,4 +1,3 @@ -{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} name: "GDAS_RESTARTB" target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas_restartb.tar" mandatory: diff --git a/parm/archive/gdas_wave_restart.yaml.j2 b/parm/archive/gdas_wave_restart.yaml.j2 deleted file mode 100644 index c18ebafce1..0000000000 --- a/parm/archive/gdas_wave_restart.yaml.j2 +++ /dev/null @@ -1,5 +0,0 @@ -{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} -name: "GDAS_WAVE_RESTART" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas_wave_restart.tar" -mandatory: - - "{{ wave_restart_dir }}/*" diff --git a/parm/archive/gdas_ice.yaml.j2 b/parm/archive/gdasice.yaml.j2 similarity index 51% rename from parm/archive/gdas_ice.yaml.j2 rename to parm/archive/gdasice.yaml.j2 index defc9acd03..47c445b859 100644 --- a/parm/archive/gdas_ice.yaml.j2 +++ b/parm/archive/gdasice.yaml.j2 @@ -1,6 +1,6 @@ -{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} +{% set head = "gdas.t" + cycle_HH + "z." %} name: "GDASICE" target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdasice.tar" mandatory: - - '{{ ice_history_dir }}/gdas.t{{ cyc }}z.*' + - '{{ ice_history_dir }}/{{ head }}*' - '{{ ice_history_dir }}/ice_in' diff --git a/parm/archive/gdas_ice_restart.yaml.j2 b/parm/archive/gdasice_restart.yaml.j2 similarity index 54% rename from parm/archive/gdas_ice_restart.yaml.j2 rename to parm/archive/gdasice_restart.yaml.j2 index a33c643cbc..c9c6bd5b7d 100644 --- a/parm/archive/gdas_ice_restart.yaml.j2 +++ b/parm/archive/gdasice_restart.yaml.j2 @@ -1,5 +1,4 @@ -{% set cycle_YYYYMMDDHH = current_cycle | strftime('%Y%m%d%H') %} -name: "GDAS_ICE_RESTART" +name: "GDASICE_RESTART" target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdasice_restart.tar" mandatory: - '{{ ice_history_dir }}/*' diff --git a/parm/archive/gdasocean.yaml.j2 b/parm/archive/gdasocean.yaml.j2 new file mode 100644 index 0000000000..362968c7bc --- /dev/null +++ b/parm/archive/gdasocean.yaml.j2 @@ -0,0 +1,6 @@ +{% set head = "gdas.t" + cycle_HH + "z." %} +name: "GDASOCEAN" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdasocean.tar" +mandatory: + - '{{ ocean_history_dir }}/{{ head }}*' + - '{{ ocean_input_dir }}/.' diff --git a/parm/archive/gdasocean_restart.yaml.j2 b/parm/archive/gdasocean_restart.yaml.j2 new file mode 100644 index 0000000000..6957c1f9e2 --- /dev/null +++ b/parm/archive/gdasocean_restart.yaml.j2 @@ -0,0 +1,5 @@ +name: "GDASOCEAN_RESTART" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdasocean_restart.tar" +mandatory: + - '{{ ocean_restart_dir }}/*' + - '{{ med_restart_dir }}/*' diff --git a/parm/archive/gdaswave.yaml.j2 b/parm/archive/gdaswave.yaml.j2 new file mode 100644 index 0000000000..295839e185 --- /dev/null +++ b/parm/archive/gdaswave.yaml.j2 @@ -0,0 +1,6 @@ +{% set head = "gdaswave.t" + cycle_HH + "z." %} +name: "GDASWAVE" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdaswave.tar" +mandatory: + - "{{ wave_grid_dir }}/{{ head }}*" + - "{{ wave_station_dir }}/{{ head }}*" diff --git a/parm/archive/gdaswave_restart.yaml.j2 b/parm/archive/gdaswave_restart.yaml.j2 new file mode 100644 index 0000000000..65e6e5caa9 --- /dev/null +++ b/parm/archive/gdaswave_restart.yaml.j2 @@ -0,0 +1,4 @@ +name: "GDASWAVE_RESTART" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdaswave_restart.tar" +mandatory: + - "{{ wave_restart_dir }}/*" diff --git a/scripts/call_archive.py b/scripts/call_archive.py index f77276d86b..c3e6ccb560 100755 --- a/scripts/call_archive.py +++ b/scripts/call_archive.py @@ -20,23 +20,19 @@ def main(): # Pull out all the configuration keys needed to run the rest of archive steps keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'PARMgfs', - 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'cyc', 'atmos_analysis_dir', - 'atmos_bufr_dir', 'NFHRS_PER_GROUP', 'HPSSARCH', 'LOCALARCH', - 'atmos_gempak_dir', 'atmos_grib_0p25_dir', 'atmos_grib_0p50_dir', - 'atmos_grib_1p00_dir', 'atmos_history_dir', - 'atmos_input_dir', 'atmos_master_dir', - 'atmos_minmon_dir', 'atmos_oznmon_dir', 'atmos_radmon_dir', - 'atmos_restart_dir', 'atmos_track_dir', - 'atmos_wmo_dir', 'chem_history_dir', 'chem_analysis_dir', 'conf_dir', - 'ice_grib_dir', 'ice_history_dir', 'med_restart_dir', 'obs_dir', - 'ocean_analysis_dir', 'ocean_grib_dir', 'ocean_history_dir', - 'ocean_input_dir', 'ocean_restart_dir', 'wave_grid_dir', - 'wave_history_dir', 'wave_restart_dir', 'wave_station_dir'] + 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'cyc', 'cycle_YYYYMMDDHH', + 'cycle_HH', 'first_cycle', 'NFHRS_PER_GROUP', 'HPSSARCH', + 'LOCALARCH'] archive_dict = AttrDict() for key in keys: archive_dict[key] = archive.task_config[key] + # Also get all relative paths + for key in archive.task_config.keys(): + if key.endswith("_dir"): + archive_dict[key] = archive.task_config[key] + # Determine which archives to create archive_sets = archive.configure(archive_dict) diff --git a/sorc/wxflow b/sorc/wxflow index 2961846ccb..63db259b85 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit 2961846ccb7f4e277e9f26bce0d6e98029e6985c +Subproject commit 63db259b856b325b802e1deb28f6ee3001f8c03b diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py old mode 100755 new mode 100644 index 8721545fc0..94e0362438 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -44,43 +44,16 @@ def __init__(self, config: Dict[str, Any]) -> None: rotdir=self.config.ROTDIR + os.sep - # Collect COM locations relative to ${ROTDIR} + # Find all absolute paths in the environment and get their relative paths from ${ROTDIR} + path_dict = self._gen_relative_paths(rotdir) local_dict = AttrDict( - {'atmos_analysis_dir': self.config.COM_ATMOS_ANALYSIS.replace(rotdir,''), - 'atmos_bufr_dir': self.config.COM_ATMOS_BUFR.replace(rotdir,''), - 'atmos_gempak_dir': self.config.COM_ATMOS_GEMPAK.replace(rotdir,''), - 'atmos_grib_0p25_dir': self.config.COM_ATMOS_GRIB_0p25.replace(rotdir,''), - 'atmos_grib_0p50_dir': self.config.COM_ATMOS_GRIB_0p50.replace(rotdir,''), - 'atmos_grib_1p00_dir': self.config.COM_ATMOS_GRIB_1p00.replace(rotdir,''), - 'atmos_history_dir': self.config.COM_ATMOS_HISTORY.replace(rotdir,''), - 'atmos_input_dir': self.config.COM_ATMOS_INPUT.replace(rotdir,''), - 'atmos_master_dir': self.config.COM_ATMOS_MASTER.replace(rotdir,''), - 'atmos_minmon_dir': self.config.COM_ATMOS_MINMON.replace(rotdir,''), - 'atmos_oznmon_dir': self.config.COM_ATMOS_OZNMON.replace(rotdir,''), - 'atmos_radmon_dir': self.config.COM_ATMOS_RADMON.replace(rotdir,''), - 'atmos_restart_dir': self.config.COM_ATMOS_RESTART.replace(rotdir,''), - 'atmos_track_dir': self.config.COM_ATMOS_TRACK.replace(rotdir,''), - 'atmos_wmo_dir': self.config.COM_ATMOS_WMO.replace(rotdir,''), - 'chem_history_dir': self.config.COM_CHEM_HISTORY.replace(rotdir,''), - 'chem_analysis_dir': self.config.COM_CHEM_ANALYSIS.replace(rotdir,''), - 'conf_dir': self.config.COM_CONF.replace(rotdir,''), - 'ice_grib_dir': self.config.COM_ICE_GRIB.replace(rotdir,''), - 'ice_history_dir': self.config.COM_ICE_HISTORY.replace(rotdir,''), - 'med_restart_dir': self.config.COM_MED_RESTART.replace(rotdir,''), - 'obs_dir': self.config.COM_OBS.replace(rotdir,''), - 'ocean_analysis_dir': self.config.COM_OCEAN_ANALYSIS.replace(rotdir,''), - 'ocean_grib_dir': self.config.COM_OCEAN_GRIB.replace(rotdir,''), - 'ocean_history_dir': self.config.COM_OCEAN_HISTORY.replace(rotdir,''), - 'ocean_input_dir': self.config.COM_OCEAN_INPUT.replace(rotdir,''), - 'ocean_restart_dir': self.config.COM_OCEAN_RESTART.replace(rotdir,''), - 'wave_grid_dir': self.config.COM_WAVE_GRID.replace(rotdir,''), - 'wave_history_dir': self.config.COM_WAVE_HISTORY.replace(rotdir,''), - 'wave_restart_dir': self.config.COM_WAVE_RESTART.replace(rotdir,''), - 'wave_station_dir': self.config.COM_WAVE_STATION.replace(rotdir,'') + {'cycle_HH': self.runtime_config.current_cycle.strftime("%H"), + 'cycle_YYYYMMDDHH': self.runtime_config.current_cycle.strftime("%Y%m%d%H"), + 'first_cycle': self.runtime_config.current_cycle == self.config.SDATE } ) - self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) + self.task_config = AttrDict(**self.config, **self.runtime_config, **path_dict, **local_dict) @classmethod @logit(logger) @@ -120,6 +93,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> list[Dict[str, Any]]: datasets.append('gdas_ocean_grib2') if(arch_dict.DO_WAVE == "YES"): + datasets.append('gdas_wave') datasets.append('gdas_wave_restart') elif (arch_dict.RUN == "gfs"): @@ -315,3 +289,31 @@ def _create_tarball(target: str, fileset: list) -> None: with tarfile.open(target, "w") as tarball: for filename in fileset: tarball.add(filename) + + @logit(logger) + def _gen_relative_paths(self, root_path: str) -> Dict: + """Generate a dict of paths in self.config relative to root_path + + Parameters + ---------- + root_path : str + Path to base all relative paths off of + + Return + ------ + rel_path_dict : Dict + Dictionary of paths relative to root_path. Members will be named + based on the dict names in self.config. For COM paths, the names will + follow COM_ --> _dir. For all other directories, the + names will follow --> _dir. + """ + + rel_path_dict = {} + for key, value in self.config.items(): + if isinstance(value, str): + if root_path in value: + rel_path = value.replace(root_path, '') + rel_key = (key[4:] if key.startswith("COM_") else key).lower() + "_dir" + rel_path_dict[rel_key] = rel_path + + return rel_path_dict From cbc815d01828a57de04567b2dc04a851c505850f Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 9 Apr 2024 18:48:54 +0000 Subject: [PATCH 010/128] Update snowda archiving to match develop. --- jobs/run_archive.sh | 1 + parm/archive/gdas_restarta.yaml.j2 | 14 ++++++++------ scripts/call_archive.py | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/jobs/run_archive.sh b/jobs/run_archive.sh index 5210e3ceb7..00c3a17c4b 100755 --- a/jobs/run_archive.sh +++ b/jobs/run_archive.sh @@ -66,6 +66,7 @@ export DO_ICE=NO export DO_AERO=NO export DO_OCN=NO export DO_WAVE=NO +export DO_JEDISNOWDA=NO export WRITE_DOPOST=.true. export ARCHIVE_RUN=$RUN export HPSSARCH="YES" diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 6010bcedbe..e76d7ceb75 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -9,6 +9,14 @@ mandatory: - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile4.nc" - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile5.nc" - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile6.nc" + {% if DO_JEDISNOWDA %} + - "{{ snow_analysis_dir }}/*0000.sfc_data.tile1.nc" + - "{{ snow_analysis_dir }}/*0000.sfc_data.tile2.nc" + - "{{ snow_analysis_dir }}/*0000.sfc_data.tile3.nc" + - "{{ snow_analysis_dir }}/*0000.sfc_data.tile4.nc" + - "{{ snow_analysis_dir }}/*0000.sfc_data.tile5.nc" + - "{{ snow_analysis_dir }}/*0000.sfc_data.tile6.nc" + {% endif %} optional: - "{{ atmos_analysis_dir }}/{{ head }}cnvstat" - "{{ atmos_analysis_dir }}/{{ head }}radstat" @@ -25,9 +33,3 @@ optional: - "{{ atmos_analysis_dir }}/{{ head }}amsua_n19.satbias_cov.nc4" - "{{ atmos_analysis_dir }}/{{ head }}amsua_n19.tlapse.txt" - "{{ conf_dir }}/{{ head }}letkfoi.yaml" - - "{{ snow_analysis_dir }}/*0000.sfc_data.tile1.nc" - - "{{ snow_analysis_dir }}/*0000.sfc_data.tile2.nc" - - "{{ snow_analysis_dir }}/*0000.sfc_data.tile3.nc" - - "{{ snow_analysis_dir }}/*0000.sfc_data.tile4.nc" - - "{{ snow_analysis_dir }}/*0000.sfc_data.tile5.nc" - - "{{ snow_analysis_dir }}/*0000.sfc_data.tile6.nc" diff --git a/scripts/call_archive.py b/scripts/call_archive.py index c3e6ccb560..41000908e6 100755 --- a/scripts/call_archive.py +++ b/scripts/call_archive.py @@ -22,7 +22,7 @@ def main(): 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'PARMgfs', 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'cyc', 'cycle_YYYYMMDDHH', 'cycle_HH', 'first_cycle', 'NFHRS_PER_GROUP', 'HPSSARCH', - 'LOCALARCH'] + 'DO_JEDISNOWDA', 'LOCALARCH'] archive_dict = AttrDict() for key in keys: From a53c56a8b20a562f8f319fbfb824f701ab041fce Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 9 Apr 2024 19:04:01 +0000 Subject: [PATCH 011/128] Address PEP8 issues. --- ush/python/pygfs/task/archive.py | 34 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 94e0362438..1ca2cf0474 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -21,6 +21,7 @@ logger = getLogger(__name__.split('.')[-1]) + class Archive(Task): """Task to archive ROTDIR data to HPSS (or locally) """ @@ -42,15 +43,15 @@ def __init__(self, config: Dict[str, Any]) -> None: """ super().__init__(config) - rotdir=self.config.ROTDIR + os.sep + rotdir = self.config.ROTDIR + os.sep # Find all absolute paths in the environment and get their relative paths from ${ROTDIR} path_dict = self._gen_relative_paths(rotdir) local_dict = AttrDict( - {'cycle_HH': self.runtime_config.current_cycle.strftime("%H"), - 'cycle_YYYYMMDDHH': self.runtime_config.current_cycle.strftime("%Y%m%d%H"), - 'first_cycle': self.runtime_config.current_cycle == self.config.SDATE - } + {'cycle_HH': self.runtime_config.current_cycle.strftime("%H"), + 'cycle_YYYYMMDDHH': self.runtime_config.current_cycle.strftime("%Y%m%d%H"), + 'first_cycle': self.runtime_config.current_cycle == self.config.SDATE + } ) self.task_config = AttrDict(**self.config, **self.runtime_config, **path_dict, **local_dict) @@ -69,13 +70,13 @@ def configure(self, arch_dict: Dict[str, Any]) -> list[Dict[str, Any]]: # Collect datasets that need to be archived # Each dataset represents one tarball - if arch_dict.HPSSARCH: + if arch_dict.HPSSARCH: self.tar_cmd = "htar" self.hsi = staticmethod(Hsi()) self.htar = staticmethod(Htar()) elif arch_dict.LOCALARCH: self.tar_cmd = "tar" - else: #Nothing to do + else: # Nothing to do raise ValueError(f"Neither HPSSARCH nor LOCALARCH are set to YES.\n" f"Unable to determine archiving method!") @@ -149,7 +150,8 @@ def execute(self, archive_sets: list[Dict[str, Any]]) -> None: try: cvf(archive_set.target, archive_set.fileset) - except: + # Regardless of exception type, attempt to remove the target + except: # noqa rm_cmd(archive_set.target) raise RuntimeError(f"Failed to create restricted archive {archive_set.target}, deleting!") @@ -189,13 +191,13 @@ def _create_fileset(archive_set: Dict[str, Any]) -> list: for item in archive_set.optional: glob_set = glob.glob(item) if len(glob_set) == 0: - print (f'WARNING: optional file/glob {item} not found!') + print(f'WARNING: optional file/glob {item} not found!') else: for entry in glob_set: fileset.append(entry) if len(fileset) == 0: - print (f'WARNING: the fileset for the {archive_set.name} archive is empty!') + print(f'WARNING: the fileset for the {archive_set.name} archive is empty!') return fileset @@ -252,18 +254,16 @@ def _protect_rstprod(self, archive_set: Dict[str, any]) -> None: else: chgrp("rstprod", archive_set.target) os.chmod(archive_set.target, 0o640) - except: + # Regardless of exception type, attempt to remove the target + except: # noqa try: if self.tar_cmd == "htar": self.hsi.rm(archive_set.target) else: rm_p(archive_set.target) - except: - pass - - raise RuntimeError(f"Failed to protect {archive_set.target}!\n" - f"Please verify that it has been deleted!!") - + finally: + raise RuntimeError(f"Failed to protect {archive_set.target}!\n" + f"Please verify that it has been deleted!!") @logit(logger) @staticmethod From aa204edf985be1bbe03758f2487d215e9faf2725 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 9 Apr 2024 19:15:17 +0000 Subject: [PATCH 012/128] Add gdasocean_analysis template --- parm/archive/gdasocean_analysis.yaml.j2 | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 parm/archive/gdasocean_analysis.yaml.j2 diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 new file mode 100644 index 0000000000..018b3b9531 --- /dev/null +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -0,0 +1,8 @@ +{% set head = "gdas.t" + cycle_HH + "z." %} +name: "GDASOCEAN_ANALYSIS" +target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdasocean_analysis.tar" +mandatory: + - '{{ ocean_analysis_dir }}/{{ head }}*' + - '{{ ocean_analysis_dir }}/gdas.t??z.ocngrid.nc' + - '{{ ocean_analysis_dir }}/diags' + - '{{ ocean_analysis_dir }}/yaml' From fa98a28170cfdfaf8f74b3f17c80c62983cef5e1 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 11 Apr 2024 19:53:49 +0000 Subject: [PATCH 013/128] Add local (ARCDIR) archiving to archive.py. --- jobs/run_archive.sh | 12 +- parm/archive/arcdir.yaml.j2 | 73 ++++++++++ scripts/call_archive.py | 10 +- sorc/wxflow | 2 +- ush/python/pygfs/task/archive.py | 236 ++++++++++++++++++++++--------- 5 files changed, 259 insertions(+), 74 deletions(-) create mode 100644 parm/archive/arcdir.yaml.j2 diff --git a/jobs/run_archive.sh b/jobs/run_archive.sh index 00c3a17c4b..aa3cd7cda6 100755 --- a/jobs/run_archive.sh +++ b/jobs/run_archive.sh @@ -3,6 +3,11 @@ export ATARDIR=/NCEPDEV/emc-global/1year/David.Huber/HERA/test_archive # Change to the head of your clone export HOMEgfs=/scratch1/NCEPDEV/global/David.Huber/GW/gw_archive +# Change to the name of an experiment +export PSLOT=minmon +export ROTDIR=/scratch1/NCEPDEV/global/David.Huber/para/comrot/${PSLOT} +# Change to where to archive local forecast products +export ARCDIR="/scratch1/NCEPDEV/global/David.Huber/archive/minmon_test" set -e module reset @@ -13,7 +18,6 @@ export wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/sorc/wxflow/src" export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export RUN=gdas -export ROTDIR=/scratch1/NCEPDEV/global/David.Huber/para/comrot/minmon export PDY=20240302 export cyc=00 export SDATE=2024030118 @@ -71,6 +75,12 @@ export WRITE_DOPOST=.true. export ARCHIVE_RUN=$RUN export HPSSARCH="YES" export LOCALARCH="NO" +export REALTIME="YES" +export ARCH_WARMICFREQ="4" +export ARCH_FCSTICFREQ="1" +export ARCH_CYC="00" +export SAVEFCSTIC="NO" +export assim_freq=6 # Not needed for task, but required by wxflow export DATA="" export CDUMP="" diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 new file mode 100644 index 0000000000..93d1148ec2 --- /dev/null +++ b/parm/archive/arcdir.yaml.j2 @@ -0,0 +1,73 @@ +{% set head = RUN + ".t" + cycle_HH + "z." %} +{% if RUN == "gdas" or RUN == "gfs" %} +deterministic: + mkdir: + - "{{ ARCDIR }}" + {% if RUN == "gfs" %} + - "{{ ARCDIR }}/tracker.{{ cycle_YYYYMMDDHH }}/{{ RUN }}" + {% endif %} + {% if FITSARC %} + {% if not VFYARC is defined %} + {% set VFYARC = ROTDIR + "/vrfyarch" %} + {% endif %} + - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YYYYMMDD }}/{{ cycle_HH }}" + {% endif %} + copy: + - ["{{ atmos_analysis_dir }}/{{ head }}gsistat", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] + {% if DO_JEDISNOWDA %} + - ["{{ snow_analysis_dir }}/{{ head }}snowstat", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] + {% endif %} + {% if DO_AERO %} + - ["{{ chem_analysis_dir }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] + {% endif %} + - ["{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.anl", "{{ ARCDIR }}/pgbanl.{{ RUN }}.{{ cycle_YYYYMMDDHH }}.grib2"] + {% if RUN == "gfs" %} + {% set fhmin, fhmax, fhout = FHMIN_GFS, FHMAX_GFS, FHOUT_GFS %} + {% elif RUN == "gdas" %} + {% set fhmin, fhmax, fhout = FHMIN, FHMAX, FHOUT %} + {% endif %} + {% for fhr in range(0, fhmax + fhout, fhout) %} + - ["{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YYYYMMDDHH }}.grib2"] + {% endfor %} + {% if RUN == "gfs" %} + - ["{{ atmos_genesis_dir }}/storms.gfso.atcf_gen.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.{{ cycle_YYYYMMDDHH }}"] + - ["{{ atmos_genesis_dir }}/storms.gfso.atcf_gen.altg.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.altg.{{ cycle_YYYYMMDDHH }}"] + - ["{{ atmos_track_dir }}/trak.gfso.atcfunix.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.{{ cycle_YYYYMMDDHH }}"] + - ["{{ atmos_track_dir }}/trak.gfso.atcfunix.altg.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.altg.{{ cycle_YYYYMMDDHH }}"] + {% if path_exists(atmos_track_dir + "/epac") %} + - ["{{ atmos_track_dir }}/epac", "{{ ARCDIR }}/epac"] + {% endif %} + {% if path_exists(atmos_track_dir + "/epac") %} + - ["{{ atmos_track_dir }}/natl", "{{ ARCDIR }}/natl"] + {% endif %} + {% endif %} + {% if RUN == "gfs" and FITSARC %} + {% if FHMAX_FITS is defined %} + {% set fhmax = FHMAX_FITS %} + {% else %} + {% set fhmax = FHMAX_GFS %} + {% endif %} + {% for fhr in range(0, fhmax + 6, 6) %} + {% set sfcfile = atmos_history_dir + "/" + head + "sfcf" + '%03d'|format(fhr) + ".nc" %} + {% set sigfile = atmos_history_dir + "/" + head + "atmf" + '%03d'|format(fhr) + ".nc" %} + - ["{{ sfcfile }}", "{{ VRFYARC }}/{{ RUN }}.{{ cycle_YYYYMMDD }}/{{ cycle_HH }}/{{ sfcfile }}"] + - ["{{ sigfile }}", "{{ VRFYARC }}/{{ RUN }}.{{ cycle_YYYYMMDD }}/{{ cycle_HH }}/{{ sigfile }}"] + {% endfor %} + {% endif %} + sed_replace: + {% set PSLOT4 = PSLOT[0:4]|upper() %} + {% if path_exists(atmos_track_dir ~ "/avno.t" ~ cycle_HH ~ "z.cycle.trackatcfunix") %} + - ["s:AVNO:{{ PSLOT4 }}:g", "{{ atmos_track_dir }}/avno.t{{ cycle_HH }}z.cycle.trackatcfunix", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] + {% endif %} + {% if path_exists(atmos_track_dir ~ "/avnop.t" ~ cycle_HH ~ "z.cycle.trackatcfunixp") %} + - ["s:AVNO:{{ PSLOT4 }}:g", "{{ atmos_track_dir }}/avnop.t{{ cycle_HH }}z.cycle.trackatcfunixp", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] + {% endif %} + {% if RUN == "gdas" %} + {% if path_exists(atmos_track_dir ~ "/gdas.t" ~ cycle_HH ~ "z.cycle.trackatcfunix") %} + - ["s:AVNO:{{ PSLOT4 }}:g", "{{ atmos_track_dir }}/gdas.t{{ cycle_HH }}z.cycle.trackatcfunix", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] + {% endif %} + {% if path_exists(atmos_track_dir ~ "/gdasp.t" ~ cycle_HH ~ "z.cycle.trackatcfunixp") %} + - ["s:AVNO:{{ PSLOT4 }}:g", "{{ atmos_track_dir }}/gdasp.t{{ cycle_HH }}z.cycle.trackatcfunixp", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] + {% endif %} + {% endif %} +{% endif %} # gfs or gdas diff --git a/scripts/call_archive.py b/scripts/call_archive.py index 41000908e6..4357d968c6 100755 --- a/scripts/call_archive.py +++ b/scripts/call_archive.py @@ -21,8 +21,10 @@ def main(): keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'PARMgfs', 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'cyc', 'cycle_YYYYMMDDHH', - 'cycle_HH', 'first_cycle', 'NFHRS_PER_GROUP', 'HPSSARCH', - 'DO_JEDISNOWDA', 'LOCALARCH'] + 'PSLOT', 'cycle_HH', 'first_cycle', 'NFHRS_PER_GROUP', 'HPSSARCH', + 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', + 'ARCH_FCSTICFREQ', 'SAVEFCSTIC', 'ARCH_CYC', 'assim_freq', 'ARCDIR', + 'path_exists'] archive_dict = AttrDict() for key in keys: @@ -34,10 +36,10 @@ def main(): archive_dict[key] = archive.task_config[key] # Determine which archives to create - archive_sets = archive.configure(archive_dict) + arcdir_set, tarball_sets = archive.configure(archive_dict) # Create the archives - archive.execute(archive_sets) + archive.execute(arcdir_set, atardir_sets) if __name__ == '__main__': main() diff --git a/sorc/wxflow b/sorc/wxflow index 63db259b85..fb0c04d2e8 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit 63db259b856b325b802e1deb28f6ee3001f8c03b +Subproject commit fb0c04d2e815d856ee7b5b9099e4a51d39acbb98 diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 1ca2cf0474..1309f49c8f 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -1,24 +1,31 @@ #!/usr/bin/env python3 +from datetime import timedelta + import glob -from yaml import load -from yaml import CLoader as Loader -from typing import Dict, Any, List -from wxflow import ( - logit, - cast_strdict_as_dtypedict, - AttrDict, - get_gid, - Task, - Htar, - Hsi, - rm_p, - mkdir_p, - chgrp, - parse_j2yaml) + from logging import getLogger + import os +from typing import Dict, Any, List + +from yaml import load +from yaml import CLoader as Loader + +from wxflow import (logit, + cast_strdict_as_dtypedict, + AttrDict, + get_gid, + Task, + Htar, + Hsi, + rm_p, + mkdir_p, + chgrp, + FileHandler, + parse_j2yaml) + logger = getLogger(__name__.split('.')[-1]) @@ -50,23 +57,47 @@ def __init__(self, config: Dict[str, Any]) -> None: local_dict = AttrDict( {'cycle_HH': self.runtime_config.current_cycle.strftime("%H"), 'cycle_YYYYMMDDHH': self.runtime_config.current_cycle.strftime("%Y%m%d%H"), + 'cycle_YYYYMMDD': self.runtime_config.current_cycle.strftime("%Y%m%d"), 'first_cycle': self.runtime_config.current_cycle == self.config.SDATE } ) + if self.config.REALTIME: + local_dict['mos_YYYYMMDDHH'] = (self.runtime_config.current_cycle - timedelta(days=1)).strftime("%Y%m%d%H") + else: + local_dict['mos_YYYYMMDDHH'] = cycle_YYYYMMDDHH + + local_dict['mos_YYYYMMDD'] = local_dict['mos_YYYYMMDDHH'][:8] + + # Add the os.path.exists function for use in parsing the jinja files + local_dict['path_exists'] = os.path.exists + self.task_config = AttrDict(**self.config, **self.runtime_config, **path_dict, **local_dict) @classmethod @logit(logger) - def configure(self, arch_dict: Dict[str, Any]) -> list[Dict[str, Any]]: + def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str, Any]]): """Determine which tarballs will need to be created. Parameters ---------- - arch_dict : Dict + arch_dict : Dict[str, Any] Task specific keys, e.g. runtime options (DO_AERO, DO_ICE, etc) + + Return + ------ + arcdir_set : Dict[str, Any] + Set of FileHandler instructions to copy files to the ARCDIR + atardir_sets : List[Dict[str, Any]] + List of tarballs and instructions for creating them via tar or htar """ + archive_parm = os.path.join(arch_dict.PARMgfs, "archive") + + # Collect the dataset to archive locally + arcdir_filename = os.path.join(archive_parm, "arcdir.yaml.j2") + arcdir_set = parse_j2yaml(arcdir_filename, arch_dict) + # Collect datasets that need to be archived # Each dataset represents one tarball @@ -76,26 +107,86 @@ def configure(self, arch_dict: Dict[str, Any]) -> list[Dict[str, Any]]: self.htar = staticmethod(Htar()) elif arch_dict.LOCALARCH: self.tar_cmd = "tar" - else: # Nothing to do - raise ValueError(f"Neither HPSSARCH nor LOCALARCH are set to YES.\n" - f"Unable to determine archiving method!") + else: # Only perform local archiving. Do not create tarballs. + self.tar_cmd = "" + return arcdir_set, [] + + if not os.path.isdir(arch_dict.ROTDIR): + raise FileNotFoundError(f"The ROTDIR ({arch_dict.ROTDIR}) does not exist!") + + # Pull out some common variables + cycle_YYYYMMDDHH = arch_dict.cycle_YYYYMMDDHH + cycle_HH = arch_dict.cycle_HH + first_cycle = arch_dict.first_cycle + ARCH_WARMICFREQ = arch_dict.ARCH_WARMICFREQ + ARCH_FCSTICFREQ = arch_dict.ARCH_FCSTICFREQ + assim_freq = arch_dict.assim_freq + + if arch_dict.RUN == "gdas" or arch_dict.RUN == "gfs": + + ARCHINC_CYC = arch_dict.ARCH_CYC + ARCHICS_CYC = ARCHINC_CYC - assim_freq + if ARCHICS_CYC < 0: + ARCHICS_CYC += 24 + + mm = cycle_YYYYMMDDHH[4:6] + dd = cycle_YYYYMMDDHH[6:8] + # TODO: This math yields multiple dates sharing the same nday + nday = (int(mm) - 1) * 30 + int(dd) + mod = nday % int(ARCH_WARMICFREQ) + + save_warm_ic_a = False + save_warm_ic_b = False + if first_cycle and cycle_HH == ARCHINC_CYC: + save_warm_ic_a = True + save_warm_ic_b = True + elif mod == 0 and cycle_HH == ARCHINC_CYC: + save_warm_ic_a = True + save_warm_ic_b = True + + if ARCHICS_CYC == "18": + nday1 = nday + 1 + mod1 = nday1 % int(ARCH_WARMICFREQ) + + if cycle_HH == ARCHICS_CYC: + if mod1 == 0: + save_warm_ic_b = True + elif first_cycle: + save_warm_ic_b = True + else: + save_warm_ic_b = False + + mod = nday % int(ARCH_FCSTICFREQ) + + SAVEFCSTIC = False + if mod == 0 or first_day: + SAVEFCSTIC = True if(arch_dict.RUN == "gdas"): - datasets = ['gdas', 'gdas_restarta', 'gdas_restartb'] + datasets = ['gdas'] + + if save_warm_ic_a or arch_dict.SAVEFCSTIC: + datasets.append("gdas_restarta") + if arch_dict.DO_WAVE: + datasets.append("gdaswave_restart") + if arch_dict.DO_OCN: + datasets.append("gdasocean_restart") + if arch_dict.DO_ICE: + datasets.append("gdasice_restart") + + if save_warm_ic_b or arch_dict.SAVEFCSTIC: + datasets.append("gdas_restartb") if(arch_dict.DO_ICE == "YES"): - datasets.append('gdas_ice') - datasets.append('gdas_ice_restart') + datasets.append('gdasice') if(arch_dict.DO_OCN == "YES"): - datasets.append('gdas_ocean_6hravg') - datasets.append('gdas_ocean_daily') - datasets.append('gdas_ocean_grib2') + datasets.append('gdasocean') + datasets.append('gdasocean_analysis') if(arch_dict.DO_WAVE == "YES"): - datasets.append('gdas_wave') - datasets.append('gdas_wave_restart') + datasets.append('gdaswave') elif (arch_dict.RUN == "gfs"): raise NotImplementedError("Archiving is not yet set up for GFS runs") @@ -112,32 +203,41 @@ def configure(self, arch_dict: Dict[str, Any]) -> list[Dict[str, Any]]: else: raise ValueError(f'Archiving is not enabled for {arch_dict.RUN} runs') - archive_sets = [] - - archive_parm = os.path.join(arch_dict.PARMgfs, "archive") + atardir_sets = [] for dataset in datasets: archive_filename = os.path.join(archive_parm, dataset + ".yaml.j2") - archive_set = parse_j2yaml(archive_filename, arch_dict) - archive_set['fileset'] = Archive._create_fileset(archive_set) - archive_set['has_rstprod'] = Archive._has_rstprod(archive_set.fileset) + atardir_set = parse_j2yaml(archive_filename, arch_dict) + atardir_set['fileset'] = Archive._create_fileset(atardir_set) + atardir_set['has_rstprod'] = Archive._has_rstprod(atardir_set.fileset) - archive_sets.append(archive_set) + atardir_sets.append(atardir_set) - return archive_sets + return arcdir_set, atardir_sets @logit(logger) - def execute(self, archive_sets: list[Dict[str, Any]]) -> None: - """Create the tarballs from the list of yaml dicts. + def execute(self, arcdir_set: Dict[str, Any], atardir_sets: List[Dict[str, Any]]) -> None: + """Perform local archiving to ARCDIR and create the tarballs from the list of yaml dicts. Parameters ---------- - arch_dict : Dict - Task specific keys, e.g. runtime options (DO_AERO, DO_ICE, etc) + arcdir_set : Dict[str, Any] + FileHandler instructions to populate ARCDIR with + atardir_sets: List[Dict[str, Any]] + Sets of files to archive via tar or htar + + Return + ------ + None """ - for archive_set in archive_sets: + # Copy files to the local ARCDIR + for key in arcdir_set.keys(): + FileHandler(arcdir_set[key]).sync() + + # Generate tarballs + for atardir_set in atardir_sets: if self.tar_cmd == "htar": cvf = self.htar.cvf @@ -146,23 +246,23 @@ def execute(self, archive_sets: list[Dict[str, Any]]) -> None: cvf = Archive._create_tarball rm_cmd = rm_p - if archive_set.has_rstprod: + if atardir_set.has_rstprod: try: - cvf(archive_set.target, archive_set.fileset) + cvf(atardir_set.target, atardir_set.fileset) # Regardless of exception type, attempt to remove the target - except: # noqa - rm_cmd(archive_set.target) - raise RuntimeError(f"Failed to create restricted archive {archive_set.target}, deleting!") + except Exception: + rm_cmd(atardir_set.target) + raise RuntimeError(f"Failed to create restricted archive {atardir_set.target}, deleting!") - self._protect_rstprod(archive_set) + self._protect_rstprod(atardir_set) else: - cvf(archive_set.target, archive_set.fileset) + cvf(atardir_set.target, atardir_set.fileset) @logit(logger) @staticmethod - def _create_fileset(archive_set: Dict[str, Any]) -> list: + def _create_fileset(atardir_set: Dict[str, Any]) -> List: """ Collect the list of all available files from the parsed yaml dict. Globs are expanded and if mandatory files are missing, an error is @@ -174,21 +274,21 @@ def _create_fileset(archive_set: Dict[str, Any]) -> list: Parameters ---------- - archive_set: Dict + atardir_set: Dict Contains full paths for mandatory and optional files to be archived. """ fileset = [] - if 'mandatory' in archive_set: - for item in archive_set.mandatory: + if 'mandatory' in atardir_set: + for item in atardir_set.mandatory: glob_set = glob.glob(item) if len(glob_set) == 0: raise FileNotFoundError(f'Mandatory file, directory, or glob {item} not found!') for entry in glob_set: fileset.append(entry) - if 'optional' in archive_set: - for item in archive_set.optional: + if 'optional' in atardir_set: + for item in atardir_set.optional: glob_set = glob.glob(item) if len(glob_set) == 0: print(f'WARNING: optional file/glob {item} not found!') @@ -197,19 +297,19 @@ def _create_fileset(archive_set: Dict[str, Any]) -> list: fileset.append(entry) if len(fileset) == 0: - print(f'WARNING: the fileset for the {archive_set.name} archive is empty!') + print(f'WARNING: the fileset for the {atardir_set.name} archive is empty!') return fileset @logit(logger) @staticmethod - def _has_rstprod(fileset: list) -> bool: + def _has_rstprod(fileset: List) -> bool: """ Checks if any files in the input fileset belongs to rstprod. Parameters ---------- - fileset : list + fileset : List List of filenames to check. """ @@ -229,7 +329,7 @@ def _has_rstprod(fileset: list) -> bool: return False @logit(logger) - def _protect_rstprod(self, archive_set: Dict[str, any]) -> None: + def _protect_rstprod(self, atardir_set: Dict[str, any]) -> None: """ Changes the group of the target tarball to rstprod and the permissions to 640. If this fails for any reason, attempt to delete the file before exiting. @@ -249,25 +349,25 @@ def _protect_rstprod(self, archive_set: Dict[str, any]) -> None: try: if self.tar_cmd == "htar": - self.hsi.chgrp("rstprod", archive_set.target) - self.hsi.chmod("640", archive_set.target) + self.hsi.chgrp("rstprod", atardir_set.target) + self.hsi.chmod("640", atardir_set.target) else: - chgrp("rstprod", archive_set.target) - os.chmod(archive_set.target, 0o640) + chgrp("rstprod", atardir_set.target) + os.chmod(atardir_set.target, 0o640) # Regardless of exception type, attempt to remove the target - except: # noqa + except Exception: try: if self.tar_cmd == "htar": - self.hsi.rm(archive_set.target) + self.hsi.rm(atardir_set.target) else: - rm_p(archive_set.target) + rm_p(atardir_set.target) finally: - raise RuntimeError(f"Failed to protect {archive_set.target}!\n" + raise RuntimeError(f"Failed to protect {atardir_set.target}!\n" f"Please verify that it has been deleted!!") @logit(logger) @staticmethod - def _create_tarball(target: str, fileset: list) -> None: + def _create_tarball(target: str, fileset: List) -> None: """Method to create a local tarball. Parameters @@ -275,7 +375,7 @@ def _create_tarball(target: str, fileset: list) -> None: target : str Tarball to create - file_list : list + file_list : List List of files to add to an archive """ import tarfile From 406c7a107e68ff94333e1bac46609d04e3b5d5fc Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 11 Apr 2024 20:01:17 +0000 Subject: [PATCH 014/128] Fix PEP8 errors. --- scripts/call_archive.py | 13 +++++++------ ush/python/pygfs/task/archive.py | 26 +++++++++++++------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/scripts/call_archive.py b/scripts/call_archive.py index 4357d968c6..87f4849fa7 100755 --- a/scripts/call_archive.py +++ b/scripts/call_archive.py @@ -19,12 +19,12 @@ def main(): # Pull out all the configuration keys needed to run the rest of archive steps keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', - 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'PARMgfs', - 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'cyc', 'cycle_YYYYMMDDHH', - 'PSLOT', 'cycle_HH', 'first_cycle', 'NFHRS_PER_GROUP', 'HPSSARCH', - 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', - 'ARCH_FCSTICFREQ', 'SAVEFCSTIC', 'ARCH_CYC', 'assim_freq', 'ARCDIR', - 'path_exists'] + 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'PARMgfs', + 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'cyc', 'cycle_YYYYMMDDHH', + 'PSLOT', 'cycle_HH', 'first_cycle', 'NFHRS_PER_GROUP', 'HPSSARCH', + 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', + 'ARCH_FCSTICFREQ', 'SAVEFCSTIC', 'ARCH_CYC', 'assim_freq', 'ARCDIR', + 'path_exists'] archive_dict = AttrDict() for key in keys: @@ -41,5 +41,6 @@ def main(): # Create the archives archive.execute(arcdir_set, atardir_sets) + if __name__ == '__main__': main() diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 1309f49c8f..9170ec8cf3 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -55,11 +55,11 @@ def __init__(self, config: Dict[str, Any]) -> None: # Find all absolute paths in the environment and get their relative paths from ${ROTDIR} path_dict = self._gen_relative_paths(rotdir) local_dict = AttrDict( - {'cycle_HH': self.runtime_config.current_cycle.strftime("%H"), - 'cycle_YYYYMMDDHH': self.runtime_config.current_cycle.strftime("%Y%m%d%H"), - 'cycle_YYYYMMDD': self.runtime_config.current_cycle.strftime("%Y%m%d"), - 'first_cycle': self.runtime_config.current_cycle == self.config.SDATE - } + {'cycle_HH': self.runtime_config.current_cycle.strftime("%H"), + 'cycle_YYYYMMDDHH': self.runtime_config.current_cycle.strftime("%Y%m%d%H"), + 'cycle_YYYYMMDD': self.runtime_config.current_cycle.strftime("%Y%m%d"), + 'first_cycle': self.runtime_config.current_cycle == self.config.SDATE + } ) if self.config.REALTIME: @@ -162,7 +162,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if mod == 0 or first_day: SAVEFCSTIC = True - if(arch_dict.RUN == "gdas"): + if arch_dict.RUN == "gdas": datasets = ['gdas'] @@ -178,26 +178,26 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if save_warm_ic_b or arch_dict.SAVEFCSTIC: datasets.append("gdas_restartb") - if(arch_dict.DO_ICE == "YES"): + if arch_dict.DO_ICE == "YES": datasets.append('gdasice') - if(arch_dict.DO_OCN == "YES"): + if arch_dict.DO_OCN == "YES": datasets.append('gdasocean') datasets.append('gdasocean_analysis') - if(arch_dict.DO_WAVE == "YES"): + if arch_dict.DO_WAVE == "YES": datasets.append('gdaswave') - elif (arch_dict.RUN == "gfs"): + elif arch_dict.RUN == "gfs": raise NotImplementedError("Archiving is not yet set up for GFS runs") - elif (arch_dict.RUN == "enkfgdas"): + elif arch_dict.RUN == "enkfgdas": raise NotImplementedError("Archiving is not yet set up for ENKF GDAS runs") - elif (arch_dict.RUN == "enkfgfs"): + elif arch_dict.RUN == "enkfgfs": raise NotImplementedError("Archiving is not yet set up for ENKF GFS runs") - elif (arch_dict.RUN == "gefs"): + elif arch_dict.RUN == "gefs": raise NotImplementedError("Archiving is not yet set up for GEFS runs") else: From a99ecb5ca64b23a93b812ec9d00f6cead897bef2 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 12 Apr 2024 15:13:12 +0000 Subject: [PATCH 015/128] Let archive.py handle tracker pslot renaming. --- parm/archive/arcdir.yaml.j2 | 25 +-- scripts/call_archive.py | 2 +- scripts/exglobal_archive.sh | 318 ------------------------------- sorc/wxflow | 2 +- ush/python/pygfs/task/archive.py | 47 ++++- 5 files changed, 53 insertions(+), 341 deletions(-) delete mode 100755 scripts/exglobal_archive.sh diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index 93d1148ec2..fd2ab6fdb7 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -34,14 +34,14 @@ deterministic: - ["{{ atmos_genesis_dir }}/storms.gfso.atcf_gen.altg.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.altg.{{ cycle_YYYYMMDDHH }}"] - ["{{ atmos_track_dir }}/trak.gfso.atcfunix.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.{{ cycle_YYYYMMDDHH }}"] - ["{{ atmos_track_dir }}/trak.gfso.atcfunix.altg.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.altg.{{ cycle_YYYYMMDDHH }}"] + ## Only created if tracking is on and there were systems to track {% if path_exists(atmos_track_dir + "/epac") %} - ["{{ atmos_track_dir }}/epac", "{{ ARCDIR }}/epac"] {% endif %} {% if path_exists(atmos_track_dir + "/epac") %} - ["{{ atmos_track_dir }}/natl", "{{ ARCDIR }}/natl"] {% endif %} - {% endif %} - {% if RUN == "gfs" and FITSARC %} + {% if FITSARC %} {% if FHMAX_FITS is defined %} {% set fhmax = FHMAX_FITS %} {% else %} @@ -53,21 +53,10 @@ deterministic: - ["{{ sfcfile }}", "{{ VRFYARC }}/{{ RUN }}.{{ cycle_YYYYMMDD }}/{{ cycle_HH }}/{{ sfcfile }}"] - ["{{ sigfile }}", "{{ VRFYARC }}/{{ RUN }}.{{ cycle_YYYYMMDD }}/{{ cycle_HH }}/{{ sigfile }}"] {% endfor %} - {% endif %} - sed_replace: - {% set PSLOT4 = PSLOT[0:4]|upper() %} - {% if path_exists(atmos_track_dir ~ "/avno.t" ~ cycle_HH ~ "z.cycle.trackatcfunix") %} - - ["s:AVNO:{{ PSLOT4 }}:g", "{{ atmos_track_dir }}/avno.t{{ cycle_HH }}z.cycle.trackatcfunix", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] - {% endif %} - {% if path_exists(atmos_track_dir ~ "/avnop.t" ~ cycle_HH ~ "z.cycle.trackatcfunixp") %} - - ["s:AVNO:{{ PSLOT4 }}:g", "{{ atmos_track_dir }}/avnop.t{{ cycle_HH }}z.cycle.trackatcfunixp", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] - {% endif %} - {% if RUN == "gdas" %} - {% if path_exists(atmos_track_dir ~ "/gdas.t" ~ cycle_HH ~ "z.cycle.trackatcfunix") %} - - ["s:AVNO:{{ PSLOT4 }}:g", "{{ atmos_track_dir }}/gdas.t{{ cycle_HH }}z.cycle.trackatcfunix", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] - {% endif %} - {% if path_exists(atmos_track_dir ~ "/gdasp.t" ~ cycle_HH ~ "z.cycle.trackatcfunixp") %} - - ["s:AVNO:{{ PSLOT4 }}:g", "{{ atmos_track_dir }}/gdasp.t{{ cycle_HH }}z.cycle.trackatcfunixp", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] - {% endif %} + {% endif %} ## FITSARC + {% endif %} ## RUN == "gfs" + {% if path_exists(atmos_track_dir ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YYYYMMDDHH) %} + - ["{{ atmos_track_dir }}/atcfunix.{{ RUN }}.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] + - ["{{ atmos_track_dir }}/atcfunixp.{{ RUN }}.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] {% endif %} {% endif %} # gfs or gdas diff --git a/scripts/call_archive.py b/scripts/call_archive.py index 87f4849fa7..25451eca66 100755 --- a/scripts/call_archive.py +++ b/scripts/call_archive.py @@ -36,7 +36,7 @@ def main(): archive_dict[key] = archive.task_config[key] # Determine which archives to create - arcdir_set, tarball_sets = archive.configure(archive_dict) + arcdir_set, atardir_sets = archive.configure(archive_dict) # Create the archives archive.execute(arcdir_set, atardir_sets) diff --git a/scripts/exglobal_archive.sh b/scripts/exglobal_archive.sh deleted file mode 100755 index 5842c76b57..0000000000 --- a/scripts/exglobal_archive.sh +++ /dev/null @@ -1,318 +0,0 @@ -#! /usr/bin/env bash - -source "${USHgfs}/preamble.sh" - -############################################## -# Begin JOB SPECIFIC work -############################################## - -# ICS are restarts and always lag INC by $assim_freq hours -ARCHINC_CYC=${ARCH_CYC} -ARCHICS_CYC=$((ARCH_CYC-assim_freq)) -if [ "${ARCHICS_CYC}" -lt 0 ]; then - ARCHICS_CYC=$((ARCHICS_CYC+24)) -fi - -# CURRENT CYCLE -APREFIX="${RUN}.t${cyc}z." - -# Realtime parallels run GFS MOS on 1 day delay -# If realtime parallel, back up CDATE_MOS one day -# Ignore possible spelling error (nothing is misspelled) -# shellcheck disable=SC2153 -CDATE_MOS=${PDY}${cyc} -if [ "${REALTIME}" = "YES" ]; then - CDATE_MOS=$(${NDATE} -24 "${PDY}${cyc}") -fi -PDY_MOS="${CDATE_MOS:0:8}" - -############################################################### -# Archive online for verification and diagnostics -############################################################### -source "${USHgfs}/file_utils.sh" - -[[ ! -d ${ARCDIR} ]] && mkdir -p "${ARCDIR}" -nb_copy "${COM_ATMOS_ANALYSIS}/${APREFIX}gsistat" "${ARCDIR}/gsistat.${RUN}.${PDY}${cyc}" -nb_copy "${COM_SNOW_ANALYSIS}/${APREFIX}snowstat" "${ARCDIR}/snowstat.${RUN}.${PDY}${cyc}" -if [[ ${DO_AERO} = "YES" ]]; then - nb_copy "${COM_CHEM_ANALYSIS}/${APREFIX}aerostat" "${ARCDIR}/aerostat.${RUN}.${PDY}${cyc}" -fi -nb_copy "${COM_ATMOS_GRIB_1p00}/${APREFIX}pgrb2.1p00.anl" "${ARCDIR}/pgbanl.${RUN}.${PDY}${cyc}.grib2" - -# Archive 1 degree forecast GRIB2 files for verification -if [[ "${RUN}" == "gfs" ]]; then - fhmax=${FHMAX_GFS} - fhr=0 - while [ "${fhr}" -le "${fhmax}" ]; do - fhr2=$(printf %02i "${fhr}") - fhr3=$(printf %03i "${fhr}") - nb_copy "${COM_ATMOS_GRIB_1p00}/${APREFIX}pgrb2.1p00.f${fhr3}" "${ARCDIR}/pgbf${fhr2}.${RUN}.${PDY}${cyc}.grib2" - fhr=$((10#${fhr} + 10#${FHOUT_GFS} )) - done -fi -if [[ "${RUN}" == "gdas" ]]; then - flist="000 003 006 009" - for fhr in ${flist}; do - fname="${COM_ATMOS_GRIB_1p00}/${APREFIX}pgrb2.1p00.f${fhr}" - # TODO Shouldn't the archived files also use three-digit tags? - fhr2=$(printf %02i $((10#${fhr}))) - nb_copy "${fname}" "${ARCDIR}/pgbf${fhr2}.${RUN}.${PDY}${cyc}.grib2" - done -fi - -if [[ -s "${COM_ATMOS_TRACK}/avno.t${cyc}z.cyclone.trackatcfunix" ]]; then - # shellcheck disable=2153 - PSLOT4=${PSLOT:0:4} - # shellcheck disable= - PSLOT4=${PSLOT4^^} - sed "s:AVNO:${PSLOT4}:g" < "${COM_ATMOS_TRACK}/avno.t${cyc}z.cyclone.trackatcfunix" \ - > "${ARCDIR}/atcfunix.${RUN}.${PDY}${cyc}" - sed "s:AVNO:${PSLOT4}:g" < "${COM_ATMOS_TRACK}/avnop.t${cyc}z.cyclone.trackatcfunix" \ - > "${ARCDIR}/atcfunixp.${RUN}.${PDY}${cyc}" -fi - -if [[ "${RUN}" == "gdas" ]] && [[ -s "${COM_ATMOS_TRACK}/gdas.t${cyc}z.cyclone.trackatcfunix" ]]; then - # shellcheck disable=2153 - PSLOT4=${PSLOT:0:4} - # shellcheck disable= - PSLOT4=${PSLOT4^^} - sed "s:AVNO:${PSLOT4}:g" < "${COM_ATMOS_TRACK}/gdas.t${cyc}z.cyclone.trackatcfunix" \ - > "${ARCDIR}/atcfunix.${RUN}.${PDY}${cyc}" - sed "s:AVNO:${PSLOT4}:g" < "${COM_ATMOS_TRACK}/gdasp.t${cyc}z.cyclone.trackatcfunix" \ - > "${ARCDIR}/atcfunixp.${RUN}.${PDY}${cyc}" -fi - -if [ "${RUN}" = "gfs" ]; then - nb_copy "${COM_ATMOS_GENESIS}/storms.gfso.atcf_gen.${PDY}${cyc}" "${ARCDIR}/." - nb_copy "${COM_ATMOS_GENESIS}/storms.gfso.atcf_gen.altg.${PDY}${cyc}" "${ARCDIR}/." - nb_copy "${COM_ATMOS_TRACK}/trak.gfso.atcfunix.${PDY}${cyc}" "${ARCDIR}/." - nb_copy "${COM_ATMOS_TRACK}/trak.gfso.atcfunix.altg.${PDY}${cyc}" "${ARCDIR}/." - - mkdir -p "${ARCDIR}/tracker.${PDY}${cyc}/${RUN}" - blist="epac natl" - for basin in ${blist}; do - if [[ -f ${basin} ]]; then - cp -rp "${COM_ATMOS_TRACK}/${basin}" "${ARCDIR}/tracker.${PDY}${cyc}/${RUN}" - fi - done -fi - -# Archive required gaussian gfs forecast files for Fit2Obs -if [[ "${RUN}" == "gfs" ]] && [[ "${FITSARC}" = "YES" ]]; then - VFYARC=${VFYARC:-${ROTDIR}/vrfyarch} - [[ ! -d ${VFYARC} ]] && mkdir -p "${VFYARC}" - mkdir -p "${VFYARC}/${RUN}.${PDY}/${cyc}" - prefix="${RUN}.t${cyc}z" - fhmax=${FHMAX_FITS:-${FHMAX_GFS}} - fhr=0 - while [[ ${fhr} -le ${fhmax} ]]; do - fhr3=$(printf %03i "${fhr}") - sfcfile="${COM_ATMOS_HISTORY}/${prefix}.sfcf${fhr3}.nc" - sigfile="${COM_ATMOS_HISTORY}/${prefix}.atmf${fhr3}.nc" - nb_copy "${sfcfile}" "${VFYARC}/${RUN}.${PDY}/${cyc}/" - nb_copy "${sigfile}" "${VFYARC}/${RUN}.${PDY}/${cyc}/" - (( fhr = 10#${fhr} + 6 )) - done -fi - - -############################################################### -# Archive data either to HPSS or locally -if [[ ${HPSSARCH} = "YES" || ${LOCALARCH} = "YES" ]]; then -############################################################### - - # --set the archiving command and create local directories, if necessary - TARCMD="htar" - HSICMD="hsi" - if [[ ${LOCALARCH} = "YES" ]]; then - TARCMD="tar" - HSICMD='' - [[ ! -d "${ATARDIR}/${PDY}${cyc}" ]] && mkdir -p "${ATARDIR}/${PDY}${cyc}" - [[ ! -d "${ATARDIR}/${CDATE_MOS}" ]] && [[ -d "${ROTDIR}/gfsmos.${PDY_MOS}" ]] && [[ "${cyc}" -eq 18 ]] && mkdir -p "${ATARDIR}/${CDATE_MOS}" - fi - - #--determine when to save ICs for warm start and forecast-only runs - SAVEWARMICA="NO" - SAVEWARMICB="NO" - SAVEFCSTIC="NO" - firstday=$(${NDATE} +24 "${SDATE}") - mm="${PDY:2:2}" - dd="${PDY:4:2}" - # TODO: This math yields multiple dates sharing the same nday - nday=$(( (10#${mm}-1)*30+10#${dd} )) - mod=$((nday % ARCH_WARMICFREQ)) - if [[ "${PDY}${cyc}" -eq "${firstday}" ]] && [[ "${cyc}" -eq "${ARCHINC_CYC}" ]]; then SAVEWARMICA="YES" ; fi - if [[ "${PDY}${cyc}" -eq "${firstday}" ]] && [[ "${cyc}" -eq "${ARCHICS_CYC}" ]]; then SAVEWARMICB="YES" ; fi - if [[ "${mod}" -eq 0 ]] && [[ "${cyc}" -eq "${ARCHINC_CYC}" ]]; then SAVEWARMICA="YES" ; fi - if [[ "${mod}" -eq 0 ]] && [[ "${cyc}" -eq "${ARCHICS_CYC}" ]]; then SAVEWARMICB="YES" ; fi - - if [[ "${ARCHICS_CYC}" -eq 18 ]]; then - nday1=$((nday+1)) - mod1=$((nday1 % ARCH_WARMICFREQ)) - if [[ "${mod1}" -eq 0 ]] && [[ "${cyc}" -eq "${ARCHICS_CYC}" ]] ; then SAVEWARMICB="YES" ; fi - if [[ "${mod1}" -ne 0 ]] && [[ "${cyc}" -eq "${ARCHICS_CYC}" ]] ; then SAVEWARMICB="NO" ; fi - if [[ "${PDY}${cyc}" -eq "${SDATE}" ]] && [[ "${cyc}" -eq "${ARCHICS_CYC}" ]] ; then SAVEWARMICB="YES" ; fi - fi - - mod=$((nday % ARCH_FCSTICFREQ)) - if [[ "${mod}" -eq 0 ]] || [[ "${PDY}${cyc}" -eq "${firstday}" ]]; then SAVEFCSTIC="YES" ; fi - - cd "${DATA}" || exit 2 - - "${USHgfs}/hpssarch_gen.sh" "${RUN}" - status=$? - if [ "${status}" -ne 0 ]; then - echo "${USHgfs}/hpssarch_gen.sh ${RUN} failed, ABORT!" - exit "${status}" - fi - - cd "${ROTDIR}" || exit 2 - - if [[ "${RUN}" = "gfs" ]]; then - - targrp_list="gfsa gfsb" - - if [ "${ARCH_GAUSSIAN:-"NO"}" = "YES" ]; then - targrp_list="${targrp_list} gfs_flux gfs_netcdfb gfs_pgrb2b" - if [ "${MODE}" = "cycled" ]; then - targrp_list="${targrp_list} gfs_netcdfa" - fi - fi - - if [ "${DO_WAVE}" = "YES" ]; then - targrp_list="${targrp_list} gfswave" - fi - - if [[ "${DO_OCN}" == "YES" ]]; then - targrp_list="${targrp_list} ocean_6hravg ocean_daily ocean_grib2 gfs_flux_1p00" - fi - - if [[ "${DO_ICE}" == "YES" ]]; then - targrp_list="${targrp_list} ice_6hravg ice_grib2" - fi - - # Aerosols - if [ "${DO_AERO}" = "YES" ]; then - for targrp in chem; do - # TODO: Why is this tar being done here instead of being added to the list? - ${TARCMD} -P -cvf "${ATARDIR}/${PDY}${cyc}/${targrp}.tar" $(cat "${DATA}/${targrp}.txt") - status=$? - if [[ "${status}" -ne 0 ]] && [[ "${PDY}${cyc}" -ge "${firstday}" ]]; then - echo "HTAR ${PDY}${cyc} ${targrp}.tar failed" - exit "${status}" - fi - done - fi - - #for restarts - if [ "${SAVEFCSTIC}" = "YES" ]; then - targrp_list="${targrp_list} gfs_restarta" - fi - - #for downstream products - if [ "${DO_BUFRSND}" = "YES" ]; then - targrp_list="${targrp_list} gfs_downstream" - fi - - #--save mdl gfsmos output from all cycles in the 18Z archive directory - if [[ -d "gfsmos.${PDY_MOS}" ]] && [[ "${cyc}" -eq 18 ]]; then - set +e - # TODO: Why is this tar being done here instead of being added to the list? - ${TARCMD} -P -cvf "${ATARDIR}/${CDATE_MOS}/gfsmos.tar" "./gfsmos.${PDY_MOS}" - status=$? - if [[ "${status}" -ne 0 ]] && [[ "${PDY}${cyc}" -ge "${firstday}" ]]; then - echo "${TARCMD^^} ${PDY}${cyc} gfsmos.tar failed" - exit "${status}" - fi - set_strict - fi - elif [[ "${RUN}" = "gdas" ]]; then - - targrp_list="gdas" - - #gdaswave - if [ "${DO_WAVE}" = "YES" ]; then - targrp_list="${targrp_list} gdaswave" - fi - - #gdasocean - if [ "${DO_OCN}" = "YES" ]; then - targrp_list="${targrp_list} gdasocean gdasocean_analysis" - fi - - #gdasice - if [ "${DO_ICE}" = "YES" ]; then - targrp_list="${targrp_list} gdasice" - fi - - if [ "${SAVEWARMICA}" = "YES" ] || [ "${SAVEFCSTIC}" = "YES" ]; then - targrp_list="${targrp_list} gdas_restarta" - if [ "${DO_WAVE}" = "YES" ]; then targrp_list="${targrp_list} gdaswave_restart"; fi - if [ "${DO_OCN}" = "YES" ]; then targrp_list="${targrp_list} gdasocean_restart"; fi - if [ "${DO_ICE}" = "YES" ]; then targrp_list="${targrp_list} gdasice_restart"; fi - fi - - if [ "${SAVEWARMICB}" = "YES" ] || [ "${SAVEFCSTIC}" = "YES" ]; then - targrp_list="${targrp_list} gdas_restartb" - fi - fi - - # Turn on extended globbing options - shopt -s extglob - for targrp in ${targrp_list}; do - set +e - - # Test whether gdas.tar or gdas_restarta.tar will have rstprod data - has_rstprod="NO" - case ${targrp} in - 'gdas'|'gdas_restarta') - # Test for rstprod in each archived file - while IFS= read -r file; do - if [[ -f ${file} ]]; then - group=$( stat -c "%G" "${file}" ) - if [[ "${group}" == "rstprod" ]]; then - has_rstprod="YES" - break - fi - fi - done < "${DATA}/${targrp}.txt" - - ;; - *) ;; - esac - - # Create the tarball - tar_fl="${ATARDIR}/${PDY}${cyc}/${targrp}.tar" - ${TARCMD} -P -cvf "${tar_fl}" $(cat "${DATA}/${targrp}.txt") - status=$? - - # Change group to rstprod if it was found even if htar/tar failed in case of partial creation - if [[ "${has_rstprod}" == "YES" ]]; then - ${HSICMD} chgrp rstprod "${tar_fl}" - stat_chgrp=$? - ${HSICMD} chmod 640 "${tar_fl}" - stat_chgrp=$((stat_chgrp+$?)) - if [[ "${stat_chgrp}" -gt 0 ]]; then - echo "FATAL ERROR: Unable to properly restrict ${tar_fl}!" - echo "Attempting to delete ${tar_fl}" - ${HSICMD} rm "${tar_fl}" - echo "Please verify that ${tar_fl} was deleted!" - exit "${stat_chgrp}" - fi - fi - - # For safety, test if the htar/tar command failed after changing groups - if [[ "${status}" -ne 0 ]] && [[ "${PDY}${cyc}" -ge "${firstday}" ]]; then - echo "FATAL ERROR: ${TARCMD} ${tar_fl} failed" - exit "${status}" - fi - set_strict - done - # Turn extended globbing back off - shopt -u extglob - -############################################################### -fi ##end of HPSS archive -############################################################### - -exit 0 diff --git a/sorc/wxflow b/sorc/wxflow index fb0c04d2e8..63db259b85 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit fb0c04d2e815d856ee7b5b9099e4a51d39acbb98 +Subproject commit 63db259b856b325b802e1deb28f6ee3001f8c03b diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 9170ec8cf3..2942587079 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -74,7 +74,6 @@ def __init__(self, config: Dict[str, Any]) -> None: self.task_config = AttrDict(**self.config, **self.runtime_config, **path_dict, **local_dict) - @classmethod @logit(logger) def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str, Any]]): """Determine which tarballs will need to be created. @@ -103,8 +102,8 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if arch_dict.HPSSARCH: self.tar_cmd = "htar" - self.hsi = staticmethod(Hsi()) - self.htar = staticmethod(Htar()) + self.hsi = Hsi() + self.htar = Htar() elif arch_dict.LOCALARCH: self.tar_cmd = "tar" else: # Only perform local archiving. Do not create tarballs. @@ -162,6 +161,9 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if mod == 0 or first_day: SAVEFCSTIC = True + # If present, copy and replace the experiment names in cycle tracking files + self._rename_cyclone_expt(arch_dict) + if arch_dict.RUN == "gdas": datasets = ['gdas'] @@ -417,3 +419,42 @@ def _gen_relative_paths(self, root_path: str) -> Dict: rel_path_dict[rel_key] = rel_path return rel_path_dict + + def _rename_cyclone_expt(self, arch_dict) -> None: + + # Rename the experiment in the tracker files from "AVNO" to the + # first 4 letters of PSLOT. + pslot4 = arch_dict.PSLOT.upper() + if len(arch_dict.PSLOT) > 4: + pslot4 = arch_dict.PSLOT[0:4].upper() + + track_dir = arch_dict.atmos_track_dir + cycle_HH = str(arch_dict.cycle_HH) + run = arch_dict.RUN + + if run == "gfs": + in_track_file = track_dir + "/avno.t" + cycle_HH + "z.cycle.trackatcfunix" + in_track_p_file = track_dir + "/avnop.t" + cycle_HH + "z.cycle.trackatcfunixp" + elif run == "gdas": + in_track_file = track_dir + "/gdas.t" + cycle_HH + "z.cycle.trackatcfunix" + in_track_p_file = track_dir + "/gdasp.t" + cycle_HH + "z.cycle.trackatcfunixp" + + if not os.path.isfile(in_track_file): + return + + cycle_YYYYMMDDHH = str(arch_dict.cycle_YYYYMMDDHH) + + out_track_file = track_dir + "/atcfunix." + run + "." + cycle_YYYYMMDDHH + out_track_p_file = track_dir + "/atcfunixp." + run + "." + cycle_YYYYMMDDHH + + def replace_string_from_to_file(filename_in, filename_out, search_str, replace_str): + with open(filename_in) as old_file: + lines = old_file.readlines() + + out_lines = [line.replace(search, replace) for line in lines] + + with open(filename_out, "w") as new_file: + new_file.writelines(out_lines) + + replace_string_from_to_file(in_track_file, out_track_file, "AVNO", pslot4) + replace_string_from_to_file(in_track_p_file, out_track_p_file, "AVNO", pslot4) From 1fcccf0966652374a8f7946d0a6144dcf6190201 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 15 Apr 2024 15:34:52 +0000 Subject: [PATCH 016/128] Rewrite archive job to use exglobal_archive.py. --- jobs/JGLOBAL_ARCHIVE | 2 +- jobs/rocoto/arch.sh | 6 ++ jobs/run_archive.sh | 93 ------------------- parm/archive/arcdir.yaml.j2 | 24 ++--- parm/archive/gdas.yaml.j2 | 81 ++++++++-------- parm/archive/gdas_restarta.yaml.j2 | 14 ++- .../{call_archive.py => exglobal_archive.py} | 11 ++- ush/python/pygfs/task/archive.py | 38 ++++---- 8 files changed, 102 insertions(+), 167 deletions(-) delete mode 100755 jobs/run_archive.sh rename scripts/{call_archive.py => exglobal_archive.py} (86%) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index e8994ed39d..b6e2a6e17b 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -30,7 +30,7 @@ done # Run archive script ############################################################### -${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive.sh} +${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive.py} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/arch.sh b/jobs/rocoto/arch.sh index d949b7d76f..9dd1a0309e 100755 --- a/jobs/rocoto/arch.sh +++ b/jobs/rocoto/arch.sh @@ -8,6 +8,12 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" +############################################################### +# setup python path for workflow utilities and tasks +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" +export PYTHONPATH + export job="arch" export jobid="${job}.$$" diff --git a/jobs/run_archive.sh b/jobs/run_archive.sh deleted file mode 100755 index aa3cd7cda6..0000000000 --- a/jobs/run_archive.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/bash -# Change to a directory that you can write to -export ATARDIR=/NCEPDEV/emc-global/1year/David.Huber/HERA/test_archive -# Change to the head of your clone -export HOMEgfs=/scratch1/NCEPDEV/global/David.Huber/GW/gw_archive -# Change to the name of an experiment -export PSLOT=minmon -export ROTDIR=/scratch1/NCEPDEV/global/David.Huber/para/comrot/${PSLOT} -# Change to where to archive local forecast products -export ARCDIR="/scratch1/NCEPDEV/global/David.Huber/archive/minmon_test" - -set -e -module reset -module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core -module load stack-intel stack-python py-jinja2 py-pyyaml -export PARMgfs=$HOMEgfs/parm -export wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/sorc/wxflow/src" -export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" - -export RUN=gdas -export PDY=20240302 -export cyc=00 -export SDATE=2024030118 -export assim_freq=6 -export NFHRS_PER_GROUP=3 -export COM_ATMOS_ANALYSIS=$ROTDIR/$RUN.$PDY/$cyc/analysis/atmos -export COM_ATMOS_BUFR=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/bufr -export COM_ATMOS_GEMPAK=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/gempak/ -export COM_ATMOS_GENESIS=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/cyclone/genesis_vital -export COM_ATMOS_HISTORY=$ROTDIR/$RUN.$PDY/$cyc/model_data/atmos/history -export COM_ATMOS_INPUT=$ROTDIR/$RUN.$PDY/$cyc/model_data/atmos/input -export COM_ATMOS_MASTER=$ROTDIR/$RUN.$PDY/$cyc/model_data/atmos/master -export COM_ATMOS_RESTART=$ROTDIR/$RUN.$PDY/$cyc/model_data/atmos/restart -export COM_ATMOS_TRACK=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/cyclone/tracks -export COM_ATMOS_WMO=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/wmo -export COM_CHEM_HISTORY=$ROTDIR/$RUN.$PDY/$cyc/model_data/chem/history -export COM_CHEM_ANALYSIS=$ROTDIR/$RUN.$PDY/$cyc/analysis/chem -export COM_MED_RESTART=$ROTDIR/$RUN.$PDY/$cyc/model_data/med/restart -export COM_ICE_HISTORY=$ROTDIR/$RUN.$PDY/$cyc/model_data/ice/history -export COM_ICE_INPUT=$ROTDIR/$RUN.$PDY/$cyc/model_data/ice/input -export COM_ICE_RESTART=$ROTDIR/$RUN.$PDY/$cyc/model_data/ice/restart -export COM_ICE_GRIB=$ROTDIR/$RUN.$PDY/$cyc/products/ice/grib2 -export COM_OBS=$ROTDIR/$RUN.$PDY/$cyc/obs -export COM_TOP=$ROTDIR/$RUN.$PDY/$cyc -export COM_OCEAN_HISTORY=$ROTDIR/$RUN.$PDY/$cyc/model_data/ocean/history -export COM_OCEAN_INPUT=$ROTDIR/$RUN.$PDY/$cyc/model_data/ocean/input -export COM_OCEAN_RESTART=$ROTDIR/$RUN.$PDY/$cyc/model_data/ocean/restart -export COM_OCEAN_GRIB=$ROTDIR/$RUN.$PDY/$cyc/products/ocean/grib2 -export COM_OCEAN_NETCDF=$ROTDIR/$RUN.$PDY/$cyc/products/ocean/netcdf -export COM_OCEAN_ANALYSIS=$ROTDIR/$RUN.$PDY/$cyc/analysis/ocean -export COM_WAVE_GRID=$ROTDIR/$RUN.$PDY/$cyc/products/wave/gridded -export COM_WAVE_HISTORY=$ROTDIR/$RUN.$PDY/$cyc/model_data/wave/history -export COM_WAVE_STATION=$ROTDIR/$RUN.$PDY/$cyc/products/wave/station -export COM_WAVE_RESTART=$ROTDIR/$RUN.$PDY/$cyc/products/wave/restart -export COM_ATMOS_OZNMON=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/oznmon -export COM_ATMOS_RADMON=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/radmon -export COM_ATMOS_MINMON=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/minmon -export COM_CONF=$ROTDIR/$RUN.$PDY/$cyc/conf -export COM_ATMOS_GRIB_0p25=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/grib2/0p25 -export COM_ATMOS_GRIB_0p50=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/grib2/0p50 -export COM_ATMOS_GRIB_1p00=$ROTDIR/$RUN.$PDY/$cyc/products/atmos/grib2/1p00 - -export FHMIN=0 -export FHMAX=9 -export FHOUT=1 -export DO_VERFRAD=YES -export DO_VMINMON=YES -export DO_VERFOZN=YES -export DO_ICE=NO -export DO_AERO=NO -export DO_OCN=NO -export DO_WAVE=NO -export DO_JEDISNOWDA=NO -export WRITE_DOPOST=.true. -export ARCHIVE_RUN=$RUN -export HPSSARCH="YES" -export LOCALARCH="NO" -export REALTIME="YES" -export ARCH_WARMICFREQ="4" -export ARCH_FCSTICFREQ="1" -export ARCH_CYC="00" -export SAVEFCSTIC="NO" -export assim_freq=6 -# Not needed for task, but required by wxflow -export DATA="" -export CDUMP="" - -#rm -rf $ATARDIR -#mkdir -p $ATARDIR - -cd ${ROTDIR} - -${HOMEgfs}/scripts/call_archive.py diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index fd2ab6fdb7..ea44437f1a 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -13,6 +13,7 @@ deterministic: - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YYYYMMDD }}/{{ cycle_HH }}" {% endif %} copy: + {% if not first_cycle and MODE == "cycled" %} - ["{{ atmos_analysis_dir }}/{{ head }}gsistat", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] {% if DO_JEDISNOWDA %} - ["{{ snow_analysis_dir }}/{{ head }}snowstat", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] @@ -21,6 +22,7 @@ deterministic: - ["{{ chem_analysis_dir }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] {% endif %} - ["{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.anl", "{{ ARCDIR }}/pgbanl.{{ RUN }}.{{ cycle_YYYYMMDDHH }}.grib2"] + {% endif %} # Full cycle {% if RUN == "gfs" %} {% set fhmin, fhmax, fhout = FHMIN_GFS, FHMAX_GFS, FHOUT_GFS %} {% elif RUN == "gdas" %} @@ -30,17 +32,6 @@ deterministic: - ["{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YYYYMMDDHH }}.grib2"] {% endfor %} {% if RUN == "gfs" %} - - ["{{ atmos_genesis_dir }}/storms.gfso.atcf_gen.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.{{ cycle_YYYYMMDDHH }}"] - - ["{{ atmos_genesis_dir }}/storms.gfso.atcf_gen.altg.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.altg.{{ cycle_YYYYMMDDHH }}"] - - ["{{ atmos_track_dir }}/trak.gfso.atcfunix.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.{{ cycle_YYYYMMDDHH }}"] - - ["{{ atmos_track_dir }}/trak.gfso.atcfunix.altg.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.altg.{{ cycle_YYYYMMDDHH }}"] - ## Only created if tracking is on and there were systems to track - {% if path_exists(atmos_track_dir + "/epac") %} - - ["{{ atmos_track_dir }}/epac", "{{ ARCDIR }}/epac"] - {% endif %} - {% if path_exists(atmos_track_dir + "/epac") %} - - ["{{ atmos_track_dir }}/natl", "{{ ARCDIR }}/natl"] - {% endif %} {% if FITSARC %} {% if FHMAX_FITS is defined %} {% set fhmax = FHMAX_FITS %} @@ -54,6 +45,17 @@ deterministic: - ["{{ sigfile }}", "{{ VRFYARC }}/{{ RUN }}.{{ cycle_YYYYMMDD }}/{{ cycle_HH }}/{{ sigfile }}"] {% endfor %} {% endif %} ## FITSARC + - ["{{ atmos_genesis_dir }}/storms.gfso.atcf_gen.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.{{ cycle_YYYYMMDDHH }}"] + - ["{{ atmos_genesis_dir }}/storms.gfso.atcf_gen.altg.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.altg.{{ cycle_YYYYMMDDHH }}"] + - ["{{ atmos_track_dir }}/trak.gfso.atcfunix.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.{{ cycle_YYYYMMDDHH }}"] + - ["{{ atmos_track_dir }}/trak.gfso.atcfunix.altg.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.altg.{{ cycle_YYYYMMDDHH }}"] + ## Only created if tracking is on and there were systems to track + {% if path_exists(atmos_track_dir + "/epac") %} + - ["{{ atmos_track_dir }}/epac", "{{ ARCDIR }}/epac"] + {% endif %} + {% if path_exists(atmos_track_dir + "/epac") %} + - ["{{ atmos_track_dir }}/natl", "{{ ARCDIR }}/natl"] + {% endif %} {% endif %} ## RUN == "gfs" {% if path_exists(atmos_track_dir ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YYYYMMDDHH) %} - ["{{ atmos_track_dir }}/atcfunix.{{ RUN }}.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 3c1975b1bc..cac9fc9a63 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -2,35 +2,34 @@ name: "GDAS" target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas.tar" mandatory: + {% if not first_cycle and MODE == "cycled" %} - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.anl" - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.anl.idx" - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.anl" - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.anl.idx" - "{{ atmos_analysis_dir }}/{{ head }}atmanl.nc" - "{{ atmos_analysis_dir }}/{{ head }}sfcanl.nc" - {% if not first_cycle %} - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmanlprod.log" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}prep.log" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}anal.log" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmanlupp.log" + - "{{ atmos_analysis_dir }}/{{ head }}atmanl.ensres.nc" + - "{{ atmos_analysis_dir }}/{{ head }}atma003.ensres.nc" + - "{{ atmos_analysis_dir }}/{{ head }}atma009.ensres.nc" + - "{{ atmos_analysis_dir }}/{{ head }}cnvstat" + - "{{ atmos_analysis_dir }}/{{ head }}oznstat" + - "{{ atmos_analysis_dir }}/{{ head }}radstat" + {% if DO_JEDIATMVAR %} + - "{{ atmos_analysis_dir }}/{{ head }}atmvar.yaml" + - "{{ atmos_analysis_dir }}/{{ head }}atmstat" {% endif %} - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_prod_f*.log" - {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} - - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" - - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" - - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" - - "{{ atmos_history_dir }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" - - "{{ atmos_history_dir }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" - - "{{ atmos_history_dir }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" - {% endfor %} - {% for fhr in range(FHMIN, FHMAX + FHOUT) %} - - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" - {% endfor %} {% if DO_VERFOZN %} - "{{ atmos_oznmon_dir }}/time/bad_cnt.{{ cycle_YYYYMMDDHH }}" - "{{ atmos_oznmon_dir }}/time/bad_diag.{{ cycle_YYYYMMDDHH }}" - "{{ atmos_oznmon_dir }}/time/bad_pen.{{ cycle_YYYYMMDDHH }}" - "{{ atmos_oznmon_dir }}/time/stdout.time.tar.gz" - "{{ atmos_oznmon_dir }}/horiz/stdout.horiz.tar.gz" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}verfozn.log" {% endif %} {% if DO_VERFRAD %} - "{{ atmos_radmon_dir }}/bad_diag.{{ cycle_YYYYMMDDHH }}" @@ -41,6 +40,7 @@ mandatory: - "{{ atmos_radmon_dir }}/radmon_bcor.tar.gz" - "{{ atmos_radmon_dir }}/radmon_time.tar.gz" - "{{ atmos_radmon_dir }}/warning.{{ cycle_YYYYMMDDHH }}" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}verfrad.log" {% endif %} {% if DO_VMINMON %} - "{{ atmos_minmon_dir }}/{{ cycle_YYYYMMDDHH }}.costs.txt" @@ -48,37 +48,33 @@ mandatory: - "{{ atmos_minmon_dir }}/{{ cycle_YYYYMMDDHH }}.gnorms.ieee_d" - "{{ atmos_minmon_dir }}/{{ cycle_YYYYMMDDHH }}.reduction.ieee_d" - "{{ atmos_minmon_dir }}/gnorm_data.txt" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}vminmon.log" {% endif %} -optional: - - "{{ atmos_analysis_dir }}/{{ head }}atmvar.yaml" - - "{{ atmos_analysis_dir }}/{{ head }}atmstat" - - "{{ atmos_analysis_dir }}/{{ head }}gsistat" - - "{{ atmos_analysis_dir }}/{{ head }}atmanl.ensres.nc" - - "{{ atmos_analysis_dir }}/{{ head }}atma003.ensres.nc" - - "{{ atmos_analysis_dir }}/{{ head }}atma009.ensres.nc" - - "{{ atmos_analysis_dir }}/{{ head }}cnvstat" - - "{{ atmos_analysis_dir }}/{{ head }}oznstat" + {% if DO_AERO %} - "{{ chem_analysis_dir }}/{{ head }}aerostat" + {% endif %} + {% if DO_JEDISNOWDA %} - "{{ snow_analysis_dir }}/{{ head }}snowstat.tgz" - - "{{ atmos_analysis_dir }}/{{ head }}radstat" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}prep.log" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}anal.log" + {% endif %} + {% endif %} # Full cycle - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}fcst.log" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}verfozn.log" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}verfrad.log" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}vminmon.log" + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_prod_f*.log" {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} + - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" + - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" + - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" + - "{{ atmos_history_dir }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" + - "{{ atmos_history_dir }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + - "{{ atmos_history_dir }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" + {% endfor %} + {% for fhr in range(FHMIN, FHMAX + FHOUT) %} - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" {% endfor %} - {% if DO_AERO %} - - "{{ chem_analysis_dir }}/{{ head }}aerostat" - {% endif %} - - "{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmanlupp.log" - {% if WRITE_DOPOST %} - # TODO set the forecast hours explicitly. This will require emulating numpy.array_split - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_upp_f*.log" - {% endif %} # WRITE_DOPOST +optional: + {% if not first_cycle and MODE == "cycled" %} + - "{{ atmos_analysis_dir }}/{{ head }}gsistat" {% if DO_VERFOZN %} {% set oznmon_types = [ "gome_metop-b", "omi_aura", "ompslp_npp", "ompsnp_n20", @@ -95,3 +91,12 @@ optional: {% endfor %} {% endfor %} {% endif %} + {% endif %} + {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} + - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + {% endfor %} + {% if not WRITE_DOPOST %} + # TODO set the forecast hours explicitly. This will require emulating numpy.array_split + - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_upp_f*.log" + {% endif %} ## not WRITE_DOPOST diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index e76d7ceb75..d677ad4d2f 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -2,6 +2,7 @@ name: "GDAS_RESTARTA" target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas_restarta.tar" mandatory: + {% if not first_cycle and MODE == "cycled" %} - "{{ atmos_analysis_dir }}/{{ head }}atmi*nc" - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile1.nc" - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile2.nc" @@ -9,6 +10,10 @@ mandatory: - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile4.nc" - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile5.nc" - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile6.nc" + - "{{ atmos_analysis_dir }}/{{ head }}abias" + - "{{ atmos_analysis_dir }}/{{ head }}abias_air" + - "{{ atmos_analysis_dir }}/{{ head }}abias_pc" + - "{{ atmos_analysis_dir }}/{{ head }}radstat" {% if DO_JEDISNOWDA %} - "{{ snow_analysis_dir }}/*0000.sfc_data.tile1.nc" - "{{ snow_analysis_dir }}/*0000.sfc_data.tile2.nc" @@ -17,19 +22,20 @@ mandatory: - "{{ snow_analysis_dir }}/*0000.sfc_data.tile5.nc" - "{{ snow_analysis_dir }}/*0000.sfc_data.tile6.nc" {% endif %} + {% endif %} optional: + {% if not first_cycle and MODE == "cycled" %} - "{{ atmos_analysis_dir }}/{{ head }}cnvstat" - - "{{ atmos_analysis_dir }}/{{ head }}radstat" - "{{ obs_dir }}/{{ head }}nsstbufr" - "{{ obs_dir }}/{{ head }}prepbur" - "{{ obs_dir }}/{{ head }}prepbufr.acft_profiles" - - "{{ atmos_analysis_dir }}/{{ head }}abias" - - "{{ atmos_analysis_dir }}/{{ head }}abias_air" - "{{ atmos_analysis_dir }}/{{ head }}abias_int" - - "{{ atmos_analysis_dir }}/{{ head }}abias_pc" - "{{ atmos_analysis_dir }}/{{ head }}dtfanl.nc" - "{{ atmos_analysis_dir }}/{{ head }}loginc.txt" - "{{ atmos_analysis_dir }}/{{ head }}amsua_n19.satbias.nc4" - "{{ atmos_analysis_dir }}/{{ head }}amsua_n19.satbias_cov.nc4" - "{{ atmos_analysis_dir }}/{{ head }}amsua_n19.tlapse.txt" + {% if DO_JEDISNOWDA %} - "{{ conf_dir }}/{{ head }}letkfoi.yaml" + {% endif %} + {% endif %} diff --git a/scripts/call_archive.py b/scripts/exglobal_archive.py similarity index 86% rename from scripts/call_archive.py rename to scripts/exglobal_archive.py index 25451eca66..a8f91400ea 100755 --- a/scripts/call_archive.py +++ b/scripts/exglobal_archive.py @@ -2,7 +2,7 @@ import os -from wxflow import AttrDict, Logger, logit, cast_strdict_as_dtypedict +from wxflow import AttrDict, Logger, logit, cast_strdict_as_dtypedict, chdir from pygfs.task.archive import Archive # initialize root logger @@ -21,9 +21,9 @@ def main(): keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'PARMgfs', 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'cyc', 'cycle_YYYYMMDDHH', - 'PSLOT', 'cycle_HH', 'first_cycle', 'NFHRS_PER_GROUP', 'HPSSARCH', + 'PSLOT', 'cycle_HH', 'first_cycle', 'HPSSARCH', 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', - 'ARCH_FCSTICFREQ', 'SAVEFCSTIC', 'ARCH_CYC', 'assim_freq', 'ARCDIR', + 'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', 'path_exists'] archive_dict = AttrDict() @@ -35,12 +35,17 @@ def main(): if key.endswith("_dir"): archive_dict[key] = archive.task_config[key] + cwd = os.getcwd() + + os.chdir(config.ROTDIR) + # Determine which archives to create arcdir_set, atardir_sets = archive.configure(archive_dict) # Create the archives archive.execute(arcdir_set, atardir_sets) + os.chdir(cwd) if __name__ == '__main__': main() diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 2942587079..00dbe92262 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -168,7 +168,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str datasets = ['gdas'] - if save_warm_ic_a or arch_dict.SAVEFCSTIC: + if save_warm_ic_a or SAVEFCSTIC: datasets.append("gdas_restarta") if arch_dict.DO_WAVE: datasets.append("gdaswave_restart") @@ -177,7 +177,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if arch_dict.DO_ICE: datasets.append("gdasice_restart") - if save_warm_ic_b or arch_dict.SAVEFCSTIC: + if save_warm_ic_b or SAVEFCSTIC: datasets.append("gdas_restartb") if arch_dict.DO_ICE == "YES": @@ -241,8 +241,13 @@ def execute(self, arcdir_set: Dict[str, Any], atardir_sets: List[Dict[str, Any]] # Generate tarballs for atardir_set in atardir_sets: + if len(atardir_set.fileset) == 0: + print(f"WARNING skipping would-be empty archive {atardir_set.target}.") + continue + if self.tar_cmd == "htar": cvf = self.htar.cvf + create = self.htar.create rm_cmd = self.hsi.rm else: cvf = Archive._create_tarball @@ -282,24 +287,23 @@ def _create_fileset(atardir_set: Dict[str, Any]) -> List: fileset = [] if 'mandatory' in atardir_set: - for item in atardir_set.mandatory: - glob_set = glob.glob(item) - if len(glob_set) == 0: - raise FileNotFoundError(f'Mandatory file, directory, or glob {item} not found!') - for entry in glob_set: - fileset.append(entry) - - if 'optional' in atardir_set: - for item in atardir_set.optional: - glob_set = glob.glob(item) - if len(glob_set) == 0: - print(f'WARNING: optional file/glob {item} not found!') - else: + if atardir_set.mandatory is not None: + for item in atardir_set.mandatory: + glob_set = glob.glob(item) + if len(glob_set) == 0: + raise FileNotFoundError(f'Mandatory file, directory, or glob {item} not found!') for entry in glob_set: fileset.append(entry) - if len(fileset) == 0: - print(f'WARNING: the fileset for the {atardir_set.name} archive is empty!') + if 'optional' in atardir_set: + if atardir_set.optional is not None: + for item in atardir_set.optional: + glob_set = glob.glob(item) + if len(glob_set) == 0: + print(f'WARNING: optional file/glob {item} not found!') + else: + for entry in glob_set: + fileset.append(entry) return fileset From bd662120eadcedddbe3eeb7cc8928efc08e22af7 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 17 Apr 2024 17:33:08 +0000 Subject: [PATCH 017/128] Enabled GFS archiving and reworked some features - Moved the cycle_YYYYMMDDHH, etc, variable declarations to templates - Use COM directories inside of the templates (instead of _dir directories) - Add for-looping for tile numbers - Added several new templates needed for the GFS - Improved ATARDIR archiving logic --- parm/archive/arcdir.yaml.j2 | 54 +++--- parm/archive/chem.yaml.j2 | 7 + parm/archive/gdas.yaml.j2 | 132 ++++++------- parm/archive/gdas_restarta.yaml.j2 | 58 +++--- parm/archive/gdas_restartb.yaml.j2 | 7 +- parm/archive/gdasice.yaml.j2 | 8 +- parm/archive/gdasice_restart.yaml.j2 | 7 +- parm/archive/gdasocean.yaml.j2 | 8 +- parm/archive/gdasocean_analysis.yaml.j2 | 12 +- parm/archive/gdasocean_restart.yaml.j2 | 9 +- parm/archive/gdaswave.yaml.j2 | 8 +- parm/archive/gdaswave_restart.yaml.j2 | 6 +- parm/archive/gfs_downstream.yaml.j2 | 11 ++ parm/archive/gfs_flux.yaml.j2 | 10 + parm/archive/gfs_flux_1p00.yaml.j2 | 10 + parm/archive/gfs_netcdfa.yaml.j2 | 12 ++ parm/archive/gfs_netcdfb.yaml.j2 | 10 + parm/archive/gfs_pgrb2b.yaml.j2 | 23 +++ parm/archive/gfs_restarta.yaml.j2 | 17 ++ parm/archive/gfs_restartb.yaml.j2 | 7 + parm/archive/gfsa.yaml.j2 | 57 ++++++ parm/archive/gfsb.yaml.j2 | 17 ++ parm/archive/gfswave.yaml.j2 | 9 + parm/archive/ice_6hravg.yaml.j2 | 8 + parm/archive/ice_grib2.yaml.j2 | 17 ++ parm/archive/ocean_6hravg.yaml.j2 | 7 + parm/archive/ocean_daily.yaml.j2 | 7 + parm/archive/ocean_grib2.yaml.j2 | 17 ++ scripts/exglobal_archive.py | 17 +- sorc/wxflow | 2 +- ush/python/pygfs/task/archive.py | 238 ++++++++++++++---------- 31 files changed, 562 insertions(+), 250 deletions(-) create mode 100644 parm/archive/chem.yaml.j2 create mode 100644 parm/archive/gfs_downstream.yaml.j2 create mode 100644 parm/archive/gfs_flux.yaml.j2 create mode 100644 parm/archive/gfs_flux_1p00.yaml.j2 create mode 100644 parm/archive/gfs_netcdfa.yaml.j2 create mode 100644 parm/archive/gfs_netcdfb.yaml.j2 create mode 100644 parm/archive/gfs_pgrb2b.yaml.j2 create mode 100644 parm/archive/gfs_restarta.yaml.j2 create mode 100644 parm/archive/gfs_restartb.yaml.j2 create mode 100644 parm/archive/gfsa.yaml.j2 create mode 100644 parm/archive/gfsb.yaml.j2 create mode 100644 parm/archive/gfswave.yaml.j2 create mode 100644 parm/archive/ice_6hravg.yaml.j2 create mode 100644 parm/archive/ice_grib2.yaml.j2 create mode 100644 parm/archive/ocean_6hravg.yaml.j2 create mode 100644 parm/archive/ocean_daily.yaml.j2 create mode 100644 parm/archive/ocean_grib2.yaml.j2 diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index ea44437f1a..a47e28dc9a 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -1,27 +1,30 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set cycle_YMD = current_cycle | to_YMD %} {% set head = RUN + ".t" + cycle_HH + "z." %} {% if RUN == "gdas" or RUN == "gfs" %} deterministic: mkdir: - "{{ ARCDIR }}" {% if RUN == "gfs" %} - - "{{ ARCDIR }}/tracker.{{ cycle_YYYYMMDDHH }}/{{ RUN }}" + - "{{ ARCDIR }}/tracker.{{ cycle_YMDH }}/{{ RUN }}" {% endif %} {% if FITSARC %} {% if not VFYARC is defined %} {% set VFYARC = ROTDIR + "/vrfyarch" %} {% endif %} - - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YYYYMMDD }}/{{ cycle_HH }}" + - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}" {% endif %} copy: - {% if not first_cycle and MODE == "cycled" %} - - ["{{ atmos_analysis_dir }}/{{ head }}gsistat", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] + {% if not current_cycle == SDATE and MODE == "cycled" %} + - ["{{ COM_ATMOS_ANALYSIS }}/{{ head }}gsistat", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}"] {% if DO_JEDISNOWDA %} - - ["{{ snow_analysis_dir }}/{{ head }}snowstat", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] + - ["{{ COM_SNOW_ANALYSIS }}/{{ head }}snowstat", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} {% if DO_AERO %} - - ["{{ chem_analysis_dir }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] + - ["{{ COM_CHEM_ANALYSIS }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} - - ["{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.anl", "{{ ARCDIR }}/pgbanl.{{ RUN }}.{{ cycle_YYYYMMDDHH }}.grib2"] + - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.anl", "{{ ARCDIR }}/pgbanl.{{ RUN }}.{{ cycle_YMDH }}.grib2"] {% endif %} # Full cycle {% if RUN == "gfs" %} {% set fhmin, fhmax, fhout = FHMIN_GFS, FHMAX_GFS, FHOUT_GFS %} @@ -29,7 +32,7 @@ deterministic: {% set fhmin, fhmax, fhout = FHMIN, FHMAX, FHOUT %} {% endif %} {% for fhr in range(0, fhmax + fhout, fhout) %} - - ["{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YYYYMMDDHH }}.grib2"] + - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"] {% endfor %} {% if RUN == "gfs" %} {% if FITSARC %} @@ -39,26 +42,29 @@ deterministic: {% set fhmax = FHMAX_GFS %} {% endif %} {% for fhr in range(0, fhmax + 6, 6) %} - {% set sfcfile = atmos_history_dir + "/" + head + "sfcf" + '%03d'|format(fhr) + ".nc" %} - {% set sigfile = atmos_history_dir + "/" + head + "atmf" + '%03d'|format(fhr) + ".nc" %} - - ["{{ sfcfile }}", "{{ VRFYARC }}/{{ RUN }}.{{ cycle_YYYYMMDD }}/{{ cycle_HH }}/{{ sfcfile }}"] - - ["{{ sigfile }}", "{{ VRFYARC }}/{{ RUN }}.{{ cycle_YYYYMMDD }}/{{ cycle_HH }}/{{ sigfile }}"] + {% set sfcfile = COM_ATMOS_HISTORY + "/" + head + "sfcf" + '%03d'|format(fhr) + ".nc" %} + {% set sigfile = COM_ATMOS_HISTORY + "/" + head + "atmf" + '%03d'|format(fhr) + ".nc" %} + - ["{{ sfcfile }}", "{{ VRFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sfcfile }}"] + - ["{{ sigfile }}", "{{ VRFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sigfile }}"] {% endfor %} {% endif %} ## FITSARC - - ["{{ atmos_genesis_dir }}/storms.gfso.atcf_gen.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.{{ cycle_YYYYMMDDHH }}"] - - ["{{ atmos_genesis_dir }}/storms.gfso.atcf_gen.altg.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.altg.{{ cycle_YYYYMMDDHH }}"] - - ["{{ atmos_track_dir }}/trak.gfso.atcfunix.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.{{ cycle_YYYYMMDDHH }}"] - - ["{{ atmos_track_dir }}/trak.gfso.atcfunix.altg.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.altg.{{ cycle_YYYYMMDDHH }}"] - ## Only created if tracking is on and there were systems to track - {% if path_exists(atmos_track_dir + "/epac") %} - - ["{{ atmos_track_dir }}/epac", "{{ ARCDIR }}/epac"] + {% if path_exists(COM_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH) %} + - ["{{ COM_ATMOS_GENESIS }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}"] + - ["{{ COM_ATMOS_GENESIS }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}"] + {% endif %} + {% if path_exists(COM_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH) %} + - ["{{ COM_ATMOS_TRACK }}/trak.gfso.atcfunix.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.{{ cycle_YMDH }}"] + - ["{{ COM_ATMOS_TRACK }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}"] {% endif %} - {% if path_exists(atmos_track_dir + "/epac") %} - - ["{{ atmos_track_dir }}/natl", "{{ ARCDIR }}/natl"] + ## Only created if tracking is on and there were systems to track + {% for basin in ["epac", "natl"] %} + {% if path_exists(COM_ATMOS_TRACK + "/" + basin) %} + - ["{{ COM_ATMOS_TRACK }}/{{ basin }}", "{{ ARCDIR }}/{{ basin }}"] {% endif %} + {% endfor %} {% endif %} ## RUN == "gfs" - {% if path_exists(atmos_track_dir ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YYYYMMDDHH) %} - - ["{{ atmos_track_dir }}/atcfunix.{{ RUN }}.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] - - ["{{ atmos_track_dir }}/atcfunixp.{{ RUN }}.{{ cycle_YYYYMMDDHH }}", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YYYYMMDDHH }}"] + {% if arch_tracks %} + - ["{{ COM_ATMOS_TRACK }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COM_ATMOS_TRACK }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} {% endif %} # gfs or gdas diff --git a/parm/archive/chem.yaml.j2 b/parm/archive/chem.yaml.j2 new file mode 100644 index 0000000000..2aebcb816d --- /dev/null +++ b/parm/archive/chem.yaml.j2 @@ -0,0 +1,7 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gocart" %} +name: "CHEM" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/chem.tar" +mandatory: + - "{{ COM_CHEM_HISTORY | relpath(ROTDIR) }}/{{ head }}*" diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index cac9fc9a63..ef39bdbe9a 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -1,80 +1,82 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} {% set head = "gdas.t" + cycle_HH + "z." %} name: "GDAS" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas.tar" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas.tar" mandatory: - {% if not first_cycle and MODE == "cycled" %} - - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.anl" - - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.anl.idx" - - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.anl" - - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.anl.idx" - - "{{ atmos_analysis_dir }}/{{ head }}atmanl.nc" - - "{{ atmos_analysis_dir }}/{{ head }}sfcanl.nc" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmanlprod.log" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}prep.log" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}anal.log" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmanlupp.log" - - "{{ atmos_analysis_dir }}/{{ head }}atmanl.ensres.nc" - - "{{ atmos_analysis_dir }}/{{ head }}atma003.ensres.nc" - - "{{ atmos_analysis_dir }}/{{ head }}atma009.ensres.nc" - - "{{ atmos_analysis_dir }}/{{ head }}cnvstat" - - "{{ atmos_analysis_dir }}/{{ head }}oznstat" - - "{{ atmos_analysis_dir }}/{{ head }}radstat" + {% if not current_cycle == SDATE and MODE == "cycled" %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma003.ensres.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma009.ensres.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}oznstat" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" {% if DO_JEDIATMVAR %} - - "{{ atmos_analysis_dir }}/{{ head }}atmvar.yaml" - - "{{ atmos_analysis_dir }}/{{ head }}atmstat" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" {% endif %} {% if DO_VERFOZN %} - - "{{ atmos_oznmon_dir }}/time/bad_cnt.{{ cycle_YYYYMMDDHH }}" - - "{{ atmos_oznmon_dir }}/time/bad_diag.{{ cycle_YYYYMMDDHH }}" - - "{{ atmos_oznmon_dir }}/time/bad_pen.{{ cycle_YYYYMMDDHH }}" - - "{{ atmos_oznmon_dir }}/time/stdout.time.tar.gz" - - "{{ atmos_oznmon_dir }}/horiz/stdout.horiz.tar.gz" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}verfozn.log" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_cnt.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_diag.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_pen.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/stdout.time.tar.gz" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/horiz/stdout.horiz.tar.gz" + - "logs/{{ cycle_YMDH }}/{{ RUN }}verfozn.log" {% endif %} {% if DO_VERFRAD %} - - "{{ atmos_radmon_dir }}/bad_diag.{{ cycle_YYYYMMDDHH }}" - - "{{ atmos_radmon_dir }}/bad_pen.{{ cycle_YYYYMMDDHH }}" - - "{{ atmos_radmon_dir }}/low_count.{{ cycle_YYYYMMDDHH }}" - - "{{ atmos_radmon_dir }}/radmon_angle.tar.gz" - - "{{ atmos_radmon_dir }}/radmon_bcoef.tar.gz" - - "{{ atmos_radmon_dir }}/radmon_bcor.tar.gz" - - "{{ atmos_radmon_dir }}/radmon_time.tar.gz" - - "{{ atmos_radmon_dir }}/warning.{{ cycle_YYYYMMDDHH }}" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}verfrad.log" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/low_count.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_angle.tar.gz" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcoef.tar.gz" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcor.tar.gz" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_time.tar.gz" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/warning.{{ cycle_YMDH }}" + - "logs/{{ cycle_YMDH }}/{{ RUN }}verfrad.log" {% endif %} {% if DO_VMINMON %} - - "{{ atmos_minmon_dir }}/{{ cycle_YYYYMMDDHH }}.costs.txt" - - "{{ atmos_minmon_dir }}/{{ cycle_YYYYMMDDHH }}.cost_terms.txt" - - "{{ atmos_minmon_dir }}/{{ cycle_YYYYMMDDHH }}.gnorms.ieee_d" - - "{{ atmos_minmon_dir }}/{{ cycle_YYYYMMDDHH }}.reduction.ieee_d" - - "{{ atmos_minmon_dir }}/gnorm_data.txt" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}vminmon.log" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" + - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" {% endif %} {% if DO_AERO %} - - "{{ chem_analysis_dir }}/{{ head }}aerostat" + - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} {% if DO_JEDISNOWDA %} - - "{{ snow_analysis_dir }}/{{ head }}snowstat.tgz" + - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snowstat.tgz" {% endif %} {% endif %} # Full cycle - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}fcst.log" - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_prod_f*.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} - - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" - - "{{ atmos_grib_0p25_dir }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" - - "{{ atmos_grib_1p00_dir }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" - - "{{ atmos_history_dir }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" - - "{{ atmos_history_dir }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" - - "{{ atmos_history_dir }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" {% endfor %} {% for fhr in range(FHMIN, FHMAX + FHOUT) %} - - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" {% endfor %} optional: - {% if not first_cycle and MODE == "cycled" %} - - "{{ atmos_analysis_dir }}/{{ head }}gsistat" + {% if not current_cycle == SDATE and MODE == "cycled" %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" {% if DO_VERFOZN %} {% set oznmon_types = [ "gome_metop-b", "omi_aura", "ompslp_npp", "ompsnp_n20", @@ -82,21 +84,21 @@ optional: ] %} {% for group in [ "horiz", "time" ] %} {% for type in oznmon_types %} - - "{{ atmos_oznmon_dir }}/{{ group }}/{{ type }}.anl.ctl" - - "{{ atmos_oznmon_dir }}/{{ group }}/{{ type }}.anl.{{ cycle_YYYYMMDDHH }}.ieee_d" - - "{{ atmos_oznmon_dir }}/{{ group }}/{{ type }}.anl.{{ cycle_YYYYMMDDHH }}.ieee_d.gz" - - "{{ atmos_oznmon_dir }}/{{ group }}/{{ type }}.ges.ctl" - - "{{ atmos_oznmon_dir }}/{{ group }}/{{ type }}.ges.{{ cycle_YYYYMMDDHH }}.ieee_d" - - "{{ atmos_oznmon_dir }}/{{ group }}/{{ type }}.ges.{{ cycle_YYYYMMDDHH }}.ieee_d.gz" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.ctl" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.{{ cycle_YMDH }}.ieee_d" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.{{ cycle_YMDH }}.ieee_d.gz" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.ctl" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.{{ cycle_YMDH }}.ieee_d" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.{{ cycle_YMDH }}.ieee_d.gz" {% endfor %} {% endfor %} {% endif %} {% endif %} {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} - - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ atmos_master_dir }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" {% endfor %} {% if not WRITE_DOPOST %} # TODO set the forecast hours explicitly. This will require emulating numpy.array_split - - "logs/{{ cycle_YYYYMMDDHH }}/{{ RUN }}atmos_upp_f*.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" {% endif %} ## not WRITE_DOPOST diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index d677ad4d2f..cc822e9424 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -1,41 +1,37 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} {% set head = "gdas.t" + cycle_HH + "z." %} name: "GDAS_RESTARTA" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas_restarta.tar" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restarta.tar" mandatory: - {% if not first_cycle and MODE == "cycled" %} - - "{{ atmos_analysis_dir }}/{{ head }}atmi*nc" - - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile1.nc" - - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile2.nc" - - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile3.nc" - - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile4.nc" - - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile5.nc" - - "{{ atmos_restart_dir }}/*0000.sfcanl_data.tile6.nc" - - "{{ atmos_analysis_dir }}/{{ head }}abias" - - "{{ atmos_analysis_dir }}/{{ head }}abias_air" - - "{{ atmos_analysis_dir }}/{{ head }}abias_pc" - - "{{ atmos_analysis_dir }}/{{ head }}radstat" + {% if not current_cycle == SDATE and MODE == "cycled" %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi*nc" + {% for i_tile in range(1,7) %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/*0000.sfcanl_data.tile{{ i_tile }}.nc" + {% endfor %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_air" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_pc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" {% if DO_JEDISNOWDA %} - - "{{ snow_analysis_dir }}/*0000.sfc_data.tile1.nc" - - "{{ snow_analysis_dir }}/*0000.sfc_data.tile2.nc" - - "{{ snow_analysis_dir }}/*0000.sfc_data.tile3.nc" - - "{{ snow_analysis_dir }}/*0000.sfc_data.tile4.nc" - - "{{ snow_analysis_dir }}/*0000.sfc_data.tile5.nc" - - "{{ snow_analysis_dir }}/*0000.sfc_data.tile6.nc" + {% for i_tile in range(1,7) %} + - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/*0000.sfc_data.tile{{ i_tile }}.nc" + {% endfor %} {% endif %} {% endif %} optional: - {% if not first_cycle and MODE == "cycled" %} - - "{{ atmos_analysis_dir }}/{{ head }}cnvstat" - - "{{ obs_dir }}/{{ head }}nsstbufr" - - "{{ obs_dir }}/{{ head }}prepbur" - - "{{ obs_dir }}/{{ head }}prepbufr.acft_profiles" - - "{{ atmos_analysis_dir }}/{{ head }}abias_int" - - "{{ atmos_analysis_dir }}/{{ head }}dtfanl.nc" - - "{{ atmos_analysis_dir }}/{{ head }}loginc.txt" - - "{{ atmos_analysis_dir }}/{{ head }}amsua_n19.satbias.nc4" - - "{{ atmos_analysis_dir }}/{{ head }}amsua_n19.satbias_cov.nc4" - - "{{ atmos_analysis_dir }}/{{ head }}amsua_n19.tlapse.txt" + {% if not current_cycle == SDATE and MODE == "cycled" %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbur" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}amsua_n19.satbias.nc4" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}amsua_n19.satbias_cov.nc4" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}amsua_n19.tlapse.txt" {% if DO_JEDISNOWDA %} - - "{{ conf_dir }}/{{ head }}letkfoi.yaml" + - "{{ COM_CONF | relpath(ROTDIR) }}/{{ head }}letkfoi.yaml" {% endif %} {% endif %} diff --git a/parm/archive/gdas_restartb.yaml.j2 b/parm/archive/gdas_restartb.yaml.j2 index d274a9cd16..fd986afeee 100644 --- a/parm/archive/gdas_restartb.yaml.j2 +++ b/parm/archive/gdas_restartb.yaml.j2 @@ -1,4 +1,7 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gdas.t" + cycle_HH + "z." %} name: "GDAS_RESTARTB" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdas_restartb.tar" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restartb.tar" mandatory: - - '{{ atmos_restart_dir }}/.' + - '{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/.' diff --git a/parm/archive/gdasice.yaml.j2 b/parm/archive/gdasice.yaml.j2 index 47c445b859..9ac864a53f 100644 --- a/parm/archive/gdasice.yaml.j2 +++ b/parm/archive/gdasice.yaml.j2 @@ -1,6 +1,8 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} {% set head = "gdas.t" + cycle_HH + "z." %} name: "GDASICE" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdasice.tar" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar" mandatory: - - '{{ ice_history_dir }}/{{ head }}*' - - '{{ ice_history_dir }}/ice_in' + - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}*' + - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/ice_in' diff --git a/parm/archive/gdasice_restart.yaml.j2 b/parm/archive/gdasice_restart.yaml.j2 index c9c6bd5b7d..f21af23d77 100644 --- a/parm/archive/gdasice_restart.yaml.j2 +++ b/parm/archive/gdasice_restart.yaml.j2 @@ -1,4 +1,7 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gdas.t" + cycle_HH + "z." %} name: "GDASICE_RESTART" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdasice_restart.tar" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice_restart.tar" mandatory: - - '{{ ice_history_dir }}/*' + - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/*' diff --git a/parm/archive/gdasocean.yaml.j2 b/parm/archive/gdasocean.yaml.j2 index 362968c7bc..6062c4e0e5 100644 --- a/parm/archive/gdasocean.yaml.j2 +++ b/parm/archive/gdasocean.yaml.j2 @@ -1,6 +1,8 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} {% set head = "gdas.t" + cycle_HH + "z." %} name: "GDASOCEAN" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdasocean.tar" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar" mandatory: - - '{{ ocean_history_dir }}/{{ head }}*' - - '{{ ocean_input_dir }}/.' + - '{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}*' + - '{{ COM_OCEAN_INPUT | relpath(ROTDIR) }}/.' diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index 018b3b9531..262607eb8d 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -1,8 +1,10 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} {% set head = "gdas.t" + cycle_HH + "z." %} name: "GDASOCEAN_ANALYSIS" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdasocean_analysis.tar" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar" mandatory: - - '{{ ocean_analysis_dir }}/{{ head }}*' - - '{{ ocean_analysis_dir }}/gdas.t??z.ocngrid.nc' - - '{{ ocean_analysis_dir }}/diags' - - '{{ ocean_analysis_dir }}/yaml' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}*' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t??z.ocngrid.nc' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml' diff --git a/parm/archive/gdasocean_restart.yaml.j2 b/parm/archive/gdasocean_restart.yaml.j2 index 6957c1f9e2..9b8a4d3729 100644 --- a/parm/archive/gdasocean_restart.yaml.j2 +++ b/parm/archive/gdasocean_restart.yaml.j2 @@ -1,5 +1,8 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gdas.t" + cycle_HH + "z." %} name: "GDASOCEAN_RESTART" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdasocean_restart.tar" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_restart.tar" mandatory: - - '{{ ocean_restart_dir }}/*' - - '{{ med_restart_dir }}/*' + - '{{ COM_OCEAN_RESTART | relpath(ROTDIR) }}/*' + - '{{ COM_MED_RESTART | relpath(ROTDIR) }}/*' diff --git a/parm/archive/gdaswave.yaml.j2 b/parm/archive/gdaswave.yaml.j2 index 295839e185..5cdc94d4e6 100644 --- a/parm/archive/gdaswave.yaml.j2 +++ b/parm/archive/gdaswave.yaml.j2 @@ -1,6 +1,8 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} {% set head = "gdaswave.t" + cycle_HH + "z." %} name: "GDASWAVE" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdaswave.tar" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave.tar" mandatory: - - "{{ wave_grid_dir }}/{{ head }}*" - - "{{ wave_station_dir }}/{{ head }}*" + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}*" diff --git a/parm/archive/gdaswave_restart.yaml.j2 b/parm/archive/gdaswave_restart.yaml.j2 index 65e6e5caa9..9a8e43f2af 100644 --- a/parm/archive/gdaswave_restart.yaml.j2 +++ b/parm/archive/gdaswave_restart.yaml.j2 @@ -1,4 +1,6 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} name: "GDASWAVE_RESTART" -target: "{{ ATARDIR }}/{{ cycle_YYYYMMDDHH }}/gdaswave_restart.tar" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave_restart.tar" mandatory: - - "{{ wave_restart_dir }}/*" + - "{{ COM_WAVE_RESTART | relpath(ROTDIR) }}/*" diff --git a/parm/archive/gfs_downstream.yaml.j2 b/parm/archive/gfs_downstream.yaml.j2 new file mode 100644 index 0000000000..ac35030168 --- /dev/null +++ b/parm/archive/gfs_downstream.yaml.j2 @@ -0,0 +1,11 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gfs.t" + cycle_HH + "z." %} +name: "GFS_DOWNSTREAM" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_downstream.tar" +mandatory: + - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.sfc" + - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.snd" + - "{{ COM_ATMOS_WMO | relpath(ROTDIR) }}/gfs_collective*.postsnd_{{ cycle_HH }}" + - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/bufr.t{{ cycle_HH }}z" + - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/gfs.t{{ cycle_HH }}z.bufrsnd.tar.gz" diff --git a/parm/archive/gfs_flux.yaml.j2 b/parm/archive/gfs_flux.yaml.j2 new file mode 100644 index 0000000000..03de6df525 --- /dev/null +++ b/parm/archive/gfs_flux.yaml.j2 @@ -0,0 +1,10 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gfs.t" + cycle_HH + "z." %} +name: "GFS_FLUX" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux.tar" +mandatory: + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ "%03d" % fhr }}.grib2" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ "%03d" % fhr }}.grib2.idx" + {% endfor %} diff --git a/parm/archive/gfs_flux_1p00.yaml.j2 b/parm/archive/gfs_flux_1p00.yaml.j2 new file mode 100644 index 0000000000..3f54c18a07 --- /dev/null +++ b/parm/archive/gfs_flux_1p00.yaml.j2 @@ -0,0 +1,10 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gfs.t" + cycle_HH + "z." %} +name: "GFS_FLUX_1P00" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux_1p00.tar" +mandatory: + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ "%03d" % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ "%03d" % fhr }}.idx" + {% endfor %} diff --git a/parm/archive/gfs_netcdfa.yaml.j2 b/parm/archive/gfs_netcdfa.yaml.j2 new file mode 100644 index 0000000000..0b7d671a24 --- /dev/null +++ b/parm/archive/gfs_netcdfa.yaml.j2 @@ -0,0 +1,12 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gfs.t" + cycle_HH + "z." %} +name: "GFS_NETCDFA" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfa.tar" +mandatory: + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi*.nc" +optional: + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" diff --git a/parm/archive/gfs_netcdfb.yaml.j2 b/parm/archive/gfs_netcdfb.yaml.j2 new file mode 100644 index 0000000000..9afab91520 --- /dev/null +++ b/parm/archive/gfs_netcdfb.yaml.j2 @@ -0,0 +1,10 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gfs.t" + cycle_HH + "z." %} +name: "GFS_NETCDFB" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfb.tar" +mandatory: + {% for fhr in range(0, ARCH_GAUSSIAN_FHMAX + ARCH_GAUSSIAN_FHINC, ARCH_GAUSSIAN_FHINC) %} + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/gfs_pgrb2b.yaml.j2 b/parm/archive/gfs_pgrb2b.yaml.j2 new file mode 100644 index 0000000000..7717d9337a --- /dev/null +++ b/parm/archive/gfs_pgrb2b.yaml.j2 @@ -0,0 +1,23 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gfs.t" + cycle_HH + "z." %} +name: "GFS_PGRB2B" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_pgrb2b.tar" +mandatory: + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" + {% if ARCH_GAUSSIAN %} + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ "%03d" % fhr }}" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ "%03d" % fhr }}.idx" + {% endfor %} + {% endif %} +optional: + {% if ARCH_GAUSSIAN %} + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ "%03d" % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ "%03d" % fhr }}.idx" + {% endfor %} + {% endif %} diff --git a/parm/archive/gfs_restarta.yaml.j2 b/parm/archive/gfs_restarta.yaml.j2 new file mode 100644 index 0000000000..349ed1fb78 --- /dev/null +++ b/parm/archive/gfs_restarta.yaml.j2 @@ -0,0 +1,17 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gfs.t" + cycle_HH + "z." %} +name: "GFS_RESTARTA" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_restarta.tar" +mandatory: + {% if MODE == "cycled" %} + {% for i_tile in range(1, 7) %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/*0000.sfcanl_data.tile{{ i_tile }}.nc" + {% endfor %} + {% elif MODE == "forecast-only" %} + - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc" + {% for i_tile in range(1, 7) %} + - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_data.tile{{ i_tile }}.nc" + - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/sfc_data.tile{{ i_tile }}.nc" + {% endfor %} + {% endif %} diff --git a/parm/archive/gfs_restartb.yaml.j2 b/parm/archive/gfs_restartb.yaml.j2 new file mode 100644 index 0000000000..4df0e09718 --- /dev/null +++ b/parm/archive/gfs_restartb.yaml.j2 @@ -0,0 +1,7 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gfs.t" + cycle_HH + "z." %} +name: "GFS_RESTARTB" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_restartb.tar" +mandatory: + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/" diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 new file mode 100644 index 0000000000..9e73844458 --- /dev/null +++ b/parm/archive/gfsa.yaml.j2 @@ -0,0 +1,57 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gfs.t" + cycle_HH + "z." %} +name: "GFSA" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsa.tar" +mandatory: + # TODO explicitly name all logs to include + {% for log in glob("logs/" ~ cycle_YMDH ~ "/gfs*.log") %} + {% if log != "gfsarch.log" %} + - "{{ log }}" + {% endif %} + {% endfor %} +# - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" +# - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" + - "{{ COM_CONF | relpath(ROTDIR) }}/ufs.input.nml" + {% if MODE == "cycled" %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" +# - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" +# - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" +# - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" +# - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" + {% if DO_VMINMON %} + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" +# - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" + {% endif %} + {% endif %} # Full cycle + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ "%03d" % fhr }}" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ "%03d" % fhr }}.idx" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ "%03d" % fhr }}.txt" + {% endfor %} +optional: + {% if MODE == "cycled" %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" + {% endif %} # cycled + - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cycle.trackatcfunix" + - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avnop.t{{ cycle_HH }}z.cycle.trackatcfunix" + - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/trak.gfso.atcfunix.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}" +# {% if not WRITE_DOPOST %} +# # TODO set the forecast hours explicitly. This will require emulating numpy.array_split +# - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" +# {% endif %} ## not WRITE_DOPOST diff --git a/parm/archive/gfsb.yaml.j2 b/parm/archive/gfsb.yaml.j2 new file mode 100644 index 0000000000..41ce3d1c3c --- /dev/null +++ b/parm/archive/gfsb.yaml.j2 @@ -0,0 +1,17 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gfs.t" + cycle_HH + "z." %} +name: "GFSB" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsb.tar" +mandatory: + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl" + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" +optional: + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ "%03d" % fhr }}" + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ "%03d" % fhr }}.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ "%03d" % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ "%03d" % fhr }}.idx" + {% endfor %} diff --git a/parm/archive/gfswave.yaml.j2 b/parm/archive/gfswave.yaml.j2 new file mode 100644 index 0000000000..f334633e6f --- /dev/null +++ b/parm/archive/gfswave.yaml.j2 @@ -0,0 +1,9 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gfswave.t" + cycle_HH + "z." %} +name: "GFSWAVE" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfswave.tar" +mandatory: + - "{{ COM_WAVE_HISTORY | relpath(ROTDIR) }}/ww3_multi*" + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}*" diff --git a/parm/archive/ice_6hravg.yaml.j2 b/parm/archive/ice_6hravg.yaml.j2 new file mode 100644 index 0000000000..1ccc57586e --- /dev/null +++ b/parm/archive/ice_6hravg.yaml.j2 @@ -0,0 +1,8 @@ +{% set head = "gfs.ice.t" + cycle_HH + "z." %} +name: "ICE_6HRAVG" +target: "{{ ATARDIR }}/{{ current_cycle | to_YMDH }}/ice_6hravg.tar" +mandatory: + - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}ic.nc" + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + 6, 6 %} + - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/ice_grib2.yaml.j2 b/parm/archive/ice_grib2.yaml.j2 new file mode 100644 index 0000000000..1fa9a37085 --- /dev/null +++ b/parm/archive/ice_grib2.yaml.j2 @@ -0,0 +1,17 @@ +{% set head = "gfs.ice.t" + cycle_HH + "z." %} +name: "ICE_GRIB2" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_grib2.tar" +mandatory: + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS %} + {% set fhr3 = '%03d' % fhr %} + {% if OCNRES == "500" %} + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" + {% elif OCNRES == "100" %} + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" + {% elif OCNRES == "025" %} + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" + {% endif %} + {% endfor %} diff --git a/parm/archive/ocean_6hravg.yaml.j2 b/parm/archive/ocean_6hravg.yaml.j2 new file mode 100644 index 0000000000..960c91867b --- /dev/null +++ b/parm/archive/ocean_6hravg.yaml.j2 @@ -0,0 +1,7 @@ +{% set head = "gfs.ocean.t" + cycle_HH + "z." %} +name: "OCEAN_6HRAVG" +target: "{{ ATARDIR }}/{{ current_cycle | to_YMDH }}/ocean_6hravg.tar" +mandatory: + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + 6, 6 %} + - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/ocean_daily.yaml.j2 b/parm/archive/ocean_daily.yaml.j2 new file mode 100644 index 0000000000..c56633177c --- /dev/null +++ b/parm/archive/ocean_daily.yaml.j2 @@ -0,0 +1,7 @@ +{% set head = "gfs.ocean.t" + cycle_HH + "z." %} +name: "OCEAN_DAILY" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_daily.tar" +mandatory: + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + 24, 24 %} + - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}daily.f{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/ocean_grib2.yaml.j2 b/parm/archive/ocean_grib2.yaml.j2 new file mode 100644 index 0000000000..2d9e54da22 --- /dev/null +++ b/parm/archive/ocean_grib2.yaml.j2 @@ -0,0 +1,17 @@ +{% set head = "gfs.ocean.t" + cycle_HH + "z." %} +name: "OCEAN_GRIB2" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_grib2.tar" +mandatory: + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS %} + {% set fhr3 = '%03d' % fhr %} + {% if OCNRES == "500" %} + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" + {% elif OCNRES == "100" %} + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" + {% elif OCNRES == "025" %} + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" + {% endif %} + {% endfor %} diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index a8f91400ea..7342b8790c 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -2,8 +2,8 @@ import os -from wxflow import AttrDict, Logger, logit, cast_strdict_as_dtypedict, chdir from pygfs.task.archive import Archive +from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, chdir, logit # initialize root logger logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) @@ -20,16 +20,22 @@ def main(): # Pull out all the configuration keys needed to run the rest of archive steps keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'PARMgfs', - 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'cyc', 'cycle_YYYYMMDDHH', - 'PSLOT', 'cycle_HH', 'first_cycle', 'HPSSARCH', + 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'PSLOT', 'HPSSARCH', 'DO_MOS', 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', - 'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', - 'path_exists'] + 'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', 'SDATE', + 'FHMIN_GFS', 'FHMAX_GFS', 'FHOUT_GFS', 'ARCH_GAUSSIAN', 'MODE', + 'FHOUT_OCNICE', 'FHOUT_OCNICE_GFS', 'DO_BUFRSND', + 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC'] archive_dict = AttrDict() for key in keys: archive_dict[key] = archive.task_config[key] + # Also import all COM* directory names + for key in archive.task_config.keys(): + if key.startswith("COM"): + archive_dict[key] = archive.task_config[key] + # Also get all relative paths for key in archive.task_config.keys(): if key.endswith("_dir"): @@ -47,5 +53,6 @@ def main(): os.chdir(cwd) + if __name__ == '__main__': main() diff --git a/sorc/wxflow b/sorc/wxflow index 63db259b85..534fe44da4 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit 63db259b856b325b802e1deb28f6ee3001f8c03b +Subproject commit 534fe44da4b88cd099a9220c654681ee3c1aaaa6 diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 00dbe92262..f73982a13b 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -1,30 +1,18 @@ #!/usr/bin/env python3 -from datetime import timedelta - import glob - -from logging import getLogger - import os +import shutil +from datetime import timedelta +from logging import getLogger +from typing import Any, Dict, List -from typing import Dict, Any, List - -from yaml import load +import numpy as np +from wxflow import (AttrDict, FileHandler, Hsi, Htar, Task, cast_strdict_as_dtypedict, + chgrp, get_gid, logit, mkdir_p, parse_j2yaml, rm_p, strftime, + to_YMD, to_YMDH) from yaml import CLoader as Loader - -from wxflow import (logit, - cast_strdict_as_dtypedict, - AttrDict, - get_gid, - Task, - Htar, - Hsi, - rm_p, - mkdir_p, - chgrp, - FileHandler, - parse_j2yaml) +from yaml import load logger = getLogger(__name__.split('.')[-1]) @@ -54,25 +42,8 @@ def __init__(self, config: Dict[str, Any]) -> None: # Find all absolute paths in the environment and get their relative paths from ${ROTDIR} path_dict = self._gen_relative_paths(rotdir) - local_dict = AttrDict( - {'cycle_HH': self.runtime_config.current_cycle.strftime("%H"), - 'cycle_YYYYMMDDHH': self.runtime_config.current_cycle.strftime("%Y%m%d%H"), - 'cycle_YYYYMMDD': self.runtime_config.current_cycle.strftime("%Y%m%d"), - 'first_cycle': self.runtime_config.current_cycle == self.config.SDATE - } - ) - - if self.config.REALTIME: - local_dict['mos_YYYYMMDDHH'] = (self.runtime_config.current_cycle - timedelta(days=1)).strftime("%Y%m%d%H") - else: - local_dict['mos_YYYYMMDDHH'] = cycle_YYYYMMDDHH - - local_dict['mos_YYYYMMDD'] = local_dict['mos_YYYYMMDDHH'][:8] - - # Add the os.path.exists function for use in parsing the jinja files - local_dict['path_exists'] = os.path.exists - self.task_config = AttrDict(**self.config, **self.runtime_config, **path_dict, **local_dict) + self.task_config = AttrDict(**self.config, **self.runtime_config, **path_dict) @logit(logger) def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str, Any]]): @@ -95,6 +66,17 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str # Collect the dataset to archive locally arcdir_filename = os.path.join(archive_parm, "arcdir.yaml.j2") + + # Add the glob.glob function for capturing log filenames + # TODO remove this kludge once log filenames are explicit + arch_dict['glob'] = glob.glob + + # Copy the cyclone track files and rename the experiments + Archive._rename_cyclone_expt(arch_dict) + # Add the os.path.exists function to the dict for yaml parsing + arch_dict['path_exists'] = os.path.exists + + # Parse the input jinja yaml template arcdir_set = parse_j2yaml(arcdir_filename, arch_dict) # Collect datasets that need to be archived @@ -113,62 +95,61 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if not os.path.isdir(arch_dict.ROTDIR): raise FileNotFoundError(f"The ROTDIR ({arch_dict.ROTDIR}) does not exist!") - # Pull out some common variables - cycle_YYYYMMDDHH = arch_dict.cycle_YYYYMMDDHH - cycle_HH = arch_dict.cycle_HH - first_cycle = arch_dict.first_cycle - ARCH_WARMICFREQ = arch_dict.ARCH_WARMICFREQ - ARCH_FCSTICFREQ = arch_dict.ARCH_FCSTICFREQ - assim_freq = arch_dict.assim_freq + arch_mos = False + if arch_dict.DO_MOS: + if self.config.REALTIME: + if arch_dict.current_cycle - timedelta(days=1) > arch_dict.SDATE: + arch_mos = True + else: + arch_mos = True + + cycle_HH = strftime(arch_dict.current_cycle, "%H") if arch_dict.RUN == "gdas" or arch_dict.RUN == "gfs": - ARCHINC_CYC = arch_dict.ARCH_CYC - ARCHICS_CYC = ARCHINC_CYC - assim_freq - if ARCHICS_CYC < 0: - ARCHICS_CYC += 24 + arch_ics_cycle = arch_dict.ARCH_CYC - arch_dict.assim_freq - mm = cycle_YYYYMMDDHH[4:6] - dd = cycle_YYYYMMDDHH[6:8] - # TODO: This math yields multiple dates sharing the same nday - nday = (int(mm) - 1) * 30 + int(dd) - mod = nday % int(ARCH_WARMICFREQ) + if arch_ics_cycle < 0: + arch_ics_cycle += 24 + + mod = (arch_dict.current_cycle - + arch_dict.SDATE).days % arch_dict.ARCH_WARMICFREQ save_warm_ic_a = False save_warm_ic_b = False - if first_cycle and cycle_HH == ARCHINC_CYC: + + if arch_dict.current_cycle == arch_dict.SDATE and cycle_HH == arch_dict.ARCH_CYC: save_warm_ic_a = True save_warm_ic_b = True - elif mod == 0 and cycle_HH == ARCHINC_CYC: + elif mod == 0 and cycle_HH == arch_dict.ARCH_CYC: save_warm_ic_a = True save_warm_ic_b = True - if ARCHICS_CYC == "18": - nday1 = nday + 1 - mod1 = nday1 % int(ARCH_WARMICFREQ) + if arch_ics_cycle == 18: - if cycle_HH == ARCHICS_CYC: + mod1 = ((arch_dict.current_cycle - + arch_dict.SDATE).days + 1) % arch_dict.ARCH_WARMICFREQ + + if cycle_HH == arch_ics_cycle: if mod1 == 0: save_warm_ic_b = True - elif first_cycle: + elif arch_dict.current_cycle == SDATE: save_warm_ic_b = True else: save_warm_ic_b = False - mod = nday % int(ARCH_FCSTICFREQ) - - SAVEFCSTIC = False - if mod == 0 or first_day: - SAVEFCSTIC = True + mod = (arch_dict.current_cycle - + arch_dict.SDATE).days % arch_dict.ARCH_FCSTICFREQ - # If present, copy and replace the experiment names in cycle tracking files - self._rename_cyclone_expt(arch_dict) + save_fcst_ic = False + if mod == 0 or arch_dict.current_cycle == SDATE: + save_fcst_ic = True if arch_dict.RUN == "gdas": - datasets = ['gdas'] + datasets = ["gdas"] - if save_warm_ic_a or SAVEFCSTIC: + if save_warm_ic_a or save_fcst_ic: datasets.append("gdas_restarta") if arch_dict.DO_WAVE: datasets.append("gdaswave_restart") @@ -177,21 +158,48 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if arch_dict.DO_ICE: datasets.append("gdasice_restart") - if save_warm_ic_b or SAVEFCSTIC: + if save_warm_ic_b or save_fcst_ic: datasets.append("gdas_restartb") - if arch_dict.DO_ICE == "YES": - datasets.append('gdasice') + if arch_dict.DO_ICE: + datasets.append("gdasice") - if arch_dict.DO_OCN == "YES": - datasets.append('gdasocean') - datasets.append('gdasocean_analysis') + if arch_dict.DO_OCN: + datasets.append("gdasocean") + datasets.append("gdasocean_analysis") - if arch_dict.DO_WAVE == "YES": - datasets.append('gdaswave') + if arch_dict.DO_WAVE: + datasets.append("gdaswave") elif arch_dict.RUN == "gfs": - raise NotImplementedError("Archiving is not yet set up for GFS runs") + datasets = ["gfsa", "gfsb"] + + if arch_dict.ARCH_GAUSSIAN: + datasets.extend(["gfs_flux", "gfs_netcdfb", "gfs_pgrb2b"]) + print(datasets) + if arch_dict.MODE == "cycled": + datasets.append("gfs_netcdfa") + + if arch_dict.DO_WAVE: + datasets.append("gfswave") + + if arch_dict.DO_OCN: + datasets.extend(["ocean_6hravg", "ocean_daily", "ocean_grib2", "gfs_flux_1p00"]) + + if arch_dict.DO_ICE: + datasets.extend(["ice_6hravg", "ice_grib2"]) + + if arch_dict.DO_AERO: + datasets.append(["aero"]) + + if save_fcst_ic: + datasets.append("gfs_restarta") + + if arch_dict.DO_BUFRSND: + datasets.append("gfs_downstream") + + if arch_mos and cycle_HH == "18": + datasets.append("gfsmos") elif arch_dict.RUN == "enkfgdas": raise NotImplementedError("Archiving is not yet set up for ENKF GDAS runs") @@ -203,7 +211,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str raise NotImplementedError("Archiving is not yet set up for GEFS runs") else: - raise ValueError(f'Archiving is not enabled for {arch_dict.RUN} runs') + raise ValueError(f"Archiving is not enabled for {arch_dict.RUN} runs") atardir_sets = [] @@ -211,8 +219,8 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str archive_filename = os.path.join(archive_parm, dataset + ".yaml.j2") atardir_set = parse_j2yaml(archive_filename, arch_dict) - atardir_set['fileset'] = Archive._create_fileset(atardir_set) - atardir_set['has_rstprod'] = Archive._has_rstprod(atardir_set.fileset) + atardir_set["fileset"] = Archive._create_fileset(atardir_set) + atardir_set["has_rstprod"] = Archive._has_rstprod(atardir_set.fileset) atardir_sets.append(atardir_set) @@ -286,21 +294,21 @@ def _create_fileset(atardir_set: Dict[str, Any]) -> List: """ fileset = [] - if 'mandatory' in atardir_set: + if "mandatory" in atardir_set: if atardir_set.mandatory is not None: for item in atardir_set.mandatory: glob_set = glob.glob(item) if len(glob_set) == 0: - raise FileNotFoundError(f'Mandatory file, directory, or glob {item} not found!') + raise FileNotFoundError(f"Mandatory file, directory, or glob {item} not found!") for entry in glob_set: fileset.append(entry) - if 'optional' in atardir_set: + if "optional" in atardir_set: if atardir_set.optional is not None: for item in atardir_set.optional: glob_set = glob.glob(item) if len(glob_set) == 0: - print(f'WARNING: optional file/glob {item} not found!') + print(f"WARNING: optional file/glob {item} not found!") else: for entry in glob_set: fileset.append(entry) @@ -387,7 +395,6 @@ def _create_tarball(target: str, fileset: List) -> None: import tarfile # TODO create a set of tar helper functions in wxflow - # Attempt to create the parent directory if it does not exist mkdir_p(os.path.dirname(os.path.realpath(target))) @@ -418,13 +425,15 @@ def _gen_relative_paths(self, root_path: str) -> Dict: for key, value in self.config.items(): if isinstance(value, str): if root_path in value: - rel_path = value.replace(root_path, '') + rel_path = value.replace(root_path, "") rel_key = (key[4:] if key.startswith("COM_") else key).lower() + "_dir" rel_path_dict[rel_key] = rel_path return rel_path_dict - def _rename_cyclone_expt(self, arch_dict) -> None: + @logit(logger) + @staticmethod + def _rename_cyclone_expt(arch_dict) -> None: # Rename the experiment in the tracker files from "AVNO" to the # first 4 letters of PSLOT. @@ -432,33 +441,60 @@ def _rename_cyclone_expt(self, arch_dict) -> None: if len(arch_dict.PSLOT) > 4: pslot4 = arch_dict.PSLOT[0:4].upper() - track_dir = arch_dict.atmos_track_dir - cycle_HH = str(arch_dict.cycle_HH) + track_dir = arch_dict.COM_ATMOS_TRACK run = arch_dict.RUN + cycle_HH = strftime(arch_dict.current_cycle, "%H") if run == "gfs": - in_track_file = track_dir + "/avno.t" + cycle_HH + "z.cycle.trackatcfunix" - in_track_p_file = track_dir + "/avnop.t" + cycle_HH + "z.cycle.trackatcfunixp" + in_track_file = (track_dir + "/avno.t" + + cycle_HH + "z.cycle.trackatcfunix") + in_track_p_file = (track_dir + "/avnop.t" + + cycle_HH + "z.cycle.trackatcfunixp") elif run == "gdas": - in_track_file = track_dir + "/gdas.t" + cycle_HH + "z.cycle.trackatcfunix" - in_track_p_file = track_dir + "/gdasp.t" + cycle_HH + "z.cycle.trackatcfunixp" + in_track_file = (track_dir + "/gdas.t" + + cycle_HH + "z.cycle.trackatcfunix") + in_track_p_file = (track_dir + "/gdasp.t" + + cycle_HH + "z.cycle.trackatcfunixp") if not os.path.isfile(in_track_file): + # Do not attempt to archive the outputs return - cycle_YYYYMMDDHH = str(arch_dict.cycle_YYYYMMDDHH) - - out_track_file = track_dir + "/atcfunix." + run + "." + cycle_YYYYMMDDHH - out_track_p_file = track_dir + "/atcfunixp." + run + "." + cycle_YYYYMMDDHH + out_track_file = track_dir + "/atcfunix." + run + "." + to_YMDH(arch_dict.current_cycle) + out_track_p_file = track_dir + "/atcfunixp." + run + "." + to_YMDH(arch_dict.current_cycle) def replace_string_from_to_file(filename_in, filename_out, search_str, replace_str): + + """Write a new file from the contents of an input file while searching + and replacing ASCII strings. To prevent partial file creation, a + temporary file is created and moved to the final location only + after the search/replace is finished. + + Parameters + ---------- + filename_in : str + Input filename + + filename_out : str + Output filename + + search_str : str + ASCII string to search for + + replace_str : str + ASCII string to replace the search_str with + """ with open(filename_in) as old_file: lines = old_file.readlines() out_lines = [line.replace(search, replace) for line in lines] - with open(filename_out, "w") as new_file: + with open("/tmp/track_file", "w") as new_file: new_file.writelines(out_lines) + shutil.move("tmp/track_file", filename_out) + replace_string_from_to_file(in_track_file, out_track_file, "AVNO", pslot4) replace_string_from_to_file(in_track_p_file, out_track_p_file, "AVNO", pslot4) + + return From dc9004de08bb8b6fc8e17987bf325fe40c047f4a Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 17 Apr 2024 17:58:12 +0000 Subject: [PATCH 018/128] Update wxflow hash. --- sorc/wxflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/wxflow b/sorc/wxflow index 534fe44da4..c237f195ac 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit 534fe44da4b88cd099a9220c654681ee3c1aaaa6 +Subproject commit c237f195ac8f80be2518747d00390fd07a7240dc From a8cbe2b26fb1ea1da0365125379a6d5945025d63 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 18 Apr 2024 19:54:09 +0000 Subject: [PATCH 019/128] Add enkf archiving. --- jobs/JGDAS_ENKF_ARCHIVE | 2 +- jobs/rocoto/earc.sh | 7 +- parm/archive/arcdir.yaml.j2 | 8 ++ parm/archive/enkf.yaml.j2 | 59 +++++++++ parm/archive/enkf_grp.yaml.j2 | 60 +++++++++ parm/archive/enkf_restarta_grp.yaml.j2 | 31 +++++ parm/archive/enkf_restartb_grp.yaml.j2 | 12 ++ parm/archive/gfs_flux.yaml.j2 | 4 +- parm/archive/gfs_flux_1p00.yaml.j2 | 4 +- parm/archive/gfs_pgrb2b.yaml.j2 | 8 +- parm/archive/gfsa.yaml.j2 | 6 +- parm/archive/gfsb.yaml.j2 | 8 +- scripts/exgdas_enkf_earc.py | 51 ++++++++ scripts/exgdas_enkf_earc.sh | 163 ------------------------- scripts/exglobal_archive.py | 8 +- ush/python/pygfs/task/archive.py | 89 +++++++++++--- 16 files changed, 316 insertions(+), 204 deletions(-) create mode 100644 parm/archive/enkf.yaml.j2 create mode 100644 parm/archive/enkf_grp.yaml.j2 create mode 100644 parm/archive/enkf_restarta_grp.yaml.j2 create mode 100644 parm/archive/enkf_restartb_grp.yaml.j2 create mode 100755 scripts/exgdas_enkf_earc.py delete mode 100755 scripts/exgdas_enkf_earc.sh diff --git a/jobs/JGDAS_ENKF_ARCHIVE b/jobs/JGDAS_ENKF_ARCHIVE index 07e171d668..5ac46a73e7 100755 --- a/jobs/JGDAS_ENKF_ARCHIVE +++ b/jobs/JGDAS_ENKF_ARCHIVE @@ -18,7 +18,7 @@ MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ # Run archive script ############################################################### -"${SCRgfs}/exgdas_enkf_earc.sh" +"${SCRgfs}/exgdas_enkf_earc.py" status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/earc.sh b/jobs/rocoto/earc.sh index c4c7341698..c24374846e 100755 --- a/jobs/rocoto/earc.sh +++ b/jobs/rocoto/earc.sh @@ -8,6 +8,12 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" +############################################################### +# setup python path for workflow utilities and tasks +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" +export PYTHONPATH + export job="earc" export jobid="${job}.$$" @@ -16,5 +22,4 @@ export jobid="${job}.$$" "${HOMEgfs}/jobs/JGDAS_ENKF_ARCHIVE" status=$? - exit "${status}" diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index a47e28dc9a..cc8fe6ffec 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -68,3 +68,11 @@ deterministic: - ["{{ COM_ATMOS_TRACK }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} {% endif %} # gfs or gdas +{% if current_cycle != SDATE and (RUN == "enkfgdas" or RUN == "enkfgfs") %} +ensemble: + mkdir: + - "{{ ARCDIR }}" + copy: + - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}enkfstat", "{{ ARCDIR }}/enkfstat.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}gsistat.ensmean", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}.ensmean"] +{% endif %} # enkfgdas or enkfgfs diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 new file mode 100644 index 0000000000..a71dbd77c4 --- /dev/null +++ b/parm/archive/enkf.yaml.j2 @@ -0,0 +1,59 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = RUN + ".t" + cycle_HH + "z." %} +{% if RUN == "enkfgdas" %} +{% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF, FHOUT_ENKF) %} +{% elif RUN == "enkfgfs" %} +{% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF_GFS, FHOUT_ENKF_GFS) %} +{% endif %} +name: "ENKF" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}.tar" +mandatory: + {% for fhr in range(fhmin, fhmax + fhout, fhout) %} + - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensmean.nc" + - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.ensmean.nc" + {% endfor %} + {% if not current_cycle == SDATE %} + {% if not DO_JEDIATMENS %} + {% set da_files = ["enkfstat", + "gsistat.ensmean", + "cnvstat.ensmean", + "oznstat.ensmean", + "radstat.ensmean"] %} + {% else %} + {% set da_files = ["atmens.yaml", + "atmensstat"] %} + {% endif %} + {% for file in da_files %} + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}{{ file }}" + {% endfor %} + {% if DOIAU %} + {% for fhr in IAUFHRS %} + {% if fhr == IAU_OFFSET %} + {% if DO_CALC_INCREMENT_ENKF_GFS %} + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmanl.ensmean.nc" + {% endif %} # calc increment + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ratminc.ensmean.nc" + {% else %} # fhr != IAU_OFFSET + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensmean.nc" + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmi{{ '%03d' % fhr }}.ensmean.nc" + {% endif %} # fhr == IAU_OFFSET + {% endfor %} # IAUFHRS + {% endif %} # DOIAU + {% set steps = ["eobs", "ecen*", "esfc", "eupd"] %} + {% for step in steps %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}{{ step }}.log" + {% endfor %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_mem*.log" + {% if lobsdiag_forenkf %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}ediag.log" + {% else %} + {% for mem in range(1, NMEM_ENS + 1) %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}eomg_mem{{ '%03d' % mem }}.log" + {% endfor %} + {% endif %} # lobsdiag_forenkf + {% endif %} # not the first cycle +optional: + {% for fhr in range(fhmin, fhmax + fhout, fhout) %} + - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensspread.nc" + {% endfor %} diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 new file mode 100644 index 0000000000..e24132331a --- /dev/null +++ b/parm/archive/enkf_grp.yaml.j2 @@ -0,0 +1,60 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = RUN + ".t" + cycle_HH + "z." %} +{% if RUN == "enkfgdas" %} +{% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF, FHOUT_ENKF) %} +{% elif RUN == "enkfgfs" %} +{% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF_GFS, FHOUT_ENKF_GFS) %} +{% endif %} +name: "ENKF_GRP" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_grp{{ ENSGRP }}.tar" +mandatory: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} + {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + + {% if not lobsdiag_forenkf %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat" + {% endif %} + {% for fhr in range(fhmin, fhmax+fhout, fhout) %} + - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf006.nc" + - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + {% endfor %} + + {% if not current_cycle == SDATE %} + {% if DO_CALC_INCR_ENKF_GFS %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + {% else %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratminc.nc" + {% endif %} + {% endif %} + + {% for iauhr in iaufhrs if iauhr != 6 %} + {% set iauhr = iauhr | int %} + {% if DO_CALC_INCREMENT_ENKF_GFS %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iauhr }}.nc" + {% else %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iauhr }}.nc" + {% endif %} + {% endfor %} # iauhr in iaufhrs + {% endfor %} # first_group_mem to last_group_mem +optional: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} + {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" + {% endif %} + {% endfor %} +optional: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} + {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + {% if not lobsdiag_forenkf %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" + {% endif %} + {% endfor %} diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 new file mode 100644 index 0000000000..b2c936c69e --- /dev/null +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -0,0 +1,31 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = RUN + ".t" + cycle_HH + "z." %} +name: "ENKF_RESTARTA_GRP" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restarta_grp{{ ENSGRP }}.tar" +mandatory: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} + {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + + {% if not lobsdiag_forenkf %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_air" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_int" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_pc" + {% endif %} + + {% for itile in range(1,7) %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/*0000.sfcanl_data.tile{{ itile }}.nc" + {% endfor %} + {% endfor %} # first_group_mem to last_group_mem +optional: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" + {% endif %} + {% endfor %} diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 new file mode 100644 index 0000000000..4e7b80560c --- /dev/null +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -0,0 +1,12 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = RUN + ".t" + cycle_HH + "z." %} +name: "ENKF_RESTARTB_GRP" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restartb_grp{{ ENSGRP }}.tar" +mandatory: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}" + {% endfor %} diff --git a/parm/archive/gfs_flux.yaml.j2 b/parm/archive/gfs_flux.yaml.j2 index 03de6df525..b0fb550a54 100644 --- a/parm/archive/gfs_flux.yaml.j2 +++ b/parm/archive/gfs_flux.yaml.j2 @@ -5,6 +5,6 @@ name: "GFS_FLUX" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux.tar" mandatory: {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ "%03d" % fhr }}.grib2" - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ "%03d" % fhr }}.grib2.idx" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" {% endfor %} diff --git a/parm/archive/gfs_flux_1p00.yaml.j2 b/parm/archive/gfs_flux_1p00.yaml.j2 index 3f54c18a07..bd86382c2e 100644 --- a/parm/archive/gfs_flux_1p00.yaml.j2 +++ b/parm/archive/gfs_flux_1p00.yaml.j2 @@ -5,6 +5,6 @@ name: "GFS_FLUX_1P00" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux_1p00.tar" mandatory: {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ "%03d" % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ "%03d" % fhr }}.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}.idx" {% endfor %} diff --git a/parm/archive/gfs_pgrb2b.yaml.j2 b/parm/archive/gfs_pgrb2b.yaml.j2 index 7717d9337a..96b0e1bd79 100644 --- a/parm/archive/gfs_pgrb2b.yaml.j2 +++ b/parm/archive/gfs_pgrb2b.yaml.j2 @@ -10,14 +10,14 @@ mandatory: - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" {% if ARCH_GAUSSIAN %} {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ "%03d" % fhr }}" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ "%03d" % fhr }}.idx" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}.idx" {% endfor %} {% endif %} optional: {% if ARCH_GAUSSIAN %} {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ "%03d" % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ "%03d" % fhr }}.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}.idx" {% endfor %} {% endif %} diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 9e73844458..e4b8ead650 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -32,9 +32,9 @@ mandatory: {% endif %} {% endif %} # Full cycle {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ "%03d" % fhr }}" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ "%03d" % fhr }}.idx" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ "%03d" % fhr }}.txt" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" {% endfor %} optional: {% if MODE == "cycled" %} diff --git a/parm/archive/gfsb.yaml.j2 b/parm/archive/gfsb.yaml.j2 index 41ce3d1c3c..b601bad029 100644 --- a/parm/archive/gfsb.yaml.j2 +++ b/parm/archive/gfsb.yaml.j2 @@ -10,8 +10,8 @@ mandatory: - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" optional: {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ "%03d" % fhr }}" - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ "%03d" % fhr }}.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ "%03d" % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ "%03d" % fhr }}.idx" + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" {% endfor %} diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py new file mode 100755 index 0000000000..48ae3836f4 --- /dev/null +++ b/scripts/exgdas_enkf_earc.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import os + +from pygfs.task.archive import Archive +from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, chdir, logit + +# initialize root logger +logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) + + +@logit(logger) +def main(): + + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the Archive object + archive = Archive(config) + + # Pull out all the configuration keys needed to run the rest of archive steps + keys = ['ATARDIR', 'current_cycle', 'IAUFHRS', 'RUN', 'PDY', + 'PSLOT', 'HPSSARCH', 'LOCALARCH', 'ROTDIR', 'PARMgfs', + 'ARCDIR', 'SDATE', 'MODE', 'ENSGRP', 'NMEM_EARCGRP', + 'NMEM_ENS', 'DO_CALC_INCREMENT_ENKF_GFS', 'DO_JEDIATMENS', + 'lobsdiag_forenkf', 'FHMIN_ENKF', 'FHMAX_ENKF_GFS', + 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF'] + + archive_dict = AttrDict() + for key in keys: + archive_dict[key] = archive.task_config[key] + + # Also import all COM* directory and template variables + for key in archive.task_config.keys(): + if key.startswith("COM"): + archive_dict[key] = archive.task_config[key] + + cwd = os.getcwd() + + os.chdir(config.ROTDIR) + + # Determine which archives to create + arcdir_set, atardir_sets = archive.configure(archive_dict) + + # Create the archives + archive.execute(arcdir_set, atardir_sets) + + os.chdir(cwd) + + +if __name__ == '__main__': + main() diff --git a/scripts/exgdas_enkf_earc.sh b/scripts/exgdas_enkf_earc.sh deleted file mode 100755 index 3e54c658e9..0000000000 --- a/scripts/exgdas_enkf_earc.sh +++ /dev/null @@ -1,163 +0,0 @@ -#! /usr/bin/env bash - -source "${USHgfs}/preamble.sh" - -############################################## -# Begin JOB SPECIFIC work -############################################## -export n=$((10#${ENSGRP})) -export CDUMP_ENKF="${EUPD_CYC:-"gdas"}" - -# ICS are restarts and always lag INC by $assim_freq hours. -EARCINC_CYC=${ARCH_CYC} -EARCICS_CYC=$((ARCH_CYC-assim_freq)) -if [ "${EARCICS_CYC}" -lt 0 ]; then - EARCICS_CYC=$((EARCICS_CYC+24)) -fi - -"${USHgfs}/hpssarch_gen.sh" "${RUN}" -status=$? -if [ "${status}" -ne 0 ]; then - echo "${USHgfs}/hpssarch_gen.sh ${RUN} failed, ABORT!" - exit "${status}" -fi - -cd "${ROTDIR}" || exit 2 - -source "${USHgfs}/file_utils.sh" - -################################################################### -# ENSGRP > 0 archives a group of ensemble members -firstday=$(${NDATE} +24 "${SDATE}") -if (( 10#${ENSGRP} > 0 )) && [[ ${HPSSARCH} = "YES" || ${LOCALARCH} = "YES" ]]; then - -#--set the archiving command and create local directories, if necessary - TARCMD="htar" - if [[ ${LOCALARCH} = "YES" ]]; then - TARCMD="tar" - if [[ ! -d "${ATARDIR}/${PDY}${cyc}" ]]; then mkdir -p "${ATARDIR}/${PDY}${cyc}"; fi - fi - -#--determine when to save ICs for warm start - SAVEWARMICA="NO" - SAVEWARMICB="NO" - mm="${PDY:4:2}" - dd="${PDY:6:2}" - nday=$(( (10#${mm}-1)*30+10#${dd} )) - mod=$((nday % ARCH_WARMICFREQ)) - if [ "${PDY}${cyc}" -eq "${firstday}" ] && [ "${cyc}" -eq "${EARCINC_CYC}" ]; then SAVEWARMICA="YES" ; fi - if [ "${PDY}${cyc}" -eq "${firstday}" ] && [ "${cyc}" -eq "${EARCICS_CYC}" ]; then SAVEWARMICB="YES" ; fi - if [ "${mod}" -eq 0 ] && [ "${cyc}" ] && [ "${EARCINC_CYC}" ]; then SAVEWARMICA="YES" ; fi - if [ "${mod}" -eq 0 ] && [ "${cyc}" ] && [ "${EARCICS_CYC}" ]; then SAVEWARMICB="YES" ; fi - - if [ "${EARCICS_CYC}" -eq 18 ]; then - nday1=$((nday+1)) - mod1=$((nday1 % ARCH_WARMICFREQ)) - if [ "${mod1}" -eq 0 ] && [ "${cyc}" -eq "${EARCICS_CYC}" ] ; then SAVEWARMICB="YES" ; fi - if [ "${mod1}" -ne 0 ] && [ "${cyc}" -eq "${EARCICS_CYC}" ] ; then SAVEWARMICB="NO" ; fi - if [ "${PDY}${cyc}" -eq "${SDATE}" ] && [ "${cyc}" -eq "${EARCICS_CYC}" ] ; then SAVEWARMICB="YES" ; fi - fi - - if [ "${PDY}${cyc}" -gt "${SDATE}" ]; then # Don't run for first half cycle - - ${TARCMD} -P -cvf "${ATARDIR}/${PDY}${cyc}/${RUN}_grp${ENSGRP}.tar" $(cat "${DATA}/${RUN}_grp${n}.txt") - status=$? - if [ "${status}" -ne 0 ] && [ "${PDY}${cyc}" -ge "${firstday}" ]; then - echo "FATAL ERROR: ${TARCMD} ${PDY}${cyc} ${RUN}_grp${ENSGRP}.tar failed" - exit "${status}" - fi - - if [ "${SAVEWARMICA}" = "YES" ] && [ "${cyc}" -eq "${EARCINC_CYC}" ]; then - ${TARCMD} -P -cvf "${ATARDIR}/${PDY}${cyc}/${RUN}_restarta_grp${ENSGRP}.tar" $(cat "${DATA}/${RUN}_restarta_grp${n}.txt") - status=$? - if [ "${status}" -ne 0 ]; then - echo "FATAL ERROR: ${TARCMD} ${PDY}${cyc} ${RUN}_restarta_grp${ENSGRP}.tar failed" - exit "${status}" - fi - fi - - if [ "${SAVEWARMICB}" = "YES" ] && [ "${cyc}" -eq "${EARCICS_CYC}" ]; then - ${TARCMD} -P -cvf "${ATARDIR}/${PDY}${cyc}/${RUN}_restartb_grp${ENSGRP}.tar" $(cat "${DATA}/${RUN}_restartb_grp${n}.txt") - status=$? - if [ "${status}" -ne 0 ]; then - echo "FATAL ERROR: ${TARCMD} ${PDY}${cyc} ${RUN}_restartb_grp${ENSGRP}.tar failed" - exit "${status}" - fi - fi - - fi # CDATE>SDATE - -fi - - -################################################################### -# ENSGRP 0 archives ensemble means and copy data to online archive -if [ "${ENSGRP}" -eq 0 ]; then - - if [[ ${HPSSARCH} = "YES" || ${LOCALARCH} = "YES" ]]; then - - #--set the archiving command and create local directories, if necessary - TARCMD="htar" - HSICMD="hsi" - if [[ ${LOCALARCH} = "YES" ]]; then - TARCMD="tar" - HSICMD="" - if [[ ! -d "${ATARDIR}/${PDY}${cyc}" ]]; then mkdir -p "${ATARDIR}/${PDY}${cyc}"; fi - fi - - set +e - # Check if the tarball will have rstprod in it - has_rstprod="NO" - while IFS= read -r file; do - if [[ -f ${file} ]]; then - group=$( stat -c "%G" "${file}" ) - if [[ "${group}" == "rstprod" ]]; then - has_rstprod="YES" - break - fi - fi - done < "${DATA}/${RUN}.txt" - - # Create the tarball - tar_fl=${ATARDIR}/${PDY}${cyc}/${RUN}.tar - ${TARCMD} -P -cvf "${tar_fl}" $(cat "${DATA}/${RUN}.txt") - status=$? - if [[ "${status}" -ne 0 ]]; then - echo "FATAL ERROR: Tarring of ${tar_fl} failed" - exit "${status}" - fi - - # If rstprod was found, change the group of the tarball - if [[ "${has_rstprod}" == "YES" ]]; then - ${HSICMD} chgrp rstprod "${tar_fl}" - stat_chgrp=$? - ${HSICMD} chmod 640 "${tar_fl}" - stat_chgrp=$((stat_chgrp+$?)) - if [[ "${stat_chgrp}" -gt 0 ]]; then - echo "FATAL ERROR: Unable to properly restrict ${tar_fl}!" - echo "Attempting to delete ${tar_fl}" - ${HSICMD} rm "${tar_fl}" - echo "Please verify that ${tar_fl} was deleted!" - exit "${stat_chgrp}" - fi - fi - - # For safety, test if the htar/tar command failed only after changing groups - if (( status != 0 && ${PDY}${cyc} >= firstday )); then - echo "FATAL ERROR: ${TARCMD} ${tar_fl} failed" - exit "${status}" - fi - set_strict - fi - - #-- Archive online for verification and diagnostics - [[ ! -d ${ARCDIR} ]] && mkdir -p "${ARCDIR}" - cd "${ARCDIR}" || exit 2 - - nb_copy "${COM_ATMOS_ANALYSIS_ENSSTAT}/${RUN}.t${cyc}z.enkfstat" \ - "enkfstat.${RUN}.${PDY}${cyc}" - nb_copy "${COM_ATMOS_ANALYSIS_ENSSTAT}/${RUN}.t${cyc}z.gsistat.ensmean" \ - "gsistat.${RUN}.${PDY}${cyc}.ensmean" -fi - -exit 0 diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 7342b8790c..2e51691944 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -31,16 +31,11 @@ def main(): for key in keys: archive_dict[key] = archive.task_config[key] - # Also import all COM* directory names + # Also import all COM* directory and template variables for key in archive.task_config.keys(): if key.startswith("COM"): archive_dict[key] = archive.task_config[key] - # Also get all relative paths - for key in archive.task_config.keys(): - if key.endswith("_dir"): - archive_dict[key] = archive.task_config[key] - cwd = os.getcwd() os.chdir(config.ROTDIR) @@ -53,6 +48,5 @@ def main(): os.chdir(cwd) - if __name__ == '__main__': main() diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index f73982a13b..f09e3bf5d8 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -10,7 +10,7 @@ import numpy as np from wxflow import (AttrDict, FileHandler, Hsi, Htar, Task, cast_strdict_as_dtypedict, chgrp, get_gid, logit, mkdir_p, parse_j2yaml, rm_p, strftime, - to_YMD, to_YMDH) + to_YMD, to_YMDH, Template, TemplateConstants) from yaml import CLoader as Loader from yaml import load @@ -71,8 +71,6 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str # TODO remove this kludge once log filenames are explicit arch_dict['glob'] = glob.glob - # Copy the cyclone track files and rename the experiments - Archive._rename_cyclone_expt(arch_dict) # Add the os.path.exists function to the dict for yaml parsing arch_dict['path_exists'] = os.path.exists @@ -95,18 +93,13 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if not os.path.isdir(arch_dict.ROTDIR): raise FileNotFoundError(f"The ROTDIR ({arch_dict.ROTDIR}) does not exist!") - arch_mos = False - if arch_dict.DO_MOS: - if self.config.REALTIME: - if arch_dict.current_cycle - timedelta(days=1) > arch_dict.SDATE: - arch_mos = True - else: - arch_mos = True - cycle_HH = strftime(arch_dict.current_cycle, "%H") if arch_dict.RUN == "gdas" or arch_dict.RUN == "gfs": + # Copy the cyclone track files and rename the experiments + Archive._rename_cyclone_expt(arch_dict) + arch_ics_cycle = arch_dict.ARCH_CYC - arch_dict.assim_freq if arch_ics_cycle < 0: @@ -198,14 +191,76 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if arch_dict.DO_BUFRSND: datasets.append("gfs_downstream") - if arch_mos and cycle_HH == "18": - datasets.append("gfsmos") + if arch_dict.DO_MOS: + arch_mos = False + if self.config.REALTIME: + if arch_dict.current_cycle - timedelta(days=1) > arch_dict.SDATE: + arch_mos = True + else: + arch_mos = True + + if arch_mos and cycle_HH == "18": + datasets.append("gfsmos") - elif arch_dict.RUN == "enkfgdas": - raise NotImplementedError("Archiving is not yet set up for ENKF GDAS runs") + elif arch_dict.RUN == "enkfgdas" or arch_dict.RUN == "enkfgfs": - elif arch_dict.RUN == "enkfgfs": - raise NotImplementedError("Archiving is not yet set up for ENKF GFS runs") + + if arch_dict.ENSGRP == 0: + datasets = [ "enkf" ] + + else: + + # Determine which members to archive + first_mem = (arch_dict.ENSGRP - 1) * arch_dict.NMEM_EARCGRP + 1 + last_mem = min(arch_dict.NMEM_ENS, + arch_dict.ENSGRP * arch_dict.NMEM_EARCGRP) + mem_list = [ f"{mem:03d}" for mem in range(first_mem, last_mem + 1) ] + + arch_dict["first_group_mem"] = first_mem + arch_dict["last_group_mem"] = last_mem + + # Create a list of IAU forecast hours from IAUFHRS + if isinstance(arch_dict.IAUFHRS, int): + # First half-cycle or if 3dvar + arch_dict["iaufhrs"] = [arch_dict.IAUFHRS] + else: + arch_dict["iaufhrs"] = [int(fhr) for fhr in arch_dict.IAUFHRS.split(",")] + # Create a dict to define COM template parameters + tmpl_dict = { + 'ROTDIR': self.task_config.ROTDIR, + 'RUN': self.task_config.RUN, + 'YMD': to_YMD(self.task_config.current_cycle), + 'HH': self.task_config.current_cycle.strftime('%H') + } + + tmpl_atm_anl = self.task_config.COM_ATMOS_ANALYSIS_TMPL + tmpl_atm_res = self.task_config.COM_ATMOS_RESTART_TMPL + tmpl_atm_hst = self.task_config.COM_ATMOS_HISTORY_TMPL + + # Construct lists of COM directories to archive data from + arch_dict["COM_ATMOS_ANALYSIS_MEM_list"] = [] + arch_dict["COM_ATMOS_RESTART_MEM_list"] = [] + arch_dict["COM_ATMOS_HISTORY_MEM_list"] = [] + DCB = TemplateConstants.DOLLAR_CURLY_BRACE + for mem in mem_list: + tmpl_dict["MEMDIR"] = "mem" + mem + com_atm_anl = Template.substitute_structure( + tmpl_atm_anl, DCB, tmpl_dict.get) + arch_dict.COM_ATMOS_ANALYSIS_MEM_list.append(com_atm_anl) + + com_atm_res = Template.substitute_structure( + tmpl_atm_res, DCB, tmpl_dict.get) + arch_dict.COM_ATMOS_RESTART_MEM_list.append(com_atm_res) + + com_atm_hst = Template.substitute_structure( + tmpl_atm_hst, DCB, tmpl_dict.get) + arch_dict.COM_ATMOS_HISTORY_MEM_list.append(com_atm_hst) + + # Declare the datasets to archive + datasets = ["enkf_grp"] + + if arch_dict.current_cycle != arch_dict.SDATE: + datasets.extend(["enkf_restarta_grp", "enkf_restartb_grp"]) elif arch_dict.RUN == "gefs": raise NotImplementedError("Archiving is not yet set up for GEFS runs") From 8611bf8a686ed5345145a509e648dedbb06fe3bd Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 18 Apr 2024 19:54:59 +0000 Subject: [PATCH 020/128] Remove now-defunct hpssarch_gen.sh. --- ush/hpssarch_gen.sh | 798 -------------------------------------------- 1 file changed, 798 deletions(-) delete mode 100755 ush/hpssarch_gen.sh diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh deleted file mode 100755 index d46fe38f18..0000000000 --- a/ush/hpssarch_gen.sh +++ /dev/null @@ -1,798 +0,0 @@ -#! /usr/bin/env bash - -################################################### -# Fanglin Yang, 20180318 -# --create bunches of files to be archived to HPSS -################################################### -source "${USHgfs}/preamble.sh" - -type=${1:-gfs} ##gfs, gdas, enkfgdas or enkfggfs - -ARCH_GAUSSIAN=${ARCH_GAUSSIAN:-"YES"} -ARCH_GAUSSIAN_FHMAX=${ARCH_GAUSSIAN_FHMAX:-36} -ARCH_GAUSSIAN_FHINC=${ARCH_GAUSSIAN_FHINC:-6} - -# Set whether to archive downstream products -DO_DOWN=${DO_DOWN:-"NO"} -if [[ ${DO_BUFRSND} = "YES" ]]; then - export DO_DOWN="YES" -fi - -#----------------------------------------------------- -if [[ ${type} = "gfs" ]]; then -#----------------------------------------------------- - FHMIN_GFS=${FHMIN_GFS:-0} - FHMAX_GFS=${FHMAX_GFS:-384} - FHOUT_GFS=${FHOUT_GFS:-3} - FHMAX_HF_GFS=${FHMAX_HF_GFS:-120} - FHOUT_HF_GFS=${FHOUT_HF_GFS:-1} - - rm -f "${DATA}/gfsa.txt" - rm -f "${DATA}/gfsb.txt" - rm -f "${DATA}/gfs_restarta.txt" - touch "${DATA}/gfsa.txt" - touch "${DATA}/gfsb.txt" - touch "${DATA}/gfs_restarta.txt" - - if [[ ${ARCH_GAUSSIAN} = "YES" ]]; then - rm -f "${DATA}/gfs_pgrb2b.txt" - rm -f "${DATA}/gfs_netcdfb.txt" - rm -f "${DATA}/gfs_flux.txt" - touch "${DATA}/gfs_pgrb2b.txt" - touch "${DATA}/gfs_netcdfb.txt" - touch "${DATA}/gfs_flux.txt" - - if [[ ${MODE} = "cycled" ]]; then - rm -f "${DATA}/gfs_netcdfa.txt" - touch "${DATA}/gfs_netcdfa.txt" - fi - fi - - if [[ ${DO_DOWN} = "YES" ]]; then - rm -f "${DATA}/gfs_downstream.txt" - touch "${DATA}/gfs_downstream.txt" - fi - - head="gfs.t${cyc}z." - - if [[ ${ARCH_GAUSSIAN} = "YES" ]]; then - { - echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2b.0p25.anl" - echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2b.0p25.anl.idx" - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2b.1p00.anl" - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2b.1p00.anl.idx" - } >> "${DATA}/gfs_pgrb2b.txt" - - if [[ ${MODE} = "cycled" ]]; then - { - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}atmanl.nc" - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}sfcanl.nc" - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}atmi*.nc" - gsida_files=("dtfanl.nc" - "loginc.txt") - for file in "${gsida_files[@]}"; do - [[ -s ${COM_ATMOS_ANALYSIS}/${head}${file} ]] && echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}${file}" - done - } >> "${DATA}/gfs_netcdfa.txt" - fi - - fh=0 - while (( fh <= ARCH_GAUSSIAN_FHMAX )); do - fhr=$(printf %03i "${fh}") - { - echo "${COM_ATMOS_HISTORY/${ROTDIR}\//}/${head}atmf${fhr}.nc" - echo "${COM_ATMOS_HISTORY/${ROTDIR}\//}/${head}sfcf${fhr}.nc" - } >> "${DATA}/gfs_netcdfb.txt" - fh=$((fh+ARCH_GAUSSIAN_FHINC)) - done - fi - - #.................. - # Exclude the gfsarch.log file, which will change during the tar operation - # This uses the bash extended globbing option - { - echo "./logs/${PDY}${cyc}/gfs!(arch).log" - echo "${COM_CONF/${ROTDIR}\//}/ufs.input.nml" - - if [[ ${MODE} = "cycled" ]]; then - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}gsistat" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}gsistat" - fi - gsiob_files=("nsstbufr" - "prepbufr" - "prepbufr.acft_profiles") - for file in "${gsiob_files[@]}"; do - [[ -s ${COM_OBS}/${head}${file} ]] && echo "${COM_OBS/${ROTDIR}\//}/${head}${file}" - done - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}atmvar.yaml" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}atmvar.yaml" - fi - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}atmstat" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}atmstat" - fi - fi - - echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2.0p25.anl" - echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2.0p25.anl.idx" - - #Only generated if there are cyclones to track - cyclone_files=("avno.t${cyc}z.cyclone.trackatcfunix" - "avnop.t${cyc}z.cyclone.trackatcfunix" - "trak.gfso.atcfunix.${PDY}${cyc}" - "trak.gfso.atcfunix.altg.${PDY}${cyc}") - - for file in "${cyclone_files[@]}"; do - [[ -s ${COM_ATMOS_TRACK}/${file} ]] && echo "${COM_ATMOS_TRACK/${ROTDIR}\//}/${file}" - done - - genesis_files=("storms.gfso.atcf_gen.${PDY}${cyc}" - "storms.gfso.atcf_gen.altg.${PDY}${cyc}") - for file in "${genesis_files[@]}"; do - [[ -s ${COM_ATMOS_GENESIS}/${file} ]] && echo "${COM_ATMOS_GENESIS/${ROTDIR}\//}/${file}" - done - - # GSI Monitor job output - - if [[ ${DO_VMINMON} = "YES" ]]; then - echo "${COM_ATMOS_MINMON/${ROTDIR}\//}/${PDY}${cyc}.costs.txt" - echo "${COM_ATMOS_MINMON/${ROTDIR}\//}/${PDY}${cyc}.cost_terms.txt" - echo "${COM_ATMOS_MINMON/${ROTDIR}\//}/${PDY}${cyc}.gnorms.ieee_d" - echo "${COM_ATMOS_MINMON/${ROTDIR}\//}/${PDY}${cyc}.reduction.ieee_d" - echo "${COM_ATMOS_MINMON/${ROTDIR}\//}/gnorm_data.txt" - fi - - } >> "${DATA}/gfsa.txt" - - { - if [[ ${DO_DOWN} = "YES" ]]; then - if [[ ${DO_BUFRSND} = "YES" ]]; then - echo "${COM_ATMOS_GEMPAK/${ROTDIR}\//}/gfs_${PDY}${cyc}.sfc" - echo "${COM_ATMOS_GEMPAK/${ROTDIR}\//}/gfs_${PDY}${cyc}.snd" - echo "${COM_ATMOS_WMO/${ROTDIR}\//}/gfs_collective*.postsnd_${cyc}" - echo "${COM_ATMOS_BUFR/${ROTDIR}\//}/bufr.t${cyc}z" - echo "${COM_ATMOS_BUFR/${ROTDIR}\//}/gfs.t${cyc}z.bufrsnd.tar.gz" - fi - fi - } >> "${DATA}/gfs_downstream.txt" - - { - echo "${COM_ATMOS_GRIB_0p50/${ROTDIR}\//}/${head}pgrb2.0p50.anl" - echo "${COM_ATMOS_GRIB_0p50/${ROTDIR}\//}/${head}pgrb2.0p50.anl.idx" - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2.1p00.anl" - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2.1p00.anl.idx" - } >> "${DATA}/gfsb.txt" - - - fh=0 - while (( fh <= FHMAX_GFS )); do - fhr=$(printf %03i "${fh}") - if [[ ${ARCH_GAUSSIAN} = "YES" ]]; then - { - echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2" - echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2.idx" - } >> "${DATA}/gfs_flux.txt" - - { - echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2b.0p25.f${fhr}" - echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2b.0p25.f${fhr}.idx" - if [[ -s "${COM_ATMOS_GRIB_1p00}/${head}pgrb2b.1p00.f${fhr}" ]]; then - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2b.1p00.f${fhr}" - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2b.1p00.f${fhr}.idx" - fi - } >> "${DATA}/gfs_pgrb2b.txt" - fi - - { - echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2.0p25.f${fhr}" - echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2.0p25.f${fhr}.idx" - echo "${COM_ATMOS_HISTORY/${ROTDIR}\//}/${head}atm.logf${fhr}.txt" - } >> "${DATA}/gfsa.txt" - - - { - if [[ -s "${COM_ATMOS_GRIB_0p50}/${head}pgrb2.0p50.f${fhr}" ]]; then - echo "${COM_ATMOS_GRIB_0p50/${ROTDIR}\//}/${head}pgrb2.0p50.f${fhr}" - echo "${COM_ATMOS_GRIB_0p50/${ROTDIR}\//}/${head}pgrb2.0p50.f${fhr}.idx" - fi - if [[ -s "${COM_ATMOS_GRIB_1p00}/${head}pgrb2.1p00.f${fhr}" ]]; then - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2.1p00.f${fhr}" - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2.1p00.f${fhr}.idx" - fi - } >> "${DATA}/gfsb.txt" - - inc=${FHOUT_GFS} - if (( FHMAX_HF_GFS > 0 && FHOUT_HF_GFS > 0 && fh < FHMAX_HF_GFS )); then - inc=${FHOUT_HF_GFS} - fi - - fh=$((fh+inc)) - done - - #.................. - { - if [[ ${MODE} = "cycled" ]]; then - echo "${COM_ATMOS_RESTART/${ROTDIR}\//}/*0000.sfcanl_data.tile1.nc" - echo "${COM_ATMOS_RESTART/${ROTDIR}\//}/*0000.sfcanl_data.tile2.nc" - echo "${COM_ATMOS_RESTART/${ROTDIR}\//}/*0000.sfcanl_data.tile3.nc" - echo "${COM_ATMOS_RESTART/${ROTDIR}\//}/*0000.sfcanl_data.tile4.nc" - echo "${COM_ATMOS_RESTART/${ROTDIR}\//}/*0000.sfcanl_data.tile5.nc" - echo "${COM_ATMOS_RESTART/${ROTDIR}\//}/*0000.sfcanl_data.tile6.nc" - elif [[ ${MODE} = "forecast-only" ]]; then - echo "${COM_ATMOS_INPUT/${ROTDIR}\//}/gfs_ctrl.nc" - echo "${COM_ATMOS_INPUT/${ROTDIR}\//}/gfs_data.tile1.nc" - echo "${COM_ATMOS_INPUT/${ROTDIR}\//}/gfs_data.tile2.nc" - echo "${COM_ATMOS_INPUT/${ROTDIR}\//}/gfs_data.tile3.nc" - echo "${COM_ATMOS_INPUT/${ROTDIR}\//}/gfs_data.tile4.nc" - echo "${COM_ATMOS_INPUT/${ROTDIR}\//}/gfs_data.tile5.nc" - echo "${COM_ATMOS_INPUT/${ROTDIR}\//}/gfs_data.tile6.nc" - echo "${COM_ATMOS_INPUT/${ROTDIR}\//}/sfc_data.tile1.nc" - echo "${COM_ATMOS_INPUT/${ROTDIR}\//}/sfc_data.tile2.nc" - echo "${COM_ATMOS_INPUT/${ROTDIR}\//}/sfc_data.tile3.nc" - echo "${COM_ATMOS_INPUT/${ROTDIR}\//}/sfc_data.tile4.nc" - echo "${COM_ATMOS_INPUT/${ROTDIR}\//}/sfc_data.tile5.nc" - echo "${COM_ATMOS_INPUT/${ROTDIR}\//}/sfc_data.tile6.nc" - fi - } >> "${DATA}/gfs_restarta.txt" - - - #.................. - if [[ ${DO_WAVE} = "YES" ]]; then - - rm -rf "${DATA}/gfswave.txt" - touch "${DATA}/gfswave.txt" - - head="gfswave.t${cyc}z." - - #........................... - { - echo "${COM_WAVE_HISTORY/${ROTDIR}\//}/ww3_multi*" - echo "${COM_WAVE_GRID/${ROTDIR}\//}/${head}*" - echo "${COM_WAVE_STATION/${ROTDIR}\//}/${head}*" - } >> "${DATA}/gfswave.txt" - fi - - if [[ "${DO_OCN}" == "YES" ]]; then - - head="gfs.ocean.t${cyc}z." - rm -f "${DATA}/ocean_6hravg.txt"; touch "${DATA}/ocean_6hravg.txt" - rm -f "${DATA}/ocean_daily.txt"; touch "${DATA}/ocean_daily.txt" - rm -f "${DATA}/ocean_grib2.txt"; touch "${DATA}/ocean_grib2.txt" - - echo "${COM_OCEAN_HISTORY/${ROTDIR}\//}/${head}6hr_avg.f*.nc" >> "${DATA}/ocean_6hravg.txt" - echo "${COM_OCEAN_HISTORY/${ROTDIR}\//}/${head}daily.f*.nc" >> "${DATA}/ocean_daily.txt" - - { - if [[ -d "${COM_OCEAN_GRIB}/5p00" ]]; then - echo "${COM_OCEAN_GRIB/${ROTDIR}\//}/5p00/${head}5p00.f*.grib2" - echo "${COM_OCEAN_GRIB/${ROTDIR}\//}/5p00/${head}5p00.f*.grib2.idx" - fi - if [[ -d "${COM_OCEAN_GRIB}/1p00" ]]; then - echo "${COM_OCEAN_GRIB/${ROTDIR}\//}/1p00/${head}1p00.f*.grib2" - echo "${COM_OCEAN_GRIB/${ROTDIR}\//}/1p00/${head}1p00.f*.grib2.idx" - fi - if [[ -d "${COM_OCEAN_GRIB}/0p25" ]]; then - echo "${COM_OCEAN_GRIB/${ROTDIR}\//}/0p25/${head}0p25.f*.grib2" - echo "${COM_OCEAN_GRIB/${ROTDIR}\//}/0p25/${head}0p25.f*.grib2.idx" - fi - } >> "${DATA}/ocean_grib2.txt" - - # Also save fluxes from atmosphere - head="gfs.t${cyc}z." - rm -f "${DATA}/gfs_flux_1p00.txt"; touch "${DATA}/gfs_flux_1p00.txt" - { - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}flux.1p00.f???" - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}flux.1p00.f???.idx" - } >> "${DATA}/gfs_flux_1p00.txt" - fi - - if [[ "${DO_ICE}" == "YES" ]]; then - head="gfs.ice.t${cyc}z." - rm -f "${DATA}/ice_6hravg.txt"; touch "${DATA}/ice_6hravg.txt" - rm -f "${DATA}/ice_grib2.txt"; touch "${DATA}/ice_grib2.txt" - - { - echo "${COM_ICE_HISTORY/${ROTDIR}\//}/${head}ic.nc" - echo "${COM_ICE_HISTORY/${ROTDIR}\//}/${head}6hr_avg.f*.nc" - } >> "${DATA}/ice_6hravg.txt" - - { - if [[ -d "${COM_ICE_GRIB}/5p00" ]]; then - echo "${COM_ICE_GRIB/${ROTDIR}\//}/5p00/${head}5p00.f*.grib2" - echo "${COM_ICE_GRIB/${ROTDIR}\//}/5p00/${head}5p00.f*.grib2.idx" - fi - if [[ -d "${COM_ICE_GRIB}/1p00" ]]; then - echo "${COM_ICE_GRIB/${ROTDIR}\//}/1p00/${head}1p00.f*.grib2" - echo "${COM_ICE_GRIB/${ROTDIR}\//}/1p00/${head}1p00.f*.grib2.idx" - fi - if [[ -d "${COM_ICE_GRIB}/0p25" ]]; then - echo "${COM_ICE_GRIB/${ROTDIR}\//}/0p25/${head}0p25.f*.grib2" - echo "${COM_ICE_GRIB/${ROTDIR}\//}/0p25/${head}0p25.f*.grib2.idx" - fi - } >> "${DATA}/ice_grib2.txt" - fi - - if [[ ${DO_AERO} = "YES" ]]; then - head="gocart" - - rm -f "${DATA}/chem.txt" - touch "${DATA}/chem.txt" - - echo "${COM_CHEM_HISTORY/${ROTDIR}\//}/${head}*" >> "${DATA}/chem.txt" - fi - -#----------------------------------------------------- -fi ##end of gfs -#----------------------------------------------------- - - - -#----------------------------------------------------- -if [[ ${type} == "gdas" ]]; then -#----------------------------------------------------- - - rm -f "${DATA}/gdas.txt" - rm -f "${DATA}/gdas_restarta.txt" - rm -f "${DATA}/gdas_restartb.txt" - touch "${DATA}/gdas.txt" - touch "${DATA}/gdas_restarta.txt" - touch "${DATA}/gdas_restartb.txt" - - head="gdas.t${cyc}z." - - #.................. - { - echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2.0p25.anl" - echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2.0p25.anl.idx" - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2.1p00.anl" - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2.1p00.anl.idx" - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}atmanl.nc" - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}sfcanl.nc" - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}atmvar.yaml" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}atmvar.yaml" - fi - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}atmstat" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}atmstat" - fi - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}gsistat" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}gsistat" - fi - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}atmanl.ensres.nc" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}atmanl.ensres.nc" - fi - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}atma003.ensres.nc" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}atma003.ensres.nc" - fi - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}atma009.ensres.nc" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}atma009.ensres.nc" - fi - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}cnvstat" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}cnvstat" - fi - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}oznstat" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}oznstat" - fi - if [[ ${DO_AERO} = "YES" ]]; then - if [[ -s "${COM_CHEM_ANALYSIS}/${head}aerostat" ]]; then - echo "${COM_CHEM_ANALYSIS/${ROTDIR}\//}/${head}aerostat" - fi - fi - if [[ -s "${COM_SNOW_ANALYSIS}/${head}snowstat.tgz" ]]; then - echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/${head}snowstat.tgz" - fi - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}radstat" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}radstat" - fi - for fstep in prep anal fcst verfozn verfrad vminmon; do - if [[ -s "${ROTDIR}/logs/${PDY}${cyc}/gdas${fstep}.log" ]]; then - echo "./logs/${PDY}${cyc}/gdas${fstep}.log" - fi - done - echo "./logs/${PDY}${cyc}/gdas*prod*.log" - if [[ "${WRITE_DOPOST}" == ".false." ]]; then - echo "./logs/${PDY}${cyc}/gdas*upp*.log" - fi - - fh=0 - while [[ ${fh} -le 9 ]]; do - fhr=$(printf %03i "${fh}") - echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2" - echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2.idx" - echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2.0p25.f${fhr}" - echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2.0p25.f${fhr}.idx" - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2.1p00.f${fhr}" - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2.1p00.f${fhr}.idx" - echo "${COM_ATMOS_HISTORY/${ROTDIR}\//}/${head}atm.logf${fhr}.txt" - echo "${COM_ATMOS_HISTORY/${ROTDIR}\//}/${head}atmf${fhr}.nc" - echo "${COM_ATMOS_HISTORY/${ROTDIR}\//}/${head}sfcf${fhr}.nc" - fh=$((fh+3)) - done - flist="001 002 004 005 007 008" - for fhr in ${flist}; do - file="${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2" - if [[ -s "${file}" ]]; then - echo "${file}" - echo "${file}.idx" - fi - done - - # GSI Monitor jobs output - - if [[ ${DO_VERFOZN} = "YES" ]]; then - for type in horiz time; do - if [[ ${type} = "horiz" ]]; then - suffix=".gz" - elif [[ ${type} = "time" ]]; then - suffix="" - echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/bad_cnt.${PDY}${cyc}" - echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/bad_diag.${PDY}${cyc}" - echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/bad_pen.${PDY}${cyc}" - fi - subtyplist="gome_metop-b omi_aura ompslp_npp ompsnp_n20 ompsnp_npp ompstc8_n20 ompstc8_npp sbuv2_n19" - for subtype in ${subtyplist}; do - # On occassion, data is not available for some of these satellites. Check for existence. - if [[ -s "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.ges.${PDY}${cyc}.ieee_d${suffix}" ]]; then - echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.anl.${PDY}${cyc}.ieee_d${suffix}" - echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.anl.ctl" - echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.ges.${PDY}${cyc}.ieee_d${suffix}" - echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.ges.ctl" - fi - done - echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/stdout.${type}.tar.gz" - done - fi - - if [[ ${DO_VERFRAD} = "YES" ]]; then - echo "${COM_ATMOS_RADMON/${ROTDIR}\//}/bad_diag.${PDY}${cyc}" - echo "${COM_ATMOS_RADMON/${ROTDIR}\//}/bad_pen.${PDY}${cyc}" - echo "${COM_ATMOS_RADMON/${ROTDIR}\//}/low_count.${PDY}${cyc}" - echo "${COM_ATMOS_RADMON/${ROTDIR}\//}/radmon_angle.tar.gz" - echo "${COM_ATMOS_RADMON/${ROTDIR}\//}/radmon_bcoef.tar.gz" - echo "${COM_ATMOS_RADMON/${ROTDIR}\//}/radmon_bcor.tar.gz" - echo "${COM_ATMOS_RADMON/${ROTDIR}\//}/radmon_time.tar.gz" - echo "${COM_ATMOS_RADMON/${ROTDIR}\//}/warning.${PDY}${cyc}" - fi - - if [[ ${DO_VMINMON} = "YES" ]]; then - echo "${COM_ATMOS_MINMON/${ROTDIR}\//}/${PDY}${cyc}.costs.txt" - echo "${COM_ATMOS_MINMON/${ROTDIR}\//}/${PDY}${cyc}.cost_terms.txt" - echo "${COM_ATMOS_MINMON/${ROTDIR}\//}/${PDY}${cyc}.gnorms.ieee_d" - echo "${COM_ATMOS_MINMON/${ROTDIR}\//}/${PDY}${cyc}.reduction.ieee_d" - echo "${COM_ATMOS_MINMON/${ROTDIR}\//}/gnorm_data.txt" - fi - - } >> "${DATA}/gdas.txt" - - #.................. - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}cnvstat" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}cnvstat" >> "${DATA}/gdas_restarta.txt" - fi - if [[ -s "${COM_ATMOS_ANALYSIS}/${head}radstat" ]]; then - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}radstat" >> "${DATA}/gdas_restarta.txt" - fi - - { - gsiob_files=("nsstbufr" - "prepbufr" - "prepbufr.acft_profiles") - for file in "${gsiob_files[@]}"; do - [[ -s ${COM_OBS}/${head}${file} ]] && echo "${COM_OBS/${ROTDIR}\//}/${head}${file}" - done - - gsida_files=("abias" - "abias_air" - "abias_int" - "abias_pc" - "dtfanl.nc" - "loginc.txt") - for file in "${gsida_files[@]}"; do - [[ -s ${COM_ATMOS_ANALYSIS}/${head}${file} ]] && echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}${file}" - done - - ufsda_files=("amsua_n19.satbias.nc4" - "amsua_n19.satbias_cov.nc4" - "amsua_n19.tlapse.txt") - for file in "${ufsda_files[@]}"; do - [[ -s ${COM_ATMOS_ANALYSIS}/${head}${file} ]] && echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}${file}" - done - - echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}atmi*nc" - - echo "${COM_ATMOS_RESTART/${ROTDIR}\//}/*0000.sfcanl_data.tile1.nc" - echo "${COM_ATMOS_RESTART/${ROTDIR}\//}/*0000.sfcanl_data.tile2.nc" - echo "${COM_ATMOS_RESTART/${ROTDIR}\//}/*0000.sfcanl_data.tile3.nc" - echo "${COM_ATMOS_RESTART/${ROTDIR}\//}/*0000.sfcanl_data.tile4.nc" - echo "${COM_ATMOS_RESTART/${ROTDIR}\//}/*0000.sfcanl_data.tile5.nc" - echo "${COM_ATMOS_RESTART/${ROTDIR}\//}/*0000.sfcanl_data.tile6.nc" - - [[ -s "${COM_CONF}/${head}letkfoi.yaml" ]] && echo "${COM_CONF/${ROTDIR}\//}/${head}letkfoi.yaml" - - if [[ "${DO_JEDISNOWDA:-}" == "YES" ]]; then - echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile1.nc" - echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile2.nc" - echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile3.nc" - echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile4.nc" - echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile5.nc" - echo "${COM_SNOW_ANALYSIS/${ROTDIR}\//}/*0000.sfc_data.tile6.nc" - fi - } >> "${DATA}/gdas_restarta.txt" - - #.................. - echo "${COM_ATMOS_RESTART/${ROTDIR}\//}" >> "${DATA}/gdas_restartb.txt" - - #.................. - if [[ ${DO_WAVE} = "YES" ]]; then - - rm -rf "${DATA}/gdaswave.txt" - touch "${DATA}/gdaswave.txt" - rm -rf "${DATA}/gdaswave_restart.txt" - touch "${DATA}/gdaswave_restart.txt" - - head="gdaswave.t${cyc}z." - - #........................... - { - echo "${COM_WAVE_GRID/${ROTDIR}\//}/${head}*" - echo "${COM_WAVE_STATION/${ROTDIR}\//}/${head}*" - } >> "${DATA}/gdaswave.txt" - - echo "${COM_WAVE_RESTART/${ROTDIR}\//}/*" >> "${DATA}/gdaswave_restart.txt" - - fi - - #.................. - if [[ ${DO_OCN} = "YES" ]]; then - - rm -rf "${DATA}/gdasocean.txt" - touch "${DATA}/gdasocean.txt" - rm -rf "${DATA}/gdasocean_restart.txt" - touch "${DATA}/gdasocean_restart.txt" - - head="gdas.t${cyc}z." - - #........................... - { - echo "${COM_OCEAN_HISTORY/${ROTDIR}\//}/${head}*" - echo "${COM_OCEAN_INPUT/${ROTDIR}\//}" - } >> "${DATA}/gdasocean.txt" - - { - echo "${COM_OCEAN_RESTART/${ROTDIR}\//}/*" - echo "${COM_MED_RESTART/${ROTDIR}\//}/*" - } >> "${DATA}/gdasocean_restart.txt" - - { - echo "${COM_OCEAN_ANALYSIS/${ROTDIR}\//}/${head}*" - echo "${COM_OCEAN_ANALYSIS/${ROTDIR}\//}/gdas.t??z.ocngrid.nc" - echo "${COM_OCEAN_ANALYSIS/${ROTDIR}\//}/diags" - echo "${COM_OCEAN_ANALYSIS/${ROTDIR}\//}/yaml" - } >> "${DATA}/gdasocean_analysis.txt" - - fi - - if [[ ${DO_ICE} = "YES" ]]; then - - rm -rf "${DATA}/gdasice.txt" - touch "${DATA}/gdasice.txt" - rm -rf "${DATA}/gdasice_restart.txt" - touch "${DATA}/gdasice_restart.txt" - - head="gdas.t${cyc}z." - - #........................... - { - echo "${COM_ICE_HISTORY/${ROTDIR}\//}/${head}*" - echo "${COM_ICE_INPUT/${ROTDIR}\//}/ice_in" - } >> "${DATA}/gdasice.txt" - - echo "${COM_ICE_RESTART/${ROTDIR}\//}/*" >> "${DATA}/gdasice_restart.txt" - - fi - - -#----------------------------------------------------- -fi ##end of gdas -#----------------------------------------------------- - - -#----------------------------------------------------- -if [[ ${type} == "enkfgdas" || ${type} == "enkfgfs" ]]; then -#----------------------------------------------------- - - IAUFHRS_ENKF=${IAUFHRS_ENKF:-6} - lobsdiag_forenkf=${lobsdiag_forenkf:-".false."} - IFS=',' read -ra nfhrs <<< ${IAUFHRS_ENKF} - NMEM_ENS=${NMEM_ENS:-80} - NMEM_EARCGRP=${NMEM_EARCGRP:-10} ##number of ens memebers included in each tarball - NTARS=$((NMEM_ENS/NMEM_EARCGRP)) - [[ ${NTARS} -eq 0 ]] && NTARS=1 - [[ $((NTARS*NMEM_EARCGRP)) -lt ${NMEM_ENS} ]] && NTARS=$((NTARS+1)) - ##NTARS2=$((NTARS/2)) # number of earc groups to include analysis/increments - NTARS2=${NTARS} - - head="${RUN}.t${cyc}z." - - #.................. - rm -f "${DATA}/${RUN}.txt" - touch "${DATA}/${RUN}.txt" - - { - gsida_files=("enkfstat" - "gsistat.ensmean" - "cnvstat.ensmean" - "oznstat.ensmean" - "radstat.ensmean") - for file in "${gsida_files[@]}"; do - [[ -s ${COM_ATMOS_ANALYSIS_ENSSTAT}/${head}${file} ]] && echo "${COM_ATMOS_ANALYSIS_ENSSTAT/${ROTDIR}\//}/${head}${file}" - done - - ufsda_files=("atmens.yaml" - "atmensstat") - for file in "${ufsda_files[@]}"; do - [[ -s ${COM_ATMOS_ANALYSIS_ENSSTAT}/${head}${file} ]] && echo "${COM_ATMOS_ANALYSIS_ENSSTAT/${ROTDIR}\//}/${head}${file}" - done - - for FHR in "${nfhrs[@]}"; do # loop over analysis times in window - if [[ ${FHR} -eq 6 ]]; then - if [[ -s "${COM_ATMOS_ANALYSIS_ENSSTAT}/${head}atmanl.ensmean.nc" ]]; then - echo "${COM_ATMOS_ANALYSIS_ENSSTAT/${ROTDIR}\//}/${head}atmanl.ensmean.nc" - fi - if [[ -s "${COM_ATMOS_ANALYSIS_ENSSTAT}/${head}atminc.ensmean.nc" ]]; then - echo "${COM_ATMOS_ANALYSIS_ENSSTAT/${ROTDIR}\//}/${head}atminc.ensmean.nc" - fi - else - if [[ -s "${COM_ATMOS_ANALYSIS_ENSSTAT}/${head}atma00${FHR}.ensmean.nc" ]]; then - echo "${COM_ATMOS_ANALYSIS_ENSSTAT/${ROTDIR}\//}/${head}atma00${FHR}.ensmean.nc" - fi - if [[ -s "${COM_ATMOS_ANALYSIS_ENSSTAT}/${head}atmi00${FHR}.ensmean.nc" ]]; then - echo "${COM_ATMOS_ANALYSIS_ENSSTAT/${ROTDIR}\//}/${head}atmi00${FHR}.ensmean.nc" - fi - fi - done # loop over FHR - for fstep in fcst epos ; do - echo "logs/${PDY}${cyc}/${RUN}${fstep}*.log" - done - - # eobs, ecen, esfc, and eupd are not run on the first cycle - for fstep in eobs ecen esfc eupd ; do - for log in "${ROTDIR}/logs/${PDY}${cyc}/${RUN}${fstep}"*".log"; do - if [[ -s "${log}" ]]; then - echo "logs/${PDY}${cyc}/${RUN}${fstep}*.log" - fi - done - done - - # eomg* are optional jobs - for log in "${ROTDIR}/logs/${PDY}${cyc}/${RUN}eomg"*".log"; do - if [[ -s "${log}" ]]; then - echo "logs/${PDY}${cyc}/${RUN}eomg*.log" - fi - break - done - - # Ensemble spread file only available with netcdf output - fh=3 - while [ $fh -le 9 ]; do - fhr=$(printf %03i $fh) - echo "${COM_ATMOS_HISTORY_ENSSTAT/${ROTDIR}\//}/${head}atmf${fhr}.ensmean.nc" - echo "${COM_ATMOS_HISTORY_ENSSTAT/${ROTDIR}\//}/${head}sfcf${fhr}.ensmean.nc" - if [[ -s "${COM_ATMOS_HISTORY_ENSSTAT}/${head}atmf${fhr}.ensspread.nc" ]]; then - echo "${COM_ATMOS_HISTORY_ENSSTAT/${ROTDIR}\//}/${head}atmf${fhr}.ensspread.nc" - fi - fh=$((fh+3)) - done - } >> "${DATA}/${RUN}.txt" - - #........................... - n=1 - while (( n <= NTARS )); do - #........................... - - rm -f "${DATA}/${RUN}_grp${n}.txt" - rm -f "${DATA}/${RUN}_restarta_grp${n}.txt" - rm -f "${DATA}/${RUN}_restartb_grp${n}.txt" - touch "${DATA}/${RUN}_grp${n}.txt" - touch "${DATA}/${RUN}_restarta_grp${n}.txt" - touch "${DATA}/${RUN}_restartb_grp${n}.txt" - - m=1 - while (( m <= NMEM_EARCGRP && (n-1)*NMEM_EARCGRP+m <= NMEM_ENS )); do - nm=$(((n-1)*NMEM_EARCGRP+m)) - mem=$(printf %03i ${nm}) - head="${RUN}.t${cyc}z." - - MEMDIR="mem${mem}" YMD=${PDY} HH=${cyc} declare_from_tmpl \ - COM_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL \ - COM_ATMOS_RESTART_MEM:COM_ATMOS_RESTART_TMPL \ - COM_ATMOS_HISTORY_MEM:COM_ATMOS_HISTORY_TMPL - - #--- - for FHR in "${nfhrs[@]}"; do # loop over analysis times in window - if [ "${FHR}" -eq 6 ]; then - { - if (( n <= NTARS2 )); then - if [[ -s "${COM_ATMOS_ANALYSIS_MEM}/${head}atmanl.nc" ]] ; then - echo "${COM_ATMOS_ANALYSIS_MEM/${ROTDIR}\//}/${head}atmanl.nc" - fi - if [[ -s "${COM_ATMOS_ANALYSIS_MEM}/${head}ratminc.nc" ]] ; then - echo "${COM_ATMOS_ANALYSIS_MEM/${ROTDIR}\//}/${head}ratminc.nc" - fi - fi - } >> "${DATA}/${RUN}_grp${n}.txt" - - if [[ -s "${COM_ATMOS_ANALYSIS_MEM}/${head}ratminc.nc" ]] ; then - echo "${COM_ATMOS_ANALYSIS_MEM/${ROTDIR}\//}/${head}ratminc.nc" \ - >> "${DATA}/${RUN}_restarta_grp${n}.txt" - fi - - else - { - if (( n <= NTARS2 )); then - if [[ -s "${COM_ATMOS_ANALYSIS_MEM}/${head}atma00${FHR}.nc" ]] ; then - echo "${COM_ATMOS_ANALYSIS_MEM/${ROTDIR}\//}/${head}atma00${FHR}.nc" - fi - if [[ -s "${COM_ATMOS_ANALYSIS_MEM}/${head}ratmi00${FHR}.nc" ]] ; then - echo "${COM_ATMOS_ANALYSIS_MEM/${ROTDIR}\//}/${head}ratmi00${FHR}.nc" - fi - fi - } >> "${DATA}/${RUN}_grp${n}.txt" - if [[ -s "${COM_ATMOS_ANALYSIS_MEM}/${head}ratmi00${FHR}.nc" ]] ; then - echo "${COM_ATMOS_ANALYSIS_MEM/${ROTDIR}\//}/${head}ratmi00${FHR}.nc" \ - >> "${DATA}/${RUN}_restarta_grp${n}.txt" - fi - fi - { - echo "${COM_ATMOS_HISTORY_MEM/${ROTDIR}\//}/${head}atmf00${FHR}.nc" - if (( FHR == 6 )); then - echo "${COM_ATMOS_HISTORY_MEM/${ROTDIR}\//}/${head}sfcf00${FHR}.nc" - fi - } >> "${DATA}/${RUN}_grp${n}.txt" - done # loop over FHR - - if [[ ${lobsdiag_forenkf} == ".false." ]] ; then - { - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/${head}gsistat" - if [[ -s "${COM_ATMOS_RESTART_MEM}/${head}cnvstat" ]] ; then - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/${head}cnvstat" - fi - } >> "${DATA}/${RUN}_grp${n}.txt" - - { - if [[ -s "${COM_ATMOS_RESTART_MEM}/${head}radstat" ]]; then - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/${head}radstat" - fi - if [[ -s "${COM_ATMOS_RESTART_MEM}/${head}cnvstat" ]]; then - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/${head}cnvstat" - fi - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/${head}abias" - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/${head}abias_air" - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/${head}abias_int" - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/${head}abias_pc" - } >> "${DATA}/${RUN}_restarta_grp${n}.txt" - fi - #--- - { - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/*0000.sfcanl_data.tile1.nc" - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/*0000.sfcanl_data.tile2.nc" - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/*0000.sfcanl_data.tile3.nc" - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/*0000.sfcanl_data.tile4.nc" - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/*0000.sfcanl_data.tile5.nc" - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}/*0000.sfcanl_data.tile6.nc" - } >> "${DATA}/${RUN}_restarta_grp${n}.txt" - #--- - echo "${COM_ATMOS_RESTART_MEM/${ROTDIR}\//}" >> "${DATA}/${RUN}_restartb_grp${n}.txt" - - m=$((m+1)) - done - - - #........................... - n=$((n+1)) - done - #........................... - - -#----------------------------------------------------- -fi ##end of enkfgdas or enkfgfs -#----------------------------------------------------- - -exit 0 From 533cccce23e8d0a279aca0b52077f55af44f4313 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 18 Apr 2024 19:58:50 +0000 Subject: [PATCH 021/128] Address PEP8 warnings --- scripts/exglobal_archive.py | 1 + ush/python/pygfs/task/archive.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 2e51691944..7653d71d61 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -48,5 +48,6 @@ def main(): os.chdir(cwd) + if __name__ == '__main__': main() diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index f09e3bf5d8..7b9e8b2edc 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -204,9 +204,8 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str elif arch_dict.RUN == "enkfgdas" or arch_dict.RUN == "enkfgfs": - if arch_dict.ENSGRP == 0: - datasets = [ "enkf" ] + datasets = ["enkf"] else: @@ -214,7 +213,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str first_mem = (arch_dict.ENSGRP - 1) * arch_dict.NMEM_EARCGRP + 1 last_mem = min(arch_dict.NMEM_ENS, arch_dict.ENSGRP * arch_dict.NMEM_EARCGRP) - mem_list = [ f"{mem:03d}" for mem in range(first_mem, last_mem + 1) ] + mem_list = [f"{mem:03d}" for mem in range(first_mem, last_mem + 1)] arch_dict["first_group_mem"] = first_mem arch_dict["last_group_mem"] = last_mem From 05f54dc8498b0cd0f1ec140453854559d87aedcf Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 18 Apr 2024 20:05:48 +0000 Subject: [PATCH 022/128] Fix additional PEP8 warnings --- ush/python/pygfs/task/archive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 7b9e8b2edc..53211abae7 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -244,15 +244,15 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str for mem in mem_list: tmpl_dict["MEMDIR"] = "mem" + mem com_atm_anl = Template.substitute_structure( - tmpl_atm_anl, DCB, tmpl_dict.get) + tmpl_atm_anl, DCB, tmpl_dict.get) arch_dict.COM_ATMOS_ANALYSIS_MEM_list.append(com_atm_anl) com_atm_res = Template.substitute_structure( - tmpl_atm_res, DCB, tmpl_dict.get) + tmpl_atm_res, DCB, tmpl_dict.get) arch_dict.COM_ATMOS_RESTART_MEM_list.append(com_atm_res) com_atm_hst = Template.substitute_structure( - tmpl_atm_hst, DCB, tmpl_dict.get) + tmpl_atm_hst, DCB, tmpl_dict.get) arch_dict.COM_ATMOS_HISTORY_MEM_list.append(com_atm_hst) # Declare the datasets to archive From 31a1d9ac321068f14e4742a9b3df0f2bf1eb4023 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 19 Apr 2024 18:00:07 +0000 Subject: [PATCH 023/128] Fixes for snow DA archiving. --- parm/archive/arcdir.yaml.j2 | 2 +- parm/archive/gdas.yaml.j2 | 6 +++--- scripts/exglobal_archive.py | 5 +++-- ush/python/pygfs/task/snow_analysis.py | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index cc8fe6ffec..6b8930bc5f 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -19,7 +19,7 @@ deterministic: {% if not current_cycle == SDATE and MODE == "cycled" %} - ["{{ COM_ATMOS_ANALYSIS }}/{{ head }}gsistat", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}"] {% if DO_JEDISNOWDA %} - - ["{{ COM_SNOW_ANALYSIS }}/{{ head }}snowstat", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COM_SNOW_ANALYSIS }}/{{ head }}snowstat.tgz", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YMDH }}.tgz"] {% endif %} {% if DO_AERO %} - ["{{ COM_CHEM_ANALYSIS }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YMDH }}"] diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index ef39bdbe9a..d41461baeb 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -15,9 +15,11 @@ mandatory: - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" + {% if DOIAU and DOHYBVAR %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma003.ensres.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma009.ensres.nc" + {% endif %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}oznstat" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" @@ -61,7 +63,7 @@ mandatory: {% endif %} # Full cycle - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" - {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} + {% for fhr in range(0, FHMAX + FHOUT, FHOUT) %} - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" @@ -69,8 +71,6 @@ mandatory: - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" - {% endfor %} - {% for fhr in range(FHMIN, FHMAX + FHOUT) %} - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" {% endfor %} diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 7653d71d61..f61268fa47 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -24,8 +24,9 @@ def main(): 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', 'SDATE', 'FHMIN_GFS', 'FHMAX_GFS', 'FHOUT_GFS', 'ARCH_GAUSSIAN', 'MODE', - 'FHOUT_OCNICE', 'FHOUT_OCNICE_GFS', 'DO_BUFRSND', - 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC'] + 'FHOUT_OCNICE', 'FHOUT_OCNICE_GFS', 'DO_BUFRSND', 'DOHYBVAR', + 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC', + 'DOIAU'] archive_dict = AttrDict() for key in keys: diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index c149f140b6..24234bc82d 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -206,7 +206,7 @@ def prepare_IMS(self) -> None: # Execute imspy to create the IMS obs data in IODA format logger.info("Create IMS obs data in IODA format") - output_file = f"ims_snow_{to_YMDH(localconf.current_cycle)}.nc4" + output_file = f"ims_snow_{to_YMDH(localconf.current_cycle)}.nc" if os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"): rm_p(output_file) @@ -333,7 +333,7 @@ def finalize(self) -> None: Instance of the SnowAnalysis object """ - logger.info("Create diagnostic tarball of diag*.nc4 files") + logger.info("Create diagnostic tarball of diag*.nc files") statfile = os.path.join(self.task_config.COM_SNOW_ANALYSIS, f"{self.task_config.APREFIX}snowstat.tgz") self.tgz_diags(statfile, self.task_config.DATA) From ec747f76b91bc0faab8edb0b769fee751db77fce Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 19 Apr 2024 18:27:42 +0000 Subject: [PATCH 024/128] Remove unused fhmin. --- parm/archive/arcdir.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index 6b8930bc5f..f4a38f7c3c 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -27,9 +27,9 @@ deterministic: - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.anl", "{{ ARCDIR }}/pgbanl.{{ RUN }}.{{ cycle_YMDH }}.grib2"] {% endif %} # Full cycle {% if RUN == "gfs" %} - {% set fhmin, fhmax, fhout = FHMIN_GFS, FHMAX_GFS, FHOUT_GFS %} + {% set fhmax, fhout = FHMAX_GFS, FHOUT_GFS %} {% elif RUN == "gdas" %} - {% set fhmin, fhmax, fhout = FHMIN, FHMAX, FHOUT %} + {% set fhmax, fhout = FHMAX, FHOUT %} {% endif %} {% for fhr in range(0, fhmax + fhout, fhout) %} - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"] From 58840c4eff0da897c029e45576bf0a1eed875b78 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 19 Apr 2024 18:31:00 +0000 Subject: [PATCH 025/128] Better explicitize enkf archiving --- parm/archive/enkf.yaml.j2 | 12 +++++++----- parm/config/gfs/config.base | 3 +++ parm/config/gfs/config.epos | 3 --- scripts/exgdas_enkf_earc.py | 3 ++- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index a71dbd77c4..b3e484d533 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -12,6 +12,9 @@ mandatory: {% for fhr in range(fhmin, fhmax + fhout, fhout) %} - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensmean.nc" - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.ensmean.nc" + {% if ENKF_SPREAD %} + - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensspread.nc" + {% endif %} {% endfor %} {% if not current_cycle == SDATE %} {% if not DO_JEDIATMENS %} @@ -40,11 +43,14 @@ mandatory: {% endif %} # fhr == IAU_OFFSET {% endfor %} # IAUFHRS {% endif %} # DOIAU + # TODO archive ecen logs based on actual groups. Will need to emulate numpy.array_split to do so. {% set steps = ["eobs", "ecen*", "esfc", "eupd"] %} {% for step in steps %} - "logs/{{ cycle_YMDH }}/{{ RUN }}{{ step }}.log" {% endfor %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_mem*.log" + {% for mem in range(1, NMEM_ENS + 1) %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_mem{{ '%03d' % mem }}.log" + {% endfor %} {% if lobsdiag_forenkf %} - "logs/{{ cycle_YMDH }}/{{ RUN }}ediag.log" {% else %} @@ -53,7 +59,3 @@ mandatory: {% endfor %} {% endif %} # lobsdiag_forenkf {% endif %} # not the first cycle -optional: - {% for fhr in range(fhmin, fhmax + fhout, fhout) %} - - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensspread.nc" - {% endfor %} diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 5a58c75220..4995560684 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -363,6 +363,9 @@ if [[ ${DOHYBVAR} == "NO" && ${DOIAU} == "YES" ]]; then export IAUFHRS_ENKF="6" fi +# Generate post-processing ensemble spread files +export ENKF_SPREAD="YES" + # Check if cycle is cold starting, DOIAU off, or free-forecast mode if [[ "${MODE}" = "cycled" && "${SDATE}" = "${PDY}${cyc}" && ${EXP_WARM_START} = ".false." ]] || [[ "${DOIAU}" = "NO" ]] || [[ "${MODE}" = "forecast-only" && ${EXP_WARM_START} = ".false." ]] ; then export IAU_OFFSET=0 diff --git a/parm/config/gfs/config.epos b/parm/config/gfs/config.epos index 8026a2ba2e..f1da929b62 100644 --- a/parm/config/gfs/config.epos +++ b/parm/config/gfs/config.epos @@ -14,7 +14,4 @@ if [ $l4densvar = ".false." ]; then export NEPOSGRP=3 fi -# Generate ensemble spread files -export ENKF_SPREAD="YES" - echo "END: config.epos" diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index 48ae3836f4..9d8ca80467 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -23,7 +23,8 @@ def main(): 'ARCDIR', 'SDATE', 'MODE', 'ENSGRP', 'NMEM_EARCGRP', 'NMEM_ENS', 'DO_CALC_INCREMENT_ENKF_GFS', 'DO_JEDIATMENS', 'lobsdiag_forenkf', 'FHMIN_ENKF', 'FHMAX_ENKF_GFS', - 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF'] + 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF', + 'ENKF_SPREAD'] archive_dict = AttrDict() for key in keys: From 7532aebab83d026633eb1b800b5c8de01b5cca04 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 22 Apr 2024 15:56:31 +0000 Subject: [PATCH 026/128] Further explicitize archiving templates. --- parm/archive/enkf_restarta_grp.yaml.j2 | 6 ++++-- parm/archive/enkf_restartb_grp.yaml.j2 | 22 ++++++++++++++++++++-- parm/archive/gdas.yaml.j2 | 7 +++---- parm/archive/gdas_restarta.yaml.j2 | 24 ++++++++++++++++++------ parm/archive/gdas_restartb.yaml.j2 | 20 +++++++++++++++++++- parm/archive/gfs_netcdfa.yaml.j2 | 5 ++++- parm/archive/gfs_pgrb2b.yaml.j2 | 9 +++------ parm/archive/gfs_restarta.yaml.j2 | 9 ++++++++- parm/archive/gfs_restartb.yaml.j2 | 23 ++++++++++++++++++++++- parm/archive/gfsa.yaml.j2 | 22 +++++++++++----------- parm/archive/gfsb.yaml.j2 | 3 ++- parm/archive/gfswave.yaml.j2 | 3 ++- parm/archive/ice_6hravg.yaml.j2 | 6 ++++-- parm/archive/ice_grib2.yaml.j2 | 22 ++++++++++++---------- parm/archive/ocean_6hravg.yaml.j2 | 6 ++++-- parm/archive/ocean_daily.yaml.j2 | 4 +++- parm/archive/ocean_grib2.yaml.j2 | 10 ++++++---- scripts/exglobal_archive.py | 2 +- sorc/wxflow | 2 +- 19 files changed, 147 insertions(+), 58 deletions(-) diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 index b2c936c69e..99c474f120 100644 --- a/parm/archive/enkf_restarta_grp.yaml.j2 +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -17,8 +17,10 @@ mandatory: - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_pc" {% endif %} - {% for itile in range(1,7) %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/*0000.sfcanl_data.tile{{ itile }}.nc" + {% set anl_delta = "-3H" | to_timedelta %} + {% set anl_time = current_cycle | add_to_datetime(anl_delta) %} + {% for itile in range(1, 7) %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" {% endfor %} {% endfor %} # first_group_mem to last_group_mem optional: diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 index 4e7b80560c..debbbe63b0 100644 --- a/parm/archive/enkf_restartb_grp.yaml.j2 +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -7,6 +7,24 @@ mandatory: {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}" + {% if DOIAU_ENKF %} + {% set rst_times = ["+3", "+6", "+9"] %} + {% else %} + {% set rst_times = ["+6"] %} + {% endif %} + {% for f_time in rst_times %} + {% set f_timedelta = (f_time + "H") | to_timedelta %} + {% set f_dt = current_cycle | add_to_datetime(f_timedelta) %} + {% set f_YMD = f_dt | to_YMD %} + {% set f_HH = f_dt | strftime("%H") %} + {% set f_prefix = f_YMD + "." + f_HH + "0000" %} + {% for itile in range(1, 7) %} + {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ f_prefix }}.{{datatype}}.tile{{ itile }}.nc" + {% endfor %} + {% endfor %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ f_prefix }}.atm_stoch.res.nc" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ f_prefix }}.coupler.res" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ f_prefix }}.fv_core.res.nc" + {% endfor %} {% endfor %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index d41461baeb..0fbc0a8641 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -83,13 +83,12 @@ optional: "ompsnp_npp", "ompstc8_n20", "ompstc8_npp", "sbuv2_n19" ] %} {% for group in [ "horiz", "time" ] %} + {% if group == "horiz" %} {% set suffix = ".gz" %} {% else %} {% set suffix = "" %} {% endif %} {% for type in oznmon_types %} - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.ctl" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.{{ cycle_YMDH }}.ieee_d" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.{{ cycle_YMDH }}.ieee_d.gz" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.{{ cycle_YMDH }}.ieee_d{{ suffix }}" - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.ctl" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.{{ cycle_YMDH }}.ieee_d" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.{{ cycle_YMDH }}.ieee_d.gz" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.{{ cycle_YMDH }}.ieee_d{{ suffix }}" {% endfor %} {% endfor %} {% endif %} diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index cc822e9424..25d40edf27 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -5,17 +5,29 @@ name: "GDAS_RESTARTA" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restarta.tar" mandatory: {% if not current_cycle == SDATE and MODE == "cycled" %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi*nc" - {% for i_tile in range(1,7) %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/*0000.sfcanl_data.tile{{ i_tile }}.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" + {% for iauhr in IAUHRS if iauhr != 6 %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" + {% endfor %} + {% if DOHYBVAR %} + {% set anl_offset = "-3H" %} + {% else %} + {% set anl_offset = "0H" %} + {% endif %} + {% set anl_delta = anl_offset | to_timedelta %} + {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} + {% for itile in range(1,7) %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" {% endfor %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_air" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_pc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" {% if DO_JEDISNOWDA %} - {% for i_tile in range(1,7) %} - - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/*0000.sfc_data.tile{{ i_tile }}.nc" + {% for itile in range(1,7) %} + # Snow analysis is 3dvar + - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/snowinc.{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" + - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" {% endfor %} {% endif %} {% endif %} @@ -23,7 +35,7 @@ optional: {% if not current_cycle == SDATE and MODE == "cycled" %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbur" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" diff --git a/parm/archive/gdas_restartb.yaml.j2 b/parm/archive/gdas_restartb.yaml.j2 index fd986afeee..147c493c05 100644 --- a/parm/archive/gdas_restartb.yaml.j2 +++ b/parm/archive/gdas_restartb.yaml.j2 @@ -4,4 +4,22 @@ name: "GDAS_RESTARTB" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restartb.tar" mandatory: - - '{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/.' + {% if DOIAU %} + {% set rst_times = ["+3", "+6", "+9"] %} + {% else %} + {% set rst_times = ["+6"] %} + {% endif %} + {% for f_time in rst_times %} + {% set f_timedelta = (f_time + "H") | to_timedelta %} + {% set f_dt = current_cycle | add_to_datetime(f_timedelta) %} + {% set f_YMD = f_dt | to_YMD %} + {% set f_HH = f_dt | strftime("%H") %} + {% set f_prefix = f_YMD + "." + f_HH + "0000" %} + {% for itile in range(1, 7) %} + {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ f_prefix }}.{{datatype}}.tile{{ itile }}.nc" + {% endfor %} + {% endfor %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ f_prefix }}.coupler.res" + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ f_prefix }}.fv_core.res.nc" + {% endfor %} diff --git a/parm/archive/gfs_netcdfa.yaml.j2 b/parm/archive/gfs_netcdfa.yaml.j2 index 0b7d671a24..4c22c6c734 100644 --- a/parm/archive/gfs_netcdfa.yaml.j2 +++ b/parm/archive/gfs_netcdfa.yaml.j2 @@ -6,7 +6,10 @@ target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfa.tar" mandatory: - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi*.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" + {% for iauhr in IAUHRS if iauhr != 6 %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" + {% endfor %} optional: - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" diff --git a/parm/archive/gfs_pgrb2b.yaml.j2 b/parm/archive/gfs_pgrb2b.yaml.j2 index 96b0e1bd79..85e80e5699 100644 --- a/parm/archive/gfs_pgrb2b.yaml.j2 +++ b/parm/archive/gfs_pgrb2b.yaml.j2 @@ -4,19 +4,16 @@ name: "GFS_PGRB2B" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_pgrb2b.tar" mandatory: + {% if MODE == "cycled" %} - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" + {% endif %} {% if ARCH_GAUSSIAN %} - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + {% for fhr in range(0, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}" - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}.idx" - {% endfor %} - {% endif %} -optional: - {% if ARCH_GAUSSIAN %} - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}" - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}.idx" {% endfor %} diff --git a/parm/archive/gfs_restarta.yaml.j2 b/parm/archive/gfs_restarta.yaml.j2 index 349ed1fb78..5f125f2121 100644 --- a/parm/archive/gfs_restarta.yaml.j2 +++ b/parm/archive/gfs_restarta.yaml.j2 @@ -5,8 +5,15 @@ name: "GFS_RESTARTA" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_restarta.tar" mandatory: {% if MODE == "cycled" %} + {% if DOHYBVAR %} + {% set anl_offset = "-3H" %} + {% else %} + {% set anl_offset = "0H" %} + {% endif %} + {% set anl_delta = anl_offset | to_timedelta %} + {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} {% for i_tile in range(1, 7) %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/*0000.sfcanl_data.tile{{ i_tile }}.nc" + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ i_tile }}.nc" {% endfor %} {% elif MODE == "forecast-only" %} - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc" diff --git a/parm/archive/gfs_restartb.yaml.j2 b/parm/archive/gfs_restartb.yaml.j2 index 4df0e09718..f085c74412 100644 --- a/parm/archive/gfs_restartb.yaml.j2 +++ b/parm/archive/gfs_restartb.yaml.j2 @@ -4,4 +4,25 @@ name: "GFS_RESTARTB" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_restartb.tar" mandatory: - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/" + {% set tiled_types = ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + {% for itile in range(1, 7) %} + {% for datatype in tiled_types %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{datatype}}.tile{{ itile }}.nc" + {% endfor %} + {% endfor %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/coupler" + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/fv_core.res.nc" + {% for f_time in range(restart_interval_gfs, FHMAX_GFS + restart_interval_gfs, restart_inverval_gfs) %} + {% set f_timedelta = (f_time + "H") | to_timedelta %} + {% set f_dt = current_cycle | add_to_datetime(f_timedelta) %} + {% set f_YMD = f_dt | to_YMD %} + {% set f_HH = f_dt | strftime("%H") %} + {% set f_prefix = f_YMD + "." + f_HH + "0000" %} + {% for itile in range(1, 7) %} + {% for datatype in tiled_types %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ f_prefix }}.{{datatype}}.tile{{ itile }}.nc" + {% endfor %} + {% endfor %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ f_prefix }}.coupler" + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ f_prefix }}.fv_core.res.nc" + {% endfor %} diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index e4b8ead650..8bc22cc1e4 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -10,25 +10,25 @@ mandatory: - "{{ log }}" {% endif %} {% endfor %} -# - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" -# - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" - "{{ COM_CONF | relpath(ROTDIR) }}/ufs.input.nml" {% if MODE == "cycled" %} - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" -# - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" -# - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" -# - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" -# - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" {% if DO_VMINMON %} - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" -# - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" {% endif %} {% endif %} # Full cycle {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} @@ -51,7 +51,7 @@ optional: - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}" - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}" - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}" -# {% if not WRITE_DOPOST %} -# # TODO set the forecast hours explicitly. This will require emulating numpy.array_split -# - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" -# {% endif %} ## not WRITE_DOPOST + # {% if not WRITE_DOPOST %} + # # TODO set the forecast hours explicitly. This will require emulating numpy.array_split + # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" + # {% endif %} ## not WRITE_DOPOST diff --git a/parm/archive/gfsb.yaml.j2 b/parm/archive/gfsb.yaml.j2 index b601bad029..cbefcdf240 100644 --- a/parm/archive/gfsb.yaml.j2 +++ b/parm/archive/gfsb.yaml.j2 @@ -4,11 +4,12 @@ name: "GFSB" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsb.tar" mandatory: + {% if MODE == "cycled" %} - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl" - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl.idx" - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" -optional: + {% endif %} {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}" - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}.idx" diff --git a/parm/archive/gfswave.yaml.j2 b/parm/archive/gfswave.yaml.j2 index f334633e6f..69355346be 100644 --- a/parm/archive/gfswave.yaml.j2 +++ b/parm/archive/gfswave.yaml.j2 @@ -4,6 +4,7 @@ name: "GFSWAVE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfswave.tar" mandatory: - - "{{ COM_WAVE_HISTORY | relpath(ROTDIR) }}/ww3_multi*" - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*" - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}*" +optional: + - "{{ COM_WAVE_HISTORY | relpath(ROTDIR) }}/ww3_multi*" diff --git a/parm/archive/ice_6hravg.yaml.j2 b/parm/archive/ice_6hravg.yaml.j2 index 1ccc57586e..a413da63da 100644 --- a/parm/archive/ice_6hravg.yaml.j2 +++ b/parm/archive/ice_6hravg.yaml.j2 @@ -1,8 +1,10 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = to_YMDH %} {% set head = "gfs.ice.t" + cycle_HH + "z." %} name: "ICE_6HRAVG" -target: "{{ ATARDIR }}/{{ current_cycle | to_YMDH }}/ice_6hravg.tar" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_6hravg.tar" mandatory: - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}ic.nc" - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + 6, 6 %} + {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/ice_grib2.yaml.j2 b/parm/archive/ice_grib2.yaml.j2 index 1fa9a37085..56f1b0dca8 100644 --- a/parm/archive/ice_grib2.yaml.j2 +++ b/parm/archive/ice_grib2.yaml.j2 @@ -1,17 +1,19 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = to_YMDH %} {% set head = "gfs.ice.t" + cycle_HH + "z." %} name: "ICE_GRIB2" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_grib2.tar" mandatory: - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS %} + {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %} {% set fhr3 = '%03d' % fhr %} - {% if OCNRES == "500" %} - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" - {% elif OCNRES == "100" %} - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" - {% elif OCNRES == "025" %} - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" + {% if ICERES == 500 %} + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" + {% elif ICERES == 100 %} + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" + {% elif ICERES == 25 or ICERES == "025" %} + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" {% endif %} {% endfor %} diff --git a/parm/archive/ocean_6hravg.yaml.j2 b/parm/archive/ocean_6hravg.yaml.j2 index 960c91867b..c46b05cda2 100644 --- a/parm/archive/ocean_6hravg.yaml.j2 +++ b/parm/archive/ocean_6hravg.yaml.j2 @@ -1,7 +1,9 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = to_YMDH %} {% set head = "gfs.ocean.t" + cycle_HH + "z." %} name: "OCEAN_6HRAVG" -target: "{{ ATARDIR }}/{{ current_cycle | to_YMDH }}/ocean_6hravg.tar" +target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_6hravg.tar" mandatory: - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + 6, 6 %} + {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/ocean_daily.yaml.j2 b/parm/archive/ocean_daily.yaml.j2 index c56633177c..2241ce55b5 100644 --- a/parm/archive/ocean_daily.yaml.j2 +++ b/parm/archive/ocean_daily.yaml.j2 @@ -1,7 +1,9 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = to_YMDH %} {% set head = "gfs.ocean.t" + cycle_HH + "z." %} name: "OCEAN_DAILY" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_daily.tar" mandatory: - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + 24, 24 %} + {% for fhr in range(24, FHMAX_GFS + 24, 24) %} - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}daily.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/ocean_grib2.yaml.j2 b/parm/archive/ocean_grib2.yaml.j2 index 2d9e54da22..4ed6c8b115 100644 --- a/parm/archive/ocean_grib2.yaml.j2 +++ b/parm/archive/ocean_grib2.yaml.j2 @@ -1,16 +1,18 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = to_YMDH %} {% set head = "gfs.ocean.t" + cycle_HH + "z." %} name: "OCEAN_GRIB2" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_grib2.tar" mandatory: - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS %} + {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %} {% set fhr3 = '%03d' % fhr %} - {% if OCNRES == "500" %} + {% if OCNRES == 500 %} - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" - {% elif OCNRES == "100" %} + {% elif OCNRES == 100 %} - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" - {% elif OCNRES == "025" %} + {% elif OCNRES == 25 or OCNRES == "025" %} - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" {% endif %} diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index f61268fa47..f8a983220e 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -26,7 +26,7 @@ def main(): 'FHMIN_GFS', 'FHMAX_GFS', 'FHOUT_GFS', 'ARCH_GAUSSIAN', 'MODE', 'FHOUT_OCNICE', 'FHOUT_OCNICE_GFS', 'DO_BUFRSND', 'DOHYBVAR', 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC', - 'DOIAU'] + 'DOIAU', 'OCNRES', 'ICERES'] archive_dict = AttrDict() for key in keys: diff --git a/sorc/wxflow b/sorc/wxflow index c237f195ac..cdc9751680 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit c237f195ac8f80be2518747d00390fd07a7240dc +Subproject commit cdc9751680cb266bd29eaa4821eae919c6c901cc From 8e5dc8da41c681b1d84ba33cc7da0977fbec4849 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 22 Apr 2024 17:31:26 +0000 Subject: [PATCH 027/128] Small fix for snow DA archiving. --- parm/archive/gdas_restarta.yaml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 25d40edf27..62c6a00a14 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -1,5 +1,6 @@ {% set cycle_HH = current_cycle | strftime("%H") %} {% set cycle_YMDH = current_cycle | to_YMDH %} +{% set cycle_YMD = current_cycle | to_YMD %} {% set head = "gdas.t" + cycle_HH + "z." %} name: "GDAS_RESTARTA" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restarta.tar" From fa870b3880bfd0e132b3b2eb8ba61af3019313f7 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 22 Apr 2024 17:31:59 +0000 Subject: [PATCH 028/128] Fix TC genesis arcdir archiving. --- parm/archive/arcdir.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index f4a38f7c3c..045cedd90c 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -53,8 +53,8 @@ deterministic: - ["{{ COM_ATMOS_GENESIS }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}"] {% endif %} {% if path_exists(COM_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH) %} - - ["{{ COM_ATMOS_TRACK }}/trak.gfso.atcfunix.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.{{ cycle_YMDH }}"] - - ["{{ COM_ATMOS_TRACK }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}"] + - ["{{ COM_ATMOS_GENESIS }}/trak.gfso.atcfunix.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.{{ cycle_YMDH }}"] + - ["{{ COM_ATMOS_GENESIS }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}"] {% endif %} ## Only created if tracking is on and there were systems to track {% for basin in ["epac", "natl"] %} From 5733738abe614ca1e282aa1d48ba0627f3a4d14d Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 22 Apr 2024 18:01:26 +0000 Subject: [PATCH 029/128] Fix pgrb2b analysis filename --- parm/archive/gfs_pgrb2b.yaml.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/archive/gfs_pgrb2b.yaml.j2 b/parm/archive/gfs_pgrb2b.yaml.j2 index 85e80e5699..5e2fefcfdf 100644 --- a/parm/archive/gfs_pgrb2b.yaml.j2 +++ b/parm/archive/gfs_pgrb2b.yaml.j2 @@ -5,10 +5,10 @@ name: "GFS_PGRB2B" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_pgrb2b.tar" mandatory: {% if MODE == "cycled" %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl.idx" {% endif %} {% if ARCH_GAUSSIAN %} {% for fhr in range(0, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} From 02bb94e8cddc1059fc8c6b43651ddf02ada7c44e Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 22 Apr 2024 18:01:56 +0000 Subject: [PATCH 030/128] Fix cyclone tracker filenames. --- parm/archive/gfsa.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 8bc22cc1e4..f93c2e9379 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -45,8 +45,8 @@ optional: - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" {% endif %} # cycled - - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cycle.trackatcfunix" - - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avnop.t{{ cycle_HH }}z.cycle.trackatcfunix" + - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cyclone.trackatcfunix" + - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avnop.t{{ cycle_HH }}z.cyclone.trackatcfunix" - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/trak.gfso.atcfunix.{{ cycle_YMDH }}" - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}" - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}" From a81ee422ef016a0c209923194ebe219695af94bf Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 23 Apr 2024 14:12:42 +0000 Subject: [PATCH 031/128] Move forecast hours and restart intervals to config.base. --- parm/config/gfs/config.base | 27 +++++++++++++++++++++++++++ parm/config/gfs/config.efcs | 9 ++++----- parm/config/gfs/config.fcst | 7 ++----- parm/config/gfs/config.wave | 10 ++-------- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 4995560684..e663e78c7a 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -278,6 +278,12 @@ export FHOUT_GFS=3 # Must be 6 for S2S until #1629 is addressed; 3 for ops export FHMAX_HF_GFS=0 export FHOUT_HF_GFS=1 export FHOUT_OCNICE_GFS=6 +export FHMIN_WAV=0 +export FHOUT_WAV=3 +export FHMAX_HF_WAV=120 +export FHOUT_HF_WAV=1 +export FHMAX_WAV=${FHMAX:-9} +export FHMAX_WAV_GFS=${FHMAX_GFS} if (( gfs_cyc != 0 )); then export STEP_GFS=$(( 24 / gfs_cyc )) else @@ -375,6 +381,24 @@ fi if [[ "${DOIAU_ENKF}" = "NO" ]]; then export IAUFHRS_ENKF="6"; fi +# Determine restart intervals +# For IAU, write restarts at beginning of window also +if [[ "${DOIAU_ENKF:-}" == "YES" ]]; then + export restart_interval_enkfgdas="3" +else + export restart_interval_enkfgdas="6" +fi + +if [[ "${DOIAU}" == "YES" ]]; then + export restart_interval_gdas="3" +else + export restart_interval_gdas="6" +fi + +# Write gfs restart files to rerun fcst from any break point +export restart_interval_gfs=${restart_interval_gfs:-12} +export restart_interval_enkfgfs=${restart_interval_enkfgfs:-12} + export GSI_SOILANAL=@GSI_SOILANAL@ # turned on nsst in anal and/or fcst steps, and turn off rtgsst @@ -431,4 +455,7 @@ if [[ ${DO_JEDIATMVAR} = "YES" ]]; then export DO_VMINMON="NO" # GSI minimization monitoring fi +# Number of regional collectives to create soundings for +export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} + echo "END: config.base" diff --git a/parm/config/gfs/config.efcs b/parm/config/gfs/config.efcs index 402ba64087..1837cf0619 100644 --- a/parm/config/gfs/config.efcs +++ b/parm/config/gfs/config.efcs @@ -85,11 +85,10 @@ if [[ ${RECENTER_ENKF:-"YES"} == "YES" ]]; then export PREFIX_ATMINC="r" fi -# For IAU, write restarts at beginning of window also -if [[ "${DOIAU_ENKF:-}" == "YES" ]]; then - export restart_interval="3" -else - export restart_interval="6" +# Set restart interval to enable restarting forecasts after failures +export restart_interval=${restart_interval_enkfgdas:-6} +if [[ ${RUN} == "enkfgfs" ]]; then + export restart_interval=${restart_interval_enkfgfs:-12} fi echo "END: config.efcs" diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index 14621307f5..8e121634a7 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -275,11 +275,8 @@ if [[ "${CDUMP}" =~ "gdas" ]] ; then # GDAS cycle specific parameters # Variables used in DA cycling export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table_da" - if [[ "${DOIAU}" == "YES" ]]; then - export restart_interval="3" - else - export restart_interval="6" - fi + # Write gfs restart files to rerun fcst from any break point + export restart_interval=${restart_interval_gdas:-6} # Turn on dry mass adjustment in GDAS export adjust_dry_mass=".true." diff --git a/parm/config/gfs/config.wave b/parm/config/gfs/config.wave index 6fbce69996..9b781130db 100644 --- a/parm/config/gfs/config.wave +++ b/parm/config/gfs/config.wave @@ -94,16 +94,10 @@ export WAVEWND_DID= export WAVEWND_FID= # The start time reflects the number of hindcast hours prior to the cycle initial time -if [[ "${CDUMP}" = "gdas" ]]; then - export FHMAX_WAV=${FHMAX:-9} -else - export FHMAX_WAV=${FHMAX_GFS} +if [[ "${CDUMP}" = "gfs" ]]; then + export FHMAX_WAV=${FHMAX_WAV_GFS} fi export WAVHINDH=0 -export FHMIN_WAV=0 -export FHOUT_WAV=3 -export FHMAX_HF_WAV=120 -export FHOUT_HF_WAV=1 export FHMAX_WAV_IBP=180 if (( FHMAX_WAV < FHMAX_WAV_IBP )); then export FHMAX_WAV_IBP=${FHMAX_GFS} ; fi From 066ffe290265d8b296c16a18b4941ebf28b7efce Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 23 Apr 2024 14:13:38 +0000 Subject: [PATCH 032/128] Archive all sfcf* files. --- parm/archive/enkf_grp.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index e24132331a..63646cfb6b 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -19,7 +19,7 @@ mandatory: - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat" {% endif %} {% for fhr in range(fhmin, fhmax+fhout, fhout) %} - - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf006.nc" + - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" {% endfor %} From 07a65378f750a50c7fe28748478f71e98536fa5e Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 23 Apr 2024 14:15:27 +0000 Subject: [PATCH 033/128] Refactor restart/forecast hour-based archiving --- parm/archive/enkf_restartb_grp.yaml.j2 | 31 +++++++++++++------------- parm/archive/gdas_restartb.yaml.j2 | 23 ++++++++----------- parm/archive/gfs_restartb.yaml.j2 | 18 +++++++-------- 3 files changed, 34 insertions(+), 38 deletions(-) diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 index debbbe63b0..2c9d4ca699 100644 --- a/parm/archive/enkf_restartb_grp.yaml.j2 +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -1,30 +1,31 @@ {% set cycle_HH = current_cycle | strftime("%H") %} {% set cycle_YMDH = current_cycle | to_YMDH %} {% set head = RUN + ".t" + cycle_HH + "z." %} +{% if RUN == "enkfgdas" %} +{% set fhmax = FHMAX_ENKF %} +{% set restart_interval = restart_interval_enkfgdas %} +{% elif RUN == "enkfgfs" %} +{% set fhmax = FHMAX_ENKF_GFS %} +{% set restart_interval = restart_interval_enkfgfs %} +{% endif %} name: "ENKF_RESTARTB_GRP" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restartb_grp{{ ENSGRP }}.tar" mandatory: {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% if DOIAU_ENKF %} - {% set rst_times = ["+3", "+6", "+9"] %} - {% else %} - {% set rst_times = ["+6"] %} - {% endif %} - {% for f_time in rst_times %} - {% set f_timedelta = (f_time + "H") | to_timedelta %} - {% set f_dt = current_cycle | add_to_datetime(f_timedelta) %} - {% set f_YMD = f_dt | to_YMD %} - {% set f_HH = f_dt | strftime("%H") %} - {% set f_prefix = f_YMD + "." + f_HH + "0000" %} + {% rst_times = range(restart_interval, fhmax + restart_interval, restart_interval) %} + {% for r_time in rst_times %} + {% set r_timedelta = (str(r_time) + "H") | to_timedelta %} + {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} + {% set r_prefix = r_dt | to_YMD + "." + r_dt | strftime("%H") + "0000" %} {% for itile in range(1, 7) %} {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ f_prefix }}.{{datatype}}.tile{{ itile }}.nc" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" {% endfor %} {% endfor %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ f_prefix }}.atm_stoch.res.nc" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ f_prefix }}.coupler.res" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ f_prefix }}.fv_core.res.nc" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.atm_stoch.res.nc" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" {% endfor %} {% endfor %} diff --git a/parm/archive/gdas_restartb.yaml.j2 b/parm/archive/gdas_restartb.yaml.j2 index 147c493c05..aa38c73830 100644 --- a/parm/archive/gdas_restartb.yaml.j2 +++ b/parm/archive/gdas_restartb.yaml.j2 @@ -4,22 +4,17 @@ name: "GDAS_RESTARTB" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restartb.tar" mandatory: - {% if DOIAU %} - {% set rst_times = ["+3", "+6", "+9"] %} - {% else %} - {% set rst_times = ["+6"] %} - {% endif %} - {% for f_time in rst_times %} - {% set f_timedelta = (f_time + "H") | to_timedelta %} - {% set f_dt = current_cycle | add_to_datetime(f_timedelta) %} - {% set f_YMD = f_dt | to_YMD %} - {% set f_HH = f_dt | strftime("%H") %} - {% set f_prefix = f_YMD + "." + f_HH + "0000" %} + {% for r_time in range(restart_interval_gdas, FHMAX + restart_interval_gdas, restart_interval_gdas %} + {% set r_timedelta = (r_time + "H") | to_timedelta %} + {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} + {% set r_YMD = r_dt | to_YMD %} + {% set r_HH = r_dt | strftime("%H") %} + {% set r_prefix = r_YMD + "." + r_HH + "0000" %} {% for itile in range(1, 7) %} {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ f_prefix }}.{{datatype}}.tile{{ itile }}.nc" + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" {% endfor %} {% endfor %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ f_prefix }}.coupler.res" - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ f_prefix }}.fv_core.res.nc" + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" {% endfor %} diff --git a/parm/archive/gfs_restartb.yaml.j2 b/parm/archive/gfs_restartb.yaml.j2 index f085c74412..4a499cd83d 100644 --- a/parm/archive/gfs_restartb.yaml.j2 +++ b/parm/archive/gfs_restartb.yaml.j2 @@ -12,17 +12,17 @@ mandatory: {% endfor %} - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/coupler" - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/fv_core.res.nc" - {% for f_time in range(restart_interval_gfs, FHMAX_GFS + restart_interval_gfs, restart_inverval_gfs) %} - {% set f_timedelta = (f_time + "H") | to_timedelta %} - {% set f_dt = current_cycle | add_to_datetime(f_timedelta) %} - {% set f_YMD = f_dt | to_YMD %} - {% set f_HH = f_dt | strftime("%H") %} - {% set f_prefix = f_YMD + "." + f_HH + "0000" %} + {% for r_time in range(restart_interval_gfs, FHMAX_GFS + restart_interval_gfs, restart_inverval_gfs) %} + {% set r_timedelta = (r_time + "H") | to_timedelta %} + {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} + {% set r_YMD = r_dt | to_YMD %} + {% set r_HH = r_dt | strftime("%H") %} + {% set r_prefix = r_YMD + "." + r_HH + "0000" %} {% for itile in range(1, 7) %} {% for datatype in tiled_types %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ f_prefix }}.{{datatype}}.tile{{ itile }}.nc" + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" {% endfor %} {% endfor %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ f_prefix }}.coupler" - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ f_prefix }}.fv_core.res.nc" + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.coupler" + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" {% endfor %} From 1c24f6bc7bc66c7c711495ec2cb42e2f08509e47 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 23 Apr 2024 14:18:19 +0000 Subject: [PATCH 034/128] Add TODOs for GDAS component archiving --- parm/archive/chem.yaml.j2 | 1 + parm/archive/gdas.yaml.j2 | 1 + parm/archive/gdasice.yaml.j2 | 1 + parm/archive/gdasice_restart.yaml.j2 | 3 ++- parm/archive/gdasocean.yaml.j2 | 1 + parm/archive/gdasocean_analysis.yaml.j2 | 1 + parm/archive/gdasocean_restart.yaml.j2 | 1 + parm/archive/gdaswave.yaml.j2 | 1 + parm/archive/gdaswave_restart.yaml.j2 | 1 + 9 files changed, 10 insertions(+), 1 deletion(-) diff --git a/parm/archive/chem.yaml.j2 b/parm/archive/chem.yaml.j2 index 2aebcb816d..d960819896 100644 --- a/parm/archive/chem.yaml.j2 +++ b/parm/archive/chem.yaml.j2 @@ -4,4 +4,5 @@ name: "CHEM" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/chem.tar" mandatory: + # TODO explicitize this set - "{{ COM_CHEM_HISTORY | relpath(ROTDIR) }}/{{ head }}*" diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 0fbc0a8641..5b710a3e6b 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -62,6 +62,7 @@ mandatory: {% endif %} {% endif %} # Full cycle - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" + # TODO explicitly name the atmos_prod log files to archive - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" {% for fhr in range(0, FHMAX + FHOUT, FHOUT) %} - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" diff --git a/parm/archive/gdasice.yaml.j2 b/parm/archive/gdasice.yaml.j2 index 9ac864a53f..df10f1eb1f 100644 --- a/parm/archive/gdasice.yaml.j2 +++ b/parm/archive/gdasice.yaml.j2 @@ -4,5 +4,6 @@ name: "GDASICE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar" mandatory: + # TODO explicitly name the files in history to archive - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}*' - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/ice_in' diff --git a/parm/archive/gdasice_restart.yaml.j2 b/parm/archive/gdasice_restart.yaml.j2 index f21af23d77..65e2b77ff6 100644 --- a/parm/archive/gdasice_restart.yaml.j2 +++ b/parm/archive/gdasice_restart.yaml.j2 @@ -4,4 +4,5 @@ name: "GDASICE_RESTART" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice_restart.tar" mandatory: - - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/*' + # TODO explicitly name the restart files to archive + - '{{ COM_ICE_RESTART | relpath(ROTDIR) }}/*' diff --git a/parm/archive/gdasocean.yaml.j2 b/parm/archive/gdasocean.yaml.j2 index 6062c4e0e5..794a491ece 100644 --- a/parm/archive/gdasocean.yaml.j2 +++ b/parm/archive/gdasocean.yaml.j2 @@ -4,5 +4,6 @@ name: "GDASOCEAN" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar" mandatory: + # TODO Explicitly name the history and input files to archive - '{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}*' - '{{ COM_OCEAN_INPUT | relpath(ROTDIR) }}/.' diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index 262607eb8d..bf3fac59c6 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -4,6 +4,7 @@ name: "GDASOCEAN_ANALYSIS" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar" mandatory: + # TODO explicitly name the analysis files to archive - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}*' - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t??z.ocngrid.nc' - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags' diff --git a/parm/archive/gdasocean_restart.yaml.j2 b/parm/archive/gdasocean_restart.yaml.j2 index 9b8a4d3729..d5ab2dcde8 100644 --- a/parm/archive/gdasocean_restart.yaml.j2 +++ b/parm/archive/gdasocean_restart.yaml.j2 @@ -4,5 +4,6 @@ name: "GDASOCEAN_RESTART" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_restart.tar" mandatory: + # TODO explicitly name the restart files to archive - '{{ COM_OCEAN_RESTART | relpath(ROTDIR) }}/*' - '{{ COM_MED_RESTART | relpath(ROTDIR) }}/*' diff --git a/parm/archive/gdaswave.yaml.j2 b/parm/archive/gdaswave.yaml.j2 index 5cdc94d4e6..6b26a3bbf1 100644 --- a/parm/archive/gdaswave.yaml.j2 +++ b/parm/archive/gdaswave.yaml.j2 @@ -4,5 +4,6 @@ name: "GDASWAVE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave.tar" mandatory: + # TODO explicitly name the wave grid/station files to archive - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*" - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}*" diff --git a/parm/archive/gdaswave_restart.yaml.j2 b/parm/archive/gdaswave_restart.yaml.j2 index 9a8e43f2af..b26f17b51a 100644 --- a/parm/archive/gdaswave_restart.yaml.j2 +++ b/parm/archive/gdaswave_restart.yaml.j2 @@ -3,4 +3,5 @@ name: "GDASWAVE_RESTART" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave_restart.tar" mandatory: + # TODO explicitly name the wave restart files to archive - "{{ COM_WAVE_RESTART | relpath(ROTDIR) }}/*" From c8c2670df486d5d046374fa84f99bf6e3f4eac0b Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 23 Apr 2024 14:19:07 +0000 Subject: [PATCH 035/128] Explicitly archive gfs_collective files --- parm/archive/gfs_downstream.yaml.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parm/archive/gfs_downstream.yaml.j2 b/parm/archive/gfs_downstream.yaml.j2 index ac35030168..2a358aa7dc 100644 --- a/parm/archive/gfs_downstream.yaml.j2 +++ b/parm/archive/gfs_downstream.yaml.j2 @@ -6,6 +6,8 @@ target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_downstream.tar" mandatory: - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.sfc" - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.snd" - - "{{ COM_ATMOS_WMO | relpath(ROTDIR) }}/gfs_collective*.postsnd_{{ cycle_HH }}" + {% for i in range(1, NUM_SND_COLLECTIVES) %} + - "{{ COM_ATMOS_WMO | relpath(ROTDIR) }}/gfs_collective{{ i }}.postsnd_{{ cycle_HH }}" + {% endfor %} - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/bufr.t{{ cycle_HH }}z" - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/gfs.t{{ cycle_HH }}z.bufrsnd.tar.gz" From a3318e243e268bbae17c9ef5856d971802da70cf Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 23 Apr 2024 14:20:05 +0000 Subject: [PATCH 036/128] More explicitly archive gridded GFS Wave products --- parm/archive/gfswave.yaml.j2 | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/parm/archive/gfswave.yaml.j2 b/parm/archive/gfswave.yaml.j2 index 69355346be..c0c07664e3 100644 --- a/parm/archive/gfswave.yaml.j2 +++ b/parm/archive/gfswave.yaml.j2 @@ -4,7 +4,13 @@ name: "GFSWAVE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfswave.tar" mandatory: - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}*" -optional: - - "{{ COM_WAVE_HISTORY | relpath(ROTDIR) }}/ww3_multi*" + {% for fh in range(0, FHMAX_HF_WAV + FHOUT_HF_WAV, FHOUT_HF_WAV) %} + # NOTE This is as explicit as possible without major logic to parse wavepostGRD. + # Matches files of the form "gfswave.tCCz...fHHH.grib2". + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" + {% endfor %} + {% for fh in range(FHMAX_HF_WAV + FHOUT_WAV, FHMAX_WAV_GFS + FHOUT_WAV, FHOUT_WAV) %} + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" + {% endfor %} From 6d932baa192214c076f01f34ab0824664bf7d9ad Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 23 Apr 2024 14:46:11 +0000 Subject: [PATCH 037/128] Add optional wave station products to archives --- parm/archive/gfswave.yaml.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/parm/archive/gfswave.yaml.j2 b/parm/archive/gfswave.yaml.j2 index c0c07664e3..632da1a383 100644 --- a/parm/archive/gfswave.yaml.j2 +++ b/parm/archive/gfswave.yaml.j2 @@ -14,3 +14,7 @@ mandatory: - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" {% endfor %} +optional: + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}bull_tar" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}cbull_tar" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}spec_tar.gz" From 5e3369398bf0eb6a318a5eb34241d651e51baa5f Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 23 Apr 2024 16:55:37 +0000 Subject: [PATCH 038/128] Fix restart intervals in archiving templates. --- parm/archive/enkf_restartb_grp.yaml.j2 | 5 ++--- parm/archive/gdas_restartb.yaml.j2 | 4 ++-- parm/archive/gfs_restartb.yaml.j2 | 2 +- scripts/exgdas_enkf_earc.py | 4 ++-- scripts/exgfs_atmos_postsnd.sh | 2 +- scripts/exglobal_archive.py | 4 +++- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 index 2c9d4ca699..d9594d0009 100644 --- a/parm/archive/enkf_restartb_grp.yaml.j2 +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -14,9 +14,8 @@ mandatory: {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% rst_times = range(restart_interval, fhmax + restart_interval, restart_interval) %} - {% for r_time in rst_times %} - {% set r_timedelta = (str(r_time) + "H") | to_timedelta %} + {% for r_time in range(restart_interval, fhmax + restart_interval, restart_interval) %} + {% set r_timedelta = (r_time | string + "H") | to_timedelta %} {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} {% set r_prefix = r_dt | to_YMD + "." + r_dt | strftime("%H") + "0000" %} {% for itile in range(1, 7) %} diff --git a/parm/archive/gdas_restartb.yaml.j2 b/parm/archive/gdas_restartb.yaml.j2 index aa38c73830..a6909ed78d 100644 --- a/parm/archive/gdas_restartb.yaml.j2 +++ b/parm/archive/gdas_restartb.yaml.j2 @@ -4,8 +4,8 @@ name: "GDAS_RESTARTB" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restartb.tar" mandatory: - {% for r_time in range(restart_interval_gdas, FHMAX + restart_interval_gdas, restart_interval_gdas %} - {% set r_timedelta = (r_time + "H") | to_timedelta %} + {% for r_time in range(restart_interval_gdas, FHMAX + restart_interval_gdas, restart_interval_gdas) %} + {% set r_timedelta = (r_time | string + "H") | to_timedelta %} {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} {% set r_YMD = r_dt | to_YMD %} {% set r_HH = r_dt | strftime("%H") %} diff --git a/parm/archive/gfs_restartb.yaml.j2 b/parm/archive/gfs_restartb.yaml.j2 index 4a499cd83d..e39c86d7f1 100644 --- a/parm/archive/gfs_restartb.yaml.j2 +++ b/parm/archive/gfs_restartb.yaml.j2 @@ -13,7 +13,7 @@ mandatory: - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/coupler" - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/fv_core.res.nc" {% for r_time in range(restart_interval_gfs, FHMAX_GFS + restart_interval_gfs, restart_inverval_gfs) %} - {% set r_timedelta = (r_time + "H") | to_timedelta %} + {% set r_timedelta = (r_time | string + "H") | to_timedelta %} {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} {% set r_YMD = r_dt | to_YMD %} {% set r_HH = r_dt | strftime("%H") %} diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index 9d8ca80467..cf51fb521b 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -23,8 +23,8 @@ def main(): 'ARCDIR', 'SDATE', 'MODE', 'ENSGRP', 'NMEM_EARCGRP', 'NMEM_ENS', 'DO_CALC_INCREMENT_ENKF_GFS', 'DO_JEDIATMENS', 'lobsdiag_forenkf', 'FHMIN_ENKF', 'FHMAX_ENKF_GFS', - 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF', - 'ENKF_SPREAD'] + 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF', 'ENKF_SPREAD', + 'restart_interval_enkfgdas', 'restart_interval_enkfgfs'] archive_dict = AttrDict() for key in keys: diff --git a/scripts/exgfs_atmos_postsnd.sh b/scripts/exgfs_atmos_postsnd.sh index 7aa97f3644..a371e0be97 100755 --- a/scripts/exgfs_atmos_postsnd.sh +++ b/scripts/exgfs_atmos_postsnd.sh @@ -113,7 +113,7 @@ fi # add appropriate WMO Headers. ######################################## rm -rf poe_col -for (( m = 1; m <10 ; m++ )); do +for (( m = 1; m <= ${NUM_SND_COLLECTIVES} ; m++ )); do echo "sh ${USHgfs}/gfs_sndp.sh ${m} " >> poe_col done diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index f8a983220e..c21764de2a 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -26,7 +26,9 @@ def main(): 'FHMIN_GFS', 'FHMAX_GFS', 'FHOUT_GFS', 'ARCH_GAUSSIAN', 'MODE', 'FHOUT_OCNICE', 'FHOUT_OCNICE_GFS', 'DO_BUFRSND', 'DOHYBVAR', 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC', - 'DOIAU', 'OCNRES', 'ICERES'] + 'DOIAU', 'OCNRES', 'ICERES', 'NUM_SND_COLLECTIVES', 'FHOUT_WAV', + 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', + 'restart_interval_gdas', 'restart_interval_gfs'] archive_dict = AttrDict() for key in keys: From 24582353885f87b92a9766bc8a0e1a3ecbae074a Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 24 Apr 2024 15:07:31 +0000 Subject: [PATCH 039/128] Address shellcheck suggestion --- scripts/exgfs_atmos_postsnd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exgfs_atmos_postsnd.sh b/scripts/exgfs_atmos_postsnd.sh index a371e0be97..23c41157fe 100755 --- a/scripts/exgfs_atmos_postsnd.sh +++ b/scripts/exgfs_atmos_postsnd.sh @@ -113,7 +113,7 @@ fi # add appropriate WMO Headers. ######################################## rm -rf poe_col -for (( m = 1; m <= ${NUM_SND_COLLECTIVES} ; m++ )); do +for (( m = 1; m <= NUM_SND_COLLECTIVES ; m++ )); do echo "sh ${USHgfs}/gfs_sndp.sh ${m} " >> poe_col done From 200b0ef95756077a3a60958ee6853e7e557f0493 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 24 Apr 2024 15:08:36 +0000 Subject: [PATCH 040/128] Fix enkf_grp optional file declarations. --- parm/archive/enkf_grp.yaml.j2 | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index 63646cfb6b..4bfc45fe00 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -15,7 +15,7 @@ mandatory: {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% if not lobsdiag_forenkf %} + {% if current_cycle != SDATE and not lobsdiag_forenkf %} - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat" {% endif %} {% for fhr in range(fhmin, fhmax+fhout, fhout) %} @@ -40,21 +40,13 @@ mandatory: {% endif %} {% endfor %} # iauhr in iaufhrs {% endfor %} # first_group_mem to last_group_mem -optional: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} - {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" - {% endif %} - {% endfor %} optional: {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% if not lobsdiag_forenkf %} + {% if current_cycle != SDATE and not lobsdiag_forenkf %} - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" {% endif %} {% endfor %} From 7d06b115ad1a33709cc1ce960b7523688385d633 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 24 Apr 2024 15:09:26 +0000 Subject: [PATCH 041/128] Move verfrad warning files to optional list --- parm/archive/gdas.yaml.j2 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 5b710a3e6b..66834ce1c3 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -36,14 +36,10 @@ mandatory: - "logs/{{ cycle_YMDH }}/{{ RUN }}verfozn.log" {% endif %} {% if DO_VERFRAD %} - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/low_count.{{ cycle_YMDH }}" - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_angle.tar.gz" - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcoef.tar.gz" - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcor.tar.gz" - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_time.tar.gz" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/warning.{{ cycle_YMDH }}" - "logs/{{ cycle_YMDH }}/{{ RUN }}verfrad.log" {% endif %} {% if DO_VMINMON %} @@ -78,6 +74,12 @@ mandatory: optional: {% if not current_cycle == SDATE and MODE == "cycled" %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" + {% if DO_VERFRAD %} + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/low_count.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/warning.{{ cycle_YMDH }}" + {% endif %} {% if DO_VERFOZN %} {% set oznmon_types = [ "gome_metop-b", "omi_aura", "ompslp_npp", "ompsnp_n20", From 00c70e1782fd0ac921f2f428b0c5d208c57e537d Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 24 Apr 2024 15:18:31 +0000 Subject: [PATCH 042/128] Fix restart_interval loop control. --- parm/archive/enkf_restartb_grp.yaml.j2 | 2 +- parm/archive/gdas_restartb.yaml.j2 | 2 +- parm/archive/gfs_restartb.yaml.j2 | 2 +- parm/config/gfs/config.base | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 index d9594d0009..b19b670ea9 100644 --- a/parm/archive/enkf_restartb_grp.yaml.j2 +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -14,7 +14,7 @@ mandatory: {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% for r_time in range(restart_interval, fhmax + restart_interval, restart_interval) %} + {% for r_time in range(restart_interval, fhmax + 1, restart_interval) %} {% set r_timedelta = (r_time | string + "H") | to_timedelta %} {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} {% set r_prefix = r_dt | to_YMD + "." + r_dt | strftime("%H") + "0000" %} diff --git a/parm/archive/gdas_restartb.yaml.j2 b/parm/archive/gdas_restartb.yaml.j2 index a6909ed78d..335376fde3 100644 --- a/parm/archive/gdas_restartb.yaml.j2 +++ b/parm/archive/gdas_restartb.yaml.j2 @@ -4,7 +4,7 @@ name: "GDAS_RESTARTB" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restartb.tar" mandatory: - {% for r_time in range(restart_interval_gdas, FHMAX + restart_interval_gdas, restart_interval_gdas) %} + {% for r_time in range(restart_interval_gdas, FHMAX + 1, restart_interval_gdas) %} {% set r_timedelta = (r_time | string + "H") | to_timedelta %} {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} {% set r_YMD = r_dt | to_YMD %} diff --git a/parm/archive/gfs_restartb.yaml.j2 b/parm/archive/gfs_restartb.yaml.j2 index e39c86d7f1..b8db246037 100644 --- a/parm/archive/gfs_restartb.yaml.j2 +++ b/parm/archive/gfs_restartb.yaml.j2 @@ -12,7 +12,7 @@ mandatory: {% endfor %} - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/coupler" - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/fv_core.res.nc" - {% for r_time in range(restart_interval_gfs, FHMAX_GFS + restart_interval_gfs, restart_inverval_gfs) %} + {% for r_time in range(restart_interval_gfs, FHMAX_GFS + 1, restart_inverval_gfs) %} {% set r_timedelta = (r_time | string + "H") | to_timedelta %} {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} {% set r_YMD = r_dt | to_YMD %} diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index e663e78c7a..89e18f8c9a 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -389,7 +389,7 @@ else export restart_interval_enkfgdas="6" fi -if [[ "${DOIAU}" == "YES" ]]; then +if [[ "${DOIAU}" == "YES" && "${DOHYBVAR}" == "YES" ]]; then export restart_interval_gdas="3" else export restart_interval_gdas="6" From a23ee5e0ee66fa48bd6372f5e61e07725d7a15dc Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 24 Apr 2024 15:19:09 +0000 Subject: [PATCH 043/128] Copy relevant changes to gefs config files. --- parm/config/gefs/config.base | 6 ++++++ parm/config/gefs/config.efcs | 2 +- parm/config/gefs/config.wave | 5 ----- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 07664f15ff..325c44a0c8 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -215,6 +215,11 @@ export FHOUT_GFS=6 export FHMAX_HF_GFS=0 export FHOUT_HF_GFS=1 export FHOUT_OCNICE_GFS=6 +export FHMIN_WAV=0 +export FHOUT_WAV=3 +export FHMAX_HF_WAV=120 +export FHOUT_HF_WAV=1 +export FHMAX_WAV_GFS=${FHMAX_GFS} if (( gfs_cyc != 0 )); then export STEP_GFS=$(( 24 / gfs_cyc )) else @@ -228,6 +233,7 @@ export FHOUT_ENKF=${FHOUT_GFS} # GFS restart interval in hours export restart_interval_gfs=12 +export restart_interval_enkfgfs=12 # NOTE: Do not set this to zero. Instead set it to $FHMAX_GFS # TODO: Remove this variable from config.base and reference from config.fcst # TODO: rework logic in config.wave and push it to parsing_nameslist_WW3.sh where it is actually used diff --git a/parm/config/gefs/config.efcs b/parm/config/gefs/config.efcs index ad90fa864c..915726b974 100644 --- a/parm/config/gefs/config.efcs +++ b/parm/config/gefs/config.efcs @@ -78,6 +78,6 @@ if [[ "${USE_OCN_PERTURB_FILES:-false}" == "true" ]]; then else export ODA_INCUPD="False" fi -export restart_interval="${restart_interval_gfs}" +export restart_interval="${restart_interval_enkfgfs:-12}" echo "END: config.efcs" diff --git a/parm/config/gefs/config.wave b/parm/config/gefs/config.wave index 7298b12aec..b5fba041df 100644 --- a/parm/config/gefs/config.wave +++ b/parm/config/gefs/config.wave @@ -82,12 +82,7 @@ export WAVEWND_DID= export WAVEWND_FID= # The start time reflects the number of hindcast hours prior to the cycle initial time -export FHMAX_WAV=${FHMAX_GFS} export WAVHINDH=0 -export FHMIN_WAV=0 -export FHOUT_WAV=3 -export FHMAX_HF_WAV=120 -export FHOUT_HF_WAV=1 export FHMAX_WAV_IBP=180 if (( FHMAX_WAV < FHMAX_WAV_IBP )); then export FHMAX_WAV_IBP=${FHMAX_GFS} ; fi From 94011b3ce9efecdd3f339136feb87afa47eb25ac Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 24 Apr 2024 15:30:43 +0000 Subject: [PATCH 044/128] Remove amsua files from archive list --- parm/archive/gdas_restarta.yaml.j2 | 3 --- 1 file changed, 3 deletions(-) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 62c6a00a14..c7212f8186 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -41,9 +41,6 @@ optional: - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}amsua_n19.satbias.nc4" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}amsua_n19.satbias_cov.nc4" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}amsua_n19.tlapse.txt" {% if DO_JEDISNOWDA %} - "{{ COM_CONF | relpath(ROTDIR) }}/{{ head }}letkfoi.yaml" {% endif %} From 78b063f55816f3047fd1f6bde4c7e7753370689c Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 25 Apr 2024 12:43:01 +0000 Subject: [PATCH 045/128] Fix wave FHMAX for GEFS. --- parm/config/gefs/config.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 325c44a0c8..ee7626c054 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -219,7 +219,7 @@ export FHMIN_WAV=0 export FHOUT_WAV=3 export FHMAX_HF_WAV=120 export FHOUT_HF_WAV=1 -export FHMAX_WAV_GFS=${FHMAX_GFS} +export FHMAX_WAV=${FHMAX_GFS} if (( gfs_cyc != 0 )); then export STEP_GFS=$(( 24 / gfs_cyc )) else From a30a05f56ecdeab8735fd7601f52da31c7f591c5 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 25 Apr 2024 13:04:17 +0000 Subject: [PATCH 046/128] Added untested enkfgfs warning to archive.py --- ush/python/pygfs/task/archive.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 53211abae7..3e092902d1 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -204,6 +204,10 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str elif arch_dict.RUN == "enkfgdas" or arch_dict.RUN == "enkfgfs": + if arch_dict.RUN == "enkfgfs": + # TODO test enkfgfs archiving and remove this warning + print("WARNING enkfgfs archiving has not been tested!") + if arch_dict.ENSGRP == 0: datasets = ["enkf"] From 1a3ac04fda867702254f78adca3741efe8d75844 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 25 Apr 2024 15:58:37 +0000 Subject: [PATCH 047/128] Remove debug print statement --- ush/python/pygfs/task/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 3e092902d1..3780f9d191 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -169,7 +169,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if arch_dict.ARCH_GAUSSIAN: datasets.extend(["gfs_flux", "gfs_netcdfb", "gfs_pgrb2b"]) - print(datasets) + if arch_dict.MODE == "cycled": datasets.append("gfs_netcdfa") From c8ede1b07273c84e0c66e6063d3aabff62bf7231 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 25 Apr 2024 15:58:59 +0000 Subject: [PATCH 048/128] Remove 'untested' warning --- ush/python/pygfs/task/archive.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 3780f9d191..2b12f15e9c 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -204,10 +204,6 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str elif arch_dict.RUN == "enkfgdas" or arch_dict.RUN == "enkfgfs": - if arch_dict.RUN == "enkfgfs": - # TODO test enkfgfs archiving and remove this warning - print("WARNING enkfgfs archiving has not been tested!") - if arch_dict.ENSGRP == 0: datasets = ["enkf"] From fc6f08575ca6fa143f35f1651eed11415de27313 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 25 Apr 2024 16:00:09 +0000 Subject: [PATCH 049/128] Corrected genesis COM path --- parm/archive/gfsa.yaml.j2 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index f93c2e9379..dfb416da68 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -42,13 +42,15 @@ optional: - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" + {% if DO_JEDIATMVAR %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" + {% endif %} {% endif %} # cycled - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cyclone.trackatcfunix" - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avnop.t{{ cycle_HH }}z.cyclone.trackatcfunix" - - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/trak.gfso.atcfunix.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}" - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}" - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}" # {% if not WRITE_DOPOST %} From fa0adb78ce7069c44b213b07ba3b60f0e4f5660f Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 25 Apr 2024 16:00:45 +0000 Subject: [PATCH 050/128] Remove unsaved atm_stoch files --- parm/archive/enkf_restartb_grp.yaml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 index b19b670ea9..529df7bc8b 100644 --- a/parm/archive/enkf_restartb_grp.yaml.j2 +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -23,7 +23,6 @@ mandatory: - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" {% endfor %} {% endfor %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.atm_stoch.res.nc" - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" {% endfor %} From 84a1dd8d1e1d5f8eca70a9cb4edb63215ddc0573 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 25 Apr 2024 16:01:32 +0000 Subject: [PATCH 051/128] Move restart_interval* to more logical locations --- parm/config/gfs/config.base | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 89e18f8c9a..993c5770ec 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -389,16 +389,14 @@ else export restart_interval_enkfgdas="6" fi +export restart_interval_enkfgfs=${restart_interval_enkfgdas} + if [[ "${DOIAU}" == "YES" && "${DOHYBVAR}" == "YES" ]]; then export restart_interval_gdas="3" else export restart_interval_gdas="6" fi -# Write gfs restart files to rerun fcst from any break point -export restart_interval_gfs=${restart_interval_gfs:-12} -export restart_interval_enkfgfs=${restart_interval_enkfgfs:-12} - export GSI_SOILANAL=@GSI_SOILANAL@ # turned on nsst in anal and/or fcst steps, and turn off rtgsst From d76c392ff1d8787bb66a802ad53fcf83e2886642 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 25 Apr 2024 16:44:36 +0000 Subject: [PATCH 052/128] Remove unused gfs_restartb template --- parm/archive/gfs_restartb.yaml.j2 | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 parm/archive/gfs_restartb.yaml.j2 diff --git a/parm/archive/gfs_restartb.yaml.j2 b/parm/archive/gfs_restartb.yaml.j2 deleted file mode 100644 index b8db246037..0000000000 --- a/parm/archive/gfs_restartb.yaml.j2 +++ /dev/null @@ -1,28 +0,0 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gfs.t" + cycle_HH + "z." %} -name: "GFS_RESTARTB" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_restartb.tar" -mandatory: - {% set tiled_types = ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - {% for itile in range(1, 7) %} - {% for datatype in tiled_types %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{datatype}}.tile{{ itile }}.nc" - {% endfor %} - {% endfor %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/coupler" - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/fv_core.res.nc" - {% for r_time in range(restart_interval_gfs, FHMAX_GFS + 1, restart_inverval_gfs) %} - {% set r_timedelta = (r_time | string + "H") | to_timedelta %} - {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} - {% set r_YMD = r_dt | to_YMD %} - {% set r_HH = r_dt | strftime("%H") %} - {% set r_prefix = r_YMD + "." + r_HH + "0000" %} - {% for itile in range(1, 7) %} - {% for datatype in tiled_types %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" - {% endfor %} - {% endfor %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.coupler" - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" - {% endfor %} From 546e38518eb052d891e5c091cefe7f47a8fdd223 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 30 Apr 2024 13:52:43 +0000 Subject: [PATCH 053/128] Add master template outline. --- parm/archive/chem.yaml.j2 | 15 +- parm/archive/enkf.yaml.j2 | 121 +++++++------ parm/archive/enkf_grp.yaml.j2 | 97 +++++----- parm/archive/enkf_restarta_grp.yaml.j2 | 65 ++++--- parm/archive/enkf_restartb_grp.yaml.j2 | 57 +++--- parm/archive/gdas.yaml.j2 | 211 +++++++++++----------- parm/archive/gdas_restarta.yaml.j2 | 92 +++++----- parm/archive/gdas_restartb.yaml.j2 | 39 ++-- parm/archive/gdasice.yaml.j2 | 17 +- parm/archive/gdasice_restart.yaml.j2 | 15 +- parm/archive/gdasocean.yaml.j2 | 17 +- parm/archive/gdasocean_analysis.yaml.j2 | 21 ++- parm/archive/gdasocean_restart.yaml.j2 | 17 +- parm/archive/gdaswave.yaml.j2 | 17 +- parm/archive/gdaswave_restart.yaml.j2 | 13 +- parm/archive/gfs_downstream.yaml.j2 | 25 ++- parm/archive/gfs_flux.yaml.j2 | 19 +- parm/archive/gfs_flux_1p00.yaml.j2 | 19 +- parm/archive/gfs_netcdfa.yaml.j2 | 29 ++- parm/archive/gfs_netcdfb.yaml.j2 | 19 +- parm/archive/gfs_pgrb2b.yaml.j2 | 39 ++-- parm/archive/gfs_restarta.yaml.j2 | 47 +++-- parm/archive/gfsa.yaml.j2 | 117 ++++++------ parm/archive/gfsb.yaml.j2 | 35 ++-- parm/archive/gfswave.yaml.j2 | 39 ++-- parm/archive/ice_6hravg.yaml.j2 | 19 +- parm/archive/ice_grib2.yaml.j2 | 37 ++-- parm/archive/master_enkf.yaml.j2 | 56 ++++++ parm/archive/master_gdas.yaml.j2 | 42 +++++ parm/archive/master_gfs.yaml.j2 | 54 ++++++ parm/archive/ocean_6hravg.yaml.j2 | 17 +- parm/archive/ocean_daily.yaml.j2 | 17 +- parm/archive/ocean_grib2.yaml.j2 | 37 ++-- scripts/exgdas_enkf_earc.py | 4 +- sorc/wxflow | 2 +- ush/python/pygfs/task/archive.py | 229 +++++++----------------- 36 files changed, 872 insertions(+), 844 deletions(-) create mode 100644 parm/archive/master_enkf.yaml.j2 create mode 100644 parm/archive/master_gdas.yaml.j2 create mode 100644 parm/archive/master_gfs.yaml.j2 diff --git a/parm/archive/chem.yaml.j2 b/parm/archive/chem.yaml.j2 index d960819896..7ac8d5ae8c 100644 --- a/parm/archive/chem.yaml.j2 +++ b/parm/archive/chem.yaml.j2 @@ -1,8 +1,7 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gocart" %} -name: "CHEM" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/chem.tar" -mandatory: - # TODO explicitize this set - - "{{ COM_CHEM_HISTORY | relpath(ROTDIR) }}/{{ head }}*" + chem: + {% set head = "gocart" %} + name: "CHEM" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/chem.tar" + mandatory: + # TODO explicitize this set + - "{{ COM_CHEM_HISTORY | relpath(ROTDIR) }}/{{ head }}*" diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index b3e484d533..e2f59fdf9b 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -1,61 +1,60 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = RUN + ".t" + cycle_HH + "z." %} -{% if RUN == "enkfgdas" %} -{% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF, FHOUT_ENKF) %} -{% elif RUN == "enkfgfs" %} -{% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF_GFS, FHOUT_ENKF_GFS) %} -{% endif %} -name: "ENKF" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}.tar" -mandatory: - {% for fhr in range(fhmin, fhmax + fhout, fhout) %} - - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensmean.nc" - - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.ensmean.nc" - {% if ENKF_SPREAD %} - - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensspread.nc" - {% endif %} - {% endfor %} - {% if not current_cycle == SDATE %} - {% if not DO_JEDIATMENS %} - {% set da_files = ["enkfstat", - "gsistat.ensmean", - "cnvstat.ensmean", - "oznstat.ensmean", - "radstat.ensmean"] %} - {% else %} - {% set da_files = ["atmens.yaml", - "atmensstat"] %} - {% endif %} - {% for file in da_files %} - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}{{ file }}" - {% endfor %} - {% if DOIAU %} - {% for fhr in IAUFHRS %} - {% if fhr == IAU_OFFSET %} - {% if DO_CALC_INCREMENT_ENKF_GFS %} - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmanl.ensmean.nc" - {% endif %} # calc increment - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ratminc.ensmean.nc" - {% else %} # fhr != IAU_OFFSET - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensmean.nc" - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmi{{ '%03d' % fhr }}.ensmean.nc" - {% endif %} # fhr == IAU_OFFSET - {% endfor %} # IAUFHRS - {% endif %} # DOIAU - # TODO archive ecen logs based on actual groups. Will need to emulate numpy.array_split to do so. - {% set steps = ["eobs", "ecen*", "esfc", "eupd"] %} - {% for step in steps %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}{{ step }}.log" - {% endfor %} - {% for mem in range(1, NMEM_ENS + 1) %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_mem{{ '%03d' % mem }}.log" - {% endfor %} - {% if lobsdiag_forenkf %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}ediag.log" - {% else %} - {% for mem in range(1, NMEM_ENS + 1) %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}eomg_mem{{ '%03d' % mem }}.log" - {% endfor %} - {% endif %} # lobsdiag_forenkf - {% endif %} # not the first cycle + enkf: + {% set head = RUN + ".t" + cycle_HH + "z." %} + {% if RUN == "enkfgdas" %} + {% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF, FHOUT_ENKF) %} + {% elif RUN == "enkfgfs" %} + {% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF_GFS, FHOUT_ENKF_GFS) %} + {% endif %} + name: "ENKF" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}.tar" + mandatory: + {% for fhr in range(fhmin, fhmax + fhout, fhout) %} + - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensmean.nc" + - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.ensmean.nc" + {% if ENKF_SPREAD %} + - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensspread.nc" + {% endif %} + {% endfor %} + {% if not current_cycle == SDATE %} + {% if not DO_JEDIATMENS %} + {% set da_files = ["enkfstat", + "gsistat.ensmean", + "cnvstat.ensmean", + "oznstat.ensmean", + "radstat.ensmean"] %} + {% else %} + {% set da_files = ["atmens.yaml", + "atmensstat"] %} + {% endif %} + {% for file in da_files %} + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}{{ file }}" + {% endfor %} + {% if DOIAU %} + {% for fhr in IAUFHRS %} + {% if fhr == IAU_OFFSET %} + {% if DO_CALC_INCREMENT_ENKF_GFS %} + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmanl.ensmean.nc" + {% endif %} # calc increment + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ratminc.ensmean.nc" + {% else %} # fhr != IAU_OFFSET + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensmean.nc" + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmi{{ '%03d' % fhr }}.ensmean.nc" + {% endif %} # fhr == IAU_OFFSET + {% endfor %} # IAUFHRS + {% endif %} # DOIAU + # TODO archive ecen logs based on actual groups. Will need to emulate numpy.array_split to do so. + {% set steps = ["eobs", "ecen*", "esfc", "eupd"] %} + {% for step in steps %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}{{ step }}.log" + {% endfor %} + {% for mem in range(1, NMEM_ENS + 1) %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_mem{{ '%03d' % mem }}.log" + {% endfor %} + {% if lobsdiag_forenkf %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}ediag.log" + {% else %} + {% for mem in range(1, NMEM_ENS + 1) %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}eomg_mem{{ '%03d' % mem }}.log" + {% endfor %} + {% endif %} # lobsdiag_forenkf + {% endif %} # not the first cycle diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index 4bfc45fe00..95d4372f0b 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -1,52 +1,51 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = RUN + ".t" + cycle_HH + "z." %} -{% if RUN == "enkfgdas" %} -{% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF, FHOUT_ENKF) %} -{% elif RUN == "enkfgfs" %} -{% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF_GFS, FHOUT_ENKF_GFS) %} -{% endif %} -name: "ENKF_GRP" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_grp{{ ENSGRP }}.tar" -mandatory: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} - {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + enkf_grp: + {% set head = RUN + ".t" + cycle_HH + "z." %} + {% if RUN == "enkfgdas" %} + {% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF, FHOUT_ENKF) %} + {% elif RUN == "enkfgfs" %} + {% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF_GFS, FHOUT_ENKF_GFS) %} + {% endif %} + name: "ENKF_GRP" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_grp{{ ENSGRP }}.tar" + mandatory: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} + {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% if current_cycle != SDATE and not lobsdiag_forenkf %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat" - {% endif %} - {% for fhr in range(fhmin, fhmax+fhout, fhout) %} - - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" - - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" - {% endfor %} + {% if current_cycle != SDATE and not lobsdiag_forenkf %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat" + {% endif %} + {% for fhr in range(fhmin, fhmax+fhout, fhout) %} + - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" + - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + {% endfor %} - {% if not current_cycle == SDATE %} - {% if DO_CALC_INCR_ENKF_GFS %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - {% else %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratminc.nc" - {% endif %} - {% endif %} + {% if not current_cycle == SDATE %} + {% if DO_CALC_INCR_ENKF_GFS %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + {% else %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratminc.nc" + {% endif %} + {% endif %} - {% for iauhr in iaufhrs if iauhr != 6 %} - {% set iauhr = iauhr | int %} - {% if DO_CALC_INCREMENT_ENKF_GFS %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iauhr }}.nc" - {% else %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iauhr }}.nc" - {% endif %} - {% endfor %} # iauhr in iaufhrs - {% endfor %} # first_group_mem to last_group_mem -optional: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} - {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% if current_cycle != SDATE and not lobsdiag_forenkf %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" - {% endif %} - {% endfor %} + {% for iauhr in iaufhrs if iauhr != 6 %} + {% set iauhr = iauhr | int %} + {% if DO_CALC_INCREMENT_ENKF_GFS %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iauhr }}.nc" + {% else %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iauhr }}.nc" + {% endif %} + {% endfor %} # iauhr in iaufhrs + {% endfor %} # first_group_mem to last_group_mem + optional: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} + {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + {% if current_cycle != SDATE and not lobsdiag_forenkf %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" + {% endif %} + {% endfor %} diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 index 99c474f120..53d1e7451d 100644 --- a/parm/archive/enkf_restarta_grp.yaml.j2 +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -1,33 +1,32 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = RUN + ".t" + cycle_HH + "z." %} -name: "ENKF_RESTARTA_GRP" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restarta_grp{{ ENSGRP }}.tar" -mandatory: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} - {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - - {% if not lobsdiag_forenkf %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_air" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_int" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_pc" - {% endif %} - - {% set anl_delta = "-3H" | to_timedelta %} - {% set anl_time = current_cycle | add_to_datetime(anl_delta) %} - {% for itile in range(1, 7) %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" - {% endfor %} - {% endfor %} # first_group_mem to last_group_mem -optional: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" - {% endif %} - {% endfor %} + enkf_restarta_grp: + {% set head = RUN + ".t" + cycle_HH + "z." %} + name: "ENKF_RESTARTA_GRP" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restarta_grp{{ ENSGRP }}.tar" + mandatory: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} + {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + + {% if not lobsdiag_forenkf %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_air" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_int" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_pc" + {% endif %} + + {% set anl_delta = "-3H" | to_timedelta %} + {% set anl_time = current_cycle | add_to_datetime(anl_delta) %} + {% for itile in range(1, 7) %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" + {% endfor %} + {% endfor %} # first_group_mem to last_group_mem + optional: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" + {% endif %} + {% endfor %} diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 index 529df7bc8b..cf50f6c76d 100644 --- a/parm/archive/enkf_restartb_grp.yaml.j2 +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -1,29 +1,28 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = RUN + ".t" + cycle_HH + "z." %} -{% if RUN == "enkfgdas" %} -{% set fhmax = FHMAX_ENKF %} -{% set restart_interval = restart_interval_enkfgdas %} -{% elif RUN == "enkfgfs" %} -{% set fhmax = FHMAX_ENKF_GFS %} -{% set restart_interval = restart_interval_enkfgfs %} -{% endif %} -name: "ENKF_RESTARTB_GRP" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restartb_grp{{ ENSGRP }}.tar" -mandatory: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% for r_time in range(restart_interval, fhmax + 1, restart_interval) %} - {% set r_timedelta = (r_time | string + "H") | to_timedelta %} - {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} - {% set r_prefix = r_dt | to_YMD + "." + r_dt | strftime("%H") + "0000" %} - {% for itile in range(1, 7) %} - {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" - {% endfor %} - {% endfor %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" - {% endfor %} - {% endfor %} + enkf_restartb_grp: + {% set head = RUN + ".t" + cycle_HH + "z." %} + {% if RUN == "enkfgdas" %} + {% set fhmax = FHMAX_ENKF %} + {% set restart_interval = restart_interval_enkfgdas %} + {% elif RUN == "enkfgfs" %} + {% set fhmax = FHMAX_ENKF_GFS %} + {% set restart_interval = restart_interval_enkfgfs %} + {% endif %} + name: "ENKF_RESTARTB_GRP" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restartb_grp{{ ENSGRP }}.tar" + mandatory: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + {% for r_time in range(restart_interval, fhmax + 1, restart_interval) %} + {% set r_timedelta = (r_time | string + "H") | to_timedelta %} + {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} + {% set r_prefix = r_dt | to_YMD + "." + r_dt | strftime("%H") + "0000" %} + {% for itile in range(1, 7) %} + {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" + {% endfor %} + {% endfor %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" + {% endfor %} + {% endfor %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 66834ce1c3..9aafbc1de2 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -1,106 +1,105 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gdas.t" + cycle_HH + "z." %} -name: "GDAS" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas.tar" -mandatory: - {% if not current_cycle == SDATE and MODE == "cycled" %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" - - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" - - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" - - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" - - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" - {% if DOIAU and DOHYBVAR %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma003.ensres.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma009.ensres.nc" - {% endif %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}oznstat" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" - {% if DO_JEDIATMVAR %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" - {% endif %} - {% if DO_VERFOZN %} - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_cnt.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_diag.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_pen.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/stdout.time.tar.gz" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/horiz/stdout.horiz.tar.gz" - - "logs/{{ cycle_YMDH }}/{{ RUN }}verfozn.log" - {% endif %} - {% if DO_VERFRAD %} - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_angle.tar.gz" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcoef.tar.gz" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcor.tar.gz" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_time.tar.gz" - - "logs/{{ cycle_YMDH }}/{{ RUN }}verfrad.log" - {% endif %} - {% if DO_VMINMON %} - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" - - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" - {% endif %} - {% if DO_AERO %} - - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" - {% endif %} - {% if DO_JEDISNOWDA %} - - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snowstat.tgz" - {% endif %} - {% endif %} # Full cycle - - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" - # TODO explicitly name the atmos_prod log files to archive - - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" - {% for fhr in range(0, FHMAX + FHOUT, FHOUT) %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" - {% endfor %} -optional: - {% if not current_cycle == SDATE and MODE == "cycled" %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" - {% if DO_VERFRAD %} - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/low_count.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/warning.{{ cycle_YMDH }}" - {% endif %} - {% if DO_VERFOZN %} - {% set oznmon_types = [ - "gome_metop-b", "omi_aura", "ompslp_npp", "ompsnp_n20", - "ompsnp_npp", "ompstc8_n20", "ompstc8_npp", "sbuv2_n19" - ] %} - {% for group in [ "horiz", "time" ] %} - {% if group == "horiz" %} {% set suffix = ".gz" %} {% else %} {% set suffix = "" %} {% endif %} - {% for type in oznmon_types %} - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.ctl" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.{{ cycle_YMDH }}.ieee_d{{ suffix }}" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.ctl" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.{{ cycle_YMDH }}.ieee_d{{ suffix }}" - {% endfor %} - {% endfor %} - {% endif %} - {% endif %} - {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" - {% endfor %} - {% if not WRITE_DOPOST %} - # TODO set the forecast hours explicitly. This will require emulating numpy.array_split - - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" - {% endif %} ## not WRITE_DOPOST + gdas: + {% set head = "gdas.t" + cycle_HH + "z." %} + name: "GDAS" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas.tar" + mandatory: + {% if not current_cycle == SDATE and MODE == "cycled" %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" + {% if DOIAU and DOHYBVAR %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma003.ensres.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma009.ensres.nc" + {% endif %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}oznstat" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" + {% if DO_JEDIATMVAR %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" + {% endif %} + {% if DO_VERFOZN %} + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_cnt.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_diag.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_pen.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/stdout.time.tar.gz" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/horiz/stdout.horiz.tar.gz" + - "logs/{{ cycle_YMDH }}/{{ RUN }}verfozn.log" + {% endif %} + {% if DO_VERFRAD %} + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_angle.tar.gz" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcoef.tar.gz" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcor.tar.gz" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_time.tar.gz" + - "logs/{{ cycle_YMDH }}/{{ RUN }}verfrad.log" + {% endif %} + {% if DO_VMINMON %} + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" + - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" + {% endif %} + {% if DO_AERO %} + - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" + {% endif %} + {% if DO_JEDISNOWDA %} + - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snowstat.tgz" + {% endif %} + {% endif %} # Full cycle + - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" + # TODO explicitly name the atmos_prod log files to archive + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" + {% for fhr in range(0, FHMAX + FHOUT, FHOUT) %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + {% endfor %} + optional: + {% if not current_cycle == SDATE and MODE == "cycled" %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" + {% if DO_VERFRAD %} + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/low_count.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/warning.{{ cycle_YMDH }}" + {% endif %} + {% if DO_VERFOZN %} + {% set oznmon_types = [ + "gome_metop-b", "omi_aura", "ompslp_npp", "ompsnp_n20", + "ompsnp_npp", "ompstc8_n20", "ompstc8_npp", "sbuv2_n19" + ] %} + {% for group in [ "horiz", "time" ] %} + {% if group == "horiz" %} {% set suffix = ".gz" %} {% else %} {% set suffix = "" %} {% endif %} + {% for type in oznmon_types %} + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.ctl" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.{{ cycle_YMDH }}.ieee_d{{ suffix }}" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.ctl" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.{{ cycle_YMDH }}.ieee_d{{ suffix }}" + {% endfor %} + {% endfor %} + {% endif %} + {% endif %} + {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + {% endfor %} + {% if not WRITE_DOPOST %} + # TODO set the forecast hours explicitly. This will require emulating numpy.array_split + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" + {% endif %} ## not WRITE_DOPOST diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index c7212f8186..3a4c375db5 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -1,47 +1,45 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set head = "gdas.t" + cycle_HH + "z." %} -name: "GDAS_RESTARTA" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restarta.tar" -mandatory: - {% if not current_cycle == SDATE and MODE == "cycled" %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" - {% for iauhr in IAUHRS if iauhr != 6 %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" - {% endfor %} - {% if DOHYBVAR %} - {% set anl_offset = "-3H" %} - {% else %} - {% set anl_offset = "0H" %} - {% endif %} - {% set anl_delta = anl_offset | to_timedelta %} - {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} - {% for itile in range(1,7) %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" - {% endfor %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_air" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_pc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" - {% if DO_JEDISNOWDA %} - {% for itile in range(1,7) %} - # Snow analysis is 3dvar - - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/snowinc.{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" - - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" - {% endfor %} - {% endif %} - {% endif %} -optional: - {% if not current_cycle == SDATE and MODE == "cycled" %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" - {% if DO_JEDISNOWDA %} - - "{{ COM_CONF | relpath(ROTDIR) }}/{{ head }}letkfoi.yaml" - {% endif %} - {% endif %} + gdas_restarta: + {% set head = "gdas.t" + cycle_HH + "z." %} + name: "GDAS_RESTARTA" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restarta.tar" + mandatory: + {% if not current_cycle == SDATE and MODE == "cycled" %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" + {% for iauhr in IAUHRS if iauhr != 6 %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" + {% endfor %} + {% if DOHYBVAR %} + {% set anl_offset = "-3H" %} + {% else %} + {% set anl_offset = "0H" %} + {% endif %} + {% set anl_delta = anl_offset | to_timedelta %} + {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} + {% for itile in range(1,7) %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" + {% endfor %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_air" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_pc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" + {% if DO_JEDISNOWDA %} + {% for itile in range(1,7) %} + # Snow analysis is 3dvar + - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/snowinc.{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" + - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" + {% endfor %} + {% endif %} + {% endif %} + optional: + {% if not current_cycle == SDATE and MODE == "cycled" %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" + {% if DO_JEDISNOWDA %} + - "{{ COM_CONF | relpath(ROTDIR) }}/{{ head }}letkfoi.yaml" + {% endif %} + {% endif %} diff --git a/parm/archive/gdas_restartb.yaml.j2 b/parm/archive/gdas_restartb.yaml.j2 index 335376fde3..56e5b5ca6e 100644 --- a/parm/archive/gdas_restartb.yaml.j2 +++ b/parm/archive/gdas_restartb.yaml.j2 @@ -1,20 +1,19 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gdas.t" + cycle_HH + "z." %} -name: "GDAS_RESTARTB" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restartb.tar" -mandatory: - {% for r_time in range(restart_interval_gdas, FHMAX + 1, restart_interval_gdas) %} - {% set r_timedelta = (r_time | string + "H") | to_timedelta %} - {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} - {% set r_YMD = r_dt | to_YMD %} - {% set r_HH = r_dt | strftime("%H") %} - {% set r_prefix = r_YMD + "." + r_HH + "0000" %} - {% for itile in range(1, 7) %} - {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" - {% endfor %} - {% endfor %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" - {% endfor %} + gdas_restartb: + {% set head = "gdas.t" + cycle_HH + "z." %} + name: "GDAS_RESTARTB" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restartb.tar" + mandatory: + {% for r_time in range(restart_interval_gdas, FHMAX + 1, restart_interval_gdas) %} + {% set r_timedelta = (r_time | string + "H") | to_timedelta %} + {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} + {% set r_YMD = r_dt | to_YMD %} + {% set r_HH = r_dt | strftime("%H") %} + {% set r_prefix = r_YMD + "." + r_HH + "0000" %} + {% for itile in range(1, 7) %} + {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" + {% endfor %} + {% endfor %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" + {% endfor %} diff --git a/parm/archive/gdasice.yaml.j2 b/parm/archive/gdasice.yaml.j2 index df10f1eb1f..09b932e328 100644 --- a/parm/archive/gdasice.yaml.j2 +++ b/parm/archive/gdasice.yaml.j2 @@ -1,9 +1,8 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gdas.t" + cycle_HH + "z." %} -name: "GDASICE" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar" -mandatory: - # TODO explicitly name the files in history to archive - - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}*' - - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/ice_in' + gdasice: + {% set head = "gdas.t" + cycle_HH + "z." %} + name: "GDASICE" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar" + mandatory: + # TODO explicitly name the files in history to archive + - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}*' + - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/ice_in' diff --git a/parm/archive/gdasice_restart.yaml.j2 b/parm/archive/gdasice_restart.yaml.j2 index 65e2b77ff6..112e8c0bd8 100644 --- a/parm/archive/gdasice_restart.yaml.j2 +++ b/parm/archive/gdasice_restart.yaml.j2 @@ -1,8 +1,7 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gdas.t" + cycle_HH + "z." %} -name: "GDASICE_RESTART" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice_restart.tar" -mandatory: - # TODO explicitly name the restart files to archive - - '{{ COM_ICE_RESTART | relpath(ROTDIR) }}/*' + gdasice_restart: + {% set head = "gdas.t" + cycle_HH + "z." %} + name: "GDASICE_RESTART" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice_restart.tar" + mandatory: + # TODO explicitly name the restart files to archive + - '{{ COM_ICE_RESTART | relpath(ROTDIR) }}/*' diff --git a/parm/archive/gdasocean.yaml.j2 b/parm/archive/gdasocean.yaml.j2 index 794a491ece..d0bb7d8606 100644 --- a/parm/archive/gdasocean.yaml.j2 +++ b/parm/archive/gdasocean.yaml.j2 @@ -1,9 +1,8 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gdas.t" + cycle_HH + "z." %} -name: "GDASOCEAN" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar" -mandatory: - # TODO Explicitly name the history and input files to archive - - '{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}*' - - '{{ COM_OCEAN_INPUT | relpath(ROTDIR) }}/.' + gdasocean: + {% set head = "gdas.t" + cycle_HH + "z." %} + name: "GDASOCEAN" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar" + mandatory: + # TODO Explicitly name the history and input files to archive + - '{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}*' + - '{{ COM_OCEAN_INPUT | relpath(ROTDIR) }}/.' diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index bf3fac59c6..b27d8c5482 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -1,11 +1,10 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gdas.t" + cycle_HH + "z." %} -name: "GDASOCEAN_ANALYSIS" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar" -mandatory: - # TODO explicitly name the analysis files to archive - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}*' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t??z.ocngrid.nc' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml' + gdasocean_analysis: + {% set head = "gdas.t" + cycle_HH + "z." %} + name: "GDASOCEAN_ANALYSIS" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar" + mandatory: + # TODO explicitly name the analysis files to archive + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}*' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t??z.ocngrid.nc' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml' diff --git a/parm/archive/gdasocean_restart.yaml.j2 b/parm/archive/gdasocean_restart.yaml.j2 index d5ab2dcde8..bd0d241f1a 100644 --- a/parm/archive/gdasocean_restart.yaml.j2 +++ b/parm/archive/gdasocean_restart.yaml.j2 @@ -1,9 +1,8 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gdas.t" + cycle_HH + "z." %} -name: "GDASOCEAN_RESTART" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_restart.tar" -mandatory: - # TODO explicitly name the restart files to archive - - '{{ COM_OCEAN_RESTART | relpath(ROTDIR) }}/*' - - '{{ COM_MED_RESTART | relpath(ROTDIR) }}/*' + gdasocean_restart: + {% set head = "gdas.t" + cycle_HH + "z." %} + name: "GDASOCEAN_RESTART" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_restart.tar" + mandatory: + # TODO explicitly name the restart files to archive + - '{{ COM_OCEAN_RESTART | relpath(ROTDIR) }}/*' + - '{{ COM_MED_RESTART | relpath(ROTDIR) }}/*' diff --git a/parm/archive/gdaswave.yaml.j2 b/parm/archive/gdaswave.yaml.j2 index 6b26a3bbf1..c88875cb34 100644 --- a/parm/archive/gdaswave.yaml.j2 +++ b/parm/archive/gdaswave.yaml.j2 @@ -1,9 +1,8 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gdaswave.t" + cycle_HH + "z." %} -name: "GDASWAVE" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave.tar" -mandatory: - # TODO explicitly name the wave grid/station files to archive - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}*" + gdaswave: + {% set head = "gdaswave.t" + cycle_HH + "z." %} + name: "GDASWAVE" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave.tar" + mandatory: + # TODO explicitly name the wave grid/station files to archive + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}*" diff --git a/parm/archive/gdaswave_restart.yaml.j2 b/parm/archive/gdaswave_restart.yaml.j2 index b26f17b51a..466c913152 100644 --- a/parm/archive/gdaswave_restart.yaml.j2 +++ b/parm/archive/gdaswave_restart.yaml.j2 @@ -1,7 +1,6 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -name: "GDASWAVE_RESTART" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave_restart.tar" -mandatory: - # TODO explicitly name the wave restart files to archive - - "{{ COM_WAVE_RESTART | relpath(ROTDIR) }}/*" + gdaswave_restart: + name: "GDASWAVE_RESTART" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave_restart.tar" + mandatory: + # TODO explicitly name the wave restart files to archive + - "{{ COM_WAVE_RESTART | relpath(ROTDIR) }}/*" diff --git a/parm/archive/gfs_downstream.yaml.j2 b/parm/archive/gfs_downstream.yaml.j2 index 2a358aa7dc..909be04cf0 100644 --- a/parm/archive/gfs_downstream.yaml.j2 +++ b/parm/archive/gfs_downstream.yaml.j2 @@ -1,13 +1,12 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gfs.t" + cycle_HH + "z." %} -name: "GFS_DOWNSTREAM" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_downstream.tar" -mandatory: - - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.sfc" - - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.snd" - {% for i in range(1, NUM_SND_COLLECTIVES) %} - - "{{ COM_ATMOS_WMO | relpath(ROTDIR) }}/gfs_collective{{ i }}.postsnd_{{ cycle_HH }}" - {% endfor %} - - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/bufr.t{{ cycle_HH }}z" - - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/gfs.t{{ cycle_HH }}z.bufrsnd.tar.gz" + gfs_downstream: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_DOWNSTREAM" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_downstream.tar" + mandatory: + - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.sfc" + - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.snd" + {% for i in range(1, NUM_SND_COLLECTIVES) %} + - "{{ COM_ATMOS_WMO | relpath(ROTDIR) }}/gfs_collective{{ i }}.postsnd_{{ cycle_HH }}" + {% endfor %} + - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/bufr.t{{ cycle_HH }}z" + - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/gfs.t{{ cycle_HH }}z.bufrsnd.tar.gz" diff --git a/parm/archive/gfs_flux.yaml.j2 b/parm/archive/gfs_flux.yaml.j2 index b0fb550a54..66e3eaa455 100644 --- a/parm/archive/gfs_flux.yaml.j2 +++ b/parm/archive/gfs_flux.yaml.j2 @@ -1,10 +1,9 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gfs.t" + cycle_HH + "z." %} -name: "GFS_FLUX" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux.tar" -mandatory: - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" - {% endfor %} + gfs_flux: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_FLUX" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux.tar" + mandatory: + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + {% endfor %} diff --git a/parm/archive/gfs_flux_1p00.yaml.j2 b/parm/archive/gfs_flux_1p00.yaml.j2 index bd86382c2e..c7168eb01a 100644 --- a/parm/archive/gfs_flux_1p00.yaml.j2 +++ b/parm/archive/gfs_flux_1p00.yaml.j2 @@ -1,10 +1,9 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gfs.t" + cycle_HH + "z." %} -name: "GFS_FLUX_1P00" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux_1p00.tar" -mandatory: - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}.idx" - {% endfor %} + gfs_flux_1p00: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_FLUX_1P00" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux_1p00.tar" + mandatory: + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}.idx" + {% endfor %} diff --git a/parm/archive/gfs_netcdfa.yaml.j2 b/parm/archive/gfs_netcdfa.yaml.j2 index 4c22c6c734..e015d8e4a1 100644 --- a/parm/archive/gfs_netcdfa.yaml.j2 +++ b/parm/archive/gfs_netcdfa.yaml.j2 @@ -1,15 +1,14 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gfs.t" + cycle_HH + "z." %} -name: "GFS_NETCDFA" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfa.tar" -mandatory: - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" - {% for iauhr in IAUHRS if iauhr != 6 %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" - {% endfor %} -optional: - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" + gfs_netcdfa: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_NETCDFA" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfa.tar" + mandatory: + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" + {% for iauhr in IAUHRS if iauhr != 6 %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" + {% endfor %} + optional: + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" diff --git a/parm/archive/gfs_netcdfb.yaml.j2 b/parm/archive/gfs_netcdfb.yaml.j2 index 9afab91520..bcf8f67861 100644 --- a/parm/archive/gfs_netcdfb.yaml.j2 +++ b/parm/archive/gfs_netcdfb.yaml.j2 @@ -1,10 +1,9 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gfs.t" + cycle_HH + "z." %} -name: "GFS_NETCDFB" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfb.tar" -mandatory: - {% for fhr in range(0, ARCH_GAUSSIAN_FHMAX + ARCH_GAUSSIAN_FHINC, ARCH_GAUSSIAN_FHINC) %} - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" - {% endfor %} + gfs_netcdfb: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_NETCDFB" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfb.tar" + mandatory: + {% for fhr in range(0, ARCH_GAUSSIAN_FHMAX + ARCH_GAUSSIAN_FHINC, ARCH_GAUSSIAN_FHINC) %} + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/gfs_pgrb2b.yaml.j2 b/parm/archive/gfs_pgrb2b.yaml.j2 index 5e2fefcfdf..f479359d9c 100644 --- a/parm/archive/gfs_pgrb2b.yaml.j2 +++ b/parm/archive/gfs_pgrb2b.yaml.j2 @@ -1,20 +1,19 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gfs.t" + cycle_HH + "z." %} -name: "GFS_PGRB2B" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_pgrb2b.tar" -mandatory: - {% if MODE == "cycled" %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl.idx" - {% endif %} - {% if ARCH_GAUSSIAN %} - {% for fhr in range(0, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}.idx" - {% endfor %} - {% endif %} + gfs_pgrb2b: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_PGRB2B" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_pgrb2b.tar" + mandatory: + {% if MODE == "cycled" %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl.idx" + {% endif %} + {% if ARCH_GAUSSIAN %} + {% for fhr in range(0, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}.idx" + {% endfor %} + {% endif %} diff --git a/parm/archive/gfs_restarta.yaml.j2 b/parm/archive/gfs_restarta.yaml.j2 index 5f125f2121..1dbe20a178 100644 --- a/parm/archive/gfs_restarta.yaml.j2 +++ b/parm/archive/gfs_restarta.yaml.j2 @@ -1,24 +1,23 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gfs.t" + cycle_HH + "z." %} -name: "GFS_RESTARTA" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_restarta.tar" -mandatory: - {% if MODE == "cycled" %} - {% if DOHYBVAR %} - {% set anl_offset = "-3H" %} - {% else %} - {% set anl_offset = "0H" %} - {% endif %} - {% set anl_delta = anl_offset | to_timedelta %} - {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} - {% for i_tile in range(1, 7) %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ i_tile }}.nc" - {% endfor %} - {% elif MODE == "forecast-only" %} - - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc" - {% for i_tile in range(1, 7) %} - - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_data.tile{{ i_tile }}.nc" - - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/sfc_data.tile{{ i_tile }}.nc" - {% endfor %} - {% endif %} + gfs_restarta: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_RESTARTA" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_restarta.tar" + mandatory: + {% if MODE == "cycled" %} + {% if DOHYBVAR %} + {% set anl_offset = "-3H" %} + {% else %} + {% set anl_offset = "0H" %} + {% endif %} + {% set anl_delta = anl_offset | to_timedelta %} + {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} + {% for i_tile in range(1, 7) %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ i_tile }}.nc" + {% endfor %} + {% elif MODE == "forecast-only" %} + - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc" + {% for i_tile in range(1, 7) %} + - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_data.tile{{ i_tile }}.nc" + - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/sfc_data.tile{{ i_tile }}.nc" + {% endfor %} + {% endif %} diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index dfb416da68..8fdd07caeb 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -1,59 +1,58 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gfs.t" + cycle_HH + "z." %} -name: "GFSA" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsa.tar" -mandatory: - # TODO explicitly name all logs to include - {% for log in glob("logs/" ~ cycle_YMDH ~ "/gfs*.log") %} - {% if log != "gfsarch.log" %} - - "{{ log }}" - {% endif %} - {% endfor %} - # - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" - # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" - - "{{ COM_CONF | relpath(ROTDIR) }}/ufs.input.nml" - {% if MODE == "cycled" %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" - # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" - # - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" - # - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" - # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" - {% if DO_VMINMON %} - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" - # - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" - {% endif %} - {% endif %} # Full cycle - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" - {% endfor %} -optional: - {% if MODE == "cycled" %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" - {% if DO_JEDIATMVAR %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" - {% endif %} - {% endif %} # cycled - - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cyclone.trackatcfunix" - - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avnop.t{{ cycle_HH }}z.cyclone.trackatcfunix" - - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}" - # {% if not WRITE_DOPOST %} - # # TODO set the forecast hours explicitly. This will require emulating numpy.array_split - # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" - # {% endif %} ## not WRITE_DOPOST + gfsa: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFSA" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsa.tar" + mandatory: + # TODO explicitly name all logs to include + {% for log in glob("logs/" ~ cycle_YMDH ~ "/gfs*.log") %} + {% if log != "gfsarch.log" %} + - "{{ log }}" + {% endif %} + {% endfor %} + # - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" + - "{{ COM_CONF | relpath(ROTDIR) }}/ufs.input.nml" + {% if MODE == "cycled" %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" + {% if DO_VMINMON %} + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" + {% endif %} + {% endif %} # Full cycle + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" + {% endfor %} + optional: + {% if MODE == "cycled" %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" + {% if DO_JEDIATMVAR %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" + {% endif %} + {% endif %} # cycled + - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cyclone.trackatcfunix" + - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avnop.t{{ cycle_HH }}z.cyclone.trackatcfunix" + - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}" + # {% if not WRITE_DOPOST %} + # # TODO set the forecast hours explicitly. This will require emulating numpy.array_split + # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" + # {% endif %} ## not WRITE_DOPOST diff --git a/parm/archive/gfsb.yaml.j2 b/parm/archive/gfsb.yaml.j2 index cbefcdf240..a550e335da 100644 --- a/parm/archive/gfsb.yaml.j2 +++ b/parm/archive/gfsb.yaml.j2 @@ -1,18 +1,17 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gfs.t" + cycle_HH + "z." %} -name: "GFSB" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsb.tar" -mandatory: - {% if MODE == "cycled" %} - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl" - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" - {% endif %} - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" - {% endfor %} + gfsb: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFSB" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsb.tar" + mandatory: + {% if MODE == "cycled" %} + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl" + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" + {% endif %} + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" + {% endfor %} diff --git a/parm/archive/gfswave.yaml.j2 b/parm/archive/gfswave.yaml.j2 index 632da1a383..74ec172d9b 100644 --- a/parm/archive/gfswave.yaml.j2 +++ b/parm/archive/gfswave.yaml.j2 @@ -1,20 +1,19 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gfswave.t" + cycle_HH + "z." %} -name: "GFSWAVE" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfswave.tar" -mandatory: - {% for fh in range(0, FHMAX_HF_WAV + FHOUT_HF_WAV, FHOUT_HF_WAV) %} - # NOTE This is as explicit as possible without major logic to parse wavepostGRD. - # Matches files of the form "gfswave.tCCz...fHHH.grib2". - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" - {% endfor %} - {% for fh in range(FHMAX_HF_WAV + FHOUT_WAV, FHMAX_WAV_GFS + FHOUT_WAV, FHOUT_WAV) %} - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" - {% endfor %} -optional: - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}bull_tar" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}cbull_tar" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}spec_tar.gz" + gfswave: + {% set head = "gfswave.t" + cycle_HH + "z." %} + name: "GFSWAVE" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfswave.tar" + mandatory: + {% for fh in range(0, FHMAX_HF_WAV + FHOUT_HF_WAV, FHOUT_HF_WAV) %} + # NOTE This is as explicit as possible without major logic to parse wavepostGRD. + # Matches files of the form "gfswave.tCCz...fHHH.grib2". + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" + {% endfor %} + {% for fh in range(FHMAX_HF_WAV + FHOUT_WAV, FHMAX_WAV_GFS + FHOUT_WAV, FHOUT_WAV) %} + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" + {% endfor %} + optional: + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}bull_tar" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}cbull_tar" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}spec_tar.gz" diff --git a/parm/archive/ice_6hravg.yaml.j2 b/parm/archive/ice_6hravg.yaml.j2 index a413da63da..060089d2c3 100644 --- a/parm/archive/ice_6hravg.yaml.j2 +++ b/parm/archive/ice_6hravg.yaml.j2 @@ -1,10 +1,9 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = to_YMDH %} -{% set head = "gfs.ice.t" + cycle_HH + "z." %} -name: "ICE_6HRAVG" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_6hravg.tar" -mandatory: - - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}ic.nc" - {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" - {% endfor %} + ice_6hravg: + {% set head = "gfs.ice.t" + cycle_HH + "z." %} + name: "ICE_6HRAVG" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_6hravg.tar" + mandatory: + - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}ic.nc" + {% for fhr in range(6, FHMAX_GFS + 6, 6) %} + - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/ice_grib2.yaml.j2 b/parm/archive/ice_grib2.yaml.j2 index 56f1b0dca8..9af8284c66 100644 --- a/parm/archive/ice_grib2.yaml.j2 +++ b/parm/archive/ice_grib2.yaml.j2 @@ -1,19 +1,18 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = to_YMDH %} -{% set head = "gfs.ice.t" + cycle_HH + "z." %} -name: "ICE_GRIB2" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_grib2.tar" -mandatory: - {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %} - {% set fhr3 = '%03d' % fhr %} - {% if ICERES == 500 %} - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" - {% elif ICERES == 100 %} - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" - {% elif ICERES == 25 or ICERES == "025" %} - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" - {% endif %} - {% endfor %} + ice_grib2: + {% set head = "gfs.ice.t" + cycle_HH + "z." %} + name: "ICE_GRIB2" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_grib2.tar" + mandatory: + {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %} + {% set fhr3 = '%03d' % fhr %} + {% if ICERES == 500 %} + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" + {% elif ICERES == 100 %} + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" + {% elif ICERES == 25 or ICERES == "025" %} + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" + {% endif %} + {% endfor %} diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 new file mode 100644 index 0000000000..3c3b00e760 --- /dev/null +++ b/parm/archive/master_enkf.yaml.j2 @@ -0,0 +1,56 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +datasets: +{% if ENSGRP == 0 %} +{% include "enkf.yaml.j2" %} +{% else %} + +# Set variables/lists needed to parse the enkf templates +{% if IAUFHRS is string %} +{% set iaufhrs = (int(fhr) for fhr in IAUFHRS.split(",")) %} +{% else %} +{% set iaufhrs = [IAUFHRS] %} +{% endif %} + +# Declare a dict to get member COM directories from templates +{% set tmpl_dict = {'ROTDIR':ROTDIR, 'RUN':RUN, 'YMD':cycle_YMD, 'HH':cycle_HH} %} + +# Declare to-be-filled lists of member COM directories +{% set COM_ATMOS_ANALYSIS_MEM_list = [] %} +{% set COM_ATMOS_RESTART_MEM_list = [] %} +{% set COM_ATMOS_HISTORY_MEM_list = [] %} + +{% set first_mem = (ENSGRP - 1) * NMEM_EARCGRP + 1 %} +{% set last_mem = [ ENSGRP * NMEM_EARCGRP, NMEM_ENS ] | min %} + +{% for mem in range(first_mem, last_mem + 1) %} + +{% set tmpl_dict['MEMDIR'] = "mem" + '%03d' % mem %} + +# Populate the member COM directories from the template +{{ COM_ATMOS_ANALYSIS_MEM_list.append( + template_substitute_structure(COM_ATMOS_ANALYSIS_TMPL, + DOLLAR_CURLY_BRACE, + tmpl_dict.get)) }} + +{{ COM_ATMOS_RESTART_MEM_list.append( + template_substitute_structure(COM_ATMOS_RESTART_TMPL, + DOLLAR_CURLY_BRACE, + tmpl_dict.get)) }} + +{{ COM_ATMOS_HISTORY_MEM_list.append( + template_substitute_structure(COM_ATMOS_HISTORY_TMPL, + DOLLAR_CURLY_BRACE, + tmpl_dict.get)) }} + +{% endfor %} + +# Determine which members to archive +{% include "enkf_grp.yaml.j2" %} +{% if current_cycle > SDATE %} +{% include "enkf_restarta_grp.yaml.j2" %} +{% include "enkf_restartb_grp.yaml.j2" %} +{% endif %} + +{% endif %} # ENSGRP != 0 diff --git a/parm/archive/master_gdas.yaml.j2 b/parm/archive/master_gdas.yaml.j2 new file mode 100644 index 0000000000..349aad12a8 --- /dev/null +++ b/parm/archive/master_gdas.yaml.j2 @@ -0,0 +1,42 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = "gdas.t" + cycle_HH + "z." %} +datasets: +{% include "gdas.yaml.j2" %} + +{% if save_warm_ic_b or save_fcst_ic %} +{% include "gdas_restartb.yaml.j2" %} +{% endif %} + +{% if DO_ICE %} {% include "gdasice.yaml.j2" %} {% endif %} + +{% if DO_OCEAN %} +{% include "gdasocean.yaml.j2" %} +{% include "gdasocean_analysis.yaml.j2" %} +{% endif %} + +{% if DO_ICE %} {% include "gdaswave.yaml.j2" %} {% endif %} + +# Determine if we will save restart ICs or not +{% set save_warm_ic_a, save_warm_ic_b = ( False, False ) %} + +%{ if ARCH_CYC == cycle_HH and current_cycle != SDATE%} +# Save the warm and forecast-only cycle ICs every ARCH_WARMICFREQ days +{% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %} +{% set save_warm_ic_a = True %} +{% set save_warm_ic_b = True %} +# Save the forecast-only restarts every ARCH_FCSTICFREQ days +{% elif (current_cycle - SDATE).days % ARCH_FCSTICFREQ == 0 %} +{% set save_warm_ic_a = True %} +{% endif %} +{% endif %} + +{% if save_warm_ic_a %} +{% include "gdas_restarta.yaml.j2" %} +{% if DO_WAVE %} {% include "gdaswave_restart.yaml.j2" %} {% endif %} +{% if DO_OCN %} {% include "gdasocean_restart.yaml.j2" %} {% endif %} +{% if DO_ICE %} {% include "gdasice_restart.yaml.j2" %} {% endif %} +{% endif %} # Save warm ICs (A) + +{% if save_warm_ic_b %} {% include "gdas_restartb.yaml.j2" %} {% endif %} diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2 new file mode 100644 index 0000000000..6c9abeb969 --- /dev/null +++ b/parm/archive/master_gfs.yaml.j2 @@ -0,0 +1,54 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +datasets: +{% include "gfsa.yaml.j2" %} +{% include "gfsb.yaml.j2" %} + +{% if ARCH_GAUSSIAN %} +{% include "gfs_flux.yaml.j2" %} +{% include "gfs_netcdfb.yaml.j2" %} +{% include "gfs_pgrb2b.yaml.j2" %} +{% if MODE == "cycled" %} +{% include "gfs_netcdfa.yaml.j2" %} +{% endif %} +{% endif %} + +{% if DO_WAVE %} {% include "gfswave.yaml.j2" %} {% endif %} +{% if DO_AERO %} {% include "chem.yaml.j2" %} {% endif %} +{% if DO_BUFRSND %} {% include "gfs_downstream.yaml.j2" %} {% endif %} + +{% if DO_OCEAN %} +{% include "ocean_6hravg.yaml.j2" %} +{% include "ocean_daily.yaml.j2" %} +{% include "ocean_grib2.yaml.j2" %} +{% include "gfs_flux_1p00.yaml.j2" %} +{% endif %} + +{% if DO_ICE %} +{% include "ice_6hravg.yaml.j2" %} +{% include "ice_grib2.yaml.j2" %} +{% endif %} + +# Determine whether to save the MOS tarball +{% if DO_MOS and cycle_HH == "18" %} +{% if not REALTIME %} +{% include "gfsmos.yaml.j2" %} +{% else %} +{% set td_from_sdate = current_cycle - SDATE %} +{% set td_one_day = "+1D" | to_timedelta %} +{% if td_from_sdate > td_one_day %} +{% include "gfsmos.yaml.j2" %} +{% endif %} +{% endif %} +{% endif %} + +# Determine if we will save restart ICs or not +%{ if ARCH_CYC == cycle_HH and current_cycle != SDATE%} +# Save the forecast-only cycle ICs every ARCH_WARMICFREQ or ARCH_FCSTICFREQ days +{% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %} +{% include "gfs_restarta.yaml.j2" %} +{% elif (current_cycle - SDATE).days % ARCH_FCSTICFREQ == 0 %} +{% include "gfs_restarta.yaml.j2" %} +{% endif %} +{% endif %} diff --git a/parm/archive/ocean_6hravg.yaml.j2 b/parm/archive/ocean_6hravg.yaml.j2 index c46b05cda2..32ffdf28a5 100644 --- a/parm/archive/ocean_6hravg.yaml.j2 +++ b/parm/archive/ocean_6hravg.yaml.j2 @@ -1,9 +1,8 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = to_YMDH %} -{% set head = "gfs.ocean.t" + cycle_HH + "z." %} -name: "OCEAN_6HRAVG" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_6hravg.tar" -mandatory: - {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" - {% endfor %} + ocean_6hravg: + {% set head = "gfs.ocean.t" + cycle_HH + "z." %} + name: "OCEAN_6HRAVG" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_6hravg.tar" + mandatory: + {% for fhr in range(6, FHMAX_GFS + 6, 6) %} + - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/ocean_daily.yaml.j2 b/parm/archive/ocean_daily.yaml.j2 index 2241ce55b5..d1d07a1897 100644 --- a/parm/archive/ocean_daily.yaml.j2 +++ b/parm/archive/ocean_daily.yaml.j2 @@ -1,9 +1,8 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = to_YMDH %} -{% set head = "gfs.ocean.t" + cycle_HH + "z." %} -name: "OCEAN_DAILY" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_daily.tar" -mandatory: - {% for fhr in range(24, FHMAX_GFS + 24, 24) %} - - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}daily.f{{ '%03d' % fhr }}.nc" - {% endfor %} + ocean_daily: + {% set head = "gfs.ocean.t" + cycle_HH + "z." %} + name: "OCEAN_DAILY" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_daily.tar" + mandatory: + {% for fhr in range(24, FHMAX_GFS + 24, 24) %} + - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}daily.f{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/ocean_grib2.yaml.j2 b/parm/archive/ocean_grib2.yaml.j2 index 4ed6c8b115..333fdade8a 100644 --- a/parm/archive/ocean_grib2.yaml.j2 +++ b/parm/archive/ocean_grib2.yaml.j2 @@ -1,19 +1,18 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMDH = to_YMDH %} -{% set head = "gfs.ocean.t" + cycle_HH + "z." %} -name: "OCEAN_GRIB2" -target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_grib2.tar" -mandatory: - {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %} - {% set fhr3 = '%03d' % fhr %} - {% if OCNRES == 500 %} - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" - {% elif OCNRES == 100 %} - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" - {% elif OCNRES == 25 or OCNRES == "025" %} - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" - {% endif %} - {% endfor %} + ocean_grib2: + {% set head = "gfs.ocean.t" + cycle_HH + "z." %} + name: "OCEAN_GRIB2" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_grib2.tar" + mandatory: + {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %} + {% set fhr3 = '%03d' % fhr %} + {% if OCNRES == 500 %} + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" + {% elif OCNRES == 100 %} + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" + {% elif OCNRES == 25 or OCNRES == "025" %} + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" + {% endif %} + {% endfor %} diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index cf51fb521b..756aa5f36b 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -24,7 +24,9 @@ def main(): 'NMEM_ENS', 'DO_CALC_INCREMENT_ENKF_GFS', 'DO_JEDIATMENS', 'lobsdiag_forenkf', 'FHMIN_ENKF', 'FHMAX_ENKF_GFS', 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF', 'ENKF_SPREAD', - 'restart_interval_enkfgdas', 'restart_interval_enkfgfs'] + 'restart_interval_enkfgdas', 'restart_interval_enkfgfs', + 'COM_ATMOS_ANALYSIS_TMPL', 'COM_ATMOS_RESTART_TMPL', + 'COM_ATMOS_HISTORY_TMPL'] archive_dict = AttrDict() for key in keys: diff --git a/sorc/wxflow b/sorc/wxflow index cdc9751680..ae7f5ed38e 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit cdc9751680cb266bd29eaa4821eae919c6c901cc +Subproject commit ae7f5ed38eac8c5d72dc0df7b739a4f9331a9404 diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 2b12f15e9c..1419ece12f 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -93,183 +93,27 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if not os.path.isdir(arch_dict.ROTDIR): raise FileNotFoundError(f"The ROTDIR ({arch_dict.ROTDIR}) does not exist!") - cycle_HH = strftime(arch_dict.current_cycle, "%H") - if arch_dict.RUN == "gdas" or arch_dict.RUN == "gfs": # Copy the cyclone track files and rename the experiments Archive._rename_cyclone_expt(arch_dict) - arch_ics_cycle = arch_dict.ARCH_CYC - arch_dict.assim_freq - - if arch_ics_cycle < 0: - arch_ics_cycle += 24 - - mod = (arch_dict.current_cycle - - arch_dict.SDATE).days % arch_dict.ARCH_WARMICFREQ - - save_warm_ic_a = False - save_warm_ic_b = False - - if arch_dict.current_cycle == arch_dict.SDATE and cycle_HH == arch_dict.ARCH_CYC: - save_warm_ic_a = True - save_warm_ic_b = True - elif mod == 0 and cycle_HH == arch_dict.ARCH_CYC: - save_warm_ic_a = True - save_warm_ic_b = True - - if arch_ics_cycle == 18: - - mod1 = ((arch_dict.current_cycle - - arch_dict.SDATE).days + 1) % arch_dict.ARCH_WARMICFREQ - - if cycle_HH == arch_ics_cycle: - if mod1 == 0: - save_warm_ic_b = True - elif arch_dict.current_cycle == SDATE: - save_warm_ic_b = True - else: - save_warm_ic_b = False - - mod = (arch_dict.current_cycle - - arch_dict.SDATE).days % arch_dict.ARCH_FCSTICFREQ - - save_fcst_ic = False - if mod == 0 or arch_dict.current_cycle == SDATE: - save_fcst_ic = True - if arch_dict.RUN == "gdas": - - datasets = ["gdas"] - - if save_warm_ic_a or save_fcst_ic: - datasets.append("gdas_restarta") - if arch_dict.DO_WAVE: - datasets.append("gdaswave_restart") - if arch_dict.DO_OCN: - datasets.append("gdasocean_restart") - if arch_dict.DO_ICE: - datasets.append("gdasice_restart") - - if save_warm_ic_b or save_fcst_ic: - datasets.append("gdas_restartb") - - if arch_dict.DO_ICE: - datasets.append("gdasice") - - if arch_dict.DO_OCN: - datasets.append("gdasocean") - datasets.append("gdasocean_analysis") - - if arch_dict.DO_WAVE: - datasets.append("gdaswave") - + master_yaml = "master_gdas.yaml.j2" elif arch_dict.RUN == "gfs": - datasets = ["gfsa", "gfsb"] - - if arch_dict.ARCH_GAUSSIAN: - datasets.extend(["gfs_flux", "gfs_netcdfb", "gfs_pgrb2b"]) - - if arch_dict.MODE == "cycled": - datasets.append("gfs_netcdfa") - - if arch_dict.DO_WAVE: - datasets.append("gfswave") - - if arch_dict.DO_OCN: - datasets.extend(["ocean_6hravg", "ocean_daily", "ocean_grib2", "gfs_flux_1p00"]) - - if arch_dict.DO_ICE: - datasets.extend(["ice_6hravg", "ice_grib2"]) - - if arch_dict.DO_AERO: - datasets.append(["aero"]) - - if save_fcst_ic: - datasets.append("gfs_restarta") - - if arch_dict.DO_BUFRSND: - datasets.append("gfs_downstream") - - if arch_dict.DO_MOS: - arch_mos = False - if self.config.REALTIME: - if arch_dict.current_cycle - timedelta(days=1) > arch_dict.SDATE: - arch_mos = True - else: - arch_mos = True - - if arch_mos and cycle_HH == "18": - datasets.append("gfsmos") - + master_yaml = "master_gfs.yaml.j2" elif arch_dict.RUN == "enkfgdas" or arch_dict.RUN == "enkfgfs": - - if arch_dict.ENSGRP == 0: - datasets = ["enkf"] - - else: - - # Determine which members to archive - first_mem = (arch_dict.ENSGRP - 1) * arch_dict.NMEM_EARCGRP + 1 - last_mem = min(arch_dict.NMEM_ENS, - arch_dict.ENSGRP * arch_dict.NMEM_EARCGRP) - mem_list = [f"{mem:03d}" for mem in range(first_mem, last_mem + 1)] - - arch_dict["first_group_mem"] = first_mem - arch_dict["last_group_mem"] = last_mem - - # Create a list of IAU forecast hours from IAUFHRS - if isinstance(arch_dict.IAUFHRS, int): - # First half-cycle or if 3dvar - arch_dict["iaufhrs"] = [arch_dict.IAUFHRS] - else: - arch_dict["iaufhrs"] = [int(fhr) for fhr in arch_dict.IAUFHRS.split(",")] - # Create a dict to define COM template parameters - tmpl_dict = { - 'ROTDIR': self.task_config.ROTDIR, - 'RUN': self.task_config.RUN, - 'YMD': to_YMD(self.task_config.current_cycle), - 'HH': self.task_config.current_cycle.strftime('%H') - } - - tmpl_atm_anl = self.task_config.COM_ATMOS_ANALYSIS_TMPL - tmpl_atm_res = self.task_config.COM_ATMOS_RESTART_TMPL - tmpl_atm_hst = self.task_config.COM_ATMOS_HISTORY_TMPL - - # Construct lists of COM directories to archive data from - arch_dict["COM_ATMOS_ANALYSIS_MEM_list"] = [] - arch_dict["COM_ATMOS_RESTART_MEM_list"] = [] - arch_dict["COM_ATMOS_HISTORY_MEM_list"] = [] - DCB = TemplateConstants.DOLLAR_CURLY_BRACE - for mem in mem_list: - tmpl_dict["MEMDIR"] = "mem" + mem - com_atm_anl = Template.substitute_structure( - tmpl_atm_anl, DCB, tmpl_dict.get) - arch_dict.COM_ATMOS_ANALYSIS_MEM_list.append(com_atm_anl) - - com_atm_res = Template.substitute_structure( - tmpl_atm_res, DCB, tmpl_dict.get) - arch_dict.COM_ATMOS_RESTART_MEM_list.append(com_atm_res) - - com_atm_hst = Template.substitute_structure( - tmpl_atm_hst, DCB, tmpl_dict.get) - arch_dict.COM_ATMOS_HISTORY_MEM_list.append(com_atm_hst) - - # Declare the datasets to archive - datasets = ["enkf_grp"] - - if arch_dict.current_cycle != arch_dict.SDATE: - datasets.extend(["enkf_restarta_grp", "enkf_restartb_grp"]) - + master_yaml = "master_enkf.yaml.j2" elif arch_dict.RUN == "gefs": raise NotImplementedError("Archiving is not yet set up for GEFS runs") - else: raise ValueError(f"Archiving is not enabled for {arch_dict.RUN} runs") + parsed_sets = parse_j2yaml(master_yaml, arch_dict) + atardir_sets = [] - for dataset in datasets: + for dataset in parsed_set.datasets: archive_filename = os.path.join(archive_parm, dataset + ".yaml.j2") atardir_set = parse_j2yaml(archive_filename, arch_dict) @@ -552,3 +396,64 @@ def replace_string_from_to_file(filename_in, filename_out, search_str, replace_s replace_string_from_to_file(in_track_p_file, out_track_p_file, "AVNO", pslot4) return + + @staticmethod + def _create_enkf_com_lists(arch_dict: Dict[str, any]) -> Dict: + + """Creates lists of ENKF member com directories from templates + + Parameters + ---------- + arch_dict : dict + Archiving context dictionary including COM_*_TMPL variables + + Return + ------ + arch_dict : dict + Updated context with lists for all ensemble members to be included + """ + + # Determine which members to archive + first_mem = (arch_dict.ENSGRP - 1) * arch_dict.NMEM_EARCGRP + 1 + last_mem = min(arch_dict.NMEM_ENS, + arch_dict.ENSGRP * arch_dict.NMEM_EARCGRP) + mem_list = [f"{mem:03d}" for mem in range(first_mem, last_mem + 1)] + + arch_dict["first_group_mem"] = first_mem + arch_dict["last_group_mem"] = last_mem + + # Create a list of IAU forecast hours from IAUFHRS + if isinstance(arch_dict.IAUFHRS, int): + # First half-cycle or if 3dvar + arch_dict["iaufhrs"] = [arch_dict.IAUFHRS] + else: + arch_dict["iaufhrs"] = [int(fhr) for fhr in arch_dict.IAUFHRS.split(",")] + # Create a dict to define COM template parameters + tmpl_dict = { + 'ROTDIR': arch_dict.ROTDIR, + 'RUN': arch_dict.RUN, + 'YMD': to_YMD(arch_dict.current_cycle), + 'HH': arch_dict.current_cycle.strftime('%H') + } + + # Get a list of all COM templates (e.g. COM_ATMOS_ANALYSIS_TMPL) + com_tmpl_names = [key for key in arch_dict.keys() + if key.startswith("COM_") and key.endswith("_TMPL")] + + # Go through each template and + for com_tmpl_name in com_tmpl_names: + + com_tmpl = arch_dict[com_tmpl_name] + + # Construct lists of COM directories to archive data from + mem_com_list = [] + for mem in mem_list: + tmpl_dict["MEMDIR"] = "mem" + mem + mem_com_list.append(Template.substitute_structure( + com_tmpl, + TemplateConstants.DOLLAR_CURLY_BRACE, + tmpl_dict.get)) + mem_com_list_name = com_tmpl_name.replace("_TMPL","_MEM_list") + arch_dict[mem_com_list_name] = mem_com_list + + return arch_dict From b412f7f32d2541a5d4b2a03633a1e51b0f5791cd Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 30 Apr 2024 17:33:45 +0000 Subject: [PATCH 054/128] Adjust earc groups by resolution --- parm/config/gfs/config.earc | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/parm/config/gfs/config.earc b/parm/config/gfs/config.earc index de73a93731..1de578e33e 100644 --- a/parm/config/gfs/config.earc +++ b/parm/config/gfs/config.earc @@ -8,7 +8,22 @@ echo "BEGIN: config.earc" # Get task specific resources . $EXPDIR/config.resources earc -export NMEM_EARCGRP=10 +# Set the number of ensemble members to archive per earc job +case "${CASE_ENS}" in + "C48" | "C96") + export NMEM_EARCGRP=80 + ;; + "C192") + export NMEM_EARCGRP=20 + ;; + "C384" | "C768") + export NMEM_EARCGRP=10 + ;; + *) + echo "FATAL ERROR: Unknown ensemble resolution ${CASE_ENS}, ABORT!" + exit 1 + ;; +esac #--starting and ending hours of previous cycles to be removed from rotating directory export RMOLDSTD_ENKF=144 From 668ca3b0c9622ace512e93aee9cc6f3bdd010011 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 30 Apr 2024 17:34:46 +0000 Subject: [PATCH 055/128] Fix template constants in jinja --- sorc/wxflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/wxflow b/sorc/wxflow index ae7f5ed38e..36fd40d513 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit ae7f5ed38eac8c5d72dc0df7b739a4f9331a9404 +Subproject commit 36fd40d5137ed6619011da2c2081dd94b5b58258 From 5f518db411d6ecaa927d9dfdd6c68337efb0c091 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 30 Apr 2024 17:37:41 +0000 Subject: [PATCH 056/128] Rename save_restart variables. --- parm/archive/master_gdas.yaml.j2 | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/parm/archive/master_gdas.yaml.j2 b/parm/archive/master_gdas.yaml.j2 index 349aad12a8..360f374b6e 100644 --- a/parm/archive/master_gdas.yaml.j2 +++ b/parm/archive/master_gdas.yaml.j2 @@ -5,10 +5,6 @@ datasets: {% include "gdas.yaml.j2" %} -{% if save_warm_ic_b or save_fcst_ic %} -{% include "gdas_restartb.yaml.j2" %} -{% endif %} - {% if DO_ICE %} {% include "gdasice.yaml.j2" %} {% endif %} {% if DO_OCEAN %} @@ -19,24 +15,25 @@ datasets: {% if DO_ICE %} {% include "gdaswave.yaml.j2" %} {% endif %} # Determine if we will save restart ICs or not -{% set save_warm_ic_a, save_warm_ic_b = ( False, False ) %} +{% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %} %{ if ARCH_CYC == cycle_HH and current_cycle != SDATE%} # Save the warm and forecast-only cycle ICs every ARCH_WARMICFREQ days {% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %} -{% set save_warm_ic_a = True %} -{% set save_warm_ic_b = True %} +{% set save_warm_start_forecast = True %} +{% set save_warm_start_cycled = True %} # Save the forecast-only restarts every ARCH_FCSTICFREQ days {% elif (current_cycle - SDATE).days % ARCH_FCSTICFREQ == 0 %} -{% set save_warm_ic_a = True %} +{% set save_warm_start_forecast = True %} {% endif %} {% endif %} -{% if save_warm_ic_a %} +{% if save_warm_start_forecast %} {% include "gdas_restarta.yaml.j2" %} {% if DO_WAVE %} {% include "gdaswave_restart.yaml.j2" %} {% endif %} {% if DO_OCN %} {% include "gdasocean_restart.yaml.j2" %} {% endif %} {% if DO_ICE %} {% include "gdasice_restart.yaml.j2" %} {% endif %} -{% endif %} # Save warm ICs (A) +{% endif %} # Save forecast-only restarts -{% if save_warm_ic_b %} {% include "gdas_restartb.yaml.j2" %} {% endif %} +# Save cycled restarts +{% if save_warm_start_cycled %} {% include "gdas_restartb.yaml.j2" %} {% endif %} From 3f16f8b4c625288f476dbba6d3872d187955f2b3 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 30 Apr 2024 17:38:16 +0000 Subject: [PATCH 057/128] Fix atardir datasets --- ush/python/pygfs/task/archive.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 1419ece12f..9f6c39ef1d 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -115,12 +115,10 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str for dataset in parsed_set.datasets: - archive_filename = os.path.join(archive_parm, dataset + ".yaml.j2") - atardir_set = parse_j2yaml(archive_filename, arch_dict) - atardir_set["fileset"] = Archive._create_fileset(atardir_set) - atardir_set["has_rstprod"] = Archive._has_rstprod(atardir_set.fileset) + dataset["fileset"] = Archive._create_fileset(dataset) + dataset["has_rstprod"] = Archive._has_rstprod(dataset.fileset) - atardir_sets.append(atardir_set) + atardir_sets.append(dataset) return arcdir_set, atardir_sets From 235830f33250ed5dc49081a9e7cfd8701e7e714e Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 1 May 2024 18:35:28 +0000 Subject: [PATCH 058/128] Update wxflow/jinja --- sorc/wxflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/wxflow b/sorc/wxflow index 36fd40d513..57afd64ffa 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit 36fd40d5137ed6619011da2c2081dd94b5b58258 +Subproject commit 57afd64ffa7c055621d4fecd288c228918910225 From 18c1271544e902e60ec06b1b3bc1958b6a8cbb03 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 1 May 2024 18:36:21 +0000 Subject: [PATCH 059/128] Fix iaufhrs typo --- parm/archive/enkf_grp.yaml.j2 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index 95d4372f0b..3d0ab8c8c4 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -30,14 +30,14 @@ {% endif %} {% endif %} - {% for iauhr in iaufhrs if iauhr != 6 %} - {% set iauhr = iauhr | int %} + {% for iaufhr in iaufhrs if iaufhr != 6 %} + {% set iaufhr = iaufhr %} {% if DO_CALC_INCREMENT_ENKF_GFS %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iauhr }}.nc" + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iaufhr }}.nc" {% else %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iauhr }}.nc" + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iaufhr }}.nc" {% endif %} - {% endfor %} # iauhr in iaufhrs + {% endfor %} # iaufhr in iaufhrs {% endfor %} # first_group_mem to last_group_mem optional: {% for mem in range(first_group_mem, last_group_mem + 1) %} From c99c8f4062b4dcffee8728a672acd4d49b6907d5 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 1 May 2024 18:37:15 +0000 Subject: [PATCH 060/128] Fix timedelta typo --- parm/archive/gdas_restarta.yaml.j2 | 2 +- parm/archive/gfs_restarta.yaml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 3a4c375db5..157e27a756 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -13,7 +13,7 @@ {% else %} {% set anl_offset = "0H" %} {% endif %} - {% set anl_delta = anl_offset | to_timedelta %} + {% set anl_timedelta = anl_offset | to_timedelta %} {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} {% for itile in range(1,7) %} - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" diff --git a/parm/archive/gfs_restarta.yaml.j2 b/parm/archive/gfs_restarta.yaml.j2 index 1dbe20a178..4bbeec04f4 100644 --- a/parm/archive/gfs_restarta.yaml.j2 +++ b/parm/archive/gfs_restarta.yaml.j2 @@ -9,7 +9,7 @@ {% else %} {% set anl_offset = "0H" %} {% endif %} - {% set anl_delta = anl_offset | to_timedelta %} + {% set anl_timedelta = anl_offset | to_timedelta %} {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} {% for i_tile in range(1, 7) %} - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ i_tile }}.nc" From 853167a8c28dbd6de01165d0f26c648e96b849f4 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 1 May 2024 18:37:58 +0000 Subject: [PATCH 061/128] Apply fixes to master templates --- parm/archive/master_enkf.yaml.j2 | 45 ++++++++++++++++---------------- parm/archive/master_gdas.yaml.j2 | 2 +- parm/archive/master_gfs.yaml.j2 | 2 +- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index 3c3b00e760..325d301b3f 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -8,41 +8,42 @@ datasets: # Set variables/lists needed to parse the enkf templates {% if IAUFHRS is string %} -{% set iaufhrs = (int(fhr) for fhr in IAUFHRS.split(",")) %} +{% set iaufhrs = [] %} +{% for iaufhr in IAUFHRS.split(",") %} +{% do iaufhrs.append(iaufhr | int) %} +{% endfor %} {% else %} {% set iaufhrs = [IAUFHRS] %} {% endif %} -# Declare a dict to get member COM directories from templates -{% set tmpl_dict = {'ROTDIR':ROTDIR, 'RUN':RUN, 'YMD':cycle_YMD, 'HH':cycle_HH} %} - # Declare to-be-filled lists of member COM directories {% set COM_ATMOS_ANALYSIS_MEM_list = [] %} {% set COM_ATMOS_RESTART_MEM_list = [] %} {% set COM_ATMOS_HISTORY_MEM_list = [] %} -{% set first_mem = (ENSGRP - 1) * NMEM_EARCGRP + 1 %} -{% set last_mem = [ ENSGRP * NMEM_EARCGRP, NMEM_ENS ] | min %} +{% set first_group_mem = (ENSGRP - 1) * NMEM_EARCGRP + 1 %} +{% set last_group_mem = [ ENSGRP * NMEM_EARCGRP, NMEM_ENS ] | min %} -{% for mem in range(first_mem, last_mem + 1) %} +{% for mem in range(first_group_mem, last_group_mem + 1) %} -{% set tmpl_dict['MEMDIR'] = "mem" + '%03d' % mem %} +# Declare a dict to get member COM directories from templates +{% set tmpl_dict = {'ROTDIR':ROTDIR, 'RUN':RUN, 'YMD':cycle_YMD, 'HH':cycle_HH, 'MEMDIR':"mem" + '%03d' % mem} %} # Populate the member COM directories from the template -{{ COM_ATMOS_ANALYSIS_MEM_list.append( - template_substitute_structure(COM_ATMOS_ANALYSIS_TMPL, - DOLLAR_CURLY_BRACE, - tmpl_dict.get)) }} - -{{ COM_ATMOS_RESTART_MEM_list.append( - template_substitute_structure(COM_ATMOS_RESTART_TMPL, - DOLLAR_CURLY_BRACE, - tmpl_dict.get)) }} - -{{ COM_ATMOS_HISTORY_MEM_list.append( - template_substitute_structure(COM_ATMOS_HISTORY_TMPL, - DOLLAR_CURLY_BRACE, - tmpl_dict.get)) }} +{% do COM_ATMOS_ANALYSIS_MEM_list.append( + COM_ATMOS_ANALYSIS_TMPL | template_substitute_structure(DOLLAR_CURLY_BRACE, + tmpl_dict.get)) +%} + +{% do COM_ATMOS_HISTORY_MEM_list.append( + COM_ATMOS_HISTORY_TMPL | template_substitute_structure(DOLLAR_CURLY_BRACE, + tmpl_dict.get)) +%} + +{% do COM_ATMOS_RESTART_MEM_list.append( + COM_ATMOS_RESTART_TMPL | template_substitute_structure(DOLLAR_CURLY_BRACE, + tmpl_dict.get)) +%} {% endfor %} diff --git a/parm/archive/master_gdas.yaml.j2 b/parm/archive/master_gdas.yaml.j2 index 360f374b6e..566c563083 100644 --- a/parm/archive/master_gdas.yaml.j2 +++ b/parm/archive/master_gdas.yaml.j2 @@ -17,7 +17,7 @@ datasets: # Determine if we will save restart ICs or not {% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %} -%{ if ARCH_CYC == cycle_HH and current_cycle != SDATE%} +{% if ARCH_CYC == cycle_HH | int and current_cycle != SDATE%} # Save the warm and forecast-only cycle ICs every ARCH_WARMICFREQ days {% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %} {% set save_warm_start_forecast = True %} diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2 index 6c9abeb969..8e0d098af4 100644 --- a/parm/archive/master_gfs.yaml.j2 +++ b/parm/archive/master_gfs.yaml.j2 @@ -44,7 +44,7 @@ datasets: {% endif %} # Determine if we will save restart ICs or not -%{ if ARCH_CYC == cycle_HH and current_cycle != SDATE%} +{% if ARCH_CYC == cycle_HH | int and current_cycle != SDATE %} # Save the forecast-only cycle ICs every ARCH_WARMICFREQ or ARCH_FCSTICFREQ days {% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %} {% include "gfs_restarta.yaml.j2" %} From f4fc98f2e17fcad9da79a9509e233a1bfb515992 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 1 May 2024 18:40:46 +0000 Subject: [PATCH 062/128] Fix dataset parsing --- ush/python/pygfs/task/archive.py | 65 +------------------------------- 1 file changed, 2 insertions(+), 63 deletions(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 9f6c39ef1d..8ee7e1799b 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -109,11 +109,11 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str else: raise ValueError(f"Archiving is not enabled for {arch_dict.RUN} runs") - parsed_sets = parse_j2yaml(master_yaml, arch_dict) + parsed_sets = parse_j2yaml(os.path.join(archive_parm,master_yaml), arch_dict) atardir_sets = [] - for dataset in parsed_set.datasets: + for dataset in parsed_sets.datasets.values(): dataset["fileset"] = Archive._create_fileset(dataset) dataset["has_rstprod"] = Archive._has_rstprod(dataset.fileset) @@ -394,64 +394,3 @@ def replace_string_from_to_file(filename_in, filename_out, search_str, replace_s replace_string_from_to_file(in_track_p_file, out_track_p_file, "AVNO", pslot4) return - - @staticmethod - def _create_enkf_com_lists(arch_dict: Dict[str, any]) -> Dict: - - """Creates lists of ENKF member com directories from templates - - Parameters - ---------- - arch_dict : dict - Archiving context dictionary including COM_*_TMPL variables - - Return - ------ - arch_dict : dict - Updated context with lists for all ensemble members to be included - """ - - # Determine which members to archive - first_mem = (arch_dict.ENSGRP - 1) * arch_dict.NMEM_EARCGRP + 1 - last_mem = min(arch_dict.NMEM_ENS, - arch_dict.ENSGRP * arch_dict.NMEM_EARCGRP) - mem_list = [f"{mem:03d}" for mem in range(first_mem, last_mem + 1)] - - arch_dict["first_group_mem"] = first_mem - arch_dict["last_group_mem"] = last_mem - - # Create a list of IAU forecast hours from IAUFHRS - if isinstance(arch_dict.IAUFHRS, int): - # First half-cycle or if 3dvar - arch_dict["iaufhrs"] = [arch_dict.IAUFHRS] - else: - arch_dict["iaufhrs"] = [int(fhr) for fhr in arch_dict.IAUFHRS.split(",")] - # Create a dict to define COM template parameters - tmpl_dict = { - 'ROTDIR': arch_dict.ROTDIR, - 'RUN': arch_dict.RUN, - 'YMD': to_YMD(arch_dict.current_cycle), - 'HH': arch_dict.current_cycle.strftime('%H') - } - - # Get a list of all COM templates (e.g. COM_ATMOS_ANALYSIS_TMPL) - com_tmpl_names = [key for key in arch_dict.keys() - if key.startswith("COM_") and key.endswith("_TMPL")] - - # Go through each template and - for com_tmpl_name in com_tmpl_names: - - com_tmpl = arch_dict[com_tmpl_name] - - # Construct lists of COM directories to archive data from - mem_com_list = [] - for mem in mem_list: - tmpl_dict["MEMDIR"] = "mem" + mem - mem_com_list.append(Template.substitute_structure( - com_tmpl, - TemplateConstants.DOLLAR_CURLY_BRACE, - tmpl_dict.get)) - mem_com_list_name = com_tmpl_name.replace("_TMPL","_MEM_list") - arch_dict[mem_com_list_name] = mem_com_list - - return arch_dict From f788e6f76539815fab46107d32e17b8e06ad737b Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 1 May 2024 18:42:15 +0000 Subject: [PATCH 063/128] Add FATAL ERROR to fatal error messages --- ush/python/pygfs/task/archive.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 8ee7e1799b..12e7c1b6e2 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -91,7 +91,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str return arcdir_set, [] if not os.path.isdir(arch_dict.ROTDIR): - raise FileNotFoundError(f"The ROTDIR ({arch_dict.ROTDIR}) does not exist!") + raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({arch_dict.ROTDIR}) does not exist!") if arch_dict.RUN == "gdas" or arch_dict.RUN == "gfs": @@ -105,9 +105,9 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str elif arch_dict.RUN == "enkfgdas" or arch_dict.RUN == "enkfgfs": master_yaml = "master_enkf.yaml.j2" elif arch_dict.RUN == "gefs": - raise NotImplementedError("Archiving is not yet set up for GEFS runs") + raise NotImplementedError("FATAL ERROR: Archiving is not yet set up for GEFS runs") else: - raise ValueError(f"Archiving is not enabled for {arch_dict.RUN} runs") + raise ValueError(f"FATAL ERROR: Archiving is not enabled for {arch_dict.RUN} runs") parsed_sets = parse_j2yaml(os.path.join(archive_parm,master_yaml), arch_dict) @@ -164,7 +164,7 @@ def execute(self, arcdir_set: Dict[str, Any], atardir_sets: List[Dict[str, Any]] # Regardless of exception type, attempt to remove the target except Exception: rm_cmd(atardir_set.target) - raise RuntimeError(f"Failed to create restricted archive {atardir_set.target}, deleting!") + raise RuntimeError(f"FATAL ERROR: Failed to create restricted archive {atardir_set.target}, deleting!") self._protect_rstprod(atardir_set) @@ -195,7 +195,7 @@ def _create_fileset(atardir_set: Dict[str, Any]) -> List: for item in atardir_set.mandatory: glob_set = glob.glob(item) if len(glob_set) == 0: - raise FileNotFoundError(f"Mandatory file, directory, or glob {item} not found!") + raise FileNotFoundError(f"FATAL ERROR: Mandatory file, directory, or glob {item} not found!") for entry in glob_set: fileset.append(entry) @@ -255,7 +255,7 @@ def _protect_rstprod(self, atardir_set: Dict[str, any]) -> None: chmod_cmd = os.fchmod rm_cmd = rm_p else: - raise KeyError(f"Invalid archiving command given: {self.tar_cmd}") + raise KeyError(f"FATAL ERROR: Invalid archiving command given: {self.tar_cmd}") try: if self.tar_cmd == "htar": @@ -272,7 +272,7 @@ def _protect_rstprod(self, atardir_set: Dict[str, any]) -> None: else: rm_p(atardir_set.target) finally: - raise RuntimeError(f"Failed to protect {atardir_set.target}!\n" + raise RuntimeError(f"FATAL ERROR: Failed to protect {atardir_set.target}!\n" f"Please verify that it has been deleted!!") @logit(logger) From 13acc222af846059d701c389b0e5f2e9a4e987b2 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 1 May 2024 18:45:13 +0000 Subject: [PATCH 064/128] Fix whitespace --- ush/python/pygfs/task/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 12e7c1b6e2..10122e9255 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -109,7 +109,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str else: raise ValueError(f"FATAL ERROR: Archiving is not enabled for {arch_dict.RUN} runs") - parsed_sets = parse_j2yaml(os.path.join(archive_parm,master_yaml), arch_dict) + parsed_sets = parse_j2yaml(os.path.join(archive_parm, master_yaml), arch_dict) atardir_sets = [] From c86964978bfa72f6557059911fc5a9b05ce9ff3e Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 2 May 2024 13:10:17 +0000 Subject: [PATCH 065/128] Update gdas component head names --- parm/archive/gdasice.yaml.j2 | 4 ++-- parm/archive/gdasice_restart.yaml.j2 | 2 +- parm/archive/gdasocean.yaml.j2 | 2 +- parm/archive/gdasocean_analysis.yaml.j2 | 2 +- parm/archive/gdasocean_restart.yaml.j2 | 2 +- parm/archive/gdaswave.yaml.j2 | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/parm/archive/gdasice.yaml.j2 b/parm/archive/gdasice.yaml.j2 index 09b932e328..c75b25d12c 100644 --- a/parm/archive/gdasice.yaml.j2 +++ b/parm/archive/gdasice.yaml.j2 @@ -1,8 +1,8 @@ gdasice: - {% set head = "gdas.t" + cycle_HH + "z." %} + {% set head = "gdas.ice.t" + cycle_HH + "z." %} name: "GDASICE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar" mandatory: # TODO explicitly name the files in history to archive - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}*' - - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/ice_in' + # - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/ice_in' diff --git a/parm/archive/gdasice_restart.yaml.j2 b/parm/archive/gdasice_restart.yaml.j2 index 112e8c0bd8..a0b809df87 100644 --- a/parm/archive/gdasice_restart.yaml.j2 +++ b/parm/archive/gdasice_restart.yaml.j2 @@ -1,5 +1,5 @@ gdasice_restart: - {% set head = "gdas.t" + cycle_HH + "z." %} + {% set head = "gdas.ice.t" + cycle_HH + "z." %} name: "GDASICE_RESTART" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice_restart.tar" mandatory: diff --git a/parm/archive/gdasocean.yaml.j2 b/parm/archive/gdasocean.yaml.j2 index d0bb7d8606..1a8ffb8a52 100644 --- a/parm/archive/gdasocean.yaml.j2 +++ b/parm/archive/gdasocean.yaml.j2 @@ -1,5 +1,5 @@ gdasocean: - {% set head = "gdas.t" + cycle_HH + "z." %} + {% set head = "gdas.ocean.t" + cycle_HH + "z." %} name: "GDASOCEAN" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar" mandatory: diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index b27d8c5482..4b7c21b16a 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -1,5 +1,5 @@ gdasocean_analysis: - {% set head = "gdas.t" + cycle_HH + "z." %} + {% set head = "gdas.ocean.t" + cycle_HH + "z." %} name: "GDASOCEAN_ANALYSIS" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar" mandatory: diff --git a/parm/archive/gdasocean_restart.yaml.j2 b/parm/archive/gdasocean_restart.yaml.j2 index bd0d241f1a..010ffafc14 100644 --- a/parm/archive/gdasocean_restart.yaml.j2 +++ b/parm/archive/gdasocean_restart.yaml.j2 @@ -1,5 +1,5 @@ gdasocean_restart: - {% set head = "gdas.t" + cycle_HH + "z." %} + {% set head = "gdas.ocean.t" + cycle_HH + "z." %} name: "GDASOCEAN_RESTART" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_restart.tar" mandatory: diff --git a/parm/archive/gdaswave.yaml.j2 b/parm/archive/gdaswave.yaml.j2 index c88875cb34..81860e0eef 100644 --- a/parm/archive/gdaswave.yaml.j2 +++ b/parm/archive/gdaswave.yaml.j2 @@ -1,5 +1,5 @@ gdaswave: - {% set head = "gdaswave.t" + cycle_HH + "z." %} + {% set head = "gdas.wave.t" + cycle_HH + "z." %} name: "GDASWAVE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave.tar" mandatory: From 4fa45b31b9caa35ed60f7f400098d79f1ad0c427 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 2 May 2024 13:11:37 +0000 Subject: [PATCH 066/128] Flat-layer the master templates, fix wave archiving --- parm/archive/master_gdas.yaml.j2 | 28 ++++++++++++++++++++++------ parm/archive/master_gfs.yaml.j2 | 14 +++++++++++--- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/parm/archive/master_gdas.yaml.j2 b/parm/archive/master_gdas.yaml.j2 index 566c563083..52e7b87af5 100644 --- a/parm/archive/master_gdas.yaml.j2 +++ b/parm/archive/master_gdas.yaml.j2 @@ -5,14 +5,18 @@ datasets: {% include "gdas.yaml.j2" %} -{% if DO_ICE %} {% include "gdasice.yaml.j2" %} {% endif %} +{% if DO_ICE %} +{% include "gdasice.yaml.j2" %} +{% endif %} {% if DO_OCEAN %} {% include "gdasocean.yaml.j2" %} {% include "gdasocean_analysis.yaml.j2" %} {% endif %} -{% if DO_ICE %} {% include "gdaswave.yaml.j2" %} {% endif %} +{% if DO_WAVE %} +{% include "gdaswave.yaml.j2" %} +{% endif %} # Determine if we will save restart ICs or not {% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %} @@ -30,10 +34,22 @@ datasets: {% if save_warm_start_forecast %} {% include "gdas_restarta.yaml.j2" %} -{% if DO_WAVE %} {% include "gdaswave_restart.yaml.j2" %} {% endif %} -{% if DO_OCN %} {% include "gdasocean_restart.yaml.j2" %} {% endif %} -{% if DO_ICE %} {% include "gdasice_restart.yaml.j2" %} {% endif %} + +{% if DO_WAVE %} +{% include "gdaswave_restart.yaml.j2" %} +{% endif %} + +{% if DO_OCN %} +{% include "gdasocean_restart.yaml.j2" %} +{% endif %} + +{% if DO_ICE %} +{% include "gdasice_restart.yaml.j2" %} +{% endif %} + {% endif %} # Save forecast-only restarts # Save cycled restarts -{% if save_warm_start_cycled %} {% include "gdas_restartb.yaml.j2" %} {% endif %} +{% if save_warm_start_cycled %} +{% include "gdas_restartb.yaml.j2" %} +{% endif %} diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2 index 8e0d098af4..a17928d684 100644 --- a/parm/archive/master_gfs.yaml.j2 +++ b/parm/archive/master_gfs.yaml.j2 @@ -14,9 +14,17 @@ datasets: {% endif %} {% endif %} -{% if DO_WAVE %} {% include "gfswave.yaml.j2" %} {% endif %} -{% if DO_AERO %} {% include "chem.yaml.j2" %} {% endif %} -{% if DO_BUFRSND %} {% include "gfs_downstream.yaml.j2" %} {% endif %} +{% if DO_WAVE %} +{% include "gfswave.yaml.j2" %} +{% endif %} + +{% if DO_AERO %} +{% include "chem.yaml.j2" %} +{% endif %} + +{% if DO_BUFRSND %} +{% include "gfs_downstream.yaml.j2" %} +{% endif %} {% if DO_OCEAN %} {% include "ocean_6hravg.yaml.j2" %} From eda06eda639a7b4a7de0f3c31d3be04732fa351d Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 2 May 2024 17:33:11 +0000 Subject: [PATCH 067/128] Correct CICE namelist. --- parm/archive/gdasice.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/archive/gdasice.yaml.j2 b/parm/archive/gdasice.yaml.j2 index c75b25d12c..0eca9500ed 100644 --- a/parm/archive/gdasice.yaml.j2 +++ b/parm/archive/gdasice.yaml.j2 @@ -5,4 +5,4 @@ mandatory: # TODO explicitly name the files in history to archive - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}*' - # - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/ice_in' + - '{{ COM_CONF | relpath(ROTDIR) }}/ufs.ice_in' From 51b7838fc5365bb32641ad55161913295d64c24d Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 2 May 2024 17:33:46 +0000 Subject: [PATCH 068/128] Add quick fix for wgrib. --- parm/config/gfs/config.base | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 993c5770ec..bb5ac2e792 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -110,6 +110,10 @@ export CHGRP_CMD="@CHGRP_CMD@" export NCDUMP="${NETCDF:-${netcdf_c_ROOT:-}}/bin/ncdump" export NCLEN="${HOMEgfs}/ush/getncdimlen" +# Stop gap fix for wgrib with spack-stack 1.6.0 +# TODO Remove this when spack-stack issue #1097 is resolved +export WGRIB="${WGRIB:-${grib_util_ROOT:-}/wgrib}" + # Machine environment, jobs, and other utility scripts export BASE_ENV="${HOMEgfs}/env" export BASE_JOB="${HOMEgfs}/jobs/rocoto" From 95d88a992d7634bcf4aa8b85f35f66db72c040e2 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 3 May 2024 17:40:42 +0000 Subject: [PATCH 069/128] Replace bash templates in Jinja templates. --- parm/archive/master_enkf.yaml.j2 | 39 ++++++++++++++++++++------------ sorc/wxflow | 2 +- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index 325d301b3f..82bb1a2a07 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -24,26 +24,35 @@ datasets: {% set first_group_mem = (ENSGRP - 1) * NMEM_EARCGRP + 1 %} {% set last_group_mem = [ ENSGRP * NMEM_EARCGRP, NMEM_ENS ] | min %} +# Construct member COM directories {% for mem in range(first_group_mem, last_group_mem + 1) %} -# Declare a dict to get member COM directories from templates -{% set tmpl_dict = {'ROTDIR':ROTDIR, 'RUN':RUN, 'YMD':cycle_YMD, 'HH':cycle_HH, 'MEMDIR':"mem" + '%03d' % mem} %} +# Declare a dict of search and replace terms to run on each template +{% set tmpl_dict = {'ROTDIR':ROTDIR, + 'RUN':RUN, + 'YMD':cycle_YMD, + 'HH':cycle_HH, + 'MEMDIR':"mem" + '%03d' % mem} %} -# Populate the member COM directories from the template -{% do COM_ATMOS_ANALYSIS_MEM_list.append( - COM_ATMOS_ANALYSIS_TMPL | template_substitute_structure(DOLLAR_CURLY_BRACE, - tmpl_dict.get)) -%} +# Replace template variables with tmpl_dict, one key at a time +# This must be done in a namespace to overcome jinja scoping +# Variables set inside of a for loop are lost at the end of the loop +# unless they are part of a namespace +{% set com_ns = namespace(COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL, + COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL, + COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL) %} -{% do COM_ATMOS_HISTORY_MEM_list.append( - COM_ATMOS_HISTORY_TMPL | template_substitute_structure(DOLLAR_CURLY_BRACE, - tmpl_dict.get)) -%} +{% for key in tmpl_dict.keys() %} +{% set search_term = '${' + key + '}' %} +{% set replace_term = tmpl_dict[key] %} +{% set com_ns.COM_ATMOS_ANALYSIS_MEM = com_ns.COM_ATMOS_ANALYSIS_MEM.replace(search_term, replace_term) %} +{% set com_ns.COM_ATMOS_HISTORY_MEM = com_ns.COM_ATMOS_HISTORY_MEM.replace(search_term, replace_term) %} +{% set com_ns.COM_ATMOS_RESTART_MEM = com_ns.COM_ATMOS_RESTART_MEM.replace(search_term, replace_term) %} +{% endfor %} -{% do COM_ATMOS_RESTART_MEM_list.append( - COM_ATMOS_RESTART_TMPL | template_substitute_structure(DOLLAR_CURLY_BRACE, - tmpl_dict.get)) -%} +{% do COM_ATMOS_ANALYSIS_MEM_list.append(com_ns.COM_ATMOS_ANALYSIS_MEM)%} +{% do COM_ATMOS_HISTORY_MEM_list.append(com_ns.COM_ATMOS_HISTORY_MEM)%} +{% do COM_ATMOS_RESTART_MEM_list.append(com_ns.COM_ATMOS_RESTART_MEM)%} {% endfor %} diff --git a/sorc/wxflow b/sorc/wxflow index 57afd64ffa..71f6b10f76 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit 57afd64ffa7c055621d4fecd288c228918910225 +Subproject commit 71f6b10f76a440993580027ba1183d61277d1299 From 32010d09ca62c35f8af98a489c85f8a3d6766cb0 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 3 May 2024 17:44:20 +0000 Subject: [PATCH 070/128] First attempt at reenabling aerosols --- parm/config/gfs/config.base | 11 +++++++---- parm/config/gfs/config.fcst | 8 ++++++++ scripts/exglobal_archive.py | 3 ++- workflow/applications/applications.py | 8 ++++++++ workflow/applications/gfs_cycled.py | 9 ++++++--- workflow/applications/gfs_forecast_only.py | 4 +++- workflow/rocoto/gfs_tasks.py | 4 ++-- 7 files changed, 36 insertions(+), 11 deletions(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index bb5ac2e792..53f5e80246 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -167,9 +167,6 @@ export APP=@APP@ shopt -s extglob # Adjust APP based on RUN case "${RUN}" in - gfs) # Turn off aerosols - APP="${APP/%A}" - ;; enkf*) # Turn off aerosols and waves APP="${APP/%+([WA])}" ;; @@ -185,6 +182,8 @@ export DO_WAVE="NO" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" +export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both +export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both export DOBNDPNT_WAVE="NO" export FRAC_GRID=".true." @@ -224,6 +223,8 @@ case "${APP}" in ;; ATMA) export DO_AERO="YES" + export AERO_ANL_CDUMP="gdas" + export AERO_FCST_CDUMP="gdas" ;; ATMW) export DO_COUPLED="YES" @@ -242,6 +243,8 @@ case "${APP}" in if [[ "${APP}" =~ A$ ]]; then export DO_AERO="YES" + export AERO_ANL_CDUMP="gdas" + export AERO_FCST_CDUMP="gdas" fi if [[ "${APP}" =~ ^S2SW ]]; then @@ -395,7 +398,7 @@ fi export restart_interval_enkfgfs=${restart_interval_enkfgdas} -if [[ "${DOIAU}" == "YES" && "${DOHYBVAR}" == "YES" ]]; then +if [[ "${DOIAU}" == "YES" ]]; then export restart_interval_gdas="3" else export restart_interval_gdas="6" diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index 8e121634a7..36c900c965 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -12,6 +12,14 @@ case ${WAVE_CDUMP} in both | "${CDUMP/enkf}" ) ;; # Don't change *) DO_WAVE="NO" ;; # Turn waves off esac + if [[ "${AERO_FCST_CDUMP}" == "both" || "AERO_FCST_CDUMP" == "$CDUMP" ]]; then + string="${string} --gocart" + +# Turn off waves if not used for this CDUMP +case ${AERO_FCST_CDUMP} in + both | "${CDUMP/enkf}" ) ;; # Don't change + *) DO_AERO="NO" ;; # Turn waves off +esac # Source model specific information that is resolution dependent string="--fv3 ${CASE}" diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index c21764de2a..74d7d73688 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -28,7 +28,8 @@ def main(): 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC', 'DOIAU', 'OCNRES', 'ICERES', 'NUM_SND_COLLECTIVES', 'FHOUT_WAV', 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', - 'restart_interval_gdas', 'restart_interval_gfs'] + 'restart_interval_gdas', 'restart_interval_gfs', + 'AERO_ANL_CDUMP', 'AERO_FCST_CDUMP'] archive_dict = AttrDict() for key in keys: diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 3a8d23f744..adfab16496 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -77,6 +77,14 @@ def __init__(self, conf: Configuration) -> None: elif wave_cdump in ['gfs', 'gdas']: self.wave_cdumps = [wave_cdump] + self.aero_anl_cdumps = None + if self.do_aero: + aero_anl_cdump = _base.get('AERO_ANL_CDUMP', 'BOTH').lower() + if aero_anl_cdump in ['both']: + self.aero_anl_cdumps = ['gfs', 'gdas'] + elif aero_anl_cdump in ['gfs', 'gdas']: + self.aero_anl_cdumps = [aero_anl_cdump] + def _init_finalize(self, conf: Configuration): print("Finalizing initialize") diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index c2a6a32f02..0f929adb68 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -150,9 +150,6 @@ def get_task_names(self): gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] - if self.do_aero: - gdas_gfs_common_tasks_before_fcst += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] - if self.do_jedisnowda: gdas_gfs_common_tasks_before_fcst += ['prepsnowobs', 'snowanl'] @@ -179,6 +176,9 @@ def get_task_names(self): if self.do_wave and 'gdas' in self.wave_cdumps: gdas_tasks += wave_prep_tasks + if self.do_aero and 'gdas' in self.aero_anl_cdumps: + gdas_tasks_before_fcst += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gdas_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] if self.do_upp: @@ -213,6 +213,9 @@ def get_task_names(self): if self.do_wave and 'gfs' in self.wave_cdumps: gfs_tasks += wave_prep_tasks + if self.do_aero and 'gfs' in self.aero_anl_cdumps: + gfs_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gfs_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] if self.do_ocean: diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 0a9648ee65..b849b50c5c 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -87,7 +87,9 @@ def get_task_names(self): tasks = ['stage_ic'] if self.do_aero: - tasks += ['aerosol_init'] + aero_fcst_cdump = _base.get('AERO_FCST_CDUMP', 'BOTH').lower() + if self._base['CDUMP'] in aero_fcst_cdump or aero_fcst_cdump == "both": + tasks += ['aerosol_init'] if self.do_wave: tasks += ['waveinit'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index bba7bac3dd..d2bc08c444 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -829,7 +829,7 @@ def _fcst_forecast_only(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}{wave_job}'} dependencies.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_aero: + if self.app_config.do_aero and self.cdump in self.app_config.aero_fcst_cdumps: # Calculate offset based on CDUMP = gfs | gdas interval = None if self.cdump in ['gfs']: @@ -873,7 +873,7 @@ def _fcst_cycled(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}ocnanalpost'} dependencies.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_aero: + if self.app_config.do_aero and self.cdump in self.app_config.aero_anl_cdumps: dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlfinal'} dependencies.append(rocoto.add_dependency(dep_dict)) From 356be5ea59206fae0dd1c00cffe96a40d7870990 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 3 May 2024 17:45:07 +0000 Subject: [PATCH 071/128] Fix snow diag tarring --- ush/python/pygfs/task/analysis.py | 1 + ush/python/pygfs/task/snow_analysis.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 078e013e7f..7e7412c885 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -348,6 +348,7 @@ def tgz_diags(statfile: str, diagdir: str) -> None: # get list of diag files to put in tarball diags = glob.glob(os.path.join(diagdir, 'diags', 'diag*nc')) + diags.extend(glob.glob(os.path.join(diagdir, 'diags', 'diag*nc4'))) logger.info(f"Compressing {len(diags)} diag files to {statfile}") diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 24234bc82d..c149f140b6 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -206,7 +206,7 @@ def prepare_IMS(self) -> None: # Execute imspy to create the IMS obs data in IODA format logger.info("Create IMS obs data in IODA format") - output_file = f"ims_snow_{to_YMDH(localconf.current_cycle)}.nc" + output_file = f"ims_snow_{to_YMDH(localconf.current_cycle)}.nc4" if os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"): rm_p(output_file) @@ -333,7 +333,7 @@ def finalize(self) -> None: Instance of the SnowAnalysis object """ - logger.info("Create diagnostic tarball of diag*.nc files") + logger.info("Create diagnostic tarball of diag*.nc4 files") statfile = os.path.join(self.task_config.COM_SNOW_ANALYSIS, f"{self.task_config.APREFIX}snowstat.tgz") self.tgz_diags(statfile, self.task_config.DATA) From 930d013c218305d7863f7742daf50adb54a6a824 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 3 May 2024 17:47:02 +0000 Subject: [PATCH 072/128] Fix build_all comment --- sorc/build_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 0797092a37..e3040fc0fa 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -183,7 +183,7 @@ echo "Building ${build_list}" # Go through all builds and adjust CPU counts up if possible if [[ ${requested_cpus} -lt ${_build_job_max} && ${big_jobs} -gt 0 ]]; then - # Add cores to the gdas, ufs, and gsi build jobs + # Add cores to the gdas and ufs build jobs extra_cores=$(( _build_job_max - requested_cpus )) extra_cores=$(( extra_cores / big_jobs )) for build in "${!build_jobs[@]}"; do From d5244830c1a2546a31b419e7cc0f023efa37c1ce Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 3 May 2024 17:47:49 +0000 Subject: [PATCH 073/128] Fix cice restart variable name --- ush/forecast_det.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/forecast_det.sh b/ush/forecast_det.sh index de2a47c921..b87218fb58 100755 --- a/ush/forecast_det.sh +++ b/ush/forecast_det.sh @@ -92,7 +92,7 @@ UFS_det(){ # Check for CICE6 restart availability if [[ "${cplice}" == ".true." ]]; then if [[ ! -f "${DATArestart}/CICE_RESTART/cice_model.res.${rdate:0:4}-${rdate:4:2}-${rdate:6:2}-${seconds}.nc" ]]; then - cice_rst_ok="NO" + cice6_rst_ok="NO" fi fi From b79e4810c46a40121bea4a11845c78ebfab9adb1 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 3 May 2024 17:54:12 +0000 Subject: [PATCH 074/128] Transfer gfs config fixes to gefs configs --- parm/config/gefs/config.base | 13 +++++++++++++ parm/config/gefs/config.fcst | 6 ++++++ parm/config/gfs/config.fcst | 2 -- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index b113a6c90e..dfe7341f66 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -81,6 +81,10 @@ export CHGRP_CMD="@CHGRP_CMD@" export NCDUMP="${NETCDF:-${netcdf_c_ROOT:-}}/bin/ncdump" export NCLEN="${HOMEgfs}/ush/getncdimlen" +# Stop gap fix for wgrib with spack-stack 1.6.0 +# TODO Remove this when spack-stack issue #1097 is resolved +export WGRIB="${WGRIB:-${grib_util_ROOT:-}/wgrib}" + # Machine environment, jobs, and other utility scripts export BASE_ENV="${HOMEgfs}/env" export BASE_JOB="${HOMEgfs}/jobs/rocoto" @@ -134,6 +138,8 @@ export DO_WAVE="NO" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" +export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both +export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both export DOBNDPNT_WAVE="NO" # The GEFS buoys file does not currently have any boundary points export FRAC_GRID=".true." @@ -179,6 +185,8 @@ case "${APP}" in ;; ATMA) export DO_AERO="YES" + export AERO_ANL_CDUMP="gdas" + export AERO_FCST_CDUMP="gdas" ;; ATMW) export DO_COUPLED="YES" @@ -197,6 +205,8 @@ case "${APP}" in if [[ "${APP}" =~ A$ ]]; then export DO_AERO="YES" + export AERO_ANL_CDUMP="gdas" + export AERO_FCST_CDUMP="gdas" fi if [[ "${APP}" =~ ^S2SW ]]; then @@ -319,4 +329,7 @@ export ARCH_FCSTICFREQ=1 # Archive frequency in days for gdas and gfs foreca export DELETE_COM_IN_ARCHIVE_JOB="YES" # NO=retain ROTDIR. YES default in arch.sh and earc.sh. +# Number of regional collectives to create soundings for +export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} + echo "END: config.base" diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index 5c592556c8..4d586821fe 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -13,6 +13,12 @@ case ${WAVE_CDUMP} in *) DO_WAVE="NO" ;; # Turn waves off esac +# Turn off waves if not used for this CDUMP +case ${AERO_FCST_CDUMP} in + both | "${CDUMP/enkf}" ) ;; # Don't change + *) DO_AERO="NO" ;; # Turn waves off +esac + # Source model specific information that is resolution dependent string="--fv3 ${CASE}" [[ "${DO_OCN}" == "YES" ]] && string="${string} --mom6 ${OCNRES}" diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index d3d767a683..179b3f59ec 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -12,8 +12,6 @@ case ${WAVE_CDUMP} in both | "${CDUMP/enkf}" ) ;; # Don't change *) DO_WAVE="NO" ;; # Turn waves off esac - if [[ "${AERO_FCST_CDUMP}" == "both" || "AERO_FCST_CDUMP" == "$CDUMP" ]]; then - string="${string} --gocart" # Turn off waves if not used for this CDUMP case ${AERO_FCST_CDUMP} in From 43d1aeda83ec396d58d006b9ec28aed6ec03e3b8 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 3 May 2024 18:30:34 +0000 Subject: [PATCH 075/128] Fix gdas_tasks --- workflow/applications/gfs_cycled.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 0f929adb68..25afb817da 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -177,7 +177,7 @@ def get_task_names(self): gdas_tasks += wave_prep_tasks if self.do_aero and 'gdas' in self.aero_anl_cdumps: - gdas_tasks_before_fcst += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gdas_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] gdas_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] From 23d6563f4ce701f28b33e91c91e06ff2ad5b7cbb Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 3 May 2024 18:58:24 +0000 Subject: [PATCH 076/128] Fix aerosol comment --- parm/config/gfs/config.fcst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index 179b3f59ec..7caa208f27 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -13,7 +13,7 @@ case ${WAVE_CDUMP} in *) DO_WAVE="NO" ;; # Turn waves off esac -# Turn off waves if not used for this CDUMP +# Turn off aerosols if not used for this CDUMP case ${AERO_FCST_CDUMP} in both | "${CDUMP/enkf}" ) ;; # Don't change *) DO_AERO="NO" ;; # Turn waves off From fa4597739f8d5ce28a80b6c8f45ee810747e1e64 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 3 May 2024 18:58:58 +0000 Subject: [PATCH 077/128] Turn aerosol analyses on for gfs and gdas runs --- parm/config/gfs/config.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 010a8d098f..2a956949fd 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -230,7 +230,7 @@ case "${APP}" in ;; ATMA) export DO_AERO="YES" - export AERO_ANL_CDUMP="gdas" + export AERO_ANL_CDUMP="both" export AERO_FCST_CDUMP="gdas" ;; ATMW) From ac9128de2c0de28e575b77b3c7c5b90687890ccc Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 3 May 2024 19:46:41 +0000 Subject: [PATCH 078/128] Use aero cdump variables to determine archiving --- parm/archive/arcdir.yaml.j2 | 2 +- parm/archive/gdas.yaml.j2 | 2 +- parm/archive/gfsa.yaml.j2 | 3 +++ parm/archive/master_gfs.yaml.j2 | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index 045cedd90c..8610980c4a 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -21,7 +21,7 @@ deterministic: {% if DO_JEDISNOWDA %} - ["{{ COM_SNOW_ANALYSIS }}/{{ head }}snowstat.tgz", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YMDH }}.tgz"] {% endif %} - {% if DO_AERO %} + {% if AERO_ANL_CDUMP == RUN or AERO_ANL_CDUMP == "both" %} - ["{{ COM_CHEM_ANALYSIS }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.anl", "{{ ARCDIR }}/pgbanl.{{ RUN }}.{{ cycle_YMDH }}.grib2"] diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 9aafbc1de2..63e7e68ad4 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -49,7 +49,7 @@ - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" {% endif %} - {% if DO_AERO %} + {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} {% if DO_JEDISNOWDA %} diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 8fdd07caeb..e7a415112a 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -35,6 +35,9 @@ - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" {% endfor %} + {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} + - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" + {% endif %} optional: {% if MODE == "cycled" %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2 index a17928d684..199efec23c 100644 --- a/parm/archive/master_gfs.yaml.j2 +++ b/parm/archive/master_gfs.yaml.j2 @@ -18,7 +18,7 @@ datasets: {% include "gfswave.yaml.j2" %} {% endif %} -{% if DO_AERO %} +{% if AERO_FCST_CDUMP == "gfs" or AERO_FCST_CDUMP == "both" %} {% include "chem.yaml.j2" %} {% endif %} From 7fd5b76486091cc40b13efabe6a1fcc092232851 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 3 May 2024 19:55:15 +0000 Subject: [PATCH 079/128] Replace waves with aerosols in config.fcst --- parm/config/gfs/config.fcst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index 7caa208f27..c38bcfead3 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -16,7 +16,7 @@ esac # Turn off aerosols if not used for this CDUMP case ${AERO_FCST_CDUMP} in both | "${CDUMP/enkf}" ) ;; # Don't change - *) DO_AERO="NO" ;; # Turn waves off + *) DO_AERO="NO" ;; # Turn aerosols off esac # Source model specific information that is resolution dependent From b483745447484e169b35cd9c4ef99cd16cb78a36 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 3 May 2024 20:28:20 +0000 Subject: [PATCH 080/128] Change mandatory to required --- parm/archive/chem.yaml.j2 | 2 +- parm/archive/enkf.yaml.j2 | 2 +- parm/archive/enkf_grp.yaml.j2 | 2 +- parm/archive/enkf_restarta_grp.yaml.j2 | 2 +- parm/archive/enkf_restartb_grp.yaml.j2 | 2 +- parm/archive/gdas.yaml.j2 | 2 +- parm/archive/gdas_restarta.yaml.j2 | 2 +- parm/archive/gdas_restartb.yaml.j2 | 2 +- parm/archive/gdasice.yaml.j2 | 2 +- parm/archive/gdasice_restart.yaml.j2 | 2 +- parm/archive/gdasocean.yaml.j2 | 2 +- parm/archive/gdasocean_analysis.yaml.j2 | 2 +- parm/archive/gdasocean_restart.yaml.j2 | 2 +- parm/archive/gdaswave.yaml.j2 | 2 +- parm/archive/gdaswave_restart.yaml.j2 | 2 +- parm/archive/gfs_downstream.yaml.j2 | 2 +- parm/archive/gfs_flux.yaml.j2 | 2 +- parm/archive/gfs_flux_1p00.yaml.j2 | 2 +- parm/archive/gfs_netcdfa.yaml.j2 | 2 +- parm/archive/gfs_netcdfb.yaml.j2 | 2 +- parm/archive/gfs_pgrb2b.yaml.j2 | 2 +- parm/archive/gfs_restarta.yaml.j2 | 2 +- parm/archive/gfsa.yaml.j2 | 2 +- parm/archive/gfsb.yaml.j2 | 2 +- parm/archive/gfswave.yaml.j2 | 2 +- parm/archive/ice_6hravg.yaml.j2 | 2 +- parm/archive/ice_grib2.yaml.j2 | 2 +- parm/archive/ocean_6hravg.yaml.j2 | 2 +- parm/archive/ocean_daily.yaml.j2 | 2 +- parm/archive/ocean_grib2.yaml.j2 | 2 +- ush/python/pygfs/task/archive.py | 12 ++++++------ 31 files changed, 36 insertions(+), 36 deletions(-) diff --git a/parm/archive/chem.yaml.j2 b/parm/archive/chem.yaml.j2 index 7ac8d5ae8c..66ecfcecb3 100644 --- a/parm/archive/chem.yaml.j2 +++ b/parm/archive/chem.yaml.j2 @@ -2,6 +2,6 @@ {% set head = "gocart" %} name: "CHEM" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/chem.tar" - mandatory: + required: # TODO explicitize this set - "{{ COM_CHEM_HISTORY | relpath(ROTDIR) }}/{{ head }}*" diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index e2f59fdf9b..9f05187bb5 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -7,7 +7,7 @@ {% endif %} name: "ENKF" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}.tar" - mandatory: + required: {% for fhr in range(fhmin, fhmax + fhout, fhout) %} - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensmean.nc" - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.ensmean.nc" diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index 3d0ab8c8c4..ae9cbc0c00 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -7,7 +7,7 @@ {% endif %} name: "ENKF_GRP" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_grp{{ ENSGRP }}.tar" - mandatory: + required: {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 index 53d1e7451d..5fcbdd1a2e 100644 --- a/parm/archive/enkf_restarta_grp.yaml.j2 +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -2,7 +2,7 @@ {% set head = RUN + ".t" + cycle_HH + "z." %} name: "ENKF_RESTARTA_GRP" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restarta_grp{{ ENSGRP }}.tar" - mandatory: + required: {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 index cf50f6c76d..2329242ba1 100644 --- a/parm/archive/enkf_restartb_grp.yaml.j2 +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -9,7 +9,7 @@ {% endif %} name: "ENKF_RESTARTB_GRP" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restartb_grp{{ ENSGRP }}.tar" - mandatory: + required: {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 63e7e68ad4..97dde36b88 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gdas.t" + cycle_HH + "z." %} name: "GDAS" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas.tar" - mandatory: + required: {% if not current_cycle == SDATE and MODE == "cycled" %} - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 157e27a756..85f45e3435 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gdas.t" + cycle_HH + "z." %} name: "GDAS_RESTARTA" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restarta.tar" - mandatory: + required: {% if not current_cycle == SDATE and MODE == "cycled" %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" {% for iauhr in IAUHRS if iauhr != 6 %} diff --git a/parm/archive/gdas_restartb.yaml.j2 b/parm/archive/gdas_restartb.yaml.j2 index 56e5b5ca6e..5d6215166f 100644 --- a/parm/archive/gdas_restartb.yaml.j2 +++ b/parm/archive/gdas_restartb.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gdas.t" + cycle_HH + "z." %} name: "GDAS_RESTARTB" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restartb.tar" - mandatory: + required: {% for r_time in range(restart_interval_gdas, FHMAX + 1, restart_interval_gdas) %} {% set r_timedelta = (r_time | string + "H") | to_timedelta %} {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} diff --git a/parm/archive/gdasice.yaml.j2 b/parm/archive/gdasice.yaml.j2 index 0eca9500ed..f735232c54 100644 --- a/parm/archive/gdasice.yaml.j2 +++ b/parm/archive/gdasice.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gdas.ice.t" + cycle_HH + "z." %} name: "GDASICE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar" - mandatory: + required: # TODO explicitly name the files in history to archive - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}*' - '{{ COM_CONF | relpath(ROTDIR) }}/ufs.ice_in' diff --git a/parm/archive/gdasice_restart.yaml.j2 b/parm/archive/gdasice_restart.yaml.j2 index a0b809df87..9291e06929 100644 --- a/parm/archive/gdasice_restart.yaml.j2 +++ b/parm/archive/gdasice_restart.yaml.j2 @@ -2,6 +2,6 @@ {% set head = "gdas.ice.t" + cycle_HH + "z." %} name: "GDASICE_RESTART" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice_restart.tar" - mandatory: + required: # TODO explicitly name the restart files to archive - '{{ COM_ICE_RESTART | relpath(ROTDIR) }}/*' diff --git a/parm/archive/gdasocean.yaml.j2 b/parm/archive/gdasocean.yaml.j2 index 1a8ffb8a52..0cd679c9d7 100644 --- a/parm/archive/gdasocean.yaml.j2 +++ b/parm/archive/gdasocean.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gdas.ocean.t" + cycle_HH + "z." %} name: "GDASOCEAN" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar" - mandatory: + required: # TODO Explicitly name the history and input files to archive - '{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}*' - '{{ COM_OCEAN_INPUT | relpath(ROTDIR) }}/.' diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index 4b7c21b16a..9fd4d4e936 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gdas.ocean.t" + cycle_HH + "z." %} name: "GDASOCEAN_ANALYSIS" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar" - mandatory: + required: # TODO explicitly name the analysis files to archive - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}*' - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t??z.ocngrid.nc' diff --git a/parm/archive/gdasocean_restart.yaml.j2 b/parm/archive/gdasocean_restart.yaml.j2 index 010ffafc14..ebb701a0c7 100644 --- a/parm/archive/gdasocean_restart.yaml.j2 +++ b/parm/archive/gdasocean_restart.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gdas.ocean.t" + cycle_HH + "z." %} name: "GDASOCEAN_RESTART" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_restart.tar" - mandatory: + required: # TODO explicitly name the restart files to archive - '{{ COM_OCEAN_RESTART | relpath(ROTDIR) }}/*' - '{{ COM_MED_RESTART | relpath(ROTDIR) }}/*' diff --git a/parm/archive/gdaswave.yaml.j2 b/parm/archive/gdaswave.yaml.j2 index 81860e0eef..092c8f176e 100644 --- a/parm/archive/gdaswave.yaml.j2 +++ b/parm/archive/gdaswave.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gdas.wave.t" + cycle_HH + "z." %} name: "GDASWAVE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave.tar" - mandatory: + required: # TODO explicitly name the wave grid/station files to archive - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*" - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}*" diff --git a/parm/archive/gdaswave_restart.yaml.j2 b/parm/archive/gdaswave_restart.yaml.j2 index 466c913152..ace018de28 100644 --- a/parm/archive/gdaswave_restart.yaml.j2 +++ b/parm/archive/gdaswave_restart.yaml.j2 @@ -1,6 +1,6 @@ gdaswave_restart: name: "GDASWAVE_RESTART" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave_restart.tar" - mandatory: + required: # TODO explicitly name the wave restart files to archive - "{{ COM_WAVE_RESTART | relpath(ROTDIR) }}/*" diff --git a/parm/archive/gfs_downstream.yaml.j2 b/parm/archive/gfs_downstream.yaml.j2 index 909be04cf0..329c426504 100644 --- a/parm/archive/gfs_downstream.yaml.j2 +++ b/parm/archive/gfs_downstream.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.t" + cycle_HH + "z." %} name: "GFS_DOWNSTREAM" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_downstream.tar" - mandatory: + required: - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.sfc" - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.snd" {% for i in range(1, NUM_SND_COLLECTIVES) %} diff --git a/parm/archive/gfs_flux.yaml.j2 b/parm/archive/gfs_flux.yaml.j2 index 66e3eaa455..202dc38855 100644 --- a/parm/archive/gfs_flux.yaml.j2 +++ b/parm/archive/gfs_flux.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.t" + cycle_HH + "z." %} name: "GFS_FLUX" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux.tar" - mandatory: + required: {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" diff --git a/parm/archive/gfs_flux_1p00.yaml.j2 b/parm/archive/gfs_flux_1p00.yaml.j2 index c7168eb01a..95c8ad6642 100644 --- a/parm/archive/gfs_flux_1p00.yaml.j2 +++ b/parm/archive/gfs_flux_1p00.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.t" + cycle_HH + "z." %} name: "GFS_FLUX_1P00" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux_1p00.tar" - mandatory: + required: {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}" - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}.idx" diff --git a/parm/archive/gfs_netcdfa.yaml.j2 b/parm/archive/gfs_netcdfa.yaml.j2 index e015d8e4a1..6ea487125e 100644 --- a/parm/archive/gfs_netcdfa.yaml.j2 +++ b/parm/archive/gfs_netcdfa.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.t" + cycle_HH + "z." %} name: "GFS_NETCDFA" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfa.tar" - mandatory: + required: - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" diff --git a/parm/archive/gfs_netcdfb.yaml.j2 b/parm/archive/gfs_netcdfb.yaml.j2 index bcf8f67861..1f2a7d0d8f 100644 --- a/parm/archive/gfs_netcdfb.yaml.j2 +++ b/parm/archive/gfs_netcdfb.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.t" + cycle_HH + "z." %} name: "GFS_NETCDFB" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfb.tar" - mandatory: + required: {% for fhr in range(0, ARCH_GAUSSIAN_FHMAX + ARCH_GAUSSIAN_FHINC, ARCH_GAUSSIAN_FHINC) %} - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" diff --git a/parm/archive/gfs_pgrb2b.yaml.j2 b/parm/archive/gfs_pgrb2b.yaml.j2 index f479359d9c..c9e0ae2f65 100644 --- a/parm/archive/gfs_pgrb2b.yaml.j2 +++ b/parm/archive/gfs_pgrb2b.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.t" + cycle_HH + "z." %} name: "GFS_PGRB2B" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_pgrb2b.tar" - mandatory: + required: {% if MODE == "cycled" %} - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl" - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl.idx" diff --git a/parm/archive/gfs_restarta.yaml.j2 b/parm/archive/gfs_restarta.yaml.j2 index 4bbeec04f4..461dba3e6f 100644 --- a/parm/archive/gfs_restarta.yaml.j2 +++ b/parm/archive/gfs_restarta.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.t" + cycle_HH + "z." %} name: "GFS_RESTARTA" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_restarta.tar" - mandatory: + required: {% if MODE == "cycled" %} {% if DOHYBVAR %} {% set anl_offset = "-3H" %} diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index e7a415112a..c287a6ba9a 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.t" + cycle_HH + "z." %} name: "GFSA" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsa.tar" - mandatory: + required: # TODO explicitly name all logs to include {% for log in glob("logs/" ~ cycle_YMDH ~ "/gfs*.log") %} {% if log != "gfsarch.log" %} diff --git a/parm/archive/gfsb.yaml.j2 b/parm/archive/gfsb.yaml.j2 index a550e335da..1b4c4f931c 100644 --- a/parm/archive/gfsb.yaml.j2 +++ b/parm/archive/gfsb.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.t" + cycle_HH + "z." %} name: "GFSB" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsb.tar" - mandatory: + required: {% if MODE == "cycled" %} - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl" - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl.idx" diff --git a/parm/archive/gfswave.yaml.j2 b/parm/archive/gfswave.yaml.j2 index 74ec172d9b..c59029209d 100644 --- a/parm/archive/gfswave.yaml.j2 +++ b/parm/archive/gfswave.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfswave.t" + cycle_HH + "z." %} name: "GFSWAVE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfswave.tar" - mandatory: + required: {% for fh in range(0, FHMAX_HF_WAV + FHOUT_HF_WAV, FHOUT_HF_WAV) %} # NOTE This is as explicit as possible without major logic to parse wavepostGRD. # Matches files of the form "gfswave.tCCz...fHHH.grib2". diff --git a/parm/archive/ice_6hravg.yaml.j2 b/parm/archive/ice_6hravg.yaml.j2 index 060089d2c3..9e01047530 100644 --- a/parm/archive/ice_6hravg.yaml.j2 +++ b/parm/archive/ice_6hravg.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.ice.t" + cycle_HH + "z." %} name: "ICE_6HRAVG" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_6hravg.tar" - mandatory: + required: - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}ic.nc" {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" diff --git a/parm/archive/ice_grib2.yaml.j2 b/parm/archive/ice_grib2.yaml.j2 index 9af8284c66..a5b3cdc164 100644 --- a/parm/archive/ice_grib2.yaml.j2 +++ b/parm/archive/ice_grib2.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.ice.t" + cycle_HH + "z." %} name: "ICE_GRIB2" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_grib2.tar" - mandatory: + required: {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %} {% set fhr3 = '%03d' % fhr %} {% if ICERES == 500 %} diff --git a/parm/archive/ocean_6hravg.yaml.j2 b/parm/archive/ocean_6hravg.yaml.j2 index 32ffdf28a5..4b7a2d3c41 100644 --- a/parm/archive/ocean_6hravg.yaml.j2 +++ b/parm/archive/ocean_6hravg.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.ocean.t" + cycle_HH + "z." %} name: "OCEAN_6HRAVG" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_6hravg.tar" - mandatory: + required: {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/ocean_daily.yaml.j2 b/parm/archive/ocean_daily.yaml.j2 index d1d07a1897..85a095136c 100644 --- a/parm/archive/ocean_daily.yaml.j2 +++ b/parm/archive/ocean_daily.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.ocean.t" + cycle_HH + "z." %} name: "OCEAN_DAILY" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_daily.tar" - mandatory: + required: {% for fhr in range(24, FHMAX_GFS + 24, 24) %} - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}daily.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/ocean_grib2.yaml.j2 b/parm/archive/ocean_grib2.yaml.j2 index 333fdade8a..2732fa3e2c 100644 --- a/parm/archive/ocean_grib2.yaml.j2 +++ b/parm/archive/ocean_grib2.yaml.j2 @@ -2,7 +2,7 @@ {% set head = "gfs.ocean.t" + cycle_HH + "z." %} name: "OCEAN_GRIB2" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_grib2.tar" - mandatory: + required: {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %} {% set fhr3 = '%03d' % fhr %} {% if OCNRES == 500 %} diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 10122e9255..cb1cdc9ffe 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -176,7 +176,7 @@ def execute(self, arcdir_set: Dict[str, Any], atardir_sets: List[Dict[str, Any]] def _create_fileset(atardir_set: Dict[str, Any]) -> List: """ Collect the list of all available files from the parsed yaml dict. - Globs are expanded and if mandatory files are missing, an error is + Globs are expanded and if required files are missing, an error is raised. TODO: expand all globs in the jinja yaml files instead of expanding @@ -186,16 +186,16 @@ def _create_fileset(atardir_set: Dict[str, Any]) -> List: Parameters ---------- atardir_set: Dict - Contains full paths for mandatory and optional files to be archived. + Contains full paths for required and optional files to be archived. """ fileset = [] - if "mandatory" in atardir_set: - if atardir_set.mandatory is not None: - for item in atardir_set.mandatory: + if "required" in atardir_set: + if atardir_set.required is not None: + for item in atardir_set.required: glob_set = glob.glob(item) if len(glob_set) == 0: - raise FileNotFoundError(f"FATAL ERROR: Mandatory file, directory, or glob {item} not found!") + raise FileNotFoundError(f"FATAL ERROR: Required file, directory, or glob {item} not found!") for entry in glob_set: fileset.append(entry) From cc89c16b243fb5c52790d2274c9ea02107fe1505 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 3 May 2024 20:29:36 +0000 Subject: [PATCH 081/128] Make wave point files required --- parm/archive/gfswave.yaml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/parm/archive/gfswave.yaml.j2 b/parm/archive/gfswave.yaml.j2 index c59029209d..65c75a9f21 100644 --- a/parm/archive/gfswave.yaml.j2 +++ b/parm/archive/gfswave.yaml.j2 @@ -13,7 +13,6 @@ - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" {% endfor %} - optional: - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}bull_tar" - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}cbull_tar" - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}spec_tar.gz" From 1ec205c113b7a835aecb5a047c77023004b70eaa Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 6 May 2024 15:09:45 +0000 Subject: [PATCH 082/128] Add wave IBP data --- parm/archive/gfswave.yaml.j2 | 5 +++++ parm/config/gefs/config.base | 1 + parm/config/gefs/config.wavepostsbs | 1 - parm/config/gfs/config.base | 1 + parm/config/gfs/config.wavepostsbs | 1 - scripts/exglobal_archive.py | 2 +- 6 files changed, 8 insertions(+), 3 deletions(-) diff --git a/parm/archive/gfswave.yaml.j2 b/parm/archive/gfswave.yaml.j2 index 65c75a9f21..a24374feca 100644 --- a/parm/archive/gfswave.yaml.j2 +++ b/parm/archive/gfswave.yaml.j2 @@ -16,3 +16,8 @@ - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}bull_tar" - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}cbull_tar" - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}spec_tar.gz" + {% if DOIBP_WAV %} + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibpbull_tar" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibpcbull_tar" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibp_tar" + {% endif %} diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index dfe7341f66..fb96830774 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -142,6 +142,7 @@ export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both export DOBNDPNT_WAVE="NO" # The GEFS buoys file does not currently have any boundary points +export DOIBP_WAV="NO" # Option to create point outputs from input boundary points export FRAC_GRID=".true." export DO_NEST="NO" # Whether to run a global-nested domain if [[ "${DO_NEST:-NO}" == "YES" ]] ; then diff --git a/parm/config/gefs/config.wavepostsbs b/parm/config/gefs/config.wavepostsbs index b3c5902e3c..82cec321da 100644 --- a/parm/config/gefs/config.wavepostsbs +++ b/parm/config/gefs/config.wavepostsbs @@ -13,7 +13,6 @@ export WAV_SUBGRBSRC="" export WAV_SUBGRB="" # Options for point output (switch on/off boundary point output) -export DOIBP_WAV='NO' # Input boundary points export DOFLD_WAV='YES' # Field data export DOPNT_WAV='YES' # Station data export DOGRB_WAV='YES' # Create grib2 files diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 2a956949fd..66ddca300c 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -186,6 +186,7 @@ export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both export DOBNDPNT_WAVE="NO" +export DOIBP_WAV="NO" # Option to create point outputs from input boundary points export FRAC_GRID=".true." export DO_NEST="NO" # Whether to run a global-nested domain if [[ "${DO_NEST:-NO}" == "YES" ]] ; then diff --git a/parm/config/gfs/config.wavepostsbs b/parm/config/gfs/config.wavepostsbs index b3c5902e3c..82cec321da 100644 --- a/parm/config/gfs/config.wavepostsbs +++ b/parm/config/gfs/config.wavepostsbs @@ -13,7 +13,6 @@ export WAV_SUBGRBSRC="" export WAV_SUBGRB="" # Options for point output (switch on/off boundary point output) -export DOIBP_WAV='NO' # Input boundary points export DOFLD_WAV='YES' # Field data export DOPNT_WAV='YES' # Station data export DOGRB_WAV='YES' # Create grib2 files diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 74d7d73688..f78750f987 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -29,7 +29,7 @@ def main(): 'DOIAU', 'OCNRES', 'ICERES', 'NUM_SND_COLLECTIVES', 'FHOUT_WAV', 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', 'restart_interval_gdas', 'restart_interval_gfs', - 'AERO_ANL_CDUMP', 'AERO_FCST_CDUMP'] + 'AERO_ANL_CDUMP', 'AERO_FCST_CDUMP', 'DOIBP_WAV'] archive_dict = AttrDict() for key in keys: From d471e9e8df1f10ea2510ad71f67651fe940f4132 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 6 May 2024 15:23:44 +0000 Subject: [PATCH 083/128] Split archive execute method, determine commands (htar/tar, chmod, etc) in configure method --- scripts/exglobal_archive.py | 8 ++- ush/python/pygfs/task/archive.py | 85 +++++++++++++++----------------- 2 files changed, 46 insertions(+), 47 deletions(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index f78750f987..f2760179c3 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -47,8 +47,12 @@ def main(): # Determine which archives to create arcdir_set, atardir_sets = archive.configure(archive_dict) - # Create the archives - archive.execute(arcdir_set, atardir_sets) + # Populate the product archive (ARCDIR) + archive.execute_store_products(arcdir_set) + + # Create the backup tarballs and store in ATARDIR + for atardir_set in atardir_sets: + archive.execute_backup_dataset(atardir_set) os.chdir(cwd) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index cb1cdc9ffe..68fb828b00 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -84,8 +84,16 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str self.tar_cmd = "htar" self.hsi = Hsi() self.htar = Htar() + self.cvf = self.htar.cvf + self.rm_cmd = self.hsi.rm + self.chgrp_cmd = self.hsi.chgrp + self.chmod_cmd = self.hsi.chmod elif arch_dict.LOCALARCH: self.tar_cmd = "tar" + self.cvf = Archive._create_tarball + self.chgrp_cmd = chgrp + self.chmod_cmd = os.fchmod + self.rm_cmd = rm_p else: # Only perform local archiving. Do not create tarballs. self.tar_cmd = "" return arcdir_set, [] @@ -123,15 +131,13 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str return arcdir_set, atardir_sets @logit(logger) - def execute(self, arcdir_set: Dict[str, Any], atardir_sets: List[Dict[str, Any]]) -> None: - """Perform local archiving to ARCDIR and create the tarballs from the list of yaml dicts. + def execute_store_products(self, arcdir_set: Dict[str, Any]) -> None: + """Perform local archiving of data products to ARCDIR. Parameters ---------- arcdir_set : Dict[str, Any] FileHandler instructions to populate ARCDIR with - atardir_sets: List[Dict[str, Any]] - Sets of files to archive via tar or htar Return ------ @@ -142,34 +148,38 @@ def execute(self, arcdir_set: Dict[str, Any], atardir_sets: List[Dict[str, Any]] for key in arcdir_set.keys(): FileHandler(arcdir_set[key]).sync() - # Generate tarballs - for atardir_set in atardir_sets: + @logit(logger) + def execute_backup_dataset(self, atardir_set: Dict[str, Any]) -> None: + """Create a backup tarball from a yaml dict. - if len(atardir_set.fileset) == 0: - print(f"WARNING skipping would-be empty archive {atardir_set.target}.") - continue + Parameters + ---------- + atardir_set: Dict[str, Any] + Dict defining set of files to backup and the target tarball. - if self.tar_cmd == "htar": - cvf = self.htar.cvf - create = self.htar.create - rm_cmd = self.hsi.rm - else: - cvf = Archive._create_tarball - rm_cmd = rm_p + Return + ------ + None + """ - if atardir_set.has_rstprod: + # Generate tarball + if len(atardir_set.fileset) == 0: + print(f"WARNING skipping would-be empty archive {atardir_set.target}.") + return - try: - cvf(atardir_set.target, atardir_set.fileset) - # Regardless of exception type, attempt to remove the target - except Exception: - rm_cmd(atardir_set.target) - raise RuntimeError(f"FATAL ERROR: Failed to create restricted archive {atardir_set.target}, deleting!") + if atardir_set.has_rstprod: - self._protect_rstprod(atardir_set) + try: + self.cvf(atardir_set.target, atardir_set.fileset) + # Regardless of exception type, attempt to remove the target + except Exception: + self.rm_cmd(atardir_set.target) + raise RuntimeError(f"FATAL ERROR: Failed to create restricted archive {atardir_set.target}, deleting!") - else: - cvf(atardir_set.target, atardir_set.fileset) + self._protect_rstprod(atardir_set) + + else: + self.cvf(atardir_set.target, atardir_set.fileset) @logit(logger) @staticmethod @@ -246,31 +256,16 @@ def _protect_rstprod(self, atardir_set: Dict[str, any]) -> None: """ - if self.tar_cmd == "htar": - chgrp_cmd = self.hsi.chgrp - chmod_cmd = self.hsi.chmod - rm_cmd = self.hsi.rm - elif self.tar_cmd == "tar": - chgrp_cmd = chgrp - chmod_cmd = os.fchmod - rm_cmd = rm_p - else: - raise KeyError(f"FATAL ERROR: Invalid archiving command given: {self.tar_cmd}") - try: + self.chgrp_cmd("rstprod", atardir_set.target) if self.tar_cmd == "htar": - self.hsi.chgrp("rstprod", atardir_set.target) - self.hsi.chmod("640", atardir_set.target) + self.chmod_cmd("640", atardir_set.target) else: - chgrp("rstprod", atardir_set.target) - os.chmod(atardir_set.target, 0o640) + self.chmod_cmd(atardir_set.target, 0o640) # Regardless of exception type, attempt to remove the target except Exception: try: - if self.tar_cmd == "htar": - self.hsi.rm(atardir_set.target) - else: - rm_p(atardir_set.target) + self.rm_cmd(atardir_set.target) finally: raise RuntimeError(f"FATAL ERROR: Failed to protect {atardir_set.target}!\n" f"Please verify that it has been deleted!!") From 196db3ead22e72b071900a01bd7dea0ec99c2317 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 6 May 2024 15:26:46 +0000 Subject: [PATCH 084/128] Remove unused numpy, timedelta modules --- ush/python/pygfs/task/archive.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 68fb828b00..efe48778bf 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -3,11 +3,9 @@ import glob import os import shutil -from datetime import timedelta from logging import getLogger from typing import Any, Dict, List -import numpy as np from wxflow import (AttrDict, FileHandler, Hsi, Htar, Task, cast_strdict_as_dtypedict, chgrp, get_gid, logit, mkdir_p, parse_j2yaml, rm_p, strftime, to_YMD, to_YMDH, Template, TemplateConstants) From 96ac4426588bcadd2ef352f277b204c29da4954e Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 6 May 2024 19:56:21 +0000 Subject: [PATCH 085/128] Split enkfgdas and enkfgfs master templates; create one tarball at a time. --- parm/archive/enkf.yaml.j2 | 13 ++++--------- parm/archive/enkf_grp.yaml.j2 | 9 ++------- parm/archive/enkf_restartb_grp.yaml.j2 | 7 ------- parm/archive/master_enkf.yaml.j2 | 13 +++++++------ parm/archive/master_enkfgdas.yaml.j2 | 6 ++++++ parm/archive/master_enkfgfs.yaml.j2 | 6 ++++++ parm/config/gefs/config.base | 4 ++-- parm/config/gfs/config.base | 3 ++- scripts/exgdas_enkf_earc.py | 8 ++++++-- ush/python/pygfs/task/archive.py | 14 ++++---------- 10 files changed, 39 insertions(+), 44 deletions(-) create mode 100644 parm/archive/master_enkfgdas.yaml.j2 create mode 100644 parm/archive/master_enkfgfs.yaml.j2 diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index 9f05187bb5..b4e2172aeb 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -1,10 +1,5 @@ enkf: {% set head = RUN + ".t" + cycle_HH + "z." %} - {% if RUN == "enkfgdas" %} - {% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF, FHOUT_ENKF) %} - {% elif RUN == "enkfgfs" %} - {% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF_GFS, FHOUT_ENKF_GFS) %} - {% endif %} name: "ENKF" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}.tar" required: @@ -30,9 +25,9 @@ - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}{{ file }}" {% endfor %} {% if DOIAU %} - {% for fhr in IAUFHRS %} + {% for fhr in iaufhrs %} {% if fhr == IAU_OFFSET %} - {% if DO_CALC_INCREMENT_ENKF_GFS %} + {% if do_calc_increment %} - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmanl.ensmean.nc" {% endif %} # calc increment - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ratminc.ensmean.nc" @@ -47,13 +42,13 @@ {% for step in steps %} - "logs/{{ cycle_YMDH }}/{{ RUN }}{{ step }}.log" {% endfor %} - {% for mem in range(1, NMEM_ENS + 1) %} + {% for mem in range(1, nmem_ens + 1) %} - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_mem{{ '%03d' % mem }}.log" {% endfor %} {% if lobsdiag_forenkf %} - "logs/{{ cycle_YMDH }}/{{ RUN }}ediag.log" {% else %} - {% for mem in range(1, NMEM_ENS + 1) %} + {% for mem in range(1, nmem_ens + 1) %} - "logs/{{ cycle_YMDH }}/{{ RUN }}eomg_mem{{ '%03d' % mem }}.log" {% endfor %} {% endif %} # lobsdiag_forenkf diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index ae9cbc0c00..13f79021a7 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -1,10 +1,5 @@ enkf_grp: {% set head = RUN + ".t" + cycle_HH + "z." %} - {% if RUN == "enkfgdas" %} - {% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF, FHOUT_ENKF) %} - {% elif RUN == "enkfgfs" %} - {% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF_GFS, FHOUT_ENKF_GFS) %} - {% endif %} name: "ENKF_GRP" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_grp{{ ENSGRP }}.tar" required: @@ -23,7 +18,7 @@ {% endfor %} {% if not current_cycle == SDATE %} - {% if DO_CALC_INCR_ENKF_GFS %} + {% if do_calc_increment %} - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atmanl.nc" {% else %} - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratminc.nc" @@ -32,7 +27,7 @@ {% for iaufhr in iaufhrs if iaufhr != 6 %} {% set iaufhr = iaufhr %} - {% if DO_CALC_INCREMENT_ENKF_GFS %} + {% if do_calc_increment %} - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iaufhr }}.nc" {% else %} - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iaufhr }}.nc" diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 index 2329242ba1..2b62f548ef 100644 --- a/parm/archive/enkf_restartb_grp.yaml.j2 +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -1,12 +1,5 @@ enkf_restartb_grp: {% set head = RUN + ".t" + cycle_HH + "z." %} - {% if RUN == "enkfgdas" %} - {% set fhmax = FHMAX_ENKF %} - {% set restart_interval = restart_interval_enkfgdas %} - {% elif RUN == "enkfgfs" %} - {% set fhmax = FHMAX_ENKF_GFS %} - {% set restart_interval = restart_interval_enkfgfs %} - {% endif %} name: "ENKF_RESTARTB_GRP" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restartb_grp{{ ENSGRP }}.tar" required: diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index 82bb1a2a07..37e2bb5769 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -1,12 +1,8 @@ +# Set variables/lists needed to parse the enkf templates {% set cycle_HH = current_cycle | strftime("%H") %} {% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_YMDH = current_cycle | to_YMDH %} -datasets: -{% if ENSGRP == 0 %} -{% include "enkf.yaml.j2" %} -{% else %} -# Set variables/lists needed to parse the enkf templates {% if IAUFHRS is string %} {% set iaufhrs = [] %} {% for iaufhr in IAUFHRS.split(",") %} @@ -16,13 +12,18 @@ datasets: {% set iaufhrs = [IAUFHRS] %} {% endif %} +datasets: +{% if ENSGRP == 0 %} +{% include "enkf.yaml.j2" %} +{% else %} + # Declare to-be-filled lists of member COM directories {% set COM_ATMOS_ANALYSIS_MEM_list = [] %} {% set COM_ATMOS_RESTART_MEM_list = [] %} {% set COM_ATMOS_HISTORY_MEM_list = [] %} {% set first_group_mem = (ENSGRP - 1) * NMEM_EARCGRP + 1 %} -{% set last_group_mem = [ ENSGRP * NMEM_EARCGRP, NMEM_ENS ] | min %} +{% set last_group_mem = [ ENSGRP * NMEM_EARCGRP, nmem_ens ] | min %} # Construct member COM directories {% for mem in range(first_group_mem, last_group_mem + 1) %} diff --git a/parm/archive/master_enkfgdas.yaml.j2 b/parm/archive/master_enkfgdas.yaml.j2 new file mode 100644 index 0000000000..e21f6a381b --- /dev/null +++ b/parm/archive/master_enkfgdas.yaml.j2 @@ -0,0 +1,6 @@ +# Set variables specific to gdasenkf runs then parse the master_enkf template +{% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF, FHOUT_ENKF) %} +{% set do_calc_increment = DO_CALC_INCREMENT %} +{% set nmem_ens = NMEM_ENS %} +{% set restart_interval = restart_interval_enkfgdas %} +{% include "master_enkf.yaml.j2" %} diff --git a/parm/archive/master_enkfgfs.yaml.j2 b/parm/archive/master_enkfgfs.yaml.j2 new file mode 100644 index 0000000000..93ec38b660 --- /dev/null +++ b/parm/archive/master_enkfgfs.yaml.j2 @@ -0,0 +1,6 @@ +# Set variables specific to gfsenkf runs then parse the master_enkf template +{% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF_GFS, FHOUT_ENKF_GFS) %} +{% set do_calc_increment = DO_CALC_INCREMENT_ENKF_GFS %} +{% set nmem_ens = NMEM_ENS_GFS %} +{% set restart_interval = restart_interval_enkfgfs %} +{% include "master_enkf.yaml.j2" %} diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index fb96830774..9ac1aa92d6 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -186,7 +186,7 @@ case "${APP}" in ;; ATMA) export DO_AERO="YES" - export AERO_ANL_CDUMP="gdas" + export AERO_ANL_CDUMP="both" export AERO_FCST_CDUMP="gdas" ;; ATMW) @@ -206,7 +206,7 @@ case "${APP}" in if [[ "${APP}" =~ A$ ]]; then export DO_AERO="YES" - export AERO_ANL_CDUMP="gdas" + export AERO_ANL_CDUMP="both" export AERO_FCST_CDUMP="gdas" fi diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 66ddca300c..17f125b8f7 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -251,7 +251,7 @@ case "${APP}" in if [[ "${APP}" =~ A$ ]]; then export DO_AERO="YES" - export AERO_ANL_CDUMP="gdas" + export AERO_ANL_CDUMP="both" export AERO_FCST_CDUMP="gdas" fi @@ -356,6 +356,7 @@ export NMEM_ENS=@NMEM_ENS@ export SMOOTH_ENKF="NO" export l4densvar=".true." export lwrite4danl=".true." +export DO_CALC_INCREMENT="NO" # Early-cycle EnKF parameters export NMEM_ENS_GFS=30 diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index 756aa5f36b..29e866bb59 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -44,8 +44,12 @@ def main(): # Determine which archives to create arcdir_set, atardir_sets = archive.configure(archive_dict) - # Create the archives - archive.execute(arcdir_set, atardir_sets) + # Populate the product archive (ARCDIR) + archive.execute_store_products(arcdir_set) + + # Create the backup tarballs and store in ATARDIR + for atardir_set in atardir_sets: + archive.execute_backup_dataset(atardir_set) os.chdir(cwd) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index efe48778bf..9aaabd4e18 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -90,7 +90,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str self.tar_cmd = "tar" self.cvf = Archive._create_tarball self.chgrp_cmd = chgrp - self.chmod_cmd = os.fchmod + self.chmod_cmd = os.chmod self.rm_cmd = rm_p else: # Only perform local archiving. Do not create tarballs. self.tar_cmd = "" @@ -104,16 +104,10 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str # Copy the cyclone track files and rename the experiments Archive._rename_cyclone_expt(arch_dict) - if arch_dict.RUN == "gdas": - master_yaml = "master_gdas.yaml.j2" - elif arch_dict.RUN == "gfs": - master_yaml = "master_gfs.yaml.j2" - elif arch_dict.RUN == "enkfgdas" or arch_dict.RUN == "enkfgfs": - master_yaml = "master_enkf.yaml.j2" - elif arch_dict.RUN == "gefs": + if arch_dict.RUN == "gefs": raise NotImplementedError("FATAL ERROR: Archiving is not yet set up for GEFS runs") - else: - raise ValueError(f"FATAL ERROR: Archiving is not enabled for {arch_dict.RUN} runs") + + master_yaml = "master_" + arch_dict.RUN + ".yaml.j2" parsed_sets = parse_j2yaml(os.path.join(archive_parm, master_yaml), arch_dict) From c018324ef6baa99af522212bbd60a06991af36dc Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 8 May 2024 12:05:09 +0000 Subject: [PATCH 086/128] Handle indention at the master level --- parm/archive/chem.yaml.j2 | 14 +- parm/archive/enkf.yaml.j2 | 110 ++++++------- parm/archive/enkf_grp.yaml.j2 | 86 +++++----- parm/archive/enkf_restarta_grp.yaml.j2 | 64 ++++---- parm/archive/enkf_restartb_grp.yaml.j2 | 42 ++--- parm/archive/gdas.yaml.j2 | 210 ++++++++++++------------ parm/archive/gdas_restarta.yaml.j2 | 90 +++++----- parm/archive/gdas_restartb.yaml.j2 | 38 ++--- parm/archive/gdasice.yaml.j2 | 16 +- parm/archive/gdasice_restart.yaml.j2 | 14 +- parm/archive/gdasocean.yaml.j2 | 16 +- parm/archive/gdasocean_analysis.yaml.j2 | 20 +-- parm/archive/gdasocean_restart.yaml.j2 | 16 +- parm/archive/gdaswave.yaml.j2 | 16 +- parm/archive/gdaswave_restart.yaml.j2 | 12 +- parm/archive/gfs_downstream.yaml.j2 | 24 +-- parm/archive/gfs_flux.yaml.j2 | 18 +- parm/archive/gfs_flux_1p00.yaml.j2 | 18 +- parm/archive/gfs_netcdfa.yaml.j2 | 28 ++-- parm/archive/gfs_netcdfb.yaml.j2 | 18 +- parm/archive/gfs_pgrb2b.yaml.j2 | 38 ++--- parm/archive/gfs_restarta.yaml.j2 | 46 +++--- parm/archive/gfsa.yaml.j2 | 122 +++++++------- parm/archive/gfsb.yaml.j2 | 34 ++-- parm/archive/gfswave.yaml.j2 | 46 +++--- parm/archive/ice_6hravg.yaml.j2 | 18 +- parm/archive/ice_grib2.yaml.j2 | 36 ++-- parm/archive/master_enkf.yaml.j2 | 6 + parm/archive/master_gdas.yaml.j2 | 18 ++ parm/archive/master_gfs.yaml.j2 | 24 +++ parm/archive/ocean_6hravg.yaml.j2 | 16 +- parm/archive/ocean_daily.yaml.j2 | 16 +- parm/archive/ocean_grib2.yaml.j2 | 36 ++-- 33 files changed, 687 insertions(+), 639 deletions(-) diff --git a/parm/archive/chem.yaml.j2 b/parm/archive/chem.yaml.j2 index 66ecfcecb3..7796912b1a 100644 --- a/parm/archive/chem.yaml.j2 +++ b/parm/archive/chem.yaml.j2 @@ -1,7 +1,7 @@ - chem: - {% set head = "gocart" %} - name: "CHEM" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/chem.tar" - required: - # TODO explicitize this set - - "{{ COM_CHEM_HISTORY | relpath(ROTDIR) }}/{{ head }}*" +chem: + {% set head = "gocart" %} + name: "CHEM" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/chem.tar" + required: + # TODO explicitize this set + - "{{ COM_CHEM_HISTORY | relpath(ROTDIR) }}/{{ head }}*" diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index b4e2172aeb..e785febfe2 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -1,55 +1,55 @@ - enkf: - {% set head = RUN + ".t" + cycle_HH + "z." %} - name: "ENKF" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}.tar" - required: - {% for fhr in range(fhmin, fhmax + fhout, fhout) %} - - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensmean.nc" - - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.ensmean.nc" - {% if ENKF_SPREAD %} - - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensspread.nc" - {% endif %} - {% endfor %} - {% if not current_cycle == SDATE %} - {% if not DO_JEDIATMENS %} - {% set da_files = ["enkfstat", - "gsistat.ensmean", - "cnvstat.ensmean", - "oznstat.ensmean", - "radstat.ensmean"] %} - {% else %} - {% set da_files = ["atmens.yaml", - "atmensstat"] %} - {% endif %} - {% for file in da_files %} - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}{{ file }}" - {% endfor %} - {% if DOIAU %} - {% for fhr in iaufhrs %} - {% if fhr == IAU_OFFSET %} - {% if do_calc_increment %} - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmanl.ensmean.nc" - {% endif %} # calc increment - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ratminc.ensmean.nc" - {% else %} # fhr != IAU_OFFSET - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensmean.nc" - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmi{{ '%03d' % fhr }}.ensmean.nc" - {% endif %} # fhr == IAU_OFFSET - {% endfor %} # IAUFHRS - {% endif %} # DOIAU - # TODO archive ecen logs based on actual groups. Will need to emulate numpy.array_split to do so. - {% set steps = ["eobs", "ecen*", "esfc", "eupd"] %} - {% for step in steps %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}{{ step }}.log" - {% endfor %} - {% for mem in range(1, nmem_ens + 1) %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_mem{{ '%03d' % mem }}.log" - {% endfor %} - {% if lobsdiag_forenkf %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}ediag.log" - {% else %} - {% for mem in range(1, nmem_ens + 1) %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}eomg_mem{{ '%03d' % mem }}.log" - {% endfor %} - {% endif %} # lobsdiag_forenkf - {% endif %} # not the first cycle +enkf: + {% set head = RUN + ".t" + cycle_HH + "z." %} + name: "ENKF" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}.tar" + required: + {% for fhr in range(fhmin, fhmax + fhout, fhout) %} + - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensmean.nc" + - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.ensmean.nc" + {% if ENKF_SPREAD %} + - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensspread.nc" + {% endif %} + {% endfor %} + {% if not current_cycle == SDATE %} + {% if not DO_JEDIATMENS %} + {% set da_files = ["enkfstat", + "gsistat.ensmean", + "cnvstat.ensmean", + "oznstat.ensmean", + "radstat.ensmean"] %} + {% else %} + {% set da_files = ["atmens.yaml", + "atmensstat"] %} + {% endif %} + {% for file in da_files %} + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}{{ file }}" + {% endfor %} + {% if DOIAU %} + {% for fhr in iaufhrs %} + {% if fhr == IAU_OFFSET %} + {% if do_calc_increment %} + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmanl.ensmean.nc" + {% endif %} # calc increment + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ratminc.ensmean.nc" + {% else %} # fhr != IAU_OFFSET + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensmean.nc" + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmi{{ '%03d' % fhr }}.ensmean.nc" + {% endif %} # fhr == IAU_OFFSET + {% endfor %} # IAUFHRS + {% endif %} # DOIAU + # TODO archive ecen logs based on actual groups. Will need to emulate numpy.array_split to do so. + {% set steps = ["eobs", "ecen*", "esfc", "eupd"] %} + {% for step in steps %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}{{ step }}.log" + {% endfor %} + {% for mem in range(1, nmem_ens + 1) %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_mem{{ '%03d' % mem }}.log" + {% endfor %} + {% if lobsdiag_forenkf %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}ediag.log" + {% else %} + {% for mem in range(1, nmem_ens + 1) %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}eomg_mem{{ '%03d' % mem }}.log" + {% endfor %} + {% endif %} # lobsdiag_forenkf + {% endif %} # not the first cycle diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index 13f79021a7..70b5d8e25e 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -1,46 +1,46 @@ - enkf_grp: - {% set head = RUN + ".t" + cycle_HH + "z." %} - name: "ENKF_GRP" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_grp{{ ENSGRP }}.tar" - required: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} - {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} +enkf_grp: + {% set head = RUN + ".t" + cycle_HH + "z." %} + name: "ENKF_GRP" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_grp{{ ENSGRP }}.tar" + required: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} + {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% if current_cycle != SDATE and not lobsdiag_forenkf %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat" - {% endif %} - {% for fhr in range(fhmin, fhmax+fhout, fhout) %} - - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" - - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" - {% endfor %} + {% if current_cycle != SDATE and not lobsdiag_forenkf %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat" + {% endif %} + {% for fhr in range(fhmin, fhmax+fhout, fhout) %} + - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" + - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + {% endfor %} - {% if not current_cycle == SDATE %} - {% if do_calc_increment %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - {% else %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratminc.nc" - {% endif %} - {% endif %} + {% if not current_cycle == SDATE %} + {% if do_calc_increment %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + {% else %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratminc.nc" + {% endif %} + {% endif %} - {% for iaufhr in iaufhrs if iaufhr != 6 %} - {% set iaufhr = iaufhr %} - {% if do_calc_increment %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iaufhr }}.nc" - {% else %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iaufhr }}.nc" - {% endif %} - {% endfor %} # iaufhr in iaufhrs - {% endfor %} # first_group_mem to last_group_mem - optional: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} - {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% if current_cycle != SDATE and not lobsdiag_forenkf %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" - {% endif %} - {% endfor %} + {% for iaufhr in iaufhrs if iaufhr != 6 %} + {% set iaufhr = iaufhr %} + {% if do_calc_increment %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iaufhr }}.nc" + {% else %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iaufhr }}.nc" + {% endif %} + {% endfor %} # iaufhr in iaufhrs + {% endfor %} # first_group_mem to last_group_mem + optional: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} + {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + {% if current_cycle != SDATE and not lobsdiag_forenkf %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" + {% endif %} + {% endfor %} diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 index 5fcbdd1a2e..4245c7e218 100644 --- a/parm/archive/enkf_restarta_grp.yaml.j2 +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -1,32 +1,32 @@ - enkf_restarta_grp: - {% set head = RUN + ".t" + cycle_HH + "z." %} - name: "ENKF_RESTARTA_GRP" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restarta_grp{{ ENSGRP }}.tar" - required: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} - {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - - {% if not lobsdiag_forenkf %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_air" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_int" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_pc" - {% endif %} - - {% set anl_delta = "-3H" | to_timedelta %} - {% set anl_time = current_cycle | add_to_datetime(anl_delta) %} - {% for itile in range(1, 7) %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" - {% endfor %} - {% endfor %} # first_group_mem to last_group_mem - optional: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" - {% endif %} - {% endfor %} +enkf_restarta_grp: + {% set head = RUN + ".t" + cycle_HH + "z." %} + name: "ENKF_RESTARTA_GRP" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restarta_grp{{ ENSGRP }}.tar" + required: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} + {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + + {% if not lobsdiag_forenkf %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_air" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_int" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_pc" + {% endif %} + + {% set anl_delta = "-3H" | to_timedelta %} + {% set anl_time = current_cycle | add_to_datetime(anl_delta) %} + {% for itile in range(1, 7) %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" + {% endfor %} + {% endfor %} # first_group_mem to last_group_mem + optional: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" + {% endif %} + {% endfor %} diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 index 2b62f548ef..268c06ec82 100644 --- a/parm/archive/enkf_restartb_grp.yaml.j2 +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -1,21 +1,21 @@ - enkf_restartb_grp: - {% set head = RUN + ".t" + cycle_HH + "z." %} - name: "ENKF_RESTARTB_GRP" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restartb_grp{{ ENSGRP }}.tar" - required: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% for r_time in range(restart_interval, fhmax + 1, restart_interval) %} - {% set r_timedelta = (r_time | string + "H") | to_timedelta %} - {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} - {% set r_prefix = r_dt | to_YMD + "." + r_dt | strftime("%H") + "0000" %} - {% for itile in range(1, 7) %} - {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" - {% endfor %} - {% endfor %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" - {% endfor %} - {% endfor %} +enkf_restartb_grp: + {% set head = RUN + ".t" + cycle_HH + "z." %} + name: "ENKF_RESTARTB_GRP" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restartb_grp{{ ENSGRP }}.tar" + required: + {% for mem in range(first_group_mem, last_group_mem + 1) %} + {% set imem = mem - first_group_mem %} + {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + {% for r_time in range(restart_interval, fhmax + 1, restart_interval) %} + {% set r_timedelta = (r_time | string + "H") | to_timedelta %} + {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} + {% set r_prefix = r_dt | to_YMD + "." + r_dt | strftime("%H") + "0000" %} + {% for itile in range(1, 7) %} + {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" + {% endfor %} + {% endfor %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" + {% endfor %} + {% endfor %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 97dde36b88..ece6cb81ab 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -1,105 +1,105 @@ - gdas: - {% set head = "gdas.t" + cycle_HH + "z." %} - name: "GDAS" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas.tar" - required: - {% if not current_cycle == SDATE and MODE == "cycled" %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" - - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" - - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" - - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" - - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" - {% if DOIAU and DOHYBVAR %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma003.ensres.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma009.ensres.nc" - {% endif %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}oznstat" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" - {% if DO_JEDIATMVAR %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" - {% endif %} - {% if DO_VERFOZN %} - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_cnt.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_diag.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_pen.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/stdout.time.tar.gz" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/horiz/stdout.horiz.tar.gz" - - "logs/{{ cycle_YMDH }}/{{ RUN }}verfozn.log" - {% endif %} - {% if DO_VERFRAD %} - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_angle.tar.gz" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcoef.tar.gz" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcor.tar.gz" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_time.tar.gz" - - "logs/{{ cycle_YMDH }}/{{ RUN }}verfrad.log" - {% endif %} - {% if DO_VMINMON %} - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" - - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" - {% endif %} - {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} - - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" - {% endif %} - {% if DO_JEDISNOWDA %} - - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snowstat.tgz" - {% endif %} - {% endif %} # Full cycle - - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" - # TODO explicitly name the atmos_prod log files to archive - - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" - {% for fhr in range(0, FHMAX + FHOUT, FHOUT) %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" - {% endfor %} - optional: - {% if not current_cycle == SDATE and MODE == "cycled" %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" - {% if DO_VERFRAD %} - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/low_count.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/warning.{{ cycle_YMDH }}" - {% endif %} - {% if DO_VERFOZN %} - {% set oznmon_types = [ - "gome_metop-b", "omi_aura", "ompslp_npp", "ompsnp_n20", - "ompsnp_npp", "ompstc8_n20", "ompstc8_npp", "sbuv2_n19" - ] %} - {% for group in [ "horiz", "time" ] %} - {% if group == "horiz" %} {% set suffix = ".gz" %} {% else %} {% set suffix = "" %} {% endif %} - {% for type in oznmon_types %} - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.ctl" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.{{ cycle_YMDH }}.ieee_d{{ suffix }}" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.ctl" - - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.{{ cycle_YMDH }}.ieee_d{{ suffix }}" - {% endfor %} - {% endfor %} - {% endif %} - {% endif %} - {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" - {% endfor %} - {% if not WRITE_DOPOST %} - # TODO set the forecast hours explicitly. This will require emulating numpy.array_split - - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" - {% endif %} ## not WRITE_DOPOST +gdas: + {% set head = "gdas.t" + cycle_HH + "z." %} + name: "GDAS" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas.tar" + required: + {% if not current_cycle == SDATE and MODE == "cycled" %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" + {% if DOIAU and DOHYBVAR %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma003.ensres.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma009.ensres.nc" + {% endif %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}oznstat" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" + {% if DO_JEDIATMVAR %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" + {% endif %} + {% if DO_VERFOZN %} + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_cnt.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_diag.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_pen.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/stdout.time.tar.gz" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/horiz/stdout.horiz.tar.gz" + - "logs/{{ cycle_YMDH }}/{{ RUN }}verfozn.log" + {% endif %} + {% if DO_VERFRAD %} + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_angle.tar.gz" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcoef.tar.gz" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcor.tar.gz" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_time.tar.gz" + - "logs/{{ cycle_YMDH }}/{{ RUN }}verfrad.log" + {% endif %} + {% if DO_VMINMON %} + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" + - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" + {% endif %} + {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} + - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" + {% endif %} + {% if DO_JEDISNOWDA %} + - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snowstat.tgz" + {% endif %} + {% endif %} # Full cycle + - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" + # TODO explicitly name the atmos_prod log files to archive + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" + {% for fhr in range(0, FHMAX + FHOUT, FHOUT) %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + {% endfor %} + optional: + {% if not current_cycle == SDATE and MODE == "cycled" %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" + {% if DO_VERFRAD %} + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/low_count.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/warning.{{ cycle_YMDH }}" + {% endif %} + {% if DO_VERFOZN %} + {% set oznmon_types = [ + "gome_metop-b", "omi_aura", "ompslp_npp", "ompsnp_n20", + "ompsnp_npp", "ompstc8_n20", "ompstc8_npp", "sbuv2_n19" + ] %} + {% for group in [ "horiz", "time" ] %} + {% if group == "horiz" %} {% set suffix = ".gz" %} {% else %} {% set suffix = "" %} {% endif %} + {% for type in oznmon_types %} + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.ctl" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.anl.{{ cycle_YMDH }}.ieee_d{{ suffix }}" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.ctl" + - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.ges.{{ cycle_YMDH }}.ieee_d{{ suffix }}" + {% endfor %} + {% endfor %} + {% endif %} + {% endif %} + {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + {% endfor %} + {% if not WRITE_DOPOST %} + # TODO set the forecast hours explicitly. This will require emulating numpy.array_split + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" + {% endif %} ## not WRITE_DOPOST diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 85f45e3435..5e60cf9868 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -1,45 +1,45 @@ - gdas_restarta: - {% set head = "gdas.t" + cycle_HH + "z." %} - name: "GDAS_RESTARTA" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restarta.tar" - required: - {% if not current_cycle == SDATE and MODE == "cycled" %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" - {% for iauhr in IAUHRS if iauhr != 6 %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" - {% endfor %} - {% if DOHYBVAR %} - {% set anl_offset = "-3H" %} - {% else %} - {% set anl_offset = "0H" %} - {% endif %} - {% set anl_timedelta = anl_offset | to_timedelta %} - {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} - {% for itile in range(1,7) %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" - {% endfor %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_air" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_pc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" - {% if DO_JEDISNOWDA %} - {% for itile in range(1,7) %} - # Snow analysis is 3dvar - - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/snowinc.{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" - - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" - {% endfor %} - {% endif %} - {% endif %} - optional: - {% if not current_cycle == SDATE and MODE == "cycled" %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" - {% if DO_JEDISNOWDA %} - - "{{ COM_CONF | relpath(ROTDIR) }}/{{ head }}letkfoi.yaml" - {% endif %} - {% endif %} +gdas_restarta: + {% set head = "gdas.t" + cycle_HH + "z." %} + name: "GDAS_RESTARTA" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restarta.tar" + required: + {% if not current_cycle == SDATE and MODE == "cycled" %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" + {% for iauhr in IAUHRS if iauhr != 6 %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" + {% endfor %} + {% if DOHYBVAR %} + {% set anl_offset = "-3H" %} + {% else %} + {% set anl_offset = "0H" %} + {% endif %} + {% set anl_timedelta = anl_offset | to_timedelta %} + {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} + {% for itile in range(1,7) %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" + {% endfor %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_air" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_pc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" + {% if DO_JEDISNOWDA %} + {% for itile in range(1,7) %} + # Snow analysis is 3dvar + - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/snowinc.{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" + - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.sfc_data.tile{{ itile }}.nc" + {% endfor %} + {% endif %} + {% endif %} + optional: + {% if not current_cycle == SDATE and MODE == "cycled" %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" + {% if DO_JEDISNOWDA %} + - "{{ COM_CONF | relpath(ROTDIR) }}/{{ head }}letkfoi.yaml" + {% endif %} + {% endif %} diff --git a/parm/archive/gdas_restartb.yaml.j2 b/parm/archive/gdas_restartb.yaml.j2 index 5d6215166f..ec7c723a1d 100644 --- a/parm/archive/gdas_restartb.yaml.j2 +++ b/parm/archive/gdas_restartb.yaml.j2 @@ -1,19 +1,19 @@ - gdas_restartb: - {% set head = "gdas.t" + cycle_HH + "z." %} - name: "GDAS_RESTARTB" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restartb.tar" - required: - {% for r_time in range(restart_interval_gdas, FHMAX + 1, restart_interval_gdas) %} - {% set r_timedelta = (r_time | string + "H") | to_timedelta %} - {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} - {% set r_YMD = r_dt | to_YMD %} - {% set r_HH = r_dt | strftime("%H") %} - {% set r_prefix = r_YMD + "." + r_HH + "0000" %} - {% for itile in range(1, 7) %} - {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" - {% endfor %} - {% endfor %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" - {% endfor %} +gdas_restartb: + {% set head = "gdas.t" + cycle_HH + "z." %} + name: "GDAS_RESTARTB" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restartb.tar" + required: + {% for r_time in range(restart_interval_gdas, FHMAX + 1, restart_interval_gdas) %} + {% set r_timedelta = (r_time | string + "H") | to_timedelta %} + {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} + {% set r_YMD = r_dt | to_YMD %} + {% set r_HH = r_dt | strftime("%H") %} + {% set r_prefix = r_YMD + "." + r_HH + "0000" %} + {% for itile in range(1, 7) %} + {% for datatype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" + {% endfor %} + {% endfor %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" + {% endfor %} diff --git a/parm/archive/gdasice.yaml.j2 b/parm/archive/gdasice.yaml.j2 index f735232c54..7a266384ef 100644 --- a/parm/archive/gdasice.yaml.j2 +++ b/parm/archive/gdasice.yaml.j2 @@ -1,8 +1,8 @@ - gdasice: - {% set head = "gdas.ice.t" + cycle_HH + "z." %} - name: "GDASICE" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar" - required: - # TODO explicitly name the files in history to archive - - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}*' - - '{{ COM_CONF | relpath(ROTDIR) }}/ufs.ice_in' +gdasice: + {% set head = "gdas.ice.t" + cycle_HH + "z." %} + name: "GDASICE" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar" + required: + # TODO explicitly name the files in history to archive + - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}*' + - '{{ COM_CONF | relpath(ROTDIR) }}/ufs.ice_in' diff --git a/parm/archive/gdasice_restart.yaml.j2 b/parm/archive/gdasice_restart.yaml.j2 index 9291e06929..39877674fb 100644 --- a/parm/archive/gdasice_restart.yaml.j2 +++ b/parm/archive/gdasice_restart.yaml.j2 @@ -1,7 +1,7 @@ - gdasice_restart: - {% set head = "gdas.ice.t" + cycle_HH + "z." %} - name: "GDASICE_RESTART" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice_restart.tar" - required: - # TODO explicitly name the restart files to archive - - '{{ COM_ICE_RESTART | relpath(ROTDIR) }}/*' +gdasice_restart: + {% set head = "gdas.ice.t" + cycle_HH + "z." %} + name: "GDASICE_RESTART" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice_restart.tar" + required: + # TODO explicitly name the restart files to archive + - '{{ COM_ICE_RESTART | relpath(ROTDIR) }}/*' diff --git a/parm/archive/gdasocean.yaml.j2 b/parm/archive/gdasocean.yaml.j2 index 0cd679c9d7..f3028a90ed 100644 --- a/parm/archive/gdasocean.yaml.j2 +++ b/parm/archive/gdasocean.yaml.j2 @@ -1,8 +1,8 @@ - gdasocean: - {% set head = "gdas.ocean.t" + cycle_HH + "z." %} - name: "GDASOCEAN" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar" - required: - # TODO Explicitly name the history and input files to archive - - '{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}*' - - '{{ COM_OCEAN_INPUT | relpath(ROTDIR) }}/.' +gdasocean: + {% set head = "gdas.ocean.t" + cycle_HH + "z." %} + name: "GDASOCEAN" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar" + required: + # TODO Explicitly name the history and input files to archive + - '{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}*' + - '{{ COM_OCEAN_INPUT | relpath(ROTDIR) }}/.' diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index 9fd4d4e936..4bc2b693d5 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -1,10 +1,10 @@ - gdasocean_analysis: - {% set head = "gdas.ocean.t" + cycle_HH + "z." %} - name: "GDASOCEAN_ANALYSIS" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar" - required: - # TODO explicitly name the analysis files to archive - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}*' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t??z.ocngrid.nc' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml' +gdasocean_analysis: + {% set head = "gdas.ocean.t" + cycle_HH + "z." %} + name: "GDASOCEAN_ANALYSIS" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar" + required: + # TODO explicitly name the analysis files to archive + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}*' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t??z.ocngrid.nc' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml' diff --git a/parm/archive/gdasocean_restart.yaml.j2 b/parm/archive/gdasocean_restart.yaml.j2 index ebb701a0c7..21bfc3955f 100644 --- a/parm/archive/gdasocean_restart.yaml.j2 +++ b/parm/archive/gdasocean_restart.yaml.j2 @@ -1,8 +1,8 @@ - gdasocean_restart: - {% set head = "gdas.ocean.t" + cycle_HH + "z." %} - name: "GDASOCEAN_RESTART" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_restart.tar" - required: - # TODO explicitly name the restart files to archive - - '{{ COM_OCEAN_RESTART | relpath(ROTDIR) }}/*' - - '{{ COM_MED_RESTART | relpath(ROTDIR) }}/*' +gdasocean_restart: + {% set head = "gdas.ocean.t" + cycle_HH + "z." %} + name: "GDASOCEAN_RESTART" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_restart.tar" + required: + # TODO explicitly name the restart files to archive + - '{{ COM_OCEAN_RESTART | relpath(ROTDIR) }}/*' + - '{{ COM_MED_RESTART | relpath(ROTDIR) }}/*' diff --git a/parm/archive/gdaswave.yaml.j2 b/parm/archive/gdaswave.yaml.j2 index 092c8f176e..74a5a64dbf 100644 --- a/parm/archive/gdaswave.yaml.j2 +++ b/parm/archive/gdaswave.yaml.j2 @@ -1,8 +1,8 @@ - gdaswave: - {% set head = "gdas.wave.t" + cycle_HH + "z." %} - name: "GDASWAVE" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave.tar" - required: - # TODO explicitly name the wave grid/station files to archive - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}*" +gdaswave: + {% set head = "gdas.wave.t" + cycle_HH + "z." %} + name: "GDASWAVE" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave.tar" + required: + # TODO explicitly name the wave grid/station files to archive + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}*" diff --git a/parm/archive/gdaswave_restart.yaml.j2 b/parm/archive/gdaswave_restart.yaml.j2 index ace018de28..8387d48616 100644 --- a/parm/archive/gdaswave_restart.yaml.j2 +++ b/parm/archive/gdaswave_restart.yaml.j2 @@ -1,6 +1,6 @@ - gdaswave_restart: - name: "GDASWAVE_RESTART" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave_restart.tar" - required: - # TODO explicitly name the wave restart files to archive - - "{{ COM_WAVE_RESTART | relpath(ROTDIR) }}/*" +gdaswave_restart: + name: "GDASWAVE_RESTART" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave_restart.tar" + required: + # TODO explicitly name the wave restart files to archive + - "{{ COM_WAVE_RESTART | relpath(ROTDIR) }}/*" diff --git a/parm/archive/gfs_downstream.yaml.j2 b/parm/archive/gfs_downstream.yaml.j2 index 329c426504..23c9383d28 100644 --- a/parm/archive/gfs_downstream.yaml.j2 +++ b/parm/archive/gfs_downstream.yaml.j2 @@ -1,12 +1,12 @@ - gfs_downstream: - {% set head = "gfs.t" + cycle_HH + "z." %} - name: "GFS_DOWNSTREAM" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_downstream.tar" - required: - - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.sfc" - - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.snd" - {% for i in range(1, NUM_SND_COLLECTIVES) %} - - "{{ COM_ATMOS_WMO | relpath(ROTDIR) }}/gfs_collective{{ i }}.postsnd_{{ cycle_HH }}" - {% endfor %} - - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/bufr.t{{ cycle_HH }}z" - - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/gfs.t{{ cycle_HH }}z.bufrsnd.tar.gz" +gfs_downstream: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_DOWNSTREAM" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_downstream.tar" + required: + - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.sfc" + - "{{ COM_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.snd" + {% for i in range(1, NUM_SND_COLLECTIVES) %} + - "{{ COM_ATMOS_WMO | relpath(ROTDIR) }}/gfs_collective{{ i }}.postsnd_{{ cycle_HH }}" + {% endfor %} + - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/bufr.t{{ cycle_HH }}z" + - "{{ COM_ATMOS_BUFR | relpath(ROTDIR) }}/gfs.t{{ cycle_HH }}z.bufrsnd.tar.gz" diff --git a/parm/archive/gfs_flux.yaml.j2 b/parm/archive/gfs_flux.yaml.j2 index 202dc38855..66c8221f60 100644 --- a/parm/archive/gfs_flux.yaml.j2 +++ b/parm/archive/gfs_flux.yaml.j2 @@ -1,9 +1,9 @@ - gfs_flux: - {% set head = "gfs.t" + cycle_HH + "z." %} - name: "GFS_FLUX" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux.tar" - required: - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" - {% endfor %} +gfs_flux: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_FLUX" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux.tar" + required: + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" + - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" + {% endfor %} diff --git a/parm/archive/gfs_flux_1p00.yaml.j2 b/parm/archive/gfs_flux_1p00.yaml.j2 index 95c8ad6642..2f5c9c8910 100644 --- a/parm/archive/gfs_flux_1p00.yaml.j2 +++ b/parm/archive/gfs_flux_1p00.yaml.j2 @@ -1,9 +1,9 @@ - gfs_flux_1p00: - {% set head = "gfs.t" + cycle_HH + "z." %} - name: "GFS_FLUX_1P00" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux_1p00.tar" - required: - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}.idx" - {% endfor %} +gfs_flux_1p00: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_FLUX_1P00" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux_1p00.tar" + required: + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}.idx" + {% endfor %} diff --git a/parm/archive/gfs_netcdfa.yaml.j2 b/parm/archive/gfs_netcdfa.yaml.j2 index 6ea487125e..4f3ed4fedc 100644 --- a/parm/archive/gfs_netcdfa.yaml.j2 +++ b/parm/archive/gfs_netcdfa.yaml.j2 @@ -1,14 +1,14 @@ - gfs_netcdfa: - {% set head = "gfs.t" + cycle_HH + "z." %} - name: "GFS_NETCDFA" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfa.tar" - required: - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" - {% for iauhr in IAUHRS if iauhr != 6 %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" - {% endfor %} - optional: - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" +gfs_netcdfa: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_NETCDFA" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfa.tar" + required: + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" + {% for iauhr in IAUHRS if iauhr != 6 %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" + {% endfor %} + optional: + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" diff --git a/parm/archive/gfs_netcdfb.yaml.j2 b/parm/archive/gfs_netcdfb.yaml.j2 index 1f2a7d0d8f..b0393d63b6 100644 --- a/parm/archive/gfs_netcdfb.yaml.j2 +++ b/parm/archive/gfs_netcdfb.yaml.j2 @@ -1,9 +1,9 @@ - gfs_netcdfb: - {% set head = "gfs.t" + cycle_HH + "z." %} - name: "GFS_NETCDFB" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfb.tar" - required: - {% for fhr in range(0, ARCH_GAUSSIAN_FHMAX + ARCH_GAUSSIAN_FHINC, ARCH_GAUSSIAN_FHINC) %} - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" - {% endfor %} +gfs_netcdfb: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_NETCDFB" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfb.tar" + required: + {% for fhr in range(0, ARCH_GAUSSIAN_FHMAX + ARCH_GAUSSIAN_FHINC, ARCH_GAUSSIAN_FHINC) %} + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/gfs_pgrb2b.yaml.j2 b/parm/archive/gfs_pgrb2b.yaml.j2 index c9e0ae2f65..b06dd14b73 100644 --- a/parm/archive/gfs_pgrb2b.yaml.j2 +++ b/parm/archive/gfs_pgrb2b.yaml.j2 @@ -1,19 +1,19 @@ - gfs_pgrb2b: - {% set head = "gfs.t" + cycle_HH + "z." %} - name: "GFS_PGRB2B" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_pgrb2b.tar" - required: - {% if MODE == "cycled" %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl.idx" - {% endif %} - {% if ARCH_GAUSSIAN %} - {% for fhr in range(0, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}.idx" - {% endfor %} - {% endif %} +gfs_pgrb2b: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_PGRB2B" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_pgrb2b.tar" + required: + {% if MODE == "cycled" %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.anl.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.anl.idx" + {% endif %} + {% if ARCH_GAUSSIAN %} + {% for fhr in range(0, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2b.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2b.1p00.f{{ '%03d' % fhr }}.idx" + {% endfor %} + {% endif %} diff --git a/parm/archive/gfs_restarta.yaml.j2 b/parm/archive/gfs_restarta.yaml.j2 index 461dba3e6f..23aed86285 100644 --- a/parm/archive/gfs_restarta.yaml.j2 +++ b/parm/archive/gfs_restarta.yaml.j2 @@ -1,23 +1,23 @@ - gfs_restarta: - {% set head = "gfs.t" + cycle_HH + "z." %} - name: "GFS_RESTARTA" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_restarta.tar" - required: - {% if MODE == "cycled" %} - {% if DOHYBVAR %} - {% set anl_offset = "-3H" %} - {% else %} - {% set anl_offset = "0H" %} - {% endif %} - {% set anl_timedelta = anl_offset | to_timedelta %} - {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} - {% for i_tile in range(1, 7) %} - - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ i_tile }}.nc" - {% endfor %} - {% elif MODE == "forecast-only" %} - - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc" - {% for i_tile in range(1, 7) %} - - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_data.tile{{ i_tile }}.nc" - - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/sfc_data.tile{{ i_tile }}.nc" - {% endfor %} - {% endif %} +gfs_restarta: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFS_RESTARTA" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_restarta.tar" + required: + {% if MODE == "cycled" %} + {% if DOHYBVAR %} + {% set anl_offset = "-3H" %} + {% else %} + {% set anl_offset = "0H" %} + {% endif %} + {% set anl_timedelta = anl_offset | to_timedelta %} + {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} + {% for i_tile in range(1, 7) %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ i_tile }}.nc" + {% endfor %} + {% elif MODE == "forecast-only" %} + - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc" + {% for i_tile in range(1, 7) %} + - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_data.tile{{ i_tile }}.nc" + - "{{ COM_ATMOS_INPUT | relpath(ROTDIR) }}/sfc_data.tile{{ i_tile }}.nc" + {% endfor %} + {% endif %} diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index c287a6ba9a..249924936c 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -1,61 +1,61 @@ - gfsa: - {% set head = "gfs.t" + cycle_HH + "z." %} - name: "GFSA" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsa.tar" - required: - # TODO explicitly name all logs to include - {% for log in glob("logs/" ~ cycle_YMDH ~ "/gfs*.log") %} - {% if log != "gfsarch.log" %} - - "{{ log }}" - {% endif %} - {% endfor %} - # - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" - # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" - - "{{ COM_CONF | relpath(ROTDIR) }}/ufs.input.nml" - {% if MODE == "cycled" %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" - # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" - # - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" - # - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" - # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" - {% if DO_VMINMON %} - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" - - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" - # - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" - {% endif %} - {% endif %} # Full cycle - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" - {% endfor %} - {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} - - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" - {% endif %} - optional: - {% if MODE == "cycled" %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" - - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" - {% if DO_JEDIATMVAR %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" - {% endif %} - {% endif %} # cycled - - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cyclone.trackatcfunix" - - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avnop.t{{ cycle_HH }}z.cyclone.trackatcfunix" - - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}" - - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}" - # {% if not WRITE_DOPOST %} - # # TODO set the forecast hours explicitly. This will require emulating numpy.array_split - # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" - # {% endif %} ## not WRITE_DOPOST +gfsa: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFSA" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsa.tar" + required: + # TODO explicitly name all logs to include + {% for log in glob("logs/" ~ cycle_YMDH ~ "/gfs*.log") %} + {% if log != "gfsarch.log" %} + - "{{ log }}" + {% endif %} + {% endfor %} + # - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" + - "{{ COM_CONF | relpath(ROTDIR) }}/ufs.input.nml" + {% if MODE == "cycled" %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" + {% if DO_VMINMON %} + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" + - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" + # - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" + {% endif %} + {% endif %} # Full cycle + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" + {% endfor %} + {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} + - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" + {% endif %} + optional: + {% if MODE == "cycled" %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" + {% if DO_JEDIATMVAR %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" + {% endif %} + {% endif %} # cycled + - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cyclone.trackatcfunix" + - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avnop.t{{ cycle_HH }}z.cyclone.trackatcfunix" + - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}" + - "{{ COM_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}" + # {% if not WRITE_DOPOST %} + # # TODO set the forecast hours explicitly. This will require emulating numpy.array_split + # - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" + # {% endif %} ## not WRITE_DOPOST diff --git a/parm/archive/gfsb.yaml.j2 b/parm/archive/gfsb.yaml.j2 index 1b4c4f931c..721b529278 100644 --- a/parm/archive/gfsb.yaml.j2 +++ b/parm/archive/gfsb.yaml.j2 @@ -1,17 +1,17 @@ - gfsb: - {% set head = "gfs.t" + cycle_HH + "z." %} - name: "GFSB" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsb.tar" - required: - {% if MODE == "cycled" %} - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl" - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" - {% endif %} - {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}.idx" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" - - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" - {% endfor %} +gfsb: + {% set head = "gfs.t" + cycle_HH + "z." %} + name: "GFSB" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsb.tar" + required: + {% if MODE == "cycled" %} + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl" + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.anl.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl.idx" + {% endif %} + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p50.f{{ '%03d' % fhr }}.idx" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" + - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}.idx" + {% endfor %} diff --git a/parm/archive/gfswave.yaml.j2 b/parm/archive/gfswave.yaml.j2 index a24374feca..8542afac0c 100644 --- a/parm/archive/gfswave.yaml.j2 +++ b/parm/archive/gfswave.yaml.j2 @@ -1,23 +1,23 @@ - gfswave: - {% set head = "gfswave.t" + cycle_HH + "z." %} - name: "GFSWAVE" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfswave.tar" - required: - {% for fh in range(0, FHMAX_HF_WAV + FHOUT_HF_WAV, FHOUT_HF_WAV) %} - # NOTE This is as explicit as possible without major logic to parse wavepostGRD. - # Matches files of the form "gfswave.tCCz...fHHH.grib2". - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" - {% endfor %} - {% for fh in range(FHMAX_HF_WAV + FHOUT_WAV, FHMAX_WAV_GFS + FHOUT_WAV, FHOUT_WAV) %} - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" - - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" - {% endfor %} - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}bull_tar" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}cbull_tar" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}spec_tar.gz" - {% if DOIBP_WAV %} - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibpbull_tar" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibpcbull_tar" - - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibp_tar" - {% endif %} +gfswave: + {% set head = "gfswave.t" + cycle_HH + "z." %} + name: "GFSWAVE" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfswave.tar" + required: + {% for fh in range(0, FHMAX_HF_WAV + FHOUT_HF_WAV, FHOUT_HF_WAV) %} + # NOTE This is as explicit as possible without major logic to parse wavepostGRD. + # Matches files of the form "gfswave.tCCz...fHHH.grib2". + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" + {% endfor %} + {% for fh in range(FHMAX_HF_WAV + FHOUT_WAV, FHMAX_WAV_GFS + FHOUT_WAV, FHOUT_WAV) %} + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" + - "{{ COM_WAVE_GRID | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" + {% endfor %} + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}bull_tar" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}cbull_tar" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}spec_tar.gz" + {% if DOIBP_WAV %} + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibpbull_tar" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibpcbull_tar" + - "{{ COM_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibp_tar" + {% endif %} diff --git a/parm/archive/ice_6hravg.yaml.j2 b/parm/archive/ice_6hravg.yaml.j2 index 9e01047530..251e51b110 100644 --- a/parm/archive/ice_6hravg.yaml.j2 +++ b/parm/archive/ice_6hravg.yaml.j2 @@ -1,9 +1,9 @@ - ice_6hravg: - {% set head = "gfs.ice.t" + cycle_HH + "z." %} - name: "ICE_6HRAVG" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_6hravg.tar" - required: - - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}ic.nc" - {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" - {% endfor %} +ice_6hravg: + {% set head = "gfs.ice.t" + cycle_HH + "z." %} + name: "ICE_6HRAVG" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_6hravg.tar" + required: + - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}ic.nc" + {% for fhr in range(6, FHMAX_GFS + 6, 6) %} + - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/ice_grib2.yaml.j2 b/parm/archive/ice_grib2.yaml.j2 index a5b3cdc164..42e6910a16 100644 --- a/parm/archive/ice_grib2.yaml.j2 +++ b/parm/archive/ice_grib2.yaml.j2 @@ -1,18 +1,18 @@ - ice_grib2: - {% set head = "gfs.ice.t" + cycle_HH + "z." %} - name: "ICE_GRIB2" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_grib2.tar" - required: - {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %} - {% set fhr3 = '%03d' % fhr %} - {% if ICERES == 500 %} - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" - {% elif ICERES == 100 %} - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" - {% elif ICERES == 25 or ICERES == "025" %} - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" - - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" - {% endif %} - {% endfor %} +ice_grib2: + {% set head = "gfs.ice.t" + cycle_HH + "z." %} + name: "ICE_GRIB2" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_grib2.tar" + required: + {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %} + {% set fhr3 = '%03d' % fhr %} + {% if ICERES == 500 %} + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" + {% elif ICERES == 100 %} + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" + {% elif ICERES == 25 or ICERES == "025" %} + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" + - "{{ COM_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" + {% endif %} + {% endfor %} diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index 37e2bb5769..885fb86ac2 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -14,7 +14,9 @@ datasets: {% if ENSGRP == 0 %} +{% filter indent(width=4) %} {% include "enkf.yaml.j2" %} +{% endfilter %} {% else %} # Declare to-be-filled lists of member COM directories @@ -58,10 +60,14 @@ datasets: {% endfor %} # Determine which members to archive +{% filter indent(width=4) %} {% include "enkf_grp.yaml.j2" %} +{% endfilter %} {% if current_cycle > SDATE %} +{% filter indent(width=4) %} {% include "enkf_restarta_grp.yaml.j2" %} {% include "enkf_restartb_grp.yaml.j2" %} +{% endfilter %} {% endif %} {% endif %} # ENSGRP != 0 diff --git a/parm/archive/master_gdas.yaml.j2 b/parm/archive/master_gdas.yaml.j2 index 52e7b87af5..3cc8eabb80 100644 --- a/parm/archive/master_gdas.yaml.j2 +++ b/parm/archive/master_gdas.yaml.j2 @@ -3,19 +3,27 @@ {% set cycle_YMDH = current_cycle | to_YMDH %} {% set head = "gdas.t" + cycle_HH + "z." %} datasets: +{% filter indent(width=4) %} {% include "gdas.yaml.j2" %} +{% endfilter %} {% if DO_ICE %} +{% filter indent(width=4) %} {% include "gdasice.yaml.j2" %} +{% endfilter %} {% endif %} {% if DO_OCEAN %} +{% filter indent(width=4) %} {% include "gdasocean.yaml.j2" %} {% include "gdasocean_analysis.yaml.j2" %} +{% endfilter %} {% endif %} {% if DO_WAVE %} +{% filter indent(width=4) %} {% include "gdaswave.yaml.j2" %} +{% endfilter %} {% endif %} # Determine if we will save restart ICs or not @@ -33,23 +41,33 @@ datasets: {% endif %} {% if save_warm_start_forecast %} +{% filter indent(width=4) %} {% include "gdas_restarta.yaml.j2" %} +{% endfilter %} {% if DO_WAVE %} +{% filter indent(width=4) %} {% include "gdaswave_restart.yaml.j2" %} +{% endfilter %} {% endif %} {% if DO_OCN %} +{% filter indent(width=4) %} {% include "gdasocean_restart.yaml.j2" %} +{% endfilter %} {% endif %} {% if DO_ICE %} +{% filter indent(width=4) %} {% include "gdasice_restart.yaml.j2" %} +{% endfilter %} {% endif %} {% endif %} # Save forecast-only restarts # Save cycled restarts {% if save_warm_start_cycled %} +{% filter indent(width=4) %} {% include "gdas_restartb.yaml.j2" %} +{% endfilter %} {% endif %} diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2 index 199efec23c..199546bf90 100644 --- a/parm/archive/master_gfs.yaml.j2 +++ b/parm/archive/master_gfs.yaml.j2 @@ -2,51 +2,71 @@ {% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_YMDH = current_cycle | to_YMDH %} datasets: +{% filter indent(width=4) %} {% include "gfsa.yaml.j2" %} {% include "gfsb.yaml.j2" %} +{% endfilter %} {% if ARCH_GAUSSIAN %} +{% filter indent(width=4) %} {% include "gfs_flux.yaml.j2" %} {% include "gfs_netcdfb.yaml.j2" %} {% include "gfs_pgrb2b.yaml.j2" %} +{% endfilter %} {% if MODE == "cycled" %} +{% filter indent(width=4) %} {% include "gfs_netcdfa.yaml.j2" %} +{% endfilter %} {% endif %} {% endif %} {% if DO_WAVE %} +{% filter indent(width=4) %} {% include "gfswave.yaml.j2" %} +{% endfilter %} {% endif %} {% if AERO_FCST_CDUMP == "gfs" or AERO_FCST_CDUMP == "both" %} +{% filter indent(width=4) %} {% include "chem.yaml.j2" %} +{% endfilter %} {% endif %} {% if DO_BUFRSND %} +{% filter indent(width=4) %} {% include "gfs_downstream.yaml.j2" %} +{% endfilter %} {% endif %} {% if DO_OCEAN %} +{% filter indent(width=4) %} {% include "ocean_6hravg.yaml.j2" %} {% include "ocean_daily.yaml.j2" %} {% include "ocean_grib2.yaml.j2" %} {% include "gfs_flux_1p00.yaml.j2" %} +{% endfilter %} {% endif %} {% if DO_ICE %} +{% filter indent(width=4) %} {% include "ice_6hravg.yaml.j2" %} {% include "ice_grib2.yaml.j2" %} +{% endfilter %} {% endif %} # Determine whether to save the MOS tarball {% if DO_MOS and cycle_HH == "18" %} {% if not REALTIME %} +{% filter indent(width=4) %} {% include "gfsmos.yaml.j2" %} +{% endfilter %} {% else %} {% set td_from_sdate = current_cycle - SDATE %} {% set td_one_day = "+1D" | to_timedelta %} {% if td_from_sdate > td_one_day %} +{% filter indent(width=4) %} {% include "gfsmos.yaml.j2" %} +{% endfilter %} {% endif %} {% endif %} {% endif %} @@ -55,8 +75,12 @@ datasets: {% if ARCH_CYC == cycle_HH | int and current_cycle != SDATE %} # Save the forecast-only cycle ICs every ARCH_WARMICFREQ or ARCH_FCSTICFREQ days {% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %} +{% filter indent(width=4) %} {% include "gfs_restarta.yaml.j2" %} +{% endfilter %} {% elif (current_cycle - SDATE).days % ARCH_FCSTICFREQ == 0 %} +{% filter indent(width=4) %} {% include "gfs_restarta.yaml.j2" %} +{% endfilter %} {% endif %} {% endif %} diff --git a/parm/archive/ocean_6hravg.yaml.j2 b/parm/archive/ocean_6hravg.yaml.j2 index 4b7a2d3c41..dac3ce262a 100644 --- a/parm/archive/ocean_6hravg.yaml.j2 +++ b/parm/archive/ocean_6hravg.yaml.j2 @@ -1,8 +1,8 @@ - ocean_6hravg: - {% set head = "gfs.ocean.t" + cycle_HH + "z." %} - name: "OCEAN_6HRAVG" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_6hravg.tar" - required: - {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" - {% endfor %} +ocean_6hravg: + {% set head = "gfs.ocean.t" + cycle_HH + "z." %} + name: "OCEAN_6HRAVG" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_6hravg.tar" + required: + {% for fhr in range(6, FHMAX_GFS + 6, 6) %} + - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}6hr_avg.f{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/ocean_daily.yaml.j2 b/parm/archive/ocean_daily.yaml.j2 index 85a095136c..0f45264973 100644 --- a/parm/archive/ocean_daily.yaml.j2 +++ b/parm/archive/ocean_daily.yaml.j2 @@ -1,8 +1,8 @@ - ocean_daily: - {% set head = "gfs.ocean.t" + cycle_HH + "z." %} - name: "OCEAN_DAILY" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_daily.tar" - required: - {% for fhr in range(24, FHMAX_GFS + 24, 24) %} - - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}daily.f{{ '%03d' % fhr }}.nc" - {% endfor %} +ocean_daily: + {% set head = "gfs.ocean.t" + cycle_HH + "z." %} + name: "OCEAN_DAILY" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_daily.tar" + required: + {% for fhr in range(24, FHMAX_GFS + 24, 24) %} + - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}daily.f{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/ocean_grib2.yaml.j2 b/parm/archive/ocean_grib2.yaml.j2 index 2732fa3e2c..2e63c0ca98 100644 --- a/parm/archive/ocean_grib2.yaml.j2 +++ b/parm/archive/ocean_grib2.yaml.j2 @@ -1,18 +1,18 @@ - ocean_grib2: - {% set head = "gfs.ocean.t" + cycle_HH + "z." %} - name: "OCEAN_GRIB2" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_grib2.tar" - required: - {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %} - {% set fhr3 = '%03d' % fhr %} - {% if OCNRES == 500 %} - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" - {% elif OCNRES == 100 %} - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" - {% elif OCNRES == 25 or OCNRES == "025" %} - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" - - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" - {% endif %} - {% endfor %} +ocean_grib2: + {% set head = "gfs.ocean.t" + cycle_HH + "z." %} + name: "OCEAN_GRIB2" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_grib2.tar" + required: + {% for fhr in range(FHOUT_OCNICE_GFS, FHMAX_GFS + FHOUT_OCNICE_GFS, FHOUT_OCNICE_GFS) %} + {% set fhr3 = '%03d' % fhr %} + {% if OCNRES == 500 %} + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2" + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ head }}5p00.f{{ fhr3 }}.grib2.idx" + {% elif OCNRES == 100 %} + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2" + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ head }}1p00.f{{ fhr3 }}.grib2.idx" + {% elif OCNRES == 25 or OCNRES == "025" %} + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2" + - "{{ COM_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ head }}0p25.f{{ fhr3 }}.grib2.idx" + {% endif %} + {% endfor %} From f1b8e88633c25d78cb7e0ff0c07e19be5b5bd0e9 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 9 May 2024 12:47:54 +0000 Subject: [PATCH 087/128] Consolidated enkf heads to master template --- parm/archive/enkf.yaml.j2 | 1 - parm/archive/enkf_grp.yaml.j2 | 1 - parm/archive/enkf_restarta_grp.yaml.j2 | 1 - parm/archive/enkf_restartb_grp.yaml.j2 | 1 - parm/archive/master_enkf.yaml.j2 | 1 + 5 files changed, 1 insertion(+), 4 deletions(-) diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index e785febfe2..e6a144b530 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -1,5 +1,4 @@ enkf: - {% set head = RUN + ".t" + cycle_HH + "z." %} name: "ENKF" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}.tar" required: diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index 70b5d8e25e..1294eb4756 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -1,5 +1,4 @@ enkf_grp: - {% set head = RUN + ".t" + cycle_HH + "z." %} name: "ENKF_GRP" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_grp{{ ENSGRP }}.tar" required: diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 index 4245c7e218..845895359a 100644 --- a/parm/archive/enkf_restarta_grp.yaml.j2 +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -1,5 +1,4 @@ enkf_restarta_grp: - {% set head = RUN + ".t" + cycle_HH + "z." %} name: "ENKF_RESTARTA_GRP" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restarta_grp{{ ENSGRP }}.tar" required: diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 index 268c06ec82..688a37fd71 100644 --- a/parm/archive/enkf_restartb_grp.yaml.j2 +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -1,5 +1,4 @@ enkf_restartb_grp: - {% set head = RUN + ".t" + cycle_HH + "z." %} name: "ENKF_RESTARTB_GRP" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restartb_grp{{ ENSGRP }}.tar" required: diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index 885fb86ac2..bd9fa7b8c0 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -2,6 +2,7 @@ {% set cycle_HH = current_cycle | strftime("%H") %} {% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_YMDH = current_cycle | to_YMDH %} +{% set head = RUN + ".t" + cycle_HH + "z." %} {% if IAUFHRS is string %} {% set iaufhrs = [] %} From 8f8ed43d443349d3f60d348b99f7548239e7250b Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 9 May 2024 12:48:40 +0000 Subject: [PATCH 088/128] Set PYTHONPATH to head of ush/python --- jobs/rocoto/arch.sh | 3 +-- jobs/rocoto/earc.sh | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/jobs/rocoto/arch.sh b/jobs/rocoto/arch.sh index 9dd1a0309e..083e319bf5 100755 --- a/jobs/rocoto/arch.sh +++ b/jobs/rocoto/arch.sh @@ -10,8 +10,7 @@ status=$? ############################################################### # setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python" export PYTHONPATH export job="arch" diff --git a/jobs/rocoto/earc.sh b/jobs/rocoto/earc.sh index c24374846e..4a9263b509 100755 --- a/jobs/rocoto/earc.sh +++ b/jobs/rocoto/earc.sh @@ -10,8 +10,7 @@ status=$? ############################################################### # setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python" export PYTHONPATH export job="earc" From be28de1ccc1ece450a2dedb85c85cfb3a226a1d9 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 9 May 2024 12:50:40 +0000 Subject: [PATCH 089/128] Clean up archive.py --- ush/python/pygfs/task/archive.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 9aaabd4e18..dade674f24 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -3,14 +3,13 @@ import glob import os import shutil +import tarfile from logging import getLogger from typing import Any, Dict, List from wxflow import (AttrDict, FileHandler, Hsi, Htar, Task, cast_strdict_as_dtypedict, chgrp, get_gid, logit, mkdir_p, parse_j2yaml, rm_p, strftime, to_YMD, to_YMDH, Template, TemplateConstants) -from yaml import CLoader as Loader -from yaml import load logger = getLogger(__name__.split('.')[-1]) @@ -156,7 +155,7 @@ def execute_backup_dataset(self, atardir_set: Dict[str, Any]) -> None: # Generate tarball if len(atardir_set.fileset) == 0: - print(f"WARNING skipping would-be empty archive {atardir_set.target}.") + logger.warning(f"WARNING: skipping would-be empty archive {atardir_set.target}.") return if atardir_set.has_rstprod: @@ -206,7 +205,7 @@ def _create_fileset(atardir_set: Dict[str, Any]) -> List: for item in atardir_set.optional: glob_set = glob.glob(item) if len(glob_set) == 0: - print(f"WARNING: optional file/glob {item} not found!") + logger.warning(f"WARNING: optional file/glob {item} not found!") else: for entry in glob_set: fileset.append(entry) @@ -275,7 +274,6 @@ def _create_tarball(target: str, fileset: List) -> None: file_list : List List of files to add to an archive """ - import tarfile # TODO create a set of tar helper functions in wxflow # Attempt to create the parent directory if it does not exist From aaa3876b7dafcb3cd194b9c37ccdfc86acb3d23a Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 9 May 2024 14:11:10 +0000 Subject: [PATCH 090/128] Fix ocean archiving --- parm/archive/master_gdas.yaml.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/parm/archive/master_gdas.yaml.j2 b/parm/archive/master_gdas.yaml.j2 index 3cc8eabb80..4a41d5cfc3 100644 --- a/parm/archive/master_gdas.yaml.j2 +++ b/parm/archive/master_gdas.yaml.j2 @@ -13,12 +13,16 @@ datasets: {% endfilter %} {% endif %} -{% if DO_OCEAN %} +{% if DO_OCN %} {% filter indent(width=4) %} {% include "gdasocean.yaml.j2" %} +{% endfilter %} +{% if DO_JEDIOCNVAR %} +{% filter indent(width=4) %} {% include "gdasocean_analysis.yaml.j2" %} {% endfilter %} {% endif %} +{% endif %} {% if DO_WAVE %} {% filter indent(width=4) %} From 48578f7f6e1d6f8c0198fcdb3dfb9dad160c01c5 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 9 May 2024 14:36:18 +0000 Subject: [PATCH 091/128] Removed unused COM, updated gdasocean template --- jobs/JGLOBAL_ARCHIVE | 2 +- parm/archive/gdasocean.yaml.j2 | 7 ++++--- ush/forecast_postdet.sh | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index b6e2a6e17b..cec3505000 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -17,7 +17,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS COM_ATMOS_BUFR COM COM_SNOW_ANALYSIS \ COM_ICE_HISTORY COM_ICE_INPUT COM_ICE_RESTART COM_ICE_GRIB \ COM_OBS COM_TOP \ - COM_OCEAN_HISTORY COM_OCEAN_INPUT COM_OCEAN_RESTART COM_OCEAN_GRIB COM_OCEAN_NETCDF \ + COM_OCEAN_HISTORY COM_OCEAN_RESTART COM_OCEAN_GRIB COM_OCEAN_NETCDF \ COM_OCEAN_ANALYSIS \ COM_WAVE_GRID COM_WAVE_HISTORY COM_WAVE_STATION COM_WAVE_RESTART \ COM_ATMOS_OZNMON COM_ATMOS_RADMON COM_ATMOS_MINMON COM_CONF diff --git a/parm/archive/gdasocean.yaml.j2 b/parm/archive/gdasocean.yaml.j2 index f3028a90ed..630618aaf1 100644 --- a/parm/archive/gdasocean.yaml.j2 +++ b/parm/archive/gdasocean.yaml.j2 @@ -3,6 +3,7 @@ gdasocean: name: "GDASOCEAN" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar" required: - # TODO Explicitly name the history and input files to archive - - '{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}*' - - '{{ COM_OCEAN_INPUT | relpath(ROTDIR) }}/.' + {% for fhr in range(FHMIN, FHMAX + 1, FHMAX) %} + - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}inst.f{{ '%03d' % fhr }}.nc" + {% endfor %} + - '{{ COM_CONF | relpath(ROTDIR) }}/ufs.MOM_input' diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 8ea556055d..032bcc3678 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -482,7 +482,7 @@ MOM6_nml() { MOM6_out() { echo "SUB ${FUNCNAME[0]}: Copying output data for MOM6" - # Copy MOM_input from DATA to COM_OCEAN_INPUT after the forecast is run (and successfull) + # Copy MOM_input from DATA to COM_CONF after the forecast is run (and successfull) ${NCP} "${DATA}/INPUT/MOM_input" "${COM_CONF}/ufs.MOM_input" # Create a list of MOM6 restart files From e4506625af536e4215ec00eb16515555ccd482b5 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 9 May 2024 14:37:11 +0000 Subject: [PATCH 092/128] Explicitized gdasocean_analysis template --- parm/archive/gdasocean_analysis.yaml.j2 | 28 +++++++++++++++++++------ scripts/exglobal_archive.py | 3 ++- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index 4bc2b693d5..a8a2742f45 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -1,10 +1,26 @@ gdasocean_analysis: - {% set head = "gdas.ocean.t" + cycle_HH + "z." %} + {% set head = "gdas.t" + cycle_HH + "z." %} name: "GDASOCEAN_ANALYSIS" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar" required: - # TODO explicitly name the analysis files to archive - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}*' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t??z.ocngrid.nc' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags' - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocninc.nc' + {% if current_cycle != SDATE and MODE == "cycled" %} + {% set ocngrid_cycle = ((cycle_HH | int) - 3) % 24 %} + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t{{ ocngrid_cycle }}z.ocngrid.nc' + {% for domain in ["ocn", "ice"] %} + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.bkgerr_stddev.nc' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.incr.nc' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}ana.nc' + {% if NMEM_ENS > 2 %} + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.recentering_error.nc" + {% endif %} + {% endfor %} + {% if NMEM_ENS > 2 %} + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_steric_stddev.nc" + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_unbal_stddev.nc" + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_total_stddev.nc" + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.steric_explained_variance.nc" + {% endif %} + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags/*.nc4' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml/*.yaml' + {% endif %} # Not the first cycle diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index f2760179c3..044a11ee7a 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -29,7 +29,8 @@ def main(): 'DOIAU', 'OCNRES', 'ICERES', 'NUM_SND_COLLECTIVES', 'FHOUT_WAV', 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', 'restart_interval_gdas', 'restart_interval_gfs', - 'AERO_ANL_CDUMP', 'AERO_FCST_CDUMP', 'DOIBP_WAV'] + 'AERO_ANL_CDUMP', 'AERO_FCST_CDUMP', 'DOIBP_WAV', 'DO_JEDIOCNVAR', + 'NMEM_ENS'] archive_dict = AttrDict() for key in keys: From f4cd9aa896bc5e96ef35c1daca1b58991b8db43f Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 9 May 2024 15:43:33 +0000 Subject: [PATCH 093/128] Clean up some of the templates --- parm/archive/arcdir.yaml.j2 | 2 +- parm/archive/enkf.yaml.j2 | 2 +- parm/archive/enkf_grp.yaml.j2 | 14 ++++++-------- parm/archive/enkf_restarta_grp.yaml.j2 | 4 ++-- parm/archive/gdas.yaml.j2 | 4 ++-- parm/archive/gdas_restarta.yaml.j2 | 4 ++-- parm/archive/gdasice.yaml.j2 | 5 +++-- parm/archive/gdasocean_analysis.yaml.j2 | 2 +- 8 files changed, 18 insertions(+), 19 deletions(-) diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index 8610980c4a..9b0d04983b 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -16,7 +16,7 @@ deterministic: - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}" {% endif %} copy: - {% if not current_cycle == SDATE and MODE == "cycled" %} + {% if current_cycle != SDATE and MODE == "cycled" %} - ["{{ COM_ATMOS_ANALYSIS }}/{{ head }}gsistat", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}"] {% if DO_JEDISNOWDA %} - ["{{ COM_SNOW_ANALYSIS }}/{{ head }}snowstat.tgz", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YMDH }}.tgz"] diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index e6a144b530..8673475e07 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -9,7 +9,7 @@ enkf: - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensspread.nc" {% endif %} {% endfor %} - {% if not current_cycle == SDATE %} + {% if current_cycle != SDATE %} {% if not DO_JEDIATMENS %} {% set da_files = ["enkfstat", "gsistat.ensmean", diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index 1294eb4756..8d6d7f3e9d 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -8,15 +8,15 @@ enkf_grp: {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% if current_cycle != SDATE and not lobsdiag_forenkf %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat" - {% endif %} {% for fhr in range(fhmin, fhmax+fhout, fhout) %} - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" {% endfor %} - {% if not current_cycle == SDATE %} + {% if current_cycle != SDATE %} + {% if not lobsdiag_forenkf %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat" + {% endif %} {% if do_calc_increment %} - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atmanl.nc" {% else %} @@ -34,12 +34,10 @@ enkf_grp: {% endfor %} # iaufhr in iaufhrs {% endfor %} # first_group_mem to last_group_mem optional: + {% if current_cycle != SDATE and not lobsdiag_forenkf %} {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} - {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% if current_cycle != SDATE and not lobsdiag_forenkf %} - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" - {% endif %} {% endfor %} + {% endif %} diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 index 845895359a..73da805f28 100644 --- a/parm/archive/enkf_restarta_grp.yaml.j2 +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -22,10 +22,10 @@ enkf_restarta_grp: {% endfor %} {% endfor %} # first_group_mem to last_group_mem optional: + {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" - {% endif %} {% endfor %} + {% endif %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index ece6cb81ab..9464b39222 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -3,7 +3,7 @@ gdas: name: "GDAS" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas.tar" required: - {% if not current_cycle == SDATE and MODE == "cycled" %} + {% if current_cycle != SDATE and MODE == "cycled" %} - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl" - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.anl.idx" - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.anl" @@ -71,7 +71,7 @@ gdas: - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" {% endfor %} optional: - {% if not current_cycle == SDATE and MODE == "cycled" %} + {% if current_cycle != SDATE and MODE == "cycled" %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" {% if DO_VERFRAD %} - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 5e60cf9868..b51684d6bd 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -3,7 +3,7 @@ gdas_restarta: name: "GDAS_RESTARTA" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restarta.tar" required: - {% if not current_cycle == SDATE and MODE == "cycled" %} + {% if current_cycle != SDATE and MODE == "cycled" %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" {% for iauhr in IAUHRS if iauhr != 6 %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" @@ -31,7 +31,7 @@ gdas_restarta: {% endif %} {% endif %} optional: - {% if not current_cycle == SDATE and MODE == "cycled" %} + {% if current_cycle != SDATE and MODE == "cycled" %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" diff --git a/parm/archive/gdasice.yaml.j2 b/parm/archive/gdasice.yaml.j2 index 7a266384ef..f2ca58b926 100644 --- a/parm/archive/gdasice.yaml.j2 +++ b/parm/archive/gdasice.yaml.j2 @@ -3,6 +3,7 @@ gdasice: name: "GDASICE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar" required: - # TODO explicitly name the files in history to archive - - '{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}*' + {% for fhr in range(FHOUT_OCNICE, FHMAX+1, FHOUT_OCNICE) %} + - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}inst.f{{ '%03d' % fhr }}.nc" + {% endfor %} - '{{ COM_CONF | relpath(ROTDIR) }}/ufs.ice_in' diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index a8a2742f45..4d822370fd 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -5,7 +5,7 @@ gdasocean_analysis: required: - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocninc.nc' {% if current_cycle != SDATE and MODE == "cycled" %} - {% set ocngrid_cycle = ((cycle_HH | int) - 3) % 24 %} + {% set ocngrid_cycle = '%02d' % (((cycle_HH | int) - 3) % 24) %} - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t{{ ocngrid_cycle }}z.ocngrid.nc' {% for domain in ["ocn", "ice"] %} - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.bkgerr_stddev.nc' From 2e3b461b9bf8e4a6537b54e6b23a2b76c750b5db Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 10 May 2024 15:12:28 +0000 Subject: [PATCH 094/128] Do not archive ocean analysis on the first cycle --- parm/archive/gdasocean_analysis.yaml.j2 | 2 -- parm/archive/master_gdas.yaml.j2 | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index 4d822370fd..74a191b024 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -4,7 +4,6 @@ gdasocean_analysis: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar" required: - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocninc.nc' - {% if current_cycle != SDATE and MODE == "cycled" %} {% set ocngrid_cycle = '%02d' % (((cycle_HH | int) - 3) % 24) %} - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t{{ ocngrid_cycle }}z.ocngrid.nc' {% for domain in ["ocn", "ice"] %} @@ -23,4 +22,3 @@ gdasocean_analysis: {% endif %} - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags/*.nc4' - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml/*.yaml' - {% endif %} # Not the first cycle diff --git a/parm/archive/master_gdas.yaml.j2 b/parm/archive/master_gdas.yaml.j2 index 4a41d5cfc3..6ff6e40b30 100644 --- a/parm/archive/master_gdas.yaml.j2 +++ b/parm/archive/master_gdas.yaml.j2 @@ -17,7 +17,7 @@ datasets: {% filter indent(width=4) %} {% include "gdasocean.yaml.j2" %} {% endfilter %} -{% if DO_JEDIOCNVAR %} +{% if DO_JEDIOCNVAR and current_cycle != SDATE and MODE == "cycled" %} {% filter indent(width=4) %} {% include "gdasocean_analysis.yaml.j2" %} {% endfilter %} From 4e9ecc0a54dc00f850bd4a89c999287945fc1ab4 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 10 May 2024 18:40:44 +0000 Subject: [PATCH 095/128] Fix restart copies in forecast_postdet.sh --- ush/forecast_postdet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 6d4ec8cfcd..0257d17e23 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -239,7 +239,7 @@ FV3_out() { if [[ "${RUN}" =~ "gdas" || "${RUN}" == "enkfgfs" ]]; then local restart_date restart_date=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${restart_interval} hours" +%Y%m%d%H) - while (( restart_date < forecast_end_cycle )); do + while (( restart_date <= forecast_end_cycle )); do echo "Copying FV3 restarts for 'RUN=${RUN}' at ${restart_date}" for fv3_restart_file in "${fv3_restart_files[@]}"; do restart_file="${restart_date:0:8}.${restart_date:8:2}0000.${fv3_restart_file}" From c945de40c3cce6a7bbd0f52fc7b8c6c5d5ab88a8 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 10 May 2024 18:44:50 +0000 Subject: [PATCH 096/128] Add archive support for jedi ATM; fix reset time offsets when DOIAU=NO --- parm/archive/arcdir.yaml.j2 | 9 +++++++ parm/archive/enkf.yaml.j2 | 23 +++++++++++----- parm/archive/enkf_restarta_grp.yaml.j2 | 4 +++ parm/archive/gdas.yaml.j2 | 37 +++++++++++++++++++++++--- parm/archive/gdas_restarta.yaml.j2 | 4 ++- parm/archive/gfs_restarta.yaml.j2 | 2 +- scripts/exgdas_enkf_earc.py | 4 +-- scripts/exglobal_archive.py | 2 +- 8 files changed, 69 insertions(+), 16 deletions(-) diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index 9b0d04983b..7d2d1c7357 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -17,7 +17,11 @@ deterministic: {% endif %} copy: {% if current_cycle != SDATE and MODE == "cycled" %} + {% if DO_JEDIATMVAR %} + - ["{{ COM_ATMOS_ANALYSIS }}/{{ head }}atmstat", "{{ ARCDIR }}/atmstat.{{ RUN }}.{{ cycle_YMDH }}"] + {% else %} - ["{{ COM_ATMOS_ANALYSIS }}/{{ head }}gsistat", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}"] + {% endif %} {% if DO_JEDISNOWDA %} - ["{{ COM_SNOW_ANALYSIS }}/{{ head }}snowstat.tgz", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YMDH }}.tgz"] {% endif %} @@ -73,6 +77,11 @@ ensemble: mkdir: - "{{ ARCDIR }}" copy: + {% if DO_JEDIATMENS %} + - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}atmensstat", "{{ ARCDIR }}/atmensstat.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}atminc.ensmean.nc", "{{ ARCDIR }}/atmensstat.{{ RUN }}.{{ cycle_YMDH }}.ensmean.nc"] + {% else %} - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}enkfstat", "{{ ARCDIR }}/enkfstat.{{ RUN }}.{{ cycle_YMDH }}"] - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}gsistat.ensmean", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}.ensmean"] + {% endif %} {% endif %} # enkfgdas or enkfgfs diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index 8673475e07..7b406fba8e 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -36,19 +36,28 @@ enkf: {% endif %} # fhr == IAU_OFFSET {% endfor %} # IAUFHRS {% endif %} # DOIAU + {% if DOHYBVAR %} + {% if DO_JEDIATMENS %} + {% set steps = ["atmensanlinit", "atmensanlrun", "atmensanlfinal"] %} + {% else %} + {% set steps = ["eobs", "eupd"] %} + {% if lobsdiag_forenkf %} + {% do steps.append("ediag") %} + {% else %} + {% for mem in range(1, nmem_ens + 1) %} + {% do steps.append("eomg_mem{{ '%03d' % mem }}") %} + {% endfor %} + {% endif %} + {% do steps.append("echgres") %} + {% do steps.append("esfc") %} # TODO archive ecen logs based on actual groups. Will need to emulate numpy.array_split to do so. - {% set steps = ["eobs", "ecen*", "esfc", "eupd"] %} + {% do steps.append("ecen*") %} + {% endif %} {% for step in steps %} - "logs/{{ cycle_YMDH }}/{{ RUN }}{{ step }}.log" {% endfor %} {% for mem in range(1, nmem_ens + 1) %} - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_mem{{ '%03d' % mem }}.log" {% endfor %} - {% if lobsdiag_forenkf %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}ediag.log" - {% else %} - {% for mem in range(1, nmem_ens + 1) %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}eomg_mem{{ '%03d' % mem }}.log" - {% endfor %} {% endif %} # lobsdiag_forenkf {% endif %} # not the first cycle diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 index 73da805f28..57659e9a2b 100644 --- a/parm/archive/enkf_restarta_grp.yaml.j2 +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -15,7 +15,11 @@ enkf_restarta_grp: - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_pc" {% endif %} + {% if DOIAU_ENKF %} {% set anl_delta = "-3H" | to_timedelta %} + {% else %} + {% set anl_delta = "0H" | to_timedelta %} + {% endif %} {% set anl_time = current_cycle | add_to_datetime(anl_delta) %} {% for itile in range(1, 7) %} - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 9464b39222..5c3e7d7084 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -12,19 +12,50 @@ gdas: - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}prep.log" + {% if DO_JEDIATMVAR %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}prepatmiodaobs.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlinit.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlprod.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlfinal.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlfv3inc.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlvar.log" + {% else %} - "logs/{{ cycle_YMDH }}/{{ RUN }}anal.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}analdiag.log" + {% endif %} - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" + {% if DO_JEDIOCNVAR %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}oceanobs.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalprep.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalbmat.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalrun.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalpost.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalchkpt.log" + {% if DOHYBVAR %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalecen.log" + {% endif %} + {% endif %} + {% if DO_OCEAN %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}ocean_prod*.log" + {% endif %} + {% if DO_ICE %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}ice_prod*.log" + {% endif %} + {% if DO_VRFY_OCEANDA %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalecen.log" + {% endif %} {% if DOIAU and DOHYBVAR %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma003.ensres.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma009.ensres.nc" {% endif %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}oznstat" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" {% if DO_JEDIATMVAR %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" + {% else %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}oznstat" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" {% endif %} {% if DO_VERFOZN %} - "{{ COM_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_cnt.{{ cycle_YMDH }}" diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index b51684d6bd..79ac3eacf5 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -8,7 +8,7 @@ gdas_restarta: {% for iauhr in IAUHRS if iauhr != 6 %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" {% endfor %} - {% if DOHYBVAR %} + {% if DOHYBVAR and DOIAU %} {% set anl_offset = "-3H" %} {% else %} {% set anl_offset = "0H" %} @@ -18,10 +18,12 @@ gdas_restarta: {% for itile in range(1,7) %} - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" {% endfor %} + {% if not DO_JEDIATMVAR %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_air" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_pc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" + {% endif %} {% if DO_JEDISNOWDA %} {% for itile in range(1,7) %} # Snow analysis is 3dvar diff --git a/parm/archive/gfs_restarta.yaml.j2 b/parm/archive/gfs_restarta.yaml.j2 index 23aed86285..c2ad717484 100644 --- a/parm/archive/gfs_restarta.yaml.j2 +++ b/parm/archive/gfs_restarta.yaml.j2 @@ -4,7 +4,7 @@ gfs_restarta: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_restarta.tar" required: {% if MODE == "cycled" %} - {% if DOHYBVAR %} + {% if DOHYBVAR and DOIAU %} {% set anl_offset = "-3H" %} {% else %} {% set anl_offset = "0H" %} diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index 29e866bb59..ae4bc6ac84 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -24,9 +24,7 @@ def main(): 'NMEM_ENS', 'DO_CALC_INCREMENT_ENKF_GFS', 'DO_JEDIATMENS', 'lobsdiag_forenkf', 'FHMIN_ENKF', 'FHMAX_ENKF_GFS', 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF', 'ENKF_SPREAD', - 'restart_interval_enkfgdas', 'restart_interval_enkfgfs', - 'COM_ATMOS_ANALYSIS_TMPL', 'COM_ATMOS_RESTART_TMPL', - 'COM_ATMOS_HISTORY_TMPL'] + 'restart_interval_enkfgdas', 'restart_interval_enkfgfs'] archive_dict = AttrDict() for key in keys: diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 044a11ee7a..120267d70c 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -30,7 +30,7 @@ def main(): 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', 'restart_interval_gdas', 'restart_interval_gfs', 'AERO_ANL_CDUMP', 'AERO_FCST_CDUMP', 'DOIBP_WAV', 'DO_JEDIOCNVAR', - 'NMEM_ENS'] + 'NMEM_ENS', 'DO_JEDIATMVAR'] archive_dict = AttrDict() for key in keys: From 655c1975a56719e0c775c49bf37c0ce062ba761d Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 10 May 2024 19:04:56 +0000 Subject: [PATCH 097/128] Reverted restart offset for enkf (does not change with IAU) --- parm/archive/enkf_restarta_grp.yaml.j2 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 index 57659e9a2b..73da805f28 100644 --- a/parm/archive/enkf_restarta_grp.yaml.j2 +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -15,11 +15,7 @@ enkf_restarta_grp: - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_pc" {% endif %} - {% if DOIAU_ENKF %} {% set anl_delta = "-3H" | to_timedelta %} - {% else %} - {% set anl_delta = "0H" | to_timedelta %} - {% endif %} {% set anl_time = current_cycle | add_to_datetime(anl_delta) %} {% for itile in range(1, 7) %} - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" From af7c968e0344971a68e901c0df5b0bcfe666cf75 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 13 May 2024 18:40:51 +0000 Subject: [PATCH 098/128] Add missing variables --- scripts/exgdas_enkf_earc.py | 3 ++- scripts/exglobal_archive.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index ae4bc6ac84..71d2fb97c8 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -24,7 +24,8 @@ def main(): 'NMEM_ENS', 'DO_CALC_INCREMENT_ENKF_GFS', 'DO_JEDIATMENS', 'lobsdiag_forenkf', 'FHMIN_ENKF', 'FHMAX_ENKF_GFS', 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF', 'ENKF_SPREAD', - 'restart_interval_enkfgdas', 'restart_interval_enkfgfs'] + 'restart_interval_enkfgdas', 'restart_interval_enkfgfs', + 'DOHYBVAR', 'DOIAU_ENKF', 'IAU_OFFSET'] archive_dict = AttrDict() for key in keys: diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 120267d70c..184cf2bca5 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -30,7 +30,8 @@ def main(): 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', 'restart_interval_gdas', 'restart_interval_gfs', 'AERO_ANL_CDUMP', 'AERO_FCST_CDUMP', 'DOIBP_WAV', 'DO_JEDIOCNVAR', - 'NMEM_ENS', 'DO_JEDIATMVAR'] + 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', + 'FITSARC'] archive_dict = AttrDict() for key in keys: From 5543214d59538165eb6fdfc6b08d3d0f31cc8e61 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 13 May 2024 18:41:27 +0000 Subject: [PATCH 099/128] Fix VFYARC archiving --- parm/archive/arcdir.yaml.j2 | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index 7d2d1c7357..0195e38524 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -10,9 +10,7 @@ deterministic: - "{{ ARCDIR }}/tracker.{{ cycle_YMDH }}/{{ RUN }}" {% endif %} {% if FITSARC %} - {% if not VFYARC is defined %} {% set VFYARC = ROTDIR + "/vrfyarch" %} - {% endif %} - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}" {% endif %} copy: @@ -46,10 +44,10 @@ deterministic: {% set fhmax = FHMAX_GFS %} {% endif %} {% for fhr in range(0, fhmax + 6, 6) %} - {% set sfcfile = COM_ATMOS_HISTORY + "/" + head + "sfcf" + '%03d'|format(fhr) + ".nc" %} - {% set sigfile = COM_ATMOS_HISTORY + "/" + head + "atmf" + '%03d'|format(fhr) + ".nc" %} - - ["{{ sfcfile }}", "{{ VRFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sfcfile }}"] - - ["{{ sigfile }}", "{{ VRFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sigfile }}"] + {% set sfcfile = "/" + head + "sfcf" + '%03d'|format(fhr) + ".nc" %} + {% set sigfile = "/" + head + "atmf" + '%03d'|format(fhr) + ".nc" %} + - ["{{COM_ATMOS_HISTORY}}/{{ sfcfile }}", "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sfcfile }}"] + - ["{{COM_ATMOS_HISTORY}}/{{ sigfile }}", "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sigfile }}"] {% endfor %} {% endif %} ## FITSARC {% if path_exists(COM_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH) %} From 4436a91fe4adbcade9c4499dcb26c360ff117ec0 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 13 May 2024 18:42:03 +0000 Subject: [PATCH 100/128] Fix offsets for ENKF IAU --- parm/archive/enkf_restarta_grp.yaml.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 index 73da805f28..0ff1fe09be 100644 --- a/parm/archive/enkf_restarta_grp.yaml.j2 +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -15,7 +15,11 @@ enkf_restarta_grp: - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_pc" {% endif %} + {% if DOHYBVAR and DOIAU_ENKF %} {% set anl_delta = "-3H" | to_timedelta %} + {% else %} + {% set anl_delta = "0H" | to_timedelta %} + {% endif %} {% set anl_time = current_cycle | add_to_datetime(anl_delta) %} {% for itile in range(1, 7) %} - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" From baf6da774c85c187984558efee9322ff8faa1125 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 13 May 2024 18:43:51 +0000 Subject: [PATCH 101/128] Fix ocean/ice GDAS products --- parm/archive/gdas.yaml.j2 | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 5c3e7d7084..0aa6319e2b 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -25,7 +25,7 @@ gdas: {% endif %} - "logs/{{ cycle_YMDH }}/{{ RUN }}atmanlupp.log" {% if DO_JEDIOCNVAR %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}oceanobs.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}prepoceanobs.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalprep.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalbmat.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalrun.log" @@ -35,12 +35,6 @@ gdas: - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalecen.log" {% endif %} {% endif %} - {% if DO_OCEAN %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}ocean_prod*.log" - {% endif %} - {% if DO_ICE %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}ice_prod*.log" - {% endif %} {% if DO_VRFY_OCEANDA %} - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalecen.log" {% endif %} @@ -103,7 +97,9 @@ gdas: {% endfor %} optional: {% if current_cycle != SDATE and MODE == "cycled" %} + {% if not DO_JEDIATMVAR %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" + {% endif %} {% if DO_VERFRAD %} - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" From 990d49b50221e18c14e9859b45d03872202bbc8e Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 13 May 2024 18:45:09 +0000 Subject: [PATCH 102/128] Wrap optional files in appropriate flags --- parm/archive/gdas_restarta.yaml.j2 | 4 +++- parm/archive/gfs_netcdfa.yaml.j2 | 2 ++ parm/archive/gfsa.yaml.j2 | 3 ++- parm/archive/master_gfs.yaml.j2 | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 79ac3eacf5..c55743bdae 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -34,13 +34,15 @@ gdas_restarta: {% endif %} optional: {% if current_cycle != SDATE and MODE == "cycled" %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" + {% if not DO_JEDIATMVAR %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" + {% endif %} {% if DO_JEDISNOWDA %} - "{{ COM_CONF | relpath(ROTDIR) }}/{{ head }}letkfoi.yaml" {% endif %} diff --git a/parm/archive/gfs_netcdfa.yaml.j2 b/parm/archive/gfs_netcdfa.yaml.j2 index 4f3ed4fedc..4ff0f8c1c9 100644 --- a/parm/archive/gfs_netcdfa.yaml.j2 +++ b/parm/archive/gfs_netcdfa.yaml.j2 @@ -10,5 +10,7 @@ gfs_netcdfa: - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" {% endfor %} optional: + {% if not DO_JEDIATMVAR %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" + {% endif %} diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 249924936c..1d2a9fd00c 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -40,13 +40,14 @@ gfsa: {% endif %} optional: {% if MODE == "cycled" %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" {% if DO_JEDIATMVAR %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" + {% else %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" {% endif %} {% endif %} # cycled - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cyclone.trackatcfunix" diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2 index 199546bf90..601c9a8a01 100644 --- a/parm/archive/master_gfs.yaml.j2 +++ b/parm/archive/master_gfs.yaml.j2 @@ -38,7 +38,7 @@ datasets: {% endfilter %} {% endif %} -{% if DO_OCEAN %} +{% if DO_OCN %} {% filter indent(width=4) %} {% include "ocean_6hravg.yaml.j2" %} {% include "ocean_daily.yaml.j2" %} From fe5d68fe84d7df71339c89388d24c18365882401 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Mon, 13 May 2024 15:09:47 -0400 Subject: [PATCH 103/128] Use RUN instead of CDUMP for aerosols. Co-authored-by: Walter Kolczynski - NOAA --- parm/config/gefs/config.fcst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index 4714879ecd..acb600abb9 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -16,7 +16,7 @@ esac # Turn off waves if not used for this CDUMP case ${AERO_FCST_CDUMP} in - both | "${CDUMP/enkf}" ) ;; # Don't change + both | "${RUN/enkf}" ) ;; # Don't change *) DO_AERO="NO" ;; # Turn waves off esac From 9a0704e2f19d9c017126d78c0d339af8a6e3d181 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 13 May 2024 19:23:36 +0000 Subject: [PATCH 104/128] Added an 1152 case for earc --- parm/config/gfs/config.earc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parm/config/gfs/config.earc b/parm/config/gfs/config.earc index 1de578e33e..4357f572c1 100644 --- a/parm/config/gfs/config.earc +++ b/parm/config/gfs/config.earc @@ -19,6 +19,8 @@ case "${CASE_ENS}" in "C384" | "C768") export NMEM_EARCGRP=10 ;; + "C1152") + export NMEM_EARCGRP=4 *) echo "FATAL ERROR: Unknown ensemble resolution ${CASE_ENS}, ABORT!" exit 1 From 50305dcbf6d6bcbd2a3f2f879e6e1c249a90948b Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 13 May 2024 19:34:30 +0000 Subject: [PATCH 105/128] Replace CDUMP with RUN for wave, aerosol configs --- parm/config/gefs/config.fcst | 6 +++--- parm/config/gfs/config.fcst | 8 ++++---- parm/config/gfs/config.wave | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index acb600abb9..9f1cad0a12 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -8,13 +8,13 @@ echo "BEGIN: config.fcst" export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM export COPY_FINAL_RESTARTS="NO" # Toggle to copy restarts from the end of GFS/GEFS Run (GDAS is handled seperately) -# Turn off waves if not used for this CDUMP +# Turn off waves if not used for this RUN case ${WAVE_CDUMP} in - both | "${CDUMP/enkf}" ) ;; # Don't change + both | "${RUN/enkf}" ) ;; # Don't change *) DO_WAVE="NO" ;; # Turn waves off esac -# Turn off waves if not used for this CDUMP +# Turn off aerosols if not used for this RUN case ${AERO_FCST_CDUMP} in both | "${RUN/enkf}" ) ;; # Don't change *) DO_AERO="NO" ;; # Turn waves off diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index 29cd5a7f0d..7de0b673a4 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -8,15 +8,15 @@ echo "BEGIN: config.fcst" export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM export COPY_FINAL_RESTARTS="NO" # Toggle to copy restarts from the end of GFS/GEFS Run (GDAS is handled seperately) -# Turn off waves if not used for this CDUMP +# Turn off waves if not used for this RUN case ${WAVE_CDUMP} in - both | "${CDUMP/enkf}" ) ;; # Don't change + both | "${RUN/enkf}" ) ;; # Don't change *) DO_WAVE="NO" ;; # Turn waves off esac -# Turn off aerosols if not used for this CDUMP +# Turn off aerosols if not used for this RUN case ${AERO_FCST_CDUMP} in - both | "${CDUMP/enkf}" ) ;; # Don't change + both | "${RUN/enkf}" ) ;; # Don't change *) DO_AERO="NO" ;; # Turn aerosols off esac diff --git a/parm/config/gfs/config.wave b/parm/config/gfs/config.wave index 6d0fd129db..3c643c7996 100644 --- a/parm/config/gfs/config.wave +++ b/parm/config/gfs/config.wave @@ -91,7 +91,7 @@ export WAVEWND_DID= export WAVEWND_FID= # The start time reflects the number of hindcast hours prior to the cycle initial time -if [[ "${CDUMP}" = "gfs" ]]; then +if [[ "${RUN}" == "gfs" ]]; then export FHMAX_WAV=${FHMAX_WAV_GFS} fi export WAVHINDH=0 @@ -107,7 +107,7 @@ export FHINCP_WAV=$(( DTPNT_WAV / 3600 )) export OUTPARS_WAV="WND CUR ICE HS T01 T02 DIR FP DP PHS PTP PDIR CHA" # Restart file config -if [[ "${CDUMP}" = "gdas" ]]; then +if [[ "${RUN}" == "gdas" ]]; then export WAVNCYC=4 export WAVHCYC=${assim_freq:-6} export FHMAX_WAV_CUR=48 # RTOFS forecasts only out to 8 days @@ -122,7 +122,7 @@ fi # Restart timing business export RSTTYPE_WAV='T' # generate second tier of restart files -if [[ "${CDUMP}" != gfs ]]; then # Setting is valid for GDAS and GEFS +if [[ "${RUN}" != gfs ]]; then # Setting is valid for GDAS and GEFS export DT_1_RST_WAV=10800 # time between restart files, set to DTRST=1 for a single restart file export DT_2_RST_WAV=43200 # restart stride for checkpointing restart export RSTIOFF_WAV=0 # first restart file offset relative to model start From dfcf804ea21d3256dbde3bc655277dfaa2dabc4b Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 13 May 2024 19:56:08 +0000 Subject: [PATCH 106/128] Move WGRIB def to module_base --- modulefiles/module_base.hera.lua | 4 ++++ modulefiles/module_base.hercules.lua | 4 ++++ modulefiles/module_base.jet.lua | 4 ++++ modulefiles/module_base.orion.lua | 4 ++++ modulefiles/module_base.s4.lua | 4 ++++ parm/config/gefs/config.base | 4 ---- parm/config/gfs/config.base | 4 ---- 7 files changed, 20 insertions(+), 8 deletions(-) diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua index c47a1bfd70..f75cf886e7 100644 --- a/modulefiles/module_base.hera.lua +++ b/modulefiles/module_base.hera.lua @@ -41,6 +41,10 @@ load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") + +-- Stop gap fix for wgrib with spack-stack 1.6.0 +-- TODO Remove this when spack-stack issue #1097 is resolved +setenv("WGRIB","wgrib") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) --prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) diff --git a/modulefiles/module_base.hercules.lua b/modulefiles/module_base.hercules.lua index 5835d013d7..998803f246 100644 --- a/modulefiles/module_base.hercules.lua +++ b/modulefiles/module_base.hercules.lua @@ -37,6 +37,10 @@ load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") + +-- Stop gap fix for wgrib with spack-stack 1.6.0 +-- TODO Remove this when spack-stack issue #1097 is resolved +setenv("WGRIB","wgrib") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua index 31f8aa676d..e53132fd6a 100644 --- a/modulefiles/module_base.jet.lua +++ b/modulefiles/module_base.jet.lua @@ -43,6 +43,10 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) load(pathJoin("perl", (os.getenv("perl_ver") or "None"))) setenv("WGRIB2","wgrib2") + +-- Stop gap fix for wgrib with spack-stack 1.6.0 +-- TODO Remove this when spack-stack issue #1097 is resolved +setenv("WGRIB","wgrib") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) --prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua index 72a480a946..4d747512db 100644 --- a/modulefiles/module_base.orion.lua +++ b/modulefiles/module_base.orion.lua @@ -38,6 +38,10 @@ load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") + +-- Stop gap fix for wgrib with spack-stack 1.6.0 +-- TODO Remove this when spack-stack issue #1097 is resolved +setenv("WGRIB","wgrib") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) --prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) diff --git a/modulefiles/module_base.s4.lua b/modulefiles/module_base.s4.lua index 6f0602c3eb..835249fb85 100644 --- a/modulefiles/module_base.s4.lua +++ b/modulefiles/module_base.s4.lua @@ -37,6 +37,10 @@ load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") + +-- Stop gap fix for wgrib with spack-stack 1.6.0 +-- TODO Remove this when spack-stack issue #1097 is resolved +setenv("WGRIB","wgrib") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) --prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index e8238db3e8..e98c942dcf 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -81,10 +81,6 @@ export CHGRP_CMD="@CHGRP_CMD@" export NCDUMP="${NETCDF:-${netcdf_c_ROOT:-}}/bin/ncdump" export NCLEN="${HOMEgfs}/ush/getncdimlen" -# Stop gap fix for wgrib with spack-stack 1.6.0 -# TODO Remove this when spack-stack issue #1097 is resolved -export WGRIB="${WGRIB:-${grib_util_ROOT:-}/wgrib}" - # Machine environment, jobs, and other utility scripts export BASE_ENV="${HOMEgfs}/env" export BASE_JOB="${HOMEgfs}/jobs/rocoto" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index fadb60b6c5..f1e25750ef 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -110,10 +110,6 @@ export CHGRP_CMD="@CHGRP_CMD@" export NCDUMP="${NETCDF:-${netcdf_c_ROOT:-}}/bin/ncdump" export NCLEN="${HOMEgfs}/ush/getncdimlen" -# Stop gap fix for wgrib with spack-stack 1.6.0 -# TODO Remove this when spack-stack issue #1097 is resolved -export WGRIB="${WGRIB:-${grib_util_ROOT:-}/wgrib}" - # Machine environment, jobs, and other utility scripts export BASE_ENV="${HOMEgfs}/env" export BASE_JOB="${HOMEgfs}/jobs/rocoto" From 0106d25d3e3ab035be9b5f891cb1432bcd3bf04e Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 13 May 2024 20:01:26 +0000 Subject: [PATCH 107/128] Added missing ;;. --- parm/config/gfs/config.earc | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/config/gfs/config.earc b/parm/config/gfs/config.earc index 4357f572c1..00a2fa95bd 100644 --- a/parm/config/gfs/config.earc +++ b/parm/config/gfs/config.earc @@ -21,6 +21,7 @@ case "${CASE_ENS}" in ;; "C1152") export NMEM_EARCGRP=4 + ;; *) echo "FATAL ERROR: Unknown ensemble resolution ${CASE_ENS}, ABORT!" exit 1 From c67c36faea512d39f476d1ff4e8981bbbd7e0024 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 14 May 2024 17:16:55 +0000 Subject: [PATCH 108/128] Fix IAUFHRS --- parm/archive/gdas_restarta.yaml.j2 | 2 +- parm/archive/gfs_netcdfa.yaml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index c55743bdae..4a84f52576 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -5,7 +5,7 @@ gdas_restarta: required: {% if current_cycle != SDATE and MODE == "cycled" %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" - {% for iauhr in IAUHRS if iauhr != 6 %} + {% for iauhr in IAUFHRS if iauhr != 6 %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" {% endfor %} {% if DOHYBVAR and DOIAU %} diff --git a/parm/archive/gfs_netcdfa.yaml.j2 b/parm/archive/gfs_netcdfa.yaml.j2 index 4ff0f8c1c9..756e7ed6fb 100644 --- a/parm/archive/gfs_netcdfa.yaml.j2 +++ b/parm/archive/gfs_netcdfa.yaml.j2 @@ -6,7 +6,7 @@ gfs_netcdfa: - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" - {% for iauhr in IAUHRS if iauhr != 6 %} + {% for iauhr in IAUFHRS if iauhr != 6 %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" {% endfor %} optional: From f27839c88f36728f41e8e250ffeb70e8fbe60f54 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 14 May 2024 17:18:20 +0000 Subject: [PATCH 109/128] Make gsistat mandatory; remove duplicate flux files --- parm/archive/gdas.yaml.j2 | 8 +------- parm/archive/gfsa.yaml.j2 | 5 +++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 0aa6319e2b..d26c852462 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -47,6 +47,7 @@ gdas: - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" {% else %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}oznstat" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat" @@ -97,9 +98,6 @@ gdas: {% endfor %} optional: {% if current_cycle != SDATE and MODE == "cycled" %} - {% if not DO_JEDIATMVAR %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" - {% endif %} {% if DO_VERFRAD %} - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" @@ -122,10 +120,6 @@ gdas: {% endfor %} {% endif %} {% endif %} - {% for fhr in range(FHMIN, FHMAX + FHOUT, FHOUT) %} - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2" - - "{{ COM_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sfluxgrbf{{ '%03d' % fhr }}.grib2.idx" - {% endfor %} {% if not WRITE_DOPOST %} # TODO set the forecast hours explicitly. This will require emulating numpy.array_split - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_upp_f*.log" diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 1d2a9fd00c..894b0836ee 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -38,6 +38,9 @@ gfsa: {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} + {% if not DO_JEDIATMVAR %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" + {% endif %} optional: {% if MODE == "cycled" %} - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" @@ -46,8 +49,6 @@ gfsa: {% if DO_JEDIATMVAR %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" - {% else %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" {% endif %} {% endif %} # cycled - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cyclone.trackatcfunix" From 6e74ce897804b37d3f9e7d409e7da3bc39be423b Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 14 May 2024 17:19:50 +0000 Subject: [PATCH 110/128] Fix ocean time step to archive --- parm/archive/gdasocean.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/archive/gdasocean.yaml.j2 b/parm/archive/gdasocean.yaml.j2 index 630618aaf1..4c7e422251 100644 --- a/parm/archive/gdasocean.yaml.j2 +++ b/parm/archive/gdasocean.yaml.j2 @@ -3,7 +3,7 @@ gdasocean: name: "GDASOCEAN" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar" required: - {% for fhr in range(FHMIN, FHMAX + 1, FHMAX) %} + {% for fhr in range(FHMIN_OCNICE, FHMAX + 1, FHOUT_OCNICE) %} - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}inst.f{{ '%03d' % fhr }}.nc" {% endfor %} - '{{ COM_CONF | relpath(ROTDIR) }}/ufs.MOM_input' From 0986bc03c14cbebd7c3346bb2e3dd546d07274fa Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 14 May 2024 17:20:42 +0000 Subject: [PATCH 111/128] Add missing ice IC file to archive. --- parm/archive/gdasice.yaml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/archive/gdasice.yaml.j2 b/parm/archive/gdasice.yaml.j2 index f2ca58b926..4cfa1eb9af 100644 --- a/parm/archive/gdasice.yaml.j2 +++ b/parm/archive/gdasice.yaml.j2 @@ -3,6 +3,7 @@ gdasice: name: "GDASICE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar" required: + - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}ic.nc" {% for fhr in range(FHOUT_OCNICE, FHMAX+1, FHOUT_OCNICE) %} - "{{ COM_ICE_HISTORY | relpath(ROTDIR) }}/{{ head }}inst.f{{ '%03d' % fhr }}.nc" {% endfor %} From 806ec5dbd92fcb93764aaf86bd00dc61360d1fc9 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 14 May 2024 17:21:01 +0000 Subject: [PATCH 112/128] Add missing ocean CSVs. --- parm/archive/gdasocean_analysis.yaml.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index 74a191b024..cbe2936277 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -20,5 +20,8 @@ gdasocean_analysis: - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_total_stddev.nc" - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.steric_explained_variance.nc" {% endif %} + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.adt_rads_all.stats.csv" + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.icec_amsr2_north.stats.csv" + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.icec_amsr2_south.stats.csv" - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags/*.nc4' - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml/*.yaml' From 7dd2fd4972b07cd86fe84640bcb373ee9ccc1f0b Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 14 May 2024 17:27:26 +0000 Subject: [PATCH 113/128] Archive *gdas hourly outputs at 3-hour intervals --- parm/archive/enkf.yaml.j2 | 2 +- parm/archive/enkf_grp.yaml.j2 | 2 +- parm/archive/gdas.yaml.j2 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index 7b406fba8e..f11a3568b5 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -2,7 +2,7 @@ enkf: name: "ENKF" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}.tar" required: - {% for fhr in range(fhmin, fhmax + fhout, fhout) %} + {% for fhr in range(0, fhmax + 1, 3) %} - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensmean.nc" - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.ensmean.nc" {% if ENKF_SPREAD %} diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index 8d6d7f3e9d..450c9aeab5 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -8,7 +8,7 @@ enkf_grp: {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% for fhr in range(fhmin, fhmax+fhout, fhout) %} + {% for fhr in range(0, fhmax+1, 3) %} - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index d26c852462..a64be8dc93 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -85,7 +85,7 @@ gdas: - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" # TODO explicitly name the atmos_prod log files to archive - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" - {% for fhr in range(0, FHMAX + FHOUT, FHOUT) %} + {% for fhr in range(0, FHMAX + 1, 3) %} - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - "{{ COM_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}" From a9fa4e3507864ea5f27120b8bcb15edfae8a39c8 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 14 May 2024 19:09:14 +0000 Subject: [PATCH 114/128] Add missing keys and fix bugs --- parm/archive/arcdir.yaml.j2 | 2 +- parm/archive/enkf.yaml.j2 | 13 ++++++++++--- parm/archive/enkf_grp.yaml.j2 | 16 +++++++--------- parm/archive/gdas_restarta.yaml.j2 | 2 +- parm/archive/gdasocean.yaml.j2 | 2 +- parm/archive/gdasocean_analysis.yaml.j2 | 16 ++++++++-------- parm/archive/gfs_netcdfa.yaml.j2 | 2 +- parm/archive/gfsa.yaml.j2 | 19 +++++++++---------- parm/archive/master_gdas.yaml.j2 | 10 ++++++++++ parm/archive/master_gfs.yaml.j2 | 9 +++++++++ scripts/exgdas_enkf_earc.py | 3 ++- scripts/exglobal_archive.py | 2 +- 12 files changed, 60 insertions(+), 36 deletions(-) diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index 0195e38524..6321f6fc41 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -65,7 +65,7 @@ deterministic: {% endif %} {% endfor %} {% endif %} ## RUN == "gfs" - {% if arch_tracks %} + {% if path_exists(COM_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH) %} - ["{{ COM_ATMOS_TRACK }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}"] - ["{{ COM_ATMOS_TRACK }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index f11a3568b5..5aeb8e3c86 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -2,7 +2,7 @@ enkf: name: "ENKF" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}.tar" required: - {% for fhr in range(0, fhmax + 1, 3) %} + {% for fhr in range(3, fhmax + 1, 3) %} - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensmean.nc" - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.ensmean.nc" {% if ENKF_SPREAD %} @@ -29,9 +29,7 @@ enkf: {% if do_calc_increment %} - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmanl.ensmean.nc" {% endif %} # calc increment - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ratminc.ensmean.nc" {% else %} # fhr != IAU_OFFSET - - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensmean.nc" - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmi{{ '%03d' % fhr }}.ensmean.nc" {% endif %} # fhr == IAU_OFFSET {% endfor %} # IAUFHRS @@ -61,3 +59,12 @@ enkf: {% endfor %} {% endif %} # lobsdiag_forenkf {% endif %} # not the first cycle + optional: + {% if current_cycle != SDATE and DOIAU %} + {% for fhr in iaufhrs %} + {% if fhr != IAU_OFFSET %} + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensmean.nc" + - "{{ COM_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmi{{ '%03d' % fhr }}.ensmean.nc" + {% endif %} # fhr == IAU_OFFSET + {% endfor %} # IAUFHRS + {% endif %} diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index 450c9aeab5..9164181698 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -8,7 +8,7 @@ enkf_grp: {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% for fhr in range(0, fhmax+1, 3) %} + {% for fhr in range(3, fhmax+1, 3) %} - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" {% endfor %} @@ -24,14 +24,6 @@ enkf_grp: {% endif %} {% endif %} - {% for iaufhr in iaufhrs if iaufhr != 6 %} - {% set iaufhr = iaufhr %} - {% if do_calc_increment %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iaufhr }}.nc" - {% else %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iaufhr }}.nc" - {% endif %} - {% endfor %} # iaufhr in iaufhrs {% endfor %} # first_group_mem to last_group_mem optional: {% if current_cycle != SDATE and not lobsdiag_forenkf %} @@ -39,5 +31,11 @@ enkf_grp: {% set imem = mem - first_group_mem %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" + {% for iaufhr in iaufhrs if iaufhr != 6 %} + {% set iaufhr = iaufhr %} + {% if do_calc_increment %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iaufhr }}.nc" + {% endif %} + {% endfor %} # iaufhr in iaufhrs {% endfor %} {% endif %} diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 4a84f52576..8dffadc6de 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -5,7 +5,7 @@ gdas_restarta: required: {% if current_cycle != SDATE and MODE == "cycled" %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" - {% for iauhr in IAUFHRS if iauhr != 6 %} + {% for iauhr in iaufhrs if iauhr != 6 %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" {% endfor %} {% if DOHYBVAR and DOIAU %} diff --git a/parm/archive/gdasocean.yaml.j2 b/parm/archive/gdasocean.yaml.j2 index 4c7e422251..9791709319 100644 --- a/parm/archive/gdasocean.yaml.j2 +++ b/parm/archive/gdasocean.yaml.j2 @@ -3,7 +3,7 @@ gdasocean: name: "GDASOCEAN" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar" required: - {% for fhr in range(FHMIN_OCNICE, FHMAX + 1, FHOUT_OCNICE) %} + {% for fhr in range(FHMIN, FHMAX + 1, FHOUT_OCNICE) %} - "{{ COM_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ head }}inst.f{{ '%03d' % fhr }}.nc" {% endfor %} - '{{ COM_CONF | relpath(ROTDIR) }}/ufs.MOM_input' diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index cbe2936277..0c43cd40ba 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -11,17 +11,17 @@ gdasocean_analysis: - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.incr.nc' - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}ana.nc' {% if NMEM_ENS > 2 %} - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.recentering_error.nc" + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.recentering_error.nc' {% endif %} {% endfor %} {% if NMEM_ENS > 2 %} - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_steric_stddev.nc" - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_unbal_stddev.nc" - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_total_stddev.nc" - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.steric_explained_variance.nc" + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_steric_stddev.nc' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_unbal_stddev.nc' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_total_stddev.nc' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.steric_explained_variance.nc' {% endif %} - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.adt_rads_all.stats.csv" - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.icec_amsr2_north.stats.csv" - - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.icec_amsr2_south.stats.csv" + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.adt_rads_all.stats.csv' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.icec_amsr2_north.stats.csv' + - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.icec_amsr2_south.stats.csv' - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags/*.nc4' - '{{ COM_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml/*.yaml' diff --git a/parm/archive/gfs_netcdfa.yaml.j2 b/parm/archive/gfs_netcdfa.yaml.j2 index 756e7ed6fb..6bcafe1b89 100644 --- a/parm/archive/gfs_netcdfa.yaml.j2 +++ b/parm/archive/gfs_netcdfa.yaml.j2 @@ -6,7 +6,7 @@ gfs_netcdfa: - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" - {% for iauhr in IAUFHRS if iauhr != 6 %} + {% for iauhr in iaufhrs if iauhr != 6 %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" {% endfor %} optional: diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 894b0836ee..3518a56fb8 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -28,6 +28,15 @@ gfsa: - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" # - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" + {% if AERO_ANL_CDUMP == "gfs" or AERO_ANL_CDUMP == "both" %} + - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" + {% endif %} + {% if DO_JEDIATMVAR %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" + {% else %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" + {% endif %} {% endif %} {% endif %} # Full cycle {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} @@ -35,21 +44,11 @@ gfsa: - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}.idx" - "{{ COM_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.logf{{ '%03d' % fhr }}.txt" {% endfor %} - {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} - - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" - {% endif %} - {% if not DO_JEDIATMVAR %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" - {% endif %} optional: {% if MODE == "cycled" %} - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" - {% if DO_JEDIATMVAR %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" - {% endif %} {% endif %} # cycled - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cyclone.trackatcfunix" - "{{ COM_ATMOS_TRACK | relpath(ROTDIR) }}/avnop.t{{ cycle_HH }}z.cyclone.trackatcfunix" diff --git a/parm/archive/master_gdas.yaml.j2 b/parm/archive/master_gdas.yaml.j2 index 6ff6e40b30..6813209e6a 100644 --- a/parm/archive/master_gdas.yaml.j2 +++ b/parm/archive/master_gdas.yaml.j2 @@ -2,6 +2,16 @@ {% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_YMDH = current_cycle | to_YMDH %} {% set head = "gdas.t" + cycle_HH + "z." %} + +{% if IAUFHRS is string %} +{% set iaufhrs = [] %} +{% for iaufhr in IAUFHRS.split(",") %} +{% do iaufhrs.append(iaufhr | int) %} +{% endfor %} +{% else %} +{% set iaufhrs = [IAUFHRS] %} +{% endif %} + datasets: {% filter indent(width=4) %} {% include "gdas.yaml.j2" %} diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2 index 601c9a8a01..5340ddb721 100644 --- a/parm/archive/master_gfs.yaml.j2 +++ b/parm/archive/master_gfs.yaml.j2 @@ -1,6 +1,15 @@ {% set cycle_HH = current_cycle | strftime("%H") %} {% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_YMDH = current_cycle | to_YMDH %} + +{% if IAUFHRS is string %} +{% set iaufhrs = [] %} +{% for iaufhr in IAUFHRS.split(",") %} +{% do iaufhrs.append(iaufhr | int) %} +{% endfor %} +{% else %} +{% set iaufhrs = [IAUFHRS] %} +{% endif %} datasets: {% filter indent(width=4) %} {% include "gfsa.yaml.j2" %} diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index 71d2fb97c8..841e551540 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -25,7 +25,8 @@ def main(): 'lobsdiag_forenkf', 'FHMIN_ENKF', 'FHMAX_ENKF_GFS', 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF', 'ENKF_SPREAD', 'restart_interval_enkfgdas', 'restart_interval_enkfgfs', - 'DOHYBVAR', 'DOIAU_ENKF', 'IAU_OFFSET'] + 'DOHYBVAR', 'DOIAU_ENKF', 'IAU_OFFSET', 'DOIAU', + 'DO_CALC_INCREMENT'] archive_dict = AttrDict() for key in keys: diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 184cf2bca5..31b5eb1186 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -31,7 +31,7 @@ def main(): 'restart_interval_gdas', 'restart_interval_gfs', 'AERO_ANL_CDUMP', 'AERO_FCST_CDUMP', 'DOIBP_WAV', 'DO_JEDIOCNVAR', 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', - 'FITSARC'] + 'FITSARC', 'IAUFHRS'] archive_dict = AttrDict() for key in keys: From c2b981c7e78a225ebd39ab844ec1034a8c99a5c7 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 16 May 2024 16:59:11 +0000 Subject: [PATCH 115/128] Flesh out enkf restarts. --- parm/archive/enkf_restarta_grp.yaml.j2 | 19 ++++++++++++++++--- parm/archive/enkf_restartb_grp.yaml.j2 | 17 +++++++++++++++++ parm/archive/master_enkf.yaml.j2 | 19 ++++++++++++++++++- scripts/exgdas_enkf_earc.py | 3 ++- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 index 0ff1fe09be..ee768a1f92 100644 --- a/parm/archive/enkf_restarta_grp.yaml.j2 +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -15,7 +15,7 @@ enkf_restarta_grp: - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_pc" {% endif %} - {% if DOHYBVAR and DOIAU_ENKF %} + {% if DOIAU_ENKF %} {% set anl_delta = "-3H" | to_timedelta %} {% else %} {% set anl_delta = "0H" | to_timedelta %} @@ -24,12 +24,25 @@ enkf_restarta_grp: {% for itile in range(1, 7) %} - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" {% endfor %} + {% if do_calc_increment %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atmanl.nc" + {% else %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratminc.nc" + {% endif %} {% endfor %} # first_group_mem to last_group_mem optional: - {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + {% set COM_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_MEM_list[imem] %} + {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" - {% endfor %} {% endif %} + {% for iaufhr in iaufhrs if iaufhr != 6 %} + {% set iaufhr = iaufhr %} + {% if do_calc_increment %} + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iaufhr }}.nc" + - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iaufhr }}.nc" + {% endif %} + {% endfor %} # iaufhr in iaufhrs + {% endfor %} diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 index 688a37fd71..34fde9d7ca 100644 --- a/parm/archive/enkf_restartb_grp.yaml.j2 +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -5,6 +5,23 @@ enkf_restartb_grp: {% for mem in range(first_group_mem, last_group_mem + 1) %} {% set imem = mem - first_group_mem %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} + + # Grab surface analysis data. + # If IAU is on, grab the beginning of the window, otherwise grab the center. + {% if DOIAU_ENKF %} + {% set offset_td = "-3H" | to_timedelta %} + {% else %} + {% set offset_td = "0H" | to_timedelta %} + {% endif %} + {% set offset_dt = current_cycle | add_to_datetime(offset_td) %} + {% set offset_YMD = offset_dt | to_YMD %} + {% set offset_HH = offset_dt | strftime("%H") %} + {% set prefix = offset_YMD + "." + offset_HH + "0000" %} + {% for itile in range(1, 7) %} + - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ prefix }}.sfcanl_data.tile{{ itile }}.nc" + {% endfor %} + + # Now get the restart files. {% for r_time in range(restart_interval, fhmax + 1, restart_interval) %} {% set r_timedelta = (r_time | string + "H") | to_timedelta %} {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index bd9fa7b8c0..658906c39b 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -64,11 +64,28 @@ datasets: {% filter indent(width=4) %} {% include "enkf_grp.yaml.j2" %} {% endfilter %} -{% if current_cycle > SDATE %} + +# Determine if restarts should be saved +{% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %} + +# Save the increments and restarts every ARCH_WARMICFREQ days +# The ensemble increments (group a) should be saved on the ARCH_CYC +{% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 %} +{% if ARCH_CYC == cycle_HH | int and current_cycle != SDATE %} {% filter indent(width=4) %} {% include "enkf_restarta_grp.yaml.j2" %} +{% endfilter %} +{% endif %} +{% endif %} + +# The ensemble ICs (group b) are restarts and always lag increments by assim_freq +{% set ics_offset = (assim_freq | string + "H") | to_timedelta %} +{% if (current_cycle | add_to_datetime(ics_offset) - SDATE).days % ARCH_WARMICFREQ == 0 %} +{% if (ARCH_CYC - assim_freq) % 24 == cycle_HH | int and current_cycle != SDATE %} +{% filter indent(width=4) %} {% include "enkf_restartb_grp.yaml.j2" %} {% endfilter %} {% endif %} +{% endif %} {% endif %} # ENSGRP != 0 diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index 841e551540..11cd8964d5 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -26,7 +26,8 @@ def main(): 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF', 'ENKF_SPREAD', 'restart_interval_enkfgdas', 'restart_interval_enkfgfs', 'DOHYBVAR', 'DOIAU_ENKF', 'IAU_OFFSET', 'DOIAU', - 'DO_CALC_INCREMENT'] + 'DO_CALC_INCREMENT', 'assim_freq', 'ARCH_CYC', + 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ'] archive_dict = AttrDict() for key in keys: From 714ab9e1044c5a120a8d2aeac975fb73a41ce571 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 16 May 2024 18:13:25 +0000 Subject: [PATCH 116/128] Key enkf atmf netcdf files off of IAUFHRS_ENKF --- parm/archive/enkf_grp.yaml.j2 | 24 ++++++------------------ scripts/exgdas_enkf_earc.py | 3 ++- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index 9164181698..cf7933ef0e 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -8,11 +8,14 @@ enkf_grp: {% set COM_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_MEM_list[imem] %} {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - {% for fhr in range(3, fhmax+1, 3) %} - - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf{{ '%03d' % fhr }}.nc" - - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.nc" + {% for iaufhr in iaufhrs_enkf %} + - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atmf{{ "%03d" % iaufhr }}.nc" {% endfor %} + {% if 6 in iaufhrs_enkf %} + - "{{ COM_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfcf006.nc" + {% endif %} + {% if current_cycle != SDATE %} {% if not lobsdiag_forenkf %} - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat" @@ -23,19 +26,4 @@ enkf_grp: - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratminc.nc" {% endif %} {% endif %} - {% endfor %} # first_group_mem to last_group_mem - optional: - {% if current_cycle != SDATE and not lobsdiag_forenkf %} - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - {% set COM_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_MEM_list[imem] %} - - "{{ COM_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" - {% for iaufhr in iaufhrs if iaufhr != 6 %} - {% set iaufhr = iaufhr %} - {% if do_calc_increment %} - - "{{ COM_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iaufhr }}.nc" - {% endif %} - {% endfor %} # iaufhr in iaufhrs - {% endfor %} - {% endif %} diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index 11cd8964d5..2febbc27f5 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -27,7 +27,8 @@ def main(): 'restart_interval_enkfgdas', 'restart_interval_enkfgfs', 'DOHYBVAR', 'DOIAU_ENKF', 'IAU_OFFSET', 'DOIAU', 'DO_CALC_INCREMENT', 'assim_freq', 'ARCH_CYC', - 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ'] + 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', + 'IAUFHRS_ENKF'] archive_dict = AttrDict() for key in keys: From 8f2d4af8af17b0b2f23d19632c19ad33e8853128 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 16 May 2024 18:14:56 +0000 Subject: [PATCH 117/128] Add missing logs and inc file to enkf*.tar --- parm/archive/enkf.yaml.j2 | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index 5aeb8e3c86..24321fd5d5 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -9,6 +9,15 @@ enkf: - "{{ COM_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmf{{ '%03d' % fhr }}.ensspread.nc" {% endif %} {% endfor %} + {% for mem in range(1, nmem_ens + 1) %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_mem{{ '%03d' % mem }}.log" + {% endfor %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}epos*.log" + # TODO archive ecen logs based on actual groups. Will need to emulate numpy.array_split to do so. + - "logs/{{ cycle_YMDH }}/{{ RUN }}ecen*.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}esfc.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}echgres.log" + {% if current_cycle != SDATE %} {% if not DO_JEDIATMENS %} {% set da_files = ["enkfstat", @@ -18,6 +27,7 @@ enkf: "radstat.ensmean"] %} {% else %} {% set da_files = ["atmens.yaml", + "atminc.ensmean.nc", "atmensstat"] %} {% endif %} {% for file in da_files %} @@ -34,7 +44,7 @@ enkf: {% endif %} # fhr == IAU_OFFSET {% endfor %} # IAUFHRS {% endif %} # DOIAU - {% if DOHYBVAR %} + {% if DO_JEDIATMENS %} {% set steps = ["atmensanlinit", "atmensanlrun", "atmensanlfinal"] %} {% else %} @@ -46,18 +56,11 @@ enkf: {% do steps.append("eomg_mem{{ '%03d' % mem }}") %} {% endfor %} {% endif %} - {% do steps.append("echgres") %} - {% do steps.append("esfc") %} - # TODO archive ecen logs based on actual groups. Will need to emulate numpy.array_split to do so. - {% do steps.append("ecen*") %} {% endif %} + {% for step in steps %} - "logs/{{ cycle_YMDH }}/{{ RUN }}{{ step }}.log" {% endfor %} - {% for mem in range(1, nmem_ens + 1) %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_mem{{ '%03d' % mem }}.log" - {% endfor %} - {% endif %} # lobsdiag_forenkf {% endif %} # not the first cycle optional: {% if current_cycle != SDATE and DOIAU %} From 1220fac90f227832548e3e5fc217f5061c864e86 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 16 May 2024 18:16:08 +0000 Subject: [PATCH 118/128] Make atma files optional, correct log name --- parm/archive/gdas.yaml.j2 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index a64be8dc93..3c7709cfac 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -36,12 +36,10 @@ gdas: {% endif %} {% endif %} {% if DO_VRFY_OCEANDA %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalecen.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalvrfy.log" {% endif %} - {% if DOIAU and DOHYBVAR %} + {% if DOHYBVAR %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma003.ensres.nc" - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma009.ensres.nc" {% endif %} {% if DO_JEDIATMVAR %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" @@ -98,6 +96,10 @@ gdas: {% endfor %} optional: {% if current_cycle != SDATE and MODE == "cycled" %} + {% if DOHYBVAR %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma003.ensres.nc" + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma009.ensres.nc" + {% endif %} {% if DO_VERFRAD %} - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" - "{{ COM_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" From f08c2b86271680d17a81771a01b1cbaa94114885 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 16 May 2024 18:17:21 +0000 Subject: [PATCH 119/128] Add sfcanl data files to gdas restart tarballs --- parm/archive/gdas_restarta.yaml.j2 | 4 ++-- parm/archive/gdas_restartb.yaml.j2 | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 8dffadc6de..9c4aa1244f 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -5,8 +5,8 @@ gdas_restarta: required: {% if current_cycle != SDATE and MODE == "cycled" %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" - {% for iauhr in iaufhrs if iauhr != 6 %} - - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" + {% for iaufhr in iaufhrs if iaufhr != 6 %} + - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iaufhr }}.nc" {% endfor %} {% if DOHYBVAR and DOIAU %} {% set anl_offset = "-3H" %} diff --git a/parm/archive/gdas_restartb.yaml.j2 b/parm/archive/gdas_restartb.yaml.j2 index ec7c723a1d..c5cb29329f 100644 --- a/parm/archive/gdas_restartb.yaml.j2 +++ b/parm/archive/gdas_restartb.yaml.j2 @@ -3,6 +3,26 @@ gdas_restartb: name: "GDAS_RESTARTB" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restartb.tar" required: + # Grab the surface analysis data. + # If IAU is on, grab the beginning of the window. + {% if DOIAU %} + {% set offset_td = "-3H" | to_timedelta %} + {% set offset_dt = current_cycle | add_to_datetime(offset_td) %} + {% set offset_YMD = offset_dt | to_YMD %} + {% set offset_HH = offset_dt | strftime("%H") %} + {% set prefix = offset_YMD + "." + offset_HH + "0000" %} + {% for itile in range(1, 7) %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ prefix }}.sfcanl_data.tile{{ itile }}.nc" + {% endfor %} + {% endif %} + + # Regardless, always grab the center surface analysis data. + {% set prefix = cycle_YMD + "." + cycle_HH + "0000" %} + {% for itile in range(1, 7) %} + - "{{ COM_ATMOS_RESTART | relpath(ROTDIR) }}/{{ prefix }}.sfcanl_data.tile{{ itile }}.nc" + {% endfor %} + + # Now get the restart files. {% for r_time in range(restart_interval_gdas, FHMAX + 1, restart_interval_gdas) %} {% set r_timedelta = (r_time | string + "H") | to_timedelta %} {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} From cfd5fbbbd474b1499a1d4dd1c2e2f80e3e104aad Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 16 May 2024 18:18:15 +0000 Subject: [PATCH 120/128] Move misplaced endif --- parm/archive/gfsa.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 3518a56fb8..7ed12819a0 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -5,7 +5,7 @@ gfsa: required: # TODO explicitly name all logs to include {% for log in glob("logs/" ~ cycle_YMDH ~ "/gfs*.log") %} - {% if log != "gfsarch.log" %} + {% if not "gfsarch.log" in log %} - "{{ log }}" {% endif %} {% endfor %} @@ -28,6 +28,7 @@ gfsa: - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" # - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" + {% endif %} {% if AERO_ANL_CDUMP == "gfs" or AERO_ANL_CDUMP == "both" %} - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} @@ -37,7 +38,6 @@ gfsa: {% else %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" {% endif %} - {% endif %} {% endif %} # Full cycle {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - "{{ COM_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pgrb2.0p25.f{{ '%03d' % fhr }}" From 83b969cea25041a63d1b934969ec218f9c9238be Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 16 May 2024 18:18:46 +0000 Subject: [PATCH 121/128] Add iaufhrs_enkf definition --- parm/archive/master_enkf.yaml.j2 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index 658906c39b..7ab7f45e30 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -13,6 +13,15 @@ {% set iaufhrs = [IAUFHRS] %} {% endif %} +{% if IAUFHRS_ENKF is string %} +{% set iaufhrs_enkf = [] %} +{% for iaufhr in IAUFHRS_ENKF.split(",") %} +{% do iaufhrs_enkf.append(iaufhr | int) %} +{% endfor %} +{% else %} +{% set iaufhrs_enkf = [IAUFHRS_ENKF] %} +{% endif %} + datasets: {% if ENSGRP == 0 %} {% filter indent(width=4) %} From 7e205e766b5934f7f1194f48c0c913fcec056dbd Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 17 May 2024 10:28:02 +0000 Subject: [PATCH 122/128] Updating nc check script to verify time dimensions --- ush/check_netcdf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/check_netcdf.sh b/ush/check_netcdf.sh index e115ae8ae3..16d9596e4c 100755 --- a/ush/check_netcdf.sh +++ b/ush/check_netcdf.sh @@ -8,7 +8,7 @@ source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" 1>/dev/null 2>&1 ncfile=${1?} -ncdump -h "${ncfile}" 1>/dev/null 2>&1 # redirect stdout and stderr to /dev/null to suppress output in cron +(( $(ncdump ${file} 2> /dev/null | grep -Po '(?<=time = UNLIMITED ; // \()\d+(?= currently)' || echo 0) > 0 )) # redirect stdout and stderr to /dev/null to suppress output in cron rc=$? # If there is no error, rc=0, else rc!=0 From 478a5ba14987bc1c2b4d5b6430efdbc8ea2b432f Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 17 May 2024 10:30:02 +0000 Subject: [PATCH 123/128] Ignore shellcheck warning --- ush/check_netcdf.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/check_netcdf.sh b/ush/check_netcdf.sh index 16d9596e4c..32c016d6a3 100755 --- a/ush/check_netcdf.sh +++ b/ush/check_netcdf.sh @@ -8,6 +8,7 @@ source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" 1>/dev/null 2>&1 ncfile=${1?} +# shellcheck disable=SC2086 (( $(ncdump ${file} 2> /dev/null | grep -Po '(?<=time = UNLIMITED ; // \()\d+(?= currently)' || echo 0) > 0 )) # redirect stdout and stderr to /dev/null to suppress output in cron rc=$? # If there is no error, rc=0, else rc!=0 From 3be3bd578a36f0fac477de36aaba84d3d219704d Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 17 May 2024 10:34:09 +0000 Subject: [PATCH 124/128] Fixed ncfile variable name --- ush/check_netcdf.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/check_netcdf.sh b/ush/check_netcdf.sh index 32c016d6a3..5f56a38aba 100755 --- a/ush/check_netcdf.sh +++ b/ush/check_netcdf.sh @@ -8,8 +8,7 @@ source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" 1>/dev/null 2>&1 ncfile=${1?} -# shellcheck disable=SC2086 -(( $(ncdump ${file} 2> /dev/null | grep -Po '(?<=time = UNLIMITED ; // \()\d+(?= currently)' || echo 0) > 0 )) # redirect stdout and stderr to /dev/null to suppress output in cron +(( $(ncdump "${ncfile}" 2> /dev/null | grep -Po '(?<=time = UNLIMITED ; // \()\d+(?= currently)' || echo 0) > 0 )) # redirect stdout and stderr to /dev/null to suppress output in cron rc=$? # If there is no error, rc=0, else rc!=0 From 4e4c0e247b82b6663d6ae1007c5985cd57fa2219 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 17 May 2024 12:56:29 +0000 Subject: [PATCH 125/128] Do not try to collect ecen, esfc logs on first cycle --- parm/archive/enkf.yaml.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index 24321fd5d5..10aee51a19 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -13,12 +13,13 @@ enkf: - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_mem{{ '%03d' % mem }}.log" {% endfor %} - "logs/{{ cycle_YMDH }}/{{ RUN }}epos*.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}echgres.log" + + {% if current_cycle != SDATE %} # TODO archive ecen logs based on actual groups. Will need to emulate numpy.array_split to do so. - "logs/{{ cycle_YMDH }}/{{ RUN }}ecen*.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}esfc.log" - - "logs/{{ cycle_YMDH }}/{{ RUN }}echgres.log" - {% if current_cycle != SDATE %} {% if not DO_JEDIATMENS %} {% set da_files = ["enkfstat", "gsistat.ensmean", From ffeaf75925f02b82ee0a9139f064651b60d60c76 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 17 May 2024 12:57:41 +0000 Subject: [PATCH 126/128] Move staticmethod for compatability with older Python versions --- ush/python/pygfs/task/archive.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index dade674f24..d66b02d7bc 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -172,8 +172,8 @@ def execute_backup_dataset(self, atardir_set: Dict[str, Any]) -> None: else: self.cvf(atardir_set.target, atardir_set.fileset) - @logit(logger) @staticmethod + @logit(logger) def _create_fileset(atardir_set: Dict[str, Any]) -> List: """ Collect the list of all available files from the parsed yaml dict. @@ -212,8 +212,8 @@ def _create_fileset(atardir_set: Dict[str, Any]) -> List: return fileset - @logit(logger) @staticmethod + @logit(logger) def _has_rstprod(fileset: List) -> bool: """ Checks if any files in the input fileset belongs to rstprod. @@ -261,8 +261,8 @@ def _protect_rstprod(self, atardir_set: Dict[str, any]) -> None: raise RuntimeError(f"FATAL ERROR: Failed to protect {atardir_set.target}!\n" f"Please verify that it has been deleted!!") - @logit(logger) @staticmethod + @logit(logger) def _create_tarball(target: str, fileset: List) -> None: """Method to create a local tarball. @@ -312,8 +312,8 @@ def _gen_relative_paths(self, root_path: str) -> Dict: return rel_path_dict - @logit(logger) @staticmethod + @logit(logger) def _rename_cyclone_expt(arch_dict) -> None: # Rename the experiment in the tracker files from "AVNO" to the From 5aeccac58aeb6bb25fee37b8b73220502fac48e9 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 21 May 2024 12:16:06 +0000 Subject: [PATCH 127/128] Separate stmp/ptmp for orion and hercules. --- workflow/hosts/hercules.yaml | 4 ++-- workflow/hosts/orion.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index 2623672709..a3c0733b8c 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -4,8 +4,8 @@ BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/work/noaa/global/${USER}' -STMP: '/work/noaa/stmp/${USER}' -PTMP: '/work/noaa/stmp/${USER}' +STMP: '/work/noaa/stmp/${USER}/HERCULES' +PTMP: '/work/noaa/stmp/${USER}/HERCULES' NOSCRUB: $HOMEDIR SCHEDULER: slurm ACCOUNT: fv3-cpu diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index dd95def386..4f8f02b4c6 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -4,8 +4,8 @@ BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/work/noaa/global/${USER}' -STMP: '/work/noaa/stmp/${USER}' -PTMP: '/work/noaa/stmp/${USER}' +STMP: '/work/noaa/stmp/${USER}/ORION' +PTMP: '/work/noaa/stmp/${USER}/ORION' NOSCRUB: $HOMEDIR SCHEDULER: slurm ACCOUNT: fv3-cpu From 5baa7f56093f374d5929193fdba8804061db2c0f Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 21 May 2024 16:35:54 +0000 Subject: [PATCH 128/128] Revert "Separate stmp/ptmp for orion and hercules." This reverts commit 5aeccac58aeb6bb25fee37b8b73220502fac48e9. --- workflow/hosts/hercules.yaml | 4 ++-- workflow/hosts/orion.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index a3c0733b8c..2623672709 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -4,8 +4,8 @@ BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/work/noaa/global/${USER}' -STMP: '/work/noaa/stmp/${USER}/HERCULES' -PTMP: '/work/noaa/stmp/${USER}/HERCULES' +STMP: '/work/noaa/stmp/${USER}' +PTMP: '/work/noaa/stmp/${USER}' NOSCRUB: $HOMEDIR SCHEDULER: slurm ACCOUNT: fv3-cpu diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index 4f8f02b4c6..dd95def386 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -4,8 +4,8 @@ BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/work/noaa/global/${USER}' -STMP: '/work/noaa/stmp/${USER}/ORION' -PTMP: '/work/noaa/stmp/${USER}/ORION' +STMP: '/work/noaa/stmp/${USER}' +PTMP: '/work/noaa/stmp/${USER}' NOSCRUB: $HOMEDIR SCHEDULER: slurm ACCOUNT: fv3-cpu