Skip to content

Commit

Permalink
Final-final GFS v16 updates / restart reproducibility bugfixes (#325)
Browse files Browse the repository at this point in the history 
* Update .gitmodules and submodule pointer for fv3atm for code review and testing
* Add GFS v16 beta restart test, update stochastics test
* Update regression test baseline date tag to 20201214; skip-ci
* tests/rt.conf: bugfix, add missing 'fv3' to new stochy tests; skip-ci
* Regression test logs for gaea.intel, hera.gnu, hera.intel, jet.intel, orion.intel; skip-ci
* Run GFS v16beta tests also on wcoss; regression test logs for wcoss; skip-ci
* Regression test logs for cheyenne.intel and cheyenne.gnu
* Revert change to .gitmodules and update submodule pointer for fv3atm
  • Loading branch information
@climbfuji
climbfuji authored Dec 15, 2020
1 parent 9bd6b59 commit b6d13d7
Show file tree
Hide file tree
Showing 19 changed files with 3,091 additions and 3,781 deletions.
2 changes: 1 addition & 1 deletion FV3
Submodule FV3 updated 2 files
+1 −1 ccpp/physics
+4 −4 io/FV3GFS_io.F90
284 changes: 207 additions & 77 deletions tests/RegressionTests_cheyenne.gnu.log
View file

Large diffs are not rendered by default.

430 changes: 280 additions & 150 deletions tests/RegressionTests_cheyenne.intel.log
View file

Large diffs are not rendered by default.

194 changes: 97 additions & 97 deletions tests/RegressionTests_gaea.intel.log
View file

Large diffs are not rendered by default.

284 changes: 207 additions & 77 deletions tests/RegressionTests_hera.gnu.log
View file

Large diffs are not rendered by default.

625 changes: 378 additions & 247 deletions tests/RegressionTests_hera.intel.log
View file

Large diffs are not rendered by default.

2,497 changes: 70 additions & 2,427 deletions tests/RegressionTests_jet.intel.log
View file

Large diffs are not rendered by default.

805 changes: 465 additions & 340 deletions tests/RegressionTests_orion.intel.log
View file

Large diffs are not rendered by default.

660 changes: 531 additions & 129 deletions tests/RegressionTests_wcoss_cray.log
View file

Large diffs are not rendered by default.

792 changes: 597 additions & 195 deletions tests/RegressionTests_wcoss_dell_p3.log
View file

Large diffs are not rendered by default.

16 changes: 9 additions & 7 deletions tests/fv3_conf/ccpp_gfs_v16_run.IN
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
rm -fr INPUT RESTART
if [ $WARM_START = .F. ]; then
cp -r @[INPUTDATA_ROOT]/FV3_input_data/INPUT .
mkdir RESTART
else
mkdir INPUT RESTART
cp -r @[INPUTDATA_ROOT]/FV3_input_data/RESTART/* ./INPUT
rsync -arv @[INPUTDATA_ROOT]/FV3_input_data/INPUT/ INPUT/
mkdir RESTART
if [ $WARM_START = .T. ]; then
rsync -arv ../fv3_ccpp_gfs_v16beta${RT_SUFFIX}/RESTART/ INPUT/
cd INPUT
rename 20161004.000000. '' 20161004.000000.*
cd ..
fi

cp @[INPUTDATA_ROOT]/FV3_input_data/INPUT/aerosol.dat .
cp @[INPUTDATA_ROOT]/FV3_input_data/INPUT/co2historicaldata_201*.txt .
cp @[INPUTDATA_ROOT]/FV3_input_data/INPUT/sfc_emissivity_idx.txt .
Expand All @@ -19,5 +21,5 @@ cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/field_table_suite2 field_table
cp @[INPUTDATA_ROOT]/FV3_input_data/*configure .

if [ $DO_RRTMGP = .T. ]; then
cp @[INPUTDATA_ROOT]/FV3_input_data_RRTMGP/* .
cp @[INPUTDATA_ROOT]/FV3_input_data_RRTMGP/* .
fi
67 changes: 35 additions & 32 deletions tests/parm/ccpp_v16beta_c96.nml.IN
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ deflate_level=1
ntiles = 6
npz = 64
grid_type = -1
make_nh = .false.
make_nh = @[MAKE_NH]
fv_debug = .false.
range_warn = .false.
reset_eta = .false.
Expand Down Expand Up @@ -84,11 +84,11 @@ deflate_level=1
delt_max = 0.002
ke_bg = 0.
do_vort_damp = .true.
external_ic = .true.
external_ic = @[EXTERNAL_IC]
external_eta = .true.
gfs_phil = .false.
nggps_ic = .true.
mountain = .false.
nggps_ic = @[NGGPS_IC]
mountain = @[MOUNTAIN]
ncep_ic = .false.
d_con = 1.
hord_mt = 5
Expand All @@ -103,7 +103,7 @@ deflate_level=1
fill = .true.
dwind_2d = .false.
print_freq = 6
warm_start = .false.
warm_start = @[WARM_START]
no_dycore = .false.
z_tracer = .true.
agrid_vel_rst = .true.
Expand Down Expand Up @@ -177,7 +177,7 @@ deflate_level=1
debug = .false.
oz_phys = .F.
oz_phys_2015 = .T.
nstf_name = 2,1,0,0,0
nstf_name = @[NSTF_NAME]
nst_anl = .true.
psautco = 0.0008,0.0005
prautco = 0.00015,0.00015
Expand All @@ -186,11 +186,11 @@ deflate_level=1
ldiag_ugwp = .false.
do_ugwp = .false.
do_tofd = .true.
do_sppt = .true.
do_shum = .true.
do_skeb = .true.
lndp_type = @[LNDP_TYPE]
n_var_lndp = @[N_VAR_LNDP]
do_sppt = @[DO_SPPT]
do_shum = @[DO_SHUM]
do_skeb = @[DO_SKEB]
lndp_type = @[LNDP_TYPE]
n_var_lndp = @[N_VAR_LNDP]
/

&gfdl_cloud_microphysics_nml
Expand Down Expand Up @@ -295,30 +295,33 @@ deflate_level=1
/

&nam_stochy
ntrunc = 766
lon_s = 1536
lat_s = 768
skeb = 0.3
iseed_skeb = 2019102712451
iseed_shum = 2019102712452
iseed_sppt = 2019102712453
skeb_tau = 21600.
skeb_lscale = 500000.
skebnorm = 0
skeb_npass = 30
skeb_vdof = 5
shum = 0.005
shum_tau = 21600.
shum_lscale = 500000.
sppt = 0.5
sppt_tau = 21600.
sppt_lscale = 500000.
sppt_logit = .true.
sppt_sfclimit = .true.
use_zmtnblck = .true.
SKEBNORM=1,
SKEB_NPASS=30,
SKEB_VDOF=5,
SKEB=@[SKEB],
SKEB_TAU=2.16E4,
SKEB_LSCALE=1000.E3,
SKEBINT=1800,
SHUM=@[SHUM],
SHUM_TAU=21600,
SHUM_LSCALE=500000,
SHUMINT=3600,
SPPT=@[SPPT],
SPPT_TAU=21600,
SPPT_LSCALE=500000,
SPPT_LOGIT=.TRUE.,
SPPT_SFCLIMIT=.TRUE.,
SPPTINT=1800,
ISEED_SHUM=1,
ISEED_SKEB=2,
ISEED_SPPT=3,
/

&nam_sfcperts
lndp_type = @[LNDP_TYPE]
LNDP_TAU=21600,
LNDP_LSCALE=500000,
ISEED_LNDP=2010,
/

&cires_ugwp_nml
Expand Down
22 changes: 22 additions & 0 deletions tests/rt.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,18 +111,34 @@ RUN | fv3_ccpp_rrfs_v1beta
RUN | fv3_ccpp_rrfs_v1beta | standard | cheyenne.intel | fv3 |
RUN | fv3_ccpp_rrfs_v1beta | standard | gaea.intel | fv3 |

COMPILE | CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_v15p2_RRTMGP,FV3_GFS_v16beta_RRTMGP | standard | wcoss_dell_p3 | fv3 |
COMPILE | CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_v15p2_RRTMGP,FV3_GFS_v16beta_RRTMGP | standard | wcoss_cray | fv3 |
COMPILE | CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_v15p2_RRTMGP,FV3_GFS_v16beta_RRTMGP | standard | hera.intel | fv3 |
COMPILE | CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_v15p2_RRTMGP,FV3_GFS_v16beta_RRTMGP | standard | orion.intel | fv3 |
COMPILE | CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_v15p2_RRTMGP,FV3_GFS_v16beta_RRTMGP | standard | cheyenne.intel | fv3 |
RUN | fv3_ccpp_gfs_v15p2 | standard | hera.intel | fv3 |
RUN | fv3_ccpp_gfs_v15p2 | standard | orion.intel | fv3 |
RUN | fv3_ccpp_gfs_v15p2 | standard | cheyenne.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta | standard | wcoss_dell_p3 | fv3 |
RUN | fv3_ccpp_gfs_v16beta | standard | wcoss_cray | fv3 |
RUN | fv3_ccpp_gfs_v16beta | standard | hera.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta | standard | orion.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta | standard | cheyenne.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta_restart | standard | wcoss_dell_p3 | | fv3_ccpp_gfs_v16beta
RUN | fv3_ccpp_gfs_v16beta_restart | standard | wcoss_cray | | fv3_ccpp_gfs_v16beta
RUN | fv3_ccpp_gfs_v16beta_restart | standard | hera.intel | | fv3_ccpp_gfs_v16beta
RUN | fv3_ccpp_gfs_v16beta_restart | standard | orion.intel | | fv3_ccpp_gfs_v16beta
RUN | fv3_ccpp_gfs_v16beta_restart | standard | cheyenne.intel | | fv3_ccpp_gfs_v16beta
RUN | fv3_ccpp_gfs_v16beta_stochy | standard | wcoss_dell_p3 | fv3 |
RUN | fv3_ccpp_gfs_v16beta_stochy | standard | wcoss_cray | fv3 |
RUN | fv3_ccpp_gfs_v16beta_stochy | standard | hera.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta_stochy | standard | orion.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta_stochy | standard | cheyenne.intel | fv3 |
RUN | fv3_ccpp_gfs_v15p2_RRTMGP | standard | hera.intel | fv3 |
RUN | fv3_ccpp_gfs_v15p2_RRTMGP | standard | orion.intel | fv3 |
RUN | fv3_ccpp_gfs_v15p2_RRTMGP | standard | cheyenne.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta_RRTMGP | standard | wcoss_dell_p3 | fv3 |
RUN | fv3_ccpp_gfs_v16beta_RRTMGP | standard | wcoss_cray | fv3 |
RUN | fv3_ccpp_gfs_v16beta_RRTMGP | standard | hera.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta_RRTMGP | standard | orion.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta_RRTMGP | standard | cheyenne.intel | fv3 |
Expand Down Expand Up @@ -152,18 +168,24 @@ RUN | fv3_ccpp_esg_HAFS_v0_hwrf_thompson
#######################################################################################################################################################################################

# Exercise compilation without specifying suites (i.e. compile all suites) in DEBUG mode (faster than in PROD mode)
COMPILE | CCPP=Y DEBUG=Y | standard | wcoss_dell_p3 | fv3 |
COMPILE | CCPP=Y DEBUG=Y | standard | wcoss_cray | fv3 |
COMPILE | CCPP=Y DEBUG=Y | standard | hera.intel | fv3 |
COMPILE | CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_v15p2_RRTMGP,FV3_GFS_v16beta_RRTMGP DEBUG=Y | standard | orion.intel | fv3 |
COMPILE | CCPP=Y DEBUG=Y | standard | cheyenne.intel | fv3 |
RUN | fv3_ccpp_gfs_v15p2_debug | standard | hera.intel | fv3 |
RUN | fv3_ccpp_gfs_v15p2_debug | standard | orion.intel | fv3 |
RUN | fv3_ccpp_gfs_v15p2_debug | standard | cheeyenne.intel| fv3 |
RUN | fv3_ccpp_gfs_v16beta_debug | standard | wcoss_dell_p3 | fv3 |
RUN | fv3_ccpp_gfs_v16beta_debug | standard | wcoss_cray | fv3 |
RUN | fv3_ccpp_gfs_v16beta_debug | standard | hera.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta_debug | standard | orion.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta_debug | standard | cheyenne.intel | fv3 |
RUN | fv3_ccpp_gfs_v15p2_RRTMGP_debug | standard | hera.intel | fv3 |
RUN | fv3_ccpp_gfs_v15p2_RRTMGP_debug | standard | orion.intel | fv3 |
RUN | fv3_ccpp_gfs_v15p2_RRTMGP_debug | standard | cheyenne.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta_RRTMGP_debug | standard | wcoss_dell_p3 | fv3 |
RUN | fv3_ccpp_gfs_v16beta_RRTMGP_debug | standard | wcoss_cray | fv3 |
RUN | fv3_ccpp_gfs_v16beta_RRTMGP_debug | standard | hera.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta_RRTMGP_debug | standard | orion.intel | fv3 |
RUN | fv3_ccpp_gfs_v16beta_RRTMGP_debug | standard | cheyenne.intel | fv3 |
Expand Down
4 changes: 2 additions & 2 deletions tests/rt.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -423,9 +423,9 @@ if [[ $TESTS_FILE =~ '35d' ]]; then
fi

if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = gaea.* ]] || [[ $MACHINE_ID = jet.* ]]; then
RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20201204/${RT_COMPILER^^}}
RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20201214/${RT_COMPILER^^}}
else
RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20201204}
RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20201214}
fi

INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20201201/}
Expand Down
2 changes: 2 additions & 0 deletions tests/rt_gnu.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ COMPILE | CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_v16beta_flake,FV3_

RUN | fv3_ccpp_gfs_v15p2 | standard | | fv3 |
RUN | fv3_ccpp_gfs_v16beta | standard | | fv3 |
RUN | fv3_ccpp_gfs_v16beta_restart | standard | | | fv3_ccpp_gfs_v16beta
RUN | fv3_ccpp_gfs_v16beta_stochy | standard | | fv3 |
RUN | fv3_ccpp_gfs_v16beta_flake | standard | | fv3 |
RUN | fv3_ccpp_gfs_v15p2_RRTMGP | standard | | fv3 |
RUN | fv3_ccpp_gfs_v16beta_RRTMGP | standard | | fv3 |
Expand Down
18 changes: 18 additions & 0 deletions tests/tests/fv3_ccpp_gfs_v16beta
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ export LIST_FILES="atmos_4xdaily.tile1.nc \
phyf024.tile4.nc \
phyf024.tile5.nc \
phyf024.tile6.nc \
phyf048.tile1.nc \
phyf048.tile2.nc \
phyf048.tile3.nc \
phyf048.tile4.nc \
phyf048.tile5.nc \
phyf048.tile6.nc \
dynf000.tile1.nc \
dynf000.tile2.nc \
dynf000.tile3.nc \
Expand All @@ -38,6 +44,12 @@ export LIST_FILES="atmos_4xdaily.tile1.nc \
dynf024.tile4.nc \
dynf024.tile5.nc \
dynf024.tile6.nc \
dynf048.tile1.nc \
dynf048.tile2.nc \
dynf048.tile3.nc \
dynf048.tile4.nc \
dynf048.tile5.nc \
dynf048.tile6.nc \
RESTART/coupler.res \
RESTART/fv_core.res.nc \
RESTART/fv_core.res.tile1.nc \
Expand Down Expand Up @@ -76,8 +88,14 @@ export_fv3

DT_ATMOS="1200"

export FHMAX=48
export RESTART_INTERVAL=24

export NSTF_NAME=2,1,0,0,0

export FV3_RUN=ccpp_gfs_v16_run.IN
export CCPP_SUITE=FV3_GFS_v16beta
export CCPP_LIB_DIR=ccpp/lib
export INPUT_NML=ccpp_v16beta_c96.nml.IN

export WLCLK=30
2 changes: 2 additions & 0 deletions tests/tests/fv3_ccpp_gfs_v16beta_debug
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ export_fv3
export FHMAX="06"
DT_ATMOS="1200"

export NSTF_NAME=2,1,0,0,0

export FV3_RUN=ccpp_gfs_v16_run.IN
export CCPP_SUITE=FV3_GFS_v16beta
export CCPP_LIB_DIR=ccpp/lib
Expand Down
75 changes: 75 additions & 0 deletions tests/tests/fv3_ccpp_gfs_v16beta_restart
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
###############################################################################
#
# FV3 CCPP GFS v16beta compiled with 32-bit dynamics test
#
###############################################################################

export TEST_DESCR="Compare FV3 32bit CCPP GFS v16beta results with previous trunk version"

export CNTL_DIR=fv3_gfs_v16beta

export LIST_FILES="phyf048.tile1.nc \
phyf048.tile2.nc \
phyf048.tile3.nc \
phyf048.tile4.nc \
phyf048.tile5.nc \
phyf048.tile6.nc \
dynf048.tile1.nc \
dynf048.tile2.nc \
dynf048.tile3.nc \
dynf048.tile4.nc \
dynf048.tile5.nc \
dynf048.tile6.nc \
RESTART/coupler.res \
RESTART/fv_core.res.nc \
RESTART/fv_core.res.tile1.nc \
RESTART/fv_core.res.tile2.nc \
RESTART/fv_core.res.tile3.nc \
RESTART/fv_core.res.tile4.nc \
RESTART/fv_core.res.tile5.nc \
RESTART/fv_core.res.tile6.nc \
RESTART/fv_srf_wnd.res.tile1.nc \
RESTART/fv_srf_wnd.res.tile2.nc \
RESTART/fv_srf_wnd.res.tile3.nc \
RESTART/fv_srf_wnd.res.tile4.nc \
RESTART/fv_srf_wnd.res.tile5.nc \
RESTART/fv_srf_wnd.res.tile6.nc \
RESTART/fv_tracer.res.tile1.nc \
RESTART/fv_tracer.res.tile2.nc \
RESTART/fv_tracer.res.tile3.nc \
RESTART/fv_tracer.res.tile4.nc \
RESTART/fv_tracer.res.tile5.nc \
RESTART/fv_tracer.res.tile6.nc \
RESTART/sfc_data.tile1.nc \
RESTART/sfc_data.tile2.nc \
RESTART/sfc_data.tile3.nc \
RESTART/sfc_data.tile4.nc \
RESTART/sfc_data.tile5.nc \
RESTART/sfc_data.tile6.nc \
RESTART/phy_data.tile1.nc \
RESTART/phy_data.tile2.nc \
RESTART/phy_data.tile3.nc \
RESTART/phy_data.tile4.nc \
RESTART/phy_data.tile5.nc \
RESTART/phy_data.tile6.nc"


export_fv3

DT_ATMOS="1200"

export FHMAX=48
export RESTART_INTERVAL=24

export WARM_START=.T.
export NGGPS_IC=.F.
export EXTERNAL_IC=.F.
export MAKE_NH=.F.
export MOUNTAIN=.T.
export NSTF_NAME=2,0,0,0,0

export FV3_RUN=ccpp_gfs_v16_run.IN
export CCPP_SUITE=FV3_GFS_v16beta
export CCPP_LIB_DIR=ccpp/lib
export INPUT_NML=ccpp_v16beta_c96.nml.IN

Loading

0 comments on commit b6d13d7

Please sign in to comment.