From d5a6fb196ae5c2ab96f1a6b9b309cd8df9eee106 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 24 Feb 2023 10:24:13 -0500 Subject: [PATCH 01/14] rename config.fv3 to config.ufs --- parm/config/config.defaults.s2sw | 2 +- parm/config/config.efcs | 2 +- parm/config/config.fcst | 2 +- parm/config/{config.fv3 => config.ufs} | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) rename parm/config/{config.fv3 => config.ufs} (99%) diff --git a/parm/config/config.defaults.s2sw b/parm/config/config.defaults.s2sw index a5f992e858..e5ae3fcc75 100644 --- a/parm/config/config.defaults.s2sw +++ b/parm/config/config.defaults.s2sw @@ -14,7 +14,7 @@ FHOUT_HF_GFS=-1 min_seaice="1.0e-6" use_cice_alb=".true." -# config.fv3 # TODO: This is hard-wired for P8 and needs to be refactored. For now, use case C384 +# config.ufs # TODO: This is hard-wired for P8 and needs to be refactored. For now, use case C384 case "${CASE}" in "C384") DELTIM=300 diff --git a/parm/config/config.efcs b/parm/config/config.efcs index 848ede9a64..9b4a2f9864 100644 --- a/parm/config/config.efcs +++ b/parm/config/config.efcs @@ -6,7 +6,7 @@ echo "BEGIN: config.efcs" # Source model specific information that is resolution dependent -. $EXPDIR/config.fv3 $CASE_ENKF +. $EXPDIR/config.ufs $CASE_ENKF # Get task specific resources . $EXPDIR/config.resources efcs diff --git a/parm/config/config.fcst b/parm/config/config.fcst index 366fff0229..85ec9d5d92 100644 --- a/parm/config/config.fcst +++ b/parm/config/config.fcst @@ -8,7 +8,7 @@ echo "BEGIN: config.fcst" # set -eu # Source model specific information that is resolution dependent -. $EXPDIR/config.fv3 $CASE +. $EXPDIR/config.ufs $CASE # Turn off waves if not used for this CDUMP case $WAVE_CDUMP in diff --git a/parm/config/config.fv3 b/parm/config/config.ufs similarity index 99% rename from parm/config/config.fv3 rename to parm/config/config.ufs index 1f6fea460d..afacd2bc93 100644 --- a/parm/config/config.fv3 +++ b/parm/config/config.ufs @@ -17,7 +17,7 @@ fi case_in=$1 -echo "BEGIN: config.fv3" +echo "BEGIN: config.ufs" if [[ "${machine}" = "WCOSS2" ]]; then @@ -179,4 +179,4 @@ case ${case_in} in ;; esac -echo "END: config.fv3" +echo "END: config.ufs" From 2574e135276f593e632595c260a2151c7a0dee54 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 24 Feb 2023 22:17:10 -0500 Subject: [PATCH 02/14] update to other configs --- parm/config/config.base.emc.dyn | 3 + parm/config/config.efcs | 33 +++- parm/config/config.fcst | 22 +-- parm/config/config.ice | 4 + parm/config/config.ocn | 9 +- parm/config/config.resources | 167 +++++++++----------- parm/config/config.ufs | 234 ++++++++++++++++++++++------- ush/nems_configure.sh | 96 ++++++------ ush/parsing_model_configure_FV3.sh | 7 + 9 files changed, 353 insertions(+), 222 deletions(-) diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index 96705ac6bd..9cdb818c04 100644 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -170,6 +170,8 @@ export OPS_RES="C768" # Do not change export LEVS=128 export CASE="@CASECTL@" export CASE_ENKF="@CASEENS@" +# TODO: This should not depend on $CASE or $CASE_ENKF +# These are the currently available grid-combinations case "$CASE" in "C48") export OCNRES=500;; "C96") export OCNRES=100;; @@ -179,6 +181,7 @@ case "$CASE" in *) export OCNRES=025;; esac export ICERES=$OCNRES +export waveGRD='gnh_10m aoc_9km gsh_15m' case "${APP}" in ATM) diff --git a/parm/config/config.efcs b/parm/config/config.efcs index 9b4a2f9864..a9b410e416 100644 --- a/parm/config/config.efcs +++ b/parm/config/config.efcs @@ -5,19 +5,36 @@ echo "BEGIN: config.efcs" +# TODO: the _ENKF counterparts need to be defined in config.base +export DO_AERO=${DO_AERO_ENKF:-"NO"} +export DO_OCN=${DO_OCN_ENKF:-"NO"} +export DO_ICE=${DO_ICE_ENKF:-"NO"} +export DO_WAVE=${DO_WAVE_ENKF:-"NO"} + +# TODO: Possibly need OCNRES_ENKF, ICERES_ENKF, WAVRES_ENKF too +if [[ ${DO_OCN} == "YES" ]]; then + case "$CASE_ENKF" in + "C48") export OCNRES=500;; + "C96") export OCNRES=100;; + "C192") export OCNRES=050;; + "C384") export OCNRES=025;; + "C768") export OCNRES=025;; + *) export OCNRES=025;; + esac +fi +[[ ${DO_ICE} == "YES" ]] && export ICERES=$OCNRES +[[ ${DO_WAVE} == "YES" ]] && export waveGRD=${waveGRD_ENKF:-$waveGRD} # TODO: will we run waves with a different resolution in the ensemble? + # Source model specific information that is resolution dependent -. $EXPDIR/config.ufs $CASE_ENKF +string="--fv3 $CASE_ENKF" +[[ ${DO_OCN} == "YES" ]] && string="$string --mom6 $OCNRES" +[[ ${DO_ICE} == "YES" ]] && string="$string --cice6 $ICERES" +[[ ${DO_WAVE} == "YES" ]] && string="$string --ww3 ${waveGRD// /;}" +source $EXPDIR/config.ufs ${string} # Get task specific resources . $EXPDIR/config.resources efcs -export npe_fv3=$npe_efcs - -if [ $QUILTING = ".true." ]; then - export npe_fv3=$(echo " $npe_fv3 + $WRITE_GROUP * $WRTTASK_PER_GROUP" | bc) - export npe_efcs=$npe_fv3 -fi - # Use serial I/O for ensemble (lustre?) export OUTPUT_FILETYPE_ATM="netcdf" export OUTPUT_FILETYPE_SFC="netcdf" diff --git a/parm/config/config.fcst b/parm/config/config.fcst index 85ec9d5d92..d06bc0a77a 100644 --- a/parm/config/config.fcst +++ b/parm/config/config.fcst @@ -5,17 +5,19 @@ echo "BEGIN: config.fcst" -# set -eu - -# Source model specific information that is resolution dependent -. $EXPDIR/config.ufs $CASE - # Turn off waves if not used for this CDUMP case $WAVE_CDUMP in both | ${CDUMP/enkf} ) ;; # Don't change *) DO_WAVE="NO" ;; # Turn waves off esac +# Source model specific information that is resolution dependent +string="--fv3 $CASE" +[[ ${DO_OCN} == "YES" ]] && string="$string --mom6 $OCNRES" +[[ ${DO_ICE} == "YES" ]] && string="$string --cice6 $ICERES" +[[ ${DO_WAVE} == "YES" ]] && string="$string --ww3 ${waveGRD// /;}" +source $EXPDIR/config.ufs ${string} + # Source component configs if necessary for component in WAVE OCN ICE AERO; do control="DO_${component}" @@ -39,13 +41,8 @@ export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_O ####################################################################### # COUPLING COMPONENTS -export OCN_model="mom6" -export ICE_model="cice6" -export WAV_model="ww3" -export CHM_model="gocart" # cpl defaults - export cpl=".false." export cplflx=".false." export cplice=".false." @@ -71,11 +68,6 @@ if [[ "$DO_WAVE" = "YES" ]]; then export cplwav=".true." fi -####################################################################### -# COUPLING COMPONENTS -export use_coldstart=".false." - - ####################################################################### export FORECASTSH="$HOMEgfs/scripts/exglobal_forecast.sh" diff --git a/parm/config/config.ice b/parm/config/config.ice index 3b82a1468f..94e84adf6b 100644 --- a/parm/config/config.ice +++ b/parm/config/config.ice @@ -1,5 +1,7 @@ #! /usr/bin/env bash +echo "BEGIN: config.ice" + case "${ICERES}" in "025") export NX_GLB="1440" @@ -15,3 +17,5 @@ case "${ICERES}" in exit 1 ;; esac + +echo "END: config.ice" diff --git a/parm/config/config.ocn b/parm/config/config.ocn index d224dd3267..50936b3ecc 100644 --- a/parm/config/config.ocn +++ b/parm/config/config.ocn @@ -1,7 +1,6 @@ #! /usr/bin/env bash -# OCNRES is currently being set in config.base - case "$CASE" in - "C48") export OCNTIM=3600;; - *) export OCNTIM=1800;; - esac +echo "BEGIN: config.ocn" + +echo "END: config.ocn" + diff --git a/parm/config/config.resources b/parm/config/config.resources index c6418c4423..c020d269b3 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -50,7 +50,7 @@ elif [[ ${machine} = "ORION" ]]; then fi if [ ${step} = "prep" ]; then - export wtime_prep='00:45:00' + export wtime_prep='00:30:00' export npe_prep=4 export npe_node_prep=2 export nth_prep=1 @@ -238,7 +238,7 @@ elif [[ "${step}" = "aeroanlrun" ]]; then layout_y=8 ;; *) - echo "FATAL: Resolution not supported'" + echo "FATAL ERROR: Resolution ${CASE} is not supported, ABORT!" exit 1 esac @@ -414,113 +414,105 @@ elif [ ${step} = "gldas" ]; then export npe_node_gaussian=$(echo "${npe_node_max} / ${nth_gaussian}" | bc) export is_exclusive=True -elif [ ${step} = "fcst" ]; then - - if [ ${CASE} = "C48" ]; then - export wtime_fcst="00:30:00" - else - export wtime_fcst="00:40:00" - fi - if [ ${CASE} = "C768" ]; then - export wtime_fcst_gfs="06:00:00" - elif [ ${CASE} = "C384" ]; then - export wtime_fcst_gfs="06:00:00" - else - export wtime_fcst_gfs="03:00:00" - fi +elif [[ ${step} = "fcst" || ${step} = "efcs" ]]; then export is_exclusive=True + if [[ ${step} == "fcst" ]]; then + _CDUMP_LIST=${CDUMP:-"gdas gfs"} + elif [[ ${step} == "efcs" ]]; then + _CDUMP_LIST=${CDUMP:-"enkfgdas enkfgfs"} + fi + # During workflow creation, we need resources for all CDUMPs and CDUMP is undefined - CDUMP_LIST=${CDUMP:-"gdas gfs enkfgdas enkfgfs"} - for CDUMP in ${CDUMP_LIST}; do - if [[ ${CDUMP} =~ "gfs" ]]; then - export layout_x=${layout_x_gfs} - export layout_y=${layout_y_gfs} - export WRITE_GROUP=${WRITE_GROUP_GFS} - export WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP_GFS} + for _CDUMP in ${_CDUMP_LIST}; do + if [[ ${_CDUMP} =~ "gfs" ]]; then + export layout_x=${layout_x_gfs} + export layout_y=${layout_y_gfs} + export WRITE_GROUP=${WRITE_GROUP_GFS} + export WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP_GFS} + ntasks_fv3=${ntasks_fv3_gfs} + ntasks_quilt=${ntasks_quilt_gfs} + nth_fv3=${nth_fv3_gfs} fi - (( ATMPETS = layout_x * layout_y * 6 )) - - # Mediator only uses the atm model PETS or less - export MEDPETS=${MEDPETS:-ATMPETS} + # PETS for the atmosphere dycore + (( FV3PETS = layout_x * layout_y * 6 )) - if [[ ${DO_AERO} == "YES" ]]; then - # Aerosol model only uses the atm model PETS - export CHMPETS=${ATMPETS} - # Aerosol model runs on same PETs as ATM, so don't add to $NTASKS_TOT - - if [[ ${machine} == "HERA" ]]; then - # Need more memory on Hera for aerosols, so increase threads to spread it out - nth_fv3_gfs=${nth_fv3_gfs:-4} - fi + # PETS for quilting + if [[ ${QUILTING:-} == ".true." ]]; then + (( QUILTPETS = ntasks_quilt )) + else + QUILTPETS=0 fi - # If using in-line post, add the write tasks to the ATMPETS - if [[ ${QUILTING} == ".true." ]]; then - (( ATMPETS = ATMPETS + WRITE_GROUP * WRTTASK_PER_GROUP )) - fi + # Total PETS for the atmosphere component + (( ATMPETS = FV3PETS + QUILTPETS)) export ATMPETS + + # Total PETS for the coupled model (starting w/ the atmosphere) NTASKS_TOT=${ATMPETS} - export nth_fcst=${nth_fv3:-2} - export nth_fcst_gfs=${nth_fv3_gfs:-2} + # The mediator PETS can overlap with other components, usually it lands on the atmosphere tasks. + # However, it is suggested limiting mediator PETS to 300, as it may cause the slow performance. + # See https://docs.google.com/document/d/1bKpi-52t5jIfv2tuNHmQkYUe3hkKsiG_DG_s6Mnukog/edit + # TODO: Update reference with moved to RTD + MEDPETS=${MEDPETS:-ATMPETS} + [[ "${MEDPETS}" -gt 300 ]] && MEDPETS=300 + export MEDPETS - export npe_node_fcst=$(echo "${npe_node_max} / ${nth_fcst}" | bc) - export npe_node_fcst_gfs=$(echo "${npe_node_max} / ${nth_fcst_gfs}" | bc) + if [[ ${DO_AERO} == "YES" ]]; then + (( CHMPETS = ntasks_fv3 )) + # GOCART shares the same grid and forecast tasks as FV3, do not add to NTASKS_TOT + export CHMPETS + fi if [[ ${DO_WAVE} == "YES" ]]; then - case ${waveGRD} in - 'gnh_10m aoc_9km gsh_15m') export WAVPETS=140 ;; - 'gwes_30m') export WAVPETS=100 ;; - 'mx050') export WAVPETS=240 ;; - 'mx025') export WAVPETS=80 ;; - *) - echo "FATAL: Number of PEs not defined for wave grid '${waveGRD}'" - echo " Please add an entry to config.resources within fcst for this grid" - exit 3 - esac - (( NTASKS_TOT = NTASKS_TOT + WAVPETS )) + (( WAVPETS = ntasks_ww3 )) + (( NTASKS_TOT = NTASKS_TOT + WAVPETS )) + export WAVPETS fi if [[ ${DO_OCN} == "YES" ]]; then - case ${OCNRES} in - # Except for 025, these are guesses for now - 500) export OCNPETS=8 ;; - 100) export OCNPETS=20 ;; - 050) export OCNPETS=60 ;; - 025) export OCNPETS=220 ;; - *) - echo "FATAL: Number of PEs not defined for ocean resolution ${OCNRES}" - echo " Please add an entry to config.resources within fcst for this resolution" - exit 3 - esac - (( NTASKS_TOT = NTASKS_TOT + OCNPETS )) + (( OCNPETS = ntasks_mom6 )) + (( NTASKS_TOT = NTASKS_TOT + OCNPETS )) + export OCNPETS fi if [[ ${DO_ICE} == "YES" ]]; then - case ${ICERES} in - # Except for 025, these are guesses for now - 500) export ICEPETS=4 ;; - 100) export ICEPETS=10 ;; - 050) export ICEPETS=30 ;; - 025) export ICEPETS=120 ;; - *) - echo "FATAL: Number of PEs not defined for ice resolution ${ICERES}" - echo " Please add an entry to config.resources within fcst for this resolution" - exit 3 - esac - (( NTASKS_TOT = NTASKS_TOT + ICEPETS )) + (( ICEPETS = ntasks_cice6 )) + (( NTASKS_TOT = NTASKS_TOT + ICEPETS )) + export ICEPETS fi - if [[ ${CDUMP} =~ "gfs" ]]; then - export npe_fcst_gfs=${NTASKS_TOT} + if [[ ${_CDUMP} =~ "gfs" ]]; then + declare -x npe_${step}_gfs=${NTASKS_TOT} + threads=${nth_fv3_gfs:-2} + declare -x nth_${step}_gfs=${threads} + declare -x npe_node_${step}_gfs=$(echo "${npe_node_max} / ${threads}" | bc) else - export npe_fcst=${NTASKS_TOT} + declare -x npe_${step}=${NTASKS_TOT} + threads=${nth_fv3:-2} + declare -x nth_${step}=${threads} + declare -x npe_node_${step}=$(echo "${npe_node_max} / ${threads}" | bc) fi done + case "${CASE}" in + "C48" | "C96" | "C192") + declare -x wtime_${step}="00:30:00" + declare -x wtime_${step}_gfs="03:00:00" + ;; + "C384" | "C768" | "C1152") + declare -x wtime_${step}="01:00:00" + declare -x wtime_${step}_gfs="06:00:00" + ;; + *) + echo "FATAL ERROR: Resolution ${CASE} not supported in ${step}" + exit 1 + ;; + esac + elif [ ${step} = "ocnpost" ]; then export wtime_ocnpost="00:30:00" @@ -786,17 +778,6 @@ elif [ ${step} = "esfc" ]; then export npe_node_cycle=$(echo "${npe_node_max} / ${nth_cycle}" | bc) export memory_esfc="80GB" -elif [ ${step} = "efcs" ]; then - - if [ ${CASE} = "C768" ]; then - export wtime_efcs="06:00:00" - else - export wtime_efcs="00:40:00" - fi - export npe_efcs=$(echo "${layout_x} * ${layout_y} * 6" | bc) - export nth_efcs=${nth_fv3:-2} - export npe_node_efcs=$(echo "${npe_node_max} / ${nth_efcs}" | bc) - elif [ ${step} = "epos" ]; then export wtime_epos="00:15:00" diff --git a/parm/config/config.ufs b/parm/config/config.ufs index afacd2bc93..cd6d76b2e6 100644 --- a/parm/config/config.ufs +++ b/parm/config/config.ufs @@ -1,49 +1,101 @@ #! /usr/bin/env bash -########## config.fv3 ########## -# FV3 model resolution specific parameters +########## config.ufs ########## +# UFS model resolution specific parameters # e.g. time-step, processor layout, physics and dynamics parameters -# This config sets default variables for FV3 for a given resolution +# This config sets default variables for FV3, MOM6, CICE6 for their resolutions # User can over-ride after sourcing this config file -if [ $# -ne 1 ]; then +echo "BEGIN: config.ufs" + +if [ $# -le 1 ]; then echo "Must specify an input resolution argument to set variables!" echo "argument can be any one of the following:" - echo "C48 C96 C192 C384 C768 C1152 C3072" + echo "--fv3 C48|C96|C192|C384|C768|C1152|C3072" + echo "--mom6 500|100|025" + echo "--cice6 500|100|025" + echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|mx050|mx025" + exit 1 fi -case_in=$1 - -echo "BEGIN: config.ufs" +# Initialize +skip_mom6=true +skip_cice6=true +skip_ww3=true +# Loop through named arguments +while [[ $# -gt 0 ]]; do + key="$1" + case "${key}" in + "--fv3") + fv3_res="$2" + ;; + "--mom6") + mom6_res="$2" + skip_mom6=false + ;; + "--cice6") + cice6_res="$2" + skip_cice6=false + ;; + "--ww3") + ww3_res="$2" + skip_ww3=false + ;; + *) # unknown option + echo "FATAL ERROR: Unknown option: ${key}, ABORT!" + exit 1 + ;; + esac + shift + shift +done -if [[ "${machine}" = "WCOSS2" ]]; then - export npe_node_max=128 -elif [[ "${machine}" = "JET" ]]; then - if [[ "${PARTITION_BATCH}" = "xjet" ]]; then - export npe_node_max=24 - elif [[ "${PARTITION_BATCH}" = "vjet" || "${PARTITION_BATCH}" = "sjet" ]]; then - export npe_node_max=16 - elif [[ "${PARTITION_BATCH}" = "kjet" ]]; then - export npe_node_max=40 - fi -elif [[ "${machine}" = "HERA" ]]; then - export npe_node_max=40 -elif [[ "${machine}" = "S4" ]]; then - if [[ "${PARTITION_BATCH}" = "s4" ]]; then - export npe_node_max=32 - elif [[ "${PARTITION_BATCH}" = "ivy" ]]; then - export npe_node_max=20 - fi -elif [[ "${machine}" = "ORION" ]]; then - export npe_node_max=40 -fi +case "${machine}" in + "WCOSS2") + export npe_node_max=128 + ;; + "HERA" | "ORION") + export npe_node_max=40 + ;; + "JET") + case "${PARTITION_BATCH}" in + "xjet") + export npe_node_max=24 + ;; + "vjet" | "sjet") + export npe_node_max=16 + ;; + "kjet") + export npe_node_max=40 + ;; + *) + echo "FATAL ERROR: Unsupported PARTITION_BATCH = ${PARTITION_BATCH}, ABORT!" + exit 1 + ;; + esac + ;; + "S4") + case "${PARTITION_BATCH}" in + "s4") + export npe_node_max=32 + ;; + "ivy") + export npe_node_max=20 + ;; + *) + echo "FATAL ERROR: Unsupported PARTITION_BATCH = ${PARTITION_BATCH}, ABORT!" + exit 1 + ;; + esac + ;; +esac # (Standard) Model resolution dependent variables -case ${case_in} in +case "${fv3_res}" in "C48") export DELTIM=1200 export layout_x=1 @@ -54,26 +106,25 @@ case ${case_in} in export nth_fv3_gfs=1 export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 - export WRTTASK_PER_GROUP=6 + export WRTTASK_PER_GROUP=2 export WRITE_GROUP_GFS=1 - export WRTTASK_PER_GROUP_GFS=6 + export WRTTASK_PER_GROUP_GFS=2 export WRTIOBUF="1M" ;; "C96") - export DELTIM=450 - export layout_x=6 - export layout_y=4 - export layout_x_gfs=6 - export layout_y_gfs=4 + export DELTIM=600 + export layout_x=2 + export layout_y=2 + export layout_x_gfs=2 + export layout_y_gfs=2 export nth_fv3=1 export nth_fv3_gfs=1 export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 - export WRTTASK_PER_GROUP=64 + export WRTTASK_PER_GROUP=4 export WRITE_GROUP_GFS=1 - export WRTTASK_PER_GROUP_GFS=64 + export WRTTASK_PER_GROUP_GFS=4 export WRTIOBUF="4M" - export n_split=6 ;; "C192") export DELTIM=450 @@ -130,9 +181,9 @@ case ${case_in} in export nth_fv3_gfs=4 export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=4 - export WRTTASK_PER_GROUP=$(echo "2*${npe_node_max}" |bc) + export WRTTASK_PER_GROUP=$(echo "2*${npe_node_max}" |bc) # TODO: this is pointless because below this is getting reset export WRITE_GROUP_GFS=4 - export WRTTASK_PER_GROUP_GFS=$(echo "2*${npe_node_max}" |bc) + export WRTTASK_PER_GROUP_GFS=$(echo "2*${npe_node_max}" |bc) # TODO: this is pointless because below this is getting reset export WRTIOBUF="48M" ;; "C3072") @@ -145,13 +196,13 @@ case ${case_in} in export nth_fv3_gfs=4 export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=4 - export WRTTASK_PER_GROUP=$(echo "3*${npe_node_max}" |bc) + export WRTTASK_PER_GROUP=$(echo "3*${npe_node_max}" |bc) # TODO: this is pointless because below this is getting reset export WRITE_GROUP_GFS=4 - export WRTTASK_PER_GROUP_GFS=$(echo "3*${npe_node_max}" |bc) + export WRTTASK_PER_GROUP_GFS=$(echo "3*${npe_node_max}" |bc) # TODO: this is pointless because below this is getting reset export WRTIOBUF="64M" ;; *) - echo "grid ${case_in} not supported, ABORT!" + echo "FATAL ERROR: Unsupported FV3 resolution = ${fv3_res}, ABORT!" exit 1 ;; esac @@ -159,16 +210,18 @@ esac if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi -# Calculate chunksize based on resolution -export RESTILE=$(echo ${case_in} |cut -c2-) -export ichunk2d=$((4*RESTILE)) -export jchunk2d=$((2*RESTILE)) -export ichunk3d=$((4*RESTILE)) -export jchunk3d=$((2*RESTILE)) -export kchunk3d=1 +(( ntasks_fv3 = layout_x * layout_y * 6 )) +(( ntasks_fv3_gfs = layout_x_gfs * layout_y_gfs * 6 )) +export ntasks_fv3 +export ntasks_fv3_gfs + +(( ntasks_quilt = WRITE_GROUP * WRTTASK_PER_GROUP )) +(( ntasks_quilt_gfs = WRITE_GROUP_GFS * WRTTASK_PER_GROUP_GFS )) +export ntasks_quilt +export ntasks_quilt_gfs # Determine whether to use parallel NetCDF based on resolution -case ${case_in} in +case ${fv3_res} in "C48" | "C96" | "C192" | "C384") export OUTPUT_FILETYPE_ATM="netcdf" export OUTPUT_FILETYPE_SFC="netcdf" @@ -179,4 +232,79 @@ case ${case_in} in ;; esac +# MOM6 specific settings +if [[ "${skip_mom6}" == "false" ]]; then + export nth_mom6=1 # MOM6 is not thread-safe (allegedly) + case "${mom6_res}" in + "500") + export ntasks_mom6=8 + export OCNTIM=3600 + ;; + "100") + export ntasks_mom6=20 + export OCNTIM=3600 + ;; + "50") + export ntasks_mom6=60 + export OCNTIM=3600 + ;; + "025") + export ntasks_mom6=220 + export OCNTIM=1800 + ;; + *) + echo "FATAL ERROR: Unsupported MOM6 resolution = ${mom6_res}, ABORT!" + exit 1 + ;; + esac +fi + +# CICE6 specific settings +if [[ "${skip_cice6}" == "false" ]]; then + export nth_cice6=${mom6_res} # CICE6 needs to run on same threads as MOM6 (allegedly) + case "${cice6_res}" in + "500") + export ntasks_cice6=4 + ;; + "100") + export ntasks_cice6=10 + echo "FATAL ERROR: Unsupported CICE6 resolution = ${cice6_res}, ABORT!" + ;; + "050") + export ntasks_cice6=30 + echo "FATAL ERROR: Unsupported CICE6 resolution = ${cice6_res}, ABORT!" + ;; + "025") + export ntasks_cice6=120 + ;; + *) + echo "FATAL ERROR: Unsupported CICE6 resolution = ${cice6_res}, ABORT!" + exit 1 + ;; + esac +fi + +# WW3 specific settings +if [[ "${skip_ww3}" == "false" ]]; then + export nth_ww3=2 + case "${ww3_res}" in + "gnh_10m;aoc_9km;gsh_15m") + export ntasks_ww3=140 + ;; + "gwes_30m") + export ntasks_ww3=100 + ;; + "mx050") + export ntasks_ww3=240 + ;; + "mx025") + export ntasks_ww3=80 + ;; + *) + echo "FATAL ERROR: Unsupported WW3 resolution = ${ww3_res}, ABORT!" + exit 1 + ;; + esac +fi + echo "END: config.ufs" diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh index c49d35a2c0..96ff91953e 100755 --- a/ush/nems_configure.sh +++ b/ush/nems_configure.sh @@ -7,7 +7,7 @@ ## $cpl** switches. ## ## This is a child script of modular -## forecast script. This script is definition only +## forecast script. This script is definition only (Is it? There is nothing defined here being used outside this script.) ##### writing_nems_configure() { @@ -17,51 +17,22 @@ if [ -e $SCRIPTDIR/nems.configure ]; then fi # Setup nems.configure -DumpFields=${NEMSDumpFields:-false} -cap_dbug_flag=${cap_dbug_flag:-0} +local DumpFields=${NEMSDumpFields:-false} +local cap_dbug_flag=${cap_dbug_flag:-0} # Determine "cmeps_run_type" based on the availability of the mediator restart file # If it is a warm_start, we already copied the mediator restart to DATA, if it was present # If the mediator restart was not present, despite being a "warm_start", we put out a WARNING # in forecast_postdet.sh if [[ -f "${DATA}/ufs.cpld.cpl.r.nc" ]]; then - cmeps_run_type='continue' + local cmeps_run_type='continue' else - cmeps_run_type='startup' + local cmeps_run_type='startup' fi -restart_interval=${restart_interval:-3024000} # Interval in seconds to write restarts - -ATM_model=${ATM_model:-'fv3'} -OCN_model=${OCN_model:-'mom6'} -ICE_model=${ICE_model:-'cice'} -WAV_model=${WAV_model:-'ww3'} -CHM_model=${CHM_model:-'gocart'} - -ATMPETS=${ATMPETS:-8} -MEDPETS=${MEDPETS:-8} -OCNPETS=${OCNPETS:-0} -ICEPETS=${ICEPETS:-0} -WAVPETS=${WAVPETS:-0} -CHMPETS=${CHMPETS:-${ATMPETS}} - -USE_MOMMESH=${USE_MOMMESH:-"true"} -MESH_OCN_ICE=${MESH_OCN_ICE:-"mesh.mx${ICERES}.nc"} - -case "${OCNRES}" in - "500") EPS_IMESH="4.0e-1";; - "100") EPS_IMESH="2.5e-1";; - *) EPS_IMESH="1.0e-1";; -esac +local res_int=${restart_interval:-3024000} # Interval in seconds to write restarts rm -f $DATA/nems.configure -med_petlist_bounds=${med_petlist_bounds:-"0 $(( $MEDPETS-1 ))"} -atm_petlist_bounds=${atm_petlist_bounds:-"0 $(( $ATMPETS-1 ))"} -ocn_petlist_bounds=${ocn_petlist_bounds:-"$ATMPETS $(( $ATMPETS+$OCNPETS-1 ))"} -ice_petlist_bounds=${ice_petlist_bounds:-"$(( $ATMPETS+$OCNPETS )) $(( $ATMPETS+$OCNPETS+$ICEPETS-1 ))"} -wav_petlist_bounds=${wav_petlist_bounds:-"$(( $ATMPETS+$OCNPETS+$ICEPETS )) $(( $ATMPETS+$OCNPETS+$ICEPETS+$WAVPETS-1 ))"} -chm_petlist_bounds=${chm_petlist_bounds:-"0 $(( $CHMPETS-1 ))"} - -esmf_logkind=${esmf_logkind:-"ESMF_LOGKIND_MULTI"} #options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE +local esmf_logkind=${esmf_logkind:-"ESMF_LOGKIND_MULTI"} #options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE # Copy the selected template into run directory infile="$SCRIPTDIR/nems.configure.$confignamevarfornems.IN" @@ -71,8 +42,12 @@ else echo "FATAL ERROR: nem.configure template '$infile' does not exist!" exit 1 fi + +local atm_petlist_bounds="0 $(( $ATMPETS-1 ))" +local med_petlist_bounds="0 $(( $MEDPETS-1 ))" + sed -i -e "s;@\[med_model\];cmeps;g" tmp1 -sed -i -e "s;@\[atm_model\];$ATM_model;g" tmp1 +sed -i -e "s;@\[atm_model\];fv3;g" tmp1 sed -i -e "s;@\[med_petlist_bounds\];$med_petlist_bounds;g" tmp1 sed -i -e "s;@\[atm_petlist_bounds\];$atm_petlist_bounds;g" tmp1 sed -i -e "s;@\[esmf_logkind\];$esmf_logkind;g" tmp1 @@ -82,39 +57,64 @@ if [ $cpl = ".true." ]; then fi if [ $cplflx = .true. ]; then - if [ $restart_interval -gt 0 ]; then - restart_interval_nems=$restart_interval + if [ $res_int -gt 0 ]; then + local restart_interval_nems=$res_int else - restart_interval_nems=$FHMAX + local restart_interval_nems=$FHMAX fi - sed -i -e "s;@\[ocn_model\];$OCN_model;g" tmp1 + + case "${OCNRES}" in + "500") local eps_imesh="4.0e-1";; + "100") local eps_imesh="2.5e-1";; + *) local eps_imesh="1.0e-1";; + esac + + local use_coldstart=${use_coldstart:-".false."} + local use_mommesh=${USE_MOMMESH:-"true"} + local restile=$(echo ${CASE} |cut -c2-) + local ocn_petlist_bounds="$ATMPETS $(( $ATMPETS+$OCNPETS-1 ))" + + sed -i -e "s;@\[ocn_model\];mom6;g" tmp1 sed -i -e "s;@\[ocn_petlist_bounds\];$ocn_petlist_bounds;g" tmp1 sed -i -e "s;@\[DumpFields\];$DumpFields;g" tmp1 sed -i -e "s;@\[cap_dbug_flag\];$cap_dbug_flag;g" tmp1 sed -i -e "s;@\[use_coldstart\];$use_coldstart;g" tmp1 sed -i -e "s;@\[RUNTYPE\];$cmeps_run_type;g" tmp1 sed -i -e "s;@\[CPLMODE\];$cplmode;g" tmp1 - sed -i -e "s;@\[restart_interval\];$restart_interval;g" tmp1 + sed -i -e "s;@\[restart_interval\];$res_int;g" tmp1 sed -i -e "s;@\[coupling_interval_fast_sec\];$CPL_FAST;g" tmp1 sed -i -e "s;@\[RESTART_N\];$restart_interval_nems;g" tmp1 - sed -i -e "s;@\[use_mommesh\];$USE_MOMMESH;g" tmp1 - sed -i -e "s;@\[eps_imesh\];$EPS_IMESH;g" tmp1 - sed -i -e "s;@\[ATMTILESIZE\];$RESTILE;g" tmp1 + sed -i -e "s;@\[use_mommesh\];$use_mommesh;g" tmp1 + sed -i -e "s;@\[eps_imesh\];$eps_imesh;g" tmp1 + sed -i -e "s;@\[ATMTILESIZE\];$restile;g" tmp1 fi + if [ $cplwav = .true. ]; then + + local wav_petlist_bounds="$(( $ATMPETS+$OCNPETS+$ICEPETS )) $(( $ATMPETS+$OCNPETS+$ICEPETS+$WAVPETS-1 ))" + sed -i -e "s;@\[wav_model\];ww3;g" tmp1 sed -i -e "s;@\[wav_petlist_bounds\];$wav_petlist_bounds;g" tmp1 sed -i -e "s;@\[MESH_WAV\];$MESH_WAV;g" tmp1 sed -i -e "s;@\[MULTIGRID\];$waveMULTIGRID;g" tmp1 fi + if [ $cplice = .true. ]; then - sed -i -e "s;@\[ice_model\];$ICE_model;g" tmp1 + + local mesh_ocn_ice=${MESH_OCN_ICE:-"mesh.mx${ICERES}.nc"} + local ice_petlist_bounds="$(( $ATMPETS+$OCNPETS )) $(( $ATMPETS+$OCNPETS+$ICEPETS-1 ))" + + sed -i -e "s;@\[ice_model\];cice6;g" tmp1 sed -i -e "s;@\[ice_petlist_bounds\];$ice_petlist_bounds;g" tmp1 - sed -i -e "s;@\[MESH_OCN_ICE\];$MESH_OCN_ICE;g" tmp1 + sed -i -e "s;@\[MESH_OCN_ICE\];$mesh_ocn_ice;g" tmp1 sed -i -e "s;@\[FHMAX\];$FHMAX_GFS;g" tmp1 fi + if [ $cplchm = .true. ]; then - sed -i -e "s;@\[chm_model\];$CHM_model;g" tmp1 + + local chm_petlist_bounds="0 $(( $CHMPETS-1 ))" + + sed -i -e "s;@\[chm_model\];gocart;g" tmp1 sed -i -e "s;@\[chm_petlist_bounds\];$chm_petlist_bounds;g" tmp1 sed -i -e "s;@\[coupling_interval_fast_sec\];$CPL_FAST;g" tmp1 fi diff --git a/ush/parsing_model_configure_FV3.sh b/ush/parsing_model_configure_FV3.sh index 4574b6e352..cc84f2599e 100755 --- a/ush/parsing_model_configure_FV3.sh +++ b/ush/parsing_model_configure_FV3.sh @@ -12,6 +12,13 @@ FV3_model_configure(){ +local restile=$(echo ${CASE} |cut -c2-) +local ichunk2d=$((4*restile)) +local jchunk2d=$((2*restile)) +local ichunk3d=$((4*restile)) +local jchunk3d=$((2*restile)) +local kchunk3d=1 + rm -f model_configure cat >> model_configure < Date: Fri, 24 Feb 2023 23:06:42 -0500 Subject: [PATCH 03/14] move ice decomp into config.ufs --- parm/config/config.ice | 16 ---------------- parm/config/config.ufs | 8 ++++++++ 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/parm/config/config.ice b/parm/config/config.ice index 94e84adf6b..7bc1f80966 100644 --- a/parm/config/config.ice +++ b/parm/config/config.ice @@ -2,20 +2,4 @@ echo "BEGIN: config.ice" -case "${ICERES}" in - "025") - export NX_GLB="1440" - export NY_GLB="1080" - ;; - "500") - export NX_GLB="72" - export NY_GLB="35" - export cice_processor_shape="slenderX1" - ;; - *) - echo "FATAL ERROR: Unsupported ICERES = ${ICERES}, ABORT!" - exit 1 - ;; -esac - echo "END: config.ice" diff --git a/parm/config/config.ufs b/parm/config/config.ufs index cd6d76b2e6..4bf80090d1 100644 --- a/parm/config/config.ufs +++ b/parm/config/config.ufs @@ -265,17 +265,25 @@ if [[ "${skip_cice6}" == "false" ]]; then case "${cice6_res}" in "500") export ntasks_cice6=4 + export NX_GLB="72" + export NY_GLB="35" + export cice_processor_shape="slenderX1" ;; "100") export ntasks_cice6=10 echo "FATAL ERROR: Unsupported CICE6 resolution = ${cice6_res}, ABORT!" + exit 1 ;; "050") export ntasks_cice6=30 echo "FATAL ERROR: Unsupported CICE6 resolution = ${cice6_res}, ABORT!" + exit 1 ;; "025") export ntasks_cice6=120 + export NX_GLB="1440" + export NY_GLB="1080" + export cice_processor_shape="slenderX2" ;; *) echo "FATAL ERROR: Unsupported CICE6 resolution = ${cice6_res}, ABORT!" From b79e2b29d1abe6bfba7c5a739ff594224cd7fcc3 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sun, 26 Feb 2023 11:00:14 -0500 Subject: [PATCH 04/14] extract configuration bits from parsing_namelists_MOM6 into config.ocn --- parm/config/config.fcst | 2 - parm/config/config.ocn | 19 +++++++- parm/config/config.ufs | 84 +++++++++++++++++++++++++---------- ush/forecast_postdet.sh | 3 -- ush/parsing_namelists_CICE.sh | 11 ++--- ush/parsing_namelists_MOM6.sh | 81 +++++---------------------------- 6 files changed, 94 insertions(+), 106 deletions(-) diff --git a/parm/config/config.fcst b/parm/config/config.fcst index d06bc0a77a..bc80fcbeae 100644 --- a/parm/config/config.fcst +++ b/parm/config/config.fcst @@ -240,8 +240,6 @@ export DO_SKEB=${DO_SKEB:-"NO"} export DO_SHUM=${DO_SHUM:-"NO"} export DO_LAND_PERT=${DO_LAND_PERT:-"NO"} export DO_CA=${DO_CA:-"YES"} -export DO_OCN_SPPT=${DO_OCN_SPPT:-"NO"} -export DO_OCN_PERT_EPBL=${DO_OCN_PERT_EPBL:-"NO"} #coupling settings export cplmode="nems_frac" diff --git a/parm/config/config.ocn b/parm/config/config.ocn index 50936b3ecc..7d14e3dd52 100644 --- a/parm/config/config.ocn +++ b/parm/config/config.ocn @@ -2,5 +2,22 @@ echo "BEGIN: config.ocn" -echo "END: config.ocn" +# MOM_input template to use +export MOM_INPUT="MOM_input_template_${OCNRES}" + +export DO_OCN_SPPT="NO" # In MOM_input, this variable is determines OCN_SPPT (OCN_SPPT = True|False) +export DO_OCN_PERT_EPBL="NO" # In MOM_input, this variable determines PERT_EPBL (PERT_EPBL = True|False) + +# Templated variables in MOM_input_template +export MOM6_USE_LI2016="True" # set to False for restart reproducibility +export MOM6_THERMO_SPAN="False" +export MOM6_ALLOW_LANDMASK_CHANGES="False" +if [[ "${DO_JEDIOCNVAR}" == "YES" ]]; then + export ODA_INCUPD="True" +else + export ODA_INCUPD="False" +fi +export ODA_INCUPD_NHOURS="3.0" # In MOM_input, this is time interval for applying increment + +echo "END: config.ocn" diff --git a/parm/config/config.ufs b/parm/config/config.ufs index 4bf80090d1..0375c1f541 100644 --- a/parm/config/config.ufs +++ b/parm/config/config.ufs @@ -237,26 +237,66 @@ if [[ "${skip_mom6}" == "false" ]]; then export nth_mom6=1 # MOM6 is not thread-safe (allegedly) case "${mom6_res}" in "500") - export ntasks_mom6=8 - export OCNTIM=3600 + ntasks_mom6=8 + OCNTIM=3600 + NX_GLB=72 + NY_GLB=35 + DT_DYNAM_MOM6='3600' + DT_THERM_MOM6='3600' + FRUNOFF="" + CHLCLIM="seawifs_1998-2006_smoothed_2X.nc" + MOM6_RESTART_SETTING='r' + MOM6_RIVER_RUNOFF='False' ;; "100") - export ntasks_mom6=20 - export OCNTIM=3600 + ntasks_mom6=20 + OCNTIM=3600 + NX_GLB=360 + NY_GLB=320 + DT_DYNAM_MOM6='1800' + DT_THERM_MOM6='3600' + FRUNOFF="" + CHLCLIM="seawifs_1998-2006_smoothed_2X.nc" + MOM6_RESTART_SETTING='n' + MOM6_RIVER_RUNOFF='False' ;; "50") - export ntasks_mom6=60 - export OCNTIM=3600 - ;; + ntasks_mom6=60 + OCNTIM=3600 + NX_GLB=720 + NY_GLB=576 + DT_DYNAM_MOM6='1800' + DT_THERM_MOM6='3600' + FRUNOFF="runoff.daitren.clim.${NX_GLB}x${NY_GLB}.v20180328.nc" + CHLCLIM="seawifs-clim-1997-2010.${NX_GLB}x${NY_GLB}.v20180328.nc" + MOM6_RESTART_SETTING='n' + MOM6_RIVER_RUNOFF='True' + ;; "025") - export ntasks_mom6=220 - export OCNTIM=1800 + ntasks_mom6=220 + OCNTIM=1800 + NX_GLB=1440 + NY_GLB=1080 + DT_DYNAM_MOM6='900' + DT_THERM_MOM6='1800' + FRUNOFF="runoff.daitren.clim.${NX_GLB}x${NY_GLB}.v20180328.nc" + CHLCLIM="seawifs-clim-1997-2010.${NX_GLB}x${NY_GLB}.v20180328.nc" + MOM6_RIVER_RUNOFF='True' + MOM6_RESTART_SETTING="r" ;; *) echo "FATAL ERROR: Unsupported MOM6 resolution = ${mom6_res}, ABORT!" exit 1 ;; esac + export ntasks_mom6 + export OCNTIM + export NX_GLB NY_GLB + export DT_DYNAM_MOM6 DT_THERM_MOM6 + export FRUNOFF + export CHLCLIM + export MOM6_RIVER_RUNOFF + export MOM6_RESTART_SETTING fi # CICE6 specific settings @@ -264,32 +304,30 @@ if [[ "${skip_cice6}" == "false" ]]; then export nth_cice6=${mom6_res} # CICE6 needs to run on same threads as MOM6 (allegedly) case "${cice6_res}" in "500") - export ntasks_cice6=4 - export NX_GLB="72" - export NY_GLB="35" - export cice_processor_shape="slenderX1" + ntasks_cice6=4 + cice6_processor_shape="slenderX1" ;; "100") - export ntasks_cice6=10 - echo "FATAL ERROR: Unsupported CICE6 resolution = ${cice6_res}, ABORT!" - exit 1 + ntasks_cice6=10 + cice6_processor_shape="slenderX1" ;; "050") - export ntasks_cice6=30 - echo "FATAL ERROR: Unsupported CICE6 resolution = ${cice6_res}, ABORT!" - exit 1 + ntasks_cice6=30 + cice6_processor_shape="slenderX2" ;; "025") - export ntasks_cice6=120 - export NX_GLB="1440" - export NY_GLB="1080" - export cice_processor_shape="slenderX2" + ntasks_cice6=120 + cice6_processor_shape="slenderX2" ;; *) echo "FATAL ERROR: Unsupported CICE6 resolution = ${cice6_res}, ABORT!" exit 1 ;; esac + # NX_GLB and NY_GLB are set in the MOM6 section above + # CICE6 runs on the same domain decomposition as MOM6 + export ntasks_cice6 + export cice6_processor_shape fi # WW3 specific settings diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index effc5cc36e..3d02c5edcd 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -755,8 +755,6 @@ CPL_out() { MOM6_postdet() { echo "SUB ${FUNCNAME[0]}: MOM6 after run type determination" - OCNRES=${OCNRES:-"025"} # TODO: remove from here and lift higher - # Copy MOM6 ICs $NLN "${ROTDIR}/${CDUMP}.${gPDY}/${gcyc}/ocean/RESTART/${PDY}.${cyc}0000.MOM.res.nc" "${DATA}/INPUT/MOM.res.nc" case $OCNRES in @@ -776,7 +774,6 @@ MOM6_postdet() { exit 111 fi $NLN "${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean/${CDUMP}.t${cyc}z.ocninc.nc" "${DATA}/INPUT/mom6_increment.nc" - export ODA_INCUPD="true" fi # Copy MOM6 fixed files diff --git a/ush/parsing_namelists_CICE.sh b/ush/parsing_namelists_CICE.sh index 90f60c5719..10abceda1d 100755 --- a/ush/parsing_namelists_CICE.sh +++ b/ush/parsing_namelists_CICE.sh @@ -23,8 +23,8 @@ else fi # Get correct MPI options for NPROC and grid -local cice_processor_shape=${cice_processor_shape:-'slenderX2'} -local shape=${cice_processor_shape#${cice_processor_shape%?}} +local processor_shape=${cice6_processor_shape:-'slenderX2'} +local shape=${processor_shape#${processor_shape%?}} local NPX=$(( ICEPETS / shape )) #number of processors in x direction local NPY=$(( ICEPETS / NPX )) #number of processors in y direction if (( $(( NX_GLB % NPX )) == 0 )); then @@ -37,10 +37,7 @@ if (( $(( NY_GLB % NPY )) == 0 )); then else local block_size_y=$(( (NY_GLB / NPY) + 1 )) fi -local max_blocks=$(( (NX_GLB * NY_GLB) / (block_size_x * block_size_y * ICEPETS) )) -if (( max_blocks == 0 )) || (( max_blocks % 2 != 0 )); then - local max_blocks=-1 -fi +local max_blocks=-1 cat > ice_in < ice_in <> input.nml <> input.nml <> input.nml <> input.nml < Date: Sun, 26 Feb 2023 11:19:47 -0500 Subject: [PATCH 05/14] apply shellcheck for ush parsing scripts --- .github/scripts/apply_shellcheck.sh | 27 ++++++++ ush/nems_configure.sh | 98 ++++++++++++++--------------- ush/parsing_model_configure_FV3.sh | 20 +++--- ush/parsing_namelists_CICE.sh | 26 ++++---- ush/parsing_namelists_MOM6.sh | 52 +++++++-------- 5 files changed, 125 insertions(+), 98 deletions(-) create mode 100755 .github/scripts/apply_shellcheck.sh diff --git a/.github/scripts/apply_shellcheck.sh b/.github/scripts/apply_shellcheck.sh new file mode 100755 index 0000000000..1e0b59e03f --- /dev/null +++ b/.github/scripts/apply_shellcheck.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -eu + +apply_shellcheck() { + local filename=${1:?} + local code=${2:-""} + patch="/tmp/patch.${RANDOM}" # Do not use PWD, as this will create tons of patch files + [[ -f "${patch}" ]] && rm -f "${patch}" + set +e + shellcheck -i "${code}" "${filename}" -f diff > "${patch}" + set -e + patch_size=$(wc -l ${patch} | awk '{print $1}') + if [[ "${patch_size}" -gt 0 ]]; then + echo "Apply patch for ${code} on ${filename}" + git apply "${patch}" + else + echo "No patch to apply for ${code} on ${filename}" + fi + rm -f "${patch}" +} + +file=${1:?} # File to fix shellcheck errors +apply_shellcheck "${file}" SC2292 # Prefer [[ ]] over [ ] for tests in Bash/Ksh +apply_shellcheck "${file}" SC2250 # Prefer putting braces around variable e.g. ${variable} +apply_shellcheck "${file}" SC2248 # Prefer double quoting +apply_shellcheck "${file}" SC2086 # Double quote to prevent globbing and word splitting diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh index 96ff91953e..5482b85403 100755 --- a/ush/nems_configure.sh +++ b/ush/nems_configure.sh @@ -12,8 +12,8 @@ writing_nems_configure() { echo "SUB ${FUNCNAME[0]}: parsing_nems_configure begins" -if [ -e $SCRIPTDIR/nems.configure ]; then - rm -f $SCRIPTDIR/nems.configure +if [[ -e ${SCRIPTDIR}/nems.configure ]]; then + rm -f "${SCRIPTDIR}"/nems.configure fi # Setup nems.configure @@ -30,37 +30,37 @@ else fi local res_int=${restart_interval:-3024000} # Interval in seconds to write restarts -rm -f $DATA/nems.configure +rm -f "${DATA}"/nems.configure local esmf_logkind=${esmf_logkind:-"ESMF_LOGKIND_MULTI"} #options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE # Copy the selected template into run directory -infile="$SCRIPTDIR/nems.configure.$confignamevarfornems.IN" -if [ -s $infile ]; then - cp $infile tmp1 +infile="${SCRIPTDIR}/nems.configure.${confignamevarfornems}.IN" +if [[ -s ${infile} ]]; then + cp "${infile}" tmp1 else - echo "FATAL ERROR: nem.configure template '$infile' does not exist!" + echo "FATAL ERROR: nem.configure template '${infile}' does not exist!" exit 1 fi -local atm_petlist_bounds="0 $(( $ATMPETS-1 ))" -local med_petlist_bounds="0 $(( $MEDPETS-1 ))" +local atm_petlist_bounds="0 $(( ${ATMPETS}-1 ))" +local med_petlist_bounds="0 $(( ${MEDPETS}-1 ))" sed -i -e "s;@\[med_model\];cmeps;g" tmp1 sed -i -e "s;@\[atm_model\];fv3;g" tmp1 -sed -i -e "s;@\[med_petlist_bounds\];$med_petlist_bounds;g" tmp1 -sed -i -e "s;@\[atm_petlist_bounds\];$atm_petlist_bounds;g" tmp1 -sed -i -e "s;@\[esmf_logkind\];$esmf_logkind;g" tmp1 +sed -i -e "s;@\[med_petlist_bounds\];${med_petlist_bounds};g" tmp1 +sed -i -e "s;@\[atm_petlist_bounds\];${atm_petlist_bounds};g" tmp1 +sed -i -e "s;@\[esmf_logkind\];${esmf_logkind};g" tmp1 -if [ $cpl = ".true." ]; then - sed -i -e "s;@\[coupling_interval_slow_sec\];$CPL_SLOW;g" tmp1 +if [[ ${cpl} = ".true." ]]; then + sed -i -e "s;@\[coupling_interval_slow_sec\];${CPL_SLOW};g" tmp1 fi -if [ $cplflx = .true. ]; then - if [ $res_int -gt 0 ]; then - local restart_interval_nems=$res_int +if [[ ${cplflx} = .true. ]]; then + if [[ ${res_int} -gt 0 ]]; then + local restart_interval_nems=${res_int} else - local restart_interval_nems=$FHMAX + local restart_interval_nems=${FHMAX} fi case "${OCNRES}" in @@ -71,59 +71,59 @@ if [ $cplflx = .true. ]; then local use_coldstart=${use_coldstart:-".false."} local use_mommesh=${USE_MOMMESH:-"true"} - local restile=$(echo ${CASE} |cut -c2-) - local ocn_petlist_bounds="$ATMPETS $(( $ATMPETS+$OCNPETS-1 ))" + local restile=$(echo "${CASE}" |cut -c2-) + local ocn_petlist_bounds="${ATMPETS} $(( ${ATMPETS}+${OCNPETS}-1 ))" sed -i -e "s;@\[ocn_model\];mom6;g" tmp1 - sed -i -e "s;@\[ocn_petlist_bounds\];$ocn_petlist_bounds;g" tmp1 - sed -i -e "s;@\[DumpFields\];$DumpFields;g" tmp1 - sed -i -e "s;@\[cap_dbug_flag\];$cap_dbug_flag;g" tmp1 - sed -i -e "s;@\[use_coldstart\];$use_coldstart;g" tmp1 - sed -i -e "s;@\[RUNTYPE\];$cmeps_run_type;g" tmp1 - sed -i -e "s;@\[CPLMODE\];$cplmode;g" tmp1 - sed -i -e "s;@\[restart_interval\];$res_int;g" tmp1 - sed -i -e "s;@\[coupling_interval_fast_sec\];$CPL_FAST;g" tmp1 - sed -i -e "s;@\[RESTART_N\];$restart_interval_nems;g" tmp1 - sed -i -e "s;@\[use_mommesh\];$use_mommesh;g" tmp1 - sed -i -e "s;@\[eps_imesh\];$eps_imesh;g" tmp1 - sed -i -e "s;@\[ATMTILESIZE\];$restile;g" tmp1 + sed -i -e "s;@\[ocn_petlist_bounds\];${ocn_petlist_bounds};g" tmp1 + sed -i -e "s;@\[DumpFields\];${DumpFields};g" tmp1 + sed -i -e "s;@\[cap_dbug_flag\];${cap_dbug_flag};g" tmp1 + sed -i -e "s;@\[use_coldstart\];${use_coldstart};g" tmp1 + sed -i -e "s;@\[RUNTYPE\];${cmeps_run_type};g" tmp1 + sed -i -e "s;@\[CPLMODE\];${cplmode};g" tmp1 + sed -i -e "s;@\[restart_interval\];${res_int};g" tmp1 + sed -i -e "s;@\[coupling_interval_fast_sec\];${CPL_FAST};g" tmp1 + sed -i -e "s;@\[RESTART_N\];${restart_interval_nems};g" tmp1 + sed -i -e "s;@\[use_mommesh\];${use_mommesh};g" tmp1 + sed -i -e "s;@\[eps_imesh\];${eps_imesh};g" tmp1 + sed -i -e "s;@\[ATMTILESIZE\];${restile};g" tmp1 fi -if [ $cplwav = .true. ]; then +if [[ ${cplwav} = .true. ]]; then - local wav_petlist_bounds="$(( $ATMPETS+$OCNPETS+$ICEPETS )) $(( $ATMPETS+$OCNPETS+$ICEPETS+$WAVPETS-1 ))" + local wav_petlist_bounds="$(( ${ATMPETS}+${OCNPETS}+${ICEPETS} )) $(( ${ATMPETS}+${OCNPETS}+${ICEPETS}+${WAVPETS}-1 ))" sed -i -e "s;@\[wav_model\];ww3;g" tmp1 - sed -i -e "s;@\[wav_petlist_bounds\];$wav_petlist_bounds;g" tmp1 - sed -i -e "s;@\[MESH_WAV\];$MESH_WAV;g" tmp1 - sed -i -e "s;@\[MULTIGRID\];$waveMULTIGRID;g" tmp1 + sed -i -e "s;@\[wav_petlist_bounds\];${wav_petlist_bounds};g" tmp1 + sed -i -e "s;@\[MESH_WAV\];${MESH_WAV};g" tmp1 + sed -i -e "s;@\[MULTIGRID\];${waveMULTIGRID};g" tmp1 fi -if [ $cplice = .true. ]; then +if [[ ${cplice} = .true. ]]; then local mesh_ocn_ice=${MESH_OCN_ICE:-"mesh.mx${ICERES}.nc"} - local ice_petlist_bounds="$(( $ATMPETS+$OCNPETS )) $(( $ATMPETS+$OCNPETS+$ICEPETS-1 ))" + local ice_petlist_bounds="$(( ${ATMPETS}+${OCNPETS} )) $(( ${ATMPETS}+${OCNPETS}+${ICEPETS}-1 ))" sed -i -e "s;@\[ice_model\];cice6;g" tmp1 - sed -i -e "s;@\[ice_petlist_bounds\];$ice_petlist_bounds;g" tmp1 - sed -i -e "s;@\[MESH_OCN_ICE\];$mesh_ocn_ice;g" tmp1 - sed -i -e "s;@\[FHMAX\];$FHMAX_GFS;g" tmp1 + sed -i -e "s;@\[ice_petlist_bounds\];${ice_petlist_bounds};g" tmp1 + sed -i -e "s;@\[MESH_OCN_ICE\];${mesh_ocn_ice};g" tmp1 + sed -i -e "s;@\[FHMAX\];${FHMAX_GFS};g" tmp1 fi -if [ $cplchm = .true. ]; then +if [[ ${cplchm} = .true. ]]; then - local chm_petlist_bounds="0 $(( $CHMPETS-1 ))" + local chm_petlist_bounds="0 $(( ${CHMPETS}-1 ))" sed -i -e "s;@\[chm_model\];gocart;g" tmp1 - sed -i -e "s;@\[chm_petlist_bounds\];$chm_petlist_bounds;g" tmp1 - sed -i -e "s;@\[coupling_interval_fast_sec\];$CPL_FAST;g" tmp1 + sed -i -e "s;@\[chm_petlist_bounds\];${chm_petlist_bounds};g" tmp1 + sed -i -e "s;@\[coupling_interval_fast_sec\];${CPL_FAST};g" tmp1 fi mv tmp1 nems.configure echo "$(cat nems.configure)" -if [ $cplflx = .true. ]; then +if [[ ${cplflx} = .true. ]]; then #Create other CMEPS mediator related files cat > pio_in << EOF @@ -180,8 +180,8 @@ echo "$(cat med_modelio.nml)" fi -cp $HOMEgfs/sorc/ufs_model.fd/tests/parm/fd_nems.yaml fd_nems.yaml +cp "${HOMEgfs}"/sorc/ufs_model.fd/tests/parm/fd_nems.yaml fd_nems.yaml -echo "SUB ${FUNCNAME[0]}: Nems configured for $confignamevarfornems" +echo "SUB ${FUNCNAME[0]}: Nems configured for ${confignamevarfornems}" } diff --git a/ush/parsing_model_configure_FV3.sh b/ush/parsing_model_configure_FV3.sh index cc84f2599e..a8997176f4 100755 --- a/ush/parsing_model_configure_FV3.sh +++ b/ush/parsing_model_configure_FV3.sh @@ -12,7 +12,7 @@ FV3_model_configure(){ -local restile=$(echo ${CASE} |cut -c2-) +local restile=$(echo "${CASE}" |cut -c2-) local ichunk2d=$((4*restile)) local jchunk2d=$((2*restile)) local ichunk3d=$((4*restile)) @@ -27,15 +27,15 @@ start_day: ${tPDY:6:2} start_hour: ${tcyc} start_minute: 0 start_second: 0 -nhours_fcst: $FHMAX +nhours_fcst: ${FHMAX} fhrot: ${IAU_FHROT:-0} -dt_atmos: $DELTIM +dt_atmos: ${DELTIM} calendar: ${calendar:-'julian'} -restart_interval: $restart_interval +restart_interval: ${restart_interval} output_1st_tstep_rst: .false. -quilting: $QUILTING +quilting: ${QUILTING} write_groups: ${WRITE_GROUP:-1} write_tasks_per_group: ${WRTTASK_PER_GROUP:-24} itasks: 1 @@ -44,8 +44,8 @@ write_dopost: ${WRITE_DOPOST:-".false."} write_nsflip: ${WRITE_NSFLIP:-".false."} num_files: ${NUM_FILES:-2} filename_base: 'atm' 'sfc' -output_grid: $OUTPUT_GRID -output_file: '$OUTPUT_FILETYPE_ATM' '$OUTPUT_FILETYPE_SFC' +output_grid: ${OUTPUT_GRID} +output_file: '${OUTPUT_FILETYPE_ATM}' '${OUTPUT_FILETYPE_SFC}' ichunk2d: ${ichunk2d:-0} jchunk2d: ${jchunk2d:-0} ichunk3d: ${ichunk3d:-0} @@ -53,9 +53,9 @@ jchunk3d: ${jchunk3d:-0} kchunk3d: ${kchunk3d:-0} ideflate: ${ideflate:-1} nbits: ${nbits:-14} -imo: $LONB_IMO -jmo: $LATB_JMO -output_fh: $OUTPUT_FH +imo: ${LONB_IMO} +jmo: ${LATB_JMO} +output_fh: ${OUTPUT_FH} iau_offset: ${IAU_OFFSET:-0} EOF diff --git a/ush/parsing_namelists_CICE.sh b/ush/parsing_namelists_CICE.sh index 10abceda1d..cf5b0fbf83 100755 --- a/ush/parsing_namelists_CICE.sh +++ b/ush/parsing_namelists_CICE.sh @@ -43,12 +43,12 @@ cat > ice_in < ice_in < ice_in < ice_in < ice_in <> input.nml <> input.nml <> input.nml < $DATA/INPUT/MOM_input -rm $DATA/INPUT/MOM_input_template_$OCNRES +${NCP} -pf "${HOMEgfs}"/parm/mom6/MOM_input_template_"${OCNRES}" "${DATA}"/INPUT/ +sed -e "s/@\[DT_THERM_MOM6\]/${DT_THERM_MOM6}/g" \ + -e "s/@\[DT_DYNAM_MOM6\]/${DT_DYNAM_MOM6}/g" \ + -e "s/@\[MOM6_RIVER_RUNOFF\]/${MOM6_RIVER_RUNOFF}/g" \ + -e "s/@\[MOM6_THERMO_SPAN\]/${MOM6_THERMO_SPAN}/g" \ + -e "s/@\[MOM6_USE_LI2016\]/${MOM6_USE_LI2016}/g" \ + -e "s/@\[MOM6_USE_WAVES\]/${MOM6_USE_WAVES}/g" \ + -e "s/@\[MOM6_ALLOW_LANDMASK_CHANGES\]/${MOM6_ALLOW_LANDMASK_CHANGES}/g" \ + -e "s/@\[NX_GLB\]/${NX_GLB}/g" \ + -e "s/@\[NY_GLB\]/${NY_GLB}/g" \ + -e "s/@\[CHLCLIM\]/${CHLCLIM}/g" \ + -e "s/@\[DO_OCN_SPPT\]/${OCN_SPPT}/g" \ + -e "s/@\[PERT_EPBL\]/${PERT_EPBL}/g" \ + -e "s/@\[ODA_INCUPD_NHOURS\]/${ODA_INCUPD_NHOURS}/g" \ + -e "s/@\[ODA_INCUPD\]/${ODA_INCUPD}/g" "${DATA}"/INPUT/MOM_input_template_"${OCNRES}" > "${DATA}"/INPUT/MOM_input +rm "${DATA}"/INPUT/MOM_input_template_"${OCNRES}" #data table for runoff: -DATA_TABLE=${DATA_TABLE:-$PARM_FV3DIAG/data_table} -$NCP $DATA_TABLE $DATA/data_table_template -sed -e "s/@\[FRUNOFF\]/$FRUNOFF/g" $DATA/data_table_template > $DATA/data_table -rm $DATA/data_table_template +DATA_TABLE=${DATA_TABLE:-${PARM_FV3DIAG}/data_table} +${NCP} "${DATA_TABLE}" "${DATA}"/data_table_template +sed -e "s/@\[FRUNOFF\]/${FRUNOFF}/g" "${DATA}"/data_table_template > "${DATA}"/data_table +rm "${DATA}"/data_table_template } From 6bce5d3a46f33380748b2fdc4c7c6fb9a7a3069a Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Sun, 26 Feb 2023 11:34:01 -0500 Subject: [PATCH 06/14] shell checker script should be free of errors, no? --- .github/scripts/apply_shellcheck.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/scripts/apply_shellcheck.sh b/.github/scripts/apply_shellcheck.sh index 1e0b59e03f..d098249bed 100755 --- a/.github/scripts/apply_shellcheck.sh +++ b/.github/scripts/apply_shellcheck.sh @@ -10,7 +10,10 @@ apply_shellcheck() { set +e shellcheck -i "${code}" "${filename}" -f diff > "${patch}" set -e - patch_size=$(wc -l ${patch} | awk '{print $1}') + # shellcheck disable=SC2312 + # Ignore the error that masks the return value + patch_size=$(wc -l "${patch}" | awk '{print $1}') + # shellcheck disable= if [[ "${patch_size}" -gt 0 ]]; then echo "Apply patch for ${code} on ${filename}" git apply "${patch}" From ec44bef918bf6b235ce150c4b9cee3dfdd1d6817 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 27 Feb 2023 10:21:27 -0500 Subject: [PATCH 07/14] reviewer comments on canonical quoting --- parm/config/config.resources | 2 +- ush/nems_configure.sh | 47 +++++++++++++++++++++-------------- ush/parsing_namelists_MOM6.sh | 12 ++++----- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index c020d269b3..e821a148cb 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -437,7 +437,7 @@ elif [[ ${step} = "fcst" || ${step} = "efcs" ]]; then fi # PETS for the atmosphere dycore - (( FV3PETS = layout_x * layout_y * 6 )) + (( FV3PETS = ntasks_fv3 )) # PETS for quilting if [[ ${QUILTING:-} == ".true." ]]; then diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh index 5482b85403..320e874939 100755 --- a/ush/nems_configure.sh +++ b/ush/nems_configure.sh @@ -30,7 +30,7 @@ else fi local res_int=${restart_interval:-3024000} # Interval in seconds to write restarts -rm -f "${DATA}"/nems.configure +rm -f "${DATA}/nems.configure" local esmf_logkind=${esmf_logkind:-"ESMF_LOGKIND_MULTI"} #options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE @@ -56,13 +56,14 @@ if [[ ${cpl} = ".true." ]]; then sed -i -e "s;@\[coupling_interval_slow_sec\];${CPL_SLOW};g" tmp1 fi -if [[ ${cplflx} = .true. ]]; then - if [[ ${res_int} -gt 0 ]]; then +if [[ "${cplflx}" = ".true." ]]; then + if [[ ${res_int} -gt 0 ]]; then local restart_interval_nems=${res_int} else local restart_interval_nems=${FHMAX} fi + # TODO: Should this be raised up to config.ufs? case "${OCNRES}" in "500") local eps_imesh="4.0e-1";; "100") local eps_imesh="2.5e-1";; @@ -72,7 +73,10 @@ if [[ ${cplflx} = .true. ]]; then local use_coldstart=${use_coldstart:-".false."} local use_mommesh=${USE_MOMMESH:-"true"} local restile=$(echo "${CASE}" |cut -c2-) - local ocn_petlist_bounds="${ATMPETS} $(( ${ATMPETS}+${OCNPETS}-1 ))" + + local start="${ATMPETS}" + local end="$(( ${start}+${OCNPETS}-1 ))" + local ocn_petlist_bounds="${start} ${end}" sed -i -e "s;@\[ocn_model\];mom6;g" tmp1 sed -i -e "s;@\[ocn_petlist_bounds\];${ocn_petlist_bounds};g" tmp1 @@ -89,20 +93,13 @@ if [[ ${cplflx} = .true. ]]; then sed -i -e "s;@\[ATMTILESIZE\];${restile};g" tmp1 fi -if [[ ${cplwav} = .true. ]]; then - - local wav_petlist_bounds="$(( ${ATMPETS}+${OCNPETS}+${ICEPETS} )) $(( ${ATMPETS}+${OCNPETS}+${ICEPETS}+${WAVPETS}-1 ))" - - sed -i -e "s;@\[wav_model\];ww3;g" tmp1 - sed -i -e "s;@\[wav_petlist_bounds\];${wav_petlist_bounds};g" tmp1 - sed -i -e "s;@\[MESH_WAV\];${MESH_WAV};g" tmp1 - sed -i -e "s;@\[MULTIGRID\];${waveMULTIGRID};g" tmp1 -fi - -if [[ ${cplice} = .true. ]]; then +if [[ "${cplice}" = ".true." ]]; then local mesh_ocn_ice=${MESH_OCN_ICE:-"mesh.mx${ICERES}.nc"} - local ice_petlist_bounds="$(( ${ATMPETS}+${OCNPETS} )) $(( ${ATMPETS}+${OCNPETS}+${ICEPETS}-1 ))" + + local start="$(( ${ATMPETS}+${OCNPETS} ))" + local end="$(( ${start}+${ICEPETS}-1 ))" + local ice_petlist_bounds="${start} ${end}" sed -i -e "s;@\[ice_model\];cice6;g" tmp1 sed -i -e "s;@\[ice_petlist_bounds\];${ice_petlist_bounds};g" tmp1 @@ -110,7 +107,19 @@ if [[ ${cplice} = .true. ]]; then sed -i -e "s;@\[FHMAX\];${FHMAX_GFS};g" tmp1 fi -if [[ ${cplchm} = .true. ]]; then +if [[ "${cplwav}" = ".true." ]]; then + + local start="$(( ${ATMPETS}+${OCNPETS:-0}+${ICEPETS:-0} ))" + local end="$(( ${start}+${WAVPETS}-1 ))" + local wav_petlist_bounds="${start} ${end}" + + sed -i -e "s;@\[wav_model\];ww3;g" tmp1 + sed -i -e "s;@\[wav_petlist_bounds\];${wav_petlist_bounds};g" tmp1 + sed -i -e "s;@\[MESH_WAV\];${MESH_WAV};g" tmp1 + sed -i -e "s;@\[MULTIGRID\];${waveMULTIGRID};g" tmp1 +fi + +if [[ "${cplchm}" = ".true." ]]; then local chm_petlist_bounds="0 $(( ${CHMPETS}-1 ))" @@ -123,7 +132,7 @@ mv tmp1 nems.configure echo "$(cat nems.configure)" -if [[ ${cplflx} = .true. ]]; then +if [[ "${cplflx}" = ".true." ]]; then #Create other CMEPS mediator related files cat > pio_in << EOF @@ -180,7 +189,7 @@ echo "$(cat med_modelio.nml)" fi -cp "${HOMEgfs}"/sorc/ufs_model.fd/tests/parm/fd_nems.yaml fd_nems.yaml +${NCP} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/fd_nems.yaml" fd_nems.yaml echo "SUB ${FUNCNAME[0]}: Nems configured for ${confignamevarfornems}" diff --git a/ush/parsing_namelists_MOM6.sh b/ush/parsing_namelists_MOM6.sh index 436a2e983e..add7090fe7 100755 --- a/ush/parsing_namelists_MOM6.sh +++ b/ush/parsing_namelists_MOM6.sh @@ -62,7 +62,7 @@ echo "$(cat input.nml)" #Copy MOM_input and edit: -${NCP} -pf "${HOMEgfs}"/parm/mom6/MOM_input_template_"${OCNRES}" "${DATA}"/INPUT/ +${NCP} -pf "${HOMEgfs}/parm/mom6/MOM_input_template_${OCNRES}" "${DATA}/INPUT/" sed -e "s/@\[DT_THERM_MOM6\]/${DT_THERM_MOM6}/g" \ -e "s/@\[DT_DYNAM_MOM6\]/${DT_DYNAM_MOM6}/g" \ -e "s/@\[MOM6_RIVER_RUNOFF\]/${MOM6_RIVER_RUNOFF}/g" \ @@ -76,13 +76,13 @@ sed -e "s/@\[DT_THERM_MOM6\]/${DT_THERM_MOM6}/g" \ -e "s/@\[DO_OCN_SPPT\]/${OCN_SPPT}/g" \ -e "s/@\[PERT_EPBL\]/${PERT_EPBL}/g" \ -e "s/@\[ODA_INCUPD_NHOURS\]/${ODA_INCUPD_NHOURS}/g" \ - -e "s/@\[ODA_INCUPD\]/${ODA_INCUPD}/g" "${DATA}"/INPUT/MOM_input_template_"${OCNRES}" > "${DATA}"/INPUT/MOM_input -rm "${DATA}"/INPUT/MOM_input_template_"${OCNRES}" + -e "s/@\[ODA_INCUPD\]/${ODA_INCUPD}/g" "${DATA}/INPUT/MOM_input_template_${OCNRES}" > "${DATA}/INPUT/MOM_input" +rm "${DATA}/INPUT/MOM_input_template_${OCNRES}" #data table for runoff: DATA_TABLE=${DATA_TABLE:-${PARM_FV3DIAG}/data_table} -${NCP} "${DATA_TABLE}" "${DATA}"/data_table_template -sed -e "s/@\[FRUNOFF\]/${FRUNOFF}/g" "${DATA}"/data_table_template > "${DATA}"/data_table -rm "${DATA}"/data_table_template +${NCP} "${DATA_TABLE}" "${DATA}/data_table_template" +sed -e "s/@\[FRUNOFF\]/${FRUNOFF}/g" "${DATA}/data_table_template" > "${DATA}/data_table" +rm "${DATA}/data_table_template" } From 5bcd7f9932047f4f440710832e52f2764b30da5c Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 27 Feb 2023 10:27:23 -0500 Subject: [PATCH 08/14] remove a helper script for applying shellcheck fixes to common errors --- .github/scripts/apply_shellcheck.sh | 30 ----------------------------- 1 file changed, 30 deletions(-) delete mode 100755 .github/scripts/apply_shellcheck.sh diff --git a/.github/scripts/apply_shellcheck.sh b/.github/scripts/apply_shellcheck.sh deleted file mode 100755 index d098249bed..0000000000 --- a/.github/scripts/apply_shellcheck.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -set -eu - -apply_shellcheck() { - local filename=${1:?} - local code=${2:-""} - patch="/tmp/patch.${RANDOM}" # Do not use PWD, as this will create tons of patch files - [[ -f "${patch}" ]] && rm -f "${patch}" - set +e - shellcheck -i "${code}" "${filename}" -f diff > "${patch}" - set -e - # shellcheck disable=SC2312 - # Ignore the error that masks the return value - patch_size=$(wc -l "${patch}" | awk '{print $1}') - # shellcheck disable= - if [[ "${patch_size}" -gt 0 ]]; then - echo "Apply patch for ${code} on ${filename}" - git apply "${patch}" - else - echo "No patch to apply for ${code} on ${filename}" - fi - rm -f "${patch}" -} - -file=${1:?} # File to fix shellcheck errors -apply_shellcheck "${file}" SC2292 # Prefer [[ ]] over [ ] for tests in Bash/Ksh -apply_shellcheck "${file}" SC2250 # Prefer putting braces around variable e.g. ${variable} -apply_shellcheck "${file}" SC2248 # Prefer double quoting -apply_shellcheck "${file}" SC2086 # Double quote to prevent globbing and word splitting From 4d26b46339a2bf9f139ae3f12d07c0d878a7be8a Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 27 Feb 2023 10:50:55 -0500 Subject: [PATCH 09/14] more quotations --- ush/nems_configure.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh index 320e874939..a3aefc9284 100755 --- a/ush/nems_configure.sh +++ b/ush/nems_configure.sh @@ -12,8 +12,8 @@ writing_nems_configure() { echo "SUB ${FUNCNAME[0]}: parsing_nems_configure begins" -if [[ -e ${SCRIPTDIR}/nems.configure ]]; then - rm -f "${SCRIPTDIR}"/nems.configure +if [[ -e "${SCRIPTDIR}/nems.configure" ]]; then + rm -f "${SCRIPTDIR}/nems.configure" fi # Setup nems.configure From e35831f0e2e4fa2290817db0b8b63184e81e0223 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 28 Feb 2023 09:26:01 -0500 Subject: [PATCH 10/14] build only GFS required ufs_utils.fd. Forgotten from PR updating hash of ufs_utils --- sorc/build_ufs_utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/build_ufs_utils.sh b/sorc/build_ufs_utils.sh index 073e3dbb47..5e2edf0737 100755 --- a/sorc/build_ufs_utils.sh +++ b/sorc/build_ufs_utils.sh @@ -4,7 +4,7 @@ set -eux script_dir=$(dirname "${BASH_SOURCE[0]}") cd "${script_dir}/ufs_utils.fd" || exit 1 -./build_all.sh +CMAKE_OPTS="-DGFS=ON" ./build_all.sh exit From 91eb2938c75ae97679851f33677a0a7c1fd388a4 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 2 Mar 2023 16:22:16 -0500 Subject: [PATCH 11/14] update comments about MOM6 threading. Ensure MOM6 is always on when enabling CICE6 --- parm/config/config.ufs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/parm/config/config.ufs b/parm/config/config.ufs index 0375c1f541..12fe9141e9 100644 --- a/parm/config/config.ufs +++ b/parm/config/config.ufs @@ -234,7 +234,7 @@ esac # MOM6 specific settings if [[ "${skip_mom6}" == "false" ]]; then - export nth_mom6=1 # MOM6 is not thread-safe (allegedly) + export nth_mom6=1 case "${mom6_res}" in "500") ntasks_mom6=8 @@ -301,7 +301,12 @@ fi # CICE6 specific settings if [[ "${skip_cice6}" == "false" ]]; then - export nth_cice6=${mom6_res} # CICE6 needs to run on same threads as MOM6 (allegedly) + # Ensure we sourced the MOM6 section + if [[ "${skip_mom6}" == "true" ]]; then + echo "FATAL ERROR: CICE6 cannot be configured without MOM6, ABORT!" + exit 1 + fi + export nth_cice6=${mom6_res} # CICE6 needs to run on same threads as MOM6 case "${cice6_res}" in "500") ntasks_cice6=4 @@ -309,7 +314,7 @@ if [[ "${skip_cice6}" == "false" ]]; then ;; "100") ntasks_cice6=10 - cice6_processor_shape="slenderX1" + cice6_processor_shape="slenderX2" ;; "050") ntasks_cice6=30 From a1cfea43abc8a90d444428d36d429f48170b92a4 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 2 Mar 2023 16:36:51 -0500 Subject: [PATCH 12/14] update layouts for C192 and C384 based on suggestions from @junwang-noaa --- parm/config/config.ufs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parm/config/config.ufs b/parm/config/config.ufs index 12fe9141e9..9658e3ca86 100644 --- a/parm/config/config.ufs +++ b/parm/config/config.ufs @@ -128,10 +128,10 @@ case "${fv3_res}" in ;; "C192") export DELTIM=450 - export layout_x=4 - export layout_y=6 - export layout_x_gfs=4 - export layout_y_gfs=6 + export layout_x=6 + export layout_y=4 + export layout_x_gfs=6 + export layout_y_gfs=4 export nth_fv3=1 export nth_fv3_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling @@ -143,8 +143,8 @@ case "${fv3_res}" in ;; "C384") export DELTIM=${DELTIM:-200} - export layout_x=6 - export layout_y=8 + export layout_x=8 + export layout_y=6 export layout_x_gfs=${layout_x_gfs:-8} export layout_y_gfs=${layout_y_gfs:-12} export nth_fv3=1 From 3a30d9605ccb6bc8126364d697a356b02a2f3402 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 2 Mar 2023 16:42:25 -0500 Subject: [PATCH 13/14] Revert "update layouts for C192 and C384 based on suggestions from @junwang-noaa" This reverts commit a1cfea43abc8a90d444428d36d429f48170b92a4. --- parm/config/config.ufs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parm/config/config.ufs b/parm/config/config.ufs index 9658e3ca86..12fe9141e9 100644 --- a/parm/config/config.ufs +++ b/parm/config/config.ufs @@ -128,10 +128,10 @@ case "${fv3_res}" in ;; "C192") export DELTIM=450 - export layout_x=6 - export layout_y=4 - export layout_x_gfs=6 - export layout_y_gfs=4 + export layout_x=4 + export layout_y=6 + export layout_x_gfs=4 + export layout_y_gfs=6 export nth_fv3=1 export nth_fv3_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling @@ -143,8 +143,8 @@ case "${fv3_res}" in ;; "C384") export DELTIM=${DELTIM:-200} - export layout_x=8 - export layout_y=6 + export layout_x=6 + export layout_y=8 export layout_x_gfs=${layout_x_gfs:-8} export layout_y_gfs=${layout_y_gfs:-12} export nth_fv3=1 From 1e6488bf46c123eb929ceb31cf0628b20f4d2896 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 3 Mar 2023 09:13:33 -0500 Subject: [PATCH 14/14] update layout_gfs for C768 and write task for C384. Remove limit on write tasks based on npe_node_max --- parm/config/config.ufs | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/parm/config/config.ufs b/parm/config/config.ufs index 12fe9141e9..4dd8cb8bc7 100644 --- a/parm/config/config.ufs +++ b/parm/config/config.ufs @@ -142,26 +142,26 @@ case "${fv3_res}" in export WRTIOBUF="8M" ;; "C384") - export DELTIM=${DELTIM:-200} + export DELTIM=200 export layout_x=6 export layout_y=8 - export layout_x_gfs=${layout_x_gfs:-8} - export layout_y_gfs=${layout_y_gfs:-12} + export layout_x_gfs=8 + export layout_y_gfs=12 export nth_fv3=1 - export nth_fv3_gfs=${nth_fv3_gfs:-2} + export nth_fv3_gfs=2 export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling - export WRITE_GROUP=2 - export WRTTASK_PER_GROUP=64 - export WRITE_GROUP_GFS=${WRITE_GROUP_GFS:-2} - export WRTTASK_PER_GROUP_GFS=${WRTTASK_PER_GROUP_GFS:-64} - export WRTIOBUF=${WRTIOBUF:-"16M"} + export WRITE_GROUP=1 + export WRTTASK_PER_GROUP=48 + export WRITE_GROUP_GFS=2 + export WRTTASK_PER_GROUP_GFS=64 + export WRTIOBUF="16M" ;; "C768") export DELTIM=150 export layout_x=8 export layout_y=12 - export layout_x_gfs=16 - export layout_y_gfs=12 + export layout_x_gfs=12 + export layout_y_gfs=16 export nth_fv3=4 export nth_fv3_gfs=4 export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling @@ -181,9 +181,9 @@ case "${fv3_res}" in export nth_fv3_gfs=4 export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=4 - export WRTTASK_PER_GROUP=$(echo "2*${npe_node_max}" |bc) # TODO: this is pointless because below this is getting reset + export WRTTASK_PER_GROUP=64 # TODO: refine these numbers when a case is available export WRITE_GROUP_GFS=4 - export WRTTASK_PER_GROUP_GFS=$(echo "2*${npe_node_max}" |bc) # TODO: this is pointless because below this is getting reset + export WRTTASK_PER_GROUP_GFS=64 # TODO: refine these numbers when a case is available export WRTIOBUF="48M" ;; "C3072") @@ -196,9 +196,9 @@ case "${fv3_res}" in export nth_fv3_gfs=4 export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=4 - export WRTTASK_PER_GROUP=$(echo "3*${npe_node_max}" |bc) # TODO: this is pointless because below this is getting reset + export WRTTASK_PER_GROUP=64 # TODO: refine these numbers when a case is available export WRITE_GROUP_GFS=4 - export WRTTASK_PER_GROUP_GFS=$(echo "3*${npe_node_max}" |bc) # TODO: this is pointless because below this is getting reset + export WRTTASK_PER_GROUP_GFS=64 # TODO: refine these numbers when a case is available export WRTIOBUF="64M" ;; *) @@ -207,9 +207,6 @@ case "${fv3_res}" in ;; esac -if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi -if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi - (( ntasks_fv3 = layout_x * layout_y * 6 )) (( ntasks_fv3_gfs = layout_x_gfs * layout_y_gfs * 6 )) export ntasks_fv3