From 47afc78568563bc4c55208e567bfb967684eae0a Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 3 Mar 2023 10:43:54 -0500 Subject: [PATCH] Consolidate ocean, ice and wave task info into config.ufs (#1334) config.fv3 is now renamed as config.ufs and contains ocean, ice, and wave task information * config.fv3 is used to record FV3 and write grid component task decomposition based on resolution. * Ocean (MOM6) and Ice (CICE6) decomposition, tasks and timesteps are added to config.ufs. * Updates are made to config.fcst and config.efcs to source config.ufs based on the configuration of the (coupled) model. * Several configurations were made deep in parsing_namelists_MOM.sh, which have now been elevated to config.ocn * Some improvements are made to limit the export of variables from functions in nems_configure.sh. * This will help with the ESMF threading work as it breaks down the work into multiple steps. --- parm/config/config.base.emc.dyn | 3 + parm/config/config.defaults.s2sw | 2 +- parm/config/config.efcs | 33 ++- parm/config/config.fcst | 24 +- parm/config/config.fv3 | 185 --------------- parm/config/config.ice | 18 +- parm/config/config.ocn | 26 ++- parm/config/config.resources | 167 ++++++-------- parm/config/config.ufs | 358 +++++++++++++++++++++++++++++ sorc/build_ufs_utils.sh | 2 +- ush/forecast_postdet.sh | 3 - ush/nems_configure.sh | 175 +++++++------- ush/parsing_model_configure_FV3.sh | 25 +- ush/parsing_namelists_CICE.sh | 37 ++- ush/parsing_namelists_MOM6.sh | 131 +++-------- 15 files changed, 654 insertions(+), 535 deletions(-) delete mode 100644 parm/config/config.fv3 create mode 100644 parm/config/config.ufs diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index a9bf44229f..4538314264 100644 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -170,6 +170,8 @@ export OPS_RES="C768" # Do not change export LEVS=128 export CASE="@CASECTL@" export CASE_ENKF="@CASEENS@" +# TODO: This should not depend on $CASE or $CASE_ENKF +# These are the currently available grid-combinations case "${CASE}" in "C48") export OCNRES=500;; "C96") export OCNRES=100;; @@ -179,6 +181,7 @@ case "${CASE}" in *) export OCNRES=025;; esac export ICERES=${OCNRES} +export waveGRD='gnh_10m aoc_9km gsh_15m' case "${APP}" in ATM) diff --git a/parm/config/config.defaults.s2sw b/parm/config/config.defaults.s2sw index d79822d3bf..a457266d76 100644 --- a/parm/config/config.defaults.s2sw +++ b/parm/config/config.defaults.s2sw @@ -14,7 +14,7 @@ FHOUT_HF_GFS=-1 min_seaice="1.0e-6" use_cice_alb=".true." -# config.fv3 # TODO: This is hard-wired for P8 and needs to be refactored. For now, use case C384 +# config.ufs # TODO: This is hard-wired for P8 and needs to be refactored. For now, use case C384 case "${CASE}" in "C384") DELTIM=300 diff --git a/parm/config/config.efcs b/parm/config/config.efcs index 848ede9a64..a9b410e416 100644 --- a/parm/config/config.efcs +++ b/parm/config/config.efcs @@ -5,19 +5,36 @@ echo "BEGIN: config.efcs" +# TODO: the _ENKF counterparts need to be defined in config.base +export DO_AERO=${DO_AERO_ENKF:-"NO"} +export DO_OCN=${DO_OCN_ENKF:-"NO"} +export DO_ICE=${DO_ICE_ENKF:-"NO"} +export DO_WAVE=${DO_WAVE_ENKF:-"NO"} + +# TODO: Possibly need OCNRES_ENKF, ICERES_ENKF, WAVRES_ENKF too +if [[ ${DO_OCN} == "YES" ]]; then + case "$CASE_ENKF" in + "C48") export OCNRES=500;; + "C96") export OCNRES=100;; + "C192") export OCNRES=050;; + "C384") export OCNRES=025;; + "C768") export OCNRES=025;; + *) export OCNRES=025;; + esac +fi +[[ ${DO_ICE} == "YES" ]] && export ICERES=$OCNRES +[[ ${DO_WAVE} == "YES" ]] && export waveGRD=${waveGRD_ENKF:-$waveGRD} # TODO: will we run waves with a different resolution in the ensemble? + # Source model specific information that is resolution dependent -. $EXPDIR/config.fv3 $CASE_ENKF +string="--fv3 $CASE_ENKF" +[[ ${DO_OCN} == "YES" ]] && string="$string --mom6 $OCNRES" +[[ ${DO_ICE} == "YES" ]] && string="$string --cice6 $ICERES" +[[ ${DO_WAVE} == "YES" ]] && string="$string --ww3 ${waveGRD// /;}" +source $EXPDIR/config.ufs ${string} # Get task specific resources . $EXPDIR/config.resources efcs -export npe_fv3=$npe_efcs - -if [ $QUILTING = ".true." ]; then - export npe_fv3=$(echo " $npe_fv3 + $WRITE_GROUP * $WRTTASK_PER_GROUP" | bc) - export npe_efcs=$npe_fv3 -fi - # Use serial I/O for ensemble (lustre?) export OUTPUT_FILETYPE_ATM="netcdf" export OUTPUT_FILETYPE_SFC="netcdf" diff --git a/parm/config/config.fcst b/parm/config/config.fcst index 4413f1f235..357b68512c 100644 --- a/parm/config/config.fcst +++ b/parm/config/config.fcst @@ -5,17 +5,19 @@ echo "BEGIN: config.fcst" -# set -eu - -# Source model specific information that is resolution dependent -. $EXPDIR/config.fv3 $CASE - # Turn off waves if not used for this CDUMP case $WAVE_CDUMP in both | ${CDUMP/enkf} ) ;; # Don't change *) DO_WAVE="NO" ;; # Turn waves off esac +# Source model specific information that is resolution dependent +string="--fv3 $CASE" +[[ ${DO_OCN} == "YES" ]] && string="$string --mom6 $OCNRES" +[[ ${DO_ICE} == "YES" ]] && string="$string --cice6 $ICERES" +[[ ${DO_WAVE} == "YES" ]] && string="$string --ww3 ${waveGRD// /;}" +source $EXPDIR/config.ufs ${string} + # Source component configs if necessary for component in WAVE OCN ICE AERO; do control="DO_${component}" @@ -39,13 +41,8 @@ export esmf_logkind="ESMF_LOGKIND_MULTI_ON_ERROR" #Options: ESMF_LOGKIND_MULTI_O ####################################################################### # COUPLING COMPONENTS -export OCN_model="mom6" -export ICE_model="cice6" -export WAV_model="ww3" -export CHM_model="gocart" # cpl defaults - export cpl=".false." export cplflx=".false." export cplice=".false." @@ -71,11 +68,6 @@ if [[ "$DO_WAVE" = "YES" ]]; then export cplwav=".true." fi -####################################################################### -# COUPLING COMPONENTS -export use_coldstart=".false." - - ####################################################################### export FORECASTSH="$HOMEgfs/scripts/exglobal_forecast.sh" @@ -253,8 +245,6 @@ export DO_SKEB=${DO_SKEB:-"NO"} export DO_SHUM=${DO_SHUM:-"NO"} export DO_LAND_PERT=${DO_LAND_PERT:-"NO"} export DO_CA=${DO_CA:-"YES"} -export DO_OCN_SPPT=${DO_OCN_SPPT:-"NO"} -export DO_OCN_PERT_EPBL=${DO_OCN_PERT_EPBL:-"NO"} #coupling settings export cplmode="nems_frac" diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 deleted file mode 100644 index a14dcda718..0000000000 --- a/parm/config/config.fv3 +++ /dev/null @@ -1,185 +0,0 @@ -#! /usr/bin/env bash - -########## config.fv3 ########## -# FV3 model resolution specific parameters -# e.g. time-step, processor layout, physics and dynamics parameters -# This config sets default variables for FV3 for a given resolution -# User can over-ride after sourcing this config file - -if [ $# -ne 1 ]; then - - echo "Must specify an input resolution argument to set variables!" - echo "argument can be any one of the following:" - echo "C48 C96 C192 C384 C768 C1152 C3072" - exit 1 - -fi - -case_in=$1 - -echo "BEGIN: config.fv3" - - -if [[ "${machine}" = "WCOSS2" ]]; then - export npe_node_max=128 -elif [[ "${machine}" = "JET" ]]; then - if [[ "${PARTITION_BATCH}" = "xjet" ]]; then - export npe_node_max=24 - elif [[ "${PARTITION_BATCH}" = "vjet" || "${PARTITION_BATCH}" = "sjet" ]]; then - export npe_node_max=16 - elif [[ "${PARTITION_BATCH}" = "kjet" ]]; then - export npe_node_max=40 - fi -elif [[ "${machine}" = "HERA" ]]; then - export npe_node_max=40 -elif [[ "${machine}" = "S4" ]]; then - if [[ "${PARTITION_BATCH}" = "s4" ]]; then - export npe_node_max=32 - elif [[ "${PARTITION_BATCH}" = "ivy" ]]; then - export npe_node_max=20 - fi -elif [[ "${machine}" = "ORION" ]]; then - export npe_node_max=40 -fi - -# (Standard) Model resolution dependent variables -case ${case_in} in - "C48") - export DELTIM=1200 - export layout_x=1 - export layout_y=1 - export layout_x_gfs=1 - export layout_y_gfs=1 - export nth_fv3=1 - export nth_fv3_gfs=1 - export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling - export WRITE_GROUP=1 - export WRTTASK_PER_GROUP=6 - export WRITE_GROUP_GFS=1 - export WRTTASK_PER_GROUP_GFS=6 - export WRTIOBUF="1M" - ;; - "C96") - export DELTIM=450 - export layout_x=6 - export layout_y=4 - export layout_x_gfs=6 - export layout_y_gfs=4 - export nth_fv3=1 - export nth_fv3_gfs=1 - export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling - export WRITE_GROUP=1 - export WRTTASK_PER_GROUP=64 - export WRITE_GROUP_GFS=1 - export WRTTASK_PER_GROUP_GFS=64 - export WRTIOBUF="4M" - export n_split=6 - ;; - "C192") - export DELTIM=450 - export layout_x=4 - export layout_y=6 - export layout_x_gfs=4 - export layout_y_gfs=6 - export nth_fv3=1 - export nth_fv3_gfs=2 - export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling - export WRITE_GROUP=1 - export WRTTASK_PER_GROUP=64 - export WRITE_GROUP_GFS=2 - export WRTTASK_PER_GROUP_GFS=64 - export WRTIOBUF="8M" - ;; - "C384") - export DELTIM=${DELTIM:-200} - export layout_x=6 - export layout_y=8 - export layout_x_gfs=${layout_x_gfs:-8} - export layout_y_gfs=${layout_y_gfs:-12} - export nth_fv3=1 - export nth_fv3_gfs=${nth_fv3_gfs:-2} - export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling - export WRITE_GROUP=2 - export WRTTASK_PER_GROUP=64 - export WRITE_GROUP_GFS=${WRITE_GROUP_GFS:-2} - export WRTTASK_PER_GROUP_GFS=${WRTTASK_PER_GROUP_GFS:-64} - export WRTIOBUF=${WRTIOBUF:-"16M"} - ;; - "C768") - export DELTIM=150 - export layout_x=8 - export layout_y=12 - export layout_x_gfs=16 - export layout_y_gfs=12 - export nth_fv3=4 - export nth_fv3_gfs=4 - export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling - export WRITE_GROUP=2 - export WRTTASK_PER_GROUP=64 - export WRITE_GROUP_GFS=4 - export WRTTASK_PER_GROUP_GFS=64 - export WRTIOBUF="32M" - ;; - "C1152") - export DELTIM=120 - export layout_x=8 - export layout_y=16 - export layout_x_gfs=8 - export layout_y_gfs=16 - export nth_fv3=4 - export nth_fv3_gfs=4 - export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling - export WRITE_GROUP=4 - export WRTTASK_PER_GROUP=$(echo "2*${npe_node_max}" |bc) - export WRITE_GROUP_GFS=4 - export WRTTASK_PER_GROUP_GFS=$(echo "2*${npe_node_max}" |bc) - export WRTIOBUF="48M" - ;; - "C3072") - export DELTIM=90 - export layout_x=16 - export layout_y=32 - export layout_x_gfs=16 - export layout_y_gfs=32 - export nth_fv3=4 - export nth_fv3_gfs=4 - export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling - export WRITE_GROUP=4 - export WRTTASK_PER_GROUP=$(echo "3*${npe_node_max}" |bc) - export WRITE_GROUP_GFS=4 - export WRTTASK_PER_GROUP_GFS=$(echo "3*${npe_node_max}" |bc) - export WRTIOBUF="64M" - ;; - *) - echo "grid ${case_in} not supported, ABORT!" - exit 1 - ;; -esac - -#While in theory this is likely the best way to go, in practice partiularly for C768 runs will fail -#when using npe_node_max. Additional write groups also did not get around errors. Should likely -#be revisted at another time. -#if [[ "${WRTTASK_PER_GROUP}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP=${npe_node_max} ; fi -#if [[ "${WRTTASK_PER_GROUP_GFS}" -gt "${npe_node_max}" ]]; then export WRTTASK_PER_GROUP_GFS=${npe_node_max} ; fi - -# Calculate chunksize based on resolution -export RESTILE=$(echo ${case_in} |cut -c2-) -export ichunk2d=$((4*RESTILE)) -export jchunk2d=$((2*RESTILE)) -export ichunk3d=$((4*RESTILE)) -export jchunk3d=$((2*RESTILE)) -export kchunk3d=1 - -# Determine whether to use parallel NetCDF based on resolution -case ${case_in} in - "C48" | "C96" | "C192" | "C384") - export OUTPUT_FILETYPE_ATM="netcdf" - export OUTPUT_FILETYPE_SFC="netcdf" - ;; - "C768" | "C1152" | "C3072") - export OUTPUT_FILETYPE_ATM="netcdf_parallel" - export OUTPUT_FILETYPE_SFC="netcdf_parallel" - ;; -esac - -echo "END: config.fv3" diff --git a/parm/config/config.ice b/parm/config/config.ice index 3b82a1468f..7bc1f80966 100644 --- a/parm/config/config.ice +++ b/parm/config/config.ice @@ -1,17 +1,5 @@ #! /usr/bin/env bash -case "${ICERES}" in - "025") - export NX_GLB="1440" - export NY_GLB="1080" - ;; - "500") - export NX_GLB="72" - export NY_GLB="35" - export cice_processor_shape="slenderX1" - ;; - *) - echo "FATAL ERROR: Unsupported ICERES = ${ICERES}, ABORT!" - exit 1 - ;; -esac +echo "BEGIN: config.ice" + +echo "END: config.ice" diff --git a/parm/config/config.ocn b/parm/config/config.ocn index d224dd3267..7d14e3dd52 100644 --- a/parm/config/config.ocn +++ b/parm/config/config.ocn @@ -1,7 +1,23 @@ #! /usr/bin/env bash -# OCNRES is currently being set in config.base - case "$CASE" in - "C48") export OCNTIM=3600;; - *) export OCNTIM=1800;; - esac +echo "BEGIN: config.ocn" + +# MOM_input template to use +export MOM_INPUT="MOM_input_template_${OCNRES}" + +export DO_OCN_SPPT="NO" # In MOM_input, this variable is determines OCN_SPPT (OCN_SPPT = True|False) +export DO_OCN_PERT_EPBL="NO" # In MOM_input, this variable determines PERT_EPBL (PERT_EPBL = True|False) + +# Templated variables in MOM_input_template +export MOM6_USE_LI2016="True" # set to False for restart reproducibility +export MOM6_THERMO_SPAN="False" +export MOM6_ALLOW_LANDMASK_CHANGES="False" + +if [[ "${DO_JEDIOCNVAR}" == "YES" ]]; then + export ODA_INCUPD="True" +else + export ODA_INCUPD="False" +fi +export ODA_INCUPD_NHOURS="3.0" # In MOM_input, this is time interval for applying increment + +echo "END: config.ocn" diff --git a/parm/config/config.resources b/parm/config/config.resources index c6418c4423..e821a148cb 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -50,7 +50,7 @@ elif [[ ${machine} = "ORION" ]]; then fi if [ ${step} = "prep" ]; then - export wtime_prep='00:45:00' + export wtime_prep='00:30:00' export npe_prep=4 export npe_node_prep=2 export nth_prep=1 @@ -238,7 +238,7 @@ elif [[ "${step}" = "aeroanlrun" ]]; then layout_y=8 ;; *) - echo "FATAL: Resolution not supported'" + echo "FATAL ERROR: Resolution ${CASE} is not supported, ABORT!" exit 1 esac @@ -414,113 +414,105 @@ elif [ ${step} = "gldas" ]; then export npe_node_gaussian=$(echo "${npe_node_max} / ${nth_gaussian}" | bc) export is_exclusive=True -elif [ ${step} = "fcst" ]; then - - if [ ${CASE} = "C48" ]; then - export wtime_fcst="00:30:00" - else - export wtime_fcst="00:40:00" - fi - if [ ${CASE} = "C768" ]; then - export wtime_fcst_gfs="06:00:00" - elif [ ${CASE} = "C384" ]; then - export wtime_fcst_gfs="06:00:00" - else - export wtime_fcst_gfs="03:00:00" - fi +elif [[ ${step} = "fcst" || ${step} = "efcs" ]]; then export is_exclusive=True + if [[ ${step} == "fcst" ]]; then + _CDUMP_LIST=${CDUMP:-"gdas gfs"} + elif [[ ${step} == "efcs" ]]; then + _CDUMP_LIST=${CDUMP:-"enkfgdas enkfgfs"} + fi + # During workflow creation, we need resources for all CDUMPs and CDUMP is undefined - CDUMP_LIST=${CDUMP:-"gdas gfs enkfgdas enkfgfs"} - for CDUMP in ${CDUMP_LIST}; do - if [[ ${CDUMP} =~ "gfs" ]]; then - export layout_x=${layout_x_gfs} - export layout_y=${layout_y_gfs} - export WRITE_GROUP=${WRITE_GROUP_GFS} - export WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP_GFS} + for _CDUMP in ${_CDUMP_LIST}; do + if [[ ${_CDUMP} =~ "gfs" ]]; then + export layout_x=${layout_x_gfs} + export layout_y=${layout_y_gfs} + export WRITE_GROUP=${WRITE_GROUP_GFS} + export WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP_GFS} + ntasks_fv3=${ntasks_fv3_gfs} + ntasks_quilt=${ntasks_quilt_gfs} + nth_fv3=${nth_fv3_gfs} fi - (( ATMPETS = layout_x * layout_y * 6 )) - - # Mediator only uses the atm model PETS or less - export MEDPETS=${MEDPETS:-ATMPETS} + # PETS for the atmosphere dycore + (( FV3PETS = ntasks_fv3 )) - if [[ ${DO_AERO} == "YES" ]]; then - # Aerosol model only uses the atm model PETS - export CHMPETS=${ATMPETS} - # Aerosol model runs on same PETs as ATM, so don't add to $NTASKS_TOT - - if [[ ${machine} == "HERA" ]]; then - # Need more memory on Hera for aerosols, so increase threads to spread it out - nth_fv3_gfs=${nth_fv3_gfs:-4} - fi + # PETS for quilting + if [[ ${QUILTING:-} == ".true." ]]; then + (( QUILTPETS = ntasks_quilt )) + else + QUILTPETS=0 fi - # If using in-line post, add the write tasks to the ATMPETS - if [[ ${QUILTING} == ".true." ]]; then - (( ATMPETS = ATMPETS + WRITE_GROUP * WRTTASK_PER_GROUP )) - fi + # Total PETS for the atmosphere component + (( ATMPETS = FV3PETS + QUILTPETS)) export ATMPETS + + # Total PETS for the coupled model (starting w/ the atmosphere) NTASKS_TOT=${ATMPETS} - export nth_fcst=${nth_fv3:-2} - export nth_fcst_gfs=${nth_fv3_gfs:-2} + # The mediator PETS can overlap with other components, usually it lands on the atmosphere tasks. + # However, it is suggested limiting mediator PETS to 300, as it may cause the slow performance. + # See https://docs.google.com/document/d/1bKpi-52t5jIfv2tuNHmQkYUe3hkKsiG_DG_s6Mnukog/edit + # TODO: Update reference with moved to RTD + MEDPETS=${MEDPETS:-ATMPETS} + [[ "${MEDPETS}" -gt 300 ]] && MEDPETS=300 + export MEDPETS - export npe_node_fcst=$(echo "${npe_node_max} / ${nth_fcst}" | bc) - export npe_node_fcst_gfs=$(echo "${npe_node_max} / ${nth_fcst_gfs}" | bc) + if [[ ${DO_AERO} == "YES" ]]; then + (( CHMPETS = ntasks_fv3 )) + # GOCART shares the same grid and forecast tasks as FV3, do not add to NTASKS_TOT + export CHMPETS + fi if [[ ${DO_WAVE} == "YES" ]]; then - case ${waveGRD} in - 'gnh_10m aoc_9km gsh_15m') export WAVPETS=140 ;; - 'gwes_30m') export WAVPETS=100 ;; - 'mx050') export WAVPETS=240 ;; - 'mx025') export WAVPETS=80 ;; - *) - echo "FATAL: Number of PEs not defined for wave grid '${waveGRD}'" - echo " Please add an entry to config.resources within fcst for this grid" - exit 3 - esac - (( NTASKS_TOT = NTASKS_TOT + WAVPETS )) + (( WAVPETS = ntasks_ww3 )) + (( NTASKS_TOT = NTASKS_TOT + WAVPETS )) + export WAVPETS fi if [[ ${DO_OCN} == "YES" ]]; then - case ${OCNRES} in - # Except for 025, these are guesses for now - 500) export OCNPETS=8 ;; - 100) export OCNPETS=20 ;; - 050) export OCNPETS=60 ;; - 025) export OCNPETS=220 ;; - *) - echo "FATAL: Number of PEs not defined for ocean resolution ${OCNRES}" - echo " Please add an entry to config.resources within fcst for this resolution" - exit 3 - esac - (( NTASKS_TOT = NTASKS_TOT + OCNPETS )) + (( OCNPETS = ntasks_mom6 )) + (( NTASKS_TOT = NTASKS_TOT + OCNPETS )) + export OCNPETS fi if [[ ${DO_ICE} == "YES" ]]; then - case ${ICERES} in - # Except for 025, these are guesses for now - 500) export ICEPETS=4 ;; - 100) export ICEPETS=10 ;; - 050) export ICEPETS=30 ;; - 025) export ICEPETS=120 ;; - *) - echo "FATAL: Number of PEs not defined for ice resolution ${ICERES}" - echo " Please add an entry to config.resources within fcst for this resolution" - exit 3 - esac - (( NTASKS_TOT = NTASKS_TOT + ICEPETS )) + (( ICEPETS = ntasks_cice6 )) + (( NTASKS_TOT = NTASKS_TOT + ICEPETS )) + export ICEPETS fi - if [[ ${CDUMP} =~ "gfs" ]]; then - export npe_fcst_gfs=${NTASKS_TOT} + if [[ ${_CDUMP} =~ "gfs" ]]; then + declare -x npe_${step}_gfs=${NTASKS_TOT} + threads=${nth_fv3_gfs:-2} + declare -x nth_${step}_gfs=${threads} + declare -x npe_node_${step}_gfs=$(echo "${npe_node_max} / ${threads}" | bc) else - export npe_fcst=${NTASKS_TOT} + declare -x npe_${step}=${NTASKS_TOT} + threads=${nth_fv3:-2} + declare -x nth_${step}=${threads} + declare -x npe_node_${step}=$(echo "${npe_node_max} / ${threads}" | bc) fi done + case "${CASE}" in + "C48" | "C96" | "C192") + declare -x wtime_${step}="00:30:00" + declare -x wtime_${step}_gfs="03:00:00" + ;; + "C384" | "C768" | "C1152") + declare -x wtime_${step}="01:00:00" + declare -x wtime_${step}_gfs="06:00:00" + ;; + *) + echo "FATAL ERROR: Resolution ${CASE} not supported in ${step}" + exit 1 + ;; + esac + elif [ ${step} = "ocnpost" ]; then export wtime_ocnpost="00:30:00" @@ -786,17 +778,6 @@ elif [ ${step} = "esfc" ]; then export npe_node_cycle=$(echo "${npe_node_max} / ${nth_cycle}" | bc) export memory_esfc="80GB" -elif [ ${step} = "efcs" ]; then - - if [ ${CASE} = "C768" ]; then - export wtime_efcs="06:00:00" - else - export wtime_efcs="00:40:00" - fi - export npe_efcs=$(echo "${layout_x} * ${layout_y} * 6" | bc) - export nth_efcs=${nth_fv3:-2} - export npe_node_efcs=$(echo "${npe_node_max} / ${nth_efcs}" | bc) - elif [ ${step} = "epos" ]; then export wtime_epos="00:15:00" diff --git a/parm/config/config.ufs b/parm/config/config.ufs new file mode 100644 index 0000000000..4dd8cb8bc7 --- /dev/null +++ b/parm/config/config.ufs @@ -0,0 +1,358 @@ +#! /usr/bin/env bash + +########## config.ufs ########## +# UFS model resolution specific parameters +# e.g. time-step, processor layout, physics and dynamics parameters +# This config sets default variables for FV3, MOM6, CICE6 for their resolutions +# User can over-ride after sourcing this config file + +echo "BEGIN: config.ufs" + +if [ $# -le 1 ]; then + + echo "Must specify an input resolution argument to set variables!" + echo "argument can be any one of the following:" + echo "--fv3 C48|C96|C192|C384|C768|C1152|C3072" + echo "--mom6 500|100|025" + echo "--cice6 500|100|025" + echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|mx050|mx025" + + exit 1 + +fi + +# Initialize +skip_mom6=true +skip_cice6=true +skip_ww3=true + +# Loop through named arguments +while [[ $# -gt 0 ]]; do + key="$1" + case "${key}" in + "--fv3") + fv3_res="$2" + ;; + "--mom6") + mom6_res="$2" + skip_mom6=false + ;; + "--cice6") + cice6_res="$2" + skip_cice6=false + ;; + "--ww3") + ww3_res="$2" + skip_ww3=false + ;; + *) # unknown option + echo "FATAL ERROR: Unknown option: ${key}, ABORT!" + exit 1 + ;; + esac + shift + shift +done + +case "${machine}" in + "WCOSS2") + export npe_node_max=128 + ;; + "HERA" | "ORION") + export npe_node_max=40 + ;; + "JET") + case "${PARTITION_BATCH}" in + "xjet") + export npe_node_max=24 + ;; + "vjet" | "sjet") + export npe_node_max=16 + ;; + "kjet") + export npe_node_max=40 + ;; + *) + echo "FATAL ERROR: Unsupported PARTITION_BATCH = ${PARTITION_BATCH}, ABORT!" + exit 1 + ;; + esac + ;; + "S4") + case "${PARTITION_BATCH}" in + "s4") + export npe_node_max=32 + ;; + "ivy") + export npe_node_max=20 + ;; + *) + echo "FATAL ERROR: Unsupported PARTITION_BATCH = ${PARTITION_BATCH}, ABORT!" + exit 1 + ;; + esac + ;; +esac + +# (Standard) Model resolution dependent variables +case "${fv3_res}" in + "C48") + export DELTIM=1200 + export layout_x=1 + export layout_y=1 + export layout_x_gfs=1 + export layout_y_gfs=1 + export nth_fv3=1 + export nth_fv3_gfs=1 + export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export WRITE_GROUP=1 + export WRTTASK_PER_GROUP=2 + export WRITE_GROUP_GFS=1 + export WRTTASK_PER_GROUP_GFS=2 + export WRTIOBUF="1M" + ;; + "C96") + export DELTIM=600 + export layout_x=2 + export layout_y=2 + export layout_x_gfs=2 + export layout_y_gfs=2 + export nth_fv3=1 + export nth_fv3_gfs=1 + export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export WRITE_GROUP=1 + export WRTTASK_PER_GROUP=4 + export WRITE_GROUP_GFS=1 + export WRTTASK_PER_GROUP_GFS=4 + export WRTIOBUF="4M" + ;; + "C192") + export DELTIM=450 + export layout_x=4 + export layout_y=6 + export layout_x_gfs=4 + export layout_y_gfs=6 + export nth_fv3=1 + export nth_fv3_gfs=2 + export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export WRITE_GROUP=1 + export WRTTASK_PER_GROUP=64 + export WRITE_GROUP_GFS=2 + export WRTTASK_PER_GROUP_GFS=64 + export WRTIOBUF="8M" + ;; + "C384") + export DELTIM=200 + export layout_x=6 + export layout_y=8 + export layout_x_gfs=8 + export layout_y_gfs=12 + export nth_fv3=1 + export nth_fv3_gfs=2 + export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export WRITE_GROUP=1 + export WRTTASK_PER_GROUP=48 + export WRITE_GROUP_GFS=2 + export WRTTASK_PER_GROUP_GFS=64 + export WRTIOBUF="16M" + ;; + "C768") + export DELTIM=150 + export layout_x=8 + export layout_y=12 + export layout_x_gfs=12 + export layout_y_gfs=16 + export nth_fv3=4 + export nth_fv3_gfs=4 + export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export WRITE_GROUP=2 + export WRTTASK_PER_GROUP=64 + export WRITE_GROUP_GFS=4 + export WRTTASK_PER_GROUP_GFS=64 + export WRTIOBUF="32M" + ;; + "C1152") + export DELTIM=120 + export layout_x=8 + export layout_y=16 + export layout_x_gfs=8 + export layout_y_gfs=16 + export nth_fv3=4 + export nth_fv3_gfs=4 + export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export WRITE_GROUP=4 + export WRTTASK_PER_GROUP=64 # TODO: refine these numbers when a case is available + export WRITE_GROUP_GFS=4 + export WRTTASK_PER_GROUP_GFS=64 # TODO: refine these numbers when a case is available + export WRTIOBUF="48M" + ;; + "C3072") + export DELTIM=90 + export layout_x=16 + export layout_y=32 + export layout_x_gfs=16 + export layout_y_gfs=32 + export nth_fv3=4 + export nth_fv3_gfs=4 + export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export WRITE_GROUP=4 + export WRTTASK_PER_GROUP=64 # TODO: refine these numbers when a case is available + export WRITE_GROUP_GFS=4 + export WRTTASK_PER_GROUP_GFS=64 # TODO: refine these numbers when a case is available + export WRTIOBUF="64M" + ;; + *) + echo "FATAL ERROR: Unsupported FV3 resolution = ${fv3_res}, ABORT!" + exit 1 + ;; +esac + +(( ntasks_fv3 = layout_x * layout_y * 6 )) +(( ntasks_fv3_gfs = layout_x_gfs * layout_y_gfs * 6 )) +export ntasks_fv3 +export ntasks_fv3_gfs + +(( ntasks_quilt = WRITE_GROUP * WRTTASK_PER_GROUP )) +(( ntasks_quilt_gfs = WRITE_GROUP_GFS * WRTTASK_PER_GROUP_GFS )) +export ntasks_quilt +export ntasks_quilt_gfs + +# Determine whether to use parallel NetCDF based on resolution +case ${fv3_res} in + "C48" | "C96" | "C192" | "C384") + export OUTPUT_FILETYPE_ATM="netcdf" + export OUTPUT_FILETYPE_SFC="netcdf" + ;; + "C768" | "C1152" | "C3072") + export OUTPUT_FILETYPE_ATM="netcdf_parallel" + export OUTPUT_FILETYPE_SFC="netcdf_parallel" + ;; +esac + +# MOM6 specific settings +if [[ "${skip_mom6}" == "false" ]]; then + export nth_mom6=1 + case "${mom6_res}" in + "500") + ntasks_mom6=8 + OCNTIM=3600 + NX_GLB=72 + NY_GLB=35 + DT_DYNAM_MOM6='3600' + DT_THERM_MOM6='3600' + FRUNOFF="" + CHLCLIM="seawifs_1998-2006_smoothed_2X.nc" + MOM6_RESTART_SETTING='r' + MOM6_RIVER_RUNOFF='False' + ;; + "100") + ntasks_mom6=20 + OCNTIM=3600 + NX_GLB=360 + NY_GLB=320 + DT_DYNAM_MOM6='1800' + DT_THERM_MOM6='3600' + FRUNOFF="" + CHLCLIM="seawifs_1998-2006_smoothed_2X.nc" + MOM6_RESTART_SETTING='n' + MOM6_RIVER_RUNOFF='False' + ;; + "50") + ntasks_mom6=60 + OCNTIM=3600 + NX_GLB=720 + NY_GLB=576 + DT_DYNAM_MOM6='1800' + DT_THERM_MOM6='3600' + FRUNOFF="runoff.daitren.clim.${NX_GLB}x${NY_GLB}.v20180328.nc" + CHLCLIM="seawifs-clim-1997-2010.${NX_GLB}x${NY_GLB}.v20180328.nc" + MOM6_RESTART_SETTING='n' + MOM6_RIVER_RUNOFF='True' + ;; + "025") + ntasks_mom6=220 + OCNTIM=1800 + NX_GLB=1440 + NY_GLB=1080 + DT_DYNAM_MOM6='900' + DT_THERM_MOM6='1800' + FRUNOFF="runoff.daitren.clim.${NX_GLB}x${NY_GLB}.v20180328.nc" + CHLCLIM="seawifs-clim-1997-2010.${NX_GLB}x${NY_GLB}.v20180328.nc" + MOM6_RIVER_RUNOFF='True' + MOM6_RESTART_SETTING="r" + ;; + *) + echo "FATAL ERROR: Unsupported MOM6 resolution = ${mom6_res}, ABORT!" + exit 1 + ;; + esac + export ntasks_mom6 + export OCNTIM + export NX_GLB NY_GLB + export DT_DYNAM_MOM6 DT_THERM_MOM6 + export FRUNOFF + export CHLCLIM + export MOM6_RIVER_RUNOFF + export MOM6_RESTART_SETTING +fi + +# CICE6 specific settings +if [[ "${skip_cice6}" == "false" ]]; then + # Ensure we sourced the MOM6 section + if [[ "${skip_mom6}" == "true" ]]; then + echo "FATAL ERROR: CICE6 cannot be configured without MOM6, ABORT!" + exit 1 + fi + export nth_cice6=${mom6_res} # CICE6 needs to run on same threads as MOM6 + case "${cice6_res}" in + "500") + ntasks_cice6=4 + cice6_processor_shape="slenderX1" + ;; + "100") + ntasks_cice6=10 + cice6_processor_shape="slenderX2" + ;; + "050") + ntasks_cice6=30 + cice6_processor_shape="slenderX2" + ;; + "025") + ntasks_cice6=120 + cice6_processor_shape="slenderX2" + ;; + *) + echo "FATAL ERROR: Unsupported CICE6 resolution = ${cice6_res}, ABORT!" + exit 1 + ;; + esac + # NX_GLB and NY_GLB are set in the MOM6 section above + # CICE6 runs on the same domain decomposition as MOM6 + export ntasks_cice6 + export cice6_processor_shape +fi + +# WW3 specific settings +if [[ "${skip_ww3}" == "false" ]]; then + export nth_ww3=2 + case "${ww3_res}" in + "gnh_10m;aoc_9km;gsh_15m") + export ntasks_ww3=140 + ;; + "gwes_30m") + export ntasks_ww3=100 + ;; + "mx050") + export ntasks_ww3=240 + ;; + "mx025") + export ntasks_ww3=80 + ;; + *) + echo "FATAL ERROR: Unsupported WW3 resolution = ${ww3_res}, ABORT!" + exit 1 + ;; + esac +fi + +echo "END: config.ufs" diff --git a/sorc/build_ufs_utils.sh b/sorc/build_ufs_utils.sh index 073e3dbb47..5e2edf0737 100755 --- a/sorc/build_ufs_utils.sh +++ b/sorc/build_ufs_utils.sh @@ -4,7 +4,7 @@ set -eux script_dir=$(dirname "${BASH_SOURCE[0]}") cd "${script_dir}/ufs_utils.fd" || exit 1 -./build_all.sh +CMAKE_OPTS="-DGFS=ON" ./build_all.sh exit diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 0eb4f870fb..5332dee66e 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -755,8 +755,6 @@ CPL_out() { MOM6_postdet() { echo "SUB ${FUNCNAME[0]}: MOM6 after run type determination" - OCNRES=${OCNRES:-"025"} # TODO: remove from here and lift higher - # Copy MOM6 ICs $NLN "${ROTDIR}/${CDUMP}.${gPDY}/${gcyc}/ocean/RESTART/${PDY}.${cyc}0000.MOM.res.nc" "${DATA}/INPUT/MOM.res.nc" case $OCNRES in @@ -776,7 +774,6 @@ MOM6_postdet() { exit 111 fi $NLN "${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean/${CDUMP}.t${cyc}z.ocninc.nc" "${DATA}/INPUT/mom6_increment.nc" - export ODA_INCUPD="true" fi # Copy MOM6 fixed files diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh index c49d35a2c0..a3aefc9284 100755 --- a/ush/nems_configure.sh +++ b/ush/nems_configure.sh @@ -7,123 +7,132 @@ ## $cpl** switches. ## ## This is a child script of modular -## forecast script. This script is definition only +## forecast script. This script is definition only (Is it? There is nothing defined here being used outside this script.) ##### writing_nems_configure() { echo "SUB ${FUNCNAME[0]}: parsing_nems_configure begins" -if [ -e $SCRIPTDIR/nems.configure ]; then - rm -f $SCRIPTDIR/nems.configure +if [[ -e "${SCRIPTDIR}/nems.configure" ]]; then + rm -f "${SCRIPTDIR}/nems.configure" fi # Setup nems.configure -DumpFields=${NEMSDumpFields:-false} -cap_dbug_flag=${cap_dbug_flag:-0} +local DumpFields=${NEMSDumpFields:-false} +local cap_dbug_flag=${cap_dbug_flag:-0} # Determine "cmeps_run_type" based on the availability of the mediator restart file # If it is a warm_start, we already copied the mediator restart to DATA, if it was present # If the mediator restart was not present, despite being a "warm_start", we put out a WARNING # in forecast_postdet.sh if [[ -f "${DATA}/ufs.cpld.cpl.r.nc" ]]; then - cmeps_run_type='continue' + local cmeps_run_type='continue' else - cmeps_run_type='startup' + local cmeps_run_type='startup' fi -restart_interval=${restart_interval:-3024000} # Interval in seconds to write restarts +local res_int=${restart_interval:-3024000} # Interval in seconds to write restarts -ATM_model=${ATM_model:-'fv3'} -OCN_model=${OCN_model:-'mom6'} -ICE_model=${ICE_model:-'cice'} -WAV_model=${WAV_model:-'ww3'} -CHM_model=${CHM_model:-'gocart'} +rm -f "${DATA}/nems.configure" -ATMPETS=${ATMPETS:-8} -MEDPETS=${MEDPETS:-8} -OCNPETS=${OCNPETS:-0} -ICEPETS=${ICEPETS:-0} -WAVPETS=${WAVPETS:-0} -CHMPETS=${CHMPETS:-${ATMPETS}} - -USE_MOMMESH=${USE_MOMMESH:-"true"} -MESH_OCN_ICE=${MESH_OCN_ICE:-"mesh.mx${ICERES}.nc"} - -case "${OCNRES}" in - "500") EPS_IMESH="4.0e-1";; - "100") EPS_IMESH="2.5e-1";; - *) EPS_IMESH="1.0e-1";; -esac - -rm -f $DATA/nems.configure - -med_petlist_bounds=${med_petlist_bounds:-"0 $(( $MEDPETS-1 ))"} -atm_petlist_bounds=${atm_petlist_bounds:-"0 $(( $ATMPETS-1 ))"} -ocn_petlist_bounds=${ocn_petlist_bounds:-"$ATMPETS $(( $ATMPETS+$OCNPETS-1 ))"} -ice_petlist_bounds=${ice_petlist_bounds:-"$(( $ATMPETS+$OCNPETS )) $(( $ATMPETS+$OCNPETS+$ICEPETS-1 ))"} -wav_petlist_bounds=${wav_petlist_bounds:-"$(( $ATMPETS+$OCNPETS+$ICEPETS )) $(( $ATMPETS+$OCNPETS+$ICEPETS+$WAVPETS-1 ))"} -chm_petlist_bounds=${chm_petlist_bounds:-"0 $(( $CHMPETS-1 ))"} - -esmf_logkind=${esmf_logkind:-"ESMF_LOGKIND_MULTI"} #options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE +local esmf_logkind=${esmf_logkind:-"ESMF_LOGKIND_MULTI"} #options: ESMF_LOGKIND_MULTI_ON_ERROR, ESMF_LOGKIND_MULTI, ESMF_LOGKIND_NONE # Copy the selected template into run directory -infile="$SCRIPTDIR/nems.configure.$confignamevarfornems.IN" -if [ -s $infile ]; then - cp $infile tmp1 +infile="${SCRIPTDIR}/nems.configure.${confignamevarfornems}.IN" +if [[ -s ${infile} ]]; then + cp "${infile}" tmp1 else - echo "FATAL ERROR: nem.configure template '$infile' does not exist!" + echo "FATAL ERROR: nem.configure template '${infile}' does not exist!" exit 1 fi + +local atm_petlist_bounds="0 $(( ${ATMPETS}-1 ))" +local med_petlist_bounds="0 $(( ${MEDPETS}-1 ))" + sed -i -e "s;@\[med_model\];cmeps;g" tmp1 -sed -i -e "s;@\[atm_model\];$ATM_model;g" tmp1 -sed -i -e "s;@\[med_petlist_bounds\];$med_petlist_bounds;g" tmp1 -sed -i -e "s;@\[atm_petlist_bounds\];$atm_petlist_bounds;g" tmp1 -sed -i -e "s;@\[esmf_logkind\];$esmf_logkind;g" tmp1 +sed -i -e "s;@\[atm_model\];fv3;g" tmp1 +sed -i -e "s;@\[med_petlist_bounds\];${med_petlist_bounds};g" tmp1 +sed -i -e "s;@\[atm_petlist_bounds\];${atm_petlist_bounds};g" tmp1 +sed -i -e "s;@\[esmf_logkind\];${esmf_logkind};g" tmp1 -if [ $cpl = ".true." ]; then - sed -i -e "s;@\[coupling_interval_slow_sec\];$CPL_SLOW;g" tmp1 +if [[ ${cpl} = ".true." ]]; then + sed -i -e "s;@\[coupling_interval_slow_sec\];${CPL_SLOW};g" tmp1 fi -if [ $cplflx = .true. ]; then - if [ $restart_interval -gt 0 ]; then - restart_interval_nems=$restart_interval +if [[ "${cplflx}" = ".true." ]]; then + if [[ ${res_int} -gt 0 ]]; then + local restart_interval_nems=${res_int} else - restart_interval_nems=$FHMAX + local restart_interval_nems=${FHMAX} fi - sed -i -e "s;@\[ocn_model\];$OCN_model;g" tmp1 - sed -i -e "s;@\[ocn_petlist_bounds\];$ocn_petlist_bounds;g" tmp1 - sed -i -e "s;@\[DumpFields\];$DumpFields;g" tmp1 - sed -i -e "s;@\[cap_dbug_flag\];$cap_dbug_flag;g" tmp1 - sed -i -e "s;@\[use_coldstart\];$use_coldstart;g" tmp1 - sed -i -e "s;@\[RUNTYPE\];$cmeps_run_type;g" tmp1 - sed -i -e "s;@\[CPLMODE\];$cplmode;g" tmp1 - sed -i -e "s;@\[restart_interval\];$restart_interval;g" tmp1 - sed -i -e "s;@\[coupling_interval_fast_sec\];$CPL_FAST;g" tmp1 - sed -i -e "s;@\[RESTART_N\];$restart_interval_nems;g" tmp1 - sed -i -e "s;@\[use_mommesh\];$USE_MOMMESH;g" tmp1 - sed -i -e "s;@\[eps_imesh\];$EPS_IMESH;g" tmp1 - sed -i -e "s;@\[ATMTILESIZE\];$RESTILE;g" tmp1 + + # TODO: Should this be raised up to config.ufs? + case "${OCNRES}" in + "500") local eps_imesh="4.0e-1";; + "100") local eps_imesh="2.5e-1";; + *) local eps_imesh="1.0e-1";; + esac + + local use_coldstart=${use_coldstart:-".false."} + local use_mommesh=${USE_MOMMESH:-"true"} + local restile=$(echo "${CASE}" |cut -c2-) + + local start="${ATMPETS}" + local end="$(( ${start}+${OCNPETS}-1 ))" + local ocn_petlist_bounds="${start} ${end}" + + sed -i -e "s;@\[ocn_model\];mom6;g" tmp1 + sed -i -e "s;@\[ocn_petlist_bounds\];${ocn_petlist_bounds};g" tmp1 + sed -i -e "s;@\[DumpFields\];${DumpFields};g" tmp1 + sed -i -e "s;@\[cap_dbug_flag\];${cap_dbug_flag};g" tmp1 + sed -i -e "s;@\[use_coldstart\];${use_coldstart};g" tmp1 + sed -i -e "s;@\[RUNTYPE\];${cmeps_run_type};g" tmp1 + sed -i -e "s;@\[CPLMODE\];${cplmode};g" tmp1 + sed -i -e "s;@\[restart_interval\];${res_int};g" tmp1 + sed -i -e "s;@\[coupling_interval_fast_sec\];${CPL_FAST};g" tmp1 + sed -i -e "s;@\[RESTART_N\];${restart_interval_nems};g" tmp1 + sed -i -e "s;@\[use_mommesh\];${use_mommesh};g" tmp1 + sed -i -e "s;@\[eps_imesh\];${eps_imesh};g" tmp1 + sed -i -e "s;@\[ATMTILESIZE\];${restile};g" tmp1 fi -if [ $cplwav = .true. ]; then - sed -i -e "s;@\[wav_model\];ww3;g" tmp1 - sed -i -e "s;@\[wav_petlist_bounds\];$wav_petlist_bounds;g" tmp1 - sed -i -e "s;@\[MESH_WAV\];$MESH_WAV;g" tmp1 - sed -i -e "s;@\[MULTIGRID\];$waveMULTIGRID;g" tmp1 + +if [[ "${cplice}" = ".true." ]]; then + + local mesh_ocn_ice=${MESH_OCN_ICE:-"mesh.mx${ICERES}.nc"} + + local start="$(( ${ATMPETS}+${OCNPETS} ))" + local end="$(( ${start}+${ICEPETS}-1 ))" + local ice_petlist_bounds="${start} ${end}" + + sed -i -e "s;@\[ice_model\];cice6;g" tmp1 + sed -i -e "s;@\[ice_petlist_bounds\];${ice_petlist_bounds};g" tmp1 + sed -i -e "s;@\[MESH_OCN_ICE\];${mesh_ocn_ice};g" tmp1 + sed -i -e "s;@\[FHMAX\];${FHMAX_GFS};g" tmp1 fi -if [ $cplice = .true. ]; then - sed -i -e "s;@\[ice_model\];$ICE_model;g" tmp1 - sed -i -e "s;@\[ice_petlist_bounds\];$ice_petlist_bounds;g" tmp1 - sed -i -e "s;@\[MESH_OCN_ICE\];$MESH_OCN_ICE;g" tmp1 - sed -i -e "s;@\[FHMAX\];$FHMAX_GFS;g" tmp1 + +if [[ "${cplwav}" = ".true." ]]; then + + local start="$(( ${ATMPETS}+${OCNPETS:-0}+${ICEPETS:-0} ))" + local end="$(( ${start}+${WAVPETS}-1 ))" + local wav_petlist_bounds="${start} ${end}" + + sed -i -e "s;@\[wav_model\];ww3;g" tmp1 + sed -i -e "s;@\[wav_petlist_bounds\];${wav_petlist_bounds};g" tmp1 + sed -i -e "s;@\[MESH_WAV\];${MESH_WAV};g" tmp1 + sed -i -e "s;@\[MULTIGRID\];${waveMULTIGRID};g" tmp1 fi -if [ $cplchm = .true. ]; then - sed -i -e "s;@\[chm_model\];$CHM_model;g" tmp1 - sed -i -e "s;@\[chm_petlist_bounds\];$chm_petlist_bounds;g" tmp1 - sed -i -e "s;@\[coupling_interval_fast_sec\];$CPL_FAST;g" tmp1 + +if [[ "${cplchm}" = ".true." ]]; then + + local chm_petlist_bounds="0 $(( ${CHMPETS}-1 ))" + + sed -i -e "s;@\[chm_model\];gocart;g" tmp1 + sed -i -e "s;@\[chm_petlist_bounds\];${chm_petlist_bounds};g" tmp1 + sed -i -e "s;@\[coupling_interval_fast_sec\];${CPL_FAST};g" tmp1 fi mv tmp1 nems.configure echo "$(cat nems.configure)" -if [ $cplflx = .true. ]; then +if [[ "${cplflx}" = ".true." ]]; then #Create other CMEPS mediator related files cat > pio_in << EOF @@ -180,8 +189,8 @@ echo "$(cat med_modelio.nml)" fi -cp $HOMEgfs/sorc/ufs_model.fd/tests/parm/fd_nems.yaml fd_nems.yaml +${NCP} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/fd_nems.yaml" fd_nems.yaml -echo "SUB ${FUNCNAME[0]}: Nems configured for $confignamevarfornems" +echo "SUB ${FUNCNAME[0]}: Nems configured for ${confignamevarfornems}" } diff --git a/ush/parsing_model_configure_FV3.sh b/ush/parsing_model_configure_FV3.sh index 4574b6e352..a8997176f4 100755 --- a/ush/parsing_model_configure_FV3.sh +++ b/ush/parsing_model_configure_FV3.sh @@ -12,6 +12,13 @@ FV3_model_configure(){ +local restile=$(echo "${CASE}" |cut -c2-) +local ichunk2d=$((4*restile)) +local jchunk2d=$((2*restile)) +local ichunk3d=$((4*restile)) +local jchunk3d=$((2*restile)) +local kchunk3d=1 + rm -f model_configure cat >> model_configure < ice_in < ice_in < ice_in < ice_in < ice_in < ice_in <> input.nml <> input.nml <> input.nml <> input.nml < $DATA/INPUT/MOM_input -rm $DATA/INPUT/MOM_input_template_$OCNRES +${NCP} -pf "${HOMEgfs}/parm/mom6/MOM_input_template_${OCNRES}" "${DATA}/INPUT/" +sed -e "s/@\[DT_THERM_MOM6\]/${DT_THERM_MOM6}/g" \ + -e "s/@\[DT_DYNAM_MOM6\]/${DT_DYNAM_MOM6}/g" \ + -e "s/@\[MOM6_RIVER_RUNOFF\]/${MOM6_RIVER_RUNOFF}/g" \ + -e "s/@\[MOM6_THERMO_SPAN\]/${MOM6_THERMO_SPAN}/g" \ + -e "s/@\[MOM6_USE_LI2016\]/${MOM6_USE_LI2016}/g" \ + -e "s/@\[MOM6_USE_WAVES\]/${MOM6_USE_WAVES}/g" \ + -e "s/@\[MOM6_ALLOW_LANDMASK_CHANGES\]/${MOM6_ALLOW_LANDMASK_CHANGES}/g" \ + -e "s/@\[NX_GLB\]/${NX_GLB}/g" \ + -e "s/@\[NY_GLB\]/${NY_GLB}/g" \ + -e "s/@\[CHLCLIM\]/${CHLCLIM}/g" \ + -e "s/@\[DO_OCN_SPPT\]/${OCN_SPPT}/g" \ + -e "s/@\[PERT_EPBL\]/${PERT_EPBL}/g" \ + -e "s/@\[ODA_INCUPD_NHOURS\]/${ODA_INCUPD_NHOURS}/g" \ + -e "s/@\[ODA_INCUPD\]/${ODA_INCUPD}/g" "${DATA}/INPUT/MOM_input_template_${OCNRES}" > "${DATA}/INPUT/MOM_input" +rm "${DATA}/INPUT/MOM_input_template_${OCNRES}" #data table for runoff: -DATA_TABLE=${DATA_TABLE:-$PARM_FV3DIAG/data_table} -$NCP $DATA_TABLE $DATA/data_table_template -sed -e "s/@\[FRUNOFF\]/$FRUNOFF/g" $DATA/data_table_template > $DATA/data_table -rm $DATA/data_table_template +DATA_TABLE=${DATA_TABLE:-${PARM_FV3DIAG}/data_table} +${NCP} "${DATA_TABLE}" "${DATA}/data_table_template" +sed -e "s/@\[FRUNOFF\]/${FRUNOFF}/g" "${DATA}/data_table_template" > "${DATA}/data_table" +rm "${DATA}/data_table_template" }