From 2435f39cca6fa7950481a080f7adf2aa6416bf57 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Mon, 17 Apr 2023 11:20:38 -0600 Subject: [PATCH 1/5] Bringing branch UTD with WIP. --- ush/python/pygfs/task/gfs_forecast.py | 17 ++- ush/python/pygfs/ufswm/gfs.py | 151 +++++++++++++++++++++++++- ush/python/pygfs/ufswm/ufs.py | 97 ++++++++++++++++- 3 files changed, 254 insertions(+), 11 deletions(-) diff --git a/ush/python/pygfs/task/gfs_forecast.py b/ush/python/pygfs/task/gfs_forecast.py index 3527c623e0..48d23c3014 100644 --- a/ush/python/pygfs/task/gfs_forecast.py +++ b/ush/python/pygfs/task/gfs_forecast.py @@ -15,7 +15,7 @@ class GFSForecast(Task): """ @logit(logger, name="GFSForecast") - def __init__(self, config: Dict[str, Any], *args, **kwargs): + def __init__(self: Task, config: Dict[str, Any], *args, **kwargs): """ Parameters ---------- @@ -33,3 +33,18 @@ def __init__(self, config: Dict[str, Any], *args, **kwargs): # Create and initialize the GFS variant of the UFS self.gfs = GFS(config) + + @logit(logger) + def configure(self: Task) -> None: + """ + + """ + + @logit(logger) + def initialize(self: Task) -> None: + """ + + """ + + # Build the directory tree and link the relevant fixed-files. + self.gfs.build_dirtree() diff --git a/ush/python/pygfs/ufswm/gfs.py b/ush/python/pygfs/ufswm/gfs.py index f86164d706..aa4d83249c 100644 --- a/ush/python/pygfs/ufswm/gfs.py +++ b/ush/python/pygfs/ufswm/gfs.py @@ -1,20 +1,163 @@ import copy import logging +import os from pygw.logger import logit from pygfs.ufswm.ufs import UFS +from pygw.file_utils import FileHandler +from pygw.yaml_file import YAMLFile, parse_yamltmpl + + logger = logging.getLogger(__name__.split('.')[-1]) +# ---- + +# Define the valid GFS configuration attributes. +VALID_APP_LIST = ["atm"] + +# HRW: Only testing C48 for now. +VALID_ATMRES_LIST = ["c48"] +VALID_ATMLEVS_LIST = [64, 128] + +# ---- + class GFS(UFS): + """ + Description + ----------- + + TBD + + Parameters + ---------- + + config: AttrDict + + A Python dictionary containing the run-time configuration + attributes. + + """ @logit(logger, name="GFS") - def __init__(self, config): + def __init__(self: UFS, config): + """ + Description + ----------- + + Creates a new GFS object. + """ + + # Define the base-class attributes. super().__init__("GFS", config) - # Start putting fixed properties of the GFS - self.ntiles = 6 + self.ufs_config.ntiles = 6 + self.config_dirtrees() + self.config_fcstapp() + + @logit(logger) + def build_dirtree(self: UFS) -> None: + """ + Description + ----------- + + This method builds the directory tree and links the relevant + files. + + """ + + fixedfile_list = [item for item in self.ufs_config.fixedfile_list if ( + "atmos" in item.lower() or "land" in item.lower())] + + for fixedfile in fixedfile_list: + msg = f"Building directory tree and linking files from {fixedfile}." + logger.info(msg=msg) + + fixedfile_model = parse_yamltmpl( + path=fixedfile, data=self.ufs_config) + FileHandler(fixedfile_model.dirtree).sync() + FileHandler(fixedfile_model.fixedfiles).sync() + + @logit(logger) + def config_dirtrees(self: UFS) -> None: + """ + Description + ----------- + + This method defines the directory paths (e.g., trees) for the + GFS forecast model and updates the base-class object + `ufs_config`. + + """ + + # Define the directory tree attributes and update the + # base-class object `ufs_config`. + self.ufs_config.DATA = self._config.DATA + + self.ufs_config.FIX_aer = os.path.join(self._config.FIXgfs, 'aer') + self.ufs_config.FIX_am = os.path.join(self._config.FIXgfs, 'am') + self.ufs_config.FIX_lut = os.path.join(self._config.FIXgfs, 'lut') + self.ufs_config.FIX_orog = os.path.join(self._config.FIXgfs, 'orog') + self.ufs_config.FIX_ugwd = os.path.join(self._config.FIXgfs, 'ugwd') + + self.ufs_config.HOMEgfs = self._config.HOMEgfs + + @logit(logger) + def config_fcstapp(self: UFS) -> None: + """ + Description + ----------- + + This method configures the base-class object `ufs_config` in + accordance with the respective GFS application. + + """ + + # HRW: `atmos_run` will need to be defined dynamically once + # additional GFS applications are added. + config_attr_dict = {'aero_run': 'DO_AERO', + 'app': 'APP', + 'atm_levs': 'LEVS', + 'atm_res': "CASE", + 'chm_cpl': 'cplchm', + 'ice_cpl': 'cplice', + 'ice_res': 'ICERES', + 'ice_run': 'DO_ICE', + 'ocn_cpl': 'cpl', + 'ocn_res': 'OCNRES', + 'ocn_run': 'DO_OCN', + 'wav_cpl': 'cplwav', + 'wav_run': 'DO_WAVE', + } + + for (config_key, config_value) in config_attr_dict.items(): + self.ufs_config[config_key] = self._config[config_value] + + # Check that the GFS application configuration is valid. + if self.ufs_config.app.lower() not in VALID_APP_LIST: + msg = (f"The GFS application {self.ufs_config.app} is not supported; " + f"supported GFS applications are {', '.join(VALID_APP_LIST).upper()}." + ) + logger.error(msg=msg) + + # NEED TO RAISE AN EXCEPTION HERE. + + if self.ufs_config.atm_res.lower() not in VALID_ATMRES_LIST: + msg = (f"The GFS cubed-sphere resolution {self.ufs_config.atm_res.upper()} " + "is not supported; supported resolutions are " + f"{', '.join(VALID_ATMRES_LIST)}." + ) + logger.error(msg=msg) + + # NEED TO RAISE AN EXCEPTION HERE. + + if self.ufs_config.atm_levs not in VALID_ATMLEVS_LIST: + msg = (f"The number of unstaggered vertical levels {self.ufs_config.atm_levs} " + "is not supported; valid number of levels are " + f"{', '.join(str(atmlev) for atmlev in VALID_ATMLEVS_LIST)}" + ) + logger.error(msg=msg) - # Determine coupled/uncoupled from config and define as appropriate + # NEED TO RAISE AN EXCEPTION HERE. diff --git a/ush/python/pygfs/ufswm/ufs.py b/ush/python/pygfs/ufswm/ufs.py index a9118801b9..d098de6c14 100644 --- a/ush/python/pygfs/ufswm/ufs.py +++ b/ush/python/pygfs/ufswm/ufs.py @@ -1,7 +1,12 @@ import re +import os import copy import logging -from typing import Dict, Any +from typing import Dict, Any, List + +from dataclasses import dataclass + +from pygw.attrdict import AttrDict from pygw.template import Template, TemplateConstants from pygw.logger import logit @@ -11,10 +16,11 @@ UFS_VARIANTS = ['GFS'] +@dataclass class UFS: @logit(logger, name="UFS") - def __init__(self, model_name: str, config: Dict[str, Any]): + def __init__(self: dataclass, model_name: str, config: Dict[str, Any]): """Initialize the UFS-weather-model generic class and check if the model_name is a valid variant Parameters @@ -29,12 +35,16 @@ def __init__(self, model_name: str, config: Dict[str, Any]): if model_name not in UFS_VARIANTS: logger.warn(f"{model_name} is not a valid UFS variant") raise NotImplementedError(f"{model_name} is not yet implemented") + self.model_name = model_name.lower() # Make a deep copy of incoming config for caching purposes. _config should not be updated self._config = copy.deepcopy(config) + # Determine the model configuration. + self.ufs_config = self.get_ufs_config() + @logit(logger) - def parse_ufs_templates(input_template, output_file, ctx: Dict) -> None: + def parse_ufs_templates(self: dataclass, input_template, output_file, ctx: Dict) -> None: """ This method parses UFS-weather-model templates of the pattern @[VARIABLE] drawing the value from ctx['VARIABLE'] @@ -50,9 +60,84 @@ def parse_ufs_templates(input_template, output_file, ctx: Dict) -> None: matches = re.findall(pattern, file_out) if matches: logger.warn(f"{input_template} was rendered incompletely") - logger.warn(f"The following variables were not substituted") - print(matches) # TODO: improve the formatting of this message - # TODO: Should we abort here? or continue to write output_file? + logger.warn(f"The following variables were not substituted:") + logger.warn(f"{', '.join(matches)}.") with open(output_file, 'w') as fho: fho.write(file_out) + + @staticmethod + def __get_fixedfile_list(fixedfile_path) -> List: + """ + Description + ----------- + + This method defines and returns a Python list of + YAML-formatted files beneath the directory tree + `fixedfile_path`. + + Parameters + ---------- + + fixedfile_path: str + + A Python string specifying the directory tree beneath + which to collect/define a list of YAML-formatted files. + + Returns + ------- + + fixedfile_list: List + + A Python list of fixed files beneath the directory tree + `fixedfile_path` attribute upon entry. + + """ + + try: + fixedfile_list = os.listdir(fixedfile_path) + except FileNotFoundError: + msg = f"The directory path {fixedfile_path} could not be found." + logger.error(msg=msg) + raise FileNotFoundError(msg) + + msg = f"Found the following fixed files: {(', '.join(fixedfile_list))}." + logger.warn(msg=msg) + + return fixedfile_list + + @logit(logger) + def get_ufs_config(self: dataclass) -> AttrDict: + """ + Description + ----------- + + This method builds a Python dictionary containing the + attributes for the respective UFS forecast model. + + Returns + ------- + + ufs_config: AttrDict + + A Python object containing the configuration attributes + for the respective UFS forecast model. + + """ + + ufs_config = AttrDict() + ufs_config.coupled = self._config.DO_COUPLED + + # Collect the list of fixed-file YAMLs. + fixedfile_path = os.path.join( + self._config.PARMgfs, "ufs", "fix", self.model_name) + fixedfile_list = self.__get_fixedfile_list( + fixedfile_path=fixedfile_path) + ufs_config.fixedfile_list = \ + [os.path.join(fixedfile_path, fixedfile) for + fixedfile in fixedfile_list] + + # Define the directories containing the forecast model static + # inputs. + + return ufs_config From cc222797fadd2181c131219b9d078456585e8bc3 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Mon, 17 Apr 2023 11:23:05 -0600 Subject: [PATCH 2/5] Temporary for rebase. --- jobs/JGLOBAL_FORECAST | 63 ----------- jobs/rocoto/fcst.sh | 53 ---------- scripts/exglobal_forecast.sh | 199 ----------------------------------- ush/module-setup.sh | 107 ------------------- 4 files changed, 422 deletions(-) delete mode 100755 jobs/JGLOBAL_FORECAST delete mode 100755 jobs/rocoto/fcst.sh delete mode 100755 scripts/exglobal_forecast.sh delete mode 100755 ush/module-setup.sh diff --git a/jobs/JGLOBAL_FORECAST b/jobs/JGLOBAL_FORECAST deleted file mode 100755 index 9027512761..0000000000 --- a/jobs/JGLOBAL_FORECAST +++ /dev/null @@ -1,63 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "fcst" -c "base fcst" - -############################################## -# Set variables used in the script -############################################## -# Set wave variables -if [ ${DO_WAVE:-"NO"} = "YES" ]; then - # WAVE component directory - export CDUMPwave=${CDUMPwave:-${CDUMP}wave} - export COMINwave=${COMINwave:-$(compath.py ${envir}/${NET}/${gfs_ver})/${CDUMP}.${PDY}/${cyc}/wave} - export COMOUTwave=${COMOUTwave:-$(compath.py -o ${NET}/${gfs_ver})/${CDUMP}.${PDY}/${cyc}/wave} -fi - -############################################## -# Begin JOB SPECIFIC work -############################################## - -# Restart conditions for GFS cycle come from GDAS -rCDUMP=${CDUMP} -[[ ${CDUMP} = "gfs" ]] && export rCDUMP="gdas" - -# Forecast length for GFS forecast -if [ ${CDUMP} = "gfs" ]; then - export FHMAX=${FHMAX_GFS} - export FHOUT=${FHOUT_GFS} - export FHMAX_HF=${FHMAX_HF_GFS} - export FHOUT_HF=${FHOUT_HF_GFS} -else - export FHMAX_HF=0 - export FHOUT_HF=0 -fi - - -############################################################### -# Run relevant exglobal script - -${FORECASTSH:-${SCRgfs}/exglobal_forecast.sh} -status=$? -[[ ${status} -ne 0 ]] && exit ${status} - - -############################################## -# End JOB SPECIFIC work -############################################## - -############################################## -# Final processing -############################################## -if [ -e "${pgmout}" ] ; then - cat ${pgmout} -fi - -########################################## -# Remove the Temporary working directory -########################################## -cd ${DATAROOT} -[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} - - -exit 0 diff --git a/jobs/rocoto/fcst.sh b/jobs/rocoto/fcst.sh deleted file mode 100755 index 512bee127f..0000000000 --- a/jobs/rocoto/fcst.sh +++ /dev/null @@ -1,53 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" - -############################################################### -# Source FV3GFS workflow modules -#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -#status=$? -#[[ ${status} -ne 0 ]] && exit ${status} - -# TODO: clean this up -source "${HOMEgfs}/ush/detect_machine.sh" -set +x -source "${HOMEgfs}/ush/module-setup.sh" -module use "${HOMEgfs}/sorc/ufs_model.fd/tests" -module load modules.ufs_model.lua -module load prod_util -if [[ "${MACHINE_ID}" = "wcoss2" ]]; then - module load cray-pals -fi -if [[ "${MACHINE_ID}" = "hera" ]]; then - module use "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/modulefiles/core" - module load "miniconda3/4.6.14" - module load "gfs_workflow/1.0.0" -# TODO: orion and wcoss2 will be uncommented when they are ready. This comment block will be removed in the next PR -#elif [[ "${MACHINE_ID}" = "orion" ]]; then -# module use "/home/rmahajan/opt/global-workflow/modulefiles/core" -# module load "python/3.7.5" -# module load "gfs_workflow/1.0.0" -#elif [[ "${MACHINE_ID}" = "wcoss2" ]]; then -# module load "python/3.7.5" -fi -module list -unset MACHINE_ID -set_trace - -############################################################### -# exglobal_forecast.py requires the following in PYTHONPATH -# This will be moved to a module load when ready -pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src:${HOMEgfs}/ush/python/pygfs" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" -export PYTHONPATH - -export job="fcst" -export jobid="${job}.$$" - -############################################################### -# Execute the JJOB -${HOMEgfs}/jobs/JGLOBAL_FORECAST -status=$? - - -exit ${status} diff --git a/scripts/exglobal_forecast.sh b/scripts/exglobal_forecast.sh deleted file mode 100755 index 8bd6776dda..0000000000 --- a/scripts/exglobal_forecast.sh +++ /dev/null @@ -1,199 +0,0 @@ -#! /usr/bin/env bash - -################################################################################ -## UNIX Script Documentation Block -## Script name: exglobal_fcst_nemsfv3gfs.sh -## Script description: Runs a global FV3GFS model forecast -## -## Author: Fanglin Yang Organization: NCEP/EMC Date: 2016-11-15 -## Abstract: This script runs a single GFS forecast with FV3 dynamical core. -## This script is created based on a C-shell script that GFDL wrote -## for the NGGPS Phase-II Dycore Comparison Project. -## -## Script history log: -## 2016-11-15 Fanglin Yang First Version. -## 2017-02-09 Rahul Mahajan Added warm start and restructured the code. -## 2017-03-10 Fanglin Yang Updated for running forecast on Cray. -## 2017-03-24 Fanglin Yang Updated to use NEMS FV3GFS with IPD4 -## 2017-05-24 Rahul Mahajan Updated for cycling with NEMS FV3GFS -## 2017-09-13 Fanglin Yang Updated for using GFDL MP and Write Component -## 2019-04-02 -## -## Attributes: -## Language: Portable Operating System Interface (POSIX) Shell -## Machines: All supported platforms -## -## Usage (Arguments) -## No command line argument -## -## Data input (location, name) -## Warm start files: -## 1. restart file except sfc_data, $gmemdir/RESTART/$PDY.$cyc.*.nc -## 2. sfcanl_data, $memdir/RESTART/$PDY.$cyc.*.nc -## 3. coupler_res, $gmemdir/RESTART/$PDY.$cyc.coupler.res -## 4. increment file, $memdir/${CDUMP}.t${cyc}z.atminc.nc -## OR $DATA/INPUT/fv3_increment.nc -## Cold start files: -## 1. initial condition, $memdir/INPUT/*.nc -## Restart files: -## -## Fix files: -## 1. computing grid, $FIXfv3/$CASE/${CASE}_grid.tile${n}.nc -## 2. orography data, $FIXfv3/$CASE/${CASE}_oro_data.tile${n}.nc -## 3. mosaic data, $FIXfv3/$CASE/${CASE}_mosaic.nc -## 4. Global O3 data, $FIX_AM/${O3FORC} -## 5. Global H2O data, $FIX_AM/${H2OFORC} -## 6. Global solar constant data, $FIX_AM/global_solarconstant_noaa_an.txt -## 7. Global surface emissivity, $FIX_AM/global_sfc_emissivity_idx.txt -## 8. Global CO2 historical data, $FIX_AM/global_co2historicaldata_glob.txt -## 8. Global CO2 monthly data, $FIX_AM/co2monthlycyc.txt -## 10. Additional global CO2 data, $FIX_AM/fix_co2_proj/global_co2historicaldata -## 11. Climatological aerosol global distribution -## $FIX_AM/global_climaeropac_global.txt -## 12. Monthly volcanic forcing $FIX_AM/global_volcanic_aerosols_YYYY-YYYY.txt -## -## Data output (location, name) -## If quilting=true and output grid is gaussian grid: -## 1. atmf data, $memdir/${CDUMP}.t${cyc}z.atmf${FH3}.$OUTPUT_FILE -## 2. sfcf data, $memdir/${CDUMP}.t${cyc}z.sfcf${FH3}.$OUTPUT_FILE -## 3. logf data, $memdir/${CDUMP}.t${cyc}z.logf${FH3}.$OUTPUT_FILE -## If quilting=false and output grid is not gaussian grid: -## 1. NGGPS2D, $memdir/nggps2d.tile${n}.nc -## 2. NGGPS3D, $memdir/nggps3d.tile${n}.nc -## 3. grid spec, $memdir/grid_spec.tile${n}.nc -## 4. atmospheric static tiles, $memdir/atmos_static.tile${n}.nc -## 5. atmospheric 4x daily tiles, $memdir/atmos_4xdaily.tile${n}.nc -## -## Status output -## 0: Normal -## others: Error -## -## Namelist input, in RUNDIR, -## 1. diag_table -## 2. nems.configure -## 3. model_configure -## 4. input.nml -####################### -# Main body starts here -####################### - -source "$HOMEgfs/ush/preamble.sh" - -SCRIPTDIR=$(dirname $(readlink -f "$0") )/../ush -echo "MAIN: environment loaded for $machine platform,Current Script locates in $SCRIPTDIR." - -# include all subroutines. Executions later. -source $SCRIPTDIR/cplvalidate.sh # validation of cpl* -source $SCRIPTDIR/forecast_predet.sh # include functions for variable definition -source $SCRIPTDIR/forecast_det.sh # include functions for run type determination -source $SCRIPTDIR/forecast_postdet.sh # include functions for variables after run type determination -source $SCRIPTDIR/nems_configure.sh # include functions for nems_configure processing -source $SCRIPTDIR/parsing_model_configure_FV3.sh -source $SCRIPTDIR/parsing_model_configure_DATM.sh - -# Compset string. For nems.configure.* template selection. Default ATM only -confignamevarfornems=${confignamevarfornems:-'atm'} - -# Coupling control switches, for coupling purpose, off by default -cpl=${cpl:-.false.} -cplflx=${cplflx:-.false.} # default off,import from outside source -cplwav=${cplwav:-.false.} # ? how to control 1-way/2-way? -cplchm=${cplchm:-.false.} # Chemistry model -cplice=${cplice:-.false.} # ICE model - -OCNTIM=${OCNTIM:-1800} -DELTIM=${DELTIM:-450} -ICETIM=${DELTIM} - -CPL_SLOW=${CPL_SLOW:-$OCNTIM} -CPL_FAST=${CPL_FAST:-$ICETIM} - -echo "MAIN: $confignamevarfornems selected" -echo "MAIN: Forecast script started for $confignamevarfornems on $machine" - -echo "MAIN: Validating $confignamevarfornems with cpl switches" -cplvalidate -echo "MAIN: $confignamevarfornems validated, continue" -# Validate the consistency between $confignamevarfornems and $CPL switches - -echo "MAIN: Loading variables before determination of run type" - -common_predet - -echo $RUN -case $RUN in - 'data') DATM_predet;; - *gfs | *gdas | 'gefs') FV3_GFS_predet;; -esac -[[ $cplflx = .true. ]] && MOM6_predet -[[ $cplwav = .true. ]] && WW3_predet -[[ $cplice = .true. ]] && CICE_predet - -case $RUN in - *gfs | *gdas | 'gefs') FV3_GFS_det;; -esac #no run type determination for data atmosphere -[[ $cplflx = .true. ]] && MOM6_det -[[ $cplwav = .true. ]] && WW3_det -[[ $cplice = .true. ]] && CICE_det - -echo "MAIN: RUN Type Determined" - -echo "MAIN: Post-determination set up of run type" -echo $RUN -case $RUN in - 'data') DATM_postdet;; - *gfs | *gdas | 'gefs') FV3_GFS_postdet;; -esac #no post determination set up for data atmosphere -[[ $cplflx = .true. ]] && MOM6_postdet -[[ $cplwav = .true. ]] && WW3_postdet -[[ $cplice = .true. ]] && CICE_postdet -[[ $cplchm = .true. ]] && GOCART_postdet -echo "MAIN: Post-determination set up of run type finished" - -echo "MAIN: Writing name lists and model configuration" -case $RUN in - 'data') DATM_nml;; - *gfs | *gdas | 'gefs') FV3_GFS_nml;; -esac -[[ $cplflx = .true. ]] && MOM6_nml -[[ $cplwav = .true. ]] && WW3_nml -[[ $cplice = .true. ]] && CICE_nml -[[ $cplchm = .true. ]] && GOCART_rc - -case $RUN in - 'data') DATM_model_configure;; - *gfs | *gdas | 'gefs') FV3_model_configure;; -esac -echo "MAIN: Name lists and model configuration written" - -echo "MAIN: Writing NEMS Configure file" -writing_nems_configure -echo "MAIN: NEMS configured" - -#------------------------------------------------------------------ -# run the executable - -if [ $esmf_profile ]; then - export ESMF_RUNTIME_PROFILE=ON - export ESMF_RUNTIME_PROFILE_OUTPUT=SUMMARY -fi - -$NCP $FCSTEXECDIR/$FCSTEXEC $DATA/. -$APRUN_UFS $DATA/$FCSTEXEC 1>&1 2>&2 -export ERR=$? -export err=$ERR -$ERRSCRIPT || exit $err - -case $RUN in - 'data') data_out_Data_ATM;; - *gfs | *gdas | 'gefs') data_out_GFS;; -esac -[[ $cplflx = .true. ]] && MOM6_out -[[ $cplwav = .true. ]] && WW3_out -[[ $cplice = .true. ]] && CICE_out -[[ $esmf_profile = .true. ]] && CPL_out -echo "MAIN: Output copied to COMROT" - -#------------------------------------------------------------------ - -exit $err diff --git a/ush/module-setup.sh b/ush/module-setup.sh deleted file mode 100755 index 9c27ab4f7c..0000000000 --- a/ush/module-setup.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash -set -u - -if [[ ${MACHINE_ID} = jet* ]] ; then - # We are on NOAA Jet - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/lmod/init/bash - fi - export LMOD_SYSTEM_DEFAULT_MODULES=contrib - module reset - -elif [[ ${MACHINE_ID} = hera* ]] ; then - # We are on NOAA Hera - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/lmod/init/bash - fi - export LMOD_SYSTEM_DEFAULT_MODULES=contrib - module reset - -elif [[ ${MACHINE_ID} = orion* ]] ; then - # We are on Orion - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/init/bash - fi - export LMOD_SYSTEM_DEFAULT_MODULES=contrib - module reset - -elif [[ ${MACHINE_ID} = s4* ]] ; then - # We are on SSEC Wisconsin S4 - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /usr/share/lmod/lmod/init/bash - fi - export LMOD_SYSTEM_DEFAULT_MODULES=license_intel - module reset - -elif [[ ${MACHINE_ID} = wcoss2 ]]; then - # We are on WCOSS2 - module reset - -elif [[ ${MACHINE_ID} = cheyenne* ]] ; then - # We are on NCAR Cheyenne - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /glade/u/apps/ch/modulefiles/default/localinit/localinit.sh - fi - module purge - -elif [[ ${MACHINE_ID} = stampede* ]] ; then - # We are on TACC Stampede - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /opt/apps/lmod/lmod/init/bash - fi - module purge - -elif [[ ${MACHINE_ID} = gaea* ]] ; then - # We are on GAEA. - if ( ! eval module help > /dev/null 2>&1 ) ; then - # We cannot simply load the module command. The GAEA - # /etc/profile modifies a number of module-related variables - # before loading the module command. Without those variables, - # the module command fails. Hence we actually have to source - # /etc/profile here. - source /etc/profile - __ms_source_etc_profile=yes - else - __ms_source_etc_profile=no - fi - module purge - # clean up after purge - unset _LMFILES_ - unset _LMFILES_000 - unset _LMFILES_001 - unset LOADEDMODULES - module load modules - if [[ -d /opt/cray/ari/modulefiles ]] ; then - module use -a /opt/cray/ari/modulefiles - fi - if [[ -d /opt/cray/pe/ari/modulefiles ]] ; then - module use -a /opt/cray/pe/ari/modulefiles - fi - if [[ -d /opt/cray/pe/craype/default/modulefiles ]] ; then - module use -a /opt/cray/pe/craype/default/modulefiles - fi - if [[ -s /etc/opt/cray/pe/admin-pe/site-config ]] ; then - source /etc/opt/cray/pe/admin-pe/site-config - fi - if [[ "${__ms_source_etc_profile}" == yes ]] ; then - source /etc/profile - unset __ms_source_etc_profile - fi - -elif [[ ${MACHINE_ID} = expanse* ]]; then - # We are on SDSC Expanse - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /etc/profile.d/modules.sh - fi - module purge - module load slurm/expanse/20.02.3 - -elif [[ ${MACHINE_ID} = discover* ]]; then - # We are on NCCS discover - export SPACK_ROOT=/discover/nobackup/mapotts1/spack - export PATH=${PATH}:${SPACK_ROOT}/bin - . "${SPACK_ROOT}"/share/spack/setup-env.sh - -else - echo WARNING: UNKNOWN PLATFORM 1>&2 -fi From fd2bc1f943b25cc22ea2237759ca8df990c43f37 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Mon, 17 Apr 2023 11:26:20 -0600 Subject: [PATCH 3/5] Updated fixed files. --- parm/ufs/fix/gfs/atmos.fixed_files.yaml | 93 +++++++++++++++++++++++++ parm/ufs/fix/gfs/land.fixed_files.yaml | 64 +++++++++++++++++ parm/ufs/fix/gfs/ocean.fixed_files.yaml | 10 +++ 3 files changed, 167 insertions(+) create mode 100644 parm/ufs/fix/gfs/atmos.fixed_files.yaml create mode 100644 parm/ufs/fix/gfs/land.fixed_files.yaml create mode 100644 parm/ufs/fix/gfs/ocean.fixed_files.yaml diff --git a/parm/ufs/fix/gfs/atmos.fixed_files.yaml b/parm/ufs/fix/gfs/atmos.fixed_files.yaml new file mode 100644 index 0000000000..57326e58b4 --- /dev/null +++ b/parm/ufs/fix/gfs/atmos.fixed_files.yaml @@ -0,0 +1,93 @@ +dirtree: + mkdir: + - $(DATA)/INPUT + - $(DATA)/RESTART + +fixedfiles: + copy: + # Atmosphere mosaic file linked as the grid_spec file (atm only) + - [$(FIX_orog)/$(atm_res)/$(atm_res)_mosaic.nc, $(DATA)/INPUT/grid_spec.nc] + + # Atmosphere grid tile files + - [$(FIX_orog)/$(atm_res)/$(atm_res)_grid.tile1.nc, $(DATA)/INPUT/] + - [$(FIX_orog)/$(atm_res)/$(atm_res)_grid.tile2.nc, $(DATA)/INPUT/] + - [$(FIX_orog)/$(atm_res)/$(atm_res)_grid.tile3.nc, $(DATA)/INPUT/] + - [$(FIX_orog)/$(atm_res)/$(atm_res)_grid.tile4.nc, $(DATA)/INPUT/] + - [$(FIX_orog)/$(atm_res)/$(atm_res)_grid.tile5.nc, $(DATA)/INPUT/] + - [$(FIX_orog)/$(atm_res)/$(atm_res)_grid.tile6.nc, $(DATA)/INPUT/] + + + + # oro_data_ls and oro_data_ss files from FIX_ugwd + - [$(FIX_ugwd)/$(atm_res)/$(atm_res)_oro_data_ls.tile1.nc, $(DATA)/INPUT/oro_data_ls.tile1.nc] + - [$(FIX_ugwd)/$(atm_res)/$(atm_res)_oro_data_ls.tile2.nc, $(DATA)/INPUT/oro_data_ls.tile2.nc] + - [$(FIX_ugwd)/$(atm_res)/$(atm_res)_oro_data_ls.tile3.nc, $(DATA)/INPUT/oro_data_ls.tile3.nc] + - [$(FIX_ugwd)/$(atm_res)/$(atm_res)_oro_data_ls.tile4.nc, $(DATA)/INPUT/oro_data_ls.tile4.nc] + - [$(FIX_ugwd)/$(atm_res)/$(atm_res)_oro_data_ls.tile5.nc, $(DATA)/INPUT/oro_data_ls.tile5.nc] + - [$(FIX_ugwd)/$(atm_res)/$(atm_res)_oro_data_ls.tile6.nc, $(DATA)/INPUT/oro_data_ls.tile6.nc] + - [$(FIX_ugwd)/$(atm_res)/$(atm_res)_oro_data_ss.tile1.nc, $(DATA)/INPUT/oro_data_ss.tile1.nc] + - [$(FIX_ugwd)/$(atm_res)/$(atm_res)_oro_data_ss.tile2.nc, $(DATA)/INPUT/oro_data_ss.tile2.nc] + - [$(FIX_ugwd)/$(atm_res)/$(atm_res)_oro_data_ss.tile3.nc, $(DATA)/INPUT/oro_data_ss.tile3.nc] + - [$(FIX_ugwd)/$(atm_res)/$(atm_res)_oro_data_ss.tile4.nc, $(DATA)/INPUT/oro_data_ss.tile4.nc] + - [$(FIX_ugwd)/$(atm_res)/$(atm_res)_oro_data_ss.tile5.nc, $(DATA)/INPUT/oro_data_ss.tile5.nc] + - [$(FIX_ugwd)/$(atm_res)/$(atm_res)_oro_data_ss.tile6.nc, $(DATA)/INPUT/oro_data_ss.tile6.nc] + + # GWD?? + - [$(FIX_ugwd)/ugwp_limb_tau.nc, $(DATA)/ugwp_limb_tau.nc] + + # CO2 climatology + - [$(FIX_am)/co2monthlycyc.txt, $(DATA)/co2monthlycyc.txt] + - [$(FIX_am)/global_co2historicaldata_glob.txt, $(DATA)/co2historicaldata_glob.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2009.txt, $(DATA)/co2historicaldata_2009.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2010.txt, $(DATA)/co2historicaldata_2010.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2011.txt, $(DATA)/co2historicaldata_2011.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2012.txt, $(DATA)/co2historicaldata_2012.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2013.txt, $(DATA)/co2historicaldata_2013.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2014.txt, $(DATA)/co2historicaldata_2014.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2015.txt, $(DATA)/co2historicaldata_2015.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2016.txt, $(DATA)/co2historicaldata_2016.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2017.txt, $(DATA)/co2historicaldata_2017.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2018.txt, $(DATA)/co2historicaldata_2018.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2019.txt, $(DATA)/co2historicaldata_2019.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2020.txt, $(DATA)/co2historicaldata_2020.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2021.txt, $(DATA)/co2historicaldata_2021.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2022.txt, $(DATA)/co2historicaldata_2022.txt] + - [$(FIX_am)/fix_co2_proj/global_co2historicaldata_2023.txt, $(DATA)/co2historicaldata_2023.txt] + + # FIX_am files + - [$(FIX_am)/global_climaeropac_global.txt, $(DATA)/aerosol.dat] + - [$(FIX_am)/ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77, $(DATA)/global_o3prdlos.f77] + - [$(FIX_am)/global_h2o_pltc.f77, $(DATA)/global_h2oprdlos.f77] + - [$(FIX_am)/global_glacier.2x2.grb, $(DATA)/global_glacier.2x2.grb] + - [$(FIX_am)/global_maxice.2x2.grb, $(DATA)/global_maxice.2x2.grb] + - [$(FIX_am)/global_snoclim.1.875.grb, $(DATA)/global_snoclim.1.875.grb] + - [$(FIX_am)/global_slmask.t1534.3072.1536.grb, $(DATA)/global_slmask.t1534.3072.1536.grb] + - [$(FIX_am)/global_soilmgldas.statsgo.t1534.3072.1536.grb, $(DATA)/global_soilmgldas.statsgo.t1534.3072.1536.grb] + - [$(FIX_am)/global_solarconstant_noaa_an.txt, $(DATA)/solarconstant_noaa_an.txt] + - [$(FIX_am)/global_sfc_emissivity_idx.txt, $(DATA)/sfc_emissivity_idx.txt] + - [$(FIX_am)/RTGSST.1982.2012.monthly.clim.grb, $(DATA)/RTGSST.1982.2012.monthly.clim.grb] + - [$(FIX_am)/IMS-NIC.blended.ice.monthly.clim.grb, $(DATA)/IMS-NIC.blended.ice.monthly.clim.grb] + + # MERRA2 Aerosol Climatology + - [$(FIX_aer)/merra2.aerclim.2003-2014.m01.nc, $(DATA)/aeroclim.m01.nc] + - [$(FIX_aer)/merra2.aerclim.2003-2014.m02.nc, $(DATA)/aeroclim.m02.nc] + - [$(FIX_aer)/merra2.aerclim.2003-2014.m03.nc, $(DATA)/aeroclim.m03.nc] + - [$(FIX_aer)/merra2.aerclim.2003-2014.m04.nc, $(DATA)/aeroclim.m04.nc] + - [$(FIX_aer)/merra2.aerclim.2003-2014.m05.nc, $(DATA)/aeroclim.m05.nc] + - [$(FIX_aer)/merra2.aerclim.2003-2014.m06.nc, $(DATA)/aeroclim.m06.nc] + - [$(FIX_aer)/merra2.aerclim.2003-2014.m07.nc, $(DATA)/aeroclim.m07.nc] + - [$(FIX_aer)/merra2.aerclim.2003-2014.m08.nc, $(DATA)/aeroclim.m08.nc] + - [$(FIX_aer)/merra2.aerclim.2003-2014.m09.nc, $(DATA)/aeroclim.m09.nc] + - [$(FIX_aer)/merra2.aerclim.2003-2014.m10.nc, $(DATA)/aeroclim.m10.nc] + - [$(FIX_aer)/merra2.aerclim.2003-2014.m11.nc, $(DATA)/aeroclim.m11.nc] + - [$(FIX_aer)/merra2.aerclim.2003-2014.m12.nc, $(DATA)/aeroclim.m12.nc] + + # Optical depth + - [$(FIX_lut)/optics_BC.v1_3.dat, $(DATA)/optics_BC.dat] + - [$(FIX_lut)/optics_DU.v15_3.dat, $(DATA)/optics_DU.dat] + - [$(FIX_lut)/optics_OC.v1_3.dat, $(DATA)/optics_OC.dat] + - [$(FIX_lut)/optics_SS.v3_3.dat, $(DATA)/optics_SS.dat] + - [$(FIX_lut)/optics_SU.v1_3.dat, $(DATA)/optics_SU.dat] + + # fd_nems.yaml file + - [$(HOMEgfs)/sorc/ufs_model.fd/tests/parm/fd_nems.yaml, $(DATA)/] diff --git a/parm/ufs/fix/gfs/land.fixed_files.yaml b/parm/ufs/fix/gfs/land.fixed_files.yaml new file mode 100644 index 0000000000..74813ebc6e --- /dev/null +++ b/parm/ufs/fix/gfs/land.fixed_files.yaml @@ -0,0 +1,64 @@ +dirtree: + mkdir: + - $(DATA)/INPUT + - $(DATA)/RESTART + +fixedfiles: + copy: + + # Files from FIX_orog/C??.mx??_frac/fix_sfc + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).facsf.tile1.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).facsf.tile2.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).facsf.tile3.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).facsf.tile4.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).facsf.tile5.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).facsf.tile6.nc, $(DATA)/] + + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).maximum_snow_albedo.tile1.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).maximum_snow_albedo.tile2.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).maximum_snow_albedo.tile3.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).maximum_snow_albedo.tile4.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).maximum_snow_albedo.tile5.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).maximum_snow_albedo.tile6.nc, $(DATA)/] + + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).slope_type.tile1.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).slope_type.tile2.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).slope_type.tile3.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).slope_type.tile4.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).slope_type.tile5.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).slope_type.tile6.nc, $(DATA)/] + + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).snowfree_albedo.tile1.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).snowfree_albedo.tile2.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).snowfree_albedo.tile3.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).snowfree_albedo.tile4.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).snowfree_albedo.tile5.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).snowfree_albedo.tile6.nc, $(DATA)/] + + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).soil_type.tile1.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).soil_type.tile2.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).soil_type.tile3.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).soil_type.tile4.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).soil_type.tile5.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).soil_type.tile6.nc, $(DATA)/] + + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).substrate_temperature.tile1.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).substrate_temperature.tile2.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).substrate_temperature.tile3.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).substrate_temperature.tile4.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).substrate_temperature.tile5.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).substrate_temperature.tile6.nc, $(DATA)/] + + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).vegetation_greenness.tile1.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).vegetation_greenness.tile2.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).vegetation_greenness.tile3.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).vegetation_greenness.tile4.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).vegetation_greenness.tile5.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).vegetation_greenness.tile6.nc, $(DATA)/] + + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).vegetation_type.tile1.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).vegetation_type.tile2.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).vegetation_type.tile3.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).vegetation_type.tile4.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).vegetation_type.tile5.nc, $(DATA)/] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/fix_sfc/$(atm_res).vegetation_type.tile6.nc, $(DATA)/] diff --git a/parm/ufs/fix/gfs/ocean.fixed_files.yaml b/parm/ufs/fix/gfs/ocean.fixed_files.yaml new file mode 100644 index 0000000000..801f070c49 --- /dev/null +++ b/parm/ufs/fix/gfs/ocean.fixed_files.yaml @@ -0,0 +1,10 @@ +copy: + + # Orography data tile files + # The following are for "frac_grid = .true." + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/oro_$(atm_res).mx$(ocn_res).tile1.nc, $(DATA)/INPUT/oro_data.tile1.nc] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/oro_$(atm_res).mx$(ocn_res).tile2.nc, $(DATA)/INPUT/oro_data.tile2.nc] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/oro_$(atm_res).mx$(ocn_res).tile3.nc, $(DATA)/INPUT/oro_data.tile3.nc] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/oro_$(atm_res).mx$(ocn_res).tile4.nc, $(DATA)/INPUT/oro_data.tile4.nc] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/oro_$(atm_res).mx$(ocn_res).tile5.nc, $(DATA)/INPUT/oro_data.tile5.nc] + - [$(FIX_orog)/$(atm_res).mx$(ocn_res)_frac/oro_$(atm_res).mx$(ocn_res).tile6.nc, $(DATA)/INPUT/oro_data.tile6.nc] From 8dcfaa6fbc5e0a94d44952f710f77c3b18ffa50d Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Mon, 17 Apr 2023 13:37:26 -0400 Subject: [PATCH 4/5] Split MPMD stdout into tasks on slurm (#1469) It can be difficult to debug MPMD jobs because their logs are all written concurrently to a single file. While the use of tags to designate which task via the preamble and PS4 can help identify which line is from which task, it is still difficult to follow a single task through the log, particularly for larger MPMD jobs with dozens of tasks. Individual stdout files are now created by using the `srun` `--output` option. These files are written to the working directory (in `$DATA`). Fixes: #1468 --- env/HERA.env | 2 +- env/JET.env | 2 +- env/ORION.env | 2 +- env/S4.env | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/env/HERA.env b/env/HERA.env index a42e3a0170..3fa6288200 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -17,7 +17,7 @@ step=$1 export npe_node_max=40 export launcher="srun -l --export=ALL" -export mpmd_opt="--multi-prog" +export mpmd_opt="--multi-prog --output=${step}.%J.%t.out" # Configure MPI environment #export I_MPI_ADJUST_ALLREDUCE=5 diff --git a/env/JET.env b/env/JET.env index 4035e8414c..66d9ed9a3b 100755 --- a/env/JET.env +++ b/env/JET.env @@ -20,7 +20,7 @@ elif [[ "${PARTITION_BATCH}" = "vjet" ]]; then export npe_node_max=16 fi export launcher="srun -l --export=ALL" -export mpmd_opt="--multi-prog" +export mpmd_opt="--multi-prog --output=${step}.%J.%t.out" # Configure STACK export OMP_STACKSIZE=2048000 diff --git a/env/ORION.env b/env/ORION.env index 04b1344609..43aa24689d 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -17,7 +17,7 @@ step=$1 export npe_node_max=40 export launcher="srun -l --export=ALL" -export mpmd_opt="--multi-prog" +export mpmd_opt="--multi-prog --output=${step}.%J.%t.out" # Configure MPI environment export MPI_BUFS_PER_PROC=2048 diff --git a/env/S4.env b/env/S4.env index e48cde8505..4933fb989e 100755 --- a/env/S4.env +++ b/env/S4.env @@ -22,7 +22,7 @@ elif [[ ${PARTITION_BATCH} = "ivy" ]]; then export npe_node_max=20 fi export launcher="srun -l --export=ALL" -export mpmd_opt="--multi-prog" +export mpmd_opt="--multi-prog --output=${step}.%J.%t.out" # Configure MPI environment export OMP_STACKSIZE=2048000 From 15ebec8a7adfb0011496a343228b06403cefe299 Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Tue, 18 Apr 2023 10:03:27 -0600 Subject: [PATCH 5/5] Bringing branch UTD prior to rebase. --- jobs/JGLOBAL_FORECAST | 68 ++++++++++++++++++++++++++ jobs/rocoto/fcst.sh | 54 +++++++++++++++++++++ ush/module-setup.sh | 109 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 231 insertions(+) create mode 100755 jobs/JGLOBAL_FORECAST create mode 100755 jobs/rocoto/fcst.sh create mode 100755 ush/module-setup.sh diff --git a/jobs/JGLOBAL_FORECAST b/jobs/JGLOBAL_FORECAST new file mode 100755 index 0000000000..9a2419654f --- /dev/null +++ b/jobs/JGLOBAL_FORECAST @@ -0,0 +1,68 @@ +#! /usr/bin/env bash + +#source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/preamble.testing.sh" # HRW +source "${HOMEgfs}/ush/jjob_header.sh" -e "fcst" -c "base fcst" + +############################################## +# Set variables used in the script +############################################## +# Set wave variables +if [ ${DO_WAVE:-"NO"} = "YES" ]; then + # WAVE component directory + export CDUMPwave=${CDUMPwave:-${CDUMP}wave} + export COMINwave=${COMINwave:-$(compath.py ${envir}/${NET}/${gfs_ver})/${CDUMP}.${PDY}/${cyc}/wave} + export COMOUTwave=${COMOUTwave:-$(compath.py -o ${NET}/${gfs_ver})/${CDUMP}.${PDY}/${cyc}/wave} +fi + +############################################## +# Begin JOB SPECIFIC work +############################################## + +# Restart conditions for GFS cycle come from GDAS +rCDUMP=${CDUMP} +[[ ${CDUMP} = "gfs" ]] && export rCDUMP="gdas" + +# Forecast length for GFS forecast +if [ ${CDUMP} = "gfs" ]; then + export FHMAX=${FHMAX_GFS} + export FHOUT=${FHOUT_GFS} + export FHMAX_HF=${FHMAX_HF_GFS} + export FHOUT_HF=${FHOUT_HF_GFS} +else + export FHMAX_HF=0 + export FHOUT_HF=0 +fi + + +############################################################### +# Run relevant exglobal script + +# ${FORECASTSH:-${SCRgfs}/exglobal_forecast.sh} # HRW +python ${SCRgfs}/exglobal_forecast.py # HRW + +exit 9999 # HRW + +status=$? +[[ ${status} -ne 0 ]] && exit ${status} + + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [ -e "${pgmout}" ] ; then + cat ${pgmout} +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd ${DATAROOT} +[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} + + +exit 0 diff --git a/jobs/rocoto/fcst.sh b/jobs/rocoto/fcst.sh new file mode 100755 index 0000000000..61d67f09e4 --- /dev/null +++ b/jobs/rocoto/fcst.sh @@ -0,0 +1,54 @@ +#! /usr/bin/env bash + +#source "${HOMEgfs}/ush/preamble.sh" # HRW +source "${HOMEgfs}/ush/preamble.testing.sh" # HRW + +############################################################### +# Source FV3GFS workflow modules +#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +#status=$? +#[[ ${status} -ne 0 ]] && exit ${status} + +# TODO: clean this up +#source "${HOMEgfs}/ush/detect_machine.sh" # HRW +#set +x # HRW +#source "${HOMEgfs}/ush/module-setup.sh" # HRW +#module use "${HOMEgfs}/sorc/ufs_model.fd/tests" # HRW +#module load modules.ufs_model.lua # HRW +#module load prod_util # HRW +if [[ "${MACHINE_ID}" = "wcoss2" ]]; then + module load cray-pals +fi +#if [[ "${MACHINE_ID}" = "hera" ]]; then # HRW +# module use "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/modulefiles/core" # HRW +# module load "miniconda3/4.6.14" # HRW +# module load "gfs_workflow/1.0.0" # HRW +# TODO: orion and wcoss2 will be uncommented when they are ready. This comment block will be removed in the next PR +#elif [[ "${MACHINE_ID}" = "orion" ]]; then +# module use "/home/rmahajan/opt/global-workflow/modulefiles/core" +# module load "python/3.7.5" +# module load "gfs_workflow/1.0.0" +#elif [[ "${MACHINE_ID}" = "wcoss2" ]]; then +# module load "python/3.7.5" +#fi # HRW +#module list # HRW +# unset MACHINE_ID # HRW +# set_trace # HRW + +############################################################### +# exglobal_forecast.py requires the following in PYTHONPATH +# This will be moved to a module load when ready +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src:${HOMEgfs}/ush/python/pygfs" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + +export job="fcst" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +${HOMEgfs}/jobs/JGLOBAL_FORECAST +status=$? + + +exit ${status} diff --git a/ush/module-setup.sh b/ush/module-setup.sh new file mode 100755 index 0000000000..a43a8db34a --- /dev/null +++ b/ush/module-setup.sh @@ -0,0 +1,109 @@ +#!/bin/bash +set -u + +exit 999 # HRW + +if [[ ${MACHINE_ID} = jet* ]] ; then + # We are on NOAA Jet + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /apps/lmod/lmod/init/bash + fi + export LMOD_SYSTEM_DEFAULT_MODULES=contrib + module reset + +elif [[ ${MACHINE_ID} = hera* ]] ; then + # We are on NOAA Hera + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /apps/lmod/lmod/init/bash + fi + export LMOD_SYSTEM_DEFAULT_MODULES=contrib + module reset + +elif [[ ${MACHINE_ID} = orion* ]] ; then + # We are on Orion + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /apps/lmod/init/bash + fi + export LMOD_SYSTEM_DEFAULT_MODULES=contrib + module reset + +elif [[ ${MACHINE_ID} = s4* ]] ; then + # We are on SSEC Wisconsin S4 + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /usr/share/lmod/lmod/init/bash + fi + export LMOD_SYSTEM_DEFAULT_MODULES=license_intel + module reset + +elif [[ ${MACHINE_ID} = wcoss2 ]]; then + # We are on WCOSS2 + module reset + +elif [[ ${MACHINE_ID} = cheyenne* ]] ; then + # We are on NCAR Cheyenne + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /glade/u/apps/ch/modulefiles/default/localinit/localinit.sh + fi + module purge + +elif [[ ${MACHINE_ID} = stampede* ]] ; then + # We are on TACC Stampede + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /opt/apps/lmod/lmod/init/bash + fi + module purge + +elif [[ ${MACHINE_ID} = gaea* ]] ; then + # We are on GAEA. + if ( ! eval module help > /dev/null 2>&1 ) ; then + # We cannot simply load the module command. The GAEA + # /etc/profile modifies a number of module-related variables + # before loading the module command. Without those variables, + # the module command fails. Hence we actually have to source + # /etc/profile here. + source /etc/profile + __ms_source_etc_profile=yes + else + __ms_source_etc_profile=no + fi + module purge + # clean up after purge + unset _LMFILES_ + unset _LMFILES_000 + unset _LMFILES_001 + unset LOADEDMODULES + module load modules + if [[ -d /opt/cray/ari/modulefiles ]] ; then + module use -a /opt/cray/ari/modulefiles + fi + if [[ -d /opt/cray/pe/ari/modulefiles ]] ; then + module use -a /opt/cray/pe/ari/modulefiles + fi + if [[ -d /opt/cray/pe/craype/default/modulefiles ]] ; then + module use -a /opt/cray/pe/craype/default/modulefiles + fi + if [[ -s /etc/opt/cray/pe/admin-pe/site-config ]] ; then + source /etc/opt/cray/pe/admin-pe/site-config + fi + if [[ "${__ms_source_etc_profile}" == yes ]] ; then + source /etc/profile + unset __ms_source_etc_profile + fi + +elif [[ ${MACHINE_ID} = expanse* ]]; then + # We are on SDSC Expanse + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /etc/profile.d/modules.sh + fi + module purge + module load slurm/expanse/20.02.3 + +elif [[ ${MACHINE_ID} = discover* ]]; then + # We are on NCCS discover + export SPACK_ROOT=/discover/nobackup/mapotts1/spack + export PATH=${PATH}:${SPACK_ROOT}/bin + . "${SPACK_ROOT}"/share/spack/setup-env.sh + +else + echo WARNING: UNKNOWN PLATFORM 1>&2 +fi