diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 007815edd2..660423c3c3 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -10,11 +10,11 @@ pipeline { parameters { // Allow job runner to filter based on platform // Use the line below to enable all PW clusters - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') // Use the line below to enable the PW AWS cluster - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use') - choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') + choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') // Allow job runner to filter based on compiler choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') // Uncomment the following line to re-enable comprehensive tests @@ -77,8 +77,8 @@ pipeline { axes { axis { name 'SRW_PLATFORM' - // values 'cheyenne', 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' - values 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + // values 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules'//, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'gaea', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { @@ -92,7 +92,7 @@ pipeline { exclude { axis { name 'SRW_PLATFORM' - values 'gaea', 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'gaea', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index fd1f7f8c9c..4be4659c87 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -6,7 +6,7 @@ Usage: source etc/lmod-setup.csh PLATFORM OPTIONS: PLATFORM - name of machine you are building on - (e.g. cheyenne | hera | jet | orion | wcoss2 ) + (e.g. cheyenne | hera | jet | orion | hercules | wcoss2 ) EOF_USAGE exit 1 else diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index 0c922abcdb..4c89d631d7 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -7,7 +7,7 @@ Usage: source etc/lmod-setup.sh PLATFORM OPTIONS: PLATFORM - name of machine you are building on - (e.g. cheyenne | hera | jet | orion | wcoss2 ) + (e.g. cheyenne | hera | jet | orion | hercules | wcoss2 ) EOF_USAGE exit 1 else diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua new file mode 100644 index 0000000000..1073952dc5 --- /dev/null +++ b/modulefiles/build_hercules_intel.lua @@ -0,0 +1,30 @@ +help([[ +This module loads libraries for building the UFS SRW App on +the MSU machine Hercules using intel-oneapi-compilers/2022.2.1 +]]) + +whatis([===[Loads libraries needed for building the UFS SRW App on Orion ]===]) + +load("contrib") +load("noaatools") + +load(pathJoin("cmake", os.getenv("cmake_ver") or "3.26.3")) + +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/hpc-stack/intel-2022.2.1/modulefiles/stack") +load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) +load(pathJoin("hpc-intel-oneapi-compilers", os.getenv("hpc_intel_ver") or "2022.2.1")) +load(pathJoin("hpc-intel-oneapi-mpi", os.getenv("hpc_mpi_ver") or "2021.7.1")) + +load("srw_common") + +load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.8.9.0")) +load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6")) + +setenv("CFLAGS","-diag-disable=10441") +setenv("FFLAGS","-diag-disable=10441") + +setenv("CMAKE_C_COMPILER","mpiicc") +setenv("CMAKE_CXX_COMPILER","mpiicpc") +setenv("CMAKE_Fortran_COMPILER","mpiifort") +setenv("CMAKE_Platform","hercules.intel") + diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua index 54b644e216..f68b4891e0 100644 --- a/modulefiles/srw_common.lua +++ b/modulefiles/srw_common.lua @@ -4,13 +4,13 @@ load("libpng/1.6.37") load_any("netcdf/4.9.2", "netcdf-c/4.9.2") load_any("netcdf/4.9.2", "netcdf-fortran/4.6.0") -load_any("pio/2.5.10","parallelio/2.5.10") +load_any("pio/2.5.10", "parallelio/2.5.9") load("esmf/8.4.2") load("fms/2023.01") load("bacio/2.4.1") -load("crtm/2.4.0") load("g2/3.4.5") +load("crtm/2.4.0") load("g2tmpl/1.10.2") load("ip/3.3.3") load("sp/2.3.3") diff --git a/modulefiles/tasks/hercules/aqm_ics.local.lua b/modulefiles/tasks/hercules/aqm_ics.local.lua new file mode 100644 index 0000000000..5a7b0cece6 --- /dev/null +++ b/modulefiles/tasks/hercules/aqm_ics.local.lua @@ -0,0 +1,2 @@ +load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/hercules/aqm_lbcs.local.lua b/modulefiles/tasks/hercules/aqm_lbcs.local.lua new file mode 100644 index 0000000000..23370a8d60 --- /dev/null +++ b/modulefiles/tasks/hercules/aqm_lbcs.local.lua @@ -0,0 +1 @@ +load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) diff --git a/modulefiles/tasks/hercules/fire_emission.local.lua b/modulefiles/tasks/hercules/fire_emission.local.lua new file mode 100644 index 0000000000..5a7b0cece6 --- /dev/null +++ b/modulefiles/tasks/hercules/fire_emission.local.lua @@ -0,0 +1,2 @@ +load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/hercules/miniconda_regional_workflow_cmaq.lua b/modulefiles/tasks/hercules/miniconda_regional_workflow_cmaq.lua new file mode 100644 index 0000000000..c785373386 --- /dev/null +++ b/modulefiles/tasks/hercules/miniconda_regional_workflow_cmaq.lua @@ -0,0 +1,4 @@ +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow_cmaq") diff --git a/modulefiles/tasks/hercules/nexus_emission.local.lua b/modulefiles/tasks/hercules/nexus_emission.local.lua new file mode 100644 index 0000000000..5a7b0cece6 --- /dev/null +++ b/modulefiles/tasks/hercules/nexus_emission.local.lua @@ -0,0 +1,2 @@ +load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/hercules/nexus_post_split.local.lua b/modulefiles/tasks/hercules/nexus_post_split.local.lua new file mode 100644 index 0000000000..5a7b0cece6 --- /dev/null +++ b/modulefiles/tasks/hercules/nexus_post_split.local.lua @@ -0,0 +1,2 @@ +load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3")) +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/hercules/plot_allvars.local.lua b/modulefiles/tasks/hercules/plot_allvars.local.lua new file mode 100644 index 0000000000..c995f83bb8 --- /dev/null +++ b/modulefiles/tasks/hercules/plot_allvars.local.lua @@ -0,0 +1,5 @@ +unload("python") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "regional_workflow") diff --git a/modulefiles/tasks/hercules/point_source.local.lua b/modulefiles/tasks/hercules/point_source.local.lua new file mode 100644 index 0000000000..07cacc8ce2 --- /dev/null +++ b/modulefiles/tasks/hercules/point_source.local.lua @@ -0,0 +1 @@ +load("miniconda_regional_workflow_cmaq") diff --git a/modulefiles/tasks/hercules/python_srw.lua b/modulefiles/tasks/hercules/python_srw.lua new file mode 100644 index 0000000000..157e6884ea --- /dev/null +++ b/modulefiles/tasks/hercules/python_srw.lua @@ -0,0 +1,5 @@ +unload("python") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "workflow_tools") diff --git a/modulefiles/tasks/hercules/run_vx.local.lua b/modulefiles/tasks/hercules/run_vx.local.lua new file mode 100644 index 0000000000..5979a8db96 --- /dev/null +++ b/modulefiles/tasks/hercules/run_vx.local.lua @@ -0,0 +1,6 @@ +--[[ +Compiler-specific modules are used for met and metplus libraries +--]] +load(pathJoin("met", os.getenv("met_ver") or "10.1.2")) +load(pathJoin("metplus", os.getenv("metplus_ver") or "4.1.3")) +load("python_srw") diff --git a/modulefiles/wflow_hercules.lua b/modulefiles/wflow_hercules.lua new file mode 100644 index 0000000000..471b1f8919 --- /dev/null +++ b/modulefiles/wflow_hercules.lua @@ -0,0 +1,21 @@ +help([[ +This module loads python environement for running SRW on +the MSU machine Hercules +]]) + +whatis([===[Loads libraries needed for running SRW on Hercules ]===]) + +load("contrib") +load("rocoto") +load("set_pythonpath") + +unload("python") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +if mode() == "load" then + LmodMsgRaw([===[Please do the following to activate conda: + > conda activate workflow_tools +]===]) +end + diff --git a/tests/WE2E/machine_suites/comprehensive.hercules b/tests/WE2E/machine_suites/comprehensive.hercules new file mode 120000 index 0000000000..18a56d9739 --- /dev/null +++ b/tests/WE2E/machine_suites/comprehensive.hercules @@ -0,0 +1 @@ +comprehensive.orion \ No newline at end of file diff --git a/tests/WE2E/machine_suites/coverage.cheyenne.gnu b/tests/WE2E/machine_suites/coverage.hercules similarity index 100% rename from tests/WE2E/machine_suites/coverage.cheyenne.gnu rename to tests/WE2E/machine_suites/coverage.hercules diff --git a/tests/build.sh b/tests/build.sh index 39c54fa509..c6cbe89c1d 100755 --- a/tests/build.sh +++ b/tests/build.sh @@ -21,7 +21,7 @@ function usage() { exit 1 } -machines=( hera jet cheyenne derecho orion wcoss2 gaea odin singularity macos noaacloud ) +machines=( hera jet cheyenne derecho orion hercules wcoss2 gaea odin singularity macos noaacloud ) [[ $# -gt 4 ]] && usage diff --git a/ush/machine/hercules.yaml b/ush/machine/hercules.yaml new file mode 100644 index 0000000000..33d5732511 --- /dev/null +++ b/ush/machine/hercules.yaml @@ -0,0 +1,45 @@ +platform: + WORKFLOW_MANAGER: rocoto + NCORES_PER_NODE: 80 + SCHED: slurm + TEST_CCPA_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc + DOMAIN_PREGEN_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen + PARTITION_DEFAULT: hercules + QUEUE_DEFAULT: batch + PARTITION_FCST: hercules + QUEUE_FCST: batch + PARTITION_HPSS: service + QUEUE_HPSS: batch + RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} + RUN_CMD_POST: srun --export=ALL + RUN_CMD_PRDGEN: srun --export=ALL + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: srun --export=ALL -n $nprocs + RUN_CMD_NEXUS: srun --export=ALL + RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS} + SCHED_NATIVE_CMD: --export=NONE + PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' + TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data + TEST_AQM_INPUT_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/aqm_data + TEST_PREGEN_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen + TEST_VX_FCST_INPUT_BASEDIR: '{{ "/work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/output_data/fcst_" }}{{ "ens" if (global.NUM_ENS_MEMBERS > 0) else "det" }}{{ "/{{workflow.PREDEF_GRID_NAME}}" }}{% raw %}{% endraw %}' + FIXaer: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_aer + FIXgsi: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_gsi + FIXgsm: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_am + FIXlut: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_lut + FIXorg: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_orog + FIXsfc: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo + FIXshp: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/NaturalEarth + EXTRN_MDL_DATA_STORES: aws +data: + ics_lbcs: + FV3GFS: + nemsio: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/nemsio/${yyyymmdd}${hh} + grib2: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/${yyyymmdd}${hh} + netcdf: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/netcdf/${yyyymmdd}${hh} + NAM: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/NAM/${yyyymmdd}${hh} + HRRR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh} + RAP: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh} + GSMGFS: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh} diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index b3e6ff3426..385c061844 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -4,7 +4,7 @@ valid_vals_RUN_ENVIR: ["nco", "community"] valid_vals_VERBOSE: [True, False] valid_vals_DEBUG: [True, False] -valid_vals_MACHINE: ["HERA", "WCOSS2", "ORION", "JET", "ODIN", "CHEYENNE", "DERECHO", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"] +valid_vals_MACHINE: ["HERA", "WCOSS2", "ORION", "HERCULES", "JET", "ODIN", "CHEYENNE", "DERECHO", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"] valid_vals_SCHED: ["slurm", "pbspro", "lsf", "lsfcray", "none"] valid_vals_FCST_MODEL: ["ufs-weather-model"] valid_vals_WORKFLOW_MANAGER: ["rocoto", "ecflow", "none"]