Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[develop] Adding Hercules as a Tier-1 platform #911

Merged
merged 10 commits into from
Sep 25, 2023
Merged

[develop] Adding Hercules as a Tier-1 platform #911

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
2b080df
Adding Hercules as a Tier-1 platform
Sep 12, 2023
5c14fdb
update build_hercules_intel.lua
Sep 15, 2023
42fbd5f
merge with develop
Sep 15, 2023
9dde8a6
update modulefile and machine file for Hercules
Sep 17, 2023
ac5e9b2
add comprehensive.hercules test suite
Sep 17, 2023
21ca4ae
update standard data locations in hercules.yaml
Sep 17, 2023
dcb6ef1
Merge branch 'develop' into develop_hercules
natalie-perlin Sep 19, 2023
bc6e479
Rename coverage.cheyenne.gnu to coverage.hercules
natalie-perlin Sep 21, 2023
48f3d39
Update build.sh
natalie-perlin Sep 21, 2023
01e852c
Merge branch 'ufs-community:develop' into develop_hercules
natalie-perlin Sep 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions .cicd/Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ pipeline {
parameters {
// Allow job runner to filter based on platform
// Use the line below to enable all PW clusters
// choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
// choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
// Use the line below to enable the PW AWS cluster
// choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
// choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use')
choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use')
// choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
// choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use')
choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use')
// Allow job runner to filter based on compiler
choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
// Uncomment the following line to re-enable comprehensive tests
Expand Down Expand Up @@ -77,8 +77,8 @@ pipeline {
axes {
axis {
name 'SRW_PLATFORM'
// values 'cheyenne', 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
values 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
// values 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules'//, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
values 'gaea', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
}

axis {
Expand All @@ -92,7 +92,7 @@ pipeline {
exclude {
axis {
name 'SRW_PLATFORM'
values 'gaea', 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
values 'gaea', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
}

axis {
Expand Down
2 changes: 1 addition & 1 deletion etc/lmod-setup.csh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ Usage: source etc/lmod-setup.csh PLATFORM

OPTIONS:
PLATFORM - name of machine you are building on
(e.g. cheyenne | hera | jet | orion | wcoss2 )
(e.g. cheyenne | hera | jet | orion | hercules | wcoss2 )
EOF_USAGE
exit 1
else
Expand Down
2 changes: 1 addition & 1 deletion etc/lmod-setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ Usage: source etc/lmod-setup.sh PLATFORM

OPTIONS:
PLATFORM - name of machine you are building on
(e.g. cheyenne | hera | jet | orion | wcoss2 )
(e.g. cheyenne | hera | jet | orion | hercules | wcoss2 )
EOF_USAGE
exit 1
else
Expand Down
30 changes: 30 additions & 0 deletions modulefiles/build_hercules_intel.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
help([[
This module loads libraries for building the UFS SRW App on
the MSU machine Hercules using intel-oneapi-compilers/2022.2.1
]])

whatis([===[Loads libraries needed for building the UFS SRW App on Orion ]===])

load("contrib")
load("noaatools")

load(pathJoin("cmake", os.getenv("cmake_ver") or "3.26.3"))

prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/hpc-stack/intel-2022.2.1/modulefiles/stack")
load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
load(pathJoin("hpc-intel-oneapi-compilers", os.getenv("hpc_intel_ver") or "2022.2.1"))
load(pathJoin("hpc-intel-oneapi-mpi", os.getenv("hpc_mpi_ver") or "2021.7.1"))

load("srw_common")

load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.8.9.0"))
load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))

setenv("CFLAGS","-diag-disable=10441")
setenv("FFLAGS","-diag-disable=10441")

setenv("CMAKE_C_COMPILER","mpiicc")
setenv("CMAKE_CXX_COMPILER","mpiicpc")
setenv("CMAKE_Fortran_COMPILER","mpiifort")
setenv("CMAKE_Platform","hercules.intel")

4 changes: 2 additions & 2 deletions modulefiles/srw_common.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ load("libpng/1.6.37")

load_any("netcdf/4.9.2", "netcdf-c/4.9.2")
load_any("netcdf/4.9.2", "netcdf-fortran/4.6.0")
load_any("pio/2.5.10","parallelio/2.5.10")
load_any("pio/2.5.10", "parallelio/2.5.9")
load("esmf/8.4.2")
load("fms/2023.01")

load("bacio/2.4.1")
load("crtm/2.4.0")
load("g2/3.4.5")
load("crtm/2.4.0")
load("g2tmpl/1.10.2")
load("ip/3.3.3")
load("sp/2.3.3")
Expand Down
2 changes: 2 additions & 0 deletions modulefiles/tasks/hercules/aqm_ics.local.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
load("miniconda_regional_workflow_cmaq")
1 change: 1 addition & 0 deletions modulefiles/tasks/hercules/aqm_lbcs.local.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
2 changes: 2 additions & 0 deletions modulefiles/tasks/hercules/fire_emission.local.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
load("miniconda_regional_workflow_cmaq")
4 changes: 4 additions & 0 deletions modulefiles/tasks/hercules/miniconda_regional_workflow_cmaq.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles")
load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))

setenv("SRW_ENV", "regional_workflow_cmaq")
2 changes: 2 additions & 0 deletions modulefiles/tasks/hercules/nexus_emission.local.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
load("miniconda_regional_workflow_cmaq")
2 changes: 2 additions & 0 deletions modulefiles/tasks/hercules/nexus_post_split.local.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
load("miniconda_regional_workflow_cmaq")
5 changes: 5 additions & 0 deletions modulefiles/tasks/hercules/plot_allvars.local.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
unload("python")
append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles")
load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))

setenv("SRW_ENV", "regional_workflow")
1 change: 1 addition & 0 deletions modulefiles/tasks/hercules/point_source.local.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
load("miniconda_regional_workflow_cmaq")
5 changes: 5 additions & 0 deletions modulefiles/tasks/hercules/python_srw.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
unload("python")
append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles")
load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))

setenv("SRW_ENV", "workflow_tools")
6 changes: 6 additions & 0 deletions modulefiles/tasks/hercules/run_vx.local.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
--[[
Compiler-specific modules are used for met and metplus libraries
--]]
load(pathJoin("met", os.getenv("met_ver") or "10.1.2"))
load(pathJoin("metplus", os.getenv("metplus_ver") or "4.1.3"))
load("python_srw")
21 changes: 21 additions & 0 deletions modulefiles/wflow_hercules.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
help([[
This module loads python environement for running SRW on
the MSU machine Hercules
]])

whatis([===[Loads libraries needed for running SRW on Hercules ]===])

load("contrib")
load("rocoto")
load("set_pythonpath")

unload("python")
append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles")
load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))

if mode() == "load" then
LmodMsgRaw([===[Please do the following to activate conda:
> conda activate workflow_tools
]===])
end

1 change: 1 addition & 0 deletions tests/WE2E/machine_suites/comprehensive.hercules
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
comprehensive.orion
0 ...WE2E/machine_suites/coverage.cheyenne.gnu → tests/WE2E/machine_suites/coverage.hercules
View file
Open in desktop
File renamed without changes.
2 changes: 1 addition & 1 deletion tests/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ function usage() {
exit 1
}

machines=( hera jet cheyenne derecho orion wcoss2 gaea odin singularity macos noaacloud )
machines=( hera jet cheyenne derecho orion hercules wcoss2 gaea odin singularity macos noaacloud )

[[ $# -gt 4 ]] && usage

Expand Down
45 changes: 45 additions & 0 deletions ush/machine/hercules.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
platform:
WORKFLOW_MANAGER: rocoto
NCORES_PER_NODE: 80
SCHED: slurm
TEST_CCPA_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc
TEST_MRMS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc
TEST_NDAS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc
DOMAIN_PREGEN_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen
PARTITION_DEFAULT: hercules
QUEUE_DEFAULT: batch
PARTITION_FCST: hercules
QUEUE_FCST: batch
PARTITION_HPSS: service
QUEUE_HPSS: batch
RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01}
RUN_CMD_POST: srun --export=ALL
RUN_CMD_PRDGEN: srun --export=ALL
RUN_CMD_SERIAL: time
RUN_CMD_UTILS: srun --export=ALL -n $nprocs
RUN_CMD_NEXUS: srun --export=ALL
RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS}
SCHED_NATIVE_CMD: --export=NONE
PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data
TEST_AQM_INPUT_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/aqm_data
TEST_PREGEN_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen
TEST_VX_FCST_INPUT_BASEDIR: '{{ "/work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/output_data/fcst_" }}{{ "ens" if (global.NUM_ENS_MEMBERS > 0) else "det" }}{{ "/{{workflow.PREDEF_GRID_NAME}}" }}{% raw %}{% endraw %}'
FIXaer: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_aer
FIXgsi: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_gsi
FIXgsm: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_am
FIXlut: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_lut
FIXorg: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_orog
FIXsfc: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo
FIXshp: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/NaturalEarth
EXTRN_MDL_DATA_STORES: aws
data:
ics_lbcs:
FV3GFS:
nemsio: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/nemsio/${yyyymmdd}${hh}
grib2: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/${yyyymmdd}${hh}
netcdf: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/netcdf/${yyyymmdd}${hh}
NAM: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/NAM/${yyyymmdd}${hh}
HRRR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh}
RAP: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh}
GSMGFS: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh}
2 changes: 1 addition & 1 deletion ush/valid_param_vals.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
valid_vals_RUN_ENVIR: ["nco", "community"]
valid_vals_VERBOSE: [True, False]
valid_vals_DEBUG: [True, False]
valid_vals_MACHINE: ["HERA", "WCOSS2", "ORION", "JET", "ODIN", "CHEYENNE", "DERECHO", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"]
valid_vals_MACHINE: ["HERA", "WCOSS2", "ORION", "HERCULES", "JET", "ODIN", "CHEYENNE", "DERECHO", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"]
valid_vals_SCHED: ["slurm", "pbspro", "lsf", "lsfcray", "none"]
valid_vals_FCST_MODEL: ["ufs-weather-model"]
valid_vals_WORKFLOW_MANAGER: ["rocoto", "ecflow", "none"]
Expand Down