From f58cd80db4521caa520bb47d1ee366f7c11df90a Mon Sep 17 00:00:00 2001 From: David Burrows Date: Fri, 30 Aug 2024 08:42:23 -0400 Subject: [PATCH 1/3] update config/version files to run CI test C96_atm3DVar --- modulefiles/module_base.gaea.lua | 9 +++++++++ versions/run.gaea.ver | 2 ++ workflow/hosts/gaea.yaml | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/modulefiles/module_base.gaea.lua b/modulefiles/module_base.gaea.lua index 55ad6b0c34..9496f31319 100644 --- a/modulefiles/module_base.gaea.lua +++ b/modulefiles/module_base.gaea.lua @@ -15,6 +15,7 @@ load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) +load(pathJoin("perlbrew", (os.getenv("perl_ver") or "None"))) load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) @@ -25,6 +26,7 @@ load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) +load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) @@ -36,4 +38,11 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) +prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/scratch/David.Burrows/aug28/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) + +prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/scratch/David.Burrows/aug28/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) + + whatis("Description: GFS run setup environment") diff --git a/versions/run.gaea.ver b/versions/run.gaea.ver index b92fe8c1db..c3aceb445d 100644 --- a/versions/run.gaea.ver +++ b/versions/run.gaea.ver @@ -2,5 +2,7 @@ export stack_intel_ver=2023.1.0 export stack_cray_mpich_ver=8.1.25 export spack_env=gsi-addon-dev +export perl_ver=5.38.2 + source "${HOMEgfs:-}/versions/run.spack.ver" export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/workflow/hosts/gaea.yaml b/workflow/hosts/gaea.yaml index 9297fed24a..79d58f6ba5 100644 --- a/workflow/hosts/gaea.yaml +++ b/workflow/hosts/gaea.yaml @@ -1,4 +1,4 @@ -BASE_GIT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git' +BASE_GIT: '/gpfs/f5/ufs-ard/scratch/David.Burrows/aug28/git' DMPDIR: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/dump' BASE_IC: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/ICSDIR' PACKAGEROOT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/nwpara' From 90ffb20ccb64a0f6d382d3fa4ab5586acc46341a Mon Sep 17 00:00:00 2001 From: David Burrows Date: Tue, 3 Sep 2024 08:20:25 -0400 Subject: [PATCH 2/3] update final git paths; trim GAEA.env to only running jobs --- env/GAEA.env | 47 ++++++++++++++++++++++++++++---- modulefiles/module_base.gaea.lua | 4 +-- workflow/hosts/gaea.yaml | 2 +- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/env/GAEA.env b/env/GAEA.env index 6809a9b186..be5e9f0ca7 100755 --- a/env/GAEA.env +++ b/env/GAEA.env @@ -34,12 +34,42 @@ else exit 2 fi -if [[ "${step}" = "waveinit" ]]; then +if [[ "${step}" = "prep" ]]; then - export CFP_MP="YES" - if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi - export wavempexec=${launcher} - export wave_mpmd=${mpmd_opt} + export POE="NO" + export BACK="NO" + export sys_tp="GAEA" + export launcher_PREP="srun" + +elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then + + export MKL_NUM_THREADS=4 + export MKL_CBWR=AUTO + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + export NTHREADS_GSI=${NTHREADSmax} + export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}" + + export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} + [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} + export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}" + + export NTHREADS_CYCLE=${threads_per_task_cycle:-12} + [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} + ntasks_cycle=${ntiles:-6} + export APRUN_CYCLE="${launcher} -n ${ntasks_cycle} --cpus-per-task=${NTHREADS_CYCLE}" + + export NTHREADS_GAUSFCANL=1 + ntasks_gausfcanl=${ntasks_gausfcanl:-1} + export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" + +elif [[ "${step}" = "sfcanl" ]]; then + + export NTHREADS_CYCLE=${threads_per_task:-14} + export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}" elif [[ "${step}" = "fcst" ]]; then @@ -49,9 +79,14 @@ elif [[ "${step}" = "fcst" ]]; then export APRUN_UFS="${launcher} -n ${ufs_ntasks}" unset nnodes ufs_ntasks +elif [[ "${step}" = "upp" ]]; then + + export NTHREADS_UPP=${NTHREADS1} + export APRUN_UPP="${APRUN} --cpus-per-task=${NTHREADS_UPP}" + elif [[ "${step}" = "atmos_products" ]]; then - export USE_CFP="YES" # Use MPMD for downstream product generation on Hera + export USE_CFP="YES" # Use MPMD for downstream product generation on Gaea elif [[ "${step}" = "oceanice_products" ]]; then diff --git a/modulefiles/module_base.gaea.lua b/modulefiles/module_base.gaea.lua index 9496f31319..f379225380 100644 --- a/modulefiles/module_base.gaea.lua +++ b/modulefiles/module_base.gaea.lua @@ -38,10 +38,10 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) -prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/scratch/David.Burrows/aug28/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) -prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/scratch/David.Burrows/aug28/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) diff --git a/workflow/hosts/gaea.yaml b/workflow/hosts/gaea.yaml index 79d58f6ba5..9297fed24a 100644 --- a/workflow/hosts/gaea.yaml +++ b/workflow/hosts/gaea.yaml @@ -1,4 +1,4 @@ -BASE_GIT: '/gpfs/f5/ufs-ard/scratch/David.Burrows/aug28/git' +BASE_GIT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git' DMPDIR: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/dump' BASE_IC: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/ICSDIR' PACKAGEROOT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/nwpara' From 9b4c012e672f0e12bb64d75aaa3b59e1dbf43cb9 Mon Sep 17 00:00:00 2001 From: David Burrows Date: Tue, 3 Sep 2024 14:59:22 -0400 Subject: [PATCH 3/3] update prepobs version number for Gaea --- modulefiles/module_base.gaea.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modulefiles/module_base.gaea.lua b/modulefiles/module_base.gaea.lua index f379225380..b08e79c274 100644 --- a/modulefiles/module_base.gaea.lua +++ b/modulefiles/module_base.gaea.lua @@ -38,8 +38,10 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) -prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) -load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) +--prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +--load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) +prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v1.1.0", "modulefiles")) +load(pathJoin("prepobs", "1.1.0")) prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None")))