From 5f461daab2eec5ae6a31ece2f7f146e27fbc36ed Mon Sep 17 00:00:00 2001 From: EdwardSnyder-NOAA <96196752+EdwardSnyder-NOAA@users.noreply.github.com> Date: Thu, 14 Mar 2024 12:23:07 -0500 Subject: [PATCH] [develop] Expand forecast fields for metric test (#1048) This PR expands the number of forecast fields for the Skill Score metric test. The forecast length in the metric WE2E test was extended to 12 hours so that the RMSE metric can be calculated for these additional forecast fields: * Specific humidity for the full column * Temperature for the full column * Wind for the full column * Dew point, pressure, temperature, and wind at the surface level for forecast hour 12. Adding these additional forecast fields will make the skill score metric test more thorough and thus making it a more inclusive test to compare against. Also, a change was made to the .cicd/scripts/srw_metric_example.sh script to reflect the new conda environment. --------- Co-authored-by: Parallel Works app-run user Co-authored-by: Parallel Works app-run user --- .cicd/scripts/srw_metric_example.sh | 139 ++++++++++++------ modulefiles/tasks/gaea/run_vx.local.lua | 4 +- modulefiles/tasks/orion/run_vx.local.lua | 5 +- parm/metplus/STATAnalysisConfig_skill_score | 70 ++++++--- ..._ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0.yaml | 2 +- 5 files changed, 157 insertions(+), 63 deletions(-) diff --git a/.cicd/scripts/srw_metric_example.sh b/.cicd/scripts/srw_metric_example.sh index 2018505735..45dd30c299 100755 --- a/.cicd/scripts/srw_metric_example.sh +++ b/.cicd/scripts/srw_metric_example.sh @@ -3,22 +3,45 @@ # The goal of this script is to provide an example of performing Indy-Severe-Weather test run and compare results to reference with # Skill score index that is calculated by MET Stat-Analysis Tools # -# Required: +# Required (these options are set in the Jenkins env): # WORKSPACE= # SRW_PLATFORM= # SRW_COMPILER= +# SRW_PROJECT= # # Optional: -[[ -n ${SRW_PROJECT} ]] || SRW_PROJECT="no_account" +#[[ -n ${SRW_PROJECT} ]] || SRW_PROJECT="no_account" [[ -n ${FORGIVE_CONDA} ]] || FORGIVE_CONDA=true set -e -u -x +BUILD_OPT=false +RUN_WE2E_OPT=false +RUN_STAT_ANLY_OPT=false + +if [[ $# -eq 0 ]]; then + BUILD_OPT=true + RUN_WE2E_OPT=true + RUN_STAT_ANLY_OPT=true +elif [[ $# -ge 4 ]]; then + echo "Too many arguments, expecting three or less" + exit 1 +else + for opt in "$@"; do + case $opt in + build) BUILD_OPT=true ;; + run_we2e) RUN_WE2E_OPT=true ;; + run_stat_anly) RUN_STAT_ANLY_OPT=true ;; + *) echo "Not valid option. Exiting!" ; exit 1 ;; + esac + done +fi + script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set # relative to script directory. declare workspace -if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then +if [[ -d "${WORKSPACE}/${SRW_PLATFORM}" ]]; then workspace="${WORKSPACE}/${SRW_PLATFORM}" else workspace="$(cd -- "${script_dir}/../.." && pwd)" @@ -39,53 +62,85 @@ we2e_test_name="grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0" pwd -# Activate the workflow environment ... +# Setup the build environment +declare srw_compiler +srw_compiler=${SRW_COMPILER} source etc/lmod-setup.sh ${platform,,} module use modulefiles -module load build_${platform,,}_${SRW_COMPILER} +module load build_${platform,,}_${srw_compiler} + +# Build srw +if [[ ${BUILD_OPT} == true ]]; then + cd ${workspace}/tests + ./build.sh ${platform,,} ${srw_compiler} +fi +cd ${workspace} + +# Activate workflow environment module load wflow_${platform,,} -[[ ${FORGIVE_CONDA} == true ]] && set +e +u # Some platforms have incomplete python3 or conda support, but wouldn't necessarily block workflow tests -conda activate workflow_tools +[[ ${FORGIVE_CONDA} == true ]] && set +e +u # Some platforms have incomplete python3 or conda support, but would not necessarily block workflow tests +conda activate srw_app set -e -u -# build srw -cd ${workspace}/tests -./build.sh ${platform,,} ${SRW_COMPILER} -cd ${workspace} - -# run test -[[ -d ${we2e_experiment_base_dir} ]] && rm -rf ${we2e_experiment_base_dir} -cd ${workspace}/tests/WE2E -./run_WE2E_tests.py -t ${we2e_test_name} -m ${platform,,} -a ${SRW_PROJECT} --expt_basedir "metric_test" --exec_subdir=install_intel/exec -q +# Run test +declare srw_project +srw_project=${SRW_PROJECT} +if [[ ${RUN_WE2E_OPT} == true ]]; then + [[ -d ${we2e_experiment_base_dir} ]] && rm -rf ${we2e_experiment_base_dir} + cd ${workspace}/tests/WE2E + ./run_WE2E_tests.py -t ${we2e_test_name} -m ${platform,,} -a ${srw_project} --expt_basedir "metric_test" --exec_subdir=install_intel/exec -q +fi cd ${workspace} -# run skill-score check -[[ ! -f Indy-Severe-Weather.tgz ]] && wget https://noaa-ufs-srw-pds.s3.amazonaws.com/sample_cases/release-public-v2.1.0/Indy-Severe-Weather.tgz -[[ ! -d Indy-Severe-Weather ]] && tar xvfz Indy-Severe-Weather.tgz -[[ -f skill-score.out ]] && rm skill-score.out -# Skill score index is computed over several terms that are defined in parm/metplus/STATAnalysisConfig_skill_score. -# It is computed by aggregating the output from earlier runs of the Point-Stat and/or Grid-Stat tools over one or more cases. -# In this example, skill score index is a weighted average of 4 skill scores of RMSE statistics for wind speed, dew point temperature, -# temperature, and pressure at lowest level in the atmosphere over 6 hour lead time. -cp ${we2e_experiment_base_dir}/${we2e_test_name}/2019061500/metprd/PointStat/*.stat ${workspace}/Indy-Severe-Weather/metprd/point_stat/ -# load met and metplus -module use modulefiles/tasks/${platform,,} -module load run_vx.local -stat_analysis -config parm/metplus/STATAnalysisConfig_skill_score -lookin ${workspace}/Indy-Severe-Weather/metprd/point_stat -v 2 -out skill-score.out +# Run skill-score check +if [[ ${RUN_STAT_ANLY_OPT} == true ]]; then + # Clear out data + rm -rf ${workspace}/Indy-Severe-Weather/ + # Check if metprd data exists locally otherwise get it from S3 + TEST_EXTRN_MDL_SOURCE_BASEDIR=$(grep TEST_EXTRN_MDL_SOURCE_BASEDIR ${workspace}/ush/machine/${SRW_PLATFORM}.yaml | awk '{print $NF}') + if [[ ! -d $(dirname ${TEST_EXTRN_MDL_SOURCE_BASEDIR})/metprd/point_stat ]] ; then + mkdir -p Indy-Severe-Weather/metprd/point_stat + cp -rp $(dirname ${TEST_EXTRN_MDL_SOURCE_BASEDIR})/metprd/point_stat Indy-Severe-Weather/metprd + elif [[ -f Indy-Severe-Weather.tgz ]]; then + tar xvfz Indy-Severe-Weather.tgz + else + wget https://noaa-ufs-srw-pds.s3.amazonaws.com/sample_cases/release-public-v2.1.0/Indy-Severe-Weather.tgz + tar xvfz Indy-Severe-Weather.tgz + fi + [[ -f skill-score.txt ]] && rm skill-score.txt + # Skill score index is computed over several terms that are defined in parm/metplus/STATAnalysisConfig_skill_score. + # It is computed by aggregating the output from earlier runs of the Point-Stat and/or Grid-Stat tools over one or more cases. + # In this example, skill score index is a weighted average of 4 skill scores of RMSE statistics for wind speed, dew point temperature, + # temperature, and pressure at lowest level in the atmosphere over 6 hour lead time. + cp ${we2e_experiment_base_dir}/${we2e_test_name}/2019061500/metprd/PointStat/*.stat ${workspace}/Indy-Severe-Weather/metprd/point_stat/ + # Remove conda for Orion due to conda env conflicts + if [[ ${platform} =~ "orion" ]]; then + sed -i 's|load("conda")|--load("conda")|g' ${workspace}/modulefiles/tasks/${platform,,}/run_vx.local.lua + fi + # Load met and metplus + module use modulefiles/tasks/${platform,,} + module load run_vx.local + # Reset Orion run_vx.local file + if [[ ${platform} =~ "orion" ]]; then + sed -i 's|--load("conda")|load("conda")|g' ${workspace}/modulefiles/tasks/${platform,,}/run_vx.local.lua + fi + # Run stat_analysis + stat_analysis -config parm/metplus/STATAnalysisConfig_skill_score -lookin ${workspace}/Indy-Severe-Weather/metprd/point_stat -v 2 -out skill-score.txt -# check skill-score.out -cat skill-score.out + # check skill-score.txt + cat skill-score.txt -# get skill-score (SS_INDEX) and check if it is significantly smaller than 1.0 -# A value greater than 1.0 indicates that the forecast model outperforms the reference, -# while a value less than 1.0 indicates that the reference outperforms the forecast. -tmp_string=$( tail -2 skill-score.out | head -1 ) -SS_INDEX=$(echo $tmp_string | awk -F " " '{print $NF}') -echo "Skill Score: ${SS_INDEX}" -if [[ ${SS_INDEX} < "0.700" ]]; then - echo "Your Skill Score is way smaller than 1.00, better check before merging" - exit 1 -else - echo "Congrats! You pass check!" + # get skill-score (SS_INDEX) and check if it is significantly smaller than 1.0 + # A value greater than 1.0 indicates that the forecast model outperforms the reference, + # while a value less than 1.0 indicates that the reference outperforms the forecast. + tmp_string=$( tail -2 skill-score.txt | head -1 ) + SS_INDEX=$(echo $tmp_string | awk -F " " '{print $NF}') + echo "Skill Score: ${SS_INDEX}" + if [[ ${SS_INDEX} < "0.700" ]]; then + echo "Your Skill Score is way smaller than 1.00, better check before merging" + exit 1 + else + echo "Congrats! You pass check!" + fi fi diff --git a/modulefiles/tasks/gaea/run_vx.local.lua b/modulefiles/tasks/gaea/run_vx.local.lua index cb64e9a38a..57cdfbb1cc 100644 --- a/modulefiles/tasks/gaea/run_vx.local.lua +++ b/modulefiles/tasks/gaea/run_vx.local.lua @@ -22,4 +22,6 @@ if (mode() == "unload") then unload(pathJoin("met", met_ver)) unload(pathJoin("metplus",metplus_ver)) end -load("python_srw") +load("conda") +setenv("SRW_ENV", "srw_app") +setenv("LD_PRELOAD", "/opt/cray/pe/gcc/12.2.0/snos/lib64/libstdc++.so.6") diff --git a/modulefiles/tasks/orion/run_vx.local.lua b/modulefiles/tasks/orion/run_vx.local.lua index 1fa9617365..5bafb4d46b 100644 --- a/modulefiles/tasks/orion/run_vx.local.lua +++ b/modulefiles/tasks/orion/run_vx.local.lua @@ -1,7 +1,7 @@ --[[ Compiler-specific modules are used for met and metplus libraries --]] -load("build_orion_intel") +--load("build_orion_intel") local met_ver = (os.getenv("met_ver") or "11.1.0") local metplus_ver = (os.getenv("metplus_ver") or "5.1.0") @@ -27,4 +27,5 @@ if (mode() == "unload") then end --load("ufs-pyenv") load("stack-python/3.10.8") -load("python_srw") +load("conda") +setenv("SRW_ENV", "srw_app") diff --git a/parm/metplus/STATAnalysisConfig_skill_score b/parm/metplus/STATAnalysisConfig_skill_score index 6fd64a6456..fba1106d6b 100644 --- a/parm/metplus/STATAnalysisConfig_skill_score +++ b/parm/metplus/STATAnalysisConfig_skill_score @@ -11,10 +11,19 @@ // model = ["FV3_WoFS_v0_SUBCONUS_3km_test_mem000", "FV3_GFS_v16_SUBCONUS_3km"]; -fcst_lead = [ "6", - "6", - "6", - "6" +fcst_lead = [ "6", "12", + "6", "12", + "6", "12", + "6", "12", + "12", + "12", + "12", + "12", + "12", + "12", + "12", + "12", + "12" ]; obs_lead = []; @@ -42,17 +51,35 @@ obs_init_inc = []; obs_init_exc = []; obs_init_hour = []; -fcst_var = [ "WIND", - "DPT", - "TMP", - "PRMSL" +fcst_var = [ "PRMSL", "PRMSL", + "WIND", "WIND", + "DPT", "DPT", + "TMP", "TMP", + "WIND", + "WIND", + "WIND", + "TMP", + "TMP", + "TMP", + "SPFH", + "SPFH", + "SPFH" ]; obs_var = []; -fcst_lev = [ "Z10", - "Z2", - "Z2", - "Z0" +fcst_lev = [ "Z0", "Z0", + "Z10", "Z10", + "Z2", "Z2", + "Z2", "Z2", + "P250", + "P400", + "P850", + "P250", + "P400", + "P850", + "P300", + "P500", + "P850" ]; obs_lev = []; @@ -74,10 +101,19 @@ line_type = [ "SL1L2" ]; column = [ "RMSE" ]; -weight = [ 10.0, - 10.0, - 10.0, - 10.0 +weight = [ 10.0, 8.0, + 10.0, 8.0, + 10.0, 8.0, + 10.0, 8.0, + 4.0, + 4.0, + 4.0, + 4.0, + 4.0, + 4.0, + 4.0, + 4.0, + 4.0 ]; //////////////////////////////////////////////////////////////////////////////// @@ -116,6 +152,6 @@ hss_ec_value = NA; rank_corr_flag = FALSE; vif_flag = FALSE; tmp_dir = "/tmp"; -version = "V10.1.1"; +version = "V11.1.0"; //////////////////////////////////////////////////////////////////////////////// diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0.yaml index 275500e692..120a38291e 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0.yaml @@ -12,7 +12,7 @@ workflow: PREDEF_GRID_NAME: SUBCONUS_Ind_3km DATE_FIRST_CYCL: '2019061500' DATE_LAST_CYCL: '2019061500' - FCST_LEN_HRS: 6 + FCST_LEN_HRS: 12 PREEXISTING_DIR_METHOD: rename rocoto: tasks: