diff --git a/.cicd/scripts/srw_metric_example.sh b/.cicd/scripts/srw_metric_example.sh index 2018505735..45dd30c299 100755 --- a/.cicd/scripts/srw_metric_example.sh +++ b/.cicd/scripts/srw_metric_example.sh @@ -3,22 +3,45 @@ # The goal of this script is to provide an example of performing Indy-Severe-Weather test run and compare results to reference with # Skill score index that is calculated by MET Stat-Analysis Tools # -# Required: +# Required (these options are set in the Jenkins env): # WORKSPACE= # SRW_PLATFORM= # SRW_COMPILER= +# SRW_PROJECT= # # Optional: -[[ -n ${SRW_PROJECT} ]] || SRW_PROJECT="no_account" +#[[ -n ${SRW_PROJECT} ]] || SRW_PROJECT="no_account" [[ -n ${FORGIVE_CONDA} ]] || FORGIVE_CONDA=true set -e -u -x +BUILD_OPT=false +RUN_WE2E_OPT=false +RUN_STAT_ANLY_OPT=false + +if [[ $# -eq 0 ]]; then + BUILD_OPT=true + RUN_WE2E_OPT=true + RUN_STAT_ANLY_OPT=true +elif [[ $# -ge 4 ]]; then + echo "Too many arguments, expecting three or less" + exit 1 +else + for opt in "$@"; do + case $opt in + build) BUILD_OPT=true ;; + run_we2e) RUN_WE2E_OPT=true ;; + run_stat_anly) RUN_STAT_ANLY_OPT=true ;; + *) echo "Not valid option. Exiting!" ; exit 1 ;; + esac + done +fi + script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set # relative to script directory. declare workspace -if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then +if [[ -d "${WORKSPACE}/${SRW_PLATFORM}" ]]; then workspace="${WORKSPACE}/${SRW_PLATFORM}" else workspace="$(cd -- "${script_dir}/../.." && pwd)" @@ -39,53 +62,85 @@ we2e_test_name="grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0" pwd -# Activate the workflow environment ... +# Setup the build environment +declare srw_compiler +srw_compiler=${SRW_COMPILER} source etc/lmod-setup.sh ${platform,,} module use modulefiles -module load build_${platform,,}_${SRW_COMPILER} +module load build_${platform,,}_${srw_compiler} + +# Build srw +if [[ ${BUILD_OPT} == true ]]; then + cd ${workspace}/tests + ./build.sh ${platform,,} ${srw_compiler} +fi +cd ${workspace} + +# Activate workflow environment module load wflow_${platform,,} -[[ ${FORGIVE_CONDA} == true ]] && set +e +u # Some platforms have incomplete python3 or conda support, but wouldn't necessarily block workflow tests -conda activate workflow_tools +[[ ${FORGIVE_CONDA} == true ]] && set +e +u # Some platforms have incomplete python3 or conda support, but would not necessarily block workflow tests +conda activate srw_app set -e -u -# build srw -cd ${workspace}/tests -./build.sh ${platform,,} ${SRW_COMPILER} -cd ${workspace} - -# run test -[[ -d ${we2e_experiment_base_dir} ]] && rm -rf ${we2e_experiment_base_dir} -cd ${workspace}/tests/WE2E -./run_WE2E_tests.py -t ${we2e_test_name} -m ${platform,,} -a ${SRW_PROJECT} --expt_basedir "metric_test" --exec_subdir=install_intel/exec -q +# Run test +declare srw_project +srw_project=${SRW_PROJECT} +if [[ ${RUN_WE2E_OPT} == true ]]; then + [[ -d ${we2e_experiment_base_dir} ]] && rm -rf ${we2e_experiment_base_dir} + cd ${workspace}/tests/WE2E + ./run_WE2E_tests.py -t ${we2e_test_name} -m ${platform,,} -a ${srw_project} --expt_basedir "metric_test" --exec_subdir=install_intel/exec -q +fi cd ${workspace} -# run skill-score check -[[ ! -f Indy-Severe-Weather.tgz ]] && wget https://noaa-ufs-srw-pds.s3.amazonaws.com/sample_cases/release-public-v2.1.0/Indy-Severe-Weather.tgz -[[ ! -d Indy-Severe-Weather ]] && tar xvfz Indy-Severe-Weather.tgz -[[ -f skill-score.out ]] && rm skill-score.out -# Skill score index is computed over several terms that are defined in parm/metplus/STATAnalysisConfig_skill_score. -# It is computed by aggregating the output from earlier runs of the Point-Stat and/or Grid-Stat tools over one or more cases. -# In this example, skill score index is a weighted average of 4 skill scores of RMSE statistics for wind speed, dew point temperature, -# temperature, and pressure at lowest level in the atmosphere over 6 hour lead time. -cp ${we2e_experiment_base_dir}/${we2e_test_name}/2019061500/metprd/PointStat/*.stat ${workspace}/Indy-Severe-Weather/metprd/point_stat/ -# load met and metplus -module use modulefiles/tasks/${platform,,} -module load run_vx.local -stat_analysis -config parm/metplus/STATAnalysisConfig_skill_score -lookin ${workspace}/Indy-Severe-Weather/metprd/point_stat -v 2 -out skill-score.out +# Run skill-score check +if [[ ${RUN_STAT_ANLY_OPT} == true ]]; then + # Clear out data + rm -rf ${workspace}/Indy-Severe-Weather/ + # Check if metprd data exists locally otherwise get it from S3 + TEST_EXTRN_MDL_SOURCE_BASEDIR=$(grep TEST_EXTRN_MDL_SOURCE_BASEDIR ${workspace}/ush/machine/${SRW_PLATFORM}.yaml | awk '{print $NF}') + if [[ ! -d $(dirname ${TEST_EXTRN_MDL_SOURCE_BASEDIR})/metprd/point_stat ]] ; then + mkdir -p Indy-Severe-Weather/metprd/point_stat + cp -rp $(dirname ${TEST_EXTRN_MDL_SOURCE_BASEDIR})/metprd/point_stat Indy-Severe-Weather/metprd + elif [[ -f Indy-Severe-Weather.tgz ]]; then + tar xvfz Indy-Severe-Weather.tgz + else + wget https://noaa-ufs-srw-pds.s3.amazonaws.com/sample_cases/release-public-v2.1.0/Indy-Severe-Weather.tgz + tar xvfz Indy-Severe-Weather.tgz + fi + [[ -f skill-score.txt ]] && rm skill-score.txt + # Skill score index is computed over several terms that are defined in parm/metplus/STATAnalysisConfig_skill_score. + # It is computed by aggregating the output from earlier runs of the Point-Stat and/or Grid-Stat tools over one or more cases. + # In this example, skill score index is a weighted average of 4 skill scores of RMSE statistics for wind speed, dew point temperature, + # temperature, and pressure at lowest level in the atmosphere over 6 hour lead time. + cp ${we2e_experiment_base_dir}/${we2e_test_name}/2019061500/metprd/PointStat/*.stat ${workspace}/Indy-Severe-Weather/metprd/point_stat/ + # Remove conda for Orion due to conda env conflicts + if [[ ${platform} =~ "orion" ]]; then + sed -i 's|load("conda")|--load("conda")|g' ${workspace}/modulefiles/tasks/${platform,,}/run_vx.local.lua + fi + # Load met and metplus + module use modulefiles/tasks/${platform,,} + module load run_vx.local + # Reset Orion run_vx.local file + if [[ ${platform} =~ "orion" ]]; then + sed -i 's|--load("conda")|load("conda")|g' ${workspace}/modulefiles/tasks/${platform,,}/run_vx.local.lua + fi + # Run stat_analysis + stat_analysis -config parm/metplus/STATAnalysisConfig_skill_score -lookin ${workspace}/Indy-Severe-Weather/metprd/point_stat -v 2 -out skill-score.txt -# check skill-score.out -cat skill-score.out + # check skill-score.txt + cat skill-score.txt -# get skill-score (SS_INDEX) and check if it is significantly smaller than 1.0 -# A value greater than 1.0 indicates that the forecast model outperforms the reference, -# while a value less than 1.0 indicates that the reference outperforms the forecast. -tmp_string=$( tail -2 skill-score.out | head -1 ) -SS_INDEX=$(echo $tmp_string | awk -F " " '{print $NF}') -echo "Skill Score: ${SS_INDEX}" -if [[ ${SS_INDEX} < "0.700" ]]; then - echo "Your Skill Score is way smaller than 1.00, better check before merging" - exit 1 -else - echo "Congrats! You pass check!" + # get skill-score (SS_INDEX) and check if it is significantly smaller than 1.0 + # A value greater than 1.0 indicates that the forecast model outperforms the reference, + # while a value less than 1.0 indicates that the reference outperforms the forecast. + tmp_string=$( tail -2 skill-score.txt | head -1 ) + SS_INDEX=$(echo $tmp_string | awk -F " " '{print $NF}') + echo "Skill Score: ${SS_INDEX}" + if [[ ${SS_INDEX} < "0.700" ]]; then + echo "Your Skill Score is way smaller than 1.00, better check before merging" + exit 1 + else + echo "Congrats! You pass check!" + fi fi diff --git a/modulefiles/tasks/gaea/run_vx.local.lua b/modulefiles/tasks/gaea/run_vx.local.lua index cb64e9a38a..57cdfbb1cc 100644 --- a/modulefiles/tasks/gaea/run_vx.local.lua +++ b/modulefiles/tasks/gaea/run_vx.local.lua @@ -22,4 +22,6 @@ if (mode() == "unload") then unload(pathJoin("met", met_ver)) unload(pathJoin("metplus",metplus_ver)) end -load("python_srw") +load("conda") +setenv("SRW_ENV", "srw_app") +setenv("LD_PRELOAD", "/opt/cray/pe/gcc/12.2.0/snos/lib64/libstdc++.so.6") diff --git a/modulefiles/tasks/orion/run_vx.local.lua b/modulefiles/tasks/orion/run_vx.local.lua index 1fa9617365..5bafb4d46b 100644 --- a/modulefiles/tasks/orion/run_vx.local.lua +++ b/modulefiles/tasks/orion/run_vx.local.lua @@ -1,7 +1,7 @@ --[[ Compiler-specific modules are used for met and metplus libraries --]] -load("build_orion_intel") +--load("build_orion_intel") local met_ver = (os.getenv("met_ver") or "11.1.0") local metplus_ver = (os.getenv("metplus_ver") or "5.1.0") @@ -27,4 +27,5 @@ if (mode() == "unload") then end --load("ufs-pyenv") load("stack-python/3.10.8") -load("python_srw") +load("conda") +setenv("SRW_ENV", "srw_app") diff --git a/parm/metplus/STATAnalysisConfig_skill_score b/parm/metplus/STATAnalysisConfig_skill_score index 6fd64a6456..fba1106d6b 100644 --- a/parm/metplus/STATAnalysisConfig_skill_score +++ b/parm/metplus/STATAnalysisConfig_skill_score @@ -11,10 +11,19 @@ // model = ["FV3_WoFS_v0_SUBCONUS_3km_test_mem000", "FV3_GFS_v16_SUBCONUS_3km"]; -fcst_lead = [ "6", - "6", - "6", - "6" +fcst_lead = [ "6", "12", + "6", "12", + "6", "12", + "6", "12", + "12", + "12", + "12", + "12", + "12", + "12", + "12", + "12", + "12" ]; obs_lead = []; @@ -42,17 +51,35 @@ obs_init_inc = []; obs_init_exc = []; obs_init_hour = []; -fcst_var = [ "WIND", - "DPT", - "TMP", - "PRMSL" +fcst_var = [ "PRMSL", "PRMSL", + "WIND", "WIND", + "DPT", "DPT", + "TMP", "TMP", + "WIND", + "WIND", + "WIND", + "TMP", + "TMP", + "TMP", + "SPFH", + "SPFH", + "SPFH" ]; obs_var = []; -fcst_lev = [ "Z10", - "Z2", - "Z2", - "Z0" +fcst_lev = [ "Z0", "Z0", + "Z10", "Z10", + "Z2", "Z2", + "Z2", "Z2", + "P250", + "P400", + "P850", + "P250", + "P400", + "P850", + "P300", + "P500", + "P850" ]; obs_lev = []; @@ -74,10 +101,19 @@ line_type = [ "SL1L2" ]; column = [ "RMSE" ]; -weight = [ 10.0, - 10.0, - 10.0, - 10.0 +weight = [ 10.0, 8.0, + 10.0, 8.0, + 10.0, 8.0, + 10.0, 8.0, + 4.0, + 4.0, + 4.0, + 4.0, + 4.0, + 4.0, + 4.0, + 4.0, + 4.0 ]; //////////////////////////////////////////////////////////////////////////////// @@ -116,6 +152,6 @@ hss_ec_value = NA; rank_corr_flag = FALSE; vif_flag = FALSE; tmp_dir = "/tmp"; -version = "V10.1.1"; +version = "V11.1.0"; //////////////////////////////////////////////////////////////////////////////// diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0.yaml index 275500e692..120a38291e 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0.yaml @@ -12,7 +12,7 @@ workflow: PREDEF_GRID_NAME: SUBCONUS_Ind_3km DATE_FIRST_CYCL: '2019061500' DATE_LAST_CYCL: '2019061500' - FCST_LEN_HRS: 6 + FCST_LEN_HRS: 12 PREEXISTING_DIR_METHOD: rename rocoto: tasks: