From 2210d4ef3fdf97d437c108b336c5850bf7b0b775 Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Thu, 31 Mar 2022 17:21:22 -0600
Subject: [PATCH 01/14] Add 3km grid over Indianapolis.  This is about 600km x
 600km in extent (200 x 200 grid points).  It is intended for use in the WE2E
 tests.

---
 ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 27 +++++++++++
 .../wflow_features/config.new_ESGgrid.sh      |  4 +-
 ush/set_predef_grid_params.sh                 | 47 +++++++++++++++++++
 ush/valid_param_vals.sh                       |  1 +
 4 files changed, 77 insertions(+), 2 deletions(-)
 create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh

diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh
new file mode 100644
index 000000000..e11ab393a
--- /dev/null
+++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh
@@ -0,0 +1,27 @@
+#
+# TEST PURPOSE/DESCRIPTION:
+# ------------------------
+#
+# This test is to ensure that the workflow running in community mode 
+# completes successfully on the SUBCONUS_Ind_3km grid using the GFS_v16
+# physics suite with ICs and LBCs derived from the FV3GFS.
+#
+
+RUN_ENVIR="community"
+PREEXISTING_DIR_METHOD="rename"
+
+PREDEF_GRID_NAME="SUBCONUS_Ind_3km"
+CCPP_PHYS_SUITE="FV3_GFS_v16"
+
+EXTRN_MDL_NAME_ICS="FV3GFS"
+FV3GFS_FILE_FMT_ICS="grib2"
+EXTRN_MDL_NAME_LBCS="FV3GFS"
+FV3GFS_FILE_FMT_LBCS="grib2"
+USE_USER_STAGED_EXTRN_FILES="TRUE"
+
+DATE_FIRST_CYCL="20190615"
+DATE_LAST_CYCL="20190615"
+CYCL_HRS=( "18" )
+
+FCST_LEN_HRS="6"
+LBC_SPEC_INTVL_HRS="6"
diff --git a/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh b/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh
index b56681f54..fc34f4a0c 100644
--- a/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh
+++ b/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh
@@ -2,7 +2,7 @@
 # TEST PURPOSE/DESCRIPTION:
 # ------------------------
 #
-# This test checks the capability of the workflow to have the user 
+# This test checks the capability of the workflow to have the user
 # specify a new grid (as opposed to one of the predefined ones in the
 # workflow) of ESGgrid type.
 
@@ -47,7 +47,7 @@ BLOCKSIZE="13"
 QUILTING="TRUE"
 if [ "$QUILTING" = "TRUE" ]; then
   WRTCMP_write_groups="1"
-  WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y ))                     
+  WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y ))
   WRTCMP_output_grid="lambert_conformal"
   WRTCMP_cen_lon="${ESGgrid_LON_CTR}"
   WRTCMP_cen_lat="${ESGgrid_LAT_CTR}"
diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh
index dd664670a..257123b6a 100644
--- a/ush/set_predef_grid_params.sh
+++ b/ush/set_predef_grid_params.sh
@@ -279,6 +279,53 @@ case ${PREDEF_GRID_NAME} in
 #
 #-----------------------------------------------------------------------
 #
+# A subconus domain over Indianapolis, Indiana with ~3km cells.  This is
+# mostly for testing on a 3km grid with a small number of cells.
+#
+#-----------------------------------------------------------------------
+#
+"SUBCONUS_Ind_3km")
+
+  GRID_GEN_METHOD="ESGgrid"
+
+  ESGgrid_LON_CTR="-86.16"
+  ESGgrid_LAT_CTR="39.77"
+
+  ESGgrid_DELX="3000.0"
+  ESGgrid_DELY="3000.0"
+
+  ESGgrid_NX="200"
+  ESGgrid_NY="200"
+
+  ESGgrid_PAZI="0.0"
+  
+  ESGgrid_WIDE_HALO_WIDTH="6"
+
+  DT_ATMOS="${DT_ATMOS:-40}"
+
+  LAYOUT_X="${LAYOUT_X:-5}"
+  LAYOUT_Y="${LAYOUT_Y:-5}"
+  BLOCKSIZE="${BLOCKSIZE:-40}"
+
+  if [ "$QUILTING" = "TRUE" ]; then
+    WRTCMP_write_groups="1"
+    WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y ))
+    WRTCMP_output_grid="lambert_conformal"
+    WRTCMP_cen_lon="${ESGgrid_LON_CTR}"
+    WRTCMP_cen_lat="${ESGgrid_LAT_CTR}"
+    WRTCMP_stdlat1="${ESGgrid_LAT_CTR}"
+    WRTCMP_stdlat2="${ESGgrid_LAT_CTR}"
+    WRTCMP_nx="197"
+    WRTCMP_ny="195"
+    WRTCMP_lon_lwr_left="-89.47120417"
+    WRTCMP_lat_lwr_left="37.07809642"
+    WRTCMP_dx="${ESGgrid_DELX}"
+    WRTCMP_dy="${ESGgrid_DELY}"
+  fi
+  ;;
+#
+#-----------------------------------------------------------------------
+#
 # The RRFS Alaska domain with ~13km cells.
 #
 # Note:
diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh
index 1ea3a86b8..a41f8eb86 100644
--- a/ush/valid_param_vals.sh
+++ b/ush/valid_param_vals.sh
@@ -27,6 +27,7 @@ valid_vals_PREDEF_GRID_NAME=( \
 "GSD_HRRR_AK_50km" \
 "RRFS_NA_13km" \
 "RRFS_NA_3km" \
+"SUBCONUS_Ind_3km" \
 )
 valid_vals_CCPP_PHYS_SUITE=( \
 "FV3_GFS_2017_gfdlmp" \

From 9ef39be1e2f037031fc13b03259ef58f140821f8 Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Thu, 31 Mar 2022 17:33:15 -0600
Subject: [PATCH 02/14] Edit comments.

---
 ush/set_predef_grid_params.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh
index 257123b6a..4ffb300ab 100644
--- a/ush/set_predef_grid_params.sh
+++ b/ush/set_predef_grid_params.sh
@@ -280,7 +280,8 @@ case ${PREDEF_GRID_NAME} in
 #-----------------------------------------------------------------------
 #
 # A subconus domain over Indianapolis, Indiana with ~3km cells.  This is
-# mostly for testing on a 3km grid with a small number of cells.
+# mostly for testing on a 3km grid with a much small number of cells than
+# on the full CONUS.
 #
 #-----------------------------------------------------------------------
 #

From fc843ac68d57fbe6b272012ba64db2d835da3d1c Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Thu, 14 Apr 2022 17:26:39 -0600
Subject: [PATCH 03/14] Generalize script "get_WE2Etest_names_subdirs_descs.sh"
 to enable including experiment variable values in the CSV file it generates.

---
 .../WE2E/get_WE2Etest_names_subdirs_descs.sh  | 155 ++++++++++++++++--
 1 file changed, 137 insertions(+), 18 deletions(-)

diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh
index 2183ead89..615cb5608 100755
--- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh
+++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh
@@ -7,8 +7,10 @@
 # the WE2E tests available in the WE2E testing system.  This information
 # consists of the test names, the category subdirectories in which the 
 # test configuration files are located (relative to a base directory), 
-# the test IDs, and the test descriptions.  These are described in more 
-# detail below.
+# the test IDs, and the test descriptions.  This function optionally 
+# also creates a CSV (Comma-Separated Value) file containing various
+# pieces of information about each of the workflow end-to-end (WE2E)
+# tests.  These are described in more detail below.  
 #
 # The function takes as inputs the following arguments:
 #
@@ -65,7 +67,7 @@
 # in the local array category_subdirs.  We refer to these as "category" 
 # subdirectories because they are used for clarity to group the tests 
 # into categories (instead of putting them all directly under the base
-# directory).  For example, one category of tests might be  those that 
+# directory).  For example, one category of tests might be those that 
 # test workflow capabilities such as running multiple cycles and ensemble
 # forecasts, another might be those that run various combinations of 
 # grids, physics suites, and external models for ICs/LBCs, etc.  Note 
@@ -208,7 +210,10 @@
 # the file will be placed in the main WE2E testing system directory 
 # specified by the input argument WE2Edir.  The CSV file can be read 
 # into a spreadsheet in Google Sheets (or another similar tool) to get
-# an overview of all the available WE2E tests.  
+# an overview of all the available WE2E tests.  The rows of the CSV file 
+# correspond to the primary WE2E tests, and the columns correspond to 
+# the test name, description, and values of various SRW App experiment
+# variables for that test.
 #
 # A CSV file will be generated in the directory specified by WE2Edir if 
 # one or more of the following conditions hold:
@@ -228,7 +233,8 @@
 # "FALSE" in the call to this function (regardless of whether or not a 
 # CSV file already exists).  If a CSV file is generated, it is placed in 
 # the directory specified by the input argment WE2Edir, and it overwrites 
-# any existing copies of the file in that directory.
+# any existing copies of the file in that directory.  The contents of
+# each column of the CSV file are described below.
 #
 #-----------------------------------------------------------------------
 #
@@ -286,7 +292,10 @@ function get_WE2Etest_names_subdirs_descs() {
         alt_test_prim_test_names \
         alt_test_subdir \
         alt_test_subdirs \
+        array_names_vars_to_extract \
+        array_names_vars_to_extract_orig \
         category_subdirs \
+        cmd \
         column_titles \
         config_fn \
         crnt_item \
@@ -299,6 +308,7 @@ function get_WE2Etest_names_subdirs_descs() {
         ii \
         j \
         jp1 \
+        k \
         line \
         mod_time_csv \
         mod_time_subdir \
@@ -308,6 +318,8 @@ function get_WE2Etest_names_subdirs_descs() {
         num_occurrences \
         num_prim_tests \
         num_tests \
+        num_vars_to_extract \
+        prim_array_names_vars_to_extract \
         prim_test_descs \
         prim_test_ids \
         prim_test_name_subdir \
@@ -349,7 +361,10 @@ function get_WE2Etest_names_subdirs_descs() {
         test_subdirs_orig \
         test_subdirs_str \
         test_type \
-        valid_vals_generate_csv_file
+        val \
+        valid_vals_generate_csv_file \
+        var_name \
+        vars_to_extract
 #
 #-----------------------------------------------------------------------
 #
@@ -839,10 +854,42 @@ they correspond to unique test names and rerun."
 #-----------------------------------------------------------------------
 #
   if [ "${get_test_descs}" = "TRUE" ]; then
+#
+# Specify in "vars_to_extract" the list of experiment variables to extract 
+# from each test configuration file (and later to place in the CSV file).  
+# Recall that the rows of the CSV file correspond to the various WE2E 
+# tests, and the columns correspond to the test name, description, and 
+# experiment variable values.  The elements of "vars_to_extract" should 
+# be the names of SRW App experiment variables that are (or can be) 
+# specified in the App's configuration file.  Note that if a variable is 
+# not specified in the test configuration file, its value is set to an 
+# empty string (and recorded as such in the CSV file).
+#
+    vars_to_extract=( "PREDEF_GRID_NAME" \
+                      "EXTRN_MDL_NAME_ICS" \
+                      "EXTRN_MDL_NAME_LBCS" \
+                      "DATE_FIRST_CYCL" \
+                    )
+    num_vars_to_extract="${#vars_to_extract[@]}"
+#
+# Create names of local arrays that will hold the value of the corresponding
+# variable for each test.  Then use these names to define them as empty 
+# arrays.  [The arrays named "prim_..." are to hold values for only the
+# primary tests, while other arrays are to hold values for all (primary
+# plus alternate) tests.]
+#
+    prim_array_names_vars_to_extract=( $( printf "prim_test_%s_vals " "${vars_to_extract[@]}" ) )
+    array_names_vars_to_extract=( $( printf "%s_vals " "${vars_to_extract[@]}" ) )
+    for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
+      cmd="${prim_array_names_vars_to_extract[$k]}=()"
+      eval $cmd
+      cmd="${array_names_vars_to_extract[$k]}=()"
+      eval $cmd
+    done
 
     print_info_msg "
-Gathering test descriptions from the configuration files of the primary 
-WE2E tests..."
+Gathering test descriptions and experiment variable values from the 
+configuration files of the primary WE2E tests..."
 
     prim_test_descs=()
     for (( i=0; i<=$((num_prim_tests-1)); i++ )); do
@@ -931,16 +978,43 @@ ${test_desc}${stripped_line} "
 #
 # First remove leading whitespace.
 #
-    test_desc="${test_desc#"${test_desc%%[![:space:]]*}"}"
+      test_desc="${test_desc#"${test_desc%%[![:space:]]*}"}"
 #
 # Now remove trailing whitespace.
 #
-    test_desc="${test_desc%"${test_desc##*[![:space:]]}"}"
+      test_desc="${test_desc%"${test_desc##*[![:space:]]}"}"
 #
 # Finally, save the description of the current test as the next element
 # of the array prim_test_descs.
 #
       prim_test_descs+=("${test_desc}")
+#
+# Get from the current test's configuration file the values of the 
+# variables specified in "vars_to_extract".  Then save the value in the
+# arrays specified by "prim_array_names_vars_to_extract".
+#
+      for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
+
+        var_name="${vars_to_extract[$k]}"
+        cmd="unset ${var_name}"
+        eval $cmd
+        cmd=$( grep "${var_name}" "${config_fn}" )
+        eval $cmd
+
+        if [ -z "${!var_name+x}" ]; then
+          print_info_msg "\
+The variable \"${var_name}\" is not defined in the current test's
+configuration file (config_fn):
+  config_fn = \"${config_fn}\"
+Setting the element in the array \"${prim_array_names_vars_to_extract[$k]}\"
+corresponding to this test to an empty string."
+          cmd="${prim_array_names_vars_to_extract[$k]}+=(\"\")"
+        else
+          cmd="${prim_array_names_vars_to_extract[$k]}+=(\"${!var_name}\")"
+        fi
+        eval $cmd
+
+      done
 
     done
 
@@ -950,13 +1024,19 @@ ${test_desc}${stripped_line} "
 #
 # Create the arrays test_ids and test_descs that initially contain the 
 # test IDs and descriptions corresponding to the primary test names
-# (those of the alternate test names will be appended below).
+# (those of the alternate test names will be appended below).  Then, in
+# the for-loop, do same for the arrays containing the experiment variable 
+# values for each test.
 #
 #-----------------------------------------------------------------------
 #
   test_ids=("${prim_test_ids[@]}")
   if [ "${get_test_descs}" = "TRUE" ]; then
     test_descs=("${prim_test_descs[@]}")
+    for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
+      cmd="${array_names_vars_to_extract[$k]}=(\"\${${prim_array_names_vars_to_extract[$k]}[@]}\")"
+      eval $cmd
+    done
   fi
 #
 #-----------------------------------------------------------------------
@@ -964,7 +1044,8 @@ ${test_desc}${stripped_line} "
 # Append to the arrays test_ids and test_descs the test IDs and descriptions
 # of the alternate test names.  We set the test ID and description of 
 # each alternate test name to those of the corresponding primary test
-# name.
+# name.  Then, in the inner for-loop, do the same for the arrays containing
+# the experiment variable values.
 #
 #-----------------------------------------------------------------------
 #
@@ -980,6 +1061,10 @@ ${test_desc}${stripped_line} "
         test_ids+=("${prim_test_ids[$j]}")
         if [ "${get_test_descs}" = "TRUE" ]; then
           test_descs+=("${prim_test_descs[$j]}")
+          for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
+            cmd="${array_names_vars_to_extract[$k]}+=(\"\${${prim_array_names_vars_to_extract[$k]}[$j]}\")"
+            eval $cmd
+          done
         fi
         num_occurrences=$((num_occurrences+1))
       fi
@@ -1003,7 +1088,9 @@ Please correct and rerun."
 #-----------------------------------------------------------------------
 #
 # Sort in order of increasing test ID the arrays containing the names, 
-# IDs, category subdirectories, and descriptions of the WE2E tests.
+# IDs, category subdirectories, and descriptions of the WE2E tests as
+# well as the arrays containing the experiment variable values for each
+# test.
 #
 # For this purpose, we first create an array (test_ids_and_inds) each
 # of whose elements consist of the test ID, the test type, and the index 
@@ -1029,8 +1116,8 @@ Please correct and rerun."
 # and the test type, which we no longer need), which is the original 
 # array index before sorting, and save the results in the array sort_inds.  
 # This array will contain the original indices in sorted order that we 
-# then use to sort the arrays containing the names, IDs, subdirectories, 
-# and descriptions of the WE2E tests.
+# then use to sort the arrays containing the WE2E test names, IDs, 
+# subdirectories, descriptions, and experiment variable values.
 #
 #-----------------------------------------------------------------------
 #
@@ -1064,11 +1151,22 @@ Please correct and rerun."
   done
 
   if [ "${get_test_descs}" = "TRUE" ]; then
+
     test_descs_orig=( "${test_descs[@]}" )
+    for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
+      cmd="${array_names_vars_to_extract[$k]}_orig=(\"\${${array_names_vars_to_extract[$k]}[@]}\")"
+      eval $cmd
+    done
+
     for (( i=0; i<=$((num_tests-1)); i++ )); do
       ii="${sort_inds[$i]}"
       test_descs[$i]="${test_descs_orig[$ii]}"
+      for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
+        cmd="${array_names_vars_to_extract[$k]}[$i]=\"\${${array_names_vars_to_extract[$k]}_orig[$ii]}\""
+        eval $cmd
+      done
     done
+
   fi
 #
 #-----------------------------------------------------------------------
@@ -1094,18 +1192,26 @@ Please correct and rerun."
 #
     csv_delimiter="|"
 #
-# Set the titles of the three columns that will be in the file.  Then
-# write them to the file.  The contents of the columns are described in
-# more detail further below.
+# Set the titles of the columns that will be in the file.  Then write 
+# them to the file.  The contents of the columns are described in more 
+# detail further below.
 #
     column_titles="\
 \"Test Name (Subdirectory)\" ${csv_delimiter} \
 \"Alternate Test Names (Subdirectories)\" ${csv_delimiter} \
 \"Test Purpose/Description\""
+    for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
+      column_titles="\
+${column_titles} ${csv_delimiter} \
+\"${vars_to_extract[$k]}\""
+    done
     printf "%s\n" "${column_titles}" >> "${csv_fp}"
 #
 # Loop through the arrays containing the WE2E test information.  Extract 
 # the necessary information and record it to the CSV file row-by-row.
+# Note that each row corresponds to a primary test.  When an alternate
+# test is encountered, its information is stored in the row of the 
+# corresponding primary test (i.e. a new row is not created).
 #
     j=0
     jp1=$((j+1))
@@ -1167,11 +1273,24 @@ ${test_names[$jp1]} (${test_subdirs[$jp1]})"
 #
 # Column 3:
 # The test description.
+#
+# Columns 4...:
+# The values of the experiment variables specified in vars_to_extract.
 #
       row_content="\
 \"${prim_test_name_subdir}\" ${csv_delimiter} \
 \"${alt_test_names_subdirs}\" ${csv_delimiter} \
 \"${test_desc}\""
+
+      for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
+        unset "val"
+        cmd="val=\"\${${array_names_vars_to_extract[$k]}[$j]}\""
+        eval $cmd
+        row_content="\
+${row_content} ${csv_delimiter} \
+\"${val}\""
+      done
+
       printf "%s\n" "${row_content}" >> "${csv_fp}"
 #
 # Update loop indices.

From f6deb7aa2629602d52cd6cb942a9806821e41bc0 Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Thu, 14 Apr 2022 17:27:43 -0600
Subject: [PATCH 04/14] Bug fix - in var_defns.sh, the definition of CRES
 should use single quotes (like for all other variables in var_defns.sh).

---
 scripts/exregional_make_grid.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh
index 742770630..3744638c8 100755
--- a/scripts/exregional_make_grid.sh
+++ b/scripts/exregional_make_grid.sh
@@ -410,7 +410,7 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then
 elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then
   CRES="C${res_equiv}"
 fi
-set_file_param "${GLOBAL_VAR_DEFNS_FP}" "CRES" "\"$CRES\""
+set_file_param "${GLOBAL_VAR_DEFNS_FP}" "CRES" "'$CRES'"
 #
 #-----------------------------------------------------------------------
 #

From 151dab30a8ad5a9d575a7fa705bc2a0036ad5c71 Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Mon, 18 Apr 2022 14:56:32 -0600
Subject: [PATCH 05/14] Minor changes (spacing).

---
 ush/check_expt_config_vars.sh | 2 +-
 ush/setup.sh                  | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/ush/check_expt_config_vars.sh b/ush/check_expt_config_vars.sh
index 53ce13a09..7476de779 100644
--- a/ush/check_expt_config_vars.sh
+++ b/ush/check_expt_config_vars.sh
@@ -63,7 +63,7 @@ function check_expt_config_vars() {
     # Note that a variable name will be found only if the equal sign immediately
     # follows the variable name.
     #
-    var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ =\"]*)=.*/\1/p")
+    var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ =\"]*)=.*/\1/p" )
 
     if [ -z "${var_name}" ]; then
 
diff --git a/ush/setup.sh b/ush/setup.sh
index 01470963d..5b96b0029 100755
--- a/ush/setup.sh
+++ b/ush/setup.sh
@@ -507,7 +507,7 @@ if [ -z "${NCORES_PER_NODE:-}" ]; then
 fi
 
 if [ -z "$FIXgsm" -o -z "$FIXaer" -o -z "$FIXlut" -o -z "$TOPO_DIR" -o -z "$SFC_CLIMO_INPUT_DIR" ]; then
-      print_err_msg_exit "\
+  print_err_msg_exit "\
 One or more fix file directories have not been specified for this machine:
   MACHINE = \"$MACHINE\"
   FIXgsm = \"${FIXgsm:-\"\"}
@@ -2141,7 +2141,6 @@ GLOBAL_VAR_DEFNS_FP="$EXPTDIR/${GLOBAL_VAR_DEFNS_FN}"
 # variable definitions file.
 #
 #-----------------------------------------------------------------------
-
 #
 print_info_msg "
 Creating list of default experiment variable definitions..." 

From 3a86f50471a2bcd64875b15294d9ee32744e69b1 Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Mon, 18 Apr 2022 14:56:55 -0600
Subject: [PATCH 06/14] Include additional information in the CSV file that the
 function get_WE2Etest_names_subdirs_descs.sh generates.  Details: 1) Add
 additional experiment variables to the CSV file (including allowing array
 variables such as CYCL_HRS) and specify default values for these variables
 (in case they are not defined in the test configuration file). 2) Calculate
 and include in the CSV file the number of forecast model runs that each test
 will make. 3) Add verbose option. 4) Add more informational messages (printed
 to screen).

---
 .../WE2E/get_WE2Etest_names_subdirs_descs.sh  | 174 ++++++++++++++++--
 1 file changed, 156 insertions(+), 18 deletions(-)

diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh
index 615cb5608..85a776f73 100755
--- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh
+++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh
@@ -23,6 +23,10 @@
 # Flag that specifies whether or not a CSV (Comma-Separated Value) file
 # containing information about the WE2E tests should be generated.
 #
+# verbose:
+# Optional verbosity flag.  Should be set to "TRUE" or "FALSE".  Default
+# is "FALSE".
+#
 # output_varname_test_configs_basedir:
 # Name of output variable in which to return the base directory of the 
 # WE2E test configuration files.
@@ -261,6 +265,7 @@ function get_WE2Etest_names_subdirs_descs() {
   local valid_args=( \
     "WE2Edir" \
     "generate_csv_file" \
+    "verbose" \
     "output_varname_test_configs_basedir" \
     "output_varname_test_names" \
     "output_varname_test_subdirs" \
@@ -281,6 +286,17 @@ function get_WE2Etest_names_subdirs_descs() {
 #
 #-----------------------------------------------------------------------
 #
+# Make the default value of "verbose" "FALSE".  Then make sure "verbose" 
+# is set to a valid value.
+#
+#-----------------------------------------------------------------------
+#
+  verbose=${verbose:-"FALSE"}
+  check_var_valid_value "verbose" "valid_vals_BOOLEAN"
+  verbose=$(boolify $verbose)
+#
+#-----------------------------------------------------------------------
+#
 # Declare local variables.
 #
 #-----------------------------------------------------------------------
@@ -303,6 +319,7 @@ function get_WE2Etest_names_subdirs_descs() {
         csv_fn \
         csv_fp \
         cwd \
+        default_val \
         hash_or_null \
         i \
         ii \
@@ -312,8 +329,14 @@ function get_WE2Etest_names_subdirs_descs() {
         line \
         mod_time_csv \
         mod_time_subdir \
+        msg \
         num_alt_tests \
         num_category_subdirs \
+        num_cdates \
+        num_cycles_per_day \
+        num_days \
+        num_fcsts \
+        num_fcsts_orig \
         num_items \
         num_occurrences \
         num_prim_tests \
@@ -324,6 +347,7 @@ function get_WE2Etest_names_subdirs_descs() {
         prim_test_ids \
         prim_test_name_subdir \
         prim_test_names \
+        prim_test_num_fcsts \
         prim_test_subdirs \
         get_test_descs \
         regex_search \
@@ -364,6 +388,7 @@ function get_WE2Etest_names_subdirs_descs() {
         val \
         valid_vals_generate_csv_file \
         var_name \
+        var_name_at \
         vars_to_extract
 #
 #-----------------------------------------------------------------------
@@ -423,6 +448,13 @@ function get_WE2Etest_names_subdirs_descs() {
     fi
 
   fi
+
+  if [ "${generate_csv_file}" = "TRUE" ]; then
+    print_info_msg "
+Will generate a CSV (Comma Separated Value) file (csv_fp) containing
+information on all WE2E tests:
+  csv_fp = \"${csv_fp}\""
+  fi
 #
 #-----------------------------------------------------------------------
 #
@@ -487,6 +519,7 @@ function get_WE2Etest_names_subdirs_descs() {
   prim_test_names=()
   prim_test_ids=()
   prim_test_subdirs=()
+  prim_test_num_fcsts=()
 
   alt_test_names=()
   alt_test_subdirs=()
@@ -862,13 +895,19 @@ they correspond to unique test names and rerun."
 # experiment variable values.  The elements of "vars_to_extract" should 
 # be the names of SRW App experiment variables that are (or can be) 
 # specified in the App's configuration file.  Note that if a variable is 
-# not specified in the test configuration file, its value is set to an 
-# empty string (and recorded as such in the CSV file).
+# not specified in the test configuration file, in most cases its value 
+# is set to an empty string (and recorded as such in the CSV file).  In
+# some cases, it is set to some other value (e.g. for the number of 
+# ensemble members NUM_ENS_MEMBERS, it is set to 1).
 #
     vars_to_extract=( "PREDEF_GRID_NAME" \
                       "EXTRN_MDL_NAME_ICS" \
                       "EXTRN_MDL_NAME_LBCS" \
                       "DATE_FIRST_CYCL" \
+                      "DATE_LAST_CYCL" \
+                      "CYCL_HRS" \
+                      "INCR_CYCL_FREQ" \
+                      "NUM_ENS_MEMBERS" \
                     )
     num_vars_to_extract="${#vars_to_extract[@]}"
 #
@@ -889,14 +928,17 @@ they correspond to unique test names and rerun."
 
     print_info_msg "
 Gathering test descriptions and experiment variable values from the 
-configuration files of the primary WE2E tests..."
+configuration files of the primary WE2E tests...
+"
 
     prim_test_descs=()
     for (( i=0; i<=$((num_prim_tests-1)); i++ )); do
 
       test_name="${prim_test_names[$i]}"
       print_info_msg "\
-  Reading in the test description for primary WE2E test:  \"${test_name}\""
+  Reading in the test description for primary WE2E test:  \"${test_name}\"
+  In category (subdirectory):  \"${subdir}\"
+"
       subdir=("${prim_test_subdirs[$i]}")
       cd_vrfy "${test_configs_basedir}/$subdir"
 #
@@ -996,25 +1038,102 @@ ${test_desc}${stripped_line} "
       for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
 
         var_name="${vars_to_extract[$k]}"
-        cmd="unset ${var_name}"
-        eval $cmd
-        cmd=$( grep "${var_name}" "${config_fn}" )
+        cmd=$( grep "^[ ]*${var_name}=" "${config_fn}" )
         eval $cmd
 
         if [ -z "${!var_name+x}" ]; then
-          print_info_msg "\
-The variable \"${var_name}\" is not defined in the current test's
-configuration file (config_fn):
-  config_fn = \"${config_fn}\"
-Setting the element in the array \"${prim_array_names_vars_to_extract[$k]}\"
-corresponding to this test to an empty string."
-          cmd="${prim_array_names_vars_to_extract[$k]}+=(\"\")"
+
+          msg="
+  The variable \"${var_name}\" is not defined in the current test's
+  configuration file (config_fn):
+    config_fn = \"${config_fn}\"
+  Setting the element in the array \"${prim_array_names_vars_to_extract[$k]}\"
+  corresponding to this test to"
+
+          case "${var_name}" in
+
+          "NUM_ENS_MEMBERS")
+            default_val="1"
+            msg=$msg":
+    ${var_name} = \"${default_val}\""
+            ;;
+
+          "INCR_CYCL_FREQ")
+            default_val="24"
+            msg=$msg":
+    ${var_name} = \"${default_val}\""
+            ;;
+
+          *)
+            default_val=""
+            msg=$msg" an empty string."
+            ;;
+          esac
+          cmd="${var_name}=\"${default_val}\""
+          eval $cmd
+
+          print_info_msg "$verbose" "$msg" 
+          cmd="${prim_array_names_vars_to_extract[$k]}+=(\"'${default_val}\")"
+
         else
-          cmd="${prim_array_names_vars_to_extract[$k]}+=(\"${!var_name}\")"
+#
+# The following are important notes regarding how the variable "cmd" 
+# containing the command that will append an element to the array 
+# specified by ${prim_array_names_vars_to_extract[$k]} is formulated: 
+#
+# 1) If all the experiment variables were scalars, then the more complex
+#    command below could be replaced with the following:
+#
+#    cmd="${prim_array_names_vars_to_extract[$k]}+=(\"${!var_name}\")"
+#
+#    But some variables are arrays, so we need the more complex approach
+#    to cover those cases.
+#
+# 2) The double quotes (which need to be escaped here, i.e. \") are needed
+#    so that for any experiment variables that are arrays, all the elements 
+#    of the array are combined together and treated as a single element.  
+#    If the experiment variable is CYCL_HRS (cycle hours) and is set to
+#    the array ("00" "12"), we want the value saved in the local array
+#    here to be a single element consisting of "00 12".  Otherwise, "00" 
+#    and "12" will be treated as separate elements, and more than one 
+#    element would be added to the array (which would be incorrect here).
+#
+# 3) The single quote (which needs to be escaped here, i.e. \') is needed
+#    so that any numbers (e.g. a set of cycle hours such as "00 12") are 
+#    treated as strings when the CSV file is opened in Google Sheets.  
+#    If this is not done, Google Sheets will remove leading zeros.
+#
+          var_name_at="${var_name}[@]"
+          cmd="${prim_array_names_vars_to_extract[$k]}+=(\'\"${!var_name_at}\")"
         fi
         eval $cmd
 
       done
+#
+# Calculate the number of forecasts that will be launched by the current
+# test.  The "10#" forces bash to treat the following number as a decimal
+# (not hexadecimal, etc).
+#
+      num_cycles_per_day=${#CYCL_HRS[@]}
+      num_days=$(( (${DATE_LAST_CYCL} - ${DATE_FIRST_CYCL} + 1)*24/10#${INCR_CYCL_FREQ} ))
+      num_cdates=$(( ${num_cycles_per_day}*${num_days} ))
+      nf=$(( ${num_cdates}*10#${NUM_ENS_MEMBERS} ))
+#
+# In the following, the single quote at the beginning forces Google Sheets 
+# to interpret this quantity as a string.  This prevents any automatic 
+# number fomatting from being applied when the CSV file is imported into
+# Google Sheets.
+#
+      prim_test_num_fcsts+=( "'$nf" )
+#
+# Unset the experiment variables defined for the current test so that 
+# they are not accidentally used for the next one.
+#
+      for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
+        var_name="${vars_to_extract[$k]}"
+        cmd="unset ${var_name}"
+        eval $cmd
+      done
 
     done
 
@@ -1033,6 +1152,7 @@ corresponding to this test to an empty string."
   test_ids=("${prim_test_ids[@]}")
   if [ "${get_test_descs}" = "TRUE" ]; then
     test_descs=("${prim_test_descs[@]}")
+    num_fcsts=("${prim_test_num_fcsts[@]}")
     for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
       cmd="${array_names_vars_to_extract[$k]}=(\"\${${prim_array_names_vars_to_extract[$k]}[@]}\")"
       eval $cmd
@@ -1061,6 +1181,7 @@ corresponding to this test to an empty string."
         test_ids+=("${prim_test_ids[$j]}")
         if [ "${get_test_descs}" = "TRUE" ]; then
           test_descs+=("${prim_test_descs[$j]}")
+          num_fcsts+=("${prim_test_num_fcsts[$j]}")
           for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
             cmd="${array_names_vars_to_extract[$k]}+=(\"\${${prim_array_names_vars_to_extract[$k]}[$j]}\")"
             eval $cmd
@@ -1153,6 +1274,7 @@ Please correct and rerun."
   if [ "${get_test_descs}" = "TRUE" ]; then
 
     test_descs_orig=( "${test_descs[@]}" )
+    num_fcsts_orig=( "${num_fcsts[@]}" )
     for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
       cmd="${array_names_vars_to_extract[$k]}_orig=(\"\${${array_names_vars_to_extract[$k]}[@]}\")"
       eval $cmd
@@ -1161,6 +1283,7 @@ Please correct and rerun."
     for (( i=0; i<=$((num_tests-1)); i++ )); do
       ii="${sort_inds[$i]}"
       test_descs[$i]="${test_descs_orig[$ii]}"
+      num_fcsts[$i]="${num_fcsts_orig[$ii]}"
       for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
         cmd="${array_names_vars_to_extract[$k]}[$i]=\"\${${array_names_vars_to_extract[$k]}_orig[$ii]}\""
         eval $cmd
@@ -1199,7 +1322,8 @@ Please correct and rerun."
     column_titles="\
 \"Test Name (Subdirectory)\" ${csv_delimiter} \
 \"Alternate Test Names (Subdirectories)\" ${csv_delimiter} \
-\"Test Purpose/Description\""
+\"Test Purpose/Description\" ${csv_delimiter} \
+\"Number of Forecast Model Runs\""
     for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
       column_titles="\
 ${column_titles} ${csv_delimiter} \
@@ -1236,6 +1360,11 @@ ${column_titles} ${csv_delimiter} \
 #
       test_desc=$( printf "%s" "${test_desc}" | sed -r -e "s/\"/\"\"/g" )
 #
+# Get the number of forecasts (number of times the forcast model is run,
+# due to a unique starting date, an ensemble member, etc).
+#
+      nf="${num_fcsts[$j]}"
+#
 # In the following inner while-loop, we step through all alternate test 
 # names (if any) that follow the current primary name and construct a 
 # string (alt_test_names_subdirs) consisting of all the alternate test 
@@ -1274,13 +1403,17 @@ ${test_names[$jp1]} (${test_subdirs[$jp1]})"
 # Column 3:
 # The test description.
 #
-# Columns 4...:
+# Columns 4:
+# The number of times the forecast model will be run by the test.
+#
+# Columns 5...:
 # The values of the experiment variables specified in vars_to_extract.
 #
       row_content="\
 \"${prim_test_name_subdir}\" ${csv_delimiter} \
 \"${alt_test_names_subdirs}\" ${csv_delimiter} \
-\"${test_desc}\""
+\"${test_desc}\" ${csv_delimiter} \
+\"${nf}\""
 
       for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do
         unset "val"
@@ -1300,6 +1433,11 @@ ${row_content} ${csv_delimiter} \
 
     done
 
+    print_info_msg "\
+Successfully generated a CSV (Comma Separated Value) file (csv_fp) 
+containing information on all WE2E tests:
+  csv_fp = \"${csv_fp}\""
+    
   fi
 #
 #-----------------------------------------------------------------------

From 212f0e18efbf6c0137560e7c80d3847ec91ac0a3 Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Mon, 18 Apr 2022 16:08:20 -0600
Subject: [PATCH 07/14] Edits to comments.

---
 tests/WE2E/get_WE2Etest_names_subdirs_descs.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh
index 85a776f73..5cf306eaa 100755
--- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh
+++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh
@@ -216,8 +216,9 @@
 # into a spreadsheet in Google Sheets (or another similar tool) to get
 # an overview of all the available WE2E tests.  The rows of the CSV file 
 # correspond to the primary WE2E tests, and the columns correspond to 
-# the test name, description, and values of various SRW App experiment
-# variables for that test.
+# the (primary) test name, alternate test names (if any), test description, 
+# number of times the test calls the forecast model, and values of various 
+# SRW App experiment variables for that test.
 #
 # A CSV file will be generated in the directory specified by WE2Edir if 
 # one or more of the following conditions hold:
@@ -1068,6 +1069,7 @@ ${test_desc}${stripped_line} "
             default_val=""
             msg=$msg" an empty string."
             ;;
+
           esac
           cmd="${var_name}=\"${default_val}\""
           eval $cmd

From e386ef70ccecb69a0da8bab90ded13cebb264148 Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Tue, 19 Apr 2022 13:05:20 -0600
Subject: [PATCH 08/14] Update the help section in run_WE2E_tests.sh to include
 examples.

---
 tests/WE2E/run_WE2E_tests.sh | 88 ++++++++++++++++++++++++++++++++----
 1 file changed, 79 insertions(+), 9 deletions(-)

diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh
index f3040208e..f059f73d7 100755
--- a/tests/WE2E/run_WE2E_tests.sh
+++ b/tests/WE2E/run_WE2E_tests.sh
@@ -133,9 +133,9 @@ tests under subdirectory testset1, another set of tests under testset2,
 etc.
 
 exec_subdir:
-Optional. Argument is used to set the EXEC_SUBDIR configuration
-variable. Please see the ush/default_configs.sh file for a full
-description.
+Optional.  Argument used to set the EXEC_SUBDIR experiment variable. 
+Please see the default experiment configuration file \"config_defaults.sh\" 
+for a full description of EXEC_SUBDIR.
 
 use_cron_to_relaunch:
 Argument used to explicitly set the experiment variable USE_CRON_TO_RELAUNCH
@@ -208,14 +208,84 @@ Same as the argument \"stmp\" described above but for setting the
 experiment variable PTMP for all tests that will run in NCO mode.
 
 compiler:
-Type of compiler to use for the workflow. Options are \"intel\" 
-and \"gnu\". Default is \"intel\",
+Type of compiler to use for the workflow. Options are \"intel\" and \"gnu\". 
+Default is \"intel\".
 
 build_env_fn:
-Specify the build environment (see ufs-srweather-app/envs) to 
-use for the workflow. (e.g. build_cheyenne_gnu.env). If a 
-\"gnu\" compiler is specified, it must also be specified with 
-the \"compiler\" option.
+Specify the build environment (see ufs-srweather-app/envs) to use for 
+the workflow. (e.g. build_cheyenne_gnu.env). If a \"gnu\" compiler is 
+specified, it must also be specified with the \"compiler\" option.
+
+
+Usage Examples:
+--------------
+Here, we give several common usage examples.  In the following, assume 
+my_tests.txt is a text file in the same directory as this script containing 
+a list of test names that we want to run, e.g.
+
+> more my_tests.txt
+new_ESGgrid
+specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE
+
+Then:
+
+1) To run the tests listed in my_tests.txt on Hera and charge the core-
+   hours used to the \"rtrr\" account, issue:
+
+     > run_WE2E_tests.sh tests_file=\"my_tests.txt\" machine=\"hera\" account=\"rtrr\"
+
+   This will create the experiment subdirectories for the two tests in
+   the directory
+
+     \${SR_WX_APP_TOP_DIR}/../expt_dirs
+
+   where {SR_WX_APP_TOP_DIR} is the directory in which the ufs-srweather-app 
+   repository is cloned.  Thus, the following two experiment directories
+   will be created:
+
+     \${SR_WX_APP_TOP_DIR}/../expt_dirs/new_ESGgrid
+     \${SR_WX_APP_TOP_DIR}/../expt_dirs/specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE
+
+   In addition, by default, cron jobs will be created in the user's cron
+   table to relaunch the workflows of these experiments every 2 minutes.
+
+2) To change the frequency with which the cron relaunch jobs are submitted
+   from the default of 2 minutes to 1 minute, issue:
+
+     > run_WE2E_tests.sh tests_file=\"my_tests.txt\" machine=\"hera\" account=\"rtrr\" cron_relaunch_intvl_mnts=\"01\"
+
+3) To disable use of cron (which means the worfkow for each test will 
+   have to be relaunched manually from within each experiment directory),
+   use
+
+     > run_WE2E_tests.sh tests_file=\"my_tests.txt\" machine=\"hera\" account=\"rtrr\" use_cron_to_relaunch=\"FALSE\"
+
+4) To place the experiment subdirectories in a subdirectory under 
+
+     \${SR_WX_APP_TOP_DIR}/../expt_dirs
+
+   instead of immediately under it, use
+
+     > run_WE2E_tests.sh tests_file=\"my_tests.txt\" machine=\"hera\" account=\"rtrr\" expt_basedir=\"test_set_01\"
+
+   In this case, the full paths to the experiment directories will be:
+
+     \${SR_WX_APP_TOP_DIR}/../expt_dirs/test_set_01/new_ESGgrid
+     \${SR_WX_APP_TOP_DIR}/../expt_dirs/test_set_01/specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE
+
+5) To use a list of tests that is not located in the directory of this
+   script but instead at
+
+     /path/to/custom/my_tests.txt
+
+   and to have the experiment directories be placed in an arbitrary 
+   location, say 
+
+     /path/to/custom/expt_dirs
+
+   use
+
+     > run_WE2E_tests.sh tests_file=\"/path/to/custom/my_tests.txt\" machine=\"hera\" account=\"rtrr\" expt_basedir=\"/path/to/custom/expt_dirs\"
 "
 #
 #-----------------------------------------------------------------------

From b474f12f2e87fa09f1e33d04d89309819fc09d2c Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Tue, 19 Apr 2022 15:41:42 -0600
Subject: [PATCH 09/14] Test commit on Mac.

---
 tests/WE2E/run_WE2E_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh
index f059f73d7..88f2fedf4 100755
--- a/tests/WE2E/run_WE2E_tests.sh
+++ b/tests/WE2E/run_WE2E_tests.sh
@@ -218,7 +218,7 @@ specified, it must also be specified with the \"compiler\" option.
 
 
 Usage Examples:
---------------
+---------------
 Here, we give several common usage examples.  In the following, assume 
 my_tests.txt is a text file in the same directory as this script containing 
 a list of test names that we want to run, e.g.

From 36a610cf692b68c8bbc688e236f00ef771c3cfb1 Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Tue, 19 Apr 2022 15:53:04 -0600
Subject: [PATCH 10/14] Another test commit from Mac.

---
 tests/WE2E/run_WE2E_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh
index 88f2fedf4..f059f73d7 100755
--- a/tests/WE2E/run_WE2E_tests.sh
+++ b/tests/WE2E/run_WE2E_tests.sh
@@ -218,7 +218,7 @@ specified, it must also be specified with the \"compiler\" option.
 
 
 Usage Examples:
----------------
+--------------
 Here, we give several common usage examples.  In the following, assume 
 my_tests.txt is a text file in the same directory as this script containing 
 a list of test names that we want to run, e.g.

From 4a3608aa54b50b44780bf37f6b21653386a1d494 Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Thu, 21 Apr 2022 12:38:17 -0600
Subject: [PATCH 11/14] Minor commits to help/usage section.

---
 tests/WE2E/run_WE2E_tests.sh | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh
index f059f73d7..ca3619d57 100755
--- a/tests/WE2E/run_WE2E_tests.sh
+++ b/tests/WE2E/run_WE2E_tests.sh
@@ -230,7 +230,7 @@ specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE
 Then:
 
 1) To run the tests listed in my_tests.txt on Hera and charge the core-
-   hours used to the \"rtrr\" account, issue:
+   hours used to the \"rtrr\" account, use:
 
      > run_WE2E_tests.sh tests_file=\"my_tests.txt\" machine=\"hera\" account=\"rtrr\"
 
@@ -239,7 +239,7 @@ Then:
 
      \${SR_WX_APP_TOP_DIR}/../expt_dirs
 
-   where {SR_WX_APP_TOP_DIR} is the directory in which the ufs-srweather-app 
+   where SR_WX_APP_TOP_DIR is the directory in which the ufs-srweather-app 
    repository is cloned.  Thus, the following two experiment directories
    will be created:
 
@@ -250,21 +250,22 @@ Then:
    table to relaunch the workflows of these experiments every 2 minutes.
 
 2) To change the frequency with which the cron relaunch jobs are submitted
-   from the default of 2 minutes to 1 minute, issue:
+   from the default of 2 minutes to 1 minute, use:
 
      > run_WE2E_tests.sh tests_file=\"my_tests.txt\" machine=\"hera\" account=\"rtrr\" cron_relaunch_intvl_mnts=\"01\"
 
 3) To disable use of cron (which means the worfkow for each test will 
    have to be relaunched manually from within each experiment directory),
-   use
+   use:
 
      > run_WE2E_tests.sh tests_file=\"my_tests.txt\" machine=\"hera\" account=\"rtrr\" use_cron_to_relaunch=\"FALSE\"
 
-4) To place the experiment subdirectories in a subdirectory under 
+4) To place the experiment subdirectories in a subdirectory named \"test_set_01\"
+   under 
 
      \${SR_WX_APP_TOP_DIR}/../expt_dirs
 
-   instead of immediately under it, use
+   (instead of immediately under the latter), use:
 
      > run_WE2E_tests.sh tests_file=\"my_tests.txt\" machine=\"hera\" account=\"rtrr\" expt_basedir=\"test_set_01\"
 
@@ -273,17 +274,16 @@ Then:
      \${SR_WX_APP_TOP_DIR}/../expt_dirs/test_set_01/new_ESGgrid
      \${SR_WX_APP_TOP_DIR}/../expt_dirs/test_set_01/specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE
 
-5) To use a list of tests that is not located in the directory of this
-   script but instead at
+5) To use a list of tests that is located in
 
      /path/to/custom/my_tests.txt
 
-   and to have the experiment directories be placed in an arbitrary 
-   location, say 
+   instead of in the same directory as this script, and to have the 
+   experiment directories be placed in an arbitrary location, say 
 
      /path/to/custom/expt_dirs
 
-   use
+   use:
 
      > run_WE2E_tests.sh tests_file=\"/path/to/custom/my_tests.txt\" machine=\"hera\" account=\"rtrr\" expt_basedir=\"/path/to/custom/expt_dirs\"
 "

From e2019b871eac8d75ad0f486a506e1d02ca492f2d Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Thu, 21 Apr 2022 13:23:00 -0600
Subject: [PATCH 12/14] Modifications to get_expts_status.sh to: 1) Add
 "official" arguments list (that gets processed via "process_args"); 2) Add
 "verbose" argument to enable less output to screen; 3) include help/usage
 string.

---
 tests/WE2E/get_expts_status.sh | 118 +++++++++++++++++++++++++--------
 1 file changed, 90 insertions(+), 28 deletions(-)

diff --git a/tests/WE2E/get_expts_status.sh b/tests/WE2E/get_expts_status.sh
index 01b127d9f..eda38c42f 100755
--- a/tests/WE2E/get_expts_status.sh
+++ b/tests/WE2E/get_expts_status.sh
@@ -70,42 +70,98 @@ ushdir="$homerrfs/ush"
 #
 #-----------------------------------------------------------------------
 #
-# Exactly one argument must be specified that consists of the full path
-# to the experiments base directory (i.e. the directory containing the 
-# experiment subdirectories).  Ensure that the number of arguments is 
-# one.
+# Set the usage message.
 #
 #-----------------------------------------------------------------------
 #
-num_args="$#"
-if [ "${num_args}" -eq 1 ]; then
-  expts_basedir="$1"
-else
-  print_err_msg_exit "
-The number of arguments to this script must be exacty one, and that 
-argument must specify the experiments base directory, i.e. the directory
-containing the experiment subdirectories.  The acutal number of arguments 
-is:
-  num_args = ${num_args}"  
+usage_str="\
+Usage:
+
+  ${scrfunc_fn} \\
+    expts_basedir=\"...\" \\
+    [verbose=\"...\"]
+
+The arguments in brackets are optional.  The arguments are defined as
+follows:
+
+expts_basedir:
+Full path to the experiments base directory, i.e. the directory containing 
+the experiment subdirectories.
+
+verbose:
+Optional verbosity flag.  Should be set to \"TRUE\" or \"FALSE\".  Default
+is \"FALSE\".
+"
+#
+#-----------------------------------------------------------------------
+#
+# Check to see if usage help for this script is being requested.  If so,
+# print it out and exit with a 0 exit code (success).
+#
+#-----------------------------------------------------------------------
+#
+help_flag="--help"
+if [ "$#" -eq 1 ] && [ "$1" = "${help_flag}" ]; then
+  print_info_msg "${usage_str}"
+  exit 0
+fi
+#
+#-----------------------------------------------------------------------
+#
+# Specify the set of valid argument names for this script or function.
+# Then process the arguments provided to it on the command line (which
+# should consist of a set of name-value pairs of the form arg1="value1",
+# arg2="value2", etc).
+#
+#-----------------------------------------------------------------------
+#
+valid_args=( \
+  "expts_basedir" \
+  "verbose" \
+  )
+process_args valid_args "$@"
+#
+#-----------------------------------------------------------------------
+#
+# Make the default value of "verbose" "FALSE".  Then make sure "verbose"
+# is set to a valid value.
+#
+#-----------------------------------------------------------------------
+#
+verbose=${verbose:-"FALSE"}
+check_var_valid_value "verbose" "valid_vals_BOOLEAN"
+verbose=$(boolify $verbose)
+#
+#-----------------------------------------------------------------------
+#
+# Verify that the required arguments to this script have been specified.
+# If not, print out an error message and exit.
+#
+#-----------------------------------------------------------------------
+#
+help_msg="\
+Use
+  ${scrfunc_fn} ${help_flag}
+to get help on how to use this script."
+
+if [ -z "${expts_basedir}" ]; then
+  print_err_msg_exit "\
+The argument \"expts_basedir\" specifying the base directory containing
+the experiment directories was not specified in the call to this script.  \
+${help_msg}"
 fi
 #
 #-----------------------------------------------------------------------
 #
 # Check that the specified experiments base directory exists and is 
 # actually a directory.  If not, print out an error message and exit.
-# If so, print out an informational message.
 #
 #-----------------------------------------------------------------------
 #
 if [ ! -d "${expts_basedir}" ]; then
   print_err_msg_exit "
-The experiments base directory (expts_basedir) does not exit or is not 
-actually a directory:
-  expts_basedir = \"${expts_basedir}\""
-else
-  print_info_msg "
-Checking the workflow status of all forecast experiments in the following
-specified experiments base directory: 
+The specified experiments base directory (expts_basedir) does not exit 
+or is not actually a directory:
   expts_basedir = \"${expts_basedir}\""
 fi
 #
@@ -116,7 +172,7 @@ fi
 #
 #-----------------------------------------------------------------------
 #
-cd "${expts_basedir}"
+cd_vrfy "${expts_basedir}"
 #
 # Get a list of all subdirectories (but not files) in the experiment base 
 # directory.  Note that the ls command below will return a string containing
@@ -175,6 +231,12 @@ var_defns_fn="var_defns.sh"
 j="0"
 expt_subdirs=()
 
+print_info_msg "\
+Checking for active experiment directories in the specified experiments
+base directory (expts_basedir):
+  expts_basedir = \"${expts_basedir}\"
+..."
+
 num_subdirs="${#subdirs_list[@]}"
 for (( i=0; i<=$((num_subdirs-1)); i++ )); do
 
@@ -184,7 +246,7 @@ $separator
 Checking whether the subdirectory 
   \"${subdir}\"
 contains an active experiment..."
-  print_info_msg "$msg"
+  print_info_msg "$verbose" "$msg"
 
   cd_vrfy "${subdir}"
 #
@@ -193,7 +255,7 @@ contains an active experiment..."
 #
   if [ ! -f "${var_defns_fn}" ]; then
 
-    print_info_msg "
+    print_info_msg "$verbose" "
 The current subdirectory (subdir) under the experiments base directory
 (expts_basedir) does not contain an experiment variable defintions file
 (var_defns_fn):
@@ -219,7 +281,7 @@ must be checked."
 #
     if [ "${EXPT_SUBDIR}" = "$subdir" ]; then
 
-      print_info_msg "
+      print_info_msg "$verbose" "
 The current subdirectory (subdir) under the experiments base directory
 (expts_basedir) contains an active experiment:
   expts_basedir = \"${expts_basedir}\"
@@ -238,7 +300,7 @@ subdirectories whose workflow status must be checked."
 #
     else
 
-      print_info_msg "
+      print_info_msg "$verbose" "
 The current subdirectory (subdir) under the experiments base directory
 (expts_basedir) contains an experiment whose original name (EXPT_SUBDIR)
 does not match the name of the current subdirectory:
@@ -254,7 +316,7 @@ status must be checked."
 
   fi
 
-  print_info_msg "\
+  print_info_msg "$verbose" "\
 $separator
 "
 #

From 9320442b8f5e2168130bac11d2ecf8890e606906 Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Thu, 21 Apr 2022 14:26:56 -0600
Subject: [PATCH 13/14] Make the number of lines at the end of the log file
 that gets written to the status report file an argument.

---
 tests/WE2E/get_expts_status.sh | 34 +++++++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/tests/WE2E/get_expts_status.sh b/tests/WE2E/get_expts_status.sh
index eda38c42f..4978f4456 100755
--- a/tests/WE2E/get_expts_status.sh
+++ b/tests/WE2E/get_expts_status.sh
@@ -13,7 +13,7 @@
 # For all such experiments, it calls the workflow (re)launch script to 
 # update the status of the workflow and prints the status out to screen.
 # It also generates a summary status file in the base directory that 
-# contains the last num_tail_lines lines (defined below) of each experiment's 
+# contains the last num_log_lines lines (defined below) of each experiment's 
 # workflow log file [which is generated by the (re)launch script] and thus 
 # has information on which tasks may have succeeded/failed].
 #
@@ -88,6 +88,11 @@ expts_basedir:
 Full path to the experiments base directory, i.e. the directory containing 
 the experiment subdirectories.
 
+num_log_lines:
+Optional integer specifying the number of lines from the end of the 
+workflow launch log file (log.launch_FV3LAM_wflow) of each test to 
+include in the status report file that this script generates.
+
 verbose:
 Optional verbosity flag.  Should be set to \"TRUE\" or \"FALSE\".  Default
 is \"FALSE\".
@@ -117,12 +122,21 @@ fi
 #
 valid_args=( \
   "expts_basedir" \
+  "num_log_lines" \
   "verbose" \
   )
 process_args valid_args "$@"
 #
 #-----------------------------------------------------------------------
 #
+# Set the default value of "num_log_lines".
+#
+#-----------------------------------------------------------------------
+#
+num_log_lines=${num_log_lines:-"40"}
+#
+#-----------------------------------------------------------------------
+#
 # Make the default value of "verbose" "FALSE".  Then make sure "verbose"
 # is set to a valid value.
 #
@@ -364,7 +378,7 @@ check_for_preexist_dir_file "${expts_status_fp}" "rename"
 # Loop through the elements of the array expt_subdirs.  For each element
 # (i.e. for each active experiment), change location to the experiment 
 # directory and call the script launch_FV3LAM_wflow.sh to update the log 
-# file log.launch_FV3LAM_wflow.  Then take the last num_tail_lines of 
+# file log.launch_FV3LAM_wflow.  Then take the last num_log_lines of 
 # this log file (along with an appropriate message) and add it to the 
 # status report file.
 #
@@ -372,7 +386,6 @@ check_for_preexist_dir_file "${expts_status_fp}" "rename"
 #
 launch_wflow_fn="launch_FV3LAM_wflow.sh"
 launch_wflow_log_fn="log.launch_FV3LAM_wflow"
-num_tail_lines="40"
 
 for (( i=0; i<=$((num_expts-1)); i++ )); do
 
@@ -388,25 +401,28 @@ Checking workflow status of experiment \"${expt_subdir}\" ..."
 #
   cd_vrfy "${expt_subdir}"
   launch_msg=$( "${launch_wflow_fn}" 2>&1 )
-  log_tail=$( tail -n ${num_tail_lines} "${launch_wflow_log_fn}" )
+  log_tail=$( tail -n ${num_log_lines} "${launch_wflow_log_fn}" )
 #
 # Print the workflow status to the screen.
 #
-  wflow_status=$( printf "${log_tail}" | grep "Workflow status:" )
-#  wflow_status="${wflow_status## }"  # Not sure why this doesn't work to strip leading spaces.
-  wflow_status=$( printf "${wflow_status}" "%s" | sed -r 's|^[ ]*||g' )  # Remove leading spaces.
+  # The "tail -1" is to get only the last occurrence of "Workflow status"
+  wflow_status=$( printf "${log_tail}" | grep "Workflow status:" | tail -1 )
+  # Not sure why this doesn't work to strip leading spaces.
+#  wflow_status="${wflow_status## }"
+  # Remove leading spaces.
+  wflow_status=$( printf "${wflow_status}" "%s" | sed -r 's|^[ ]*||g' )
   print_info_msg "${wflow_status}"
   print_info_msg "\
 $separator
 "
 #
-# Combine message above with the last num_tail_lines lines from the workflow 
+# Combine message above with the last num_log_lines lines from the workflow 
 # launch log file and place the result in the status report file.
 #
   msg=$msg"
 ${wflow_status}
 
-The last ${num_tail_lines} lines of this experiment's workflow launch log file 
+The last ${num_log_lines} lines of this experiment's workflow launch log file 
 (\"${launch_wflow_log_fn}\") are:
 
 ${log_tail}

From 7ea8f4bf2613951b5b522b1d6e4d230c4d36b780 Mon Sep 17 00:00:00 2001
From: gerard ketefian <gerard.ketefian@noaa.gov>
Date: Thu, 21 Apr 2022 15:25:58 -0600
Subject: [PATCH 14/14] Add informational message and edit comments.

---
 tests/WE2E/get_expts_status.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/WE2E/get_expts_status.sh b/tests/WE2E/get_expts_status.sh
index 4978f4456..91de215d2 100755
--- a/tests/WE2E/get_expts_status.sh
+++ b/tests/WE2E/get_expts_status.sh
@@ -12,7 +12,7 @@
 # directory represent active experiments (see below for how this is done).
 # For all such experiments, it calls the workflow (re)launch script to 
 # update the status of the workflow and prints the status out to screen.
-# It also generates a summary status file in the base directory that 
+# It also generates a status report file in the base directory that 
 # contains the last num_log_lines lines (defined below) of each experiment's 
 # workflow log file [which is generated by the (re)launch script] and thus 
 # has information on which tasks may have succeeded/failed].
@@ -438,4 +438,7 @@ ${log_tail}
 done
 
 print_info_msg "\
+A status report has been created in:
+  expts_status_fp = \"${expts_status_fp}\"
+
 DONE."