From 52ed745db8678e340b578922b0978c09ca1f8210 Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Thu, 2 Jun 2022 09:34:01 -0400
Subject: [PATCH 01/17] Normalize Parallel Works cluster platform value

Set the value of platform to 'noaacloud' when SRW_PLATFORM matches a
Parallel Works cluster name.
---
 .cicd/scripts/srw_build.sh |  12 +++-
 .cicd/scripts/srw_test.sh  | 123 +++++++++++++++++++++++++++++++++++++
 2 files changed, 133 insertions(+), 2 deletions(-)

diff --git a/.cicd/scripts/srw_build.sh b/.cicd/scripts/srw_build.sh
index 638d538fa6..154ba23fbc 100755
--- a/.cicd/scripts/srw_build.sh
+++ b/.cicd/scripts/srw_build.sh
@@ -16,12 +16,20 @@ else
     workspace="$(cd -- "${script_dir}/../.." && pwd)"
 fi
 
+# Normalize Parallel Works cluster platform value.
+declare platform
+if [[ "${SRW_PLATFORM}" =~ ^(az|g|p)cluster_noaa ]]; then
+    platform='noaacloud'
+else
+    platform="${SRW_PLATFORM}"
+fi
+
 build_dir="${workspace}/build"
 
 # Set build related environment variables and load required modules.
-source "${workspace}/etc/lmod-setup.sh" "${SRW_PLATFORM}"
+source "${workspace}/etc/lmod-setup.sh" "${platform}"
 module use "${workspace}/modulefiles"
-module load "build_${SRW_PLATFORM}_${SRW_COMPILER}"
+module load "build_${platform}_${SRW_COMPILER}"
 
 # Compile SRW application and install to repository root.
 mkdir "${build_dir}"
diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh
index 43b9935888..39e0f36a25 100755
--- a/.cicd/scripts/srw_test.sh
+++ b/.cicd/scripts/srw_test.sh
@@ -17,5 +17,128 @@ else
     workspace="$(cd -- "${script_dir}/../.." && pwd)"
 fi
 
+# Normalize Parallel Works cluster platform value.
+declare platform
+if [[ "${SRW_PLATFORM}" =~ ^(az|g|p)cluster_noaa ]]; then
+    platform='noaacloud'
+else
+    platform="${SRW_PLATFORM}"
+fi
+
+we2e_experiment_base_dir="${workspace}/experiments"
+we2e_test_dir="${workspace}/regional_workflow/tests/WE2E"
+
+we2e_test_file="${we2e_test_dir}/experiments.txt"
+
+# The default set of end-to-end tests to run.
+# TODO: Create a list of additional tests that can be run when a parameter
+# is set to true.
+declare -a we2e_default_tests
+we2e_default_tests=('grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16')
+
+# Parses the test log for the status of a specific test.
+function workflow_status() {
+    local test="$1"
+
+    local test_dir="${we2e_experiment_base_dir}/${test}"
+    local log_file="${test_dir}/log.launch_FV3LAM_wflow"
+
+    if [[ -f "${log_file}" ]]; then
+        local status
+        status="$(awk 'BEGIN {FS=":";} $1 ~ "^[[:space:]]+Workflow status" {print $2}' "${log_file}" |\
+            tail -1 |\
+            sed --regexp-extended --expression 's/^[[:space:]]*(.*)[[:space:]]*$/\1/')"
+        if [[ "${status}" == 'IN PROGRESS' || "${status}" == 'SUCCESS' || "${status}" == 'FAILURE' ]]; then
+            echo "${status}"
+        else
+            echo 'UNKNOWN'
+        fi
+    else
+        echo 'NOT FOUND'
+    fi
+}
+
+# Gets the status of all tests. Prints the number of tests that are running.
+# Returns a non-zero code when all tests reach a final state.
+function check_progress() {
+    local in_progress=false
+    local remaining=0
+
+    for test in "${we2e_default_tests[@]}"; do
+        local status
+        status="$(workflow_status "${test}")"
+        if [[ "${status}" == 'IN PROGRESS' ]]; then
+            in_progress=true
+            (( remaining++ ))
+        fi
+    done
+
+    if "${in_progress}"; then
+        echo "Tests remaining: ${remaining}"
+    else
+        return 1
+    fi
+}
+
+# Prints the status of all tests.
+function get_results() {
+    for test in "${we2e_default_tests[@]}"; do
+        local status
+        status="$(workflow_status "${test}")"
+        echo "${test} ${status}"
+    done
+}
+
 # Verify that there is a non-zero sized weather model executable.
 [[ -s "${workspace}/bin/ufs_model" ]] || [[ -s "${workspace}/bin/NEMS.exe" ]]
+
+# Set test related environment variables and load required modules.
+source "${workspace}/etc/lmod-setup.sh" "${platform}"
+module use "${workspace}/modulefiles"
+module load "build_${platform}_${SRW_COMPILER}"
+module load "wflow_${platform}"
+
+if [[ "${platform}" == 'cheyenne' ]]; then
+    export PATH="/glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427/bin:${PATH}"
+else
+    conda activate regional_workflow
+fi
+
+# Create the experiments/tests base directory.
+mkdir "${we2e_experiment_base_dir}"
+
+# Generate the experiments/tests file.
+for test in "${we2e_default_tests[@]}"; do
+    echo "${test}" >> "${we2e_test_file}"
+done
+
+# Run the end-to-end tests.
+"${we2e_test_dir}/run_WE2E_tests.sh" \
+    tests_file="${we2e_test_file}" \
+    machine="${platform}" \
+    account="${SRW_PROJECT}" \
+    expt_basedir="${we2e_experiment_base_dir}" \
+    compiler="${SRW_COMPILER}"
+
+# Allow the tests to start before checking for status.
+# TODO: Create a parameter that sets the initial start delay.
+sleep 180
+
+# Wait for all tests to complete.
+while check_progress; do
+    # TODO: Create a paremeter that sets the poll frequency.
+    sleep 60
+done
+
+# Get test results and write to a file.
+results="$(get_results |\
+    tee "${workspace}/test_results-${platform}-${SRW_COMPILER}.txt")"
+
+# Check that the number of tests equals the number of successes, otherwise
+# exit with a non-zero code that equals the difference.
+successes="$(awk '$2 == "SUCCESS" {print $1}' <<< "${results}" | wc -l)"
+if [[ "${#we2e_default_tests[@]}" -ne "${successes}" ]]; then
+    exit "$(( "${#we2e_default_tests[@]}" - "${successes}" ))"
+fi

From 3ba8f18e38680e8ea3026b6e120c3ffa6a13acae Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Thu, 2 Jun 2022 09:39:18 -0400
Subject: [PATCH 02/17] Enable the AWS Parallel Works platform

---
 .cicd/Jenkinsfile | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 095eecb200..109e5ac6d0 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -8,13 +8,12 @@ pipeline {
     parameters {
         // Allow job runner to filter based on platform
         // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pcluster_noaa_v2_use1', 'azcluster_noaa', 'gcluster_noaa_v2_usc1'], description: 'Specify the platform(s) to use')
-        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use')
+        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pcluster_noaa_v2_use1'], description: 'Specify the platform(s) to use')
         // Allow job runner to filter based on compiler
         choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
     }
 
     stages {
-        /*
         // Start the NOAA Parallel Works clusters, if necessary
         stage('Start Parallel Works Clusters') {
             matrix {
@@ -29,7 +28,7 @@ pipeline {
                 axes {
                     axis {
                         name 'SRW_PLATFORM'
-                        values 'pcluster_noaa_v2_use1', 'azcluster_noaa', 'gcluster_noaa_v2_usc1'
+                        values 'pcluster_noaa_v2_use1' //, 'azcluster_noaa', 'gcluster_noaa_v2_usc1'
                     }
                 }
 
@@ -44,7 +43,6 @@ pipeline {
                 }
             }
         }
-        */
 
         // Build and test the SRW application on all supported platforms using the supported compilers for each platform
         stage('Build and Test') {
@@ -68,8 +66,7 @@ pipeline {
                 axes {
                     axis {
                         name 'SRW_PLATFORM'
-                        // values 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pcluster_noaa_v2_use1', 'azcluster_noaa', 'gcluster_noaa_v2_usc1'
-                        values 'cheyenne', 'gaea', 'hera', 'jet', 'orion'
+                        values 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pcluster_noaa_v2_use1' //, 'azcluster_noaa', 'gcluster_noaa_v2_usc1'
                     }
 
                     axis {
@@ -83,7 +80,7 @@ pipeline {
                     exclude {
                             axis {
                             name 'SRW_PLATFORM'
-                            values 'gaea', 'hera', 'jet', 'orion'
+                            values 'gaea', 'hera', 'jet', 'orion', 'pcluster_noaa_v2_use1' //, 'azcluster_noaa', 'gcluster_noaa_v2_usc1'
                         }
 
                         axis {
@@ -141,12 +138,12 @@ pipeline {
         }
     }
 
-    /*
     post {
         always {
             // Stop any Parallel Works clusters that were started during the pipeline execution
             script {
-                def pw_clusters = ['pcluster_noaa_v2_use1', 'azcluster_noaa', 'gcluster_noaa_v2_usc1']
+                // def pw_clusters = ['pcluster_noaa_v2_use1', 'azcluster_noaa', 'gcluster_noaa_v2_usc1']
+                def pw_clusters = ['pcluster_noaa_v2_use1']
                 def clusters = []
 
                 // Determine which clusters need to be stopped, if any
@@ -166,5 +163,4 @@ pipeline {
             }
         }
     }
-    */
 }

From 01435ad550c453b5390cb2beb272b41c40a838db Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Thu, 2 Jun 2022 11:55:20 -0400
Subject: [PATCH 03/17] Move agent declaration to stages

This change allows the platform filter to work correctly, otherwise, the
Parallel Works clusters would block indefinitely waiting to execute the
matrix on a agent/node that was not started.
---
 .cicd/Jenkinsfile | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 109e5ac6d0..b96f3af046 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -90,10 +90,6 @@ pipeline {
                     }
                 }
 
-                agent {
-                    label env.SRW_PLATFORM
-                }
-
                 environment {
                     BRANCH_NAME_ESCAPED = env.BRANCH_NAME.replace('/', '_')
                     BUILD_VERSION = "${env.SRW_PLATFORM}-${env.SRW_COMPILER}-${env.BRANCH_NAME_ESCAPED}-${env.BUILD_NUMBER}"
@@ -103,6 +99,10 @@ pipeline {
                 stages {
                     // Clean the workspace, checkout the repository, and run checkout_externals
                     stage('Initialize') {
+                        agent {
+                            label env.SRW_PLATFORM
+                        }
+
                         steps {
                             echo "Initializing SRW (${env.SRW_COMPILER}) build environment on ${env.SRW_PLATFORM}"
                             cleanWs()
@@ -113,6 +113,10 @@ pipeline {
 
                     // Run the unified build script; if successful create a tarball of the build and upload to S3
                     stage('Build') {
+                        agent {
+                            label env.SRW_PLATFORM
+                        }
+
                         steps {
                             echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM}"
                             sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_build.sh"'
@@ -128,6 +132,10 @@ pipeline {
 
                     // Run the unified test script
                     stage('Test') {
+                        agent {
+                            label env.SRW_PLATFORM
+                        }
+
                         steps {
                             echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM}"
                             sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_test.sh"'

From c65b1e8044d78129476ef9642a30da24816f3bc3 Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Fri, 3 Jun 2022 11:37:40 -0400
Subject: [PATCH 04/17] Ensure PROJ_LIB is set on Parallel Works platforms

---
 .cicd/scripts/srw_test.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh
index 39e0f36a25..29e5892629 100755
--- a/.cicd/scripts/srw_test.sh
+++ b/.cicd/scripts/srw_test.sh
@@ -103,6 +103,10 @@ module load "wflow_${platform}"
 if [[ "${platform}" == 'cheyenne' ]]; then
     export PATH="/glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427/bin:${PATH}"
 else
+    if [[ "${platform}" == 'noaacloud' && -z "${PROJ_LIB-}" ]]; then
+        PROJ_LIB=''
+    fi
+
     conda activate regional_workflow
 fi
 

From 2a6348c286fcc8e4d681d46402a87a0f8467782a Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Sat, 4 Jun 2022 08:50:49 -0400
Subject: [PATCH 05/17] Fix final exit status of srw_test script

Some platforms do not recognize quoted variables within an arithmetic
expression. This change removes the quotes.
---
 .cicd/scripts/srw_test.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh
index 29e5892629..d7150cbe86 100755
--- a/.cicd/scripts/srw_test.sh
+++ b/.cicd/scripts/srw_test.sh
@@ -143,6 +143,4 @@ results="$(get_results |\
 # Check that the number of tests equals the number of successes, otherwise
 # exit with a non-zero code that equals the difference.
 successes="$(awk '$2 == "SUCCESS" {print $1}' <<< "${results}" | wc -l)"
-if [[ "${#we2e_default_tests[@]}" -ne "${successes}" ]]; then
-    exit "$(( "${#we2e_default_tests[@]}" - "${successes}" ))"
-fi
+exit "$(( ${#we2e_default_tests[@]} - ${successes} ))"

From ec1ce52b0cc1e9c92e1e5a1a240cbe8fd0ddf337 Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Mon, 6 Jun 2022 11:43:58 -0400
Subject: [PATCH 06/17] Add comprehensive end-to-end tests option

* Add a parameter to the Jenkins pipeline that allows the comprehensive
set of workflow and end-to-end tests to be executed during the test
stage.
* Add logic to the Jenkins pipeline that checks for a specific Pull
Request label, then overrides the comprehensive end-to-end test
parameter's value if set.
---
 .cicd/Jenkinsfile         | 19 +++++++++++-
 .cicd/scripts/srw_test.sh | 63 +++++++++++++++++++++++++++++++++------
 2 files changed, 72 insertions(+), 10 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index b96f3af046..2f4105c2ee 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -11,6 +11,7 @@ pipeline {
         choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pcluster_noaa_v2_use1'], description: 'Specify the platform(s) to use')
         // Allow job runner to filter based on compiler
         choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
+        booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests'
     }
 
     stages {
@@ -138,7 +139,23 @@ pipeline {
 
                         steps {
                             echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM}"
-                            sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_test.sh"'
+
+                            // If executing for a Pull Request, check for the run_we2e_comprehensive_tests. If set,
+                            // override the value of the SRW_WE2E_COMPREHENSIVE_TESTS parameter
+                            script {
+                                def run_we2e_comprehensive_tests = params.SRW_WE2E_COMPREHENSIVE_TESTS
+                                def run_we2e_comprehensive_tests_label = 'run_we2e_comprehensive_tests'
+
+                                if (env.CHANGE_ID) {
+                                    pullRequest.labels.each {
+                                        if (it == run_we2e_comprehensive_tests_label) {
+                                            run_we2e_comprehensive_tests = true
+                                        }
+                                    }
+                                }
+
+                                sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests} bash --login ${env.WORKSPACE}/.cicd/scripts/srw_test.sh"
+                            }
                         }
                     }
                 }
diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh
index d7150cbe86..c55bd40f8b 100755
--- a/.cicd/scripts/srw_test.sh
+++ b/.cicd/scripts/srw_test.sh
@@ -30,14 +30,59 @@ we2e_test_dir="${workspace}/regional_workflow/tests/WE2E"
 
 we2e_test_file="${we2e_test_dir}/experiments.txt"
 
-# The default set of end-to-end tests to run.
-# TODO: Create a list of additional tests that can be run when a parameter
-# is set to true.
-declare -a we2e_default_tests
-we2e_default_tests=('grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+# The fundamental set of end-to-end tests to run.
+declare -a we2e_fundamental_tests
+we2e_fundamental_tests=('grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
     'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
     'grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16')
 
+# The comprehensive set of end-to-end tests to run.
+declare -a we2e_comprehensive_tests
+we2e_comprehensive_tests=('grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
+    'grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
+    'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
+    'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR'
+    'grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR'
+    'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR'
+)
+
+# The set of workflow tests to run.
+declare -a we2e_workflow_tests
+we2e_workflow_tests=('community_ensemble_008mems'
+    'community_ensemble_2mems'
+    'deactivate_tasks'
+    'inline_post'
+    'MET_ensemble_verification'
+    'MET_verification'
+    'custom_ESGgrid'
+    'custom_GFDLgrid'
+    'custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE'
+    'custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE'
+    'specify_DOT_OR_USCORE'
+    'specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE'
+    'specify_RESTART_INTERVAL'
+    'specify_template_filenames'
+    # 'subhourly_post_ensemble_2mems'
+    # 'subhourly_post'
+)
+
+declare -a we2e_tests
+we2e_tests=("${we2e_fundamental_tests[@]}")
+if "${SRW_WE2E_COMPREHENSIVE_TESTS}"; then
+    we2e_tests+=("${we2e_comprehensive_tests[@]}" "${we2e_workflow_tests[@]}")
+
+    # Add additional tests for Hera.
+    if [[ "${platform}" == 'hera' ]]; then
+        we2e_tests+=('specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS')
+    fi
+
+    # Add additional tests for all platforms, except Gaea and Parallel Works.
+    if [[ "${platform}" != 'gaea' && "${platform}" != 'noaacloud' ]]; then
+        we2e_tests+=('pregen_grid_orog_sfc_climo')
+    fi
+fi
+
 # Parses the test log for the status of a specific test.
 function workflow_status() {
     local test="$1"
@@ -66,7 +111,7 @@ function check_progress() {
     local in_progress=false
     local remaining=0
 
-    for test in "${we2e_default_tests[@]}"; do
+    for test in "${we2e_tests[@]}"; do
         local status
         status="$(workflow_status "${test}")"
         if [[ "${status}" == 'IN PROGRESS' ]]; then
@@ -84,7 +129,7 @@ function check_progress() {
 
 # Prints the status of all tests.
 function get_results() {
-    for test in "${we2e_default_tests[@]}"; do
+    for test in "${we2e_tests[@]}"; do
         local status
         status="$(workflow_status "${test}")"
         echo "${test} ${status}"
@@ -114,7 +159,7 @@ fi
 mkdir "${we2e_experiment_base_dir}"
 
 # Generate the experiments/tests file.
-for test in "${we2e_default_tests[@]}"; do
+for test in "${we2e_tests[@]}"; do
     echo "${test}" >> "${we2e_test_file}"
 done
 
@@ -143,4 +188,4 @@ results="$(get_results |\
 # Check that the number of tests equals the number of successes, otherwise
 # exit with a non-zero code that equals the difference.
 successes="$(awk '$2 == "SUCCESS" {print $1}' <<< "${results}" | wc -l)"
-exit "$(( ${#we2e_default_tests[@]} - ${successes} ))"
+exit "$(( ${#we2e_tests[@]} - ${successes} ))"

From 80fe8b7666aa0d4d9d0f1151e27db8cf3372a5f9 Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Tue, 7 Jun 2022 11:13:26 -0400
Subject: [PATCH 07/17] Clean up the workspace after a we2e test run

The experiments directory uses a lot of disk space. Removing it after
the end-to-end tests complete will allow us to keep the workspaces
longer. However, the test logs should be preserved. This change creates
a tarball containing the test logs in the workspace, which is archived,
then removes the experiments directory.
---
 .cicd/Jenkinsfile         | 13 +++++++++++++
 .cicd/scripts/srw_test.sh | 10 ++++++++--
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 2f4105c2ee..ed0e50b1ef 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -137,6 +137,10 @@ pipeline {
                             label env.SRW_PLATFORM
                         }
 
+                        environment {
+                            SRW_WE2E_EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/experiments"
+                        }
+
                         steps {
                             echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM}"
 
@@ -157,6 +161,15 @@ pipeline {
                                 sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests} bash --login ${env.WORKSPACE}/.cicd/scripts/srw_test.sh"
                             }
                         }
+
+                        post {
+                            always {
+                                // Archive the test log files and remove the experiments directory to conserve disk space
+                                sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --file "${WORKSPACE}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log.launch_FV3LAM_wflow */log/*'
+                                sh 'rm -rf "${SRW_WE2E_EXPERIMENT_BASE_DIR}"'
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_logs-*-*.tgz', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                            }
+                        }
                     }
                 }
             }
diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh
index c55bd40f8b..9e8d8f3432 100755
--- a/.cicd/scripts/srw_test.sh
+++ b/.cicd/scripts/srw_test.sh
@@ -25,7 +25,13 @@ else
     platform="${SRW_PLATFORM}"
 fi
 
-we2e_experiment_base_dir="${workspace}/experiments"
+declare we2e_experiment_base_dir
+if [[ -n "${SRW_WE2E_EXPERIMENT_BASE_DIR}" ]]; then
+    we2e_experiment_base_dir="${SRW_WE2E_EXPERIMENT_BASE_DIR}"
+else
+    we2e_experiment_base_dir="${workspace}/experiments"
+fi
+
 we2e_test_dir="${workspace}/regional_workflow/tests/WE2E"
 
 we2e_test_file="${we2e_test_dir}/experiments.txt"
@@ -183,7 +189,7 @@ done
 
 # Get test results and write to a file.
 results="$(get_results |\
-    tee "${workspace}/test_results-${platform}-${SRW_COMPILER}.txt")"
+    tee "${workspace}/we2e_test_results-${platform}-${SRW_COMPILER}.txt")"
 
 # Check that the number of tests equals the number of successes, otherwise
 # exit with a non-zero code that equals the difference.

From 9beeca1d16f46447b931cfe660ad0e724ab46cf9 Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Tue, 14 Jun 2022 08:17:59 -0400
Subject: [PATCH 08/17] Disable concurrent builds for branches and PRs

Prevent Jenkins from executing multiple pipelines at the same time for a
given branch or change request.
---
 .cicd/Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index ed0e50b1ef..1e65e5c29b 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -2,6 +2,7 @@ pipeline {
     agent none
 
     options {
+        disableConcurrentBuilds()
         skipDefaultCheckout(true)
     }
 

From 2a8cfc7b8dfeb1ebe95b26ab53045650f3ec02a7 Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Tue, 14 Jun 2022 10:32:46 -0400
Subject: [PATCH 09/17] Disable branch indexing triggers for pipeline

---
 .cicd/Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 1e65e5c29b..25b9879883 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -3,6 +3,7 @@ pipeline {
 
     options {
         disableConcurrentBuilds()
+        overrideIndexTriggers(false)
         skipDefaultCheckout(true)
     }
 

From d309fc3da6d9b0c2d5cf6e9c9b850ae1b453877b Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Tue, 14 Jun 2022 15:55:58 -0400
Subject: [PATCH 10/17] Update we2e fundamental tests for srw_test script

---
 .cicd/scripts/srw_test.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh
index 9e8d8f3432..5392abfaa8 100755
--- a/.cicd/scripts/srw_test.sh
+++ b/.cicd/scripts/srw_test.sh
@@ -40,7 +40,10 @@ we2e_test_file="${we2e_test_dir}/experiments.txt"
 declare -a we2e_fundamental_tests
 we2e_fundamental_tests=('grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
     'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
-    'grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16')
+    'grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
+    'grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR'
+    'grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0')
 
 # The comprehensive set of end-to-end tests to run.
 declare -a we2e_comprehensive_tests

From 2e4887e8c040088f54c9fddd6752757ccf05fc87 Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Wed, 15 Jun 2022 16:02:01 -0400
Subject: [PATCH 11/17] Log and archive the output of the srw build

---
 .cicd/Jenkinsfile          | 2 +-
 .cicd/scripts/srw_build.sh | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index 25b9879883..f57218bfeb 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -128,7 +128,7 @@ pipeline {
                         post {
                             success {
                                 sh 'tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" bin include lib share'
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "build/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.log", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                         }
                     }
diff --git a/.cicd/scripts/srw_build.sh b/.cicd/scripts/srw_build.sh
index 154ba23fbc..1fe338d306 100755
--- a/.cicd/scripts/srw_build.sh
+++ b/.cicd/scripts/srw_build.sh
@@ -34,6 +34,7 @@ module load "build_${platform}_${SRW_COMPILER}"
 # Compile SRW application and install to repository root.
 mkdir "${build_dir}"
 pushd "${build_dir}"
-    cmake -DCMAKE_INSTALL_PREFIX="${workspace}" -DENABLE_RRFS=on "${workspace}"
-    make -j "${MAKE_JOBS}"
+    build_log_file="${build_dir}/srw_build-${platform}-${SRW_COMPILER}.log"
+    cmake -DCMAKE_INSTALL_PREFIX="${workspace}" "${workspace}" | tee "${build_log_file}"
+    make -j "${MAKE_JOBS}" | tee --append "${build_log_file}"
 popd

From 0bb18c28fe4efc9e0f8845db0539e1bac6d6bb10 Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Wed, 22 Jun 2022 10:21:38 -0400
Subject: [PATCH 12/17] Update we2e comprehensive tests in srw_test script

---
 .cicd/scripts/srw_test.sh | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh
index 5392abfaa8..d850b3a205 100755
--- a/.cicd/scripts/srw_test.sh
+++ b/.cicd/scripts/srw_test.sh
@@ -54,7 +54,12 @@ we2e_comprehensive_tests=('grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_
     'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR'
     'grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR'
     'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR'
-)
+    'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
+    'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
+    'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
+    'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
+    'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
+    'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR')
 
 # The set of workflow tests to run.
 declare -a we2e_workflow_tests
@@ -62,8 +67,6 @@ we2e_workflow_tests=('community_ensemble_008mems'
     'community_ensemble_2mems'
     'deactivate_tasks'
     'inline_post'
-    'MET_ensemble_verification'
-    'MET_verification'
     'custom_ESGgrid'
     'custom_GFDLgrid'
     'custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE'
@@ -79,7 +82,8 @@ we2e_workflow_tests=('community_ensemble_008mems'
 declare -a we2e_tests
 we2e_tests=("${we2e_fundamental_tests[@]}")
 if "${SRW_WE2E_COMPREHENSIVE_TESTS}"; then
-    we2e_tests+=("${we2e_comprehensive_tests[@]}" "${we2e_workflow_tests[@]}")
+    we2e_tests+=("${we2e_comprehensive_tests[@]}"
+        "${we2e_workflow_tests[@]}")
 
     # Add additional tests for Hera.
     if [[ "${platform}" == 'hera' ]]; then
@@ -88,7 +92,9 @@ if "${SRW_WE2E_COMPREHENSIVE_TESTS}"; then
 
     # Add additional tests for all platforms, except Gaea and Parallel Works.
     if [[ "${platform}" != 'gaea' && "${platform}" != 'noaacloud' ]]; then
-        we2e_tests+=('pregen_grid_orog_sfc_climo')
+        we2e_tests+=('pregen_grid_orog_sfc_climo'
+            'MET_ensemble_verification'
+            'MET_verification')
     fi
 fi
 

From ba83efc14fb793ae4c16b8e9a5d447a14b1e21b2 Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Fri, 29 Jul 2022 12:45:09 -0400
Subject: [PATCH 13/17] Update we2e fundamental/comprehensive tests

---
 .cicd/scripts/srw_test.sh | 65 +++++++++++++++++++--------------------
 1 file changed, 32 insertions(+), 33 deletions(-)

diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh
index d850b3a205..b58ccba965 100755
--- a/.cicd/scripts/srw_test.sh
+++ b/.cicd/scripts/srw_test.sh
@@ -39,63 +39,62 @@ we2e_test_file="${we2e_test_dir}/experiments.txt"
 # The fundamental set of end-to-end tests to run.
 declare -a we2e_fundamental_tests
 we2e_fundamental_tests=('grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
-    'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
+    'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR'
+    'grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR'
+    'grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta'
+    'grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR'
+    'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR'
+    'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
     'grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
-    'grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
     'grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR'
-    'grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0')
+    'grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
+    'nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR'
+    'community_ensemble_2mems'
+    'custom_ESGgrid'
+    'deactivate_tasks'
+    'inline_post'
+    'nco_ensemble'
+    'specify_DOT_OR_USCORE'
+    'specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE'
+    'specify_RESTART_INTERVAL'
+    'specify_template_filenames')
+
+if [[ "${platform}" != 'gaea' && "${platform}" != 'noaacloud' ]]; then
+    we2e_fundamental_tests+=('MET_ensemble_verification'
+        'MET_verification'
+        'pregen_grid_orog_sfc_climo')
+fi
 
 # The comprehensive set of end-to-end tests to run.
 declare -a we2e_comprehensive_tests
-we2e_comprehensive_tests=('grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
-    'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
+we2e_comprehensive_tests=('grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
     'grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
     'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
-    'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR'
     'grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR'
     'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR'
-    'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
     'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
     'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
     'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
     'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
-    'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR')
-
-# The set of workflow tests to run.
-declare -a we2e_workflow_tests
-we2e_workflow_tests=('community_ensemble_008mems'
-    'community_ensemble_2mems'
-    'deactivate_tasks'
-    'inline_post'
-    'custom_ESGgrid'
+    'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
+    'grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0'
+    'community_ensemble_008mems'
     'custom_GFDLgrid'
     'custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE'
-    'custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE'
-    'specify_DOT_OR_USCORE'
-    'specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE'
-    'specify_RESTART_INTERVAL'
-    'specify_template_filenames'
-    # 'subhourly_post_ensemble_2mems'
-    # 'subhourly_post'
-)
+    'custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE')
 
 declare -a we2e_tests
 we2e_tests=("${we2e_fundamental_tests[@]}")
 if "${SRW_WE2E_COMPREHENSIVE_TESTS}"; then
-    we2e_tests+=("${we2e_comprehensive_tests[@]}"
-        "${we2e_workflow_tests[@]}")
+    we2e_tests+=("${we2e_comprehensive_tests[@]}")
 
     # Add additional tests for Hera.
     if [[ "${platform}" == 'hera' ]]; then
         we2e_tests+=('specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS')
     fi
-
-    # Add additional tests for all platforms, except Gaea and Parallel Works.
-    if [[ "${platform}" != 'gaea' && "${platform}" != 'noaacloud' ]]; then
-        we2e_tests+=('pregen_grid_orog_sfc_climo'
-            'MET_ensemble_verification'
-            'MET_verification')
-    fi
 fi
 
 # Parses the test log for the status of a specific test.

From 87d656df95cec5313d28c08d205353a4af5148dc Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Thu, 4 Aug 2022 10:45:19 -0400
Subject: [PATCH 14/17] Update we2e comprehensive tests

---
 .cicd/scripts/srw_test.sh | 66 +++++++++++++++++++++++++++++++--------
 1 file changed, 53 insertions(+), 13 deletions(-)

diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh
index b58ccba965..7bd3f4ecb5 100755
--- a/.cicd/scripts/srw_test.sh
+++ b/.cicd/scripts/srw_test.sh
@@ -69,22 +69,62 @@ fi
 
 # The comprehensive set of end-to-end tests to run.
 declare -a we2e_comprehensive_tests
-we2e_comprehensive_tests=('grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+we2e_comprehensive_tests=('community_ensemble_008mems'
+    'custom_GFDLgrid'
+    'custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE'
+    'custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_TRUE'
+    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200'
+    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818'
+    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518'
+    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600'
+    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100'
+    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio'
+    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200'
+    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818'
+    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518'
+    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600'
+    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100'
+    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000'
+    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2022060112_48h'
+    'get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS'
+    'get_from_HPSS_ics_HRRR_lbcs_RAP'
+    'get_from_HPSS_ics_RAP_lbcs_RAP'
+    'get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio'
+    'specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS'
+    'grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2'
+    'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
+    'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
+    'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp'
+    'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional'
+    'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2'
+    'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
+    'grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp'
+    'grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2'
+    'grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16'
+    'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km'
+    'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2'
     'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
-    'grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
-    'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
+    'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
+    'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
+    'grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
     'grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR'
+    'grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
+    'grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta'
+    'grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2'
     'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR'
-    'grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
-    'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
-    'grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
-    'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
-    'grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR'
-    'grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0'
-    'community_ensemble_008mems'
-    'custom_GFDLgrid'
-    'custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE'
-    'custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE')
+    'grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta'
+    'grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
+    'grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta'
+    'grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2'
+    'nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
+    'nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km')
 
 declare -a we2e_tests
 we2e_tests=("${we2e_fundamental_tests[@]}")

From 92b843371776eaa4d52b94d031eda4e82068d04a Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Wed, 17 Aug 2022 14:51:56 -0400
Subject: [PATCH 15/17] Remove invalid tests from comprehensive list

---
 .cicd/scripts/srw_test.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh
index 7bd3f4ecb5..bca6aa57a6 100755
--- a/.cicd/scripts/srw_test.sh
+++ b/.cicd/scripts/srw_test.sh
@@ -85,12 +85,10 @@ we2e_comprehensive_tests=('community_ensemble_008mems'
     'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600'
     'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100'
     'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000'
-    'get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2022060112_48h'
     'get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS'
     'get_from_HPSS_ics_HRRR_lbcs_RAP'
     'get_from_HPSS_ics_RAP_lbcs_RAP'
     'get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio'
-    'specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS'
     'grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
     'grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'
     'grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16'

From bf5a9b689c67bc59928bf56f28c21c9f359e79ca Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Tue, 20 Sep 2022 12:32:51 -0400
Subject: [PATCH 16/17] Update Parallel Works cluster names

---
 .cicd/Jenkinsfile          | 14 +++++++-------
 .cicd/scripts/srw_build.sh |  2 +-
 .cicd/scripts/srw_test.sh  |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
index f57218bfeb..79174c4d20 100644
--- a/.cicd/Jenkinsfile
+++ b/.cicd/Jenkinsfile
@@ -9,8 +9,8 @@ pipeline {
 
     parameters {
         // Allow job runner to filter based on platform
-        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pcluster_noaa_v2_use1', 'azcluster_noaa', 'gcluster_noaa_v2_usc1'], description: 'Specify the platform(s) to use')
-        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pcluster_noaa_v2_use1'], description: 'Specify the platform(s) to use')
+        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
+        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
         // Allow job runner to filter based on compiler
         choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
         booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests'
@@ -31,7 +31,7 @@ pipeline {
                 axes {
                     axis {
                         name 'SRW_PLATFORM'
-                        values 'pcluster_noaa_v2_use1' //, 'azcluster_noaa', 'gcluster_noaa_v2_usc1'
+                        values 'pclusternoaav2use1' //, 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                     }
                 }
 
@@ -69,7 +69,7 @@ pipeline {
                 axes {
                     axis {
                         name 'SRW_PLATFORM'
-                        values 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pcluster_noaa_v2_use1' //, 'azcluster_noaa', 'gcluster_noaa_v2_usc1'
+                        values 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1' //, 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                     }
 
                     axis {
@@ -83,7 +83,7 @@ pipeline {
                     exclude {
                             axis {
                             name 'SRW_PLATFORM'
-                            values 'gaea', 'hera', 'jet', 'orion', 'pcluster_noaa_v2_use1' //, 'azcluster_noaa', 'gcluster_noaa_v2_usc1'
+                            values 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1' //, 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                         }
 
                         axis {
@@ -182,8 +182,8 @@ pipeline {
         always {
             // Stop any Parallel Works clusters that were started during the pipeline execution
             script {
-                // def pw_clusters = ['pcluster_noaa_v2_use1', 'azcluster_noaa', 'gcluster_noaa_v2_usc1']
-                def pw_clusters = ['pcluster_noaa_v2_use1']
+                // def pw_clusters = ['pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1']
+                def pw_clusters = ['pclusternoaav2use1']
                 def clusters = []
 
                 // Determine which clusters need to be stopped, if any
diff --git a/.cicd/scripts/srw_build.sh b/.cicd/scripts/srw_build.sh
index 1fe338d306..fed4170e3a 100755
--- a/.cicd/scripts/srw_build.sh
+++ b/.cicd/scripts/srw_build.sh
@@ -18,7 +18,7 @@ fi
 
 # Normalize Parallel Works cluster platform value.
 declare platform
-if [[ "${SRW_PLATFORM}" =~ ^(az|g|p)cluster_noaa ]]; then
+if [[ "${SRW_PLATFORM}" =~ ^(az|g|p)clusternoaa ]]; then
     platform='noaacloud'
 else
     platform="${SRW_PLATFORM}"
diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh
index bca6aa57a6..0d0301f9cd 100755
--- a/.cicd/scripts/srw_test.sh
+++ b/.cicd/scripts/srw_test.sh
@@ -19,7 +19,7 @@ fi
 
 # Normalize Parallel Works cluster platform value.
 declare platform
-if [[ "${SRW_PLATFORM}" =~ ^(az|g|p)cluster_noaa ]]; then
+if [[ "${SRW_PLATFORM}" =~ ^(az|g|p)clusternoaa ]]; then
     platform='noaacloud'
 else
     platform="${SRW_PLATFORM}"

From d8c9c8bf0d94d87f50bffa4e1bb7cb62d173e136 Mon Sep 17 00:00:00 2001
From: Jesse McFarland <jesse@mcfarland.sh>
Date: Tue, 20 Sep 2022 12:40:17 -0400
Subject: [PATCH 17/17] Remove regional_workflow from we2e_test_dir path

---
 .cicd/scripts/srw_test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh
index 0d0301f9cd..0582815e74 100755
--- a/.cicd/scripts/srw_test.sh
+++ b/.cicd/scripts/srw_test.sh
@@ -32,7 +32,7 @@ else
     we2e_experiment_base_dir="${workspace}/experiments"
 fi
 
-we2e_test_dir="${workspace}/regional_workflow/tests/WE2E"
+we2e_test_dir="${workspace}/tests/WE2E"
 
 we2e_test_file="${we2e_test_dir}/experiments.txt"