diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index f4e0294441..6231fd7e11 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -117,32 +117,38 @@ pipeline { // Clean the workspace, checkout the repository, and run checkout_externals stage('Initialize') { steps { - echo "Initializing SRW (${env.SRW_COMPILER}) build environment on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" + dir ("${env.SRW_PLATFORM}") { + echo "Initializing SRW (${env.SRW_COMPILER}) build environment on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" cleanWs() checkout scm - sh '"${WORKSPACE}/manage_externals/checkout_externals"' + sh '"${WORKSPACE}/${SRW_PLATFORM}/manage_externals/checkout_externals"' + } } } // Run the unittest functional tests that require an HPC platform stage('Functional UnitTests') { steps { + dir ("${env.SRW_PLATFORM}") { echo "Running unittest on retrieve_data.py" - sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_unittest.sh"' + sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_unittest.sh"' + } } } // Run the unified build script; if successful create a tarball of the build and upload to S3 stage('Build') { steps { - echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" - sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_build.sh"' + dir ("${env.SRW_PLATFORM}") { + echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" + sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_build.sh"' + } } post { success { - sh 'cd "${WORKSPACE}/${INSTALL_NAME}" && tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" *' - s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "build_${env.SRW_COMPILER}/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + sh 'cd "${WORKSPACE}/${SRW_PLATFORM}/${INSTALL_NAME}" && tar --create --gzip --verbose --file "${WORKSPACE}/${SRW_PLATFORM}/${BUILD_NAME}.tgz" *' + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/build_${env.SRW_COMPILER}/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] } } } @@ -150,22 +156,25 @@ pipeline { // Try a few Workflow Task scripts to make sure E2E tests can be launched in a follow-on 'Test' stage stage('Functional WorkflowTaskTests') { steps { - echo "Running simple workflow script task tests on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" - sh 'bash --login "${WORKSPACE}/.cicd/scripts/wrapper_srw_ftest.sh"' + dir ("${env.SRW_PLATFORM}") { + echo "Running simple workflow script task tests on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" + sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/wrapper_srw_ftest.sh"' + } } } // Run the unified test script stage('Test') { environment { - SRW_WE2E_EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/expt_dirs" + SRW_WE2E_EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/${env.SRW_PLATFORM}/expt_dirs" } steps { - echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})" + dir ("${env.SRW_PLATFORM}") { + echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" // Remove the following line to re-enable comprehensive tests - sh 'SRW_WE2E_COMPREHENSIVE_TESTS=false bash --login "${WORKSPACE}/.cicd/scripts/srw_test.sh"' + sh 'SRW_WE2E_COMPREHENSIVE_TESTS=false bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"' // Uncomment the following block to re-enable comprehensive tests /* @@ -183,18 +192,19 @@ pipeline { } } - sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests}" + ' bash --login "${WORKSPACE}/.cicd/scripts/srw_test.sh"' + sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests}" + ' bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"' } */ + } } post { always { // Archive the test log files - sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --dereference --file "${WORKSPACE}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log/* ${WORKSPACE}/tests/WE2E/WE2E_tests_*yaml WE2E_summary*txt ${WORKSPACE}/tests/WE2E/log.*' + sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --dereference --file "${WORKSPACE}/${SRW_PLATFORM}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log/* ${WORKSPACE}/${SRW_PLATFORM}/tests/WE2E/WE2E_tests_*yaml WE2E_summary*txt ${WORKSPACE}/${SRW_PLATFORM}/tests/WE2E/log.*' // Remove the data sets from the experiments directory to conserve disk space sh 'find "${SRW_WE2E_EXPERIMENT_BASE_DIR}" -regextype posix-extended -regex "^.*(orog|[0-9]{10})$" -type d | xargs rm -rf' - s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: '*_test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_logs-*-*.tgz', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/*_test_results-*-*.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.SRW_PLATFORM}/we2e_test_logs-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] } } } diff --git a/.cicd/scripts/qsub_srw_ftest.sh b/.cicd/scripts/qsub_srw_ftest.sh index 2a669fe61a..e9f0170a05 100644 --- a/.cicd/scripts/qsub_srw_ftest.sh +++ b/.cicd/scripts/qsub_srw_ftest.sh @@ -12,4 +12,4 @@ #PBS -o log_wrap.%j.log #PBS -e err_wrap.%j.err -bash ${WORKSPACE}/.cicd/scripts/srw_ftest.sh +bash ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_ftest.sh diff --git a/.cicd/scripts/sbatch_srw_ftest.sh b/.cicd/scripts/sbatch_srw_ftest.sh index 1655148068..5add5368b5 100644 --- a/.cicd/scripts/sbatch_srw_ftest.sh +++ b/.cicd/scripts/sbatch_srw_ftest.sh @@ -13,4 +13,4 @@ #SBATCH -o log_wrap.%j.log #SBATCH -e err_wrap.%j.err -bash ${WORKSPACE}/.cicd/scripts/srw_ftest.sh +bash ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_ftest.sh diff --git a/.cicd/scripts/srw_build.sh b/.cicd/scripts/srw_build.sh index 7d8e70b321..196d984a05 100755 --- a/.cicd/scripts/srw_build.sh +++ b/.cicd/scripts/srw_build.sh @@ -10,8 +10,8 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set # relative to script directory. declare workspace -if [[ -n "${WORKSPACE}" ]]; then - workspace="${WORKSPACE}" +if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${SRW_PLATFORM}" else workspace="$(cd -- "${script_dir}/../.." && pwd)" fi @@ -24,10 +24,6 @@ else platform="${SRW_PLATFORM}" fi -if [[ "${SRW_PLATFORM}" = jet-epic ]]; then - platform='jet' -fi - # Build and install cd ${workspace}/tests set +e diff --git a/.cicd/scripts/srw_ftest.sh b/.cicd/scripts/srw_ftest.sh index 52c0cc45f5..7a787a7137 100755 --- a/.cicd/scripts/srw_ftest.sh +++ b/.cicd/scripts/srw_ftest.sh @@ -28,8 +28,8 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set # relative to script directory. declare workspace -if [[ -n "${WORKSPACE}" ]]; then - workspace="${WORKSPACE}" +if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${SRW_PLATFORM}" cd $workspace else workspace="$(cd -- "${script_dir}/../.." && pwd)" @@ -43,10 +43,6 @@ else platform="${SRW_PLATFORM}" fi -if [[ "${SRW_PLATFORM}" = jet-epic ]]; then - platform='jet' -fi - # Test directories we2e_experiment_base_dir="${workspace}/expt_dirs" we2e_test_dir="${workspace}/tests/WE2E" diff --git a/.cicd/scripts/srw_metric_example.sh b/.cicd/scripts/srw_metric_example.sh index 67d795f11f..2018505735 100755 --- a/.cicd/scripts/srw_metric_example.sh +++ b/.cicd/scripts/srw_metric_example.sh @@ -18,8 +18,8 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set # relative to script directory. declare workspace -if [[ -n "${WORKSPACE}" ]]; then - workspace="${WORKSPACE}" +if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${SRW_PLATFORM}" else workspace="$(cd -- "${script_dir}/../.." && pwd)" fi diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh index 2b0041a847..1bffe083bd 100755 --- a/.cicd/scripts/srw_test.sh +++ b/.cicd/scripts/srw_test.sh @@ -11,8 +11,8 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) # Get repository root from Jenkins WORKSPACE variable if set, otherwise, set # relative to script directory. declare workspace -if [[ -n "${WORKSPACE}" ]]; then - workspace="${WORKSPACE}" +if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${SRW_PLATFORM}" else workspace="$(cd -- "${script_dir}/../.." && pwd)" fi @@ -25,10 +25,6 @@ else platform="${SRW_PLATFORM}" fi -if [[ "${SRW_PLATFORM}" = jet-epic ]]; then - platform='jet' -fi - # Test directories we2e_experiment_base_dir="${workspace}/expt_dirs" we2e_test_dir="${workspace}/tests/WE2E" diff --git a/.cicd/scripts/srw_unittest.sh b/.cicd/scripts/srw_unittest.sh index 9424fd4f68..cfc1719de4 100755 --- a/.cicd/scripts/srw_unittest.sh +++ b/.cicd/scripts/srw_unittest.sh @@ -10,8 +10,8 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd) # Get repository root from Jenkins WORKSPACE variable if set, otherwise, # set relative to script directory. declare workspace -if [[ -n "${WORKSPACE:-}" ]]; then - workspace="${WORKSPACE}" +if [[ -n "${WORKSPACE}/${SRW_PLATFORM}" ]]; then + workspace="${WORKSPACE}/${SRW_PLATFORM}" else workspace="$(cd -- "${script_dir}/../.." && pwd)" fi diff --git a/.cicd/scripts/wrapper_srw_ftest.sh b/.cicd/scripts/wrapper_srw_ftest.sh index 02ece830f9..403233354a 100755 --- a/.cicd/scripts/wrapper_srw_ftest.sh +++ b/.cicd/scripts/wrapper_srw_ftest.sh @@ -24,18 +24,18 @@ fi # Customize wrapper scripts if [[ "${SRW_PLATFORM}" == gaea ]]; then - sed -i '15i #SBATCH --clusters=c4' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh - sed -i 's|qos=batch|qos=windfall|g' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i '15i #SBATCH --clusters=c4' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i 's|qos=batch|qos=windfall|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh fi if [[ "${SRW_PLATFORM}" == gaea-c5 ]]; then - sed -i '15i #SBATCH --clusters=c5' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh - sed -i 's|qos=batch|qos=normal|g' ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i '15i #SBATCH --clusters=c5' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh + sed -i 's|qos=batch|qos=normal|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh fi # Call job card and return job_id -echo "Running: ${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh" -job_id=$(${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh) +echo "Running: ${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh" +job_id=$(${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh) echo "Waiting ten seconds for node to initialize" sleep 10 @@ -51,7 +51,7 @@ do echo "Job has completed." # Return exit code and check for results file first - results_file="${WORKSPACE}/functional_test_results_${SRW_PLATFORM}_${SRW_COMPILER}.txt" + results_file="${WORKSPACE}/${SRW_PLATFORM}/functional_test_results_${SRW_PLATFORM}_${SRW_COMPILER}.txt" if [ ! -f "$results_file" ]; then echo "Missing results file! \nexit 1" exit 1 diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS.yaml b/tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS.yaml index 338edeb186..cc07f37bc8 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS.yaml +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS.yaml @@ -21,3 +21,5 @@ task_get_extrn_lbcs: EXTRN_MDL_NAME_LBCS: FV3GFS LBC_SPEC_INTVL_HRS: 3 FV3GFS_FILE_FMT_LBCS: netcdf +task_run_fcst: + DT_ATMOS: 60