Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable AWS Parallel Works platform and Add Comprehensive End-To-End Tests #333

Merged
merged 17 commits into from
Sep 29, 2022
Merged

Enable AWS Parallel Works platform and Add Comprehensive End-To-End Tests #333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
52ed745
Normalize Parallel Works cluster platform value
Jun 2, 2022
3ba8f18
Enable the AWS Parallel Works platform
Jun 2, 2022
01435ad
Move agent declaration to stages
Jun 2, 2022
c65b1e8
Ensure PROJ_LIB is set on Parallel Works platforms
Jun 3, 2022
2a6348c
Fix final exit status of srw_test script
Jun 4, 2022
ec1ce52
Add comprehensive end-to-end tests option
Jun 6, 2022
80fe8b7
Clean up the workspace after a we2e test run
Jun 7, 2022
9beeca1
Disable concurrent builds for branches and PRs
Jun 14, 2022
2a8cfc7
Disable branch indexing triggers for pipeline
Jun 14, 2022
d309fc3
Update we2e fundamental tests for srw_test script
Jun 14, 2022
2e4887e
Log and archive the output of the srw build
Jun 15, 2022
0bb18c2
Update we2e comprehensive tests in srw_test script
Jun 22, 2022
ba83efc
Update we2e fundamental/comprehensive tests
Jul 29, 2022
87d656d
Update we2e comprehensive tests
Aug 4, 2022
92b8433
Remove invalid tests from comprehensive list
Aug 17, 2022
bf5a9b6
Update Parallel Works cluster names
Sep 20, 2022
d8c9c8b
Remove regional_workflow from we2e_test_dir path
Sep 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 53 additions & 17 deletions .cicd/Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,21 @@ pipeline {
agent none

options {
disableConcurrentBuilds()
overrideIndexTriggers(false)
skipDefaultCheckout(true)
}

parameters {
// Allow job runner to filter based on platform
// choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pcluster_noaa_v2_use1', 'azcluster_noaa', 'gcluster_noaa_v2_usc1'], description: 'Specify the platform(s) to use')
choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use')
// choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
// Allow job runner to filter based on compiler
choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests'
}

stages {
/*
// Start the NOAA Parallel Works clusters, if necessary
stage('Start Parallel Works Clusters') {
matrix {
Expand All @@ -29,7 +31,7 @@ pipeline {
axes {
axis {
name 'SRW_PLATFORM'
values 'pcluster_noaa_v2_use1', 'azcluster_noaa', 'gcluster_noaa_v2_usc1'
values 'pclusternoaav2use1' //, 'azclusternoaav2eus1', 'gclusternoaav2usc1'
}
}

Expand All @@ -44,7 +46,6 @@ pipeline {
}
}
}
*/

// Build and test the SRW application on all supported platforms using the supported compilers for each platform
stage('Build and Test') {
Expand All @@ -68,8 +69,7 @@ pipeline {
axes {
axis {
name 'SRW_PLATFORM'
// values 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pcluster_noaa_v2_use1', 'azcluster_noaa', 'gcluster_noaa_v2_usc1'
values 'cheyenne', 'gaea', 'hera', 'jet', 'orion'
values 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1' //, 'azclusternoaav2eus1', 'gclusternoaav2usc1'
}

axis {
Expand All @@ -83,7 +83,7 @@ pipeline {
exclude {
axis {
name 'SRW_PLATFORM'
values 'gaea', 'hera', 'jet', 'orion'
values 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1' //, 'azclusternoaav2eus1', 'gclusternoaav2usc1'
}

axis {
Expand All @@ -93,10 +93,6 @@ pipeline {
}
}

agent {
label env.SRW_PLATFORM
}

environment {
BRANCH_NAME_ESCAPED = env.BRANCH_NAME.replace('/', '_')
BUILD_VERSION = "${env.SRW_PLATFORM}-${env.SRW_COMPILER}-${env.BRANCH_NAME_ESCAPED}-${env.BUILD_NUMBER}"
Expand All @@ -106,6 +102,10 @@ pipeline {
stages {
// Clean the workspace, checkout the repository, and run checkout_externals
stage('Initialize') {
agent {
label env.SRW_PLATFORM
}

steps {
echo "Initializing SRW (${env.SRW_COMPILER}) build environment on ${env.SRW_PLATFORM}"
cleanWs()
Expand All @@ -116,6 +116,10 @@ pipeline {

// Run the unified build script; if successful create a tarball of the build and upload to S3
stage('Build') {
agent {
label env.SRW_PLATFORM
}

steps {
echo "Building SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM}"
sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_build.sh"'
Expand All @@ -124,29 +128,62 @@ pipeline {
post {
success {
sh 'tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" bin include lib share'
s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "build/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.log", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
}
}
}

// Run the unified test script
stage('Test') {
agent {
label env.SRW_PLATFORM
}

environment {
SRW_WE2E_EXPERIMENT_BASE_DIR = "${env.WORKSPACE}/experiments"
}

steps {
echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM}"
sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_test.sh"'

// If executing for a Pull Request, check for the run_we2e_comprehensive_tests. If set,
// override the value of the SRW_WE2E_COMPREHENSIVE_TESTS parameter
script {
def run_we2e_comprehensive_tests = params.SRW_WE2E_COMPREHENSIVE_TESTS
def run_we2e_comprehensive_tests_label = 'run_we2e_comprehensive_tests'

if (env.CHANGE_ID) {
pullRequest.labels.each {
if (it == run_we2e_comprehensive_tests_label) {
run_we2e_comprehensive_tests = true
}
}
}

sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests} bash --login ${env.WORKSPACE}/.cicd/scripts/srw_test.sh"
}
}

post {
always {
// Archive the test log files and remove the experiments directory to conserve disk space
sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --file "${WORKSPACE}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log.launch_FV3LAM_wflow */log/*'
sh 'rm -rf "${SRW_WE2E_EXPERIMENT_BASE_DIR}"'
s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_logs-*-*.tgz', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
}
}
}
}
}
}
}

/*
post {
always {
// Stop any Parallel Works clusters that were started during the pipeline execution
script {
def pw_clusters = ['pcluster_noaa_v2_use1', 'azcluster_noaa', 'gcluster_noaa_v2_usc1']
// def pw_clusters = ['pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1']
def pw_clusters = ['pclusternoaav2use1']
def clusters = []

// Determine which clusters need to be stopped, if any
Expand All @@ -166,5 +203,4 @@ pipeline {
}
}
}
*/
}
17 changes: 13 additions & 4 deletions .cicd/scripts/srw_build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,25 @@ else
workspace="$(cd -- "${script_dir}/../.." && pwd)"
fi

# Normalize Parallel Works cluster platform value.
declare platform
if [[ "${SRW_PLATFORM}" =~ ^(az|g|p)clusternoaa ]]; then
platform='noaacloud'
else
platform="${SRW_PLATFORM}"
fi

build_dir="${workspace}/build"

# Set build related environment variables and load required modules.
source "${workspace}/etc/lmod-setup.sh" "${SRW_PLATFORM}"
source "${workspace}/etc/lmod-setup.sh" "${platform}"
module use "${workspace}/modulefiles"
module load "build_${SRW_PLATFORM}_${SRW_COMPILER}"
module load "build_${platform}_${SRW_COMPILER}"

# Compile SRW application and install to repository root.
mkdir "${build_dir}"
pushd "${build_dir}"
cmake -DCMAKE_INSTALL_PREFIX="${workspace}" -DENABLE_RRFS=on "${workspace}"
make -j "${MAKE_JOBS}"
build_log_file="${build_dir}/srw_build-${platform}-${SRW_COMPILER}.log"
cmake -DCMAKE_INSTALL_PREFIX="${workspace}" "${workspace}" | tee "${build_log_file}"
make -j "${MAKE_JOBS}" | tee --append "${build_log_file}"
popd
Loading