Skip to content

Commit

Permalink
Fix Jenkins success reporting (NOAA-EMC#2302)
Browse files Browse the repository at this point in the history 
Moving the post section back outside of main Run Experiments
stage. This allows the system to correctly report the **Success**
status until after all tests pass.

_Had originally moved them in attempts to solve "Not an SCM
GitHub Job" issue and cause the reporting to misbehave._

Also ran through Jenkins linter and updated some messaging
that was incorrectly reporting system build type.
  • Loading branch information
@TerrenceMcGuinness-NOAA
TerrenceMcGuinness-NOAA authored Feb 10, 2024
1 parent 54daa31 commit 061992b
Showing 1 changed file with 77 additions and 77 deletions.
154 changes: 77 additions & 77 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,15 @@ pipeline {
machine = 'none'
for (label in pullRequest.labels) {
echo "Label: ${label}"
if ((label.matches("CI-Hera-Ready"))) {
if ((label.matches('CI-Hera-Ready'))) {
machine = 'hera'
} else if ((label.matches("CI-Orion-Ready"))) {
} else if ((label.matches('CI-Orion-Ready'))) {
machine = 'orion'
} else if ((label.matches("CI-Hercules-Ready"))) {
} else if ((label.matches('CI-Hercules-Ready'))) {
machine = 'hercules'
}
} // createing a second machine varible with first letter capital
// because the first letter of the machine name is captitalized in the GitHub labels
} // createing a second machine varible with first letter capital
// because the first letter of the machine name is captitalized in the GitHub labels
Machine = machine[0].toUpperCase() + machine.substring(1)
}
}
Expand All @@ -42,17 +42,17 @@ pipeline {
agent { label "${machine}-emc" }
steps {
script {
properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
HOME = "${WORKSPACE}/TESTDIR"
commonworkspace = "${WORKSPACE}"
sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
sh(script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true)
pullRequest.addLabel("CI-${Machine}-Building")
if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Ready") } ) {
if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
pullRequest.removeLabel("CI-${Machine}-Ready")
}
}
}
}
}

stage('Build System') {
matrix {
Expand All @@ -61,121 +61,82 @@ pipeline {
// throttle(['global_matrix_build'])
//}
axes {
axis {
name "system"
values "gfs", "gefs"
axis {
name 'system'
values 'gfs', 'gefs'
}
}
stages {
stage("build system") {
stage('build system') {
steps {
script {
def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME
sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true)
sh(script: "mkdir -p ${HOMEgfs}", returnStatus: true)
ws(HOMEgfs) {
env.MACHINE_ID = machine // MACHINE_ID is used in the build scripts to determine the machine and is added to the shell environment
if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") }
} else {
checkout scm
sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true)
def builds_file = readYaml file: "ci/cases/yamls/build.yaml"
sh(script: 'source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive', returnStatus: true)
def builds_file = readYaml file: 'ci/cases/yamls/build.yaml'
def build_args_list = builds_file['builds']
def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "")
def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '')
dir("${HOMEgfs}/sorc") {
sh( script: "${build_args}", returnStatus: true)
sh( script: "./link_workflow.sh", returnStatus: true)
sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
sh(script: "${build_args}", returnStatus: true)
sh(script: './link_workflow.sh', returnStatus: true)
sh(script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true)
}
}
if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) {
pullRequest.removeLabel("CI-${Machine}-Building")
}
pullRequest.addLabel("CI-${Machine}-Running")
if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) {
pullRequest.removeLabel("CI-${Machine}-Building")
}
pullRequest.addLabel("CI-${Machine}-Running")
}
}
}
}
}
}
}

stage('Run Tests') {
matrix {
agent { label "${machine}-emc" }
axes {
axis {
name "Case"
values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" // TODO add dynamic list of cases from env vars (needs addtional plugins)
name 'Case'
values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar' // TODO add dynamic list of cases from env vars (needs addtional plugins)
}
}
stages {
stage('Create Experiment') {
steps {
steps {
script {
sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true)
sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true)
def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp"
system = yaml_case.experiment.system
def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to populate the XML on per system basis
env.RUNTESTS = "${HOME}/RUNTESTS"
sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
}
sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true)
}
}
}
stage('Run Experiments') {
steps {
script {
HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
ws(HOMEgfs) {
pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim()
pullRequest.comment("**Running experiments: ${Case} on ${Machine}**<br>Built against system **${system}** in directory:<br>`${HOMEgfs}`<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
try {
sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
pullRequest.comment("**Running** experiment: ${Case} on ${Machine}<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
try {
sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true)
} catch (Exception e) {
pullRequest.comment("**FAILURE** running experiments: ${Case} on ${Machine}")
error("Failed to run experiments ${Case} on ${Machine}")
}
pullRequest.comment("**SUCCESS** running experiments: ${Case} on ${Machine}")
}
}
}
post {
always {
script {
ws (HOMEgfs) {
for (label in pullRequest.labels) {
if (label.contains("${Machine}")) {
pullRequest.removeLabel(label)
}
}
}
}
}
success {
script {
ws (HOMEgfs) {
pullRequest.addLabel("CI-${Machine}-Passed")
def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
}
}
}
failure {
script {
ws (HOMEgfs) {
pullRequest.addLabel("CI-${Machine}-Failed")
def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York'))
pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
if (fileExists('${HOME}/RUNTESTS/ci.log')) {
def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
fileContent.eachLine { line ->
if( line.contains(".log")) {
archiveArtifacts artifacts: "${line}", fingerprint: true
}
}
}
pullRequest.comment("**FAILURE** running experiment: ${Case} on ${Machine}")
error("Failed to run experiments ${Case} on ${Machine}")
}
pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}")
}
}
}
Expand All @@ -185,4 +146,43 @@ pipeline {
}
}

post {
always {
script {
if(env.CHANGE_ID) {
for (label in pullRequest.labels) {
if (label.contains("${Machine}")) {
pullRequest.removeLabel(label)
}
}
}
}
}
success {
script {
if(env.CHANGE_ID) {
pullRequest.addLabel("CI-${Machine}-Passed")
def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
}
}
}
failure {
script {
if(env.CHANGE_ID) {
pullRequest.addLabel("CI-${Machine}-Failed")
def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
}
if (fileExists('${HOME}/RUNTESTS/ci.log')) {
def fileContent = readFile '${HOME}/RUNTESTS/ci.log'
fileContent.eachLine { line ->
if (line.contains('.log')) {
archiveArtifacts artifacts: "${line}", fingerprint: true
}
}
}
}
}
}
}

0 comments on commit 061992b

Please sign in to comment.