From f3d7e5e2b0b48b427c52c8510d76943817e8eccc Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 2 Jan 2020 16:57:02 -0500 Subject: [PATCH 01/23] Playing around with retryable ci groups --- vars/kibanaPipeline.groovy | 8 ++++++-- vars/retryable.groovy | 39 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 vars/retryable.groovy diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index 18f214554b444..d364e272ea78e 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -70,7 +70,9 @@ def getOssCiGroupWorker(ciGroup) { "CI_GROUP=${ciGroup}", "JOB=kibana-ciGroup${ciGroup}", ]) { - runbld("./test/scripts/jenkins_ci_group.sh", "Execute kibana-ciGroup${ciGroup}") + retryable("kibana-ciGroup${ciGroup}") { + runbld("./test/scripts/jenkins_ci_group.sh", "Execute kibana-ciGroup${ciGroup}") + } } }) } @@ -81,7 +83,9 @@ def getXpackCiGroupWorker(ciGroup) { "CI_GROUP=${ciGroup}", "JOB=xpack-kibana-ciGroup${ciGroup}", ]) { - runbld("./test/scripts/jenkins_xpack_ci_group.sh", "Execute xpack-kibana-ciGroup${ciGroup}") + retryable("xpack-kibana-ciGroup${ciGroup}") { + runbld("./test/scripts/jenkins_xpack_ci_group.sh", "Execute xpack-kibana-ciGroup${ciGroup}") + } } }) } diff --git a/vars/retryable.groovy b/vars/retryable.groovy new file mode 100644 index 0000000000000..4045488f813af --- /dev/null +++ b/vars/retryable.groovy @@ -0,0 +1,39 @@ +MAX_GLOBAL_RETRIES = 1 +CURRENT_GLOBAL_RETRIES = 0 +FLAKY_FAILURES = [] + +def setMax(max) { + MAX_GLOBAL_RETRIES = max +} + +def haveReachedMaxRetries() { + return CURRENT_GLOBAL_RETRIES >= MAX_GLOBAL_RETRIES +} + +def getFlakyFailures() { + return FLAKY_FAILURES +} + +def call(label, Closure closure) { + try { + closure() + } catch (ex) { + if (haveReachedMaxRetries()) { + print "Couldn't retry '${label}', have already reached the max number of retries for this build." + throw ex + } + + CURRENT_GLOBAL_RETRIES++ + print ex.toString() + unstable "${label} failed but is retryable, trying a second time..." + + closure() + + FLAKY_FAILURES << [ + label: label, + exception: ex, + ] + + unstable "${label} failed on the first attempt, but succeeded on the second. Marking it as flaky." + } +} From 72a0673401cad53a7cd1878f2776d0bcd33e0694 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 2 Jan 2020 17:18:40 -0500 Subject: [PATCH 02/23] Fix retryable global vars --- Jenkinsfile | 2 ++ vars/retryable.groovy | 20 ++++++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f6844c7f03fe6..9e5f371850a49 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,6 +9,8 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a ansiColor('xterm') { githubPr.withDefaultPrComments { catchError { + retryable.enable() + parallel([ 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), diff --git a/vars/retryable.groovy b/vars/retryable.groovy index 4045488f813af..183319b08d987 100644 --- a/vars/retryable.groovy +++ b/vars/retryable.groovy @@ -1,11 +1,18 @@ -MAX_GLOBAL_RETRIES = 1 -CURRENT_GLOBAL_RETRIES = 0 -FLAKY_FAILURES = [] +import groovy.transform.Field + +@Field GLOBAL_RETRIES_ENABLED = false +@Field MAX_GLOBAL_RETRIES = 1 +@Field CURRENT_GLOBAL_RETRIES = 0 +@Field FLAKY_FAILURES = [] def setMax(max) { MAX_GLOBAL_RETRIES = max } +def enable() { + GLOBAL_RETRIES_ENABLED = true +} + def haveReachedMaxRetries() { return CURRENT_GLOBAL_RETRIES >= MAX_GLOBAL_RETRIES } @@ -15,6 +22,11 @@ def getFlakyFailures() { } def call(label, Closure closure) { + if (!GLOBAL_RETRIES_ENABLED) { + closure() + return + } + try { closure() } catch (ex) { @@ -24,7 +36,7 @@ def call(label, Closure closure) { } CURRENT_GLOBAL_RETRIES++ - print ex.toString() + print ex.toString() // TODO replace with unsandboxed hudson.Functions.printThrowable(ex) unstable "${label} failed but is retryable, trying a second time..." closure() From e3cb852be81b66a21383262c79290433ff67f61a Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 13:12:10 -0500 Subject: [PATCH 03/23] Add UNSTABLE build support to PR comments --- vars/githubPr.groovy | 2 ++ vars/retryable.groovy | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/vars/githubPr.groovy b/vars/githubPr.groovy index ce164ab98ab1e..0332bb864ca9b 100644 --- a/vars/githubPr.groovy +++ b/vars/githubPr.groovy @@ -88,6 +88,8 @@ def getHistoryText(builds) { .collect { build -> if (build.status == "SUCCESS") { return "* :green_heart: [Build #${build.number}](${build.url}) succeeded ${build.commit}" + } else if(build.status == "UNSTABLE") { + return "* :yellow_heart: [Build #${build.number}](${build.url}) was flaky ${build.commit}" } else { return "* :broken_heart: [Build #${build.number}](${build.url}) failed ${build.commit}" } diff --git a/vars/retryable.groovy b/vars/retryable.groovy index 183319b08d987..060e4466dafe6 100644 --- a/vars/retryable.groovy +++ b/vars/retryable.groovy @@ -39,7 +39,12 @@ def call(label, Closure closure) { print ex.toString() // TODO replace with unsandboxed hudson.Functions.printThrowable(ex) unstable "${label} failed but is retryable, trying a second time..." - closure() + def JOB = env.JOB ? "" : "${env.JOB}-retry" + withEnv([ + "JOB=${JOB}", + ]) { + closure() + } FLAKY_FAILURES << [ label: label, From 9e1cbdfdc6a1f45d6f1e8a609b26cd020e1eccc9 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 13:16:20 -0500 Subject: [PATCH 04/23] Add a bunch of no-ops for faster retryables testing --- vars/kibanaPipeline.groovy | 9 ++++++++- vars/runbld.groovy | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index d364e272ea78e..bf1a4831ecee8 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -143,7 +143,8 @@ def jobRunner(label, useRamDisk, closure) { // Try to clone from Github up to 8 times, waiting 15 secs between attempts retry(8) { try { - scmVars = checkout scm + // scmVars = checkout scm + print "No-Op: checkout scm" } catch (ex) { sleep 15 throw ex @@ -186,6 +187,9 @@ def uploadGcsArtifact(uploadPrefix, pattern) { } def withGcsArtifactUpload(workerName, closure) { + print "No-Op: gcsUpload" + return + def uploadPrefix = "kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}" def ARTIFACT_PATTERNS = [ 'target/kibana-*', @@ -256,6 +260,9 @@ def sendKibanaMail() { } def bash(script, label) { + print "No-Op: [${label}] ${script}" + return + sh( script: "#!/bin/bash\n${script}", label: label diff --git a/vars/runbld.groovy b/vars/runbld.groovy index e52bc244c65cb..e23a6fdef1b20 100644 --- a/vars/runbld.groovy +++ b/vars/runbld.groovy @@ -1,4 +1,7 @@ def call(script, label, enableJunitProcessing = false) { + print "No-Op: [${label}] ${script}" + return + def extraConfig = enableJunitProcessing ? "" : "--config ${env.WORKSPACE}/kibana/.ci/runbld_no_junit.yml" sh( @@ -8,6 +11,9 @@ def call(script, label, enableJunitProcessing = false) { } def junit() { + print "No-Op: junit" + return + sh( script: "/usr/local/bin/runbld -d '${pwd()}' ${env.WORKSPACE}/kibana/test/scripts/jenkins_runbld_junit.sh", label: "Process JUnit reports with runbld" From 822ce796b9c684b062f965489575587f15f82dd9 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 13:19:21 -0500 Subject: [PATCH 05/23] Add a fake error for testing --- Jenkinsfile | 11 ++++++++++- vars/kibanaPipeline.groovy | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9e5f371850a49..7015d9250626f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,12 +10,21 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a githubPr.withDefaultPrComments { catchError { retryable.enable() + def tries = 1 parallel([ 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ - 'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') }), + 'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { + retryable('kibana-firefoxSmoke') { + if (tries == 1) { + tries++ + error "Fake Error" + } + runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') + } + }), 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index bf1a4831ecee8..a0c4e4ee56571 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -14,7 +14,7 @@ def withWorkers(machineName, preWorkerClosure = {}, workerClosures = [:]) { return { // This delay helps smooth out CPU load caused by ES/Kibana instances starting up at the same time def delay = (workerNumber-1)*20 - sleep(delay) + // sleep(delay) workerClosure(workerNumber) } From 1a0617adfa7eaa390a5b52806c22c3af876de4d3 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 13:22:52 -0500 Subject: [PATCH 06/23] Fake scmVars --- vars/kibanaPipeline.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index a0c4e4ee56571..b1d2768d1b6ee 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -144,6 +144,10 @@ def jobRunner(label, useRamDisk, closure) { retry(8) { try { // scmVars = checkout scm + scmVars = [ + GIT_BRANCH: 'retry-flaky-tests', + ] + print "No-Op: checkout scm" } catch (ex) { sleep 15 From 00b79f51fa36ec9c24a970c0a2c1ecb8d6a92dfc Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 13:27:49 -0500 Subject: [PATCH 07/23] Fix test no-op --- vars/kibanaPipeline.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index b1d2768d1b6ee..2aad276b44351 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -181,6 +181,9 @@ def jobRunner(label, useRamDisk, closure) { // TODO what should happen if GCS, Junit, or email publishing fails? Unstable build? Failed build? def uploadGcsArtifact(uploadPrefix, pattern) { + print "No-Op: gcsUpload" + return + googleStorageUpload( credentialsId: 'kibana-ci-gcs-plugin', bucket: "gs://${uploadPrefix}", @@ -191,9 +194,6 @@ def uploadGcsArtifact(uploadPrefix, pattern) { } def withGcsArtifactUpload(workerName, closure) { - print "No-Op: gcsUpload" - return - def uploadPrefix = "kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}" def ARTIFACT_PATTERNS = [ 'target/kibana-*', From 0a191e6ead9424e15a3ea606bb5d4389e3c71341 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 13:30:50 -0500 Subject: [PATCH 08/23] Missed an unstable message --- vars/githubPr.groovy | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/vars/githubPr.groovy b/vars/githubPr.groovy index 0332bb864ca9b..ba7df12e87adf 100644 --- a/vars/githubPr.groovy +++ b/vars/githubPr.groovy @@ -104,13 +104,20 @@ def getNextCommentMessage(previousCommentInfo = [:]) { info.builds = previousCommentInfo.builds ?: [] def messages = [] + def status = buildUtils.getBuildStatus() - if (buildUtils.getBuildStatus() == 'SUCCESS') { + if (status == 'SUCCESS') { messages << """ ## :green_heart: Build Succeeded * [continuous-integration/kibana-ci/pull-request](${env.BUILD_URL}) * Commit: ${getCommitHash()} """ + } else if(status == 'UNSTABLE') { + messages << """ + ## :yellow_heart: Build suceeded, but was flaky + * [continuous-integration/kibana-ci/pull-request](${env.BUILD_URL}) + * Commit: ${getCommitHash()} + """ } else { messages << """ ## :broken_heart: Build Failed From d184f34d6fd7efdb9728b8122dca8b8825338e8b Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 14:45:22 -0500 Subject: [PATCH 09/23] Add another flaky task, which should cause a build error --- Jenkinsfile | 14 +++++++++++++- vars/retryable.groovy | 5 +++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7015d9250626f..8a6b853e373e1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,6 +11,7 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a catchError { retryable.enable() def tries = 1 + def tries2 = 1 parallel([ 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), @@ -37,7 +38,16 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), - 'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', { runbld('./test/scripts/jenkins_accessibility.sh', 'Execute kibana-accessibility') }), + 'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', { + retryable('kibana-accessibility') { + if (tries2 == 1) { + tries2++ + error "Fake error" + } + runbld('./test/scripts/jenkins_accessibility.sh', 'Execute kibana-accessibility') + } + }), + // 'oss-visualRegression': kibanaPipeline.getPostBuildWorker('visualRegression', { runbld('./test/scripts/jenkins_visual_regression.sh', 'Execute kibana-visualRegression') }), ]), 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ @@ -58,6 +68,8 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a ]) } } + + // TODO spit out flaky failures to the console kibanaPipeline.sendMail() } } diff --git a/vars/retryable.groovy b/vars/retryable.groovy index 060e4466dafe6..aadd73afc424b 100644 --- a/vars/retryable.groovy +++ b/vars/retryable.groovy @@ -13,6 +13,11 @@ def enable() { GLOBAL_RETRIES_ENABLED = true } +def enable(max) { + enable() + setMax(max) +} + def haveReachedMaxRetries() { return CURRENT_GLOBAL_RETRIES >= MAX_GLOBAL_RETRIES } From c69432cad36fe7bdd0e8306bbb1e624bf068973c Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 14:54:25 -0500 Subject: [PATCH 10/23] Remove no-ops and such that were for testing --- Jenkinsfile | 24 ++++++++++-------------- vars/kibanaPipeline.groovy | 15 ++------------- vars/runbld.groovy | 6 ------ 3 files changed, 12 insertions(+), 33 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8a6b853e373e1..0239cd5018e82 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,19 +10,12 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a githubPr.withDefaultPrComments { catchError { retryable.enable() - def tries = 1 - def tries2 = 1 - parallel([ 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ 'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { retryable('kibana-firefoxSmoke') { - if (tries == 1) { - tries++ - error "Fake Error" - } runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') } }), @@ -40,18 +33,17 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), 'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', { retryable('kibana-accessibility') { - if (tries2 == 1) { - tries2++ - error "Fake error" - } runbld('./test/scripts/jenkins_accessibility.sh', 'Execute kibana-accessibility') } }), - // 'oss-visualRegression': kibanaPipeline.getPostBuildWorker('visualRegression', { runbld('./test/scripts/jenkins_visual_regression.sh', 'Execute kibana-visualRegression') }), ]), 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ - 'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') }), + 'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { + retryable('xpack-firefoxSmoke') { + runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') + } + }), 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), @@ -62,7 +54,11 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), - 'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', { runbld('./test/scripts/jenkins_xpack_accessibility.sh', 'Execute xpack-accessibility') }), + 'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', { + retryable('xpack-accessibility') { + runbld('./test/scripts/jenkins_xpack_accessibility.sh', 'Execute xpack-accessibility') + } + }), // 'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld('./test/scripts/jenkins_xpack_visual_regression.sh', 'Execute xpack-visualRegression') }), ]), ]) diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index 2aad276b44351..d364e272ea78e 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -14,7 +14,7 @@ def withWorkers(machineName, preWorkerClosure = {}, workerClosures = [:]) { return { // This delay helps smooth out CPU load caused by ES/Kibana instances starting up at the same time def delay = (workerNumber-1)*20 - // sleep(delay) + sleep(delay) workerClosure(workerNumber) } @@ -143,12 +143,7 @@ def jobRunner(label, useRamDisk, closure) { // Try to clone from Github up to 8 times, waiting 15 secs between attempts retry(8) { try { - // scmVars = checkout scm - scmVars = [ - GIT_BRANCH: 'retry-flaky-tests', - ] - - print "No-Op: checkout scm" + scmVars = checkout scm } catch (ex) { sleep 15 throw ex @@ -181,9 +176,6 @@ def jobRunner(label, useRamDisk, closure) { // TODO what should happen if GCS, Junit, or email publishing fails? Unstable build? Failed build? def uploadGcsArtifact(uploadPrefix, pattern) { - print "No-Op: gcsUpload" - return - googleStorageUpload( credentialsId: 'kibana-ci-gcs-plugin', bucket: "gs://${uploadPrefix}", @@ -264,9 +256,6 @@ def sendKibanaMail() { } def bash(script, label) { - print "No-Op: [${label}] ${script}" - return - sh( script: "#!/bin/bash\n${script}", label: label diff --git a/vars/runbld.groovy b/vars/runbld.groovy index e23a6fdef1b20..e52bc244c65cb 100644 --- a/vars/runbld.groovy +++ b/vars/runbld.groovy @@ -1,7 +1,4 @@ def call(script, label, enableJunitProcessing = false) { - print "No-Op: [${label}] ${script}" - return - def extraConfig = enableJunitProcessing ? "" : "--config ${env.WORKSPACE}/kibana/.ci/runbld_no_junit.yml" sh( @@ -11,9 +8,6 @@ def call(script, label, enableJunitProcessing = false) { } def junit() { - print "No-Op: junit" - return - sh( script: "/usr/local/bin/runbld -d '${pwd()}' ${env.WORKSPACE}/kibana/test/scripts/jenkins_runbld_junit.sh", label: "Process JUnit reports with runbld" From e6d3b0cd891a4de64c53bba88a2d500b2eb89e51 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 15:07:33 -0500 Subject: [PATCH 11/23] Add a temporary flaky test --- test/functional/apps/home/_home.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/functional/apps/home/_home.js b/test/functional/apps/home/_home.js index 6587c2f113b7f..799b019c47453 100644 --- a/test/functional/apps/home/_home.js +++ b/test/functional/apps/home/_home.js @@ -27,6 +27,10 @@ export default function({ getService, getPageObjects }) { describe('Kibana takes you home', function describeIndexTests() { this.tags('smoke'); + it('is flaky', () => { + expect(process.env.JOB).to.match(/retry$/); + }); + it('clicking on kibana logo should take you to home page', async () => { await PageObjects.common.navigateToApp('settings'); await globalNav.clickLogo(); From f4ad2b33c5e582a0bac381aea8656adf7eb1fe97 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 15:48:32 -0500 Subject: [PATCH 12/23] Skip most test suites for now --- Jenkinsfile | 92 ++++++++++++++++++++++++++--------------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0239cd5018e82..d5627b7e6d8ee 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,56 +11,56 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a catchError { retryable.enable() parallel([ - 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), - 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), + // 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), + // 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ - 'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { - retryable('kibana-firefoxSmoke') { - runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') - } - }), - 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), - 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), - 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), - 'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), - 'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), + // 'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { + // retryable('kibana-firefoxSmoke') { + // runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') + // } + // }), + // 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), + // 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), + // 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), + // 'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), + // 'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), 'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6), - 'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), - 'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), - 'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), - 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), - 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), - 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), - 'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', { - retryable('kibana-accessibility') { - runbld('./test/scripts/jenkins_accessibility.sh', 'Execute kibana-accessibility') - } - }), + // 'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), + // 'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), + // 'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), + // 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), + // 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), + // 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), + // 'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', { + // retryable('kibana-accessibility') { + // runbld('./test/scripts/jenkins_accessibility.sh', 'Execute kibana-accessibility') + // } + // }), // 'oss-visualRegression': kibanaPipeline.getPostBuildWorker('visualRegression', { runbld('./test/scripts/jenkins_visual_regression.sh', 'Execute kibana-visualRegression') }), ]), - 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ - 'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { - retryable('xpack-firefoxSmoke') { - runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') - } - }), - 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), - 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), - 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), - 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), - 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), - 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), - 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), - 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), - 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), - 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), - 'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', { - retryable('xpack-accessibility') { - runbld('./test/scripts/jenkins_xpack_accessibility.sh', 'Execute xpack-accessibility') - } - }), - // 'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld('./test/scripts/jenkins_xpack_visual_regression.sh', 'Execute xpack-visualRegression') }), - ]), + // 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ + // 'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { + // retryable('xpack-firefoxSmoke') { + // runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') + // } + // }), + // 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), + // 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), + // 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), + // 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), + // 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), + // 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), + // 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), + // 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), + // 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), + // 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), + // 'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', { + // retryable('xpack-accessibility') { + // runbld('./test/scripts/jenkins_xpack_accessibility.sh', 'Execute xpack-accessibility') + // } + // }), + // // 'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld('./test/scripts/jenkins_xpack_visual_regression.sh', 'Execute xpack-visualRegression') }), + // ]), ]) } } From f57b1754e49d7fa95bcfd8d288b83c09807ec4ad Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 3 Jan 2020 17:08:26 -0500 Subject: [PATCH 13/23] Try to fix scoping issue --- vars/retryable.groovy | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/vars/retryable.groovy b/vars/retryable.groovy index aadd73afc424b..5f5473e8bbe90 100644 --- a/vars/retryable.groovy +++ b/vars/retryable.groovy @@ -1,16 +1,16 @@ import groovy.transform.Field -@Field GLOBAL_RETRIES_ENABLED = false -@Field MAX_GLOBAL_RETRIES = 1 -@Field CURRENT_GLOBAL_RETRIES = 0 -@Field FLAKY_FAILURES = [] +public static @Field GLOBAL_RETRIES_ENABLED = false +public static @Field MAX_GLOBAL_RETRIES = 1 +public static @Field CURRENT_GLOBAL_RETRIES = 0 +public static @Field FLAKY_FAILURES = [] def setMax(max) { - MAX_GLOBAL_RETRIES = max + retryable.MAX_GLOBAL_RETRIES = max } def enable() { - GLOBAL_RETRIES_ENABLED = true + retryable.GLOBAL_RETRIES_ENABLED = true } def enable(max) { @@ -19,15 +19,15 @@ def enable(max) { } def haveReachedMaxRetries() { - return CURRENT_GLOBAL_RETRIES >= MAX_GLOBAL_RETRIES + return retryable.CURRENT_GLOBAL_RETRIES >= retryable.MAX_GLOBAL_RETRIES } def getFlakyFailures() { - return FLAKY_FAILURES + return retryable.FLAKY_FAILURES } def call(label, Closure closure) { - if (!GLOBAL_RETRIES_ENABLED) { + if (!retryable.GLOBAL_RETRIES_ENABLED) { closure() return } @@ -40,7 +40,7 @@ def call(label, Closure closure) { throw ex } - CURRENT_GLOBAL_RETRIES++ + retryable.CURRENT_GLOBAL_RETRIES++ print ex.toString() // TODO replace with unsandboxed hudson.Functions.printThrowable(ex) unstable "${label} failed but is retryable, trying a second time..." @@ -51,7 +51,7 @@ def call(label, Closure closure) { closure() } - FLAKY_FAILURES << [ + retryable.FLAKY_FAILURES << [ label: label, exception: ex, ] From 54466a871f26e9ff8b4cd03dd4c29fa3d943f936 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Sun, 5 Jan 2020 14:59:47 -0500 Subject: [PATCH 14/23] Fix bad ternary --- vars/retryable.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/retryable.groovy b/vars/retryable.groovy index 5f5473e8bbe90..260c16e107e06 100644 --- a/vars/retryable.groovy +++ b/vars/retryable.groovy @@ -44,7 +44,7 @@ def call(label, Closure closure) { print ex.toString() // TODO replace with unsandboxed hudson.Functions.printThrowable(ex) unstable "${label} failed but is retryable, trying a second time..." - def JOB = env.JOB ? "" : "${env.JOB}-retry" + def JOB = env.JOB ? "${env.JOB}-retry" : "" withEnv([ "JOB=${JOB}", ]) { From 4d78370c86f729715cef1e27826a276f40db60e6 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Sun, 5 Jan 2020 20:34:38 -0500 Subject: [PATCH 15/23] Fix typo --- vars/githubPr.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/githubPr.groovy b/vars/githubPr.groovy index ba7df12e87adf..a1afe451fe171 100644 --- a/vars/githubPr.groovy +++ b/vars/githubPr.groovy @@ -114,7 +114,7 @@ def getNextCommentMessage(previousCommentInfo = [:]) { """ } else if(status == 'UNSTABLE') { messages << """ - ## :yellow_heart: Build suceeded, but was flaky + ## :yellow_heart: Build succeeded, but was flaky * [continuous-integration/kibana-ci/pull-request](${env.BUILD_URL}) * Commit: ${getCommitHash()} """ From 7188866d8348413765f7ca3f053ee5fed50cb2aa Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 14 Jan 2020 11:35:01 -0500 Subject: [PATCH 16/23] Print flaky failures to console, add flaky failures to github pr message, print exception stacktraces --- Jenkinsfile | 2 +- vars/githubPr.groovy | 6 ++++++ vars/retryable.groovy | 16 +++++++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d5627b7e6d8ee..dd7873556f010 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -65,7 +65,7 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a } } - // TODO spit out flaky failures to the console + retryable.printFlakyFailures() kibanaPipeline.sendMail() } } diff --git a/vars/githubPr.groovy b/vars/githubPr.groovy index a1afe451fe171..4f3e448536b02 100644 --- a/vars/githubPr.groovy +++ b/vars/githubPr.groovy @@ -118,6 +118,12 @@ def getNextCommentMessage(previousCommentInfo = [:]) { * [continuous-integration/kibana-ci/pull-request](${env.BUILD_URL}) * Commit: ${getCommitHash()} """ + + def failures = retryable.getFlakyFailures() + if (failures && failures.size() > 0) { + def list = failures.collect { message += " * ${it.label}" }.join("\n") + messages << "* Flaky failures:\n${list}" + } } else { messages << """ ## :broken_heart: Build Failed diff --git a/vars/retryable.groovy b/vars/retryable.groovy index 260c16e107e06..cc34024958aed 100644 --- a/vars/retryable.groovy +++ b/vars/retryable.groovy @@ -26,6 +26,20 @@ def getFlakyFailures() { return retryable.FLAKY_FAILURES } +def printFlakyFailures() { + catchError { + def failures = getFlakyFailures() + + if (failures && failures.size() > 0) { + print "This build had the following flaky failures:" + failures.each { + print "\n${it.label}" + buildUtils.printStacktrace(it.exception) + } + } + } +} + def call(label, Closure closure) { if (!retryable.GLOBAL_RETRIES_ENABLED) { closure() @@ -41,7 +55,7 @@ def call(label, Closure closure) { } retryable.CURRENT_GLOBAL_RETRIES++ - print ex.toString() // TODO replace with unsandboxed hudson.Functions.printThrowable(ex) + buildUtils.printStacktrace(ex) unstable "${label} failed but is retryable, trying a second time..." def JOB = env.JOB ? "${env.JOB}-retry" : "" From ac6a829bb209ee9e8437596903ed2dc2e9e80c5d Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 14 Jan 2020 11:44:05 -0500 Subject: [PATCH 17/23] Remove unnecessary variable --- vars/githubPr.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/githubPr.groovy b/vars/githubPr.groovy index 4f3e448536b02..87dcac13049bc 100644 --- a/vars/githubPr.groovy +++ b/vars/githubPr.groovy @@ -121,7 +121,7 @@ def getNextCommentMessage(previousCommentInfo = [:]) { def failures = retryable.getFlakyFailures() if (failures && failures.size() > 0) { - def list = failures.collect { message += " * ${it.label}" }.join("\n") + def list = failures.collect { " * ${it.label}" }.join("\n") messages << "* Flaky failures:\n${list}" } } else { From a3e7cd57b6b4cef88156fbc324f115a7ccd3ed16 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 14 Jan 2020 11:50:11 -0500 Subject: [PATCH 18/23] Change spacing so github pr message looks better --- vars/githubPr.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vars/githubPr.groovy b/vars/githubPr.groovy index 87dcac13049bc..888f873bd6530 100644 --- a/vars/githubPr.groovy +++ b/vars/githubPr.groovy @@ -113,17 +113,19 @@ def getNextCommentMessage(previousCommentInfo = [:]) { * Commit: ${getCommitHash()} """ } else if(status == 'UNSTABLE') { - messages << """ + def message = """ ## :yellow_heart: Build succeeded, but was flaky * [continuous-integration/kibana-ci/pull-request](${env.BUILD_URL}) * Commit: ${getCommitHash()} - """ + """.stripIndent() def failures = retryable.getFlakyFailures() if (failures && failures.size() > 0) { def list = failures.collect { " * ${it.label}" }.join("\n") - messages << "* Flaky failures:\n${list}" + message += "* Flaky failures:\n${list}" } + + messages << message } else { messages << """ ## :broken_heart: Build Failed From 3fe035194ed748f8d6f236fc95ede917731ebbde Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 14 Jan 2020 16:49:41 -0500 Subject: [PATCH 19/23] Add test failures from junit report to github pr comment --- Jenkinsfile | 2 +- test/functional/apps/home/_home.js | 2 +- vars/githubPr.groovy | 35 +++++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dd7873556f010..b87e1464f67e0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ #!/bin/groovy -library 'kibana-pipeline-library' +library 'kibana-pipeline-library@retry-flaky-tests' kibanaLibrary.load() stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit diff --git a/test/functional/apps/home/_home.js b/test/functional/apps/home/_home.js index 799b019c47453..e8c443be656c9 100644 --- a/test/functional/apps/home/_home.js +++ b/test/functional/apps/home/_home.js @@ -27,7 +27,7 @@ export default function({ getService, getPageObjects }) { describe('Kibana takes you home', function describeIndexTests() { this.tags('smoke'); - it('is flaky', () => { + it.only('is flaky', () => { expect(process.env.JOB).to.match(/retry$/); }); diff --git a/vars/githubPr.groovy b/vars/githubPr.groovy index 888f873bd6530..443fd0acec9e2 100644 --- a/vars/githubPr.groovy +++ b/vars/githubPr.groovy @@ -99,6 +99,37 @@ def getHistoryText(builds) { return "### History\n${list}" } +def getTestFailuresMessage() { + def failures = testUtils.getFailures() + if (!failures) { + return "" + } + + def messages = [] + + failures.take(5).each { failure -> + messages << """ +--- + +### Test Failures +
${failure.fullDisplayName} + +``` +${failure.stdOut} +``` +
+ +--- + """ + } + + if (failures.size() > 3) { + messages << "and ${failures.size() - 3} more failures, only showing the first 3." + } + + return messages.join("\n") +} + def getNextCommentMessage(previousCommentInfo = [:]) { info = previousCommentInfo ?: [:] info.builds = previousCommentInfo.builds ?: [] @@ -122,7 +153,7 @@ def getNextCommentMessage(previousCommentInfo = [:]) { def failures = retryable.getFlakyFailures() if (failures && failures.size() > 0) { def list = failures.collect { " * ${it.label}" }.join("\n") - message += "* Flaky failures:\n${list}" + message += "* Flaky suites:\n${list}" } messages << message @@ -134,6 +165,8 @@ def getNextCommentMessage(previousCommentInfo = [:]) { """ } + messages << getTestFailuresMessage() + if (info.builds && info.builds.size() > 0) { messages << getHistoryText(info.builds) } From 430a07dd4f353859b96d2831f720cd5b65e1db1e Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 15 Jan 2020 17:58:05 -0500 Subject: [PATCH 20/23] Add test report urls to failures in pr comment --- vars/githubPr.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vars/githubPr.groovy b/vars/githubPr.groovy index 443fd0acec9e2..ed2c0a2afbc99 100644 --- a/vars/githubPr.groovy +++ b/vars/githubPr.groovy @@ -111,9 +111,11 @@ def getTestFailuresMessage() { messages << """ --- -### Test Failures +### [Test Failures](${env.BUILD_URL}testReport)
${failure.fullDisplayName} +[Link to Jenkins](${failure.url}) + ``` ${failure.stdOut} ``` From c9d81a206a3cf7ed8afb9c0f835dd20a2ea9a28d Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 16 Jan 2020 14:59:18 -0500 Subject: [PATCH 21/23] Removing kibana-pipeline-library pinning and increase build timeout a little bit because of retries --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b87e1464f67e0..d9b98fb1edc25 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,10 +1,10 @@ #!/bin/groovy -library 'kibana-pipeline-library@retry-flaky-tests' +library 'kibana-pipeline-library' kibanaLibrary.load() stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit - timeout(time: 120, unit: 'MINUTES') { + timeout(time: 135, unit: 'MINUTES') { timestamps { ansiColor('xterm') { githubPr.withDefaultPrComments { From 9f3bc1186df2223ab230234c9f86bd4e221e047c Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 16 Jan 2020 17:37:47 -0500 Subject: [PATCH 22/23] Missing keyword and add a comment --- vars/githubPr.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/githubPr.groovy b/vars/githubPr.groovy index ed2c0a2afbc99..4c19511bb8953 100644 --- a/vars/githubPr.groovy +++ b/vars/githubPr.groovy @@ -133,7 +133,7 @@ ${failure.stdOut} } def getNextCommentMessage(previousCommentInfo = [:]) { - info = previousCommentInfo ?: [:] + def info = previousCommentInfo ?: [:] info.builds = previousCommentInfo.builds ?: [] def messages = [] @@ -185,7 +185,7 @@ def getNextCommentMessage(previousCommentInfo = [:]) { return messages .findAll { !!it } // No blank strings - .collect { it.stripIndent().trim() } + .collect { it.stripIndent().trim() } // This just allows us to indent various strings above, but leaves them un-indented in the comment .join("\n\n") } From 545c10189693956305901c9d3f297c963504232a Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 16 Jan 2020 17:40:34 -0500 Subject: [PATCH 23/23] Remove temporary test changes --- Jenkinsfile | 92 +++++++++++++++--------------- test/functional/apps/home/_home.js | 4 -- 2 files changed, 46 insertions(+), 50 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d9b98fb1edc25..4695004cd010a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,56 +11,56 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a catchError { retryable.enable() parallel([ - // 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), - // 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), + 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), + 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ - // 'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { - // retryable('kibana-firefoxSmoke') { - // runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') - // } - // }), - // 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), - // 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), - // 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), - // 'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), - // 'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), + 'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { + retryable('kibana-firefoxSmoke') { + runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') + } + }), + 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), + 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), + 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), + 'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), + 'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), 'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6), - // 'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), - // 'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), - // 'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), - // 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), - // 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), - // 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), - // 'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', { - // retryable('kibana-accessibility') { - // runbld('./test/scripts/jenkins_accessibility.sh', 'Execute kibana-accessibility') - // } - // }), + 'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), + 'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), + 'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), + 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), + 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), + 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), + 'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', { + retryable('kibana-accessibility') { + runbld('./test/scripts/jenkins_accessibility.sh', 'Execute kibana-accessibility') + } + }), // 'oss-visualRegression': kibanaPipeline.getPostBuildWorker('visualRegression', { runbld('./test/scripts/jenkins_visual_regression.sh', 'Execute kibana-visualRegression') }), ]), - // 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ - // 'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { - // retryable('xpack-firefoxSmoke') { - // runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') - // } - // }), - // 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), - // 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), - // 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), - // 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), - // 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), - // 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), - // 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), - // 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), - // 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), - // 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), - // 'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', { - // retryable('xpack-accessibility') { - // runbld('./test/scripts/jenkins_xpack_accessibility.sh', 'Execute xpack-accessibility') - // } - // }), - // // 'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld('./test/scripts/jenkins_xpack_visual_regression.sh', 'Execute xpack-visualRegression') }), - // ]), + 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ + 'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { + retryable('xpack-firefoxSmoke') { + runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') + } + }), + 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), + 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), + 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), + 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), + 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), + 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), + 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), + 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), + 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), + 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), + 'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', { + retryable('xpack-accessibility') { + runbld('./test/scripts/jenkins_xpack_accessibility.sh', 'Execute xpack-accessibility') + } + }), + // 'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld('./test/scripts/jenkins_xpack_visual_regression.sh', 'Execute xpack-visualRegression') }), + ]), ]) } } diff --git a/test/functional/apps/home/_home.js b/test/functional/apps/home/_home.js index e8c443be656c9..6587c2f113b7f 100644 --- a/test/functional/apps/home/_home.js +++ b/test/functional/apps/home/_home.js @@ -27,10 +27,6 @@ export default function({ getService, getPageObjects }) { describe('Kibana takes you home', function describeIndexTests() { this.tags('smoke'); - it.only('is flaky', () => { - expect(process.env.JOB).to.match(/retry$/); - }); - it('clicking on kibana logo should take you to home page', async () => { await PageObjects.common.navigateToApp('settings'); await globalNav.clickLogo();