[7.4] Experimenting with delay in test startup (#51459) (#52012)

* Experimenting with delay in test startup

* Move firefox tests sooner becuase they take a long time to run

* Move functional test parallel process delay to a spot that should work better for flaky test runner

Jenkinsfile

@@ -12,6 +12,7 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
             'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'),
             'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'),
             'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
+              'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') }),
               'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1),
               'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2),
               'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3),
@@ -24,10 +25,10 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
               'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10),
               'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
               'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
-              'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') }),
               'oss-visualRegression': kibanaPipeline.getPostBuildWorker('visualRegression', { runbld('./test/scripts/jenkins_visual_regression.sh', 'Execute kibana-visualRegression') }),
             ]),
             'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
+              'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') }),
               'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1),
               'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2),
               'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3),
@@ -38,7 +39,6 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
               'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8),
               'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
               'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
-              'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') }),
               'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld('./test/scripts/jenkins_xpack_visual_regression.sh', 'Execute xpack-visualRegression') }),
             ]),
           ])

vars/kibanaPipeline.groovy

@@ -11,6 +11,10 @@ def withWorkers(name, preWorkerClosure = {}, workerClosures = [:]) {
           nextWorker++
 
           return {
+            // This delay helps smooth out CPU load caused by ES/Kibana instances starting up at the same time
+            def delay = (workerNumber-1)*20
+            sleep(delay)
+
             workerClosure(workerNumber)
           }
         }