[8.x] Fixing flaky backfill tests (#194328) (#195489)

# Backport

This will backport the following commits from `main` to `8.x`:
- [Fixing flaky backfill tests
(#194328)](#194328)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Ying
Mao","email":"ying.mao@elastic.co"},"sourceCommit":{"committedDate":"2024-10-08T18:25:23Z","message":"Fixing
flaky backfill tests (#194328)\n\nResolves
#192144
and\r\nhttps://github.com//issues/181862\r\n\r\n##
Summary\r\n\r\nUnskipped the test that was skipped
in\r\nhttps://github.com//issues/181862 but it did not
fail at\r\nall during the 2400 times total I ran the flaky test runner
so I'm going\r\nto unskip and see if it comes back ever\r\n\r\nWhile
running the flaky test runner, I saw this test
from\r\nhttps://github.com//issues/192144 be flaky so I
addressed\r\nin this PR. This was caused by the query for the alert
documents\r\nsometimes not returning the alert docs in timestamp order,
which is\r\nimportant because I'm trying to compare the actual alert
timestamp to\r\nthe expected alert timestamp (in order). Added a sort
order to the alert\r\nquery.\r\n\r\n---------\r\n\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"2d6874981ec036dd992c2481a13228aed5bd68a2","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Feature:Alerting","release_note:skip","Team:ResponseOps","v9.0.0","backport:prev-minor","v8.16.0"],"title":"Fixing
flaky backfill
tests","number":194328,"url":"https://github.com/elastic/kibana/pull/194328","mergeCommit":{"message":"Fixing
flaky backfill tests (#194328)\n\nResolves
#192144
and\r\nhttps://github.com//issues/181862\r\n\r\n##
Summary\r\n\r\nUnskipped the test that was skipped
in\r\nhttps://github.com//issues/181862 but it did not
fail at\r\nall during the 2400 times total I ran the flaky test runner
so I'm going\r\nto unskip and see if it comes back ever\r\n\r\nWhile
running the flaky test runner, I saw this test
from\r\nhttps://github.com//issues/192144 be flaky so I
addressed\r\nin this PR. This was caused by the query for the alert
documents\r\nsometimes not returning the alert docs in timestamp order,
which is\r\nimportant because I'm trying to compare the actual alert
timestamp to\r\nthe expected alert timestamp (in order). Added a sort
order to the alert\r\nquery.\r\n\r\n---------\r\n\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"2d6874981ec036dd992c2481a13228aed5bd68a2"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/194328","number":194328,"mergeCommit":{"message":"Fixing
flaky backfill tests (#194328)\n\nResolves
#192144
and\r\nhttps://github.com//issues/181862\r\n\r\n##
Summary\r\n\r\nUnskipped the test that was skipped
in\r\nhttps://github.com//issues/181862 but it did not
fail at\r\nall during the 2400 times total I ran the flaky test runner
so I'm going\r\nto unskip and see if it comes back ever\r\n\r\nWhile
running the flaky test runner, I saw this test
from\r\nhttps://github.com//issues/192144 be flaky so I
addressed\r\nin this PR. This was caused by the query for the alert
documents\r\nsometimes not returning the alert docs in timestamp order,
which is\r\nimportant because I'm trying to compare the actual alert
timestamp to\r\nthe expected alert timestamp (in order). Added a sort
order to the alert\r\nquery.\r\n\r\n---------\r\n\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"2d6874981ec036dd992c2481a13228aed5bd68a2"}},{"branch":"8.x","label":"v8.16.0","branchLabelMappingKey":"^v8.16.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Ying Mao <ying.mao@elastic.co>

x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/backfill/find.ts

@@ -135,8 +135,7 @@ export default function findBackfillTests({ getService }: FtrProviderContext) {
 
     for (const scenario of UserAtSpaceScenarios) {
       const { user, space } = scenario;
-      // FLAKY: https://github.com/elastic/kibana/issues/181862
-      describe.skip(scenario.id, () => {
+      describe(scenario.id, () => {
         const apiOptions = {
           spaceId: space.id,
           username: user.username,

x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/backfill/task_runner.ts

@@ -53,6 +53,7 @@ import {
 export default function createBackfillTaskRunnerTests({ getService }: FtrProviderContext) {
   const es = getService('es');
   const retry = getService('retry');
+  const log = getService('log');
   const esTestIndexTool = new ESTestIndexTool(es, retry);
   const supertestWithoutAuth = getService('supertestWithoutAuth');
   const supertest = getService('supertest');
@@ -65,25 +66,25 @@ export default function createBackfillTaskRunnerTests({ getService }: FtrProvide
     moment().utc().subtract(14, 'days').toISOString(),
 
     // backfill execution set 1
-    moment().utc().startOf('day').subtract(13, 'days').add(64, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(13, 'days').add(65, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(13, 'days').add(66, 'seconds').toISOString(),
+    moment().utc().startOf('day').subtract(13, 'days').add(10, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(13, 'days').add(11, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(13, 'days').add(12, 'minutes').toISOString(),
 
     // backfill execution set 2
-    moment().utc().startOf('day').subtract(12, 'days').add(89, 'seconds').toISOString(),
+    moment().utc().startOf('day').subtract(12, 'days').add(20, 'minutes').toISOString(),
 
     // backfill execution set 3
-    moment().utc().startOf('day').subtract(11, 'days').add(785, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(11, 'days').add(888, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(11, 'days').add(954, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(11, 'days').add(1045, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(11, 'days').add(1145, 'seconds').toISOString(),
+    moment().utc().startOf('day').subtract(11, 'days').add(30, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(11, 'days').add(31, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(11, 'days').add(32, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(11, 'days').add(33, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(11, 'days').add(34, 'minutes').toISOString(),
 
     // backfill execution set 4 purposely left empty
 
     // after last backfill
-    moment().utc().startOf('day').subtract(9, 'days').add(666, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(9, 'days').add(667, 'seconds').toISOString(),
+    moment().utc().startOf('day').subtract(9, 'days').add(40, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(9, 'days').add(41, 'minutes').toISOString(),
   ];
 
   describe('ad hoc backfill task', () => {
@@ -175,6 +176,9 @@ export default function createBackfillTaskRunnerTests({ getService }: FtrProvide
         .send([{ rule_id: ruleId, start, end }])
         .expect(200);
 
+      log.info(`originalDocTimestamps ${JSON.stringify(originalDocTimestamps)}`);
+      log.info(`scheduledBackfill ${JSON.stringify(response2.body)}`);
+
       const scheduleResult = response2.body;
 
       expect(scheduleResult.length).to.eql(1);
@@ -668,7 +672,10 @@ export default function createBackfillTaskRunnerTests({ getService }: FtrProvide
   async function queryForAlertDocs<T>(): Promise<Array<SearchHit<T>>> {
     const searchResult = await es.search({
       index: alertsAsDataIndex,
-      body: { query: { match_all: {} } },
+      body: {
+        sort: [{ [ALERT_ORIGINAL_TIME]: { order: 'asc' } }],
+        query: { match_all: {} },
+      },
     });
     return searchResult.hits.hits as Array<SearchHit<T>>;
   }