[8.x] Fix flaky test around search cancellation (#193008) (#193063)

# Backport

This will backport the following commits from `main` to `8.x`:
- [Fix flaky test around search cancellation
(#193008)](#193008)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Mike
Côté","email":"mikecote@users.noreply.github.com"},"sourceCommit":{"committedDate":"2024-09-16T17:45:08Z","message":"Fix
flaky test around search cancellation (#193008)\n\nResolves
https://github.com/elastic/kibana/issues/192914\r\n\r\nIn this PR, I'm
fixing the flakiness in tests where sometimes rules fail\r\nwith a
different message on timeout. This is expected as it's a
race\r\ncondition between the Elasticsearch request timing out and the
alerting\r\nrule getting cancelled. So we can expect one of two
messages.\r\n\r\nNote: Test is not skipped as of PR
creation","sha":"0c63a7b73394e9bfe348de8b2d0755c557098eae","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Feature:Alerting","release_note:skip","Team:ResponseOps","v9.0.0","backport:prev-minor","v8.16.0"],"title":"Fix
flaky test around search
cancellation","number":193008,"url":"https://github.com/elastic/kibana/pull/193008","mergeCommit":{"message":"Fix
flaky test around search cancellation (#193008)\n\nResolves
https://github.com/elastic/kibana/issues/192914\r\n\r\nIn this PR, I'm
fixing the flakiness in tests where sometimes rules fail\r\nwith a
different message on timeout. This is expected as it's a
race\r\ncondition between the Elasticsearch request timing out and the
alerting\r\nrule getting cancelled. So we can expect one of two
messages.\r\n\r\nNote: Test is not skipped as of PR
creation","sha":"0c63a7b73394e9bfe348de8b2d0755c557098eae"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/193008","number":193008,"mergeCommit":{"message":"Fix
flaky test around search cancellation (#193008)\n\nResolves
https://github.com/elastic/kibana/issues/192914\r\n\r\nIn this PR, I'm
fixing the flakiness in tests where sometimes rules fail\r\nwith a
different message on timeout. This is expected as it's a
race\r\ncondition between the Elasticsearch request timing out and the
alerting\r\nrule getting cancelled. So we can expect one of two
messages.\r\n\r\nNote: Test is not skipped as of PR
creation","sha":"0c63a7b73394e9bfe348de8b2d0755c557098eae"}},{"branch":"8.x","label":"v8.16.0","branchLabelMappingKey":"^v8.16.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Mike Côté <mikecote@users.noreply.github.com>

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/builtin_alert_types/cancellable/rule.ts

@@ -127,19 +127,25 @@ export default function ruleTests({ getService }: FtrProviderContext) {
       events.filter((event) => event?.event?.action === 'execute');
       expect(events[0]?.event?.outcome).to.eql('failure');
       expect(events[0]?.kibana?.alerting?.status).to.eql('error');
-      expect(events[0]?.error?.message).to.eql(
-        'Search has been aborted due to cancelled execution'
-      );
+      // Timeouts will encounter one of the following two messages
+      const expectedMessages = [
+        'Request timed out',
+        'Search has been aborted due to cancelled execution',
+      ];
+      expect(expectedMessages.includes(events[0]?.error?.message || '')).to.be(true);
 
       // rule execution status should be in error with reason timeout
       const { status, body: rule } = await supertest.get(
         `${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule/${ruleId}`
       );
       expect(status).to.eql(200);
       expect(rule.execution_status.status).to.eql('error');
-      expect(rule.execution_status.error.message).to.eql(
-        `test.cancellableRule:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of 3s`
-      );
+      expect(
+        [
+          'Request timed out',
+          `test.cancellableRule:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of 3s`,
+        ].includes(rule.execution_status.error.message)
+      ).to.eql(true);
       expect(rule.execution_status.error.reason).to.eql('timeout');
     });
 
@@ -183,9 +189,12 @@ export default function ruleTests({ getService }: FtrProviderContext) {
       );
       expect(status).to.eql(200);
       expect(rule.execution_status.status).to.eql('error');
-      expect(rule.execution_status.error.message).to.eql(
-        `test.cancellableRule:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of 3s`
-      );
+      expect(
+        [
+          'Request timed out',
+          `test.cancellableRule:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of 3s`,
+        ].includes(rule.execution_status.error.message)
+      ).to.eql(true);
       expect(rule.execution_status.error.reason).to.eql('timeout');
     });
 

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/builtin_alert_types/long_running/rule.ts

@@ -74,9 +74,12 @@ export default function ruleTests({ getService }: FtrProviderContext) {
       expect(errorStatuses.length).to.be.greaterThan(0);
       const lastErrorStatus = errorStatuses.pop();
       expect(lastErrorStatus?.status).to.eql('error');
-      expect(lastErrorStatus?.error.message).to.eql(
-        `test.patternLongRunning.cancelAlertsOnRuleTimeout:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of 3s`
-      );
+      expect(
+        [
+          'Request timed out',
+          `test.patternLongRunning.cancelAlertsOnRuleTimeout:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of 3s`,
+        ].includes(lastErrorStatus?.error.message || '')
+      ).to.eql(true);
       expect(lastErrorStatus?.error.reason).to.eql('timeout');
     });