From 0c63a7b73394e9bfe348de8b2d0755c557098eae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20C=C3=B4t=C3=A9?= <mikecote@users.noreply.github.com>
Date: Mon, 16 Sep 2024 13:45:08 -0400
Subject: [PATCH] Fix flaky test around search cancellation (#193008)

Resolves https://github.com/elastic/kibana/issues/192914

In this PR, I'm fixing the flakiness in tests where sometimes rules fail
with a different message on timeout. This is expected as it's a race
condition between the Elasticsearch request timing out and the alerting
rule getting cancelled. So we can expect one of two messages.

Note: Test is not skipped as of PR creation
---
 .../builtin_alert_types/cancellable/rule.ts   | 27 ++++++++++++-------
 .../builtin_alert_types/long_running/rule.ts  |  9 ++++---
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/builtin_alert_types/cancellable/rule.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/builtin_alert_types/cancellable/rule.ts
index 41a12004e256a..8a254447b4a4e 100644
--- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/builtin_alert_types/cancellable/rule.ts
+++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/builtin_alert_types/cancellable/rule.ts
@@ -127,9 +127,12 @@ export default function ruleTests({ getService }: FtrProviderContext) {
       events.filter((event) => event?.event?.action === 'execute');
       expect(events[0]?.event?.outcome).to.eql('failure');
       expect(events[0]?.kibana?.alerting?.status).to.eql('error');
-      expect(events[0]?.error?.message).to.eql(
-        'Search has been aborted due to cancelled execution'
-      );
+      // Timeouts will encounter one of the following two messages
+      const expectedMessages = [
+        'Request timed out',
+        'Search has been aborted due to cancelled execution',
+      ];
+      expect(expectedMessages.includes(events[0]?.error?.message || '')).to.be(true);
 
       // rule execution status should be in error with reason timeout
       const { status, body: rule } = await supertest.get(
@@ -137,9 +140,12 @@ export default function ruleTests({ getService }: FtrProviderContext) {
       );
       expect(status).to.eql(200);
       expect(rule.execution_status.status).to.eql('error');
-      expect(rule.execution_status.error.message).to.eql(
-        `test.cancellableRule:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of 3s`
-      );
+      expect(
+        [
+          'Request timed out',
+          `test.cancellableRule:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of 3s`,
+        ].includes(rule.execution_status.error.message)
+      ).to.eql(true);
       expect(rule.execution_status.error.reason).to.eql('timeout');
     });
 
@@ -183,9 +189,12 @@ export default function ruleTests({ getService }: FtrProviderContext) {
       );
       expect(status).to.eql(200);
       expect(rule.execution_status.status).to.eql('error');
-      expect(rule.execution_status.error.message).to.eql(
-        `test.cancellableRule:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of 3s`
-      );
+      expect(
+        [
+          'Request timed out',
+          `test.cancellableRule:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of 3s`,
+        ].includes(rule.execution_status.error.message)
+      ).to.eql(true);
       expect(rule.execution_status.error.reason).to.eql('timeout');
     });
 
diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/builtin_alert_types/long_running/rule.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/builtin_alert_types/long_running/rule.ts
index 43524a57cb225..effd35d392a3f 100644
--- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/builtin_alert_types/long_running/rule.ts
+++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/builtin_alert_types/long_running/rule.ts
@@ -74,9 +74,12 @@ export default function ruleTests({ getService }: FtrProviderContext) {
       expect(errorStatuses.length).to.be.greaterThan(0);
       const lastErrorStatus = errorStatuses.pop();
       expect(lastErrorStatus?.status).to.eql('error');
-      expect(lastErrorStatus?.error.message).to.eql(
-        `test.patternLongRunning.cancelAlertsOnRuleTimeout:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of 3s`
-      );
+      expect(
+        [
+          'Request timed out',
+          `test.patternLongRunning.cancelAlertsOnRuleTimeout:${ruleId}: execution cancelled due to timeout - exceeded rule type timeout of 3s`,
+        ].includes(lastErrorStatus?.error.message || '')
+      ).to.eql(true);
       expect(lastErrorStatus?.error.reason).to.eql('timeout');
     });