Scout: run tests in parallel (with spaces) (#207253)

## Summary

This PR adds `spaceTest` interface to `kbn-scout` to run space aware
tests, that can be executed in parallel. Most of Discover tests were
converted to parallel run because we see runtime improvement with 2
parallel workers.

Experiment 1: **ES data pre-ingested**, running 9 Discover **stateful**
tests in **5 files** locally
| Run setup  | Took time |
| ------------- | ------------- |
| 1 worker  | `1.3` min |
| 2 workers | `58.7` sec  |
| 3 workers | `48.3` sec  |
| 4 workers | **tests fail**  |

Conclusion: using **2** workers is the optimal solution to continue

Experiment 2: Running Discover tests for stateful/serverless in **Kibana
CI** (starting servers, ingesting ES data, running tests)
| Run setup  | 1 worker | 2 workers | diff
| ------------- | ------------- |------------- |------------- |
| stateful, 9 tests / 5 files  | `1.7` min | `1.2` min | `-29.4%`|
| svl ES, 8 tests / 4 files  | `1.7` min | `1.3` min | `-23.5%`|
| svl Oblt, 8 tests / 4 files  | `1.8` min | `1.4` min | `-22.2%`|
| svl Search, 5 tests / 2 files  | `59.9` sec | `51.6` sec | `-13.8%`|

Conclusion: parallel run effectiveness benefits from tests being split
in **more test files**.

Experiment 3: Clone existing tests to have **3 times more test files**
and re-run tests for stateful/serverless in **Kibana CI** (starting
servers, ingesting ES data, running tests)
| Run setup  | 1 worker | 2 workers | diff
| ------------- | ------------- |------------- |------------- |
| stateful, 27 tests / 15 files  | `4.3` min | `2.7` min | `-37.2%`|
| svl ES, 24 tests / 12 files  | `4.3` min | `2.7` min | `-37.2%`|

Conclusion: parallel run effectiveness is **increasing** with more test
files in place, **not linear** but with good test design we can expect
**up to 40%** or maybe a bit more.

How parallel run works:
- `scoutSpace` fixture is loaded on Playwright worker setup (using
`auto: true` config), creates a new Kibana Space, expose its id to other
fixtures and deletes the space on teardown.
- `browserAuth` fixture for parallel run caches Cookie per worker/space
like `role:spaceId`. It is needed because Playwright doesn't spin up new
browser for worker, but only new context.
- kbnClient was updated to allow passing `createNewCopies: true` in
query, it is needed to load the same Saved Objects in parallel
workers/spaces and generate new ids to work with them. `scoutSpace`
caches ids and allows to reach saved object by its name. This logic is
different from single thread run, where we can use default ids from
kbnArchives.

How to run parallel tests locally, e.g. for stateful: 
```
node scripts/scout run-tests --stateful --config x-pack/platform/plugins/private/discover_enhanced/ui_tests/parallel.playwright.config.ts
```

.buildkite/scripts/steps/functional/scout_ui_tests.sh

@@ -26,6 +26,7 @@ EXIT_CODE=0
 
 # Discovery Enhanced
 for run_mode in "--stateful" "--serverless=es" "--serverless=oblt" "--serverless=security"; do
+  run_tests "Discovery Enhanced: Parallel Workers" "x-pack/platform/plugins/private/discover_enhanced/ui_tests/parallel.playwright.config.ts" "$run_mode"
   run_tests "Discovery Enhanced" "x-pack/platform/plugins/private/discover_enhanced/ui_tests/playwright.config.ts" "$run_mode"
 done
 

packages/kbn-scout/index.ts

@@ -8,15 +8,24 @@
  */
 
 export * as cli from './src/cli';
-export { expect, test, tags, createPlaywrightConfig, createLazyPageObject } from './src/playwright';
+export {
+  expect,
+  test,
+  spaceTest,
+  tags,
+  createPlaywrightConfig,
+  createLazyPageObject,
+  ingestTestDataHook,
+} from './src/playwright';
 export type {
-  ScoutPage,
   ScoutPlaywrightOptions,
   ScoutTestOptions,
+  ScoutPage,
   PageObjects,
   ScoutTestFixtures,
   ScoutWorkerFixtures,
-  EsArchiverFixture,
+  ScoutParallelTestFixtures,
+  ScoutParallelWorkerFixtures,
 } from './src/playwright';
 
 export type { Client, KbnClient, KibanaUrl, SamlSessionManager, ToolingLog } from './src/types';

packages/kbn-scout/src/common/services/kibana_url.ts

@@ -55,8 +55,9 @@ export class KibanaUrl {
    * @param appName name of the app to get the URL for
    * @param options optional modifications to apply to the URL
    */
-  app(appName: string, options?: PathOptions) {
-    return this.get(`/app/${appName}`, options);
+  app(appName: string, options?: { space?: string; pathOptions?: PathOptions }) {
+    const relPath = options?.space ? `s/${options.space}/app/${appName}` : `/app/${appName}`;
+    return this.get(relPath, options?.pathOptions);
   }
 
   toString() {

packages/kbn-scout/src/playwright/config/create_config.ts

@@ -21,6 +21,7 @@ export function createPlaywrightConfig(options: ScoutPlaywrightOptions): Playwri
 
   return defineConfig<ScoutTestOptions>({
     testDir: options.testDir,
+    globalSetup: options.globalSetup,
     /* Run tests in files in parallel */
     fullyParallel: false,
     /* Fail the build on CI if you accidentally left test.only in the source code. */

packages/kbn-scout/src/playwright/fixtures/index.ts

@@ -7,20 +7,5 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { mergeTests } from '@playwright/test';
-
-import { scoutWorkerFixtures } from './worker';
-import { scoutTestFixtures } from './test';
-
-export const scoutCoreFixtures = mergeTests(scoutWorkerFixtures, scoutTestFixtures);
-
-export type {
-  EsArchiverFixture,
-  ScoutTestFixtures,
-  ScoutWorkerFixtures,
-  ScoutPage,
-  Client,
-  KbnClient,
-  KibanaUrl,
-  ToolingLog,
-} from './types';
+export * from './single_thread_fixtures';
+export * from './parallel_run_fixtures';

packages/kbn-scout/src/playwright/fixtures/parallel_run_fixtures.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { mergeTests } from 'playwright/test';
+import { coreWorkerFixtures, scoutSpaceParallelFixture } from './worker';
+import type {
+  EsClient,
+  KbnClient,
+  KibanaUrl,
+  ScoutSpaceParallelFixture,
+  ScoutTestConfig,
+  ToolingLog,
+} from './worker';
+import {
+  scoutPageParallelFixture,
+  browserAuthParallelFixture,
+  pageObjectsParallelFixture,
+  validateTagsFixture,
+} from './test';
+import type { BrowserAuthFixture, ScoutPage, PageObjects } from './test';
+
+export const scoutParallelFixtures = mergeTests(
+  // worker scope fixtures
+  coreWorkerFixtures,
+  scoutSpaceParallelFixture,
+  // test scope fixtures
+  browserAuthParallelFixture,
+  scoutPageParallelFixture,
+  pageObjectsParallelFixture,
+  validateTagsFixture
+);
+
+export interface ScoutParallelTestFixtures {
+  browserAuth: BrowserAuthFixture;
+  page: ScoutPage;
+  pageObjects: PageObjects;
+}
+
+export interface ScoutParallelWorkerFixtures {
+  log: ToolingLog;
+  config: ScoutTestConfig;
+  kbnUrl: KibanaUrl;
+  kbnClient: KbnClient;
+  esClient: EsClient;
+  scoutSpace: ScoutSpaceParallelFixture;
+}

packages/kbn-scout/src/playwright/fixtures/single_thread_fixtures.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { mergeTests } from 'playwright/test';
+import { coreWorkerFixtures, esArchiverFixture, uiSettingsFixture } from './worker';
+import type {
+  EsArchiverFixture,
+  EsClient,
+  KbnClient,
+  KibanaUrl,
+  ScoutTestConfig,
+  ToolingLog,
+  UiSettingsFixture,
+} from './worker';
+import {
+  scoutPageFixture,
+  browserAuthFixture,
+  pageObjectsFixture,
+  validateTagsFixture,
+  BrowserAuthFixture,
+  ScoutPage,
+  PageObjects,
+} from './test';
+export type { PageObjects, ScoutPage } from './test';
+
+export const scoutFixtures = mergeTests(
+  // worker scope fixtures
+  coreWorkerFixtures,
+  esArchiverFixture,
+  uiSettingsFixture,
+  // test scope fixtures
+  browserAuthFixture,
+  scoutPageFixture,
+  pageObjectsFixture,
+  validateTagsFixture
+);
+
+export interface ScoutTestFixtures {
+  browserAuth: BrowserAuthFixture;
+  page: ScoutPage;
+  pageObjects: PageObjects;
+}
+
+export interface ScoutWorkerFixtures {
+  log: ToolingLog;
+  config: ScoutTestConfig;
+  kbnUrl: KibanaUrl;
+  kbnClient: KbnClient;
+  esClient: EsClient;
+  esArchiver: EsArchiverFixture;
+  uiSettings: UiSettingsFixture;
+}

packages/kbn-scout/src/playwright/fixtures/test/browser_auth/index.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export type LoginFunction = (role: string) => Promise<void>;
+
+export interface BrowserAuthFixture {
+  /**
+   * Logs in as a user with viewer-only permissions.
+   * @returns A Promise that resolves once the cookie in browser is set.
+   */
+  loginAsViewer: () => Promise<void>;
+  /**
+   * Logs in as a user with administrative privileges
+   * @returns A Promise that resolves once the cookie in browser is set.
+   */
+  loginAsAdmin: () => Promise<void>;
+  /**
+   * Logs in as a user with elevated, but not admin, permissions.
+   * @returns A Promise that resolves once the cookie in browser is set.
+   */
+  loginAsPrivilegedUser: () => Promise<void>;
+}
+
+export { browserAuthParallelFixture } from './parallel';
+export { browserAuthFixture } from './single_thread';

packages/kbn-scout/src/playwright/fixtures/test/browser_auth/parallel.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { BrowserAuthFixture, LoginFunction } from '.';
+import { PROJECT_DEFAULT_ROLES } from '../../../../common';
+import { serviceLoadedMsg } from '../../../utils';
+import { coreWorkerFixtures } from '../../worker';
+import { ScoutSpaceParallelFixture } from '../../worker/scout_space';
+
+/**
+ * The "browserAuth" fixture simplifies the process of logging into Kibana with
+ * different roles during tests. It uses the "samlAuth" fixture to create an authentication session
+ * for the specified role and the "context" fixture to update the cookie with the role-scoped session.
+ */
+export const browserAuthParallelFixture = coreWorkerFixtures.extend<
+  { browserAuth: BrowserAuthFixture },
+  { scoutSpace: ScoutSpaceParallelFixture }
+>({
+  browserAuth: async ({ log, context, samlAuth, config, scoutSpace }, use) => {
+    const setSessionCookie = async (cookieValue: string) => {
+      await context.clearCookies();
+      await context.addCookies([
+        {
+          name: 'sid',
+          value: cookieValue,
+          path: '/',
+          domain: 'localhost',
+        },
+      ]);
+    };
+
+    const loginAs: LoginFunction = async (role) => {
+      const spaceId = scoutSpace.id;
+      const cookie = await samlAuth.getInteractiveUserSessionCookieWithRoleScope(role, { spaceId });
+      await setSessionCookie(cookie);
+    };
+
+    const loginAsAdmin = () => loginAs('admin');
+    const loginAsViewer = () => loginAs('viewer');
+    const loginAsPrivilegedUser = () => {
+      const roleName = config.serverless
+        ? PROJECT_DEFAULT_ROLES.get(config.projectType!)!
+        : 'editor';
+      return loginAs(roleName);
+    };
+
+    log.debug(serviceLoadedMsg(`browserAuth:${scoutSpace.id}`));
+    await use({ loginAsAdmin, loginAsViewer, loginAsPrivilegedUser });
+  },
+});

packages/kbn-scout/src/playwright/fixtures/test/browser_auth.ts → packages/kbn-scout/src/playwright/fixtures/test/browser_auth/single_thread.ts

@@ -7,19 +7,17 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { test as base } from '@playwright/test';
-import { PROJECT_DEFAULT_ROLES } from '../../../common';
-import { LoginFixture, ScoutWorkerFixtures } from '../types';
-import { serviceLoadedMsg } from '../../utils';
-
-type LoginFunction = (role: string) => Promise<void>;
+import { PROJECT_DEFAULT_ROLES } from '../../../../common';
+import { serviceLoadedMsg } from '../../../utils';
+import { coreWorkerFixtures } from '../../worker';
+import { BrowserAuthFixture, LoginFunction } from '.';
 
 /**
  * The "browserAuth" fixture simplifies the process of logging into Kibana with
  * different roles during tests. It uses the "samlAuth" fixture to create an authentication session
  * for the specified role and the "context" fixture to update the cookie with the role-scoped session.
  */
-export const browserAuthFixture = base.extend<{ browserAuth: LoginFixture }, ScoutWorkerFixtures>({
+export const browserAuthFixture = coreWorkerFixtures.extend<{ browserAuth: BrowserAuthFixture }>({
   browserAuth: async ({ log, context, samlAuth, config }, use) => {
     const setSessionCookie = async (cookieValue: string) => {
       await context.clearCookies();

packages/kbn-scout/src/playwright/fixtures/test/index.ts

@@ -7,15 +7,10 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { mergeTests } from '@playwright/test';
-import { browserAuthFixture } from './browser_auth';
-import { scoutPageFixture } from './page';
-import { pageObjectsFixture } from './page_objects';
-import { validateTagsFixture } from './validate_tags';
-
-export const scoutTestFixtures = mergeTests(
-  browserAuthFixture,
-  scoutPageFixture,
-  pageObjectsFixture,
-  validateTagsFixture
-);
+export { browserAuthFixture, browserAuthParallelFixture } from './browser_auth';
+export type { BrowserAuthFixture } from './browser_auth';
+export { scoutPageFixture, scoutPageParallelFixture } from './scout_page';
+export type { ScoutPage } from './scout_page';
+export { validateTagsFixture } from './validate_tags';
+export { pageObjectsFixture, pageObjectsParallelFixture } from './page_objects';
+export type { PageObjects } from './page_objects';

packages/kbn-scout/src/playwright/fixtures/test/page_objects/index.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export type LoginFunction = (role: string) => Promise<void>;
+
+export type { PageObjects } from '../../../page_objects';
+
+export { pageObjectsParallelFixture } from './parallel';
+export { pageObjectsFixture } from './single_thread';

packages/kbn-scout/src/playwright/fixtures/test/page_objects/parallel.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { PageObjects, createCorePageObjects } from '../../../page_objects';
+import { serviceLoadedMsg } from '../../../utils';
+import { ScoutSpaceParallelFixture } from '../../worker';
+import { scoutPageParallelFixture } from '../scout_page';
+
+/**
+ * The "pageObjects" fixture provides a centralized and consistent way to access and
+ * interact with reusable Page Objects in tests. This fixture automatically
+ * initializes core Page Objects and makes them available to tests, promoting
+ * modularity and reducing redundant setup.
+ *
+ * Note: Page Objects are lazily instantiated on first access.
+ */
+export const pageObjectsParallelFixture = scoutPageParallelFixture.extend<
+  {
+    pageObjects: PageObjects;
+  },
+  { scoutSpace: ScoutSpaceParallelFixture }
+>({
+  pageObjects: async ({ page, log, scoutSpace }, use) => {
+    const corePageObjects = createCorePageObjects(page);
+    log.debug(serviceLoadedMsg(`pageObjects:${scoutSpace.id}`));
+    await use(corePageObjects);
+  },
+});