Merge branch 'stack-mon-add-ftr-logs' of https://github.com/elastic/k…

…ibana into stack-mon-add-ftr-logs

.buildkite/ftr_oblt_stateful_configs.yml

@@ -1,16 +1,12 @@
 disabled:
   # Cypress configs, for now these are still run manually
+  - x-pack/test/apm_cypress/cli_config.ts
   - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_open.ts
   - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_runner.ts
   - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config.ts
-  - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config_run.ts
-  - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config.ts
   - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config_run.ts
   - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config.ts
-  - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config_open.ts
-  - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config_runner.ts
-  - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config.ts
-
+  - x-pack/test/profiling_cypress/cli_config.ts
   #FTR configs
   - x-pack/solutions/observability/plugins/uptime/e2e/config.ts
 
@@ -32,7 +28,6 @@ enabled:
   - x-pack/test/api_integration/apis/synthetics/config.ts
   - x-pack/test/api_integration/apis/uptime/config.ts
   - x-pack/test/api_integration/apis/entity_manager/config.ts
-  - x-pack/test/api_integration/apis/streams/config.ts
   - x-pack/test/apm_api_integration/basic/config.ts
   - x-pack/test/apm_api_integration/cloud/config.ts
   - x-pack/test/apm_api_integration/rules/config.ts

.buildkite/ftr_platform_stateful_configs.yml

@@ -66,6 +66,7 @@ enabled:
   - test/functional/apps/dashboard/group4/config.ts
   - test/functional/apps/dashboard/group5/config.ts
   - test/functional/apps/dashboard/group6/config.ts
+  - test/functional/apps/dashboard/esql_controls/config.ts
   - test/functional/apps/discover/ccs_compatibility/config.ts
   - test/functional/apps/discover/embeddable/config.ts
   - test/functional/apps/discover/esql/config.ts
@@ -285,7 +286,6 @@ enabled:
   - x-pack/test/functional_cloud/saml.config.ts
   - x-pack/test/functional_solution_sidenav/config.ts
   - x-pack/test/functional_search/config.ts
-  - x-pack/test/kubernetes_security/basic/config.ts
   - x-pack/test/licensing_plugin/config.public.ts
   - x-pack/test/licensing_plugin/config.ts
   - x-pack/test/monitoring_api_integration/config.ts
@@ -343,9 +343,7 @@ enabled:
   - x-pack/test/security_functional/expired_session.config.ts
   - x-pack/test/session_view/basic/config.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
-  - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_basic.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/config_trial.ts
-  - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_trial.ts
   - x-pack/test/spaces_api_integration/spaces_only/config.ts
   - x-pack/test/task_manager_claimer_update_by_query/config.ts
   - x-pack/test/ui_capabilities/security_and_spaces/config.ts

.buildkite/ftr_security_serverless_configs.yml

@@ -121,7 +121,10 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/spaces/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/siem_migrations/rules/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
   # serverless config files that run deployment-agnostic tests
   - x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts
+  - x-pack/test/spaces_api_integration/deployment_agnostic/security_and_spaces/serverless.config.ts
+  - x-pack/test/spaces_api_integration/deployment_agnostic/security_and_spaces/serverless.copy_to_space.config.ts

.buildkite/ftr_security_stateful_configs.yml

@@ -95,6 +95,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/spaces/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/siem_migrations/rules/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_endpoint/configs/endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/integrations.config.ts
   - x-pack/test/api_integration/apis/cloud_security_posture/config.ts
@@ -104,3 +105,9 @@ enabled:
   - x-pack/test/cloud_security_posture_functional/data_views/config.ts
   - x-pack/test/automatic_import_api_integration/apis/config_basic.ts
   - x-pack/test/automatic_import_api_integration/apis/config_graphs.ts
+  - x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/spaces_api_integration/deployment_agnostic/spaces_only/config.ts
+  - x-pack/test/spaces_api_integration/deployment_agnostic/security_and_spaces/stateful.config_basic.ts
+  - x-pack/test/spaces_api_integration/deployment_agnostic/security_and_spaces/stateful.config_trial.ts
+  - x-pack/test/spaces_api_integration/deployment_agnostic/security_and_spaces/stateful.copy_to_space.config_trial.ts
+  - x-pack/test/spaces_api_integration/deployment_agnostic/security_and_spaces/stateful.copy_to_space.config_basic.ts

.buildkite/package.json

@@ -15,12 +15,14 @@
     "globby": "^11.1.0",
     "js-yaml": "^4.1.0",
     "minimatch": "^5.0.1",
+    "minimist": "^1.2.8",
     "tslib": "*"
   },
   "devDependencies": {
     "@types/chai": "^4.3.3",
-    "@types/js-yaml": "^4.0.5",
+    "@types/js-yaml": "^4.0.9",
     "@types/minimatch": "^3.0.5",
+    "@types/minimist": "^1.2.5",
     "@types/mocha": "^10.0.1",
     "@types/node": "^15.12.2",
     "chai": "^4.3.10",

.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync.yml

@@ -22,7 +22,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-management'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: false
-      branch_configuration: main
+      branch_configuration: main 8.x
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/console_definitions_sync.yml

.buildkite/pipeline-resource-definitions/kibana-deploy-cloud.yml

@@ -0,0 +1,45 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-deploy-cloud-from-pr
+  description: 'Builds Kibana and initiates a Kibana cloud deployment from a PR'
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-deploy-cloud-from-pr'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: 'group:kibana-operations'
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / deploy cloud from PR
+      description: 'Builds Kibana and initiates a Kibana cloud deployment from a PR'
+    spec:
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false'
+
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/build_pr_and_deploy_cloud.yml
+      provider_settings:
+        build_pull_requests: true
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+        trigger_mode: none
+      cancel_intermediate_builds: true
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+      tags:
+        - kibana

.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml

@@ -25,7 +25,7 @@ spec:
       branch_configuration: main
       default_branch: main
       repository: elastic/kibana
-      pipeline_file: .buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml
+      pipeline_file: .buildkite/pipelines/serverless_deployment/build_pr_and_deploy_project.yml
       provider_settings:
         build_pull_requests: true
         prefix_pull_request_fork_branch_names: false

.buildkite/pipeline-resource-definitions/kibana-on-merge.yml

@@ -24,6 +24,7 @@ spec:
         GITHUB_COMMIT_STATUS_CONTEXT: buildkite/on-merge
         REPORT_FAILED_TESTS_TO_GITHUB: 'true'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SCOUT_REPORTER_ENABLED: 'true'
       allow_rebuilds: true
       branch_configuration: main 8.x 8.17 8.16 7.17
       default_branch: main

.buildkite/pipeline-resource-definitions/kibana-pointer-compression.yml

@@ -27,10 +27,6 @@ spec:
       pipeline_file: ".buildkite/pipelines/pointer_compression.yml"
       provider_settings:
         trigger_mode: none
-      schedules:
-        Daily run:
-          branch: main
-          cronline: "@daily"
       teams:
         kibana-operations:
           access_level: MANAGE_BUILD_AND_READ

.buildkite/pipeline-resource-definitions/kibana-pr.yml

@@ -22,6 +22,7 @@ spec:
         ELASTIC_PR_COMMENTS_ENABLED: 'true'
         ELASTIC_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
         GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-ci
+        SCOUT_REPORTER_ENABLED: 'true'
       allow_rebuilds: true
       branch_configuration: ''
       cancel_intermediate_builds: true

.buildkite/pipeline-resource-definitions/locations.yml

@@ -17,6 +17,7 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-codeql.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-deploy-cloud.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing-9-dot-0.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml

.buildkite/pipeline-resource-definitions/scripts/validate-pipeline-definition.sh

@@ -15,7 +15,7 @@ ABSOLUTE_PATH=$(realpath "$TARGET_FILE")
 FILE_NAME=$(basename "$ABSOLUTE_PATH")
 FOLDER_NAME=$(dirname "$ABSOLUTE_PATH")
 
-docker run -it \
+docker run \
  --mount type=bind,source="$FOLDER_NAME",target=/home/app/ \
   docker.elastic.co/ci-agent-images/pipelib:0.8.0@sha256:641d7fc6cfe473900a1fbe49876762916d804b09fdf2945f74e9f803f3073779 \
   rre validate --backstage-entity-aware "/home/app/$FILE_NAME"

.buildkite/pipeline-utils/github/github.ts

@@ -9,6 +9,8 @@
 
 import { Octokit, RestEndpointMethodTypes } from '@octokit/rest';
 
+export const KIBANA_COMMENT_SIGIL = 'kbn-message-context';
+
 const github = new Octokit({
   auth: process.env.GITHUB_TOKEN,
 });
@@ -113,6 +115,56 @@ export function addComment(
   });
 }
 
+export async function upsertComment(
+  messageOpts: {
+    commentBody: string;
+    commentContext: string;
+    clearPrevious: boolean;
+  },
+  owner = process.env.GITHUB_PR_BASE_OWNER,
+  repo = process.env.GITHUB_PR_BASE_REPO,
+  prNumber: undefined | string | number = process.env.GITHUB_PR_NUMBER
+) {
+  const { commentBody, commentContext, clearPrevious } = messageOpts;
+  if (!owner || !repo || !prNumber) {
+    throw Error(
+      "Couldn't retrieve Github PR info from environment variables in order to add a comment"
+    );
+  }
+  if (!commentContext) {
+    throw Error('Comment context is required when updating a comment');
+  }
+
+  const commentMarker = `<!-- ${KIBANA_COMMENT_SIGIL}:${commentContext} -->`;
+  const body = `${commentMarker}\n${commentBody}`;
+
+  const existingComment = (
+    await github.paginate(github.issues.listComments, {
+      owner,
+      repo,
+      issue_number: typeof prNumber === 'number' ? prNumber : parseInt(prNumber, 10),
+    })
+  ).find((comment) => comment.body?.includes(commentMarker));
+
+  if (!existingComment) {
+    return addComment(body, owner, repo, prNumber);
+  } else if (clearPrevious) {
+    await github.issues.deleteComment({
+      owner,
+      repo,
+      comment_id: existingComment.id,
+    });
+    return addComment(body, owner, repo, prNumber);
+  } else {
+    return github.issues.updateComment({
+      owner,
+      repo,
+      comment_id: existingComment.id,
+      body,
+    });
+  }
+}
+
 export function getGithubClient() {
   return github;
 }