Merge branch 'refs/heads/main' into hide-hie-tab-when-no-access

.buildkite/scripts/steps/checks/renovate.sh

@@ -4,5 +4,11 @@
 
 set -euo pipefail
 
+source .buildkite/scripts/common/util.sh
+
 echo --- Check renovate.json
-docker run -v "$(pwd)"/renovate.json:/home/app/renovate.json docker.elastic.co/ci-agent-images/pipelib:0.8.0@sha256:641d7fc6cfe473900a1fbe49876762916d804b09fdf2945f74e9f803f3073779 renovate-config-validator
+
+retry 3 3 docker run \
+  -v "$(pwd)"/renovate.json:/home/app/renovate.json \
+  docker.elastic.co/ci-agent-images/pipelib:0.8.0@sha256:641d7fc6cfe473900a1fbe49876762916d804b09fdf2945f74e9f803f3073779 \
+  renovate-config-validator

docs/developer/plugin-list.asciidoc

@@ -840,12 +840,13 @@ Kibana.
 
 
 |{kib-repo}blob/{branch}/x-pack/plugins/security_solution_ess/README.md[securitySolutionEss]
-|This plugin contains the ESS/on-prem deployments (non-serverless) specific logic for Security Solution.
+|The security_solution_ess plugin is an internal plugin for Kibana's Security Solution, designed to encapsulate ESS-specific logic.
+This plugin is only enabled when the application is built for stateful deployments (ESS or on-prem), keeping the main security_solution plugin clean and agnostic of the offering model.
 
 
-|{kib-repo}blob/{branch}/x-pack/plugins/security_solution_serverless/README.mdx[securitySolutionServerless]
-|This plugin contains configuration and code used to create a Serverless Security project.
-It leverages universal configuration and other APIs in the serverless plugin to configure Kibana.
+|{kib-repo}blob/{branch}/x-pack/plugins/security_solution_serverless/README.md[securitySolutionServerless]
+|The security_solution_serverless plugin is an internal plugin for Kibana's Security Solution, designed to encapsulate serverless-specific logic.
+This plugin is only enabled when the application is built for serverless project, keeping the main security_solution plugin clean and agnostic of the offering model.
 
 
 |{kib-repo}blob/{branch}/x-pack/plugins/serverless/README.mdx[serverless]

packages/kbn-ci-stats-reporter/src/ci_stats_reporter.ts

@@ -170,7 +170,7 @@ export class CiStatsReporter {
       const { stdout } = await execa('git', ['config', 'user.email']);
       email = stdout;
     } catch (e) {
-      this.log.debug(e.message);
+      // no-op - we're ok with email being undefined
     }
 
     try {
@@ -181,7 +181,7 @@ export class CiStatsReporter {
     }
 
     const memUsage = process.memoryUsage();
-    const isElasticCommitter = email && email.endsWith('@elastic.co') ? true : false;
+    const isElasticCommitter = email && email.endsWith('@elastic.co');
 
     const defaultMeta = {
       kibanaUuid,
@@ -200,14 +200,16 @@ export class CiStatsReporter {
       memoryUsageHeapUsed: memUsage.heapUsed,
       memoryUsageExternal: memUsage.external,
       memoryUsageArrayBuffers: memUsage.arrayBuffers,
-      nestedTiming: process.env.CI_STATS_NESTED_TIMING ? true : false,
+      nestedTiming: !!process.env.CI_STATS_NESTED_TIMING,
       osArch: Os.arch(),
       osPlatform: Os.platform(),
       osRelease: Os.release(),
       totalMem: Os.totalmem(),
     };
 
-    this.log.debug('CIStatsReporter committerHash: %s', defaultMeta.committerHash);
+    if (defaultMeta.committerHash) {
+      this.log.debug('CIStatsReporter committerHash: %s', defaultMeta.committerHash);
+    }
 
     return !!(await this.req({
       auth: !!buildId,

packages/kbn-discover-utils/index.ts

@@ -34,6 +34,8 @@ export {
   buildDataTableRecord,
   buildDataTableRecordList,
   createLogsContextService,
+  createDegradedDocsControl,
+  createStacktraceControl,
   fieldConstants,
   formatFieldValue,
   formatHit,
@@ -52,6 +54,8 @@ export {
   LogLevelCoalescedValue,
   LogLevelBadge,
   getFieldValue,
+  getVisibleColumns,
+  canPrependTimeFieldColumn,
 } from './src';
 
 export type { LogsContextService } from './src';

packages/kbn-discover-utils/src/__mocks__/data_view.ts

@@ -140,3 +140,9 @@ export const dataViewMock = buildDataViewMock({
   name: 'the-data-view',
   fields: shallowMockedFields,
 });
+
+export const dataViewMockWithTimeField = buildDataViewMock({
+  name: 'the-data-view',
+  fields: shallowMockedFields,
+  timeFieldName: '@timestamp',
+});

packages/kbn-discover-utils/src/components/custom_control_columns/degraded_docs_control.tsx

@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { i18n } from '@kbn/i18n';
+import React from 'react';
+import { EuiCode, EuiSpacer } from '@elastic/eui';
+import {
+  RowControlColumn,
+  RowControlComponent,
+  RowControlProps,
+  RowControlRowProps,
+} from './types';
+import { DEGRADED_DOCS_FIELDS } from '../../field_constants';
+
+interface DegradedDocsControlProps extends Partial<RowControlProps> {
+  enabled?: boolean;
+}
+
+/**
+ * Degraded docs control factory function.
+ * @param props Optional props for the generated Control component, useful to override onClick, etc
+ */
+export const createDegradedDocsControl = (props?: DegradedDocsControlProps): RowControlColumn => ({
+  id: 'connectedDegradedDocs',
+  headerAriaLabel: actionsHeaderAriaLabelDegradedAction,
+  renderControl: (Control, rowProps) => {
+    return <DegradedDocs Control={Control} rowProps={rowProps} {...props} />;
+  },
+});
+
+const actionsHeaderAriaLabelDegradedAction = i18n.translate(
+  'discover.customControl.degradedDocArialLabel',
+  { defaultMessage: 'Access to degraded docs' }
+);
+
+const degradedDocButtonLabelWhenPresent = i18n.translate(
+  'discover.customControl.degradedDocPresent',
+  {
+    defaultMessage:
+      "This document couldn't be parsed correctly. Not all fields are properly populated",
+  }
+);
+
+const degradedDocButtonLabelWhenNotPresent = i18n.translate(
+  'discover.customControl.degradedDocNotPresent',
+  { defaultMessage: 'All fields in this document were parsed correctly' }
+);
+
+const degradedDocButtonLabelWhenDisabled = i18n.translate(
+  'discover.customControl.degradedDocDisabled',
+  {
+    defaultMessage:
+      'Degraded document field detection is currently disabled for this search. To enable it, include the METADATA directive for the `_ignored` field in your ES|QL query. For example:',
+  }
+);
+
+const DegradedDocs = ({
+  Control,
+  enabled = true,
+  rowProps: { record },
+  ...props
+}: {
+  Control: RowControlComponent;
+  rowProps: RowControlRowProps;
+} & DegradedDocsControlProps) => {
+  const isDegradedDocumentExists = DEGRADED_DOCS_FIELDS.some(
+    (field) => field in record.raw && record.raw[field] !== null
+  );
+
+  if (!enabled) {
+    const codeSample = 'FROM logs-* METADATA _ignored';
+
+    const tooltipContent = (
+      <div>
+        {degradedDocButtonLabelWhenDisabled}
+        <EuiSpacer size="s" />
+        <EuiCode>{codeSample}</EuiCode>
+      </div>
+    );
+
+    return (
+      <Control
+        disabled
+        data-test-subj="docTableDegradedDocDisabled"
+        tooltipContent={tooltipContent}
+        label={`${degradedDocButtonLabelWhenDisabled} ${codeSample}`}
+        iconType="indexClose"
+        onClick={undefined}
+        {...props}
+      />
+    );
+  }
+
+  return isDegradedDocumentExists ? (
+    <Control
+      data-test-subj="docTableDegradedDocExist"
+      color="danger"
+      tooltipContent={degradedDocButtonLabelWhenPresent}
+      label={degradedDocButtonLabelWhenPresent}
+      iconType="indexClose"
+      onClick={undefined}
+      {...props}
+    />
+  ) : (
+    <Control
+      data-test-subj="docTableDegradedDocDoesNotExist"
+      color="text"
+      tooltipContent={degradedDocButtonLabelWhenNotPresent}
+      label={degradedDocButtonLabelWhenNotPresent}
+      iconType="indexClose"
+      onClick={undefined}
+      {...props}
+    />
+  );
+};

packages/kbn-discover-utils/src/components/custom_control_columns/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export { createDegradedDocsControl } from './degraded_docs_control';
+export { createStacktraceControl } from './stacktrace_control';

packages/kbn-discover-utils/src/components/custom_control_columns/stacktrace_control.tsx

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { i18n } from '@kbn/i18n';
+import React from 'react';
+import {
+  RowControlColumn,
+  RowControlComponent,
+  RowControlProps,
+  RowControlRowProps,
+} from './types';
+import { LogDocument } from '../../data_types';
+import { getStacktraceFields } from '../../utils/get_stack_trace_fields';
+
+/**
+ * Stacktrace control factory function.
+ * @param props Optional props for the generated Control component, useful to override onClick, etc
+ */
+export const createStacktraceControl = (props?: Partial<RowControlProps>): RowControlColumn => ({
+  id: 'connectedStacktraceDocs',
+  headerAriaLabel: actionsHeaderAriaLabelStacktraceAction,
+  renderControl: (Control, rowProps) => {
+    return <Stacktrace Control={Control} rowProps={rowProps} {...props} />;
+  },
+});
+
+const actionsHeaderAriaLabelStacktraceAction = i18n.translate(
+  'discover.customControl.stacktraceArialLabel',
+  { defaultMessage: 'Access to available stacktraces' }
+);
+
+const stacktraceAvailableControlButton = i18n.translate(
+  'discover.customControl.stacktrace.available',
+  { defaultMessage: 'Stacktraces available' }
+);
+
+const stacktraceNotAvailableControlButton = i18n.translate(
+  'discover.customControl.stacktrace.notAvailable',
+  { defaultMessage: 'Stacktraces not available' }
+);
+
+const Stacktrace = ({
+  Control,
+  rowProps: { record },
+  ...props
+}: {
+  Control: RowControlComponent;
+  rowProps: RowControlRowProps;
+} & Partial<RowControlProps>) => {
+  const stacktrace = getStacktraceFields(record as LogDocument);
+  const hasValue = Object.values(stacktrace).some(Boolean);
+
+  return hasValue ? (
+    <Control
+      data-test-subj="docTableStacktraceExist"
+      label={stacktraceAvailableControlButton}
+      tooltipContent={stacktraceAvailableControlButton}
+      iconType="apmTrace"
+      onClick={undefined}
+      {...props}
+    />
+  ) : (
+    <Control
+      disabled
+      data-test-subj="docTableStacktraceDoesNotExist"
+      label={stacktraceNotAvailableControlButton}
+      tooltipContent={stacktraceNotAvailableControlButton}
+      iconType="apmTrace"
+      onClick={undefined}
+      {...props}
+    />
+  );
+};

packages/kbn-discover-utils/src/components/custom_control_columns/types.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EuiButtonIconProps, EuiDataGridControlColumn, IconType } from '@elastic/eui';
+import React, { FC, ReactElement } from 'react';
+import { DataTableRecord } from '../../types';
+
+export interface RowControlRowProps {
+  rowIndex: number;
+  record: DataTableRecord;
+}
+
+export interface RowControlProps {
+  'data-test-subj'?: string;
+  color?: EuiButtonIconProps['color'];
+  disabled?: boolean;
+  label: string;
+  iconType: IconType;
+  onClick: ((props: RowControlRowProps) => void) | undefined;
+  tooltipContent?: React.ReactNode;
+}
+
+export type RowControlComponent = FC<RowControlProps>;
+
+export interface RowControlColumn {
+  id: string;
+  headerAriaLabel: string;
+  headerCellRender?: EuiDataGridControlColumn['headerCellRender'];
+  renderControl: (Control: RowControlComponent, props: RowControlRowProps) => ReactElement;
+}

packages/kbn-discover-utils/src/field_constants.ts

@@ -35,7 +35,7 @@ export const CONTAINER_NAME_FIELD = 'container.name';
 export const CONTAINER_ID_FIELD = 'container.id';
 
 // Degraded Docs
-export const DEGRADED_DOCS_FIELD = 'ignored_field_values';
+export const DEGRADED_DOCS_FIELDS = ['ignored_field_values', '_ignored'] as const;
 
 // Error Stacktrace
 export const ERROR_STACK_TRACE = 'error.stack_trace';

packages/kbn-discover-utils/src/index.ts

@@ -12,3 +12,5 @@ export * as fieldConstants from './field_constants';
 export * from './hooks';
 export * from './utils';
 export * from './data_types';
+
+export * from './components/custom_control_columns';

packages/kbn-discover-utils/src/types.ts

@@ -8,8 +8,15 @@
  */
 
 import type { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { DatatableColumnMeta } from '@kbn/expressions-plugin/common';
 
 export type { IgnoredReason, ShouldShowFieldInTableHandler } from './utils';
+export type {
+  RowControlColumn,
+  RowControlComponent,
+  RowControlProps,
+  RowControlRowProps,
+} from './components/custom_control_columns/types';
 
 type DiscoverSearchHit = SearchHit<Record<string, unknown>>;
 
@@ -41,6 +48,17 @@ export interface DataTableRecord {
   isAnchor?: boolean;
 }
 
+/**
+ * Custom column types per column name
+ */
+export type DataTableColumnsMeta = Record<
+  string,
+  {
+    type: DatatableColumnMeta['type'];
+    esType?: DatatableColumnMeta['esType'];
+  }
+>;
+
 type FormattedHitPair = readonly [
   fieldDisplayName: string,
   formattedValue: string,