From b38941be7a253c80d426a49af806575ba15652a5 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula <123897612+bhapas@users.noreply.github.com> Date: Thu, 3 Oct 2024 11:14:39 +0200 Subject: [PATCH 01/42] [Automatic Import] Fix Non-ecs compatible fields in grok processor (#194727) ## Release Note Fixes a bug to resolve non-ecs compatible fields in Structured / Unstructured syslog processing in Automatic Import. ## Summary https://github.com/elastic/kibana/issues/194205 explains the issue. This PR fixes `packageName.dataStreamName` for handling header values from grok processor for KV graph so that ecs mapping gets the header values in the converted json Samples too.. ### Before this PR ![image](https://github.com/user-attachments/assets/d2660f7d-2cca-413c-ab90-1a0f3e1b4a03) ### After this PR image - Closes https://github.com/elastic/kibana/issues/194205 ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../server/graphs/kv/constants.ts | 7 ++- .../server/graphs/kv/error.ts | 2 + .../server/graphs/kv/header.ts | 2 + .../server/graphs/kv/prompts.ts | 6 ++- .../server/graphs/kv/validate.ts | 51 +++++++++++-------- .../server/graphs/related/prompts.ts | 2 + .../server/graphs/unstructured/error.ts | 2 + .../server/graphs/unstructured/prompts.ts | 2 + .../server/graphs/unstructured/types.ts | 5 +- .../graphs/unstructured/unstructured.ts | 2 + .../graphs/unstructured/validate.test.ts | 12 +++-- .../server/graphs/unstructured/validate.ts | 11 ++-- .../server/templates/pipeline.yml.njk | 4 +- 13 files changed, 74 insertions(+), 34 deletions(-) diff --git a/x-pack/plugins/integration_assistant/server/graphs/kv/constants.ts b/x-pack/plugins/integration_assistant/server/graphs/kv/constants.ts index 92cc55841bb98..183898ec31354 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/kv/constants.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/kv/constants.ts @@ -17,12 +17,13 @@ export const KV_HEADER_EXAMPLE_ANSWER = { rfc: 'RFC2454', regex: '/(?:(d{4}[-]d{2}[-]d{2}[T]d{2}[:]d{2}[:]d{2}(?:.d{1,6})?(?:[+-]d{2}[:]d{2}|Z)?)|-)s(?:([w][wd.@-]*)|-)s(.*)$/', - grok_pattern: '%{WORD:key1}:%{WORD:value1};%{WORD:key2}:%{WORD:value2}:%{GREEDYDATA:message}', + grok_pattern: + '%{WORD:cisco.audit.key1}:%{WORD:cisco.audit.value1};%{WORD:cisco.audit.key2}:%{WORD:cisco.audit.value2}:%{GREEDYDATA:message}', }; export const KV_HEADER_ERROR_EXAMPLE_ANSWER = { grok_pattern: - '%{TIMESTAMP:timestamp}:%{WORD:value1};%{WORD:key2}:%{WORD:value2}:%{GREEDYDATA:message}', + '%{TIMESTAMP:cisco.audit.timestamp}:%{WORD:cisco.audit.value1};%{WORD:cisco.audit.key2}:%{WORD:cisco.audit.value2}:%{GREEDYDATA:message}', }; export const onFailure = { @@ -33,6 +34,8 @@ export const onFailure = { }, }; +export const removeProcessor = { remove: { field: 'message', ignore_missing: true } }; + export const COMMON_ERRORS = [ { error: 'field [message] does not contain value_split [=]', diff --git a/x-pack/plugins/integration_assistant/server/graphs/kv/error.ts b/x-pack/plugins/integration_assistant/server/graphs/kv/error.ts index b1b7c12a68d5a..dabaea0769442 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/kv/error.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/kv/error.ts @@ -46,6 +46,8 @@ export async function handleHeaderError({ const currentPattern = state.grokPattern; const pattern = await kvHeaderErrorGraph.invoke({ + packageName: state.packageName, + dataStreamName: state.dataStreamName, current_pattern: JSON.stringify(currentPattern, null, 2), errors: JSON.stringify(state.errors, null, 2), ex_answer: JSON.stringify(KV_HEADER_ERROR_EXAMPLE_ANSWER, null, 2), diff --git a/x-pack/plugins/integration_assistant/server/graphs/kv/header.ts b/x-pack/plugins/integration_assistant/server/graphs/kv/header.ts index 36d8968ab9e67..532bcfb9672c3 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/kv/header.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/kv/header.ts @@ -21,6 +21,8 @@ export async function handleHeader({ const pattern = await kvHeaderGraph.invoke({ samples: state.logSamples, + packageName: state.packageName, + dataStreamName: state.dataStreamName, ex_answer: JSON.stringify(KV_HEADER_EXAMPLE_ANSWER, null, 2), }); diff --git a/x-pack/plugins/integration_assistant/server/graphs/kv/prompts.ts b/x-pack/plugins/integration_assistant/server/graphs/kv/prompts.ts index 2ab1073a4ba8b..21889be26cfb2 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/kv/prompts.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/kv/prompts.ts @@ -79,8 +79,9 @@ Follow these steps to identify the header pattern: You ALWAYS follow these guidelines when writing your response: - - Do not parse the message part in the regex. Just the header part should be in regex nad grok_pattern. + - Do not parse the message part in the regex. Just the header part should be in regex and grok_pattern. - Make sure to map the remaining message body to \'message\' in grok pattern. + - Make sure to add \`{packageName}.{dataStreamName}\` as a prefix to each field in the pattern. Refer to example response. - Do not respond with anything except the processor as a JSON object enclosed with 3 backticks (\`), see example response above. Use strict JSON response format. @@ -121,8 +122,9 @@ Follow these steps to fix the errors in the header pattern: 4. Make sure the regex and grok pattern matches all the header information. Only the structured message body should be under GREEDYDATA in grok pattern. You ALWAYS follow these guidelines when writing your response: - - Do not parse the message part in the regex. Just the header part should be in regex nad grok_pattern. + - Do not parse the message part in the regex. Just the header part should be in regex and grok_pattern. - Make sure to map the remaining message body to \'message\' in grok pattern. + - Make sure to add \`{packageName}.{dataStreamName}\` as a prefix to each field in the pattern. Refer to example response. - Do not respond with anything except the processor as a JSON object enclosed with 3 backticks (\`), see example response above. Use strict JSON response format. diff --git a/x-pack/plugins/integration_assistant/server/graphs/kv/validate.ts b/x-pack/plugins/integration_assistant/server/graphs/kv/validate.ts index b0601de74aa5e..e130a69910076 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/kv/validate.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/kv/validate.ts @@ -10,10 +10,10 @@ import { ESProcessorItem } from '../../../common'; import type { KVState } from '../../types'; import type { HandleKVNodeParams } from './types'; import { testPipeline } from '../../util'; -import { onFailure } from './constants'; +import { onFailure, removeProcessor } from './constants'; import { createGrokProcessor } from '../../util/processors'; -interface KVResult { +interface StructuredLogResult { [packageName: string]: { [dataStreamName: string]: unknown }; } @@ -32,25 +32,24 @@ export async function handleKVValidate({ // Pick logSamples if there was no header detected. const samples = state.header ? state.kvLogMessages : state.logSamples; - - const { pipelineResults: kvOutputSamples, errors } = (await createJSONInput( - kvProcessor, - samples, - client, - state - )) as { pipelineResults: KVResult[]; errors: object[] }; - + const { errors } = await verifyKVProcessor(kvProcessor, samples, client); if (errors.length > 0) { return { errors, lastExecutedChain: 'kvValidate' }; } // Converts JSON Object into a string and parses it as a array of JSON strings - const jsonSamples = kvOutputSamples + const additionalProcessors = state.additionalProcessors; + additionalProcessors.push(kvProcessor[0]); + const samplesObject: StructuredLogResult[] = await buildJSONSamples( + state.logSamples, + additionalProcessors, + client + ); + + const jsonSamples = samplesObject .map((log) => log[packageName]) .map((log) => log[dataStreamName]) .map((log) => JSON.stringify(log)); - const additionalProcessors = state.additionalProcessors; - additionalProcessors.push(kvProcessor[0]); return { jsonSamples, @@ -89,15 +88,25 @@ export async function handleHeaderValidate({ }; } -async function createJSONInput( +async function verifyKVProcessor( kvProcessor: ESProcessorItem, formattedSamples: string[], - client: IScopedClusterClient, - state: KVState -): Promise<{ pipelineResults: object[]; errors: object[] }> { - // This processor removes the original message field in the JSON output - const removeProcessor = { remove: { field: 'message', ignore_missing: true } }; + client: IScopedClusterClient +): Promise<{ errors: object[] }> { + // This processor removes the original message field in the output const pipeline = { processors: [kvProcessor[0], removeProcessor], on_failure: [onFailure] }; - const { pipelineResults, errors } = await testPipeline(formattedSamples, pipeline, client); - return { pipelineResults, errors }; + const { errors } = await testPipeline(formattedSamples, pipeline, client); + return { errors }; +} + +async function buildJSONSamples( + samples: string[], + processors: object[], + client: IScopedClusterClient +): Promise { + const pipeline = { processors: [...processors, removeProcessor], on_failure: [onFailure] }; + const { pipelineResults } = (await testPipeline(samples, pipeline, client)) as { + pipelineResults: StructuredLogResult[]; + }; + return pipelineResults; } diff --git a/x-pack/plugins/integration_assistant/server/graphs/related/prompts.ts b/x-pack/plugins/integration_assistant/server/graphs/related/prompts.ts index 87947eb8763af..9fa50d5900806 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/related/prompts.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/related/prompts.ts @@ -33,6 +33,7 @@ For each pipeline result you find matching values that would fit any of the rela You ALWAYS follow these guidelines when writing your response: +- The \`message\` field may not be part of related fields. - You can use as many processor objects as needed to map all relevant pipeline result fields to any of the ECS related fields. - If no relevant fields or values are found that could be mapped confidently to any of the related fields, then respond with an empty array [] as valid JSON enclosed with 3 backticks (\`). - Do not respond with anything except the array of processors as a valid JSON objects enclosed with 3 backticks (\`), see example response below. @@ -79,6 +80,7 @@ Follow these steps to help resolve the current ingest pipeline issues: You ALWAYS follow these guidelines when writing your response: +- The \`message\` field may not be part of related fields. - Never use "split" in template values, only use the field name inside the triple brackets. If the error mentions "Improperly closed variable in query-template" then check each "value" field for any special characters and remove them. - If solving an error means removing the last processor in the list, then return an empty array [] as valid JSON enclosed with 3 backticks (\`). - Do not respond with anything except the complete updated array of processors as a valid JSON object enclosed with 3 backticks (\`), see example response below. diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/error.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/error.ts index d002dd19d5439..c353ae4d24c43 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/unstructured/error.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/error.ts @@ -20,6 +20,8 @@ export async function handleUnstructuredError({ const currentPatterns = state.grokPatterns; const pattern = await grokErrorGraph.invoke({ + packageName: state.packageName, + dataStreamName: state.dataStreamName, current_pattern: JSON.stringify(currentPatterns, null, 2), errors: JSON.stringify(state.errors, null, 2), ex_answer: JSON.stringify(GROK_ERROR_EXAMPLE_ANSWER, null, 2), diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/prompts.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/prompts.ts index 5cf5c67135d53..7f19b2b0d28bc 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/unstructured/prompts.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/prompts.ts @@ -39,6 +39,7 @@ export const GROK_MAIN_PROMPT = ChatPromptTemplate.fromMessages([ You ALWAYS follow these guidelines when writing your response: - Make sure to map the remaining message part to \'message\' in grok pattern. + - Make sure to add \`{packageName}.{dataStreamName}\` as a prefix to each field in the pattern. Refer to example response. - Do not respond with anything except the processor as a JSON object enclosed with 3 backticks (\`), see example response above. Use strict JSON response format. @@ -89,6 +90,7 @@ Follow these steps to help improve the grok patterns and apply it step by step: You ALWAYS follow these guidelines when writing your response: - Make sure to map the remaining message part to \'message\' in grok pattern. + - Make sure to add \`{packageName}.{dataStreamName}\` as a prefix to each field in the pattern. Refer to example response. - Do not respond with anything except the processor as a JSON object enclosed with 3 backticks (\`), see example response above. Use strict JSON response format. diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/types.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/types.ts index 218d3856cb661..8c1a32d5d74d1 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/unstructured/types.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/types.ts @@ -25,7 +25,10 @@ export interface HandleUnstructuredNodeParams extends UnstructuredNodeParams { } export interface GrokResult { - [key: string]: unknown; grok_patterns: string[]; message: string; } + +export interface LogResult { + [packageName: string]: { [dataStreamName: string]: unknown }; +} diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/unstructured.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/unstructured.ts index 42186e796275f..c00d33a78b2d8 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/unstructured/unstructured.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/unstructured.ts @@ -21,6 +21,8 @@ export async function handleUnstructured({ const samples = state.logSamples; const pattern = (await grokMainGraph.invoke({ + packageName: state.packageName, + dataStreamName: state.dataStreamName, samples: samples[0], ex_answer: JSON.stringify(GROK_EXAMPLE_ANSWER, null, 2), })) as GrokResult; diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.test.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.test.ts index 493834e3220f9..5251fa1a730a9 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.test.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.test.ts @@ -28,9 +28,15 @@ describe('Testing unstructured validation without errors', () => { const client = { asCurrentUser: { ingest: { - simulate: jest - .fn() - .mockReturnValue({ docs: [{ doc: { _source: { message: 'dummy data' } } }] }), + simulate: jest.fn().mockReturnValue({ + docs: [ + { + doc: { + _source: { testPackage: { testDatastream: { message: 'dummy data' } } }, + }, + }, + ], + }), }, }, } as unknown as IScopedClusterClient; diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.ts index 043e38be0983f..eea7602b641d6 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.ts @@ -6,7 +6,7 @@ */ import type { UnstructuredLogState } from '../../types'; -import type { GrokResult, HandleUnstructuredNodeParams } from './types'; +import type { HandleUnstructuredNodeParams, LogResult } from './types'; import { testPipeline } from '../../util'; import { onFailure } from './constants'; import { createGrokProcessor } from '../../util/processors'; @@ -18,9 +18,11 @@ export async function handleUnstructuredValidate({ const grokPatterns = state.grokPatterns; const grokProcessor = createGrokProcessor(grokPatterns); const pipeline = { processors: grokProcessor, on_failure: [onFailure] }; + const packageName = state.packageName; + const dataStreamName = state.dataStreamName; const { pipelineResults, errors } = (await testPipeline(state.logSamples, pipeline, client)) as { - pipelineResults: GrokResult[]; + pipelineResults: LogResult[]; errors: object[]; }; @@ -28,7 +30,10 @@ export async function handleUnstructuredValidate({ return { errors, lastExecutedChain: 'unstructuredValidate' }; } - const jsonSamples: string[] = pipelineResults.map((entry) => JSON.stringify(entry)); + const jsonSamples = pipelineResults + .map((log) => log[packageName]) + .map((log) => log[dataStreamName]) + .map((log) => JSON.stringify(log)); const additionalProcessors = state.additionalProcessors; additionalProcessors.push(grokProcessor[0]); diff --git a/x-pack/plugins/integration_assistant/server/templates/pipeline.yml.njk b/x-pack/plugins/integration_assistant/server/templates/pipeline.yml.njk index d583d68c4b733..ba846dc50fba9 100644 --- a/x-pack/plugins/integration_assistant/server/templates/pipeline.yml.njk +++ b/x-pack/plugins/integration_assistant/server/templates/pipeline.yml.njk @@ -19,11 +19,11 @@ processors: field: originalMessage ignore_missing: true tag: remove_copied_message - if: 'ctx.event?.original != null'{% if log_format != 'unstructured' %} + if: 'ctx.event?.original != null' - remove: field: message ignore_missing: true - tag: remove_message{% endif %}{% if (log_format == 'json') or (log_format == 'ndjson') %} + tag: remove_message{% if (log_format == 'json') or (log_format == 'ndjson') %} - json: field: event.original tag: json_original From 9f58ffd52755335aa4a8558cdad408c6b32526f4 Mon Sep 17 00:00:00 2001 From: Elena Stoeva <59341489+ElenaStoeva@users.noreply.github.com> Date: Thu, 3 Oct 2024 10:37:56 +0100 Subject: [PATCH 02/42] [Mappings editor] Only add defined advanced options to request (#194148) Fixes https://github.com/elastic/kibana/issues/106006 Fixes https://github.com/elastic/kibana/issues/106151 Fixes https://github.com/elastic/kibana/issues/150395 ## Summary This PR makes the Advanced options (configuration) form add to the request only the field values that have set values. This works by adding the `stripUnsetFields` option to the `useForm` hook (similar to the `stripEmptyFields` option) which determines if the unset values will be returned by the form (unset means that the field hasn't been set/modified by the user (is not dirty) and its initial value is undefined). https://github.com/user-attachments/assets/b46af90d-6886-4232-ae0f-66910902e238 --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../static/forms/docs/core/use_form_hook.mdx | 45 +++++++++++++++++++ .../forms/hook_form_lib/hooks/use_form.ts | 36 ++++++++++----- .../static/forms/hook_form_lib/types.ts | 4 ++ .../configuration_form/configuration_form.tsx | 38 ++++++---------- .../index_management/index_template_wizard.ts | 35 +++++++++++++++ .../page_objects/index_management_page.ts | 14 ++++++ 6 files changed, 138 insertions(+), 34 deletions(-) diff --git a/src/plugins/es_ui_shared/static/forms/docs/core/use_form_hook.mdx b/src/plugins/es_ui_shared/static/forms/docs/core/use_form_hook.mdx index 82cd0c88834a3..b8186aac3ac84 100644 --- a/src/plugins/es_ui_shared/static/forms/docs/core/use_form_hook.mdx +++ b/src/plugins/es_ui_shared/static/forms/docs/core/use_form_hook.mdx @@ -261,3 +261,48 @@ With this option you can decide if you want empty string value to be returned by "role": "" } ``` + +#### stripUnsetFields + +**Type:** `boolean` +**Default:** `false` + +Sometimes, we only want to include fields that have a defined initial value or if their value has been set by the user. +In this case, set `stripUnsetFields` to `true`. + +Suppose we have a toggle field `autocompleteEnabled`, which doesn't have a specified default value passed to `useForm`: + +```js +const { form } = useForm({ + defaultValue: { + darkModeEnabled: false, + accessibilityEnabled: true, + autocompleteEnabled: undefined, + }, + options: { stripUnsetFields: true }, +}); +``` + +Initially, the form data includes only `darkModeEnabled` and `accessibilityEnabled` because `autocompleteEnabled` is stripped. + +```js +{ + "darkModeEnabled": false, + "accessibilityEnabled": true, +} +``` + +Then the user toggles the `autocompleteEnabled` field to `false`. Now the field is included in the form data: + +```js +{ + "darkModeEnabled": false, + "accessibilityEnabled": true, + "autocompleteEnabled": false, +} +``` + +Note: This option only considers the `defaultValue` config passed to `useForm()` to determine if the initial value is +undefined. If a default value has been specified as a prop to the `` component or in the form schema, +but not in the `defaultValue` config for `useForm()`, the field would initially be populated with the specified default +value, but it won't be included in the form data until the user explicitly sets its value. diff --git a/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_form.ts b/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_form.ts index ecf3a242f0f16..84b338fb95106 100644 --- a/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_form.ts +++ b/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_form.ts @@ -25,6 +25,7 @@ import { createArrayItem, getInternalArrayFieldPath } from '../components/use_ar const DEFAULT_OPTIONS = { valueChangeDebounceTime: 500, stripEmptyFields: true, + stripUnsetFields: false, }; export interface UseFormReturn { @@ -66,13 +67,18 @@ export function useForm( return initDefaultValue(defaultValue); }, [defaultValue, initDefaultValue]); - const { valueChangeDebounceTime, stripEmptyFields: doStripEmptyFields } = options ?? {}; + const { + valueChangeDebounceTime, + stripEmptyFields: doStripEmptyFields, + stripUnsetFields, + } = options ?? {}; const formOptions = useMemo( () => ({ stripEmptyFields: doStripEmptyFields ?? DEFAULT_OPTIONS.stripEmptyFields, valueChangeDebounceTime: valueChangeDebounceTime ?? DEFAULT_OPTIONS.valueChangeDebounceTime, + stripUnsetFields: stripUnsetFields ?? DEFAULT_OPTIONS.stripUnsetFields, }), - [valueChangeDebounceTime, doStripEmptyFields] + [valueChangeDebounceTime, doStripEmptyFields, stripUnsetFields] ); const [isSubmitted, setIsSubmitted] = useState(false); @@ -177,8 +183,16 @@ export function useForm( const fieldsToArray = useCallback<() => FieldHook[]>(() => Object.values(fieldsRefs.current), []); + const getFieldDefaultValue: FormHook['getFieldDefaultValue'] = useCallback( + (fieldName) => get(defaultValueDeserialized.current ?? {}, fieldName), + [] + ); + const getFieldsForOutput = useCallback( - (fields: FieldsMap, opts: { stripEmptyFields: boolean }): FieldsMap => { + ( + fields: FieldsMap, + opts: { stripEmptyFields: boolean; stripUnsetFields: boolean } + ): FieldsMap => { return Object.entries(fields).reduce((acc, [key, field]) => { if (!field.__isIncludedInOutput) { return acc; @@ -191,11 +205,17 @@ export function useForm( } } + if (opts.stripUnsetFields) { + if (!field.isDirty && getFieldDefaultValue(field.path) === undefined) { + return acc; + } + } + acc[key] = field; return acc; }, {} as FieldsMap); }, - [] + [getFieldDefaultValue] ); const updateFormDataAt: FormHook['__updateFormDataAt'] = useCallback( @@ -396,12 +416,13 @@ export function useForm( const getFormData: FormHook['getFormData'] = useCallback(() => { const fieldsToOutput = getFieldsForOutput(fieldsRefs.current, { stripEmptyFields: formOptions.stripEmptyFields, + stripUnsetFields: formOptions.stripUnsetFields, }); const fieldsValue = mapFormFields(fieldsToOutput, (field) => field.__serializeValue()); return serializer ? serializer(unflattenObject(fieldsValue)) : unflattenObject(fieldsValue); - }, [getFieldsForOutput, formOptions.stripEmptyFields, serializer]); + }, [getFieldsForOutput, formOptions.stripEmptyFields, formOptions.stripUnsetFields, serializer]); const getErrors: FormHook['getErrors'] = useCallback(() => { if (isValid === true) { @@ -455,11 +476,6 @@ export function useForm( const getFields: FormHook['getFields'] = useCallback(() => fieldsRefs.current, []); - const getFieldDefaultValue: FormHook['getFieldDefaultValue'] = useCallback( - (fieldName) => get(defaultValueDeserialized.current ?? {}, fieldName), - [] - ); - const updateFieldValues: FormHook['updateFieldValues'] = useCallback( (updatedFormData, { runDeserializer = true } = {}) => { if ( diff --git a/src/plugins/es_ui_shared/static/forms/hook_form_lib/types.ts b/src/plugins/es_ui_shared/static/forms/hook_form_lib/types.ts index f700faa66e6fd..282b9de8d3ee4 100644 --- a/src/plugins/es_ui_shared/static/forms/hook_form_lib/types.ts +++ b/src/plugins/es_ui_shared/static/forms/hook_form_lib/types.ts @@ -141,6 +141,10 @@ export interface FormOptions { * Remove empty string field ("") from form data */ stripEmptyFields?: boolean; + /** + * Remove fields from form data that don't have initial value and are not modified by the user. + */ + stripUnsetFields?: boolean; } export interface FieldHook { diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/configuration_form/configuration_form.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/configuration_form/configuration_form.tsx index b326779d8a76c..e3e32c55aada0 100644 --- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/configuration_form/configuration_form.tsx +++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/configuration_form/configuration_form.tsx @@ -29,30 +29,19 @@ interface Props { } const formSerializer = (formData: GenericObject, sourceFieldMode?: string) => { - const { - dynamicMapping: { - enabled: dynamicMappingsEnabled, - throwErrorsForUnmappedFields, - /* eslint-disable @typescript-eslint/naming-convention */ - numeric_detection, - date_detection, - dynamic_date_formats, - /* eslint-enable @typescript-eslint/naming-convention */ - }, - sourceField, - metaField, - _routing, - _size, - subobjects, - } = formData; + const { dynamicMapping, sourceField, metaField, _routing, _size, subobjects } = formData; - const dynamic = dynamicMappingsEnabled ? true : throwErrorsForUnmappedFields ? 'strict' : false; + const dynamic = dynamicMapping?.enabled + ? true + : dynamicMapping?.throwErrorsForUnmappedFields + ? 'strict' + : dynamicMapping?.enabled; const serialized = { dynamic, - numeric_detection, - date_detection, - dynamic_date_formats, + numeric_detection: dynamicMapping?.numeric_detection, + date_detection: dynamicMapping?.date_detection, + dynamic_date_formats: dynamicMapping?.dynamic_date_formats, _source: sourceFieldMode ? { mode: sourceFieldMode } : sourceField, _meta: metaField, _routing, @@ -85,18 +74,18 @@ const formDeserializer = (formData: GenericObject) => { return { dynamicMapping: { - enabled: dynamic === true || dynamic === undefined, - throwErrorsForUnmappedFields: dynamic === 'strict', + enabled: dynamic === 'strict' ? false : dynamic, + throwErrorsForUnmappedFields: dynamic === 'strict' ? true : undefined, numeric_detection, date_detection, dynamic_date_formats, }, sourceField: { - enabled: enabled === true || enabled === undefined, + enabled, includes, excludes, }, - metaField: _meta ?? {}, + metaField: _meta, _routing, _size, subobjects, @@ -121,6 +110,7 @@ export const ConfigurationForm = React.memo(({ value, esNodesPlugins }: Props) = deserializer: formDeserializer, defaultValue: value, id: 'configurationForm', + options: { stripUnsetFields: true }, }); const dispatch = useDispatch(); const { subscribe, submit, reset, getFormData } = form; diff --git a/x-pack/test/functional/apps/index_management/index_template_wizard.ts b/x-pack/test/functional/apps/index_management/index_template_wizard.ts index e9eb68f749e78..5b49286f6182b 100644 --- a/x-pack/test/functional/apps/index_management/index_template_wizard.ts +++ b/x-pack/test/functional/apps/index_management/index_template_wizard.ts @@ -15,6 +15,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const comboBox = getService('comboBox'); const find = getService('find'); const browser = getService('browser'); + const log = getService('log'); describe('Index template wizard', function () { before(async () => { @@ -162,6 +163,40 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { expect(await testSubjects.exists('fieldSubType')).to.be(true); expect(await testSubjects.exists('nextButton')).to.be(true); }); + + it("advanced options tab doesn't add default values to request by default", async () => { + await pageObjects.indexManagement.changeMappingsEditorTab('advancedOptions'); + await testSubjects.click('previewIndexTemplate'); + const templatePreview = await testSubjects.getVisibleText('simulateTemplatePreview'); + + await log.debug(`Template preview text: ${templatePreview}`); + + // All advanced options should not be part of the request + expect(templatePreview).to.not.contain('"dynamic"'); + expect(templatePreview).to.not.contain('"subobjects"'); + expect(templatePreview).to.not.contain('"dynamic_date_formats"'); + expect(templatePreview).to.not.contain('"date_detection"'); + expect(templatePreview).to.not.contain('"numeric_detection"'); + }); + + it('advanced options tab adds the set values to the request', async () => { + await pageObjects.indexManagement.changeMappingsEditorTab('advancedOptions'); + + // Toggle the subobjects field to false + await testSubjects.click('subobjectsToggle'); + + await testSubjects.click('previewIndexTemplate'); + const templatePreview = await testSubjects.getVisibleText('simulateTemplatePreview'); + + await log.debug(`Template preview text: ${templatePreview}`); + + // Only the subobjects option should be part of the request + expect(templatePreview).to.contain('"subobjects": false'); + expect(templatePreview).to.not.contain('"dynamic"'); + expect(templatePreview).to.not.contain('"dynamic_date_formats"'); + expect(templatePreview).to.not.contain('"date_detection"'); + expect(templatePreview).to.not.contain('"numeric_detection"'); + }); }); }); }; diff --git a/x-pack/test/functional/page_objects/index_management_page.ts b/x-pack/test/functional/page_objects/index_management_page.ts index 8730d2807cd21..8077581bbbb48 100644 --- a/x-pack/test/functional/page_objects/index_management_page.ts +++ b/x-pack/test/functional/page_objects/index_management_page.ts @@ -118,6 +118,20 @@ export function IndexManagementPageProvider({ getService }: FtrProviderContext) await testSubjects.click(tab); }, + async changeMappingsEditorTab( + tab: 'mappedFields' | 'runtimeFields' | 'dynamicTemplates' | 'advancedOptions' + ) { + const index = [ + 'mappedFields', + 'runtimeFields', + 'dynamicTemplates', + 'advancedOptions', + ].indexOf(tab); + + const tabs = await testSubjects.findAll('formTab'); + await tabs[index].click(); + }, + async clickNextButton() { await testSubjects.click('nextButton'); }, From d22b8306236d98c376c42e94a268fd5f9d8bca85 Mon Sep 17 00:00:00 2001 From: Michael DeFazio Date: Thu, 3 Oct 2024 06:04:56 -0400 Subject: [PATCH 03/42] [ResponseOps][Connectors] Fix flex issues of create connector flyout with technical badge (#194775) ## Summary When multiple badges are present, the Technical Preview badge in the Connector flyout gets pushed away from the title. ![CleanShot 2024-10-02 at 20 51 04@2x](https://github.com/user-attachments/assets/2e12e082-9a53-4046-a7a0-7c3bb2d6a100) With the flexbox fixes ![CleanShot 2024-10-02 at 20 49 40@2x](https://github.com/user-attachments/assets/6a2f92b9-226b-4497-9058-18a5b7cf4ebf) ### Checklist Delete any items that are not applicable to this PR. - [ ] ~Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)~ - [ ] ~[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials~ - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] ~Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/))~ - [ ] ~Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))~ - [ ] ~If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)~ - [x] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [x] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) --- .../action_connector_form/create_connector_flyout/header.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/create_connector_flyout/header.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/create_connector_flyout/header.tsx index a986a8d67f9b0..0b252ca6660c6 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/create_connector_flyout/header.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/create_connector_flyout/header.tsx @@ -47,8 +47,8 @@ const FlyoutHeaderComponent: React.FC = ({ {actionTypeName && actionTypeMessage ? ( <> - - + +

Date: Thu, 3 Oct 2024 11:14:41 +0100 Subject: [PATCH 04/42] [FTR] fix flaky: `cmn/mgmt/data_views/_edit_field.ts` (#194738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary What I tried before pushing 1. Better specificity - Couldn't get it just right (due to the dom structure) 1. Changing dom structure - Couldn't do that how I would like, due to the structure of `.tsx` file So, I went with adding a service method that simply filters down by two table dropdowns, instead of one. _Sadly I spend most of my time with the specificity (different css combinators), so lesson learned there_. Also, cleaned up the test code to simply call the service method and nothing more. ### Note ~~Will probably drop the other added service method (`openEditFlyoutByRowNumber`)~~ ✅ Resolves: https://github.com/elastic/kibana/issues/194662 --- test/functional/page_objects/settings_page.ts | 20 ++++++++++++++++++- .../management/data_views/_edit_field.ts | 18 ++--------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/test/functional/page_objects/settings_page.ts b/test/functional/page_objects/settings_page.ts index 01dbc848f3b15..94f3b9f3e3e40 100644 --- a/test/functional/page_objects/settings_page.ts +++ b/test/functional/page_objects/settings_page.ts @@ -9,7 +9,6 @@ import expect from '@kbn/expect'; import { FtrService } from '../ftr_provider_context'; - export class SettingsPageObject extends FtrService { private readonly log = this.ctx.getService('log'); private readonly retry = this.ctx.getService('retry'); @@ -387,6 +386,11 @@ export class SettingsPageObject extends FtrService { const input = await this.testSubjects.find('indexPatternFieldFilter'); await input.clearValueWithKeyboard(); await input.type(name); + const value = await this.testSubjects.getAttribute('indexPatternFieldFilter', 'value'); + expect(value).to.eql( + name, + `Expected new value to be the input: [${name}}], but got: [${value}]` + ); } async openControlsByName(name: string) { @@ -1048,4 +1052,18 @@ export class SettingsPageObject extends FtrService { [data-test-subj="indexPatternOption-${newIndexPatternTitle}"]` ); } + + async changeAndValidateFieldFormat(name: string, fieldType: string) { + await this.filterField(name); + await this.setFieldTypeFilter(fieldType); + await this.testSubjects.click('editFieldFormat'); + + expect(await this.testSubjects.getVisibleText('flyoutTitle')).to.eql(`Edit field '${name}'`); + + await this.retry.tryForTime(5000, async () => { + const previewText = await this.testSubjects.getVisibleText('fieldPreviewItem > value'); + expect(previewText).to.be('css'); + }); + await this.closeIndexPatternFieldEditor(); + } } diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_edit_field.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_edit_field.ts index b14687ad12003..cd19615dc8304 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_edit_field.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_edit_field.ts @@ -5,12 +5,10 @@ * 2.0. */ -import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const kibanaServer = getService('kibanaServer'); - const retry = getService('retry'); const PageObjects = getPageObjects(['settings', 'common']); const testSubjects = getService('testSubjects'); @@ -32,23 +30,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should show preview for fields in _source', async function () { - await PageObjects.settings.filterField('extension'); - await testSubjects.click('editFieldFormat'); - await retry.tryForTime(5000, async () => { - const previewText = await testSubjects.getVisibleText('fieldPreviewItem > value'); - expect(previewText).to.be('css'); - }); - await PageObjects.settings.closeIndexPatternFieldEditor(); + await PageObjects.settings.changeAndValidateFieldFormat('extension', 'text'); }); it('should show preview for fields not in _source', async function () { - await PageObjects.settings.filterField('extension.raw'); - await testSubjects.click('editFieldFormat'); - await retry.tryForTime(5000, async () => { - const previewText = await testSubjects.getVisibleText('fieldPreviewItem > value'); - expect(previewText).to.be('css'); - }); - await PageObjects.settings.closeIndexPatternFieldEditor(); + await PageObjects.settings.changeAndValidateFieldFormat('extension.raw', 'keyword'); }); }); }); From 22e36117c41caa757cdcf962e5c2c065f21bbc0b Mon Sep 17 00:00:00 2001 From: Jeramy Soucy Date: Thu, 3 Oct 2024 13:03:25 +0200 Subject: [PATCH 05/42] Removes visible features column from spaces grid when in serverless (#194438) Closes #194403 ## Summary Removes the 'Features visible' column from the Spaces management page grid when in serverless. ### Tests - x-pack/plugins/spaces/public/management/spaces_grid/spaces_grid_page.test.tsx ### Manual tesing - Start ES & Kibana in serverless mode, with a `xpack.spaces.maxSpaces` setting > 1 - Navigate to spaces management page and verify that the 'Features visible' column is not present - Start ES & Kibana in stateful mode - Navigate to spaces management page and verify that the 'Features visible' column is present --- .../management/management_service.test.ts | 3 + .../public/management/management_service.tsx | 3 + .../spaces_grid/spaces_grid_page.test.tsx | 79 +++++++++++++++++++ .../spaces_grid/spaces_grid_page.tsx | 4 +- .../management/spaces_management_app.test.tsx | 4 +- .../management/spaces_management_app.tsx | 3 + x-pack/plugins/spaces/public/plugin.tsx | 1 + 7 files changed, 95 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/spaces/public/management/management_service.test.ts b/x-pack/plugins/spaces/public/management/management_service.test.ts index 40a61397e286f..d1d7fe8d160a9 100644 --- a/x-pack/plugins/spaces/public/management/management_service.test.ts +++ b/x-pack/plugins/spaces/public/management/management_service.test.ts @@ -50,6 +50,7 @@ describe('ManagementService', () => { getRolesAPIClient: getRolesAPIClientMock, getPrivilegesAPIClient: jest.fn(), eventTracker, + isServerless: false, }); expect(mockKibanaSection.registerApp).toHaveBeenCalledTimes(1); @@ -73,6 +74,7 @@ describe('ManagementService', () => { getRolesAPIClient: getRolesAPIClientMock, getPrivilegesAPIClient: jest.fn(), eventTracker, + isServerless: false, }); }); }); @@ -97,6 +99,7 @@ describe('ManagementService', () => { getRolesAPIClient: jest.fn(), getPrivilegesAPIClient: jest.fn(), eventTracker, + isServerless: false, }); service.stop(); diff --git a/x-pack/plugins/spaces/public/management/management_service.tsx b/x-pack/plugins/spaces/public/management/management_service.tsx index 0379189e192c3..ba66229323bc8 100644 --- a/x-pack/plugins/spaces/public/management/management_service.tsx +++ b/x-pack/plugins/spaces/public/management/management_service.tsx @@ -28,6 +28,7 @@ interface SetupDeps { eventTracker: EventTracker; getPrivilegesAPIClient: () => Promise; logger: Logger; + isServerless: boolean; } export class ManagementService { @@ -42,6 +43,7 @@ export class ManagementService { getRolesAPIClient, eventTracker, getPrivilegesAPIClient, + isServerless, }: SetupDeps) { this.registeredSpacesManagementApp = management.sections.section.kibana.registerApp( spacesManagementApp.create({ @@ -52,6 +54,7 @@ export class ManagementService { getRolesAPIClient, eventTracker, getPrivilegesAPIClient, + isServerless, }) ); } diff --git a/x-pack/plugins/spaces/public/management/spaces_grid/spaces_grid_page.test.tsx b/x-pack/plugins/spaces/public/management/spaces_grid/spaces_grid_page.test.tsx index c8707f8959f0c..abb184bcb4382 100644 --- a/x-pack/plugins/spaces/public/management/spaces_grid/spaces_grid_page.test.tsx +++ b/x-pack/plugins/spaces/public/management/spaces_grid/spaces_grid_page.test.tsx @@ -60,6 +60,7 @@ featuresStart.getFeatures.mockResolvedValue([ const spacesGridCommonProps = { serverBasePath: '', maxSpaces: 1000, + isServerless: false, }; describe('SpacesGridPage', () => { @@ -326,6 +327,7 @@ describe('SpacesGridPage', () => { maxSpaces={1} allowSolutionVisibility serverBasePath={spacesGridCommonProps.serverBasePath} + isServerless={false} /> ); @@ -410,4 +412,81 @@ describe('SpacesGridPage', () => { title: 'Error loading spaces', }); }); + + it(`renders the 'Features visible' column when not serverless`, async () => { + const httpStart = httpServiceMock.createStartContract(); + httpStart.get.mockResolvedValue([]); + + const error = new Error('something awful happened'); + + const notifications = notificationServiceMock.createStartContract(); + + const wrapper = shallowWithIntl( + Promise.reject(error)} + notifications={notifications} + getUrlForApp={getUrlForApp} + history={history} + capabilities={{ + navLinks: {}, + management: {}, + catalogue: {}, + spaces: { manage: true }, + }} + allowSolutionVisibility + {...spacesGridCommonProps} + /> + ); + + // allow spacesManager to load spaces and lazy-load SpaceAvatar + await act(async () => {}); + wrapper.update(); + + expect(wrapper.find('EuiInMemoryTable').prop('columns')).toContainEqual( + expect.objectContaining({ + field: 'disabledFeatures', + name: 'Features visible', + }) + ); + }); + + it(`does not render the 'Features visible' column when serverless`, async () => { + const httpStart = httpServiceMock.createStartContract(); + httpStart.get.mockResolvedValue([]); + + const error = new Error('something awful happened'); + + const notifications = notificationServiceMock.createStartContract(); + + const wrapper = shallowWithIntl( + Promise.reject(error)} + notifications={notifications} + getUrlForApp={getUrlForApp} + history={history} + capabilities={{ + navLinks: {}, + management: {}, + catalogue: {}, + spaces: { manage: true }, + }} + allowSolutionVisibility + {...spacesGridCommonProps} + isServerless={true} + /> + ); + + // allow spacesManager to load spaces and lazy-load SpaceAvatar + await act(async () => {}); + wrapper.update(); + + expect(wrapper.find('EuiInMemoryTable').prop('columns')).not.toContainEqual( + expect.objectContaining({ + field: 'disabledFeatures', + name: 'Features visible', + }) + ); + }); }); diff --git a/x-pack/plugins/spaces/public/management/spaces_grid/spaces_grid_page.tsx b/x-pack/plugins/spaces/public/management/spaces_grid/spaces_grid_page.tsx index 462b65f327ebc..3049fb00d8977 100644 --- a/x-pack/plugins/spaces/public/management/spaces_grid/spaces_grid_page.tsx +++ b/x-pack/plugins/spaces/public/management/spaces_grid/spaces_grid_page.tsx @@ -63,6 +63,7 @@ interface Props { getUrlForApp: ApplicationStart['getUrlForApp']; maxSpaces: number; allowSolutionVisibility: boolean; + isServerless: boolean; } interface State { @@ -335,7 +336,8 @@ export class SpacesGridPage extends Component { }, ]; - const shouldShowFeaturesColumn = !activeSolution || activeSolution === SOLUTION_VIEW_CLASSIC; + const shouldShowFeaturesColumn = + !this.props.isServerless && (!activeSolution || activeSolution === SOLUTION_VIEW_CLASSIC); if (shouldShowFeaturesColumn) { config.push({ field: 'disabledFeatures', diff --git a/x-pack/plugins/spaces/public/management/spaces_management_app.test.tsx b/x-pack/plugins/spaces/public/management/spaces_management_app.test.tsx index a04335613e59b..ffafe432a5a3b 100644 --- a/x-pack/plugins/spaces/public/management/spaces_management_app.test.tsx +++ b/x-pack/plugins/spaces/public/management/spaces_management_app.test.tsx @@ -78,6 +78,7 @@ async function mountApp(basePath: string, pathname: string, spaceId?: string) { getRolesAPIClient: jest.fn(), getPrivilegesAPIClient: jest.fn(), eventTracker, + isServerless: false, }) .mount({ basePath, @@ -102,6 +103,7 @@ describe('spacesManagementApp', () => { getRolesAPIClient: jest.fn(), getPrivilegesAPIClient: jest.fn(), eventTracker, + isServerless: false, }) ).toMatchInlineSnapshot(` Object { @@ -126,7 +128,7 @@ describe('spacesManagementApp', () => { css="You have tried to stringify object returned from \`css\` function. It isn't supposed to be used directly (e.g. as value of the \`className\` prop), but rather handed to emotion so it can handle it (e.g. as value of \`css\` prop)." data-test-subj="kbnRedirectAppLink" > - Spaces Page: {"capabilities":{"catalogue":{},"management":{},"navLinks":{}},"notifications":{"toasts":{}},"spacesManager":{"onActiveSpaceChange$":{}},"serverBasePath":"","history":{"action":"PUSH","length":1,"location":{"pathname":"/","search":"","hash":""}},"maxSpaces":1000,"allowSolutionVisibility":true} + Spaces Page: {"capabilities":{"catalogue":{},"management":{},"navLinks":{}},"notifications":{"toasts":{}},"spacesManager":{"onActiveSpaceChange$":{}},"serverBasePath":"","history":{"action":"PUSH","length":1,"location":{"pathname":"/","search":"","hash":""}},"maxSpaces":1000,"allowSolutionVisibility":true,"isServerless":false} `); diff --git a/x-pack/plugins/spaces/public/management/spaces_management_app.tsx b/x-pack/plugins/spaces/public/management/spaces_management_app.tsx index fa74316779a7e..7ad85d0ef7c52 100644 --- a/x-pack/plugins/spaces/public/management/spaces_management_app.tsx +++ b/x-pack/plugins/spaces/public/management/spaces_management_app.tsx @@ -36,6 +36,7 @@ interface CreateParams { getRolesAPIClient: () => Promise; eventTracker: EventTracker; getPrivilegesAPIClient: () => Promise; + isServerless: boolean; } export const spacesManagementApp = Object.freeze({ @@ -48,6 +49,7 @@ export const spacesManagementApp = Object.freeze({ eventTracker, getRolesAPIClient, getPrivilegesAPIClient, + isServerless, }: CreateParams) { const title = i18n.translate('xpack.spaces.displayName', { defaultMessage: 'Spaces', @@ -92,6 +94,7 @@ export const spacesManagementApp = Object.freeze({ getUrlForApp={application.getUrlForApp} maxSpaces={config.maxSpaces} allowSolutionVisibility={config.allowSolutionVisibility} + isServerless={isServerless} /> ); }; diff --git a/x-pack/plugins/spaces/public/plugin.tsx b/x-pack/plugins/spaces/public/plugin.tsx index 86196333c0883..8450aaff32657 100644 --- a/x-pack/plugins/spaces/public/plugin.tsx +++ b/x-pack/plugins/spaces/public/plugin.tsx @@ -129,6 +129,7 @@ export class SpacesPlugin implements Plugin Date: Thu, 3 Oct 2024 13:37:47 +0200 Subject: [PATCH 06/42] Clean up `cloud_chat` (#194571) ## Summary Close https://github.com/elastic/kibana-team/issues/1017 This PR removes the unused Cloud Chat functionality from Kibana. The chat was not used for some time. Moreover, we've seen some issues with it where users saw it when it wasn't expected. Given the absence of automated tests and the fact that the feature is no longer needed, we are removing it to improve the overall maintainability and reliability of the codebase. This will also decrease the amount of code loaded for trial users of Kibana in cloud making the app slightly faster. --- .../steps/storybooks/build_and_upload.ts | 1 - docs/developer/plugin-list.asciidoc | 3 +- packages/kbn-optimizer/limits.yml | 1 - src/dev/storybook/aliases.ts | 1 - test/plugin_functional/config.ts | 2 - .../test_suites/core_plugins/rendering.ts | 2 - x-pack/.i18nrc.json | 1 - .../cloud_chat/.storybook/decorator.tsx | 38 -- .../cloud_chat/.storybook/index.ts | 8 - .../cloud_chat/.storybook/main.ts | 10 - .../cloud_chat/.storybook/manager.ts | 20 - .../cloud_chat/.storybook/preview.ts | 11 - .../cloud_integrations/cloud_chat/README.md | 5 +- .../cloud_chat/common/constants.ts | 9 - .../cloud_chat/common/types.ts | 15 - .../cloud_chat/common/util.ts | 19 - .../cloud_chat/jest.config.js | 18 - .../cloud_chat/kibana.jsonc | 2 +- .../public/components/chat/chat.tsx | 70 ---- .../components/chat/get_chat_context.test.ts | 65 --- .../components/chat/get_chat_context.ts | 36 -- .../public/components/chat/index.tsx | 28 -- .../public/components/chat/use_chat_config.ts | 213 ---------- .../public/components/chat/when_idle.tsx | 34 -- .../chat_experiment_switcher/index.tsx | 40 -- .../chat_floating_bubble/chat.stories.tsx | 123 ------ .../chat_floating_bubble.tsx | 101 ----- .../components/chat_floating_bubble/index.tsx | 30 -- .../chat_header_menu_items.tsx | 109 ------ .../chat_header_menu_item/chat_icon_dark.svg | 5 - .../chat_header_menu_item/chat_icon_light.svg | 5 - .../components/chat_header_menu_item/index.ts | 8 - .../cloud_chat/public/components/index.tsx | 9 - .../cloud_chat/public/index.ts | 13 - .../cloud_chat/public/plugin.test.ts | 103 ----- .../cloud_chat/public/plugin.tsx | 139 ------- .../cloud_chat/public/services/index.tsx | 45 --- .../cloud_chat/server/config.ts | 81 +--- .../cloud_chat/server/index.ts | 18 +- .../cloud_chat/server/plugin.ts | 45 --- .../cloud_chat/server/routes/chat.test.ts | 369 ------------------ .../cloud_chat/server/routes/chat.ts | 100 ----- .../cloud_chat/server/routes/index.ts | 8 - .../server/util/generate_jwt.test.ts | 23 -- .../cloud_chat/server/util/generate_jwt.ts | 24 -- .../cloud_chat/tsconfig.json | 9 +- .../translations/translations/fr-FR.json | 5 - .../translations/translations/ja-JP.json | 5 - .../translations/translations/zh-CN.json | 5 - x-pack/test/cloud_integration/config.ts | 20 +- x-pack/test/cloud_integration/tests/chat.ts | 31 -- 51 files changed, 41 insertions(+), 2044 deletions(-) delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/.storybook/decorator.tsx delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/.storybook/index.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/.storybook/main.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/.storybook/manager.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/.storybook/preview.ts delete mode 100755 x-pack/plugins/cloud_integrations/cloud_chat/common/constants.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/common/types.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/common/util.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/jest.config.js delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat/chat.tsx delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat/get_chat_context.test.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat/get_chat_context.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat/index.tsx delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat/use_chat_config.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat/when_idle.tsx delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat_experiment_switcher/index.tsx delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat_floating_bubble/chat.stories.tsx delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat_floating_bubble/chat_floating_bubble.tsx delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat_floating_bubble/index.tsx delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat_header_menu_item/chat_header_menu_items.tsx delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat_header_menu_item/chat_icon_dark.svg delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat_header_menu_item/chat_icon_light.svg delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat_header_menu_item/index.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/components/index.tsx delete mode 100755 x-pack/plugins/cloud_integrations/cloud_chat/public/index.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/plugin.test.ts delete mode 100755 x-pack/plugins/cloud_integrations/cloud_chat/public/plugin.tsx delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/public/services/index.tsx delete mode 100755 x-pack/plugins/cloud_integrations/cloud_chat/server/plugin.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/server/routes/chat.test.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/server/routes/chat.ts delete mode 100755 x-pack/plugins/cloud_integrations/cloud_chat/server/routes/index.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/server/util/generate_jwt.test.ts delete mode 100644 x-pack/plugins/cloud_integrations/cloud_chat/server/util/generate_jwt.ts delete mode 100644 x-pack/test/cloud_integration/tests/chat.ts diff --git a/.buildkite/scripts/steps/storybooks/build_and_upload.ts b/.buildkite/scripts/steps/storybooks/build_and_upload.ts index 13b346794f7b4..10128470005ce 100644 --- a/.buildkite/scripts/steps/storybooks/build_and_upload.ts +++ b/.buildkite/scripts/steps/storybooks/build_and_upload.ts @@ -18,7 +18,6 @@ const STORYBOOKS = [ 'canvas', 'cases', 'cell_actions', - 'cloud_chat', 'coloring', 'chart_icons', 'content_management_examples', diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 55a2a19040aec..a99e030a4adc1 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -487,7 +487,8 @@ The plugin exposes the static DefaultEditorController class to consume. |{kib-repo}blob/{branch}/x-pack/plugins/cloud_integrations/cloud_chat/README.md[cloudChat] -|Integrates with DriftChat in order to provide live support to our Elastic Cloud users. This plugin should only run on Elastic Cloud. +|The plugin was meant to integrate with DriftChat in order to provide live support to our Elastic Cloud users. +It was removed, but the plugin was left behind to register no longer used config keys. |{kib-repo}blob/{branch}/x-pack/plugins/cloud_integrations/cloud_data_migration/README.md[cloudDataMigration] diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index da5ba0500eeb9..8af1cb4460ecb 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -11,7 +11,6 @@ pageLoadAssetSize: cases: 180037 charts: 55000 cloud: 21076 - cloudChat: 19894 cloudDataMigration: 19170 cloudDefend: 18697 cloudExperiments: 109746 diff --git a/src/dev/storybook/aliases.ts b/src/dev/storybook/aliases.ts index 4d52c77fc8e20..07e92a23f9bc5 100644 --- a/src/dev/storybook/aliases.ts +++ b/src/dev/storybook/aliases.ts @@ -16,7 +16,6 @@ export const storybookAliases = { canvas: 'x-pack/plugins/canvas/storybook', cases: 'packages/kbn-cases-components/.storybook', cell_actions: 'packages/kbn-cell-actions/.storybook', - cloud_chat: 'x-pack/plugins/cloud_integrations/cloud_chat/.storybook', cloud: 'packages/cloud/.storybook', coloring: 'packages/kbn-coloring/.storybook', language_documentation_popover: 'packages/kbn-language-documentation/.storybook', diff --git a/test/plugin_functional/config.ts b/test/plugin_functional/config.ts index 88eef1ff3ee15..dbdf8f033848a 100644 --- a/test/plugin_functional/config.ts +++ b/test/plugin_functional/config.ts @@ -59,8 +59,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { // We want to test when the banner is shown '--telemetry.banner=true', // explicitly enable the cloud integration plugins to validate the rendered config keys - '--xpack.cloud_integrations.chat.enabled=true', - '--xpack.cloud_integrations.chat.chatURL=a_string', '--xpack.cloud_integrations.experiments.enabled=true', '--xpack.cloud_integrations.experiments.launch_darkly.sdk_key=a_string', '--xpack.cloud_integrations.experiments.launch_darkly.client_id=a_string', diff --git a/test/plugin_functional/test_suites/core_plugins/rendering.ts b/test/plugin_functional/test_suites/core_plugins/rendering.ts index f1922eee52380..b0e915cbbc735 100644 --- a/test/plugin_functional/test_suites/core_plugins/rendering.ts +++ b/test/plugin_functional/test_suites/core_plugins/rendering.ts @@ -233,8 +233,6 @@ export default function ({ getService }: PluginFunctionalProviderContext) { 'xpack.cloud.deployments_url (string?)', 'xpack.cloud.is_elastic_staff_owned (boolean?)', 'xpack.cloud.trial_end_date (string?)', - 'xpack.cloud_integrations.chat.chatURL (string?)', - 'xpack.cloud_integrations.chat.trialBuffer (number?)', // Commented because it's inside a schema conditional, and the test is not able to resolve it. But it's shared. // Added here for documentation purposes. // 'xpack.cloud_integrations.experiments.launch_darkly.client_id (string)', diff --git a/x-pack/.i18nrc.json b/x-pack/.i18nrc.json index 72e35fb16de2f..a46e291093411 100644 --- a/x-pack/.i18nrc.json +++ b/x-pack/.i18nrc.json @@ -18,7 +18,6 @@ "xpack.canvas": "plugins/canvas", "xpack.cases": "plugins/cases", "xpack.cloud": "plugins/cloud", - "xpack.cloudChat": "plugins/cloud_integrations/cloud_chat", "xpack.cloudDefend": "plugins/cloud_defend", "xpack.cloudLinks": "plugins/cloud_integrations/cloud_links", "xpack.cloudDataMigration": "plugins/cloud_integrations/cloud_data_migration", diff --git a/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/decorator.tsx b/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/decorator.tsx deleted file mode 100644 index 6331b82951bf0..0000000000000 --- a/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/decorator.tsx +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React, { FC, PropsWithChildren } from 'react'; -import { DecoratorFn } from '@storybook/react'; -import { ServicesProvider, CloudChatServices } from '../public/services'; - -// TODO: move to a storybook implementation of the service using parameters. -const services: CloudChatServices = { - chat: { - chatURL: 'https://elasticcloud-production-chat-us-east-1.s3.amazonaws.com/drift-iframe.html', - chatVariant: 'bubble', - user: { - id: 'user-id', - email: 'test-user@elastic.co', - // this doesn't affect chat appearance, - // but a user identity in Drift only - jwt: 'identity-jwt', - trialEndDate: new Date(), - kbnVersion: '8.9.0', - kbnBuildNum: 12345, - }, - }, -}; - -export const getCloudContextProvider: () => FC> = - () => - ({ children }) => - {children}; - -export const getCloudContextDecorator: DecoratorFn = (storyFn) => { - const CloudContextProvider = getCloudContextProvider(); - return {storyFn()}; -}; diff --git a/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/index.ts b/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/index.ts deleted file mode 100644 index 321df983cb20d..0000000000000 --- a/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { getCloudContextDecorator, getCloudContextProvider } from './decorator'; diff --git a/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/main.ts b/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/main.ts deleted file mode 100644 index bf63e08d64c32..0000000000000 --- a/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/main.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { defaultConfig } from '@kbn/storybook'; - -module.exports = defaultConfig; diff --git a/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/manager.ts b/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/manager.ts deleted file mode 100644 index 54c3d31c2002f..0000000000000 --- a/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/manager.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { addons } from '@storybook/addons'; -import { create } from '@storybook/theming'; -import { PANEL_ID } from '@storybook/addon-actions'; - -addons.setConfig({ - theme: create({ - base: 'light', - brandTitle: 'Cloud Storybook', - brandUrl: 'https://github.com/elastic/kibana/tree/main/x-pack/plugins/cloud', - }), - showPanel: true.valueOf, - selectedPanel: PANEL_ID, -}); diff --git a/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/preview.ts b/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/preview.ts deleted file mode 100644 index 83c512e516d5a..0000000000000 --- a/x-pack/plugins/cloud_integrations/cloud_chat/.storybook/preview.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { addDecorator } from '@storybook/react'; -import { getCloudContextDecorator } from './decorator'; - -addDecorator(getCloudContextDecorator); diff --git a/x-pack/plugins/cloud_integrations/cloud_chat/README.md b/x-pack/plugins/cloud_integrations/cloud_chat/README.md index cee3d9f5a6671..edc102df9b7d3 100755 --- a/x-pack/plugins/cloud_integrations/cloud_chat/README.md +++ b/x-pack/plugins/cloud_integrations/cloud_chat/README.md @@ -1,3 +1,4 @@ -# Cloud Chat +# Cloud Chat - Deprecated / Removed -Integrates with DriftChat in order to provide live support to our Elastic Cloud users. This plugin should only run on Elastic Cloud. +The plugin was meant to integrate with DriftChat in order to provide live support to our Elastic Cloud users. +It was removed, but the plugin was left behind to register no longer used config keys. diff --git a/x-pack/plugins/cloud_integrations/cloud_chat/common/constants.ts b/x-pack/plugins/cloud_integrations/cloud_chat/common/constants.ts deleted file mode 100755 index b196959a582d3..0000000000000 --- a/x-pack/plugins/cloud_integrations/cloud_chat/common/constants.ts +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const GET_CHAT_USER_DATA_ROUTE_PATH = '/internal/cloud/chat_user'; -export const DEFAULT_TRIAL_BUFFER = 90; diff --git a/x-pack/plugins/cloud_integrations/cloud_chat/common/types.ts b/x-pack/plugins/cloud_integrations/cloud_chat/common/types.ts deleted file mode 100644 index 8b7d316ceedfa..0000000000000 --- a/x-pack/plugins/cloud_integrations/cloud_chat/common/types.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export type ChatVariant = 'header' | 'bubble'; - -export interface GetChatUserDataResponseBody { - token: string; - email: string; - id: string; - chatVariant: ChatVariant; -} diff --git a/x-pack/plugins/cloud_integrations/cloud_chat/common/util.ts b/x-pack/plugins/cloud_integrations/cloud_chat/common/util.ts deleted file mode 100644 index 9d6cddac77f99..0000000000000 --- a/x-pack/plugins/cloud_integrations/cloud_chat/common/util.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -/** - * Returns true if today's date is within the an end date + buffer, false otherwise. - * - * @param endDate The end date of the trial. - * @param buffer The number of days to add to the end date. - * @returns true if today's date is within the an end date + buffer, false otherwise. - */ -export const isTodayInDateWindow = (endDate: Date, buffer: number) => { - const endDateWithBuffer = new Date(endDate); - endDateWithBuffer.setDate(endDateWithBuffer.getDate() + buffer); - return endDateWithBuffer > new Date(); -}; diff --git a/x-pack/plugins/cloud_integrations/cloud_chat/jest.config.js b/x-pack/plugins/cloud_integrations/cloud_chat/jest.config.js deleted file mode 100644 index 44f6f241d44d0..0000000000000 --- a/x-pack/plugins/cloud_integrations/cloud_chat/jest.config.js +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -module.exports = { - preset: '@kbn/test', - rootDir: '../../../../', - roots: ['/x-pack/plugins/cloud_integrations/cloud_chat'], - coverageDirectory: - '/target/kibana-coverage/jest/x-pack/plugins/cloud_integrations/cloud_chat', - coverageReporters: ['text', 'html'], - collectCoverageFrom: [ - '/x-pack/plugins/cloud_integrations/cloud_chat/{common,public,server}/**/*.{ts,tsx}', - ], -}; diff --git a/x-pack/plugins/cloud_integrations/cloud_chat/kibana.jsonc b/x-pack/plugins/cloud_integrations/cloud_chat/kibana.jsonc index 6394ccc7b53f1..dad2a22752df1 100644 --- a/x-pack/plugins/cloud_integrations/cloud_chat/kibana.jsonc +++ b/x-pack/plugins/cloud_integrations/cloud_chat/kibana.jsonc @@ -6,7 +6,7 @@ "plugin": { "id": "cloudChat", "server": true, - "browser": true, + "browser": false, "configPath": [ "xpack", "cloud_integrations", diff --git a/x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat/chat.tsx b/x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat/chat.tsx deleted file mode 100644 index debc600de80a7..0000000000000 --- a/x-pack/plugins/cloud_integrations/cloud_chat/public/components/chat/chat.tsx +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React from 'react'; -import { i18n } from '@kbn/i18n'; -import { WhenIdle } from './when_idle'; -import { useChatConfig, ChatApi } from './use_chat_config'; -export type { ChatApi } from './use_chat_config'; - -export interface Props { - /** Handler invoked when the chat widget signals it is ready. */ - onReady?: (chatApi: ChatApi) => void; - /** Handler invoked when the chat widget signals to be resized. */ - onResize?: () => void; - /** Handler invoked when the playbook is fired. */ - onPlaybookFired?: () => void; - /** The offset from the top of the page to the chat widget. */ - topOffset?: number; -} - -/** - * A component that will display a trigger that will allow the user to chat with a human operator, - * when the service is enabled; otherwise, it renders nothing. - */ -export const Chat = ({ onReady, onResize, onPlaybookFired, topOffset = 0 }: Props) => { - const config = useChatConfig({ - onReady, - onResize, - onPlaybookFired, - }); - - if (!config.enabled) { - return null; - } - - return ( - - + +Taking a closer look at the decompiled function (`syscfg_setAjySnParams`) which overwrites the values stored in `app_ajy_sn.ini`, we can see that input parameters, extracted from the `MSG_DRW` command are used to pass along string data which will be used to overwrite the model, vendor, and serial number fields in the file. memset is used to overwrite three global variables, intended to store these input strings, with null bytes. strcpy is then used to transfer the input parameters into these globals. In each instance, this will result in bytes being copied directly from the `MSG_DRW` command buffer until it encounters a null character. + +![](/assets/images/storm-on-the-horizon/image21.png "image_tooltip") + +Because no validation is enforced on the length of these input parameters extracted from the command, it is trivial to craft a message of sufficient length which will trigger a buffer overflow. While we did not leverage this vulnerability as part of our attack to brick the camera, this appears to be an instance where an exploit could be developed which would allow for an attacker to achieve remote code execution on the camera. + +## Impact + +We have confirmed that a broad range of devices across several vendors affiliated with AJCloud and several different firmware versions are affected by these vulnerabilities and flaws. Overall, we successfully demonstrated our attacks against fifteen different camera products from Wansview, Galayou, Cinnado, and Faleemi. Based on our findings, it is safe to assume that all devices which operate AJCloud firmware and connect to the AJCloud platform are affected. + +All attempts to contact both AJCloud and Wansview in order to disclose these vulnerabilities and flaws were unsuccessful. + +## What did the vendors do right? + +Despite the vulnerabilities we discovered and discussed previously, there are a number of the security controls that AJCloud and the camera vendors implemented well. For such a low cost device, many best practices were implemented. First, the network communications are secured well using certificate based WebSocket authentication. In addition to adding encryption, putting many of the API endpoints behind the certificate auth makes man in the middle attacks significantly more challenging. Furthermore, the APKs for the mobile apps were signed and obfuscated making manipulating these apps very time consuming. + +Additionally, the vendors also made some sound decisions with the camera hardware and firmware. The local OS for the camera is effectively limited, focusing on just the needed functionality for their product. The file system is configured to be read only, outside of logging, and the kernel watchdog is an effective method of ensuring uptime and reducing risk of being stuck in a failed state. The Ingenic Xburst T31 SoC, provides a capable platform with a wide range of support including secure boot, a Power-On Reset (POR) watchdog, and a separate RISC-V processor capable of running some rudimentary machine learning on the camera input. + +## What did the vendors do wrong? + +Unfortunately, there were a number of missed opportunities with these available features. Potentially the most egregious is the unauthenticated cloud access. Given the API access controls established for many of the endpoints, having the camera user access endpoints available via serial number without authentication is a huge and avoidable misstep. The P2P protocol is also vulnerable as we showcased, but compared to the API access which should be immediately fixable, this may take some more time to fix the protocol. It is a very dangerous vulnerability, but it is a little bit more understandable as it requires considerably more time investment to both discover and fix. + +From the application side, the primary issue is with the Windows app which has extensive debug logging which should have been removed before releasing publicly. As for the hardware, it can be easily manipulated with physical access (exposed reset button, etc.). This is not so much an issue given the target consumer audience. It is expected to err on the side of usability rather than security, especially given physical access to the device. On a similar note, secure boot should be enabled, especially given that the T31 SoC supports it. While not strictly necessary, this would make it much harder to debug the source code and firmware of the device directly, making it more difficult to discover vulnerabilities that may be present. Ideally it would be implemented in such a way that the bootloader could still load an unsigned OS to allow for easier tinkering and development, but would prevent the signed OS from loading until the boot loader configuration is restored. However, one significant flaw in the current firmware is the dependence on the original serial number that is not stored in a read only mount point while the system is running. Manipulating the serial number should not permanently brick the device. It should either have a mechanism for requesting a new serial number (or restoring its original serial number) should its serial number be overwritten, or the serial number should be immutable. + +## Mitigations + +Certain steps can be taken in order to reduce the attack surface and limit potential adverse effects in the event of an attack, though they vary in their effectiveness. + +Segmenting Wi-Fi cameras and other IoT devices off from the rest of your network is a highly recommended countermeasure which will prevent attackers from pivoting laterally to more critical systems. However, this approach does not prevent an attacker from obtaining sensitive user data through exploiting the access control vulnerability we discovered in the AJCloud platform. Also, considering the ease in which we were able to demonstrate how cameras could be accessed and manipulated remotely via P2P, any device connected to the AJCloud platform is still at significant risk of compromise regardless of its local network configuration. + +Restricting all network communications to and from these cameras would not be feasible due to how essential connectivity to the AJCloud platform is to their operation. As previously mentioned, the devices will simply not operate if they are unable to connect to various API endpoints upon startup. + +A viable approach could be restricting communications beyond the initial startup routine. However, this would prevent remote access and control via mobile and desktop apps, which would defeat the entire purpose of these cameras in the first place. For further research in this area, please refer to “[Blocking Without Breaking: Identification and Mitigation of Non-Essential IoT Traffic](https://petsymposium.org/popets/2021/popets-2021-0075.pdf)”, which explored this approach more in-depth across a myriad of IoT devices and vendors. + +The best approach to securing any Wi-Fi camera, regardless of vendor, while maintaining core functionality would be to flash it with alternative open source firmware such as [OpenIPC](https://openipc.org) or [thingino](https://thingino.com). Switching to open source firmware avoids the headaches associated with forced connectivity to vendor cloud platforms by providing users with fine grain control of device configuration and remote network accessibility. Open access to the firmware source helps to ensure that critical flaws and vulnerabilities are quickly identified and patched by diligent project contributors. + +## Key Takeaways + +Our research revealed several critical vulnerabilities that span all aspects of cameras operating AJCloud firmware which are connected to their platform. Significant flaws in access control management on their platform and the PPPP peer protocol provides an expansive attack surface which affects millions of active devices across the world. Exploiting these flaws and vulnerabilities leads to the exposure of sensitive user data and provides attackers with full remote control of any camera connected to the AJCloud platform. Furthermore, a built-in P2P command, which intentionally provides arbitrary write access to a key configuration file, can be leveraged to either permanently disable cameras or facilitate remote code execution through triggering a buffer overflow. + +Please visit our [GitHub repository](https://github.com/elastic/camera-hacks) for custom tools and scripts we have built along with data and notes we have captured which we felt would provide the most benefit to the security research community. diff --git a/x-pack/plugins/elastic_assistant/server/knowledge_base/security_labs/streamlining_esql_query_and_rule_validation.md b/x-pack/plugins/elastic_assistant/server/knowledge_base/security_labs/streamlining_esql_query_and_rule_validation.md new file mode 100644 index 0000000000000..d5950d772d400 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/knowledge_base/security_labs/streamlining_esql_query_and_rule_validation.md @@ -0,0 +1,219 @@ +--- +title: "Streamlining ES|QL Query and Rule Validation: Integrating with GitHub CI" +slug: "streamlining-esql-query-and-rule-validation" +date: "2023-11-17" +description: "ES|QL is Elastic's new piped query language. Taking full advantage of this new feature, Elastic Security Labs walks through how to run validation of ES|QL rules for the Detection Engine." +author: + - slug: mika-ayenson + - slug: eric-forte +image: "photo-edited-01.png" +category: + - slug: security-research +--- + +One of the amazing, recently premiered [8.11.0 features](https://www.elastic.co/guide/en/elasticsearch/reference/current/release-highlights.html), is the Elasticsearch Query Language ([ES|QL](https://www.elastic.co/guide/en/elasticsearch/reference/current/esql.html)). As highlighted in an earlier [post by Costin Leau](https://www.elastic.co/blog/elasticsearch-query-language-esql), it’s a full-blown, specialized query and compute engine for Elasitcsearch. Now that it’s in technical preview, we wanted to share some options to _validate_ your ES|QL queries. This overview is for engineers new to ES|QL. Whether you’re searching for insights in Kibana or investigating security threats in [Timelines](https://www.elastic.co/guide/en/security/current/timelines-ui.html), you’ll see how this capability is seamlessly interwoven throughout Elastic. + +## ES|QL validation basics ft. Kibana & Elasticsearch + +If you want to quickly validate a single query, or feel comfortable manually testing queries one-by-one, the Elastic Stack UI is all you need. After navigating to the Discover tab in Kibana, click on the "**Try ES|QL**" Technical Preview button in the Data View dropdown to load the query pane. You can also grab sample queries from the [ES|QL Examples](https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-examples.html) to get up and running. Introducing non-[ECS](https://www.elastic.co/guide/en/ecs/current/index.html) fields will immediately highlight errors prioritizing syntax errors, then unknown column errors. + +![](/assets/images/streamlining-esql-query-and-rule-validation/image7.png) + +In this example, there are two syntax errors that are highlighted: +* the invalid syntax error on the input `wheres` which should be `where` and +* the unknown column `process.worsking_directory`, which should be `process.working_directory`. + +![](/assets/images/streamlining-esql-query-and-rule-validation/image3.png) + +After resolving the syntax error in this example, you’ll observe the Unknown column errors. Here are a couple reasons this error may appear: + + - **Fix Field Name Typos**: Sometimes you simply need to fix the name as suggested in the error; consult the ECS or any integration schemas and confirm the fields are correct + - **Add Missing Data**: If you’re confident the fields are correct, sometimes adding data to your stack, which will populate the columns + - **Update Mapping**: You can configure [Mappings](https://www.elastic.co/guide/en/elasticsearch/reference/8.11/mapping.html) to set explicit fields, or add new fields to an existing data stream or index using the [Update Mapping API](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-put-mapping.html) + +## ES|QL warnings + +Not all fields will appear as errors, in which case you’re presented with warnings and a dropdown list. Hard failures (e.g. errors), imply that the rule cannot execute, whereas warnings indicate that the rule can run, but the functions may be degraded. + +![](/assets/images/streamlining-esql-query-and-rule-validation/image6.png) + +When utilizing broad ES|QL queries that span multiple indices, such as `logs-* | limit 10`, there might be instances where certain fields fail to appear in the results. This is often due to the fields being undefined in the indexed data, or not yet supported by ES|QL. In cases where the expected fields are not retrieved, it's typically a sign that the data was ingested into Elasticsearch without these fields being indexed, as per the established mappings. Instead of causing the query to fail, ES|QL handles this by returning "null" for the unavailable fields, serving as a warning that something in the query did not execute as expected. This approach ensures the query still runs, distinguishing it from a hard failure, which occurs when the query cannot execute at all, such as when a non-existent field is referenced. + +![](/assets/images/streamlining-esql-query-and-rule-validation/image12.png) + +There are also helpful performance warnings that may appear. Providing a `LIMIT` parameter to the query will help address performance warnings. Note this example highlights that there is a default limit of 500 events returned. This limit may significantly increase once this feature is generally available. + +## Security + +In an investigative workflow, security practitioners prefer to iteratively hunt for threats, which may encompass manually testing, refining, and tuning a query in the UI. Conveniently, security analysts and engineers can natively leverage ES|QL in timelines, with no need to interrupt workflows by pivoting back and forth to a different view in Kibana. You’ll receive the same errors and warnings in the same security component, which shows Elasticsearch feedback under the hood. + +![](/assets/images/streamlining-esql-query-and-rule-validation/image1.png) + +In some components, you will receive additional feedback based on the context of where ES|QL is implemented. One scenario is when you create an ES|QL rule using the create new rule feature under the Detection Rules (SIEM) tab. + +![](/assets/images/streamlining-esql-query-and-rule-validation/image8.png) + +For example, this query could easily be converted to an [EQL](https://www.elastic.co/guide/en/elasticsearch/reference/current/eql.html) or [KQL](https://www.elastic.co/guide/en/kibana/current/kuery-query.html) query as it does not leverage powerful features of ES|QL like statistics, frequency analysis, or parsing unstructured data. If you want to learn more about the benefits of queries using ES|QL check out this [blog by Costin](https://www.elastic.co/blog/elasticsearch-query-language-esql), which covers performance boosts. In this case, we must add `[metadata _id, _version, _index]` to the query, which informs the UI which components to return in the results. + +## API calls? Of course! + +Prior to this section, all of the examples referenced creating ES|QL queries and receiving feedback directly from the UI. For illustrative purposes, the following examples leverage Dev Tools, but these calls are easily migratable to cURL bash commands or the language / tool of your choice that can send an HTTP request. + +![](/assets/images/streamlining-esql-query-and-rule-validation/image4.png) + +Here is the same query as previously shown throughout other examples, sent via a POST request to the [query API](https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-query-api.html) with a valid query. + +![](/assets/images/streamlining-esql-query-and-rule-validation/image10.png) + +As expected, if you supply an invalid query, you’ll receive similar feedback observed in the UI. In this example, we’ve also supplied the `?error_trace` flag which can provide the stack trace if you need additional context for why the query failed validation. + +As you can imagine, we can use the API to programmatically validate ES|QL queries. You can also still use the [Create rule](https://www.elastic.co/guide/en/kibana/current/create-rule-api.html) Kibana API, which requires a bit more metadata associated with a security rule. However, if you want to only validate a query, the `_query` API comes in handy. From here you can use the [Elasticsearch Python Client](https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/index.html) to connect to your stack and validate queries. + +``` +from elasticsearch import Elasticsearch +client = Elasticsearch(...) +data = { +"query": """ + from logs-endpoint.events.* + | keep host.os.type, process.name, process.working_directory, event.type, event.action + | where host.os.type == "linux" and process.name == "unshadow" and event.type == "start" and event.action in ("exec", "exec_event") +""" +} + +# Execute the query +headers = {"Content-Type": "application/json", "Accept": "application/json"} +response = client.perform_request( +"POST", "/_query", params={"pretty": True}, headers=headers, body=data +) +``` + +## Leverage the grammar + +One of the best parts of Elastic developing in the open is the [antlr ES|QL grammar](https://github.com/elastic/elasticsearch/tree/main/x-pack/plugin/esql/src/main/antlr) is also available. + +![](/assets/images/streamlining-esql-query-and-rule-validation/image5.png) + +If you’re comfortable with [ANTLR](https://www.antlr.org), you can also download the latest JAR to build a lexer and parser. + +``` +pip install antlr4-tools # for antlr4 +git clone git@github.com:elastic/elasticsearch.git # large repo +cd elasticsearch/x-pack/plugin/esql/src/main/antlr # navigate to grammar +antlr4 -Dlanguage=Python3 -o build EsqlBaseLexer.g4 # generate lexer +antlr4 -Dlanguage=Python3 -o build EsqlBaseParser.g4 # generate parser +``` + +This process will require more lifting to get ES|QL validation started, but you’ll at least have a tree object built, that provides more granular control and access to the parsed fields. + +![](/assets/images/streamlining-esql-query-and-rule-validation/image13.png) + +However, as you can see the listeners are stubs, which means you’ll need to build in semantics _manually_ if you want to go this route. + +## The security rule GitHub CI use case + +For our internal Elastic EQL and KQL query rule validation, we utilize the parsed abstract syntax tree (AST) objects of our queries to perform nuanced semantic validation across multiple stack versions. For example, having the AST allows us to validate proper field usage, verify new features are not used in older stack versions before being introduced, or even more, ensure related integrations are built based on datastreams used in the query. Fundamentally, local validation allows us to streamline a broader range of support for many stack features and versions. If you’re interested in seeing more of the design and rigorous validation that we can do with the AST, check out our [detection-rules repo](https://github.com/elastic/detection-rules/tree/main). + +If you do not need granular access to the specific parsed tree objects and do not need to control the semantics of ES|QL validation, then out-of-the-box APIs may be all you need to validate queries. In this use case, we want to validate security detection rules using continuous integration. Managing detection rules through systems like GitHub helps garner all the benefits of using a version-controlled like tracking rule changes, receiving feedback via pull requests, and more. Conceptually, rule authors should be able to create these rules (which contain ES|QL queries) locally and exercise the git rule development lifecycle. + +CI checks help to ensure queries still pass ES|QL validation without having to manually check the query in the UI. Based on the examples shown thus far, you have to either stand up a persistent stack and validate queries against the API, or build a parser implementation based on the available grammar outside of the Elastic stack. + +One approach to using a short-lived Elastic stack versus leveraging a managed persistent stack is to use the [Elastic Container Project (ECP)](https://github.com/peasead/elastic-container). As advertised, this project will: + +_Stand up a 100% containerized Elastic stack, TLS secured, with Elasticsearch, Kibana, Fleet, and the Detection Engine all pre-configured, enabled, and ready to use, within minutes._ + +![](/assets/images/streamlining-esql-query-and-rule-validation/image11.png) + +With a combination of: + + - Elastic Containers (e.g. ECP) + - CI (e.g. Github Action Workflow) + - ES|QL rules + - Automation Foo (e.g. python & bash scripts) + +You can validate ES|QL rules via CI against the _latest stack version_ relatively easily, but there are some nuances involved in this approach. + +![](/assets/images/streamlining-esql-query-and-rule-validation/image2.gif) + +Feel free to check out the sample [GitHub action workflow](https://gist.github.com/Mikaayenson/7fa8f908ab7e8466178679a9a0cd9ecc) if you’re interested in a high-level overview of how it can be implemented. + +**Note:** if you're interested in using the GitHub action workflow, check out their documentation on using GitHub [secrets in Actions](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions) and [setting up Action workflows](https://docs.github.com/en/actions/quickstart). + +## CI nuances + + 1. Any custom configuration needs to be scripted away (e.g. setting up additional policies, [enrichments](https://www.elastic.co/guide/en/elasticsearch/reference/current/match-enrich-policy-type.html), etc.) In our POC, we created a step and bash script that executed a series of POST requests to our temporary CI Elastic Stack, which created the new enrichments used in our detection rules. + +``` +- name: Add Enrich Policy + env: + ELASTICSEARCH_SERVER: "https://localhost:9200" + ELASTICSEARCH_USERNAME: "elastic" + ELASTICSEARCH_PASSWORD: "${{ secrets.PASSWORD }}" + run: | + set -x + chmod +x ./add_enrich.sh + bash ./add_enrich.sh +``` + + 2. Without data in our freshly deployed CI Elastic stack, there will be many `Unknown Column` issues as previously mentioned. One approach to address this is to build indices with the proper mappings for the queries to match. For example, if you have a query that searches the index `logs-endpoint.events.*`, then create an index called `logs-endpoint.events.ci`, with the proper mappings from the integration used in the query. + + 3. Once the temporary stack is configured, you’ll need extra logic to iterate over all the rules and validate using the `_query` API. For example, you can create a unit test that iterates over all the rules. We do this today by leveraging our default `RuleCollection.default()` that loads all rules, in our detection-rules repo, but here is a snippet that quickly loads only ES|QL rules. + + +``` +# tests/test_all_rules.py +class TestESQLRules: + """Test ESQL Rules.""" + + @unittest.skipIf(not os.environ.get("DR_VALIDATE_ESQL"), + "Test only run when DR_VALIDATE_ESQL environment variable set.") + def test_environment_variables_set(self): + collection = RuleCollection() + + # Iterate over all .toml files in the given directory recursively + for rule in Path(DEFAULT_RULES_DIR).rglob('*.toml'): + # Read file content + content = rule.read_text(encoding='utf-8') + # Search for the pattern + if re.search(r'language = "esql"', content): + print(f"Validating {str(rule)}") + collection.load_file(rule) +``` + + Each rule would run through a validator method once the file is loaded with `load_file`. + +``` +# detection_rules/rule_validator.py +class ESQLValidator(QueryValidator): + """Specific fields for ESQL query event types.""" + + def validate(self, data: 'QueryRuleData', meta: RuleMeta) -> None: + """Validate an ESQL query while checking TOMLRule.""" + if not os.environ.get("DR_VALIDATE_ESQL"): + return + + if Version.parse(meta.min_stack_version) < Version.parse("8.11.0"): + raise ValidationError(f"Rule minstack must be greater than 8.10.0 {data.rule_id}") + + client = Elasticsearch(...) + client.info() + client.perform_request("POST", "/_query", params={"pretty": True}, + headers={"accept": "application/json", + "content-type": "application/json"}, + body={"query": f"{self.query} | LIMIT 0"}) +``` + + As highlighted earlier, we can `POST` to the query API and validate given the credentials that were set as GitHub action secrets and passed to the validation as environment variables. Note, the `LIMIT 0` is so the query does not return data intentionally. It’s meant to only perform validation. Finally the single CI step would be a bash call to run the unit tests (e.g. `pytest tests/test_all_rules.py::TestESQLRules`). + + 4. Finally, CI leveraging containers may not scale well when validating many rules against multiple Elastic stack versions and configurations. Especially if you would like to test on a commit-basis. The time to deploy one stack took slightly over five minutes to complete. This measurement could greatly increase or decrease depending on your CI setup. + +## Conclusion + +Elasticsearch's new feature, Elasticsearch Query Language (ES|QL), is a specialized query and compute engine for Elasticsearch, now in technical preview. It offers seamless integration across various Elastic services like Kibana and Timelines, with validation options for ES|QL queries. Users can validate queries through the Elastic Stack UI or API calls, receiving immediate feedback on syntax or column errors. + +Additionally, ES|QL's ANTLR grammar is [available](https://github.com/elastic/elasticsearch/tree/d5f5d0908ff7d1bfb3978e4c57aa6ff517f6ed29/x-pack/plugin/esql/src/main/antlr) for those who prefer a more hands-on approach to building lexers and parsers. We’re exploring ways to validate ES|QL queries in an automated fashion and now it’s your turn. Just know that we’re not done exploring, so check out ES|QL and let us know if you have ideas! We’d love to hear how you plan to use it within the stack natively or in CI. + +We’re always interested in hearing use cases and workflows like these, so as always, reach out to us via [GitHub issues](https://github.com/elastic/detection-rules/issues), chat with us in our [community Slack](http://ela.st/slack), and ask questions in our [Discuss forums](https://discuss.elastic.co/c/security/endpoint-security/80). + +Check out these additional resources to learn more about how we’re bringing the latest AI capabilities to the hands of the analyst: +Learn everything [ES|QL](https://www.elastic.co/guide/en/elasticsearch/reference/current/esql.html) +Checkout the 8.11.0 release blog [introducing ES|QL](https://www.elastic.co/blog/whats-new-elasticsearch-platform-8-11-0) diff --git a/x-pack/plugins/elastic_assistant/server/knowledge_base/security_labs/testing_okta_visibility_and_detection_dorothy.md b/x-pack/plugins/elastic_assistant/server/knowledge_base/security_labs/testing_okta_visibility_and_detection_dorothy.md new file mode 100644 index 0000000000000..3943b671a591c --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/knowledge_base/security_labs/testing_okta_visibility_and_detection_dorothy.md @@ -0,0 +1,123 @@ +--- +title: "Testing your Okta visibility and detection with Dorothy and Elastic Security" +slug: "testing-okta-visibility-and-detection-dorothy" +date: "2022-06-02" +description: "Dorothy is a tool for security teams to test their visibility and detection capabilities for their Okta environment. IAM solutions are frequently targeted by adversaries but poorly monitored. Learn how to get started with Dorothy in this post." +author: + - slug: david-french +image: "blog-thumb-dorothy-cow.jpg" +category: + - slug: security-research +--- + +When approached by stakeholders in their organization, few security teams can confidently demonstrate that logging and alerting capabilities are working as expected. Organizations have become more distributed and reliant on cloud offerings for use cases such as identity and access management, user productivity, and file storage. Meanwhile, adversaries have extended their operational capabilities in cloud environments. It is crucial that security teams are able to monitor these systems for abuse in order to protect their organization’s data from attack. + +[Dorothy](https://github.com/elastic/dorothy) is a free and open tool to help security teams test their visibility, monitoring, and detection capabilities for Okta Single Sign-On (SSO) environments. We’ll demonstrate how Dorothy can be used to execute tests and how [Elastic Security](https://www.elastic.co/security) can be used to alert on relevant and suspicious behavior using our [free and open detection rules](https://github.com/elastic/detection-rules/). + +## What is Okta SSO? + +For those who aren’t familiar, [Okta SSO](https://www.okta.com/products/single-sign-on/) is a cloud-based identity management solution that allows users to authenticate to a variety of systems and applications within their organization using a single user account. Informing end users that they only have to remember _one_ username and password instead of ten or more reduces the risk that they’ll develop poor password hygiene and enables system administrators to enforce stronger password policies. Further, multi-factor authentication (MFA) policies can be configured in Okta, which raises the barrier to entry for attackers. Many attackers will simply move on and look for an easier target when they discover that MFA is enforced in their target’s network or user account. + +While SSO solutions can provide a convenient user experience and reduce cybersecurity risk for an organization, these centralized systems offer a type of skeleton key to many systems and applications, and are often an attractive target for attackers. It’s critical that security teams understand what normal behavior looks like in their Okta environment so that they can identify suspicious activity more easily. + +## Meet Dorothy + +[Dorothy](https://github.com/elastic/dorothy) has 25+ modules to simulate actions an attacker may take while operating in an Okta environment and behavior that security teams should monitor for, detect, and alert on. All modules are mapped to the relevant [MITRE ATT&CK®](https://attack.mitre.org/) tactics, such as Persistence, Defense Evasion, Discovery, and Impact. + +![Figure 1 - Starting Dorothy and listing its modules](/assets/images/testing-okta-visibility-and-detection-dorothy/1-Dorothy-blog-listing-modules.png) + +Dorothy was created to help defenders test their security visibility and controls, and does not provide any modules to obtain initial access or escalate privileges in an Okta environment. To execute actions using Dorothy, a valid Okta API token is required that is linked to a user with one or more administrator roles assigned. + +A user-friendly shell interface with contextual help is provided for navigation between menus and modules, helping guide the user through simulated intruder scenarios. Other features include configuration profiles to manage connections to individual Okta environments and detailed logging with the option of indexing events into Elasticsearch to provide an audit trail of the actions that were executed using Dorothy. + +## Executing actions in an Okta environment using Dorothy + +In this section, we demonstrate how to execute some of Dorothy’s modules in an Okta environment. Figure 2 below shows the typical workflow for an Elastic Security user. After this demonstration, you should be comfortable with heading over to Dorothy’s GitHub repository and following the “Getting Started” steps in the project’s [wiki](https://github.com/elastic/dorothy/wiki). + +![Figure 2 - Example workflow for executing actions in an Okta environment using Dorothy](/assets/images/testing-okta-visibility-and-detection-dorothy/2-Dorothy-blog-example_workflow.png) + +### whoami? + +Let’s put ourselves in an attacker's shoes and think about what actions they might take while operating in an Okta environment. As an attacker with an initial foothold, the first questions I'll have are about the user for which I have an API token. Let's simulate this attacker action through Dorothy's whoami command to look at the associated user’s login ID, last login time, and last password change. + +Now that we have a better understanding of the user account we have control of, we’ll list Dorothy’s modules and check out the help menu before making our next move. + +