From 734012a0708cc766cb81b1deeefdb6b4858bb4a1 Mon Sep 17 00:00:00 2001 From: Garrett Spong Date: Fri, 12 May 2023 14:02:00 -0600 Subject: [PATCH] Adds new markdown parser for send to timeline actions for eql, kql and queryDSL --- .../security_assistant/security_assistant.tsx | 154 +++++++++--------- .../send_to_timeline_button.tsx | 105 +++++++----- .../use_conversation/helpers.tsx | 49 ++++++ .../use_conversation/sample_conversations.tsx | 104 ++++++++++++ .../flyout/add_timeline_button/index.tsx | 18 ++ .../timeline/data_providers/data_provider.ts | 4 + 6 files changed, 317 insertions(+), 117 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/security_assistant/use_conversation/helpers.tsx create mode 100644 x-pack/plugins/security_solution/public/security_assistant/use_conversation/sample_conversations.tsx diff --git a/x-pack/plugins/security_solution/public/security_assistant/security_assistant.tsx b/x-pack/plugins/security_solution/public/security_assistant/security_assistant.tsx index 6df284527f475..e703739782e2c 100644 --- a/x-pack/plugins/security_solution/public/security_assistant/security_assistant.tsx +++ b/x-pack/plugins/security_solution/public/security_assistant/security_assistant.tsx @@ -6,7 +6,6 @@ */ import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { createPortal } from 'react-dom'; import type { EuiCommentProps } from '@elastic/eui'; import { EuiFlexGroup, @@ -21,17 +20,18 @@ import { EuiAvatar, EuiPageHeader, EuiMarkdownFormat, - EuiIcon, EuiToolTip, + EuiIcon, } from '@elastic/eui'; -import type { DataProvider } from '@kbn/timelines-plugin/common'; import { CommentType } from '@kbn/cases-plugin/common'; import styled from 'styled-components'; +import { createPortal } from 'react-dom'; import * as i18n from './translations'; import { useKibana } from '../common/lib/kibana'; import { getCombinedMessage, getMessageFromRawResponse, isFileHash } from './helpers'; import { SendToTimelineButton } from './send_to_timeline_button'; + import { SettingsPopover } from './settings_popover'; import { useSecurityAssistantContext } from './security_assistant_context'; import { ContextPills } from './context_pills'; @@ -46,7 +46,7 @@ import { getDefaultSystemPrompt, getSuperheroPrompt } from './prompt/helpers'; import type { Prompt } from './types'; import { getPromptById } from './prompt_editor/helpers'; -const isMac = navigator.platform.toLowerCase().indexOf('mac') >= 0; +import { analyzeMarkdown } from './use_conversation/helpers'; const CommentsContainer = styled.div` max-height: 600px; @@ -100,6 +100,7 @@ export const SecurityAssistant: React.FC = const { cases } = useKibana().services; const bottomRef = useRef(null); + const lastCommentRef = useRef(null); const [promptTextPreview, setPromptTextPreview] = useState(''); const [systemPrompts] = useState([getDefaultSystemPrompt(), getSuperheroPrompt()]); @@ -220,49 +221,48 @@ export const SecurityAssistant: React.FC = ); // Drill in `Add To Timeline` action - // Grab all relevant dom elements - const commentBlocks = [...document.getElementsByClassName('euiMarkdownFormat')]; - // Filter if no code block exists as to not make extra portals - commentBlocks.filter((cb) => cb.querySelectorAll('.euiCodeBlock__code').length > 0); - - let commentDetails: Array<{ - commentBlock: Element; - codeBlocks: Element[]; - codeBlockControls: Element[]; - dataProviders: DataProvider[]; - }> = - currentConversation.messages.length > 0 - ? commentBlocks.map((commentBlock) => { - return { - commentBlock, - codeBlocks: [...commentBlock.querySelectorAll('.euiCodeBlock__code')], - codeBlockControls: [...commentBlock.querySelectorAll('.euiCodeBlock__controls')], - dataProviders: [], - }; - }) - : []; - commentDetails = commentDetails.map((details) => { - const dataProviders: DataProvider[] = details.codeBlocks.map((codeBlock, i) => { - return { - id: 'assistant-data-provider', - name: 'Assistant Query', - enabled: true, - // overriding to use as isEQL - excluded: details.commentBlock?.textContent?.includes('EQL') ?? false, - kqlQuery: codeBlock.textContent ?? '', - queryMatch: { - field: 'host.name', - operator: ':', - value: 'test', - }, - and: [], - }; + // First let's find + const messageCodeBlocks = useMemo(() => { + const cbd = currentConversation.messages.map(({ content }) => { + return analyzeMarkdown(content); }); - return { - ...details, - dataProviders, - }; - }); + return cbd.map((codeBlocks, messageIndex) => { + return codeBlocks.map((codeBlock, codeBlockIndex) => { + return { + ...codeBlock, + controlContainer: document.querySelectorAll( + `.message-${messageIndex} .euiCodeBlock__controls` + )[codeBlockIndex], + button: ( + + + + + + ), + }; + }); + }); + }, [currentConversation.messages, lastCommentRef.current]); //// // Add min-height to all codeblocks so timeline icon doesn't overflow @@ -324,31 +324,15 @@ export const SecurityAssistant: React.FC = )} {/* Create portals for each EuiCodeBlock to add the `Investigate in Timeline` action */} - {currentConversation.messages.length > 0 && - commentDetails.length > 0 && - // eslint-disable-next-line array-callback-return - commentDetails.map((e) => { - if (e.dataProviders != null && e.dataProviders.length > 0) { - return e.codeBlocks.map((block, i) => { - if (e.codeBlockControls[i] != null) { - return createPortal( - - - - - , - e.codeBlockControls[i] - ); - } else { - return <>; - } - }); - } - })} + {messageCodeBlocks.map((codeBlocks) => { + return codeBlocks.map((codeBlock) => { + return codeBlock.controlContainer != null ? ( + createPortal(codeBlock.button, codeBlock.controlContainer) + ) : ( + <> + ); + }); + })} = username: isUser ? 'You' : 'Assistant', actions: ( <> + + handleAddToExistingCaseClick(message.content)} + iconType="editorComment" + color="primary" + aria-label="Add message content as a timeline note" + /> + handleAddToExistingCaseClick(message.content)} @@ -389,11 +381,21 @@ export const SecurityAssistant: React.FC = ), // event: isUser ? 'Asked a question' : 'Responded with', - children: ( - - {message.content} - - ), + children: + index !== currentConversation.messages.length - 1 ? ( + + + {message.content} + + + ) : ( + + + {message.content} + + + + ), timelineAvatar: isUser ? ( ) : ( diff --git a/x-pack/plugins/security_solution/public/security_assistant/send_to_timeline_button.tsx b/x-pack/plugins/security_solution/public/security_assistant/send_to_timeline_button.tsx index 9ef80d6ed57c6..1d13927d0970a 100644 --- a/x-pack/plugins/security_solution/public/security_assistant/send_to_timeline_button.tsx +++ b/x-pack/plugins/security_solution/public/security_assistant/send_to_timeline_button.tsx @@ -10,10 +10,10 @@ import { EuiButton, EuiButtonEmpty } from '@elastic/eui'; import type { Filter } from '@kbn/es-query'; import { useDispatch } from 'react-redux'; -import { sourcererSelectors } from '../../public/common/store'; +import { sourcererSelectors } from '../common/store'; import { InputsModelId } from '../common/store/inputs/constants'; -import { TimeRange } from '../common/store/inputs/model'; -import { inputsActions } from '../../public/common/store/inputs'; +import type { TimeRange } from '../common/store/inputs/model'; +import { inputsActions } from '../common/store/inputs'; import { applyKqlFilterQuery, setActiveTabTimeline, @@ -21,11 +21,10 @@ import { updateDataView, updateEqlOptions, } from '../timelines/store/timeline/actions'; -import { sourcererActions } from '../../public/common/store/actions'; +import { sourcererActions } from '../common/store/actions'; import { SourcererScopeName } from '../common/store/sourcerer/model'; -import { TimelineTabs } from '../../common/types'; -import { TimelineId, TimelineType } from '../../common/types'; -import { DataProvider } from '../timelines/components/timeline/data_providers/data_provider'; +import { TimelineTabs, TimelineId, TimelineType } from '../../common/types'; +import type { DataProvider } from '../timelines/components/timeline/data_providers/data_provider'; import { useCreateTimeline } from '../timelines/components/timeline/properties/use_create_timeline'; import { useDeepEqualSelector } from '../common/hooks/use_selector'; import { ACTION_INVESTIGATE_IN_TIMELINE } from '../detections/components/alerts_table/translations'; @@ -86,42 +85,66 @@ export const SendToTimelineButton: React.FunctionComponent { + const codeBlockRegex = /```(\w+)?\s([\s\S]*?)```/g; + const matches = [...markdown.matchAll(codeBlockRegex)]; + const types = { + eql: ['Event Query Language', 'EQL sequence query'], + kql: ['Kibana Query Language', 'KQL Query'], + dsl: ['Elasticsearch QueryDSL', 'Elasticsearch Query DSL', 'Elasticsearch DSL'], + }; + + const result: CodeBlockDetails[] = matches.map((match) => { + let type = match[1] || 'no-type'; + if (type === 'no-type' || type === 'json') { + const start = match.index || 0; + const precedingText = markdown.slice(0, start); + for (const [typeKey, keywords] of Object.entries(types)) { + if (keywords.some((kw) => precedingText.includes(kw))) { + type = typeKey; + break; + } + } + } + + const content = match[2].trim(); + const start = match.index || 0; + const end = start + match[0].length; + return { type: type as QueryType, content, start, end }; + }); + + return result; +}; diff --git a/x-pack/plugins/security_solution/public/security_assistant/use_conversation/sample_conversations.tsx b/x-pack/plugins/security_solution/public/security_assistant/use_conversation/sample_conversations.tsx new file mode 100644 index 0000000000000..a988218b1cb80 --- /dev/null +++ b/x-pack/plugins/security_solution/public/security_assistant/use_conversation/sample_conversations.tsx @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Conversation } from '../security_assistant_context/types'; + +export const ALL_QUERIES_CONVERSATION: Record = { + default: { + id: 'default', + messages: [ + { + role: 'user', + content: 'Can you provide a sample KQL query please? (Kibana)', + timestamp: '5/12/2023, 12:04:07 AM', + }, + { + role: 'assistant', + content: + 'Certainly! Here\'s a sample KQL (Kibana Query Language) query for a fictional web server log data:\n\n```\nresponse.keyword: "200" AND extension.keyword: "html" AND NOT (geo.src: "US" OR geo.src: "CA")\n```\n\nThis query will return all documents where:\n\n1. The `response` field has a value of "200" (HTTP status code for a successful request).\n2. The `extension` field has a value of "html" (indicating an HTML file was requested).\n3. The `geo.src` field is not "US" or "CA" (meaning the request did not come from the United States or Canada).\n\nYou can use this query in Kibana\'s Discover or Dashboard panels to filter the data based on these conditions.', + timestamp: '5/12/2023, 12:04:21 AM', + }, + { + role: 'user', + content: + 'What about an EQL sequence query for 4 failed logons followed by a successful login?', + timestamp: '5/12/2023, 12:08:31 AM', + }, + { + role: 'assistant', + content: + 'Here\'s an EQL (Event Query Language) sequence query to detect 4 failed logons followed by a successful login for a fictional logon event data:\n\n```eql\nsequence by user.name with maxspan=5m\n [authentication where result == "failure" and event.action == "logon_attempt"]{4}\n [authentication where result == "success" and event.action == "logon_attempt"]\n```\n\nThis query will return sequences where:\n\n1. The events are grouped by the `user.name` field.\n2. The maximum time span between the first and last event in the sequence is 5 minutes (`maxspan=5m`).\n3. There are 4 consecutive `authentication` events with a `result` field value of "failure" and an `event.action` field value of "logon_attempt".\n4. After the 4 failed logon attempts, there\'s an `authentication` event with a `result` field value of "success" and an `event.action` field value of "logon_attempt".\n\nYou can use this EQL query in Elasticsearch or Kibana to detect potential brute force attacks or suspicious logon behavior.', + timestamp: '5/12/2023, 12:08:46 AM', + }, + { + role: 'user', + content: "Can you fix the markdown please, it's not rendering", + timestamp: '5/12/2023, 12:09:00 AM', + }, + { + role: 'assistant', + content: + 'Apologies for the formatting issue. Here\'s the corrected EQL sequence query:\n\n```\nsequence by user.name with maxspan=5m\n [authentication where result == "failure" and event.action == "logon_attempt"]{4}\n [authentication where result == "success" and event.action == "logon_attempt"]\n```\n\nThis query will return sequences where:\n\n1. The events are grouped by the `user.name` field.\n2. The maximum time span between the first and last event in the sequence is 5 minutes (`maxspan=5m`).\n3. There are 4 consecutive `authentication` events with a `result` field value of "failure" and an `event.action` field value of "logon_attempt".\n4. After the 4 failed logon attempts, there\'s an `authentication` event with a `result` field value of "success" and an `event.action` field value of "logon_attempt".\n\nYou can use this EQL query in Elasticsearch or Kibana to detect potential brute force attacks or suspicious logon behavior.', + timestamp: '5/12/2023, 12:09:16 AM', + }, + { + role: 'user', + content: + 'Can you response with a message that contains both the first KQL query you sent me, and also the EQL sequence query you sent?', + timestamp: '5/12/2023, 12:11:49 AM', + }, + { + role: 'assistant', + content: + 'Certainly! Here are both the KQL and EQL queries in a single response:\n\n**KQL Query (Kibana Query Language):**\n\n```\nresponse.keyword: "200" AND extension.keyword: "html" AND NOT (geo.src: "US" OR geo.src: "CA")\n```\n\nThis KQL query returns all documents where:\n\n1. The `response` field has a value of "200" (HTTP status code for a successful request).\n2. The `extension` field has a value of "html" (indicating an HTML file was requested).\n3. The `geo.src` field is not "US" or "CA" (meaning the request did not come from the United States or Canada).\n\nYou can use this query in Kibana\'s Discover or Dashboard panels to filter the data based on these conditions.\n\n**EQL Sequence Query (Event Query Language):**\n\n```\nsequence by user.name with maxspan=5m\n [authentication where result == "failure" and event.action == "logon_attempt"]{4}\n [authentication where result == "success" and event.action == "logon_attempt"]\n```\n\nThis EQL query returns sequences where:\n\n1. The events are grouped by the `user.name` field.\n2. The maximum time span between the first and last event in the sequence is 5 minutes (`maxspan=5m`).\n3. There are 4 consecutive `authentication` events with a `result` field value of "failure" and an `event.action` field value of "logon_attempt".\n4. After the 4 failed logon attempts, there\'s an `authentication` event with a `result` field value of "success" and an `event.action` field value of "logon_attempt".\n\nYou can use this EQL query in Elasticsearch or Kibana to detect potential brute force attacks or suspicious logon behavior.', + timestamp: '5/12/2023, 12:12:09 AM', + }, + { + role: 'user', + content: + 'Last but not least, could you write an Elasticsearch QueryDSL for filtering events where `event.action` is `logon_attempt`', + timestamp: '5/12/2023, 12:30:38 AM', + }, + { + role: 'assistant', + content: + 'Certainly! Here\'s an Elasticsearch QueryDSL for filtering events where `event.action` is `logon_attempt`:\n\n```json\n{\n "query": {\n "bool": {\n "filter": [\n {\n "term": {\n "event.action": "logon_attempt"\n }\n }\n ]\n }\n }\n}\n```\n\nThis QueryDSL will return all documents where the `event.action` field has a value of "logon_attempt". You can use this query in Elasticsearch to filter the data based on this condition.', + timestamp: '5/12/2023, 12:30:49 AM', + }, + ], + apiConfig: { + openAI: { + apiKey: '', + baseUrl: '', + model: '', + prompt: '', + temperature: 0.2, + }, + virusTotal: { + apiKey: '', + baseUrl: '', + }, + }, + }, + timeline: { + id: 'timeline', + messages: [], + apiConfig: { + openAI: { + apiKey: '', + baseUrl: '', + model: '', + prompt: '', + temperature: 0.2, + }, + virusTotal: { + apiKey: '', + baseUrl: '', + }, + }, + }, +}; diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/add_timeline_button/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/flyout/add_timeline_button/index.tsx index 74662e7563201..bc77c7299738e 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/add_timeline_button/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/flyout/add_timeline_button/index.tsx @@ -14,6 +14,10 @@ import type { ActionTimelineToShow } from '../../open_timeline/types'; import * as i18n from '../../timeline/properties/translations'; import { NewTimeline } from '../../timeline/properties/helpers'; import { NewTemplateTimeline } from '../../timeline/properties/new_template_timeline'; +import { + CREATE_RULE_FROM_TIMELINE, + CREATE_RULE_FROM_TIMELINE_CORRELATION, +} from '../../open_timeline/translations'; interface AddTimelineButtonComponentProps { timelineId: string; @@ -81,6 +85,20 @@ const AddTimelineButtonComponent: React.FC = ({ title={i18n.NEW_TEMPLATE_TIMELINE} /> + + + + + + diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/data_provider.ts b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/data_provider.ts index 3ed66744c760f..42afc52e7eb94 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/data_provider.ts +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/data_providers/data_provider.ts @@ -32,6 +32,8 @@ export interface QueryMatch { operator: QueryOperator; } +export type QueryType = 'eql' | 'kql' | 'dsl' | 'json' | 'no-type'; + export interface DataProvider { /** Uniquely identifies a data provider */ id: string; @@ -67,6 +69,8 @@ export interface DataProvider { * Array of multiple values for a field */ valueArray?: string[] | null; + + queryType?: QueryType; } export type DataProvidersAnd = Pick>;