diff --git a/x-pack/legacy/plugins/siem/index.test.ts b/x-pack/legacy/plugins/siem/index.test.ts deleted file mode 100644 index 5b7c488eb174c..0000000000000 --- a/x-pack/legacy/plugins/siem/index.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { getRequiredPlugins } from '.'; - -// This test is a temporary test which is so we do not accidentally check-in -// feature flags turned on from "alerting" and "actions". If those get -// turned on during a check-in it will cause everyone's Kibana to not start. -// Once alerting and actions are part of the plugins by default this test -// should be removed. -describe('siem plugin tests', () => { - describe('getRequiredPlugins', () => { - test('null settings returns regular kibana and elasticsearch plugins', () => { - expect(getRequiredPlugins(null, null)).toEqual(['kibana', 'elasticsearch']); - }); - - test('undefined settings returns regular kibana and elasticsearch plugins', () => { - expect(getRequiredPlugins(undefined, undefined)).toEqual(['kibana', 'elasticsearch']); - }); - - test('alertingFeatureEnabled being false returns regular kibana and elasticsearch plugins', () => { - expect(getRequiredPlugins('false', undefined)).toEqual(['kibana', 'elasticsearch']); - }); - - test('alertingFeatureEnabled being true returns action and alerts', () => { - expect(getRequiredPlugins('true', undefined)).toEqual([ - 'kibana', - 'elasticsearch', - 'alerting', - 'actions', - ]); - }); - - test('alertingFeatureEnabled being false but a string for siemIndex returns alerting and actions', () => { - expect(getRequiredPlugins('false', '.siem-signals-frank')).toEqual([ - 'kibana', - 'elasticsearch', - 'alerting', - 'actions', - ]); - }); - - test('alertingFeatureEnabled being true and a string for siemIndex returns alerting and actions', () => { - expect(getRequiredPlugins('true', '.siem-signals-frank')).toEqual([ - 'kibana', - 'elasticsearch', - 'alerting', - 'actions', - ]); - }); - - test('alertingFeatureEnabled being true and an empty string for siemIndex returns regular kibana and elasticsearch plugins', () => { - expect(getRequiredPlugins(undefined, '')).toEqual(['kibana', 'elasticsearch']); - }); - - test('alertingFeatureEnabled being true and a string of spaces for siemIndex returns regular kibana and elasticsearch plugins', () => { - expect(getRequiredPlugins(undefined, ' ')).toEqual(['kibana', 'elasticsearch']); - }); - - test('alertingFeatureEnabled being null and a string for siemIndex returns alerting and actions', () => { - expect(getRequiredPlugins(null, '.siem-signals-frank')).toEqual([ - 'kibana', - 'elasticsearch', - 'alerting', - 'actions', - ]); - }); - - test('alertingFeatureEnabled being undefined and a string for siemIndex returns alerting and actions', () => { - expect(getRequiredPlugins(undefined, '.siem-signals-frank')).toEqual([ - 'kibana', - 'elasticsearch', - 'alerting', - 'actions', - ]); - }); - }); -}); diff --git a/x-pack/legacy/plugins/siem/index.ts b/x-pack/legacy/plugins/siem/index.ts index 06aaec631be66..72b4ec588a5a4 100644 --- a/x-pack/legacy/plugins/siem/index.ts +++ b/x-pack/legacy/plugins/siem/index.ts @@ -28,36 +28,13 @@ import { } from './common/constants'; import { defaultIndexPattern } from './default_index_pattern'; -// This is VERY TEMPORARY as we need a way to turn on alerting and actions -// for the server without having to manually edit this file. Once alerting -// and actions has their enabled true by default this can be removed. -// 'alerting', 'actions' are hidden behind feature flags at the moment so if you turn -// these on without the feature flags turned on then Kibana will crash since we are a legacy plugin -// and legacy plugins cannot have optional requirements. -// This returns ['alerting', 'actions', 'kibana', 'elasticsearch'] iff alertingFeatureEnabled is true -// or if the developer signalsIndex is setup. Otherwise this returns ['kibana', 'elasticsearch'] -export const getRequiredPlugins = ( - alertingFeatureEnabled: string | null | undefined, - signalsIndex: string | null | undefined -) => { - const baseRequire = ['kibana', 'elasticsearch']; - if ( - (signalsIndex != null && signalsIndex.trim() !== '') || - (alertingFeatureEnabled && alertingFeatureEnabled.toLowerCase() === 'true') - ) { - return [...baseRequire, 'alerting', 'actions']; - } else { - return baseRequire; - } -}; - // eslint-disable-next-line @typescript-eslint/no-explicit-any export const siem = (kibana: any) => { return new kibana.Plugin({ id: APP_ID, configPrefix: 'xpack.siem', publicDir: resolve(__dirname, 'public'), - require: getRequiredPlugins(process.env.ALERTING_FEATURE_ENABLED, process.env.SIGNALS_INDEX), + require: ['kibana', 'elasticsearch', 'alerting', 'actions'], uiExports: { app: { description: i18n.translate('xpack.siem.securityDescription', { diff --git a/x-pack/legacy/plugins/siem/server/kibana.index.ts b/x-pack/legacy/plugins/siem/server/kibana.index.ts index 2f1530a777042..3d73b9f4d90b0 100644 --- a/x-pack/legacy/plugins/siem/server/kibana.index.ts +++ b/x-pack/legacy/plugins/siem/server/kibana.index.ts @@ -42,21 +42,15 @@ export const initServerWithKibana = ( const libs = compose(kbnServer, mode); initServer(libs); - if ( - kbnServer.config().has('xpack.actions.enabled') && - kbnServer.config().get('xpack.actions.enabled') === true && - kbnServer.config().has('xpack.alerting.enabled') && - kbnServer.config().has('xpack.alerting.enabled') === true - ) { - logger.info( - 'Detected feature flags for actions and alerting and enabling detection engine API endpoints' - ); - createRulesRoute(kbnServer); - readRulesRoute(kbnServer); - updateRulesRoute(kbnServer); - deleteRulesRoute(kbnServer); - findRulesRoute(kbnServer); - } + + // Signals/Alerting Rules routes for + // routes such as ${DETECTION_ENGINE_RULES_URL} + // that have the REST endpoints of /api/detection_engine/rules + createRulesRoute(kbnServer); + readRulesRoute(kbnServer); + updateRulesRoute(kbnServer); + deleteRulesRoute(kbnServer); + findRulesRoute(kbnServer); const xpackMainPlugin = kbnServer.plugins.xpack_main; xpackMainPlugin.registerFeature({ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md b/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md index 4b1dbf62d0dd4..75757bbaa0c1f 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md @@ -1,62 +1,20 @@ -Temporary README.md for users and developers working on the backend detection engine -for how to get started. +README.md for developers working on the backend detection engine on how to get started +using the CURL scripts in the scripts folder. -# Setup for Users +The scripts rely on CURL and jq: +* [CURL](https://curl.haxx.se) +* [jq](https://stedolan.github.io/jq/) -If you're just a user and want to enable the REST interfaces and UI screens do the following. -NOTE: this is very temporary and once alerting and actions is enabled by default you will no -longer have to do these steps - -Set the environment variable ALERTING_FEATURE_ENABLED to be true in your .profile or your windows -global environment variable. - -```sh -export ALERTING_FEATURE_ENABLED=true -``` - -In your `kibana.yml` file enable alerting and actions like so: - -```sh -# Feature flag to turn on alerting -xpack.alerting.enabled: true - -# Feature flag to turn on actions which goes with alerting -xpack.actions.enabled: true -``` - -Start Kibana and you will see these messages indicating detection engine is activated like so: - -```sh -server log [11:39:05.561] [info][siem] Detected feature flags for actions and alerting and enabling detection engine API endpoints -``` - -If you see crashes like this: - -```ts - FATAL Error: Unmet requirement "alerting" for plugin "siem" -``` - -It is because Kibana is not picking up your changes from `kibana.yml` and not seeing that alerting and actions is enabled. - -# For Developers - -See these two other pages for references: -https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/alerting/README.md -https://github.com/elastic/kibana/tree/master/x-pack/legacy/plugins/actions - -Since there is no UI yet and a lot of backend areas that are not created, you -should install the kbn-action and kbn-alert project from here: -https://github.com/pmuellr/kbn-action - -The scripts rely on CURL and jq, ensure both of these are installed: +Install curl and jq ```sh brew update brew install curl brew install jq ``` -Open up your .zshrc/.bashrc and add these lines with the variables filled in: +Open `$HOME/.zshrc` or `${HOME}.bashrc` depending on your SHELL output from `echo $SHELL` +and add these environment variables: ```sh export ELASTICSEARCH_USERNAME=${user} @@ -66,52 +24,30 @@ export KIBANA_URL=http://localhost:5601 export SIGNALS_INDEX=.siem-signals-${your user id} export TASK_MANAGER_INDEX=.kibana-task-manager-${your user id} export KIBANA_INDEX=.kibana-${your user id} - -# This is for the kbn-action and kbn-alert tool -export KBN_URLBASE=http://${user}:${password}@localhost:5601 ``` -source your .zhsrc/.bashrc or open a new terminal to ensure you get the new values set. - -Optional env var when set to true will utilize `reindex` api for reindexing -instead of the scroll and bulk index combination. +source `$HOME/.zshrc` or `${HOME}.bashrc` to ensure variables are set: ```sh -export USE_REINDEX_API=true -``` - -Add these lines to your `kibana.dev.yml` to turn on the feature toggles of alerting and actions: - -```sh -# Feature flag to turn on alerting -xpack.alerting.enabled: true - -# Feature flag to turn on actions which goes with alerting -xpack.actions.enabled: true +source ~/.zshrc ``` Restart Kibana and ensure that you are using `--no-base-path` as changing the base path is a feature but will get in the way of the CURL scripts written as is. You should see alerting and actions starting up like so afterwards ```sh -server log [22:05:22.277] [info][status][plugin:alerting@8.0.0] Status changed from uninitialized to green - Ready -server log [22:05:22.270] [info][status][plugin:actions@8.0.0] Status changed from uninitialized to green - Ready -``` - -You should also see the SIEM detect the feature flags and start the API endpoints for detection engine - -```sh -server log [11:39:05.561] [info][siem] Detected feature flags for actions and alerting and enabling detection engine API endpoints +server log [22:05:22.277] [info][status][plugin:alerting@8.0.0] Status changed from uninitialized to green - Ready +server log [22:05:22.270] [info][status][plugin:actions@8.0.0] Status changed from uninitialized to green - Ready ``` Go into your SIEM Advanced settings and underneath the setting of `siem:defaultSignalsIndex`, set that to the same -value as you did with the environment variable of SIGNALS_INDEX, which should be `.siem-signals-${your user id}` +value as you did with the environment variable of `${SIGNALS_INDEX}`, which should be `.siem-signals-${your user id}` ``` .siem-signals-${your user id} ``` -Open a terminal and go into the scripts folder `cd kibana/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts` and run: +Go to the scripts folder `cd kibana/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts` and run: ```sh ./hard_reset.sh @@ -124,7 +60,7 @@ which will: - Delete any existing alerts you have - Delete any existing alert tasks you have - Delete any existing signal mapping you might have had. -- Add the latest signal index and its mappings using your settings from `SIGNALS_INDEX` environment variable. +- Add the latest signal index and its mappings using your settings from `${SIGNALS_INDEX}` environment variable. - Posts the sample rule from `rules/root_or_admin_1.json` by replacing its `output_index` with your `SIGNALS_INDEX` environment variable - The sample rule checks for root or admin every 5 minutes and reports that as a signal if it is a positive hit @@ -181,21 +117,20 @@ You should see the new rules created like so: Every 5 minutes if you get positive hits you will see messages on info like so: ```sh -server log [09:54:59.013] [info][plugins][siem] Total signals found from signal rule "id: a556065c-0656-4ba1-ad64-a77ca9d2013b", "ruleId: rule-1": 10000 +server log [09:54:59.013] [info][plugins][siem] Total signals found from signal rule "id: a556065c-0656-4ba1-ad64-a77ca9d2013b", "ruleId: rule-1": 10000 ``` -Rules are space aware and default to the "default" space for these scripts if you do not export -the variable of SPACE_URL. For example, if you want to post rules to the space `test-space` you would -set your SPACE_URL to be: +Rules are [space aware](https://www.elastic.co/guide/en/kibana/master/xpack-spaces.html) and default +to the "default" (empty) URL space if you do not export the variable of `SPACE_URL`. Example, if you want to +post rules to `test-space` you set `SPACE_URL` to be: ```sh export SPACE_URL=/s/test-space ``` -So that the scripts prepend a `/s/test-space` in front of all the APIs to correctly create, modify, delete, and update -them from within that space. - -See the scripts folder and the tools for more command line fun. +The `${SPACE_URL}` is in front of all the APIs to correctly create, modify, delete, and update +them from within the defined space. If this variable is not defined the default which is the url of an +empty string will be used. Add the `.siem-signals-${your user id}` to your advanced SIEM settings to see any signals created which should update once every 5 minutes at this point. @@ -216,3 +151,7 @@ logging.events: ops: __no-ops__, } ``` + +See these two README.md's pages for more references on the alerting and actions API: +https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/alerting/README.md +https://github.com/elastic/kibana/tree/master/x-pack/legacy/plugins/actions diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/README.md b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/README.md deleted file mode 100644 index 8d617a8de3fcd..0000000000000 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/README.md +++ /dev/null @@ -1,39 +0,0 @@ -A set of scripts for developers to utilize command line functionality of Kibana/Elastic -search which is not available in the DEV console for the detection engine. - -Before beginning ensure in your .zshrc/.bashrc you have your user, password, and url set: - -Open up your .zshrc/.bashrc and add these lines with the variables filled in: - -``` -export ELASTICSEARCH_USERNAME=${user} -export ELASTICSEARCH_PASSWORD=${password} -export ELASTICSEARCH_URL=https://${ip}:9200 -export KIBANA_URL=http://localhost:5601 -export SIGNALS_INDEX=.siem-signals-${your user id} -export TASK_MANAGER_INDEX=.kibana-task-manager-${your user id} -export KIBANA_INDEX=.kibana-${your user id} - -# This is for the kbn-action and kbn-alert tool -export KBN_URLBASE=http://${user}:${password}@localhost:5601 -``` - -And that you have the latest version of [NodeJS](https://nodejs.org/en/), -[CURL](https://curl.haxx.se), and [jq](https://stedolan.github.io/jq/) installed. - -If you have homebrew you can install using brew like so - -``` -brew install jq -``` - -After that you can execute scripts within this folder by first ensuring -your current working directory is `./scripts` and then running any scripts within -that folder. - -Example to add a rule to the system - -``` -cd ./scripts -./post_rule.sh ./rules/root_or_admin_1.json -``` diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/check_env_variables.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/check_env_variables.sh index c534b33d28413..c2406dc7f6231 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/check_env_variables.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/check_env_variables.sh @@ -11,36 +11,36 @@ set -e if [ -z "${ELASTICSEARCH_USERNAME}" ]; then - echo "Set ELASTICSEARCH_USERNAME in your enviornment" + echo "Set ELASTICSEARCH_USERNAME in your environment" exit 1 fi if [ -z "${ELASTICSEARCH_PASSWORD}" ]; then - echo "Set ELASTICSEARCH_PASSWORD in your enviornment" + echo "Set ELASTICSEARCH_PASSWORD in your environment" exit 1 fi if [ -z "${ELASTICSEARCH_URL}" ]; then - echo "Set ELASTICSEARCH_URL in your enviornment" + echo "Set ELASTICSEARCH_URL in your environment" exit 1 fi if [ -z "${KIBANA_URL}" ]; then - echo "Set KIBANA_URL in your enviornment" + echo "Set KIBANA_URL in your environment" exit 1 fi if [ -z "${SIGNALS_INDEX}" ]; then - echo "Set SIGNALS_INDEX in your enviornment" + echo "Set SIGNALS_INDEX in your environment" exit 1 fi if [ -z "${TASK_MANAGER_INDEX}" ]; then - echo "Set TASK_MANAGER_INDEX in your enviornment" + echo "Set TASK_MANAGER_INDEX in your environment" exit 1 fi if [ -z "${KIBANA_INDEX}" ]; then - echo "Set KIBANA_INDEX in your enviornment" + echo "Set KIBANA_INDEX in your environment" exit 1 fi