[SIEM][Detection Engine] Adds signal data index per spaces through index naming conventions

x-pack/legacy/plugins/siem/common/constants.ts

@@ -15,7 +15,11 @@ export const DEFAULT_TIME_RANGE = 'timepicker:timeDefaults';
 export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults';
 export const DEFAULT_SIEM_TIME_RANGE = 'siem:timeDefaults';
 export const DEFAULT_SIEM_REFRESH_INTERVAL = 'siem:refreshIntervalDefaults';
+
+// DEPRECATED: THIS WILL BE REMOVED VERY SOON AND IS NO LONGER USED ON THE BACKEND
+// TODO: Remove this as soon as no code is left that is pulling data from it.
 export const DEFAULT_SIGNALS_INDEX_KEY = 'siem:defaultSignalsIndex';
+
 export const DEFAULT_SIGNALS_INDEX = '.siem-signals';
 export const DEFAULT_MAX_SIGNALS = 100;
 export const DEFAULT_SEARCH_AFTER_PAGE_SIZE = 100;
@@ -32,12 +36,18 @@ export const DEFAULT_INTERVAL_VALUE = 300000; // ms
 export const DEFAULT_TIMEPICKER_QUICK_RANGES = 'timepicker:quickRanges';
 
 /**
- * Id for the SIGNALS alerting type
+ * Id for the signals alerting type
  */
 export const SIGNALS_ID = `${APP_ID}.signals`;
 
 /**
- * Detection engine route
+ * Detection engine routes
  */
 export const DETECTION_ENGINE_URL = '/api/detection_engine';
 export const DETECTION_ENGINE_RULES_URL = `${DETECTION_ENGINE_URL}/rules`;
+export const DETECTION_ENGINE_INDEX_URL = `${DETECTION_ENGINE_URL}/index`;
+
+/**
+ * Default signals index key for kibana.dev.yml
+ */
+export const SIGNALS_INDEX_KEY = 'signalsIndex';

x-pack/legacy/plugins/siem/index.ts

@@ -7,6 +7,7 @@
 import { i18n } from '@kbn/i18n';
 import { resolve } from 'path';
 import { Server } from 'hapi';
+import { Root } from 'joi';
 
 import { PluginInitializerContext } from 'src/core/server';
 import { plugin } from './server';
@@ -24,6 +25,7 @@ import {
   DEFAULT_FROM,
   DEFAULT_TO,
   DEFAULT_SIGNALS_INDEX,
+  SIGNALS_INDEX_KEY,
   DEFAULT_SIGNALS_INDEX_KEY,
 } from './common/constants';
 import { defaultIndexPattern } from './default_index_pattern';
@@ -103,6 +105,8 @@ export const siem = (kibana: any) => {
           category: ['siem'],
           requiresPageReload: true,
         },
+        // DEPRECATED: This should be removed once the front end is no longer using any parts of it.
+        // TODO: Remove this as soon as no code is left that is pulling data from it.
         [DEFAULT_SIGNALS_INDEX_KEY]: {
           name: i18n.translate('xpack.siem.uiSettings.defaultSignalsIndexLabel', {
             defaultMessage: 'Elasticsearch signals index',
@@ -155,7 +159,11 @@ export const siem = (kibana: any) => {
         getInjectedUiAppVars,
         indexPatternsServiceFactory,
         injectUiAppVars,
-        plugins: { alerting: plugins.alerting, xpack_main: plugins.xpack_main },
+        plugins: {
+          alerting: plugins.alerting,
+          xpack_main: plugins.xpack_main,
+          spaces: plugins.spaces,
+        },
         route: route.bind(server),
         savedObjects,
       };
@@ -166,5 +174,13 @@ export const siem = (kibana: any) => {
         serverFacade
       );
     },
+    config(Joi: Root) {
+      return Joi.object()
+        .keys({
+          enabled: Joi.boolean().default(true),
+          [SIGNALS_INDEX_KEY]: Joi.string().default(DEFAULT_SIGNALS_INDEX),
+        })
+        .default();
+    },
   });
 };

x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_rules.js

@@ -42,9 +42,10 @@ const allRulesNdJson = 'all_rules.ndjson';
 // For converting, if you want to use these instead of rely on the defaults then
 // comment these in and use them for the script. Otherwise this is commented out
 // so we can utilize the defaults of input and output which are based on saved objects
-// of siem:defaultIndex and siem:defaultSignalsIndex
+// of siem:defaultIndex and your kibana.dev.yml setting of xpack.siem.signalsIndex. If
+// the setting of xpack.siem.signalsIndex is not set it defaults to .siem-signals
 // const INDEX = ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'];
-// const OUTPUT_INDEX = process.env.SIGNALS_INDEX || '.siem-signals';
+// const OUTPUT_INDEX = '.siem-signals-some-other-index';
 
 const walk = dir => {
   const list = fs.readdirSync(dir);

x-pack/legacy/plugins/siem/server/kibana.index.ts

@@ -18,11 +18,14 @@ import {
 import { rulesAlertType } from './lib/detection_engine/alerts/rules_alert_type';
 import { isAlertExecutor } from './lib/detection_engine/alerts/types';
 import { createRulesRoute } from './lib/detection_engine/routes/create_rules_route';
+import { createIndexRoute } from './lib/detection_engine/routes/index/create_index_route';
+import { readIndexRoute } from './lib/detection_engine/routes/index/read_index_route';
 import { readRulesRoute } from './lib/detection_engine/routes/read_rules_route';
 import { findRulesRoute } from './lib/detection_engine/routes/find_rules_route';
 import { deleteRulesRoute } from './lib/detection_engine/routes/delete_rules_route';
 import { updateRulesRoute } from './lib/detection_engine/routes/update_rules_route';
 import { ServerFacade } from './types';
+import { deleteIndexRoute } from './lib/detection_engine/routes/index/delete_index_route';
 
 const APP_ID = 'siem';
 
@@ -43,15 +46,20 @@ export const initServerWithKibana = (
   const libs = compose(kbnServer, mode);
   initServer(libs);
 
-  // Signals/Alerting Rules routes for
-  // routes such as ${DETECTION_ENGINE_RULES_URL}
-  // that have the REST endpoints of /api/detection_engine/rules
+  // Detection Engine Rule routes that have the REST endpoints of /api/detection_engine/rules
+  // All REST rule creation, deletion, updating, etc...
   createRulesRoute(kbnServer);
   readRulesRoute(kbnServer);
   updateRulesRoute(kbnServer);
   deleteRulesRoute(kbnServer);
   findRulesRoute(kbnServer);
 
+  // Detection Engine index routes that have the REST endpoints of /api/detection_engine/index
+  // All REST index creation, policy management for spaces
+  createIndexRoute(kbnServer);
+  readIndexRoute(kbnServer);
+  deleteIndexRoute(kbnServer);
+
   const xpackMainPlugin = kbnServer.plugins.xpack_main;
   xpackMainPlugin.registerFeature({
     id: APP_ID,

x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md

@@ -2,11 +2,12 @@ README.md for developers working on the backend detection engine on how to get s
 using the CURL scripts in the scripts folder.
 
 The scripts rely on CURL and jq:
-* [CURL](https://curl.haxx.se)
-* [jq](https://stedolan.github.io/jq/)
 
+- [CURL](https://curl.haxx.se)
+- [jq](https://stedolan.github.io/jq/)
 
 Install curl and jq
+
 ```sh
 brew update
 brew install curl
@@ -21,7 +22,6 @@ export ELASTICSEARCH_USERNAME=${user}
 export ELASTICSEARCH_PASSWORD=${password}
 export ELASTICSEARCH_URL=https://${ip}:9200
 export KIBANA_URL=http://localhost:5601
-export SIGNALS_INDEX=.siem-signals-${your user id}
 export TASK_MANAGER_INDEX=.kibana-task-manager-${your user id}
 export KIBANA_INDEX=.kibana-${your user id}
 ```
@@ -32,6 +32,12 @@ source `$HOME/.zshrc` or `${HOME}.bashrc` to ensure variables are set:
 source ~/.zshrc
 ```
 
+Open your `kibana.dev.yml` file and add these lines:
+
+```sh
+xpack.siem.signalsIndex: .siem-signals-${your user id}
+```
+
 Restart Kibana and ensure that you are using `--no-base-path` as changing the base path is a feature but will
 get in the way of the CURL scripts written as is. You should see alerting and actions starting up like so afterwards
 
@@ -40,13 +46,6 @@ server log [22:05:22.277] [info][status][plugin:alerting@8.0.0] Status changed f
 server log [22:05:22.270] [info][status][plugin:actions@8.0.0] Status changed from uninitialized to green - Ready
 ```
 
-Go into your SIEM Advanced settings and underneath the setting of `siem:defaultSignalsIndex`, set that to the same
-value as you did with the environment variable of `${SIGNALS_INDEX}`, which should be `.siem-signals-${your user id}`
-
-```
-.siem-signals-${your user id}
-```
-
 Go to the scripts folder `cd kibana/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts` and run:
 
 ```sh
@@ -59,9 +58,9 @@ which will:
 - Delete any existing actions you have
 - Delete any existing alerts you have
 - Delete any existing alert tasks you have
-- Delete any existing signal mapping you might have had.
-- Add the latest signal index and its mappings using your settings from `${SIGNALS_INDEX}` environment variable.
-- Posts the sample rule from `rules/root_or_admin_1.json` by replacing its `output_index` with your `SIGNALS_INDEX` environment variable
+- Delete any existing signal mapping, policies, and template, you might have previously had.
+- Add the latest signal index and its mappings using your settings from `kibana.dev.yml` environment variable of `xpack.siem.signalsIndex`.
+- Posts the sample rule from `rules/root_or_admin_1.json`
 - The sample rule checks for root or admin every 5 minutes and reports that as a signal if it is a positive hit
 
 Now you can run
@@ -128,9 +127,9 @@ post rules to `test-space` you set `SPACE_URL` to be:
 export SPACE_URL=/s/test-space
 ```
 
-The  `${SPACE_URL}` is in front of all the APIs to correctly create, modify, delete, and update
+The `${SPACE_URL}` is in front of all the APIs to correctly create, modify, delete, and update
 them from within the defined space. If this variable is not defined the default which is the url of an
-empty string will be used. 
+empty string will be used.
 
 Add the `.siem-signals-${your user id}` to your advanced SIEM settings to see any signals
 created which should update once every 5 minutes at this point.