[Security Solution][Detections] Add new fields to the rule model: Related Integrations, Required Fields, and Setup

x-pack/plugins/security_solution/common/detection_engine/schemas/common/index.ts

@@ -6,4 +6,5 @@
  */
 
 export * from './rule_monitoring';
+export * from './rule_params';
 export * from './schemas';

x-pack/plugins/security_solution/common/detection_engine/schemas/common/rule_params.ts

@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as t from 'io-ts';
+import { NonEmptyString } from '@kbn/securitysolution-io-ts-types';
+
+// -------------------------------------------------------------------------------------------------
+// Related integrations
+
+/**
+ * Related integration is a potential dependency of a rule. It's assumed that if the user installs
+ * one of the related integrations of a rule, the rule might start to work properly because it will
+ * have source events (generated by this integration) potentially matching the rule's query.
+ *
+ *   NOTE: Proper work is not guaranteed, because a related integration, if installed, can be
+ *   configured differently or generate data that is not necessarily relevant for this rule.
+ *
+ * Related integration is a combination of a Fleet package and (optionally) one of the
+ * package's "integrations" that this package contains. It is represented by 3 properties:
+ *
+ *   - `package`: name of the package (required, unique id)
+ *   - `version`: version of the package (required, semver-compatible)
+ *   - `integration`: name of the integration of this package (optional, id within the package)
+ *
+ * There are Fleet packages like `windows` that contain only one integration; in this case,
+ * `integration` should be unspecified. There are also packages like `aws` and `azure` that contain
+ * several integrations; in this case, `integration` should be specified.
+ *
+ * @example
+ * const x: RelatedIntegration = {
+ *   package: 'windows',
+ *   version: '1.5.x',
+ * };
+ *
+ * @example
+ * const x: RelatedIntegration = {
+ *   package: 'azure',
+ *   version: '~1.1.6',
+ *   integration: 'activitylogs',
+ * };
+ */
+export type RelatedIntegration = t.TypeOf<typeof RelatedIntegration>;
+export const RelatedIntegration = t.exact(
+  t.intersection([
+    t.type({
+      package: NonEmptyString,
+      version: NonEmptyString,
+    }),
+    t.partial({
+      integration: NonEmptyString,
+    }),
+  ])
+);
+
+/**
+ * Array of related integrations.
+ *
+ * @example
+ * const x: RelatedIntegrationArray = [
+ *   {
+ *     package: 'windows',
+ *     version: '1.5.x',
+ *   },
+ *   {
+ *     package: 'azure',
+ *     version: '~1.1.6',
+ *     integration: 'activitylogs',
+ *   },
+ * ];
+ */
+export type RelatedIntegrationArray = t.TypeOf<typeof RelatedIntegrationArray>;
+export const RelatedIntegrationArray = t.array(RelatedIntegration);
+
+// -------------------------------------------------------------------------------------------------
+// Required fields
+
+/**
+ * Almost all types of Security rules check source event documents for a match to some kind of
+ * query or filter. If a document has certain field with certain values, then it's a match and
+ * the rule will generate an alert.
+ *
+ * Required field is an event field that must be present in the source indices of a given rule.
+ *
+ * @example
+ * const standardEcsField: RequiredField = {
+ *   name: 'event.action',
+ *   type: 'keyword',
+ *   ecs: true,
+ * };
+ *
+ * @example
+ * const nonEcsField: RequiredField = {
+ *   name: 'winlog.event_data.AttributeLDAPDisplayName',
+ *   type: 'keyword',
+ *   ecs: false,
+ * };
+ */
+export type RequiredField = t.TypeOf<typeof RequiredField>;
+export const RequiredField = t.exact(
+  t.type({
+    name: NonEmptyString,
+    type: NonEmptyString,
+    ecs: t.boolean,
+  })
+);
+
+/**
+ * Array of event fields that must be present in the source indices of a given rule.
+ *
+ * @example
+ * const x: RequiredFieldArray = [
+ *   {
+ *     name: 'event.action',
+ *     type: 'keyword',
+ *     ecs: true,
+ *   },
+ *   {
+ *     name: 'event.code',
+ *     type: 'keyword',
+ *     ecs: true,
+ *   },
+ *   {
+ *     name: 'winlog.event_data.AttributeLDAPDisplayName',
+ *     type: 'keyword',
+ *     ecs: false,
+ *   },
+ * ];
+ */
+export type RequiredFieldArray = t.TypeOf<typeof RequiredFieldArray>;
+export const RequiredFieldArray = t.array(RequiredField);
+
+// -------------------------------------------------------------------------------------------------
+// Setup guide
+
+/**
+ * Any instructions for the user for setting up their environment in order to start receiving
+ * source events for a given rule.
+ *
+ * It's a multiline text. Markdown is supported.
+ */
+export type SetupGuide = t.TypeOf<typeof SetupGuide>;
+export const SetupGuide = t.string;

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts

@@ -72,7 +72,10 @@ import {
   Author,
   event_category_override,
   namespace,
-} from '../common/schemas';
+  RelatedIntegrationArray,
+  RequiredFieldArray,
+  SetupGuide,
+} from '../common';
 
 /**
  * Big differences between this schema and the createRulesSchema
@@ -117,8 +120,11 @@ export const addPrepackagedRulesSchema = t.intersection([
       meta, // defaults to "undefined" if not set during decode
       machine_learning_job_id, // defaults to "undefined" if not set during decode
       max_signals: DefaultMaxSignalsNumber, // defaults to DEFAULT_MAX_SIGNALS (100) if not set during decode
+      related_integrations: RelatedIntegrationArray, // defaults to "undefined" if not set during decode
+      required_fields: RequiredFieldArray, // defaults to "undefined" if not set during decode
       risk_score_mapping: DefaultRiskScoreMappingArray, // defaults to empty risk score mapping array if not set during decode
       rule_name_override, // defaults to "undefined" if not set during decode
+      setup: SetupGuide, // defaults to "undefined" if not set during decode
       severity_mapping: DefaultSeverityMappingArray, // defaults to empty actions array if not set during decode
       tags: DefaultStringArray, // defaults to empty string array if not set during decode
       to: DefaultToString, // defaults to "now" if not set during decode

x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts

@@ -80,7 +80,10 @@ import {
   timestamp_override,
   Author,
   event_category_override,
-} from '../common/schemas';
+  RelatedIntegrationArray,
+  RequiredFieldArray,
+  SetupGuide,
+} from '../common';
 
 /**
  * Differences from this and the createRulesSchema are
@@ -129,8 +132,11 @@ export const importRulesSchema = t.intersection([
       meta, // defaults to "undefined" if not set during decode
       machine_learning_job_id, // defaults to "undefined" if not set during decode
       max_signals: DefaultMaxSignalsNumber, // defaults to DEFAULT_MAX_SIGNALS (100) if not set during decode
+      related_integrations: RelatedIntegrationArray, // defaults to "undefined" if not set during decode
+      required_fields: RequiredFieldArray, // defaults to "undefined" if not set during decode
       risk_score_mapping: DefaultRiskScoreMappingArray, // defaults to empty risk score mapping array if not set during decode
       rule_name_override, // defaults to "undefined" if not set during decode
+      setup: SetupGuide, // defaults to "undefined" if not set during decode
       severity_mapping: DefaultSeverityMappingArray, // defaults to empty actions array if not set during decode
       tags: DefaultStringArray, // defaults to empty string array if not set during decode
       to: DefaultToString, // defaults to "now" if not set during decode

x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts

@@ -61,7 +61,7 @@ import {
   rule_name_override,
   timestamp_override,
   event_category_override,
-} from '../common/schemas';
+} from '../common';
 
 /**
  * All of the patch elements should default to undefined if not set

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.ts

@@ -67,6 +67,9 @@ import {
   created_by,
   namespace,
   ruleExecutionSummary,
+  RelatedIntegrationArray,
+  RequiredFieldArray,
+  SetupGuide,
 } from '../common';
 
 export const createSchema = <
@@ -412,6 +415,14 @@ const responseRequiredFields = {
   updated_by,
   created_at,
   created_by,
+
+  // NOTE: For now, Related Integrations, Required Fields and Setup Guide are supported for prebuilt
+  // rules only. We don't want to allow users to edit these 3 fields via the API. If we added them
+  // to baseParams.defaultable, they would become a part of the request schema as optional fields.
+  // This is why we add them here, in order to add them only to the response schema.
+  related_integrations: RelatedIntegrationArray,
+  required_fields: RequiredFieldArray,
+  setup: SetupGuide,
 };
 
 const responseOptionalFields = {

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts

@@ -68,6 +68,9 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem
   rule_id: 'query-rule-id',
   interval: '5m',
   exceptions_list: getListArrayMock(),
+  related_integrations: [],
+  required_fields: [],
+  setup: '',
 });
 
 export const getRulesMlSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchema => {
@@ -132,6 +135,9 @@ export const getThreatMatchingSchemaPartialMock = (enabled = false): Partial<Rul
     risk_score_mapping: [],
     name: 'Query with a rule id',
     references: [],
+    related_integrations: [],
+    required_fields: [],
+    setup: '',
     severity: 'high',
     severity_mapping: [],
     updated_by: 'elastic',

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts

@@ -74,6 +74,9 @@ import {
   timestamp_override,
   namespace,
   ruleExecutionSummary,
+  RelatedIntegrationArray,
+  RequiredFieldArray,
+  SetupGuide,
 } from '../common';
 
 import { typeAndTimelineOnlySchema, TypeAndTimelineOnly } from './type_timeline_only_schema';
@@ -111,6 +114,9 @@ export const requiredRulesSchema = t.type({
   created_by,
   version,
   exceptions_list: DefaultListArray,
+  related_integrations: RelatedIntegrationArray,
+  required_fields: RequiredFieldArray,
+  setup: SetupGuide,
 });
 
 export type RequiredRulesSchema = t.TypeOf<typeof requiredRulesSchema>;

x-pack/plugins/security_solution/cypress/objects/rule.ts

@@ -442,7 +442,9 @@ export const expectedExportedRule = (ruleResponse: Cypress.Response<RulesSchema>
     severity,
     query,
   } = ruleResponse.body;
-  const rule = {
+
+  // NOTE: Order of the properties in this object matters for the tests to work.
+  const rule: RulesSchema = {
     id,
     updated_at: updatedAt,
     updated_by: updatedBy,
@@ -469,13 +471,18 @@ export const expectedExportedRule = (ruleResponse: Cypress.Response<RulesSchema>
     version: 1,
     exceptions_list: [],
     immutable: false,
+    related_integrations: [],
+    required_fields: [],
+    setup: '',
     type: 'query',
     language: 'kuery',
     index: getIndexPatterns(),
     query,
     throttle: 'no_actions',
     actions: [],
   };
+
+  // NOTE: Order of the properties in this object matters for the tests to work.
   const details = {
     exported_count: 1,
     exported_rules_count: 1,

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/mock.ts

@@ -38,6 +38,9 @@ export const savedRuleMock: Rule = {
   max_signals: 100,
   query: "user.email: 'root@elastic.co'",
   references: [],
+  related_integrations: [],
+  required_fields: [],
+  setup: '',
   severity: 'high',
   severity_mapping: [],
   tags: ['APM'],
@@ -80,6 +83,9 @@ export const rulesMock: FetchRulesResponse = {
         'event.kind:alert and event.module:endgame and event.action:cred_theft_event and endgame.metadata.type:detection',
       filters: [],
       references: [],
+      related_integrations: [],
+      required_fields: [],
+      setup: '',
       severity: 'high',
       severity_mapping: [],
       updated_by: 'elastic',
@@ -115,6 +121,9 @@ export const rulesMock: FetchRulesResponse = {
       query: 'event.kind:alert and event.module:endgame and event.action:rules_engine_event',
       filters: [],
       references: [],
+      related_integrations: [],
+      required_fields: [],
+      setup: '',
       severity: 'medium',
       severity_mapping: [],
       updated_by: 'elastic',

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts

@@ -34,6 +34,9 @@ import {
   BulkAction,
   BulkActionEditPayload,
   ruleExecutionSummary,
+  RelatedIntegrationArray,
+  RequiredFieldArray,
+  SetupGuide,
 } from '../../../../../common/detection_engine/schemas/common';
 
 import {
@@ -102,11 +105,14 @@ export const RuleSchema = t.intersection([
     name: t.string,
     max_signals: t.number,
     references: t.array(t.string),
+    related_integrations: RelatedIntegrationArray,
+    required_fields: RequiredFieldArray,
     risk_score: t.number,
     risk_score_mapping,
     rule_id: t.string,
     severity,
     severity_mapping,
+    setup: SetupGuide,
     tags: t.array(t.string),
     type,
     to: t.string,

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule.test.tsx

@@ -67,9 +67,12 @@ describe('useRule', () => {
           max_signals: 100,
           query: "user.email: 'root@elastic.co'",
           references: [],
+          related_integrations: [],
+          required_fields: [],
           risk_score: 75,
           risk_score_mapping: [],
           rule_id: 'bbd3106e-b4b5-4d7c-a1a2-47531d6a2baf',
+          setup: '',
           severity: 'high',
           severity_mapping: [],
           tags: ['APM'],

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule_with_fallback.test.tsx

@@ -78,9 +78,12 @@ describe('useRuleWithFallback', () => {
             "name": "Test rule",
             "query": "user.email: 'root@elastic.co'",
             "references": Array [],
+            "related_integrations": Array [],
+            "required_fields": Array [],
             "risk_score": 75,
             "risk_score_mapping": Array [],
             "rule_id": "bbd3106e-b4b5-4d7c-a1a2-47531d6a2baf",
+            "setup": "",
             "severity": "high",
             "severity_mapping": Array [],
             "tags": Array [