From 5a049098197b3de121aaf404aa10064b4127a631 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 28 Jul 2020 22:43:40 -0400 Subject: [PATCH 1/4] [Security Solution][Exceptions] Use semantic version for manifest version + Scaling Tweaks (#73388) * Manifest version is semantic version * Configurable task interval * Use task interval over scheduled when provided * Fix crash on download of large artifact * Don't need to generate linux artifacts * Configurable artifact validation * Test fixes * Test fixes * Type/test fixes * Final tweaks * Remove linux endpoint exception generation from UI * Fix paging so that we stop before 10k * Fix pagination * Fix pagination test Co-authored-by: Elastic Machine --- .../common/endpoint/generate_data.ts | 2 +- .../common/endpoint/schema/common.ts | 5 +- .../common/endpoint/schema/manifest.ts | 4 +- .../exceptions/add_exception_modal/index.tsx | 2 +- .../security_solution/server/config.ts | 6 ++ .../server/endpoint/config.ts | 6 ++ .../endpoint/ingest_integration.test.ts | 25 +---- .../server/endpoint/ingest_integration.ts | 10 +- .../server/endpoint/lib/artifacts/common.ts | 4 +- .../endpoint/lib/artifacts/lists.test.ts | 12 ++- .../server/endpoint/lib/artifacts/lists.ts | 9 +- .../endpoint/lib/artifacts/manifest.test.ts | 49 +++------ .../server/endpoint/lib/artifacts/manifest.ts | 100 +++++++----------- .../server/endpoint/lib/artifacts/mocks.ts | 32 ++---- .../lib/artifacts/saved_object_mappings.ts | 8 +- .../endpoint/lib/artifacts/task.test.ts | 2 +- .../server/endpoint/lib/artifacts/task.ts | 31 ++++-- .../artifacts/download_exception_list.ts | 10 +- .../endpoint/schemas/artifacts/manifest.ts | 23 ++++ .../schemas/artifacts/saved_objects.mock.ts | 2 + .../schemas/artifacts/saved_objects.ts | 4 + .../services/artifacts/manifest_client.ts | 1 - .../manifest_manager/manifest_manager.test.ts | 82 ++++---------- .../manifest_manager/manifest_manager.ts | 62 +++++------ .../routes/__mocks__/index.ts | 2 + .../endpoint/artifacts/api_feature/data.json | 28 +---- .../apps/endpoint/policy_details.ts | 12 --- .../apis/artifacts/index.ts | 78 ++++++++++++-- 28 files changed, 284 insertions(+), 327 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest.ts diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 97ac5c9030a3d..9a92270fc9c14 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -1041,7 +1041,7 @@ export class EndpointDocGenerator { config: { artifact_manifest: { value: { - manifest_version: 'WzAsMF0=', + manifest_version: '1.0.0', schema_version: 'v1', artifacts: {}, }, diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts index 8f2ea1f8a6452..1c910927a7afa 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts @@ -23,8 +23,6 @@ export const encryptionAlgorithm = t.keyof({ export const identifier = t.string; -export const manifestVersion = t.string; - export const manifestSchemaVersion = t.keyof({ v1: null, }); @@ -34,4 +32,7 @@ export const relativeUrl = t.string; export const sha256 = t.string; +export const semanticVersion = t.string; +export type SemanticVersion = t.TypeOf; + export const size = t.number; diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts index f8bb8b70f2d5b..f03db881837d5 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts @@ -11,9 +11,9 @@ import { encryptionAlgorithm, identifier, manifestSchemaVersion, - manifestVersion, relativeUrl, sha256, + semanticVersion, size, } from './common'; @@ -50,7 +50,7 @@ export type ManifestEntryDispatchSchema = t.TypeOf { - const osDefaults = ['windows', 'macos', 'linux']; + const osDefaults = ['windows', 'macos']; if (alertData) { const osTypes = getMappedNonEcsValue({ data: alertData.nonEcsData, diff --git a/x-pack/plugins/security_solution/server/config.ts b/x-pack/plugins/security_solution/server/config.ts index f7d961ae3ec5c..e2c06ae9f931f 100644 --- a/x-pack/plugins/security_solution/server/config.ts +++ b/x-pack/plugins/security_solution/server/config.ts @@ -29,6 +29,12 @@ export const configSchema = schema.object({ from: schema.string({ defaultValue: 'now-15m' }), to: schema.string({ defaultValue: 'now' }), }), + + /** + * Artifacts Configuration + */ + packagerTaskInterval: schema.string({ defaultValue: '60s' }), + validateArtifactDownloads: schema.boolean({ defaultValue: true }), }); export const createConfig$ = (context: PluginInitializerContext) => diff --git a/x-pack/plugins/security_solution/server/endpoint/config.ts b/x-pack/plugins/security_solution/server/endpoint/config.ts index 908e14468c5c7..6a3644f7aaf71 100644 --- a/x-pack/plugins/security_solution/server/endpoint/config.ts +++ b/x-pack/plugins/security_solution/server/endpoint/config.ts @@ -27,6 +27,12 @@ export const EndpointConfigSchema = schema.object({ from: schema.string({ defaultValue: 'now-15m' }), to: schema.string({ defaultValue: 'now' }), }), + + /** + * Artifacts Configuration + */ + packagerTaskInterval: schema.string({ defaultValue: '60s' }), + validateArtifactDownloads: schema.boolean({ defaultValue: true }), }); export function createConfig$(context: PluginInitializerContext) { diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts index be749b2ebd25a..03999715dfc71 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts @@ -12,7 +12,6 @@ import { ManifestManagerMockType, } from './services/artifacts/manifest_manager/manifest_manager.mock'; import { getPackageConfigCreateCallback } from './ingest_integration'; -import { ManifestConstants } from './lib/artifacts'; describe('ingest_integration tests ', () => { describe('ingest_integration sanity checks', () => { @@ -30,16 +29,6 @@ describe('ingest_integration tests ', () => { expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual({ artifacts: { - 'endpoint-exceptionlist-linux-v1': { - compression_algorithm: 'zlib', - decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - decoded_size: 14, - encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - encoded_size: 22, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - }, 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', @@ -61,7 +50,7 @@ describe('ingest_integration tests ', () => { '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', }, }, - manifest_version: 'a9b7ef358a363f327f479e31efc4f228b2277a7fb4d1914ca9b4e7ca9ffcf537', + manifest_version: '1.0.0', schema_version: 'v1', }); }); @@ -70,9 +59,7 @@ describe('ingest_integration tests ', () => { const logger = loggingSystemMock.create().get('ingest_integration.test'); const manifestManager = getManifestManagerMock(); manifestManager.pushArtifacts = jest.fn().mockResolvedValue([new Error('error updating')]); - const lastComputed = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); + const lastComputed = await manifestManager.getLastComputedManifest(); const callback = getPackageConfigCreateCallback(logger, manifestManager); const policyConfig = createNewPackageConfigMock(); @@ -90,9 +77,7 @@ describe('ingest_integration tests ', () => { const manifestManager = getManifestManagerMock({ mockType: ManifestManagerMockType.InitialSystemState, }); - const lastComputed = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); + const lastComputed = await manifestManager.getLastComputedManifest(); expect(lastComputed).toEqual(null); manifestManager.buildNewManifest = jest.fn().mockRejectedValue(new Error('abcd')); @@ -107,9 +92,7 @@ describe('ingest_integration tests ', () => { test('subsequent policy creations succeed', async () => { const logger = loggingSystemMock.create().get('ingest_integration.test'); const manifestManager = getManifestManagerMock(); - const lastComputed = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); + const lastComputed = await manifestManager.getLastComputedManifest(); manifestManager.buildNewManifest = jest.fn().mockResolvedValue(lastComputed); // no diffs const callback = getPackageConfigCreateCallback(logger, manifestManager); diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index 11d4b12d0b76a..695267f322857 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -10,7 +10,7 @@ import { factory as policyConfigFactory } from '../../common/endpoint/models/pol import { NewPolicyData } from '../../common/endpoint/types'; import { ManifestManager } from './services/artifacts'; import { Manifest } from './lib/artifacts'; -import { reportErrors, ManifestConstants } from './lib/artifacts/common'; +import { reportErrors } from './lib/artifacts/common'; import { InternalArtifactCompleteSchema } from './schemas/artifacts'; import { manifestDispatchSchema } from '../../common/endpoint/schema/manifest'; @@ -18,14 +18,14 @@ const getManifest = async (logger: Logger, manifestManager: ManifestManager): Pr let manifest: Manifest | null = null; try { - manifest = await manifestManager.getLastComputedManifest(ManifestConstants.SCHEMA_VERSION); + manifest = await manifestManager.getLastComputedManifest(); // If we have not yet computed a manifest, then we have to do so now. This should only happen // once. if (manifest == null) { // New computed manifest based on current state of exception list - const newManifest = await manifestManager.buildNewManifest(ManifestConstants.SCHEMA_VERSION); - const diffs = newManifest.diff(Manifest.getDefault(ManifestConstants.SCHEMA_VERSION)); + const newManifest = await manifestManager.buildNewManifest(); + const diffs = newManifest.diff(Manifest.getDefault()); // Compress new artifacts const adds = diffs.filter((diff) => diff.type === 'add').map((diff) => diff.id); @@ -63,7 +63,7 @@ const getManifest = async (logger: Logger, manifestManager: ManifestManager): Pr logger.error(err); } - return manifest ?? Manifest.getDefault(ManifestConstants.SCHEMA_VERSION); + return manifest ?? Manifest.getDefault(); }; /** diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts index 7298a9bfa72a6..7f90aa7b91063 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -13,13 +13,11 @@ import { export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist', SAVED_OBJECT_TYPE: 'endpoint:user-artifact', - SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'], - SCHEMA_VERSION: 'v1', + SUPPORTED_OPERATING_SYSTEMS: ['macos', 'windows'], }; export const ManifestConstants = { SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest', - SCHEMA_VERSION: 'v1', }; export const getArtifactId = (artifact: InternalArtifactSchema) => { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index bb8b4fb3d5ce7..fea3b2b9a4526 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -314,21 +314,23 @@ describe('buildEventTypeSignal', () => { test('it should convert the exception lists response to the proper endpoint format while paging', async () => { // The first call returns two exceptions const first = getFoundExceptionListItemSchemaMock(); + first.per_page = 2; + first.total = 4; first.data.push(getExceptionListItemSchemaMock()); // The second call returns two exceptions const second = getFoundExceptionListItemSchemaMock(); + second.per_page = 2; + second.total = 4; second.data.push(getExceptionListItemSchemaMock()); - // The third call returns no exceptions, paging stops - const third = getFoundExceptionListItemSchemaMock(); - third.data = []; mockExceptionClient.findExceptionListItem = jest .fn() .mockReturnValueOnce(first) - .mockReturnValueOnce(second) - .mockReturnValueOnce(third); + .mockReturnValueOnce(second); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); + // Expect 2 exceptions, the first two calls returned the same exception list items expect(resp.entries.length).toEqual(2); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 5998a88527f2f..e41781dd605a0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -79,10 +79,10 @@ export async function getFullEndpointExceptionList( schemaVersion: string ): Promise { const exceptions: WrappedTranslatedExceptionList = { entries: [] }; - let numResponses = 0; let page = 1; + let paging = true; - do { + while (paging) { const response = await eClient.findExceptionListItem({ listId: ENDPOINT_LIST_ID, namespaceType: 'agnostic', @@ -94,17 +94,16 @@ export async function getFullEndpointExceptionList( }); if (response?.data !== undefined) { - numResponses = response.data.length; - exceptions.entries = exceptions.entries.concat( translateToEndpointExceptions(response, schemaVersion) ); + paging = (page - 1) * 100 + response.data.length < response.total; page++; } else { break; } - } while (numResponses > 0); + } const [validated, errors] = validate(exceptions, wrappedTranslatedExceptionList); if (errors != null) { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index 95587c6fc105d..3d70f7266277f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -6,7 +6,7 @@ import { ManifestSchemaVersion } from '../../../../common/endpoint/schema/common'; import { InternalArtifactCompleteSchema } from '../../schemas'; -import { ManifestConstants, getArtifactId } from './common'; +import { getArtifactId } from './common'; import { Manifest } from './manifest'; import { getMockArtifacts, @@ -30,29 +30,21 @@ describe('manifest', () => { }); test('Can create manifest with valid schema version', () => { - const manifest = new Manifest('v1'); + const manifest = new Manifest(); expect(manifest).toBeInstanceOf(Manifest); }); test('Cannot create manifest with invalid schema version', () => { expect(() => { - new Manifest('abcd' as ManifestSchemaVersion); + new Manifest({ + schemaVersion: 'abcd' as ManifestSchemaVersion, + }); }).toThrow(); }); test('Empty manifest transforms correctly to expected endpoint format', async () => { expect(emptyManifest.toEndpointFormat()).toStrictEqual({ artifacts: { - 'endpoint-exceptionlist-linux-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - decoded_size: 14, - encoded_size: 22, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - }, 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', encryption_algorithm: 'none', @@ -74,7 +66,7 @@ describe('manifest', () => { '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', }, }, - manifest_version: 'a9b7ef358a363f327f479e31efc4f228b2277a7fb4d1914ca9b4e7ca9ffcf537', + manifest_version: '1.0.0', schema_version: 'v1', }); }); @@ -82,16 +74,6 @@ describe('manifest', () => { test('Manifest transforms correctly to expected endpoint format', async () => { expect(manifest1.toEndpointFormat()).toStrictEqual({ artifacts: { - 'endpoint-exceptionlist-linux-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', - decoded_size: 432, - encoded_size: 147, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - }, 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', encryption_algorithm: 'none', @@ -113,15 +95,16 @@ describe('manifest', () => { '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', }, }, - manifest_version: 'a7f4760bfa2662e85e30fe4fb8c01b4c4a20938c76ab21d3c5a3e781e547cce7', + manifest_version: '1.0.0', schema_version: 'v1', }); }); test('Manifest transforms correctly to expected saved object format', async () => { expect(manifest1.toSavedObject()).toStrictEqual({ + schemaVersion: 'v1', + semanticVersion: '1.0.0', ids: [ - 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', ], @@ -133,12 +116,12 @@ describe('manifest', () => { expect(diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', type: 'delete', }, { id: - 'endpoint-exceptionlist-linux-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', + 'endpoint-exceptionlist-macos-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', type: 'add', }, ]); @@ -154,7 +137,6 @@ describe('manifest', () => { const entries = manifest1.getEntries(); const keys = Object.keys(entries); expect(keys).toEqual([ - 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', ]); @@ -168,13 +150,8 @@ describe('manifest', () => { }); test('Manifest can be created from list of artifacts', async () => { - const oldManifest = new Manifest(ManifestConstants.SCHEMA_VERSION); - const manifest = Manifest.fromArtifacts(artifacts, 'v1', oldManifest); - expect( - manifest.contains( - 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' - ) - ).toEqual(true); + const oldManifest = new Manifest(); + const manifest = Manifest.fromArtifacts(artifacts, oldManifest); expect( manifest.contains( 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index 6ece2bf0f48e8..9e0e940ea9a1d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -3,8 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - -import { createHash } from 'crypto'; +import semver from 'semver'; import { validate } from '../../../../common/validate'; import { InternalArtifactSchema, @@ -13,13 +12,15 @@ import { InternalArtifactCompleteSchema, } from '../../schemas/artifacts'; import { - manifestSchemaVersion, ManifestSchemaVersion, + SemanticVersion, + semanticVersion, } from '../../../../common/endpoint/schema/common'; -import { ManifestSchema, manifestSchema } from '../../../../common/endpoint/schema/manifest'; +import { manifestSchema, ManifestSchema } from '../../../../common/endpoint/schema/manifest'; import { ManifestEntry } from './manifest_entry'; import { maybeCompressArtifact, isCompressed } from './lists'; import { getArtifactId } from './common'; +import { ManifestVersion, manifestVersion } from '../../schemas/artifacts/manifest'; export interface ManifestDiff { type: string; @@ -28,37 +29,39 @@ export interface ManifestDiff { export class Manifest { private entries: Record; - private schemaVersion: ManifestSchemaVersion; - - // For concurrency control - private version: string | undefined; + private version: ManifestVersion; - constructor(schemaVersion: string, version?: string) { + constructor(version?: Partial) { this.entries = {}; - this.version = version; - const [validated, errors] = validate( - (schemaVersion as unknown) as object, - manifestSchemaVersion - ); + const decodedVersion = { + schemaVersion: version?.schemaVersion ?? 'v1', + semanticVersion: version?.semanticVersion ?? '1.0.0', + soVersion: version?.soVersion, + }; + const [validated, errors] = validate(decodedVersion, manifestVersion); if (errors != null || validated === null) { - throw new Error(`Invalid manifest schema version: ${schemaVersion}`); + throw new Error(errors ?? 'Invalid version format.'); } - this.schemaVersion = validated; + this.version = validated; } - public static getDefault(schemaVersion: string) { - return new Manifest(schemaVersion); + public static getDefault(schemaVersion?: ManifestSchemaVersion) { + return new Manifest({ schemaVersion, semanticVersion: '1.0.0' }); } public static fromArtifacts( artifacts: InternalArtifactCompleteSchema[], - schemaVersion: string, - oldManifest: Manifest + oldManifest: Manifest, + schemaVersion?: ManifestSchemaVersion ): Manifest { - const manifest = new Manifest(schemaVersion, oldManifest.getSoVersion()); + const manifest = new Manifest({ + schemaVersion, + semanticVersion: oldManifest.getSemanticVersion(), + soVersion: oldManifest.getSavedObjectVersion(), + }); artifacts.forEach((artifact) => { const id = getArtifactId(artifact); const existingArtifact = oldManifest.getArtifact(id); @@ -71,25 +74,12 @@ export class Manifest { return manifest; } - public static fromPkgConfig(manifestPkgConfig: ManifestSchema): Manifest | null { - if (manifestSchema.is(manifestPkgConfig)) { - const manifest = new Manifest(manifestPkgConfig.schema_version); - for (const [identifier, artifactRecord] of Object.entries(manifestPkgConfig.artifacts)) { - const artifact = { - identifier, - compressionAlgorithm: artifactRecord.compression_algorithm, - encryptionAlgorithm: artifactRecord.encryption_algorithm, - decodedSha256: artifactRecord.decoded_sha256, - decodedSize: artifactRecord.decoded_size, - encodedSha256: artifactRecord.encoded_sha256, - encodedSize: artifactRecord.encoded_size, - }; - manifest.addEntry(artifact); - } - return manifest; - } else { - return null; + public bumpSemanticVersion() { + const newSemanticVersion = semver.inc(this.getSemanticVersion(), 'patch'); + if (!semanticVersion.is(newSemanticVersion)) { + throw new Error(`Invalid semver: ${newSemanticVersion}`); } + this.version.semanticVersion = newSemanticVersion; } public async compressArtifact(id: string): Promise { @@ -112,30 +102,16 @@ export class Manifest { return null; } - public equals(manifest: Manifest): boolean { - return this.getSha256() === manifest.getSha256(); - } - - public getSha256(): string { - let sha256 = createHash('sha256'); - Object.keys(this.entries) - .sort() - .forEach((docId) => { - sha256 = sha256.update(docId); - }); - return sha256.digest('hex'); - } - public getSchemaVersion(): ManifestSchemaVersion { - return this.schemaVersion; + return this.version.schemaVersion; } - public getSoVersion(): string | undefined { - return this.version; + public getSavedObjectVersion(): string | undefined { + return this.version.soVersion; } - public setSoVersion(version: string) { - this.version = version; + public getSemanticVersion(): SemanticVersion { + return this.version.semanticVersion; } public addEntry(artifact: InternalArtifactSchema) { @@ -179,8 +155,8 @@ export class Manifest { public toEndpointFormat(): ManifestSchema { const manifestObj: ManifestSchema = { - manifest_version: this.getSha256(), - schema_version: this.schemaVersion, + manifest_version: this.getSemanticVersion(), + schema_version: this.getSchemaVersion(), artifacts: {}, }; @@ -198,7 +174,9 @@ export class Manifest { public toSavedObject(): InternalManifestSchema { return { - ids: Object.keys(this.entries), + ids: Object.keys(this.getEntries()), + schemaVersion: this.getSchemaVersion(), + semanticVersion: this.getSemanticVersion(), }; } } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts index 0ec6cb2bd61b3..62fff4715b562 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts @@ -29,7 +29,7 @@ export const getMockArtifactsWithDiff = async (opts?: { compress: boolean }) => return Promise.all( ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map>( async (os) => { - if (os === 'linux') { + if (os === 'macos') { return getInternalArtifactMockWithDiffs(os, 'v1'); } return getInternalArtifactMock(os, 'v1', opts); @@ -49,21 +49,21 @@ export const getEmptyMockArtifacts = async (opts?: { compress: boolean }) => { }; export const getMockManifest = async (opts?: { compress: boolean }) => { - const manifest = new Manifest('v1'); + const manifest = new Manifest(); const artifacts = await getMockArtifacts(opts); artifacts.forEach((artifact) => manifest.addEntry(artifact)); return manifest; }; export const getMockManifestWithDiffs = async (opts?: { compress: boolean }) => { - const manifest = new Manifest('v1'); + const manifest = new Manifest(); const artifacts = await getMockArtifactsWithDiff(opts); artifacts.forEach((artifact) => manifest.addEntry(artifact)); return manifest; }; export const getEmptyMockManifest = async (opts?: { compress: boolean }) => { - const manifest = new Manifest('v1'); + const manifest = new Manifest(); const artifacts = await getEmptyMockArtifacts(opts); artifacts.forEach((artifact) => manifest.addEntry(artifact)); return manifest; @@ -74,16 +74,6 @@ export const createPackageConfigWithInitialManifestMock = (): PackageConfig => { packageConfig.inputs[0].config!.artifact_manifest = { value: { artifacts: { - 'endpoint-exceptionlist-linux-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - decoded_size: 14, - encoded_size: 22, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - }, 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', encryption_algorithm: 'none', @@ -105,7 +95,7 @@ export const createPackageConfigWithInitialManifestMock = (): PackageConfig => { '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', }, }, - manifest_version: 'a9b7ef358a363f327f479e31efc4f228b2277a7fb4d1914ca9b4e7ca9ffcf537', + manifest_version: '1.0.0', schema_version: 'v1', }, }; @@ -117,16 +107,6 @@ export const createPackageConfigWithManifestMock = (): PackageConfig => { packageConfig.inputs[0].config!.artifact_manifest = { value: { artifacts: { - 'endpoint-exceptionlist-linux-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: '0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', - encoded_sha256: '57941169bb2c5416f9bd7224776c8462cb9a2be0fe8b87e6213e77a1d29be824', - decoded_size: 292, - encoded_size: 131, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', - }, 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', encryption_algorithm: 'none', @@ -148,7 +128,7 @@ export const createPackageConfigWithManifestMock = (): PackageConfig => { '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', }, }, - manifest_version: '520f6cf88b3f36a065c6ca81058d5f8690aadadf6fe857f8dec4cc37589e7283', + manifest_version: '1.0.1', schema_version: 'v1', }, }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts index 0fb433df95de3..734304516e37e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts @@ -55,7 +55,13 @@ export const manifestSavedObjectMappings: SavedObjectsType['mappings'] = { type: 'date', index: false, }, - // array of doc ids + schemaVersion: { + type: 'keyword', + }, + semanticVersion: { + type: 'keyword', + index: false, + }, ids: { type: 'keyword', index: false, diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts index daa8a7dd83ee0..32d58da5c3b78 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts @@ -38,7 +38,7 @@ describe('task', () => { taskManager: mockTaskManagerSetup, }); const mockTaskManagerStart = taskManagerMock.createStart(); - manifestTask.start({ taskManager: mockTaskManagerStart }); + await manifestTask.start({ taskManager: mockTaskManagerStart }); expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled(); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index ba164059866ea..4f2dbdf7644e2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - import { Logger } from 'src/core/server'; import { ConcreteTaskInstance, @@ -11,7 +10,7 @@ import { TaskManagerStartContract, } from '../../../../../task_manager/server'; import { EndpointAppContext } from '../../types'; -import { reportErrors, ManifestConstants } from './common'; +import { reportErrors } from './common'; import { InternalArtifactCompleteSchema } from '../../schemas/artifacts'; export const ManifestTaskConstants = { @@ -45,7 +44,23 @@ export class ManifestTask { createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { return { run: async () => { + const taskInterval = (await this.endpointAppContext.config()).packagerTaskInterval; await this.runTask(taskInstance.id); + const nextRun = new Date(); + if (taskInterval.endsWith('s')) { + const seconds = parseInt(taskInterval.slice(0, -1), 10); + nextRun.setSeconds(nextRun.getSeconds() + seconds); + } else if (taskInterval.endsWith('m')) { + const minutes = parseInt(taskInterval.slice(0, -1), 10); + nextRun.setMinutes(nextRun.getMinutes() + minutes); + } else { + this.logger.error(`Invalid task interval: ${taskInterval}`); + return; + } + return { + state: {}, + runAt: nextRun, + }; }, cancel: async () => {}, }; @@ -61,7 +76,7 @@ export class ManifestTask { taskType: ManifestTaskConstants.TYPE, scope: ['securitySolution'], schedule: { - interval: '60s', + interval: (await this.endpointAppContext.config()).packagerTaskInterval, }, state: {}, params: { version: ManifestTaskConstants.VERSION }, @@ -92,19 +107,14 @@ export class ManifestTask { try { // Last manifest we computed, which was saved to ES - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); + const oldManifest = await manifestManager.getLastComputedManifest(); if (oldManifest == null) { this.logger.debug('User manifest not available yet.'); return; } // New computed manifest based on current state of exception list - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest - ); + const newManifest = await manifestManager.buildNewManifest(oldManifest); const diffs = newManifest.diff(oldManifest); // Compress new artifacts @@ -131,6 +141,7 @@ export class ManifestTask { // Commit latest manifest state, if different if (diffs.length) { + newManifest.bumpSemanticVersion(); const error = await manifestManager.commit(newManifest); if (error) { throw error; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 38e900c4d5015..d825841f1e257 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - import { IRouter, SavedObjectsClientContract, @@ -14,7 +13,6 @@ import { import LRU from 'lru-cache'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { authenticateAgentWithAccessToken } from '../../../../../ingest_manager/server/services/agents/authenticate'; -import { validate } from '../../../../common/validate'; import { LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG } from '../../../../common/endpoint/constants'; import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; import { ArtifactConstants } from '../../lib/artifacts'; @@ -63,6 +61,7 @@ export function registerDownloadExceptionListRoute( } } + const validateDownload = (await endpointContext.config()).validateArtifactDownloads; const buildAndValidateResponse = (artName: string, body: Buffer): IKibanaResponse => { const artifact: HttpResponseOptions = { body, @@ -72,11 +71,10 @@ export function registerDownloadExceptionListRoute( }, }; - const [validated, errors] = validate(artifact, downloadArtifactResponseSchema); - if (errors !== null || validated === null) { - return res.internalError({ body: errors! }); + if (validateDownload && !downloadArtifactResponseSchema.is(artifact)) { + return res.internalError({ body: 'Artifact failed to validate.' }); } else { - return res.ok((validated as unknown) as HttpResponseOptions); + return res.ok(artifact); } }; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest.ts new file mode 100644 index 0000000000000..707d4c1374fe2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { manifestSchemaVersion, semanticVersion } from '../../../../common/endpoint/schema/common'; + +const optionalVersions = t.partial({ + soVersion: t.string, +}); + +export const manifestVersion = t.intersection([ + optionalVersions, + t.exact( + t.type({ + schemaVersion: manifestSchemaVersion, + semanticVersion, + }) + ), +]); +export type ManifestVersion = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts index d95627601a183..ae565f785c399 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -53,4 +53,6 @@ export const getInternalArtifactMockWithDiffs = async ( export const getInternalManifestMock = (): InternalManifestSchema => ({ ids: [], + schemaVersion: 'v1', + semanticVersion: '1.0.0', }); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts index 4dea916dcb436..56f247b65d802 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -9,8 +9,10 @@ import { compressionAlgorithm, encryptionAlgorithm, identifier, + semanticVersion, sha256, size, + manifestSchemaVersion, } from '../../../../common/endpoint/schema/common'; import { created } from './common'; @@ -58,6 +60,8 @@ export type InternalArtifactCreateSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts index 385f115e6301a..e55243f0650a5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - import { SavedObject, SavedObjectsClientContract, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index c838f772fb66b..d99d6a959d7aa 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -16,22 +16,17 @@ describe('manifest_manager', () => { describe('ManifestManager sanity checks', () => { test('ManifestManager can retrieve and diff manifests', async () => { const manifestManager = getManifestManagerMock(); - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest! - ); + const oldManifest = await manifestManager.getLastComputedManifest(); + const newManifest = await manifestManager.buildNewManifest(oldManifest!); expect(newManifest.diff(oldManifest!)).toEqual([ { id: - 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', type: 'delete', }, { id: - 'endpoint-exceptionlist-linux-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', + 'endpoint-exceptionlist-macos-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', type: 'add', }, ]); @@ -40,23 +35,18 @@ describe('manifest_manager', () => { test('ManifestManager populates cache properly', async () => { const cache = new LRU({ max: 10, maxAge: 1000 * 60 * 60 }); const manifestManager = getManifestManagerMock({ cache }); - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest! - ); + const oldManifest = await manifestManager.getLastComputedManifest(); + const newManifest = await manifestManager.buildNewManifest(oldManifest!); const diffs = newManifest.diff(oldManifest!); expect(diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', type: 'delete', }, { id: - 'endpoint-exceptionlist-linux-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', + 'endpoint-exceptionlist-macos-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', type: 'add', }, ]); @@ -104,13 +94,8 @@ describe('manifest_manager', () => { test('ManifestManager cannot dispatch incomplete (uncompressed) artifact', async () => { const packageConfigService = createPackageConfigServiceMock(); const manifestManager = getManifestManagerMock({ packageConfigService }); - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest! - ); + const oldManifest = await manifestManager.getLastComputedManifest(); + const newManifest = await manifestManager.buildNewManifest(oldManifest!); const dispatchErrors = await manifestManager.tryDispatch(newManifest); expect(dispatchErrors.length).toEqual(1); expect(dispatchErrors[0].message).toEqual('Invalid manifest'); @@ -119,17 +104,14 @@ describe('manifest_manager', () => { test('ManifestManager can dispatch manifest', async () => { const packageConfigService = createPackageConfigServiceMock(); const manifestManager = getManifestManagerMock({ packageConfigService }); - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest! - ); + const oldManifest = await manifestManager.getLastComputedManifest(); + const newManifest = await manifestManager.buildNewManifest(oldManifest!); const diffs = newManifest.diff(oldManifest!); const newArtifactId = diffs[1].id; await newManifest.compressArtifact(newArtifactId); + newManifest.bumpSemanticVersion(); + const dispatchErrors = await manifestManager.tryDispatch(newManifest); expect(dispatchErrors).toEqual([]); @@ -140,10 +122,10 @@ describe('manifest_manager', () => { expect( packageConfigService.update.mock.calls[0][2].inputs[0].config!.artifact_manifest.value ).toEqual({ - manifest_version: '520f6cf88b3f36a065c6ca81058d5f8690aadadf6fe857f8dec4cc37589e7283', + manifest_version: '1.0.1', schema_version: 'v1', artifacts: { - 'endpoint-exceptionlist-linux-v1': { + 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', encryption_algorithm: 'none', decoded_sha256: '0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', @@ -151,17 +133,7 @@ describe('manifest_manager', () => { decoded_size: 292, encoded_size: 131, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', - }, - 'endpoint-exceptionlist-macos-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', - decoded_size: 432, - encoded_size: 147, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', }, 'endpoint-exceptionlist-windows-v1': { compression_algorithm: 'zlib', @@ -180,17 +152,14 @@ describe('manifest_manager', () => { test('ManifestManager fails to dispatch on conflict', async () => { const packageConfigService = createPackageConfigServiceMock(); const manifestManager = getManifestManagerMock({ packageConfigService }); - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest! - ); + const oldManifest = await manifestManager.getLastComputedManifest(); + const newManifest = await manifestManager.buildNewManifest(oldManifest!); const diffs = newManifest.diff(oldManifest!); const newArtifactId = diffs[1].id; await newManifest.compressArtifact(newArtifactId); + newManifest.bumpSemanticVersion(); + packageConfigService.update.mockRejectedValueOnce({ status: 409 }); const dispatchErrors = await manifestManager.tryDispatch(newManifest); expect(dispatchErrors).toEqual([{ status: 409 }]); @@ -202,13 +171,8 @@ describe('manifest_manager', () => { savedObjectsClient, }); - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest! - ); + const oldManifest = await manifestManager.getLastComputedManifest(); + const newManifest = await manifestManager.buildNewManifest(oldManifest!); const diffs = newManifest.diff(oldManifest!); const oldArtifactId = diffs[0].id; const newArtifactId = diffs[1].id; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index b52c51ba789af..217fd6de2ba68 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - +import semver from 'semver'; import { Logger, SavedObjectsClientContract } from 'src/core/server'; import LRU from 'lru-cache'; import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server'; @@ -13,7 +13,6 @@ import { manifestDispatchSchema } from '../../../../../common/endpoint/schema/ma import { ArtifactConstants, - ManifestConstants, Manifest, buildArtifact, getFullEndpointExceptionList, @@ -52,6 +51,7 @@ export class ManifestManager { protected savedObjectsClient: SavedObjectsClientContract; protected logger: Logger; protected cache: LRU; + protected schemaVersion: ManifestSchemaVersion; constructor(context: ManifestManagerContext) { this.artifactClient = context.artifactClient; @@ -60,28 +60,27 @@ export class ManifestManager { this.savedObjectsClient = context.savedObjectsClient; this.logger = context.logger; this.cache = context.cache; + this.schemaVersion = 'v1'; } /** - * Gets a ManifestClient for the provided schemaVersion. + * Gets a ManifestClient for this manager's schemaVersion. * - * @param schemaVersion The schema version of the manifest. - * @returns {ManifestClient} A ManifestClient scoped to the provided schemaVersion. + * @returns {ManifestClient} A ManifestClient scoped to the appropriate schemaVersion. */ - protected getManifestClient(schemaVersion: string): ManifestClient { - return new ManifestClient(this.savedObjectsClient, schemaVersion as ManifestSchemaVersion); + protected getManifestClient(): ManifestClient { + return new ManifestClient(this.savedObjectsClient, this.schemaVersion); } /** * Builds an array of artifacts (one per supported OS) based on the current * state of exception-list-agnostic SOs. * - * @param schemaVersion The schema version of the artifact * @returns {Promise} An array of uncompressed artifacts built from exception-list-agnostic SOs. * @throws Throws/rejects if there are errors building the list. */ protected async buildExceptionListArtifacts( - schemaVersion: string + artifactSchemaVersion?: string ): Promise { return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.reduce< Promise @@ -89,10 +88,10 @@ export class ManifestManager { const exceptionList = await getFullEndpointExceptionList( this.exceptionListClient, os, - schemaVersion + artifactSchemaVersion ?? 'v1' ); const artifacts = await acc; - const artifact = await buildArtifact(exceptionList, os, schemaVersion); + const artifact = await buildArtifact(exceptionList, os, artifactSchemaVersion ?? 'v1'); return Promise.resolve([...artifacts, artifact]); }, Promise.resolve([])); } @@ -168,20 +167,23 @@ export class ManifestManager { * Returns the last computed manifest based on the state of the * user-artifact-manifest SO. * - * @param schemaVersion The schema version of the manifest. * @returns {Promise} The last computed manifest, or null if does not exist. * @throws Throws/rejects if there is an unexpected error retrieving the manifest. */ - public async getLastComputedManifest(schemaVersion: string): Promise { + public async getLastComputedManifest(): Promise { try { - const manifestClient = this.getManifestClient(schemaVersion); + const manifestClient = this.getManifestClient(); const manifestSo = await manifestClient.getManifest(); if (manifestSo.version === undefined) { throw new Error('No version returned for manifest.'); } - const manifest = new Manifest(schemaVersion, manifestSo.version); + const manifest = new Manifest({ + schemaVersion: this.schemaVersion, + semanticVersion: manifestSo.attributes.semanticVersion, + soVersion: manifestSo.version, + }); for (const id of manifestSo.attributes.ids) { const artifactSo = await this.artifactClient.getArtifact(id); @@ -199,22 +201,17 @@ export class ManifestManager { /** * Builds a new manifest based on the current user exception list. * - * @param schemaVersion The schema version of the manifest. * @param baselineManifest A baseline manifest to use for initializing pre-existing artifacts. * @returns {Promise} A new Manifest object reprenting the current exception list. */ - public async buildNewManifest( - schemaVersion: string, - baselineManifest?: Manifest - ): Promise { + public async buildNewManifest(baselineManifest?: Manifest): Promise { // Build new exception list artifacts - const artifacts = await this.buildExceptionListArtifacts(ArtifactConstants.SCHEMA_VERSION); + const artifacts = await this.buildExceptionListArtifacts(); // Build new manifest const manifest = Manifest.fromArtifacts( artifacts, - ManifestConstants.SCHEMA_VERSION, - baselineManifest ?? Manifest.getDefault(schemaVersion) + baselineManifest ?? Manifest.getDefault(this.schemaVersion) ); return manifest; @@ -247,14 +244,12 @@ export class ManifestManager { for (const packageConfig of items) { const { id, revision, updated_at, updated_by, ...newPackageConfig } = packageConfig; if (newPackageConfig.inputs.length > 0 && newPackageConfig.inputs[0].config !== undefined) { - const artifactManifest = newPackageConfig.inputs[0].config.artifact_manifest ?? { + const oldManifest = newPackageConfig.inputs[0].config.artifact_manifest ?? { value: {}, }; - const oldManifest = - Manifest.fromPkgConfig(artifactManifest.value) ?? - Manifest.getDefault(ManifestConstants.SCHEMA_VERSION); - if (!manifest.equals(oldManifest)) { + const newManifestVersion = manifest.getSemanticVersion(); + if (semver.gt(newManifestVersion, oldManifest.value.manifest_version)) { newPackageConfig.inputs[0].config.artifact_manifest = { value: serializedManifest, }; @@ -262,7 +257,7 @@ export class ManifestManager { try { await this.packageConfigService.update(this.savedObjectsClient, id, newPackageConfig); this.logger.debug( - `Updated package config ${id} with manifest version ${manifest.getSha256()}` + `Updated package config ${id} with manifest version ${manifest.getSemanticVersion()}` ); } catch (err) { errors.push(err); @@ -274,8 +269,7 @@ export class ManifestManager { errors.push(new Error(`Package config ${id} has no config.`)); } } - - paging = page * items.length < total; + paging = (page - 1) * 20 + items.length < total; page++; } @@ -290,11 +284,11 @@ export class ManifestManager { */ public async commit(manifest: Manifest): Promise { try { - const manifestClient = this.getManifestClient(manifest.getSchemaVersion()); + const manifestClient = this.getManifestClient(); // Commit the new manifest const manifestSo = manifest.toSavedObject(); - const version = manifest.getSoVersion(); + const version = manifest.getSavedObjectVersion(); if (version == null) { await manifestClient.createManifest(manifestSo); @@ -304,7 +298,7 @@ export class ManifestManager { }); } - this.logger.info(`Committed manifest ${manifest.getSha256()}`); + this.logger.info(`Committed manifest ${manifest.getSemanticVersion()}`); } catch (err) { return err; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts index bfaab096a5013..196d816b6b257 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts @@ -25,6 +25,8 @@ export const createMockConfig = () => ({ from: 'now-15m', to: 'now', }, + packagerTaskInterval: '60s', + validateArtifactDownloads: true, }); export const mockGetCurrentUser = { diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index 47390f0428742..3af1070597671 100644 --- a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -1,28 +1,3 @@ -{ - "type": "doc", - "value": { - "id": "endpoint:user-artifact:endpoint-exceptionlist-linux-v1-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", - "index": ".kibana", - "source": { - "references": [ - ], - "endpoint:user-artifact": { - "body": "eJylkM8KwjAMxl9Fci59gN29iicvMqR02QjUbiSpKGPvbiw6ETwpuX1/fh9kBszKhALNcQa9TQgNCJ2nhOA+vJ4wdWaGqJSHPY8RRXxPCb3QkJEtP07IQUe2GOWYSoedqU8qXq16ikGqeAmpPNRtCqIU3WbnDx4WN38d/WvhQqmCXzDlIlojP9CsjLC0bqWtHwhaGN/1jHVkae3u+6N6Sg==", - "created": 1593016187465, - "compressionAlgorithm": "zlib", - "encryptionAlgorithm": "none", - "identifier": "endpoint-exceptionlist-linux-v1", - "encodedSha256": "5caaeabcb7864d47157fc7c28d5a7398b4f6bbaaa565d789c02ee809253b7613", - "encodedSize": 160, - "decodedSha256": "d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", - "decodedSize": 358 - }, - "type": "endpoint:user-artifact", - "updated_at": "2020-06-24T16:29:47.584Z" - } - } -} - { "type": "doc", "value": { @@ -83,8 +58,9 @@ ], "endpoint:user-artifact-manifest": { "created": 1593183699663, + "schemaVersion": "v1", + "semanticVersion": "1.0.1", "ids": [ - "endpoint-exceptionlist-linux-v1-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", "endpoint-exceptionlist-macos-v1-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658", "endpoint-exceptionlist-windows-v1-8d2bcc37e82fad5d06e2c9e4bd96793ea8905ace1d528a57d0d0579ecc8c647e" ] diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index ca84384390a29..d4947222a6cc0 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -118,18 +118,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }, artifact_manifest: { artifacts: { - 'endpoint-exceptionlist-linux-v1': { - compression_algorithm: 'zlib', - decoded_sha256: - 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - decoded_size: 14, - encoded_sha256: - 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - encoded_size: 22, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - }, 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', decoded_sha256: diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/artifacts/index.ts b/x-pack/test/security_solution_endpoint_api_int/apis/artifacts/index.ts index a4a8de418157f..d5106f5549924 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/artifacts/index.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/artifacts/index.ts @@ -83,24 +83,24 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with list items', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/8d2bcc37e82fad5d06e2c9e4bd96793ea8905ace1d528a57d0d0579ecc8c647e' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() .expect(200) .expect((response) => { - expect(response.body.byteLength).to.equal(160); + expect(response.body.byteLength).to.equal(191); const encodedHash = createHash('sha256').update(response.body).digest('hex'); expect(encodedHash).to.equal( - '5caaeabcb7864d47157fc7c28d5a7398b4f6bbaaa565d789c02ee809253b7613' + '73015ee5131dabd1b48aa4776d3e766d836f8dd8c9fa8999c9b931f60027f07f' ); const decodedBody = inflateSync(response.body); const decodedHash = createHash('sha256').update(decodedBody).digest('hex'); expect(decodedHash).to.equal( - 'd2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '8d2bcc37e82fad5d06e2c9e4bd96793ea8905ace1d528a57d0d0579ecc8c647e' ); - expect(decodedBody.byteLength).to.equal(358); + expect(decodedBody.byteLength).to.equal(704); const artifactJson = JSON.parse(decodedBody.toString()); expect(artifactJson).to.eql({ entries: [ @@ -113,6 +113,35 @@ export default function (providerContext: FtrProviderContext) { type: 'exact_cased', value: 'Elastic, N.V.', }, + { + entries: [ + { + field: 'signer', + operator: 'included', + type: 'exact_cased', + value: '๐Ÿ˜ˆ', + }, + { + field: 'trusted', + operator: 'included', + type: 'exact_cased', + value: 'true', + }, + ], + field: 'file.signature', + type: 'nested', + }, + ], + }, + { + type: 'simple', + entries: [ + { + field: 'actingProcess.file.signer', + operator: 'included', + type: 'exact_cased', + value: 'Another signer', + }, { entries: [ { @@ -280,7 +309,7 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact from cache', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/8d2bcc37e82fad5d06e2c9e4bd96793ea8905ace1d528a57d0d0579ecc8c647e' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -292,22 +321,24 @@ export default function (providerContext: FtrProviderContext) { .then(async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/8d2bcc37e82fad5d06e2c9e4bd96793ea8905ace1d528a57d0d0579ecc8c647e' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() .expect(200) .expect((response) => { + expect(response.body.byteLength).to.equal(191); const encodedHash = createHash('sha256').update(response.body).digest('hex'); expect(encodedHash).to.equal( - '5caaeabcb7864d47157fc7c28d5a7398b4f6bbaaa565d789c02ee809253b7613' + '73015ee5131dabd1b48aa4776d3e766d836f8dd8c9fa8999c9b931f60027f07f' ); const decodedBody = inflateSync(response.body); const decodedHash = createHash('sha256').update(decodedBody).digest('hex'); expect(decodedHash).to.equal( - 'd2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '8d2bcc37e82fad5d06e2c9e4bd96793ea8905ace1d528a57d0d0579ecc8c647e' ); + expect(decodedBody.byteLength).to.equal(704); const artifactJson = JSON.parse(decodedBody.toString()); expect(artifactJson).to.eql({ entries: [ @@ -320,6 +351,35 @@ export default function (providerContext: FtrProviderContext) { type: 'exact_cased', value: 'Elastic, N.V.', }, + { + entries: [ + { + field: 'signer', + operator: 'included', + type: 'exact_cased', + value: '๐Ÿ˜ˆ', + }, + { + field: 'trusted', + operator: 'included', + type: 'exact_cased', + value: 'true', + }, + ], + field: 'file.signature', + type: 'nested', + }, + ], + }, + { + type: 'simple', + entries: [ + { + field: 'actingProcess.file.signer', + operator: 'included', + type: 'exact_cased', + value: 'Another signer', + }, { entries: [ { From 7314c1ba8fe896473de9b1a081ee8c3133eb7075 Mon Sep 17 00:00:00 2001 From: Kevin Logan <56395104+kevinlog@users.noreply.github.com> Date: Tue, 28 Jul 2020 23:05:52 -0400 Subject: [PATCH 2/4] [SECURITY_SOLUTION] Task/add detections rule text (#73596) --- .../view/policy_forms/protections/malware.tsx | 29 ++++++++++++++++++- .../components/endpoint_notice/index.tsx | 4 +-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx index dee1e27782e69..84d4bf5355cd9 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx @@ -7,9 +7,18 @@ import React, { useCallback, useMemo } from 'react'; import { useDispatch } from 'react-redux'; import styled from 'styled-components'; -import { EuiRadio, EuiSwitch, EuiTitle, EuiSpacer, htmlIdGenerator } from '@elastic/eui'; +import { + EuiRadio, + EuiSwitch, + EuiTitle, + EuiSpacer, + htmlIdGenerator, + EuiCallOut, +} from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; +import { APP_ID } from '../../../../../../../common/constants'; +import { SecurityPageName } from '../../../../../../app/types'; import { Immutable, ProtectionModes } from '../../../../../../../common/endpoint/types'; import { OS, MalwareProtectionOSes } from '../../../types'; @@ -17,6 +26,7 @@ import { ConfigForm } from '../config_form'; import { policyConfig } from '../../../store/policy_details/selectors'; import { usePolicyDetailsSelector } from '../../policy_hooks'; import { clone } from '../../../models/policy_details_config'; +import { LinkToApp } from '../../../../../../common/components/endpoint/link_to_app'; const ProtectionRadioGroup = styled.div` display: flex; @@ -177,6 +187,23 @@ export const MalwareProtections = React.memo(() => { rightCorner={protectionSwitch} > {radioButtons} + + + + + + ), + }} + /> + ); }); diff --git a/x-pack/plugins/security_solution/public/overview/components/endpoint_notice/index.tsx b/x-pack/plugins/security_solution/public/overview/components/endpoint_notice/index.tsx index 7170412cb55ad..1d726a7dbd901 100644 --- a/x-pack/plugins/security_solution/public/overview/components/endpoint_notice/index.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/endpoint_notice/index.tsx @@ -33,7 +33,7 @@ export const EndpointNotice = memo<{ onDismiss: () => void }>(({ onDismiss }) => } @@ -49,7 +49,7 @@ export const EndpointNotice = memo<{ onDismiss: () => void }>(({ onDismiss }) => From 7059270ce975e2b142a2f6238857dd24cc9446b8 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 28 Jul 2020 22:09:04 -0500 Subject: [PATCH 3/4] [deb/rpm] fix config folder path (#73001) Co-authored-by: Elastic Machine Co-authored-by: Tyler Smalley --- docs/setup/production.asciidoc | 2 +- .../config/deprecation/core_deprecations.test.ts | 2 +- .../server/config/deprecation/core_deprecations.ts | 2 +- src/core/server/path/index.ts | 10 +++++++--- src/dev/build/tasks/bin/scripts/kibana-keystore | 2 +- .../build/tasks/bin/scripts/kibana-keystore.bat | 4 ++-- src/dev/build/tasks/bin/scripts/kibana-plugin | 2 +- src/dev/build/tasks/bin/scripts/kibana-plugin.bat | 4 ++-- src/dev/build/tasks/bin/scripts/kibana.bat | 4 ++-- .../os_packages/package_scripts/post_install.sh | 14 ++++++++++++++ .../service_templates/sysv/etc/default/kibana | 2 +- 11 files changed, 33 insertions(+), 15 deletions(-) diff --git a/docs/setup/production.asciidoc b/docs/setup/production.asciidoc index afb4b37df6a28..23dbb3346b7ef 100644 --- a/docs/setup/production.asciidoc +++ b/docs/setup/production.asciidoc @@ -167,7 +167,7 @@ These can be used to automatically update the list of hosts as a cluster is resi Kibana has a default maximum memory limit of 1.4 GB, and in most cases, we recommend leaving this unconfigured. In some scenarios, such as large reporting jobs, it may make sense to tweak limits to meet more specific requirements. -You can modify this limit by setting `--max-old-space-size` in the `node.options` config file that can be found inside `kibana/config` folder or any other configured with the environment variable `KIBANA_PATH_CONF` (for example in debian based system would be `/etc/kibana`). +You can modify this limit by setting `--max-old-space-size` in the `node.options` config file that can be found inside `kibana/config` folder or any other configured with the environment variable `KBN_PATH_CONF` (for example in debian based system would be `/etc/kibana`). The option accepts a limit in MB: -------- diff --git a/src/core/server/config/deprecation/core_deprecations.test.ts b/src/core/server/config/deprecation/core_deprecations.test.ts index ebdb6f1c88b16..adf0f52339366 100644 --- a/src/core/server/config/deprecation/core_deprecations.test.ts +++ b/src/core/server/config/deprecation/core_deprecations.test.ts @@ -51,7 +51,7 @@ describe('core deprecations', () => { const { messages } = applyCoreDeprecations(); expect(messages).toMatchInlineSnapshot(` Array [ - "Environment variable CONFIG_PATH is deprecated. It has been replaced with KIBANA_PATH_CONF pointing to a config folder", + "Environment variable CONFIG_PATH is deprecated. It has been replaced with KBN_PATH_CONF pointing to a config folder", ] `); }); diff --git a/src/core/server/config/deprecation/core_deprecations.ts b/src/core/server/config/deprecation/core_deprecations.ts index 715f5b883139f..6cc0e5ef138d5 100644 --- a/src/core/server/config/deprecation/core_deprecations.ts +++ b/src/core/server/config/deprecation/core_deprecations.ts @@ -23,7 +23,7 @@ import { ConfigDeprecationProvider, ConfigDeprecation } from './types'; const configPathDeprecation: ConfigDeprecation = (settings, fromPath, log) => { if (has(process.env, 'CONFIG_PATH')) { log( - `Environment variable CONFIG_PATH is deprecated. It has been replaced with KIBANA_PATH_CONF pointing to a config folder` + `Environment variable CONFIG_PATH is deprecated. It has been replaced with KBN_PATH_CONF pointing to a config folder` ); } return settings; diff --git a/src/core/server/path/index.ts b/src/core/server/path/index.ts index 1bb650518c47a..7c1a81643fbc8 100644 --- a/src/core/server/path/index.ts +++ b/src/core/server/path/index.ts @@ -25,14 +25,18 @@ import { fromRoot } from '../utils'; const isString = (v: any): v is string => typeof v === 'string'; const CONFIG_PATHS = [ + process.env.KBN_PATH_CONF && join(process.env.KBN_PATH_CONF, 'kibana.yml'), process.env.KIBANA_PATH_CONF && join(process.env.KIBANA_PATH_CONF, 'kibana.yml'), process.env.CONFIG_PATH, // deprecated fromRoot('config/kibana.yml'), ].filter(isString); -const CONFIG_DIRECTORIES = [process.env.KIBANA_PATH_CONF, fromRoot('config'), '/etc/kibana'].filter( - isString -); +const CONFIG_DIRECTORIES = [ + process.env.KBN_PATH_CONF, + process.env.KIBANA_PATH_CONF, + fromRoot('config'), + '/etc/kibana', +].filter(isString); const DATA_PATHS = [ process.env.DATA_PATH, // deprecated diff --git a/src/dev/build/tasks/bin/scripts/kibana-keystore b/src/dev/build/tasks/bin/scripts/kibana-keystore index f83df118d24e8..d811e70095548 100755 --- a/src/dev/build/tasks/bin/scripts/kibana-keystore +++ b/src/dev/build/tasks/bin/scripts/kibana-keystore @@ -14,7 +14,7 @@ while [ -h "$SCRIPT" ] ; do done DIR="$(dirname "${SCRIPT}")/.." -CONFIG_DIR=${KIBANA_PATH_CONF:-"$DIR/config"} +CONFIG_DIR=${KBN_PATH_CONF:-"$DIR/config"} NODE="${DIR}/node/bin/node" test -x "$NODE" if [ ! -x "$NODE" ]; then diff --git a/src/dev/build/tasks/bin/scripts/kibana-keystore.bat b/src/dev/build/tasks/bin/scripts/kibana-keystore.bat index 389eb5bf488e4..7e227141c8ba3 100755 --- a/src/dev/build/tasks/bin/scripts/kibana-keystore.bat +++ b/src/dev/build/tasks/bin/scripts/kibana-keystore.bat @@ -12,8 +12,8 @@ If Not Exist "%NODE%" ( Exit /B 1 ) -set CONFIG_DIR=%KIBANA_PATH_CONF% -If [%KIBANA_PATH_CONF%] == [] ( +set CONFIG_DIR=%KBN_PATH_CONF% +If [%KBN_PATH_CONF%] == [] ( set CONFIG_DIR=%DIR%\config ) diff --git a/src/dev/build/tasks/bin/scripts/kibana-plugin b/src/dev/build/tasks/bin/scripts/kibana-plugin index f1102e1ef5a32..f4486e9cf85fb 100755 --- a/src/dev/build/tasks/bin/scripts/kibana-plugin +++ b/src/dev/build/tasks/bin/scripts/kibana-plugin @@ -14,7 +14,7 @@ while [ -h "$SCRIPT" ] ; do done DIR="$(dirname "${SCRIPT}")/.." -CONFIG_DIR=${KIBANA_PATH_CONF:-"$DIR/config"} +CONFIG_DIR=${KBN_PATH_CONF:-"$DIR/config"} NODE="${DIR}/node/bin/node" test -x "$NODE" if [ ! -x "$NODE" ]; then diff --git a/src/dev/build/tasks/bin/scripts/kibana-plugin.bat b/src/dev/build/tasks/bin/scripts/kibana-plugin.bat index 6815b1b9eab8c..4fb30977fda06 100755 --- a/src/dev/build/tasks/bin/scripts/kibana-plugin.bat +++ b/src/dev/build/tasks/bin/scripts/kibana-plugin.bat @@ -13,8 +13,8 @@ If Not Exist "%NODE%" ( Exit /B 1 ) -set CONFIG_DIR=%KIBANA_PATH_CONF% -If [%KIBANA_PATH_CONF%] == [] ( +set CONFIG_DIR=%KBN_PATH_CONF% +If [%KBN_PATH_CONF%] == [] ( set CONFIG_DIR=%DIR%\config ) diff --git a/src/dev/build/tasks/bin/scripts/kibana.bat b/src/dev/build/tasks/bin/scripts/kibana.bat index d3edc92f110a5..98dd9ec05a48c 100755 --- a/src/dev/build/tasks/bin/scripts/kibana.bat +++ b/src/dev/build/tasks/bin/scripts/kibana.bat @@ -14,8 +14,8 @@ If Not Exist "%NODE%" ( Exit /B 1 ) -set CONFIG_DIR=%KIBANA_PATH_CONF% -If [%KIBANA_PATH_CONF%] == [] ( +set CONFIG_DIR=%KBN_PATH_CONF% +If [%KBN_PATH_CONF%] == [] ( set CONFIG_DIR=%DIR%\config ) diff --git a/src/dev/build/tasks/os_packages/package_scripts/post_install.sh b/src/dev/build/tasks/os_packages/package_scripts/post_install.sh index c49b291d1a0c9..1c679bdb40b59 100644 --- a/src/dev/build/tasks/os_packages/package_scripts/post_install.sh +++ b/src/dev/build/tasks/os_packages/package_scripts/post_install.sh @@ -31,6 +31,10 @@ case $1 in --ingroup "<%= group %>" --shell /bin/false "<%= user %>" fi + if [ -n "$2" ]; then + IS_UPGRADE=true + fi + set_access ;; abort-deconfigure|abort-upgrade|abort-remove) @@ -47,6 +51,10 @@ case $1 in -c "kibana service user" "<%= user %>" fi + if [ "$1" = "2" ]; then + IS_UPGRADE=true + fi + set_access ;; @@ -55,3 +63,9 @@ case $1 in exit 1 ;; esac + +if [ "$IS_UPGRADE" = "true" ]; then + if command -v systemctl >/dev/null; then + systemctl daemon-reload + fi +fi diff --git a/src/dev/build/tasks/os_packages/service_templates/sysv/etc/default/kibana b/src/dev/build/tasks/os_packages/service_templates/sysv/etc/default/kibana index 092dc6482fa1d..ee019d348ed97 100644 --- a/src/dev/build/tasks/os_packages/service_templates/sysv/etc/default/kibana +++ b/src/dev/build/tasks/os_packages/service_templates/sysv/etc/default/kibana @@ -12,4 +12,4 @@ KILL_ON_STOP_TIMEOUT=0 BABEL_CACHE_PATH="/var/lib/kibana/optimize/.babel_register_cache.json" -KIBANA_PATH_CONF="/etc/kibana" +KBN_PATH_CONF="/etc/kibana" From e64573231905158121187f07f8aadc26ba4ac287 Mon Sep 17 00:00:00 2001 From: Yara Tercero Date: Tue, 28 Jul 2020 23:27:14 -0400 Subject: [PATCH 4/4] [Security Solution][Exceptions] - Update rule.exceptions_list to include exception list list_id (#73349) ## Summary This PR addresses the following: - Adds `list_id` to `rule.exceptions_list` - this is needed in a number of features - Updated `getExceptions` in `x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts` to use the latest exception item find endpoint that accepts an array of lists (previously was looping through lists and conducting a `find` for each) - Updated prepackaged rule that makes reference to global endpoint list to include `list_id` - Updates `formatAboutStepData` in `x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts` to include exception list `list_id` --- .../hooks/use_exception_list.test.ts | 28 +++++++--- .../plugins/lists/public/exceptions/types.ts | 1 + .../add_prepackged_rules_schema.test.ts | 5 +- .../request/create_rules_schema.test.ts | 5 +- .../request/import_rules_schema.test.ts | 5 +- .../request/patch_rules_schema.test.ts | 5 +- .../request/update_rules_schema.test.ts | 5 +- .../schemas/types/lists.mock.ts | 9 ++- .../schemas/types/lists.test.ts | 10 ++-- .../detection_engine/schemas/types/lists.ts | 5 +- ...se_fetch_or_create_rule_exception_list.tsx | 1 + .../exceptions/viewer/index.test.tsx | 2 + .../components/exceptions/viewer/index.tsx | 2 - .../rules/all/__mocks__/mock.ts | 7 --- .../rules/create/helpers.test.ts | 31 ++++------- .../detection_engine/rules/create/helpers.ts | 7 ++- .../detection_engine/rules/details/index.tsx | 4 +- .../prepackaged_rules/elastic_endpoint.json | 14 ++--- .../prepackaged_rules/external_alerts.json | 9 +-- .../detection_engine/signals/utils.test.ts | 55 ++++++++----------- .../lib/detection_engine/signals/utils.ts | 49 +++++------------ 21 files changed, 122 insertions(+), 137 deletions(-) diff --git a/x-pack/plugins/lists/public/exceptions/hooks/use_exception_list.test.ts b/x-pack/plugins/lists/public/exceptions/hooks/use_exception_list.test.ts index 918397d01ce2c..f678ed4faeeda 100644 --- a/x-pack/plugins/lists/public/exceptions/hooks/use_exception_list.test.ts +++ b/x-pack/plugins/lists/public/exceptions/hooks/use_exception_list.test.ts @@ -41,7 +41,9 @@ describe('useExceptionList', () => { useExceptionList({ filterOptions: { filter: '', tags: [] }, http: mockKibanaHttpService, - lists: [{ id: 'myListId', namespaceType: 'single', type: 'detection' }], + lists: [ + { id: 'myListId', listId: 'list_id', namespaceType: 'single', type: 'detection' }, + ], onError: onErrorMock, pagination: { page: 1, @@ -76,7 +78,9 @@ describe('useExceptionList', () => { useExceptionList({ filterOptions: { filter: '', tags: [] }, http: mockKibanaHttpService, - lists: [{ id: 'myListId', namespaceType: 'single', type: 'detection' }], + lists: [ + { id: 'myListId', listId: 'list_id', namespaceType: 'single', type: 'detection' }, + ], onError: onErrorMock, onSuccess: onSuccessMock, pagination: { @@ -131,7 +135,9 @@ describe('useExceptionList', () => { initialProps: { filterOptions: { filter: '', tags: [] }, http: mockKibanaHttpService, - lists: [{ id: 'myListId', namespaceType: 'single', type: 'detection' }], + lists: [ + { id: 'myListId', listId: 'list_id', namespaceType: 'single', type: 'detection' }, + ], onError: onErrorMock, onSuccess: onSuccessMock, pagination: { @@ -146,7 +152,9 @@ describe('useExceptionList', () => { rerender({ filterOptions: { filter: '', tags: [] }, http: mockKibanaHttpService, - lists: [{ id: 'newListId', namespaceType: 'single', type: 'detection' }], + lists: [ + { id: 'newListId', listId: 'new_list_id', namespaceType: 'single', type: 'detection' }, + ], onError: onErrorMock, onSuccess: onSuccessMock, pagination: { @@ -173,7 +181,9 @@ describe('useExceptionList', () => { useExceptionList({ filterOptions: { filter: '', tags: [] }, http: mockKibanaHttpService, - lists: [{ id: 'myListId', namespaceType: 'single', type: 'detection' }], + lists: [ + { id: 'myListId', listId: 'list_id', namespaceType: 'single', type: 'detection' }, + ], onError: onErrorMock, pagination: { page: 1, @@ -210,7 +220,9 @@ describe('useExceptionList', () => { useExceptionList({ filterOptions: { filter: '', tags: [] }, http: mockKibanaHttpService, - lists: [{ id: 'myListId', namespaceType: 'single', type: 'detection' }], + lists: [ + { id: 'myListId', listId: 'list_id', namespaceType: 'single', type: 'detection' }, + ], onError: onErrorMock, pagination: { page: 1, @@ -238,7 +250,9 @@ describe('useExceptionList', () => { useExceptionList({ filterOptions: { filter: '', tags: [] }, http: mockKibanaHttpService, - lists: [{ id: 'myListId', namespaceType: 'single', type: 'detection' }], + lists: [ + { id: 'myListId', listId: 'list_id', namespaceType: 'single', type: 'detection' }, + ], onError: onErrorMock, pagination: { page: 1, diff --git a/x-pack/plugins/lists/public/exceptions/types.ts b/x-pack/plugins/lists/public/exceptions/types.ts index f99323b384781..c0ec72e1c19eb 100644 --- a/x-pack/plugins/lists/public/exceptions/types.ts +++ b/x-pack/plugins/lists/public/exceptions/types.ts @@ -60,6 +60,7 @@ export interface UseExceptionListProps { export interface ExceptionIdentifiers { id: string; + listId: string; namespaceType: NamespaceType; type: ExceptionListType; } diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts index 5fd2c3dbbf894..3cad48ec18fc1 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts @@ -1446,11 +1446,13 @@ describe('add prepackaged rules schema', () => { exceptions_list: [ { id: 'some_uuid', + list_id: 'list_id_single', namespace_type: 'single', type: 'detection', }, { - id: 'some_uuid', + id: 'endpoint_list', + list_id: 'endpoint_list', namespace_type: 'agnostic', type: 'endpoint', }, @@ -1535,6 +1537,7 @@ describe('add prepackaged rules schema', () => { const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "undefined" supplied to "exceptions_list,list_id"', 'Invalid value "undefined" supplied to "exceptions_list,type"', 'Invalid value "not a namespace type" supplied to "exceptions_list,namespace_type"', ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts index 71f3964956249..c2c2f4784f2b5 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts @@ -1513,11 +1513,13 @@ describe('create rules schema', () => { exceptions_list: [ { id: 'some_uuid', + list_id: 'list_id_single', namespace_type: 'single', type: 'detection', }, { - id: 'some_uuid', + id: 'endpoint_list', + list_id: 'endpoint_list', namespace_type: 'agnostic', type: 'endpoint', }, @@ -1600,6 +1602,7 @@ describe('create rules schema', () => { const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "undefined" supplied to "exceptions_list,list_id"', 'Invalid value "undefined" supplied to "exceptions_list,type"', 'Invalid value "not a namespace type" supplied to "exceptions_list,namespace_type"', ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts index 828626ef26d6f..00a3f2126f44a 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts @@ -1642,11 +1642,13 @@ describe('import rules schema', () => { exceptions_list: [ { id: 'some_uuid', + list_id: 'list_id_single', namespace_type: 'single', type: 'detection', }, { - id: 'some_uuid', + id: 'endpoint_list', + list_id: 'endpoint_list', namespace_type: 'agnostic', type: 'endpoint', }, @@ -1730,6 +1732,7 @@ describe('import rules schema', () => { const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "undefined" supplied to "exceptions_list,list_id"', 'Invalid value "undefined" supplied to "exceptions_list,type"', 'Invalid value "not a namespace type" supplied to "exceptions_list,namespace_type"', ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.test.ts index e75aff1abe3e9..e4fc53b934f58 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.test.ts @@ -1176,11 +1176,13 @@ describe('patch_rules_schema', () => { exceptions_list: [ { id: 'some_uuid', + list_id: 'list_id_single', namespace_type: 'single', type: 'detection', }, { - id: 'some_uuid', + id: 'endpoint_list', + list_id: 'endpoint_list', namespace_type: 'agnostic', type: 'endpoint', }, @@ -1251,6 +1253,7 @@ describe('patch_rules_schema', () => { const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "undefined" supplied to "exceptions_list,list_id"', 'Invalid value "undefined" supplied to "exceptions_list,type"', 'Invalid value "not a namespace type" supplied to "exceptions_list,namespace_type"', 'Invalid value "[{"id":"uuid_here","namespace_type":"not a namespace type"}]" supplied to "exceptions_list"', diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.test.ts index d18d2d91b963c..024198d783048 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.test.ts @@ -1448,11 +1448,13 @@ describe('update rules schema', () => { exceptions_list: [ { id: 'some_uuid', + list_id: 'list_id_single', namespace_type: 'single', type: 'detection', }, { - id: 'some_uuid', + id: 'endpoint_list', + list_id: 'endpoint_list', namespace_type: 'agnostic', type: 'endpoint', }, @@ -1534,6 +1536,7 @@ describe('update rules schema', () => { const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "undefined" supplied to "exceptions_list,list_id"', 'Invalid value "undefined" supplied to "exceptions_list,type"', 'Invalid value "not a namespace type" supplied to "exceptions_list,namespace_type"', ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.mock.ts index 0c7853bc3c08a..fec7548811820 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.mock.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.mock.ts @@ -4,17 +4,20 @@ * you may not use this file except in compliance with the Elastic License. */ import { List, ListArray } from './lists'; +import { ENDPOINT_LIST_ID } from '../../../shared_imports'; export const getListMock = (): List => ({ id: 'some_uuid', + list_id: 'list_id_single', namespace_type: 'single', type: 'detection', }); -export const getListAgnosticMock = (): List => ({ - id: 'some_uuid', +export const getEndpointListMock = (): List => ({ + id: ENDPOINT_LIST_ID, + list_id: ENDPOINT_LIST_ID, namespace_type: 'agnostic', type: 'endpoint', }); -export const getListArrayMock = (): ListArray => [getListMock(), getListAgnosticMock()]; +export const getListArrayMock = (): ListArray => [getListMock(), getEndpointListMock()]; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.test.ts index 56ee4630996fd..7a2c167bfd855 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.test.ts @@ -9,7 +9,7 @@ import { left } from 'fp-ts/lib/Either'; import { foldLeftRight, getPaths } from '../../../test_utils'; -import { getListAgnosticMock, getListMock, getListArrayMock } from './lists.mock'; +import { getEndpointListMock, getListMock, getListArrayMock } from './lists.mock'; import { List, ListArray, @@ -31,7 +31,7 @@ describe('Lists', () => { }); test('it should validate a list with "namespace_type" of "agnostic"', () => { - const payload = getListAgnosticMock(); + const payload = getEndpointListMock(); const decoded = list.decode(payload); const message = pipe(decoded, foldLeftRight); @@ -91,7 +91,7 @@ describe('Lists', () => { const message = pipe(decoded, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ - 'Invalid value "1" supplied to "Array<{| id: string, type: "detection" | "endpoint", namespace_type: "agnostic" | "single" |}>"', + 'Invalid value "1" supplied to "Array<{| id: NonEmptyString, list_id: NonEmptyString, type: "detection" | "endpoint", namespace_type: "agnostic" | "single" |}>"', ]); expect(message.schema).toEqual({}); }); @@ -122,8 +122,8 @@ describe('Lists', () => { const message = pipe(decoded, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([ - 'Invalid value "1" supplied to "(Array<{| id: string, type: "detection" | "endpoint", namespace_type: "agnostic" | "single" |}> | undefined)"', - 'Invalid value "[1]" supplied to "(Array<{| id: string, type: "detection" | "endpoint", namespace_type: "agnostic" | "single" |}> | undefined)"', + 'Invalid value "1" supplied to "(Array<{| id: NonEmptyString, list_id: NonEmptyString, type: "detection" | "endpoint", namespace_type: "agnostic" | "single" |}> | undefined)"', + 'Invalid value "[1]" supplied to "(Array<{| id: NonEmptyString, list_id: NonEmptyString, type: "detection" | "endpoint", namespace_type: "agnostic" | "single" |}> | undefined)"', ]); expect(message.schema).toEqual({}); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.ts index e5aaee6d3ec74..fecdd0761aade 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/lists.ts @@ -8,9 +8,12 @@ import * as t from 'io-ts'; import { exceptionListType, namespaceType } from '../../../shared_imports'; +import { NonEmptyString } from './non_empty_string'; + export const list = t.exact( t.type({ - id: t.string, + id: NonEmptyString, + list_id: NonEmptyString, type: exceptionListType, namespace_type: namespaceType, }) diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/use_fetch_or_create_rule_exception_list.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/use_fetch_or_create_rule_exception_list.tsx index 2a5ef7b21b519..0d367e03a799f 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/use_fetch_or_create_rule_exception_list.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/use_fetch_or_create_rule_exception_list.tsx @@ -102,6 +102,7 @@ export const useFetchOrCreateRuleExceptionList = ({ const newExceptionListReference = { id: newExceptionList.id, + list_id: newExceptionList.list_id, type: newExceptionList.type, namespace_type: newExceptionList.namespace_type, }; diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/index.test.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/index.test.tsx index 986f27f6495ec..84613d1c73536 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/index.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/index.test.tsx @@ -72,6 +72,7 @@ describe('ExceptionsViewer', () => { exceptionListsMeta={[ { id: '5b543420', + listId: 'list_id', type: 'endpoint', namespaceType: 'single', }, @@ -124,6 +125,7 @@ describe('ExceptionsViewer', () => { exceptionListsMeta={[ { id: '5b543420', + listId: 'list_id', type: 'endpoint', namespaceType: 'single', }, diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/index.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/index.tsx index 16eaef4136983..7f4d825276406 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/index.tsx @@ -176,8 +176,6 @@ const ExceptionsViewerComponent = ({ const handleEditException = useCallback( (exception: ExceptionListItemSchema): void => { - // TODO: Added this just for testing. Update - // modal state logic as needed once ready dispatch({ type: 'updateExceptionToEdit', exception, diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts index 7a6b61f0dfd89..8c6e91254314e 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts @@ -6,7 +6,6 @@ import { esFilters } from '../../../../../../../../../../src/plugins/data/public'; import { Rule, RuleError } from '../../../../../containers/detection_engine/rules'; -import { List } from '../../../../../../../common/detection_engine/schemas/types'; import { AboutStepRule, ActionsStepRule, DefineStepRule, ScheduleStepRule } from '../../types'; import { FieldValueQueryBar } from '../../../../../components/rules/query_bar'; import { fillEmptySeverityMappings } from '../../helpers'; @@ -242,9 +241,3 @@ export const mockRules: Rule[] = [ mockRule('abe6c564-050d-45a5-aaf0-386c37dd1f61'), mockRule('63f06f34-c181-4b2d-af35-f2ace572a1ee'), ]; - -export const mockExceptionsList: List = { - namespace_type: 'single', - id: '75cd4380-cc5e-11ea-9101-5b34f44aeb44', - type: 'detection', -}; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts index 6458d2faa2468..4f2055424ca61 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts @@ -5,9 +5,11 @@ */ import { List } from '../../../../../../common/detection_engine/schemas/types'; -import { ENDPOINT_LIST_ID } from '../../../../../shared_imports'; import { NewRule } from '../../../../containers/detection_engine/rules'; - +import { + getListMock, + getEndpointListMock, +} from '../../../../../../common/detection_engine/schemas/types/lists.mock'; import { DefineStepRuleJson, ScheduleStepRuleJson, @@ -29,19 +31,12 @@ import { } from './helpers'; import { mockDefineStepRule, - mockExceptionsList, mockQueryBar, mockScheduleStepRule, mockAboutStepRule, mockActionsStepRule, } from '../all/__mocks__/mock'; -const ENDPOINT_LIST = { - id: ENDPOINT_LIST_ID, - namespace_type: 'agnostic', - type: 'endpoint', -} as List; - describe('helpers', () => { describe('getTimeTypeValue', () => { test('returns timeObj with value 0 if no time value found', () => { @@ -391,14 +386,12 @@ describe('helpers', () => { }, [] ); - expect(result.exceptions_list).toEqual([ - { id: ENDPOINT_LIST_ID, namespace_type: 'agnostic', type: 'endpoint' }, - ]); + expect(result.exceptions_list).toEqual([getEndpointListMock()]); }); test('returns formatted object with detections exceptions_list', () => { - const result: AboutStepRuleJson = formatAboutStepData(mockData, [mockExceptionsList]); - expect(result.exceptions_list).toEqual([mockExceptionsList]); + const result: AboutStepRuleJson = formatAboutStepData(mockData, [getListMock()]); + expect(result.exceptions_list).toEqual([getListMock()]); }); test('returns formatted object with both exceptions_lists', () => { @@ -407,13 +400,13 @@ describe('helpers', () => { ...mockData, isAssociatedToEndpointList: true, }, - [mockExceptionsList] + [getListMock()] ); - expect(result.exceptions_list).toEqual([ENDPOINT_LIST, mockExceptionsList]); + expect(result.exceptions_list).toEqual([getEndpointListMock(), getListMock()]); }); test('returns formatted object with pre-existing exceptions lists', () => { - const exceptionsLists: List[] = [ENDPOINT_LIST, mockExceptionsList]; + const exceptionsLists: List[] = [getEndpointListMock(), getListMock()]; const result: AboutStepRuleJson = formatAboutStepData( { ...mockData, @@ -425,9 +418,9 @@ describe('helpers', () => { }); test('returns formatted object with pre-existing endpoint exceptions list disabled', () => { - const exceptionsLists: List[] = [ENDPOINT_LIST, mockExceptionsList]; + const exceptionsLists: List[] = [getEndpointListMock(), getListMock()]; const result: AboutStepRuleJson = formatAboutStepData(mockData, exceptionsLists); - expect(result.exceptions_list).toEqual([mockExceptionsList]); + expect(result.exceptions_list).toEqual([getListMock()]); }); test('returns formatted object with empty falsePositive and references filtered out', () => { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts index 8b03f62fc82bd..434a33ac37722 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts @@ -177,7 +177,12 @@ export const formatAboutStepData = ( ...(isAssociatedToEndpointList ? { exceptions_list: [ - { id: ENDPOINT_LIST_ID, namespace_type: 'agnostic', type: 'endpoint' }, + { + id: ENDPOINT_LIST_ID, + list_id: ENDPOINT_LIST_ID, + namespace_type: 'agnostic', + type: 'endpoint', + }, ...detectionExceptionLists, ] as AboutStepRuleJson['exceptions_list'], } diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx index 90424e1fb9dd0..789469e981fb4 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx @@ -328,13 +328,13 @@ export const RuleDetailsPageComponent: FC = ({ lists: ExceptionIdentifiers[]; allowedExceptionListTypes: ExceptionListTypeEnum[]; }>( - (acc, { id, namespace_type, type }) => { + (acc, { id, list_id, namespace_type, type }) => { const { allowedExceptionListTypes, lists } = acc; const shouldAddEndpoint = type === ExceptionListTypeEnum.ENDPOINT && !allowedExceptionListTypes.includes(ExceptionListTypeEnum.ENDPOINT); return { - lists: [...lists, { id, namespaceType: namespace_type, type }], + lists: [...lists, { id, listId: list_id, namespaceType: namespace_type, type }], allowedExceptionListTypes: shouldAddEndpoint ? [...allowedExceptionListTypes, ExceptionListTypeEnum.ENDPOINT] : allowedExceptionListTypes, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json index e6a517d85db81..05601ec8ffb4c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json @@ -1,20 +1,17 @@ { - "author": [ - "Elastic" - ], + "author": ["Elastic"], "description": "Generates a detection alert each time an Elastic Endpoint Security alert is received. Enabling this rule allows you to immediately begin investigating your Elastic Endpoint alerts.", "enabled": true, "exceptions_list": [ { "id": "endpoint_list", + "list_id": "endpoint_list", "namespace_type": "agnostic", "type": "endpoint" } ], "from": "now-10m", - "index": [ - "logs-endpoint.alerts-*" - ], + "index": ["logs-endpoint.alerts-*"], "language": "kuery", "license": "Elastic License", "max_signals": 10000, @@ -57,10 +54,7 @@ "value": "99" } ], - "tags": [ - "Elastic", - "Endpoint" - ], + "tags": ["Elastic", "Endpoint"], "timestamp_override": "event.ingested", "type": "query", "version": 1 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json index 678ad9eb03b50..8b627c48d2904 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json @@ -1,12 +1,9 @@ { - "author": [ - "Elastic" - ], + "author": ["Elastic"], "description": "Generates a detection alert for each external alert written to the configured indices. Enabling this rule allows you to immediately begin investigating external alerts in the app.", "index": [ "apm-*-transaction*", "auditbeat-*", - "endgame-*", "filebeat-*", "logs-*", "packetbeat-*", @@ -54,9 +51,7 @@ "value": "99" } ], - "tags": [ - "Elastic" - ], + "tags": ["Elastic"], "timestamp_override": "event.ingested", "type": "query", "version": 1 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index a610970907bf8..3c41f29625a51 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -716,26 +716,31 @@ describe('utils', () => { describe('#getExceptions', () => { test('it successfully returns array of exception list items', async () => { + listMock.getExceptionListClient = () => + (({ + findExceptionListsItem: jest.fn().mockResolvedValue({ + data: [getExceptionListItemSchemaMock()], + page: 1, + per_page: 10000, + total: 1, + }), + } as unknown) as ExceptionListClient); const client = listMock.getExceptionListClient(); const exceptions = await getExceptions({ client, lists: getListArrayMock(), }); - expect(client.getExceptionList).toHaveBeenNthCalledWith(1, { - id: 'some_uuid', - listId: undefined, - namespaceType: 'single', - }); - expect(client.getExceptionList).toHaveBeenNthCalledWith(2, { - id: 'some_uuid', - listId: undefined, - namespaceType: 'agnostic', + expect(client.findExceptionListsItem).toHaveBeenCalledWith({ + listId: ['list_id_single', 'endpoint_list'], + namespaceType: ['single', 'agnostic'], + page: 1, + perPage: 10000, + filter: [], + sortOrder: undefined, + sortField: undefined, }); - expect(exceptions).toEqual([ - getExceptionListItemSchemaMock(), - getExceptionListItemSchemaMock(), - ]); + expect(exceptions).toEqual([getExceptionListItemSchemaMock()]); }); test('it throws if "client" is undefined', async () => { @@ -747,7 +752,7 @@ describe('utils', () => { ).rejects.toThrowError('lists plugin unavailable during rule execution'); }); - test('it returns empty array if no "lists" is undefined', async () => { + test('it returns empty array if "lists" is undefined', async () => { const exceptions = await getExceptions({ client: listMock.getExceptionListClient(), lists: undefined, @@ -771,11 +776,11 @@ describe('utils', () => { ).rejects.toThrowError('unable to fetch exception list items'); }); - test('it throws if "findExceptionListItem" fails', async () => { + test('it throws if "findExceptionListsItem" fails', async () => { const err = new Error('error fetching list'); listMock.getExceptionListClient = () => (({ - findExceptionListItem: jest.fn().mockRejectedValue(err), + findExceptionListsItem: jest.fn().mockRejectedValue(err), } as unknown) as ExceptionListClient); await expect(() => @@ -786,24 +791,10 @@ describe('utils', () => { ).rejects.toThrowError('unable to fetch exception list items'); }); - test('it returns empty array if "getExceptionList" returns null', async () => { - listMock.getExceptionListClient = () => - (({ - getExceptionList: jest.fn().mockResolvedValue(null), - } as unknown) as ExceptionListClient); - - const exceptions = await getExceptions({ - client: listMock.getExceptionListClient(), - lists: undefined, - }); - - expect(exceptions).toEqual([]); - }); - - test('it returns empty array if "findExceptionListItem" returns null', async () => { + test('it returns empty array if "findExceptionListsItem" returns null', async () => { listMock.getExceptionListClient = () => (({ - findExceptionListItem: jest.fn().mockResolvedValue(null), + findExceptionListsItem: jest.fn().mockResolvedValue(null), } as unknown) as ExceptionListClient); const exceptions = await getExceptions({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index ae4274f31e145..9519720d0bbec 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -161,43 +161,20 @@ export const getExceptions = async ({ throw new Error('lists plugin unavailable during rule execution'); } - if (lists != null) { + if (lists != null && lists.length > 0) { try { - // Gather all exception items of all exception lists linked to rule - const exceptions = await Promise.all( - lists - .map(async (list) => { - const { id, namespace_type: namespaceType } = list; - try { - // TODO update once exceptions client `findExceptionListItem` - // accepts an array of list ids - const foundList = await client.getExceptionList({ - id, - namespaceType, - listId: undefined, - }); - - if (foundList == null) { - return []; - } else { - const items = await client.findExceptionListItem({ - listId: foundList.list_id, - namespaceType, - page: 1, - perPage: MAX_EXCEPTION_LIST_SIZE, - filter: undefined, - sortOrder: undefined, - sortField: undefined, - }); - return items != null ? items.data : []; - } - } catch { - throw new Error('unable to fetch exception list items'); - } - }) - .flat() - ); - return exceptions.flat(); + const listIds = lists.map(({ list_id: listId }) => listId); + const namespaceTypes = lists.map(({ namespace_type: namespaceType }) => namespaceType); + const items = await client.findExceptionListsItem({ + listId: listIds, + namespaceType: namespaceTypes, + page: 1, + perPage: MAX_EXCEPTION_LIST_SIZE, + filter: [], + sortOrder: undefined, + sortField: undefined, + }); + return items != null ? items.data : []; } catch { throw new Error('unable to fetch exception list items'); }