Write tests for ingest callback and ensure policy is returned when er…

…rors occur

x-pack/plugins/ingest_manager/common/mocks.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { NewPackageConfig } from './types/models/package_config';
+
+export const createNewPackageConfigMock = () => {
+  return {
+    name: 'endpoint-1',
+    description: '',
+    namespace: 'default',
+    enabled: true,
+    config_id: '93c46720-c217-11ea-9906-b5b8a21b268e',
+    output_id: '',
+    package: {
+      name: 'endpoint',
+      title: 'Elastic Endpoint',
+      version: '0.9.0',
+    },
+    inputs: [],
+  } as NewPackageConfig;
+};

x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { loggerMock } from 'src/core/server/logging/logger.mock';
+import { createNewPackageConfigMock } from '../../../ingest_manager/common/mocks';
+import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config';
+import { getManifestManagerMock } from './services/artifacts/manifest_manager/manifest_manager.mock';
+import { getPackageConfigCreateCallback } from './ingest_integration';
+
+describe('ingest_integration tests ', () => {
+  describe('ingest_integration sanity checks', () => {
+    test('policy is updated with manifest', async () => {
+      const logger = loggerMock.create();
+      const manifestManager = getManifestManagerMock();
+      const callback = getPackageConfigCreateCallback(logger, manifestManager);
+      const policyConfig = createNewPackageConfigMock();
+      const newPolicyConfig = await callback(policyConfig);
+      expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint');
+      expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory());
+      expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual({
+        artifacts: {
+          'endpoint-exceptionlist-linux-v1': {
+            compression_algorithm: 'none',
+            decoded_sha256: '1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc',
+            decoded_size: 287,
+            encoded_sha256: '1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc',
+            encoded_size: 287,
+            encryption_algorithm: 'none',
+            relative_url:
+              '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc',
+          },
+        },
+        manifest_version: 'WzAsMF0=',
+        schema_version: 'v1',
+      });
+    });
+
+    test('policy is returned even if error is encountered during artifact sync', async () => {
+      const logger = loggerMock.create();
+      const manifestManager = getManifestManagerMock();
+      manifestManager.syncArtifacts = jest.fn().mockRejectedValue([new Error('error updating')]);
+      const callback = getPackageConfigCreateCallback(logger, manifestManager);
+      const policyConfig = createNewPackageConfigMock();
+      const newPolicyConfig = await callback(policyConfig);
+      expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint');
+      expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory());
+      expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual({
+        manifest_version: 'WzAsMF0=',
+        schema_version: 'v1',
+        artifacts: {},
+      });
+    });
+  });
+});

x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts

@@ -8,8 +8,9 @@ import { Logger } from '../../../../../src/core/server';
 import { NewPackageConfig } from '../../../ingest_manager/common/types/models';
 import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config';
 import { NewPolicyData } from '../../common/endpoint/types';
-import { ManifestManager } from './services/artifacts';
-import { reportErrors } from './lib/artifacts/common';
+import { ManifestManager, ManifestSnapshot } from './services/artifacts';
+import { reportErrors, ManifestConstants } from './lib/artifacts/common';
+import { ManifestSchemaVersion } from '../../common/endpoint/schema/common';
 
 /**
  * Callback to handle creation of PackageConfigs in Ingest Manager
@@ -30,43 +31,60 @@ export const getPackageConfigCreateCallback = (
     // follow the types/schema expected
     let updatedPackageConfig = newPackageConfig as NewPolicyData;
 
-    // get snapshot based on exception-list-agnostic SOs
-    // with diffs from last dispatched manifest, if it exists
-    const snapshot = await manifestManager.getSnapshot({ initialize: true });
+    // get current manifest from SO (last dispatched)
+    const manifest = (
+      await manifestManager.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION)
+    )?.toEndpointFormat() ?? {
+      manifest_version: 'default',
+      schema_version: ManifestConstants.SCHEMA_VERSION as ManifestSchemaVersion,
+      artifacts: {},
+    };
 
+    // Until we get the Default Policy Configuration in the Endpoint package,
+    // we will add it here manually at creation time.
+    if (newPackageConfig.inputs.length === 0) {
+      updatedPackageConfig = {
+        ...newPackageConfig,
+        inputs: [
+          {
+            type: 'endpoint',
+            enabled: true,
+            streams: [],
+            config: {
+              artifact_manifest: {
+                value: manifest,
+              },
+              policy: {
+                value: policyConfigFactory(),
+              },
+            },
+          },
+        ],
+      };
+    }
+
+    let snapshot: ManifestSnapshot | null = null;
     let success = true;
     try {
-      if (snapshot && snapshot.diffs.length > 0) {
+      // Try to get most up-to-date manifest data.
+
+      // get snapshot based on exception-list-agnostic SOs
+      // with diffs from last dispatched manifest, if it exists
+      snapshot = await manifestManager.getSnapshot({ initialize: true });
+
+      if (snapshot && snapshot.diffs.length) {
         // create new artifacts
         const errors = await manifestManager.syncArtifacts(snapshot, 'add');
         if (errors.length) {
           reportErrors(logger, errors);
           throw new Error('Error writing new artifacts.');
         }
+      }
 
-        // Until we get the Default Policy Configuration in the Endpoint package,
-        // we will add it here manually at creation time.
-        // @ts-ignore
-        if (newPackageConfig.inputs.length === 0) {
-          updatedPackageConfig = {
-            ...newPackageConfig,
-            inputs: [
-              {
-                type: 'endpoint',
-                enabled: true,
-                streams: [],
-                config: {
-                  artifact_manifest: {
-                    value: snapshot.manifest.toEndpointFormat(),
-                  },
-                  policy: {
-                    value: policyConfigFactory(),
-                  },
-                },
-              },
-            ],
-          };
-        }
+      if (snapshot) {
+        updatedPackageConfig.inputs[0].config.artifact_manifest = {
+          value: snapshot.manifest.toEndpointFormat(),
+        };
       }
 
       return updatedPackageConfig;

x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts

@@ -4,9 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-// eslint-disable-next-line max-classes-per-file
 import { savedObjectsClientMock, loggingSystemMock } from 'src/core/server/mocks';
 import { Logger } from 'src/core/server';
+import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server';
+import { createPackageConfigServiceMock } from '../../../../../../ingest_manager/server/mocks';
 import { getFoundExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock';
 import { listMock } from '../../../../../../lists/server/mocks';
 import {
@@ -21,40 +22,6 @@ import { getArtifactClientMock } from '../artifact_client.mock';
 import { getManifestClientMock } from '../manifest_client.mock';
 import { ManifestManager } from './manifest_manager';
 
-function getMockPackageConfig() {
-  return {
-    id: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1',
-    inputs: [
-      {
-        config: {},
-      },
-    ],
-    revision: 1,
-    version: 'abcd', // TODO: not yet implemented in ingest_manager (https://github.com/elastic/kibana/issues/69992)
-    updated_at: '2020-06-25T16:03:38.159292',
-    updated_by: 'kibana',
-    created_at: '2020-06-25T16:03:38.159292',
-    created_by: 'kibana',
-  };
-}
-
-class PackageConfigServiceMock {
-  public create = jest.fn().mockResolvedValue(getMockPackageConfig());
-  public get = jest.fn().mockResolvedValue(getMockPackageConfig());
-  public getByIds = jest.fn().mockResolvedValue([getMockPackageConfig()]);
-  public list = jest.fn().mockResolvedValue({
-    items: [getMockPackageConfig()],
-    total: 1,
-    page: 1,
-    perPage: 20,
-  });
-  public update = jest.fn().mockResolvedValue(getMockPackageConfig());
-}
-
-export function getPackageConfigServiceMock() {
-  return new PackageConfigServiceMock();
-}
-
 async function mockBuildExceptionListArtifacts(
   os: string,
   schemaVersion: string
@@ -66,35 +33,33 @@ async function mockBuildExceptionListArtifacts(
   return [await buildArtifact(exceptions, os, schemaVersion)];
 }
 
-// @ts-ignore
 export class ManifestManagerMock extends ManifestManager {
-  // @ts-ignore
-  private buildExceptionListArtifacts = async () => {
-    return mockBuildExceptionListArtifacts('linux', 'v1');
-  };
+  protected buildExceptionListArtifacts = jest
+    .fn()
+    .mockResolvedValue(mockBuildExceptionListArtifacts('linux', 'v1'));
 
-  // @ts-ignore
-  private getLastDispatchedManifest = jest
+  public getLastDispatchedManifest = jest
     .fn()
     .mockResolvedValue(new Manifest(new Date(), 'v1', ManifestConstants.INITIAL_VERSION));
 
-  // @ts-ignore
-  private getManifestClient = jest
+  protected getManifestClient = jest
     .fn()
     .mockReturnValue(getManifestClientMock(this.savedObjectsClient));
+
+  public syncArtifacts = jest.fn().mockResolvedValue([]);
 }
 
 export const getManifestManagerMock = (opts?: {
   cache?: ExceptionsCache;
-  packageConfigService?: PackageConfigServiceMock;
+  packageConfigService?: jest.Mocked<PackageConfigServiceInterface>;
   savedObjectsClient?: ReturnType<typeof savedObjectsClientMock.create>;
 }): ManifestManagerMock => {
   let cache = new ExceptionsCache(5);
   if (opts?.cache !== undefined) {
     cache = opts.cache;
   }
 
-  let packageConfigService = getPackageConfigServiceMock();
+  let packageConfigService = createPackageConfigServiceMock();
   if (opts?.packageConfigService !== undefined) {
     packageConfigService = opts.packageConfigService;
   }
@@ -107,7 +72,6 @@ export const getManifestManagerMock = (opts?: {
   const manifestManager = new ManifestManagerMock({
     artifactClient: getArtifactClientMock(savedObjectsClient),
     cache,
-    // @ts-ignore
     packageConfigService,
     exceptionListClient: listMock.getExceptionListClient(),
     logger: loggingSystemMock.create().get() as jest.Mocked<Logger>,

x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts

@@ -64,7 +64,7 @@ export class ManifestManager {
    * @param schemaVersion The schema version of the manifest.
    * @returns {ManifestClient} A ManifestClient scoped to the provided schemaVersion.
    */
-  private getManifestClient(schemaVersion: string): ManifestClient {
+  protected getManifestClient(schemaVersion: string): ManifestClient {
     return new ManifestClient(this.savedObjectsClient, schemaVersion as ManifestSchemaVersion);
   }
 
@@ -76,7 +76,7 @@ export class ManifestManager {
    * @returns {Promise<InternalArtifactSchema[]>} An array of uncompressed artifacts built from exception-list-agnostic SOs.
    * @throws Throws/rejects if there are errors building the list.
    */
-  private async buildExceptionListArtifacts(
+  protected async buildExceptionListArtifacts(
     schemaVersion: string
   ): Promise<InternalArtifactSchema[]> {
     // TODO: should wrap in try/catch?
@@ -96,42 +96,6 @@ export class ManifestManager {
     );
   }
 
-  /**
-   * Returns the last dispatched manifest based on the current state of the
-   * user-artifact-manifest SO.
-   *
-   * @param schemaVersion The schema version of the manifest.
-   * @returns {Promise<Manifest | null>} The last dispatched manifest, or null if does not exist.
-   * @throws Throws/rejects if there is an unexpected error retrieving the manifest.
-   */
-  private async getLastDispatchedManifest(schemaVersion: string): Promise<Manifest | null> {
-    try {
-      const manifestClient = this.getManifestClient(schemaVersion);
-      const manifestSo = await manifestClient.getManifest();
-
-      if (manifestSo.version === undefined) {
-        throw new Error('No version returned for manifest.');
-      }
-
-      const manifest = new Manifest(
-        new Date(manifestSo.attributes.created),
-        schemaVersion,
-        manifestSo.version
-      );
-
-      for (const id of manifestSo.attributes.ids) {
-        const artifactSo = await this.artifactClient.getArtifact(id);
-        manifest.addEntry(artifactSo.attributes);
-      }
-      return manifest;
-    } catch (err) {
-      if (err.output.statusCode !== 404) {
-        throw err;
-      }
-      return null;
-    }
-  }
-
   /**
    * Writes new artifact SOs based on provided snapshot.
    *
@@ -186,6 +150,42 @@ export class ManifestManager {
     return errors;
   }
 
+  /**
+   * Returns the last dispatched manifest based on the current state of the
+   * user-artifact-manifest SO.
+   *
+   * @param schemaVersion The schema version of the manifest.
+   * @returns {Promise<Manifest | null>} The last dispatched manifest, or null if does not exist.
+   * @throws Throws/rejects if there is an unexpected error retrieving the manifest.
+   */
+  public async getLastDispatchedManifest(schemaVersion: string): Promise<Manifest | null> {
+    try {
+      const manifestClient = this.getManifestClient(schemaVersion);
+      const manifestSo = await manifestClient.getManifest();
+
+      if (manifestSo.version === undefined) {
+        throw new Error('No version returned for manifest.');
+      }
+
+      const manifest = new Manifest(
+        new Date(manifestSo.attributes.created),
+        schemaVersion,
+        manifestSo.version
+      );
+
+      for (const id of manifestSo.attributes.ids) {
+        const artifactSo = await this.artifactClient.getArtifact(id);
+        manifest.addEntry(artifactSo.attributes);
+      }
+      return manifest;
+    } catch (err) {
+      if (err.output.statusCode !== 404) {
+        throw err;
+      }
+      return null;
+    }
+  }
+
   /**
    * Snapshots a manifest based on current state of exception-list-agnostic SOs.
    *