From 786e4db2ff6658688ad696be045c28d7929f4669 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 11 Dec 2019 14:21:32 +0100 Subject: [PATCH 1/3] [EPM] Cleanup ILM loading (#52632) Before the check for the ILM policy to exist triggered an exception. With this change it is a normal response also if the policy does not exist yet. A follow up issue will be created in Elasticsearch to get a HEAD request for this available. --- .../server/lib/elasticsearch/ilm/install.ts | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/x-pack/legacy/plugins/epm/server/lib/elasticsearch/ilm/install.ts b/x-pack/legacy/plugins/epm/server/lib/elasticsearch/ilm/install.ts index e84b247665e5a..0af6b6255a4d5 100644 --- a/x-pack/legacy/plugins/epm/server/lib/elasticsearch/ilm/install.ts +++ b/x-pack/legacy/plugins/epm/server/lib/elasticsearch/ilm/install.ts @@ -25,17 +25,11 @@ export async function policyExists( name: string, callCluster: CallESAsCurrentUser ): Promise { - try { - // TODO: Figure out if there is a better way to check for an ILM policy to exist that - // does not throw an exception. - await callCluster('transport.request', { - method: 'GET', - path: '/_ilm/policy/' + name, - }); - return true; - } catch (e) { - return false; - } + const response = await callCluster('transport.request', { + method: 'GET', + path: '/_ilm/policy/?filter_path=' + name, + }); - return false; + // If the response contains a key, it means the policy exists + return Object.keys(response).length > 0; } From 00b36c45900fd70802e88869c7d2a863ad03dc12 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 11 Dec 2019 14:26:20 +0100 Subject: [PATCH 2/3] [EPM] Switch to staging URL for registry (#52626) The old cluster with the registry will be removed as soon as this is merged. --- x-pack/legacy/plugins/epm/server/config.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/epm/server/config.ts b/x-pack/legacy/plugins/epm/server/config.ts index 4076b906a360a..bd32ea986c92c 100644 --- a/x-pack/legacy/plugins/epm/server/config.ts +++ b/x-pack/legacy/plugins/epm/server/config.ts @@ -34,7 +34,10 @@ export const getConfigSchema = (Joi: typeof JoiNamespace) => { const DEFAULT_CONFIG = { enabled: true, - registryUrl: 'http://package-registry.app.elstc.co', + // This is the staging url and should be later switched to https://epr.elastic.co for production + // Both are behind a CDN with caching, so upgrading the registry will not immidiately invalidate + // all the cached information. + registryUrl: 'https://epr.ea-web.elastic.dev', }; // As of 2019, this is a Singleton because of the way JavaScript modules are specified. From 189d375976f54701630a2e8a3457996ec732456b Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 11 Dec 2019 16:00:18 +0100 Subject: [PATCH 3/3] [EPM] Use Dataset interface to generate template (#52255) This will make sure we have to pass much feature params and can fully rely on the datasource object to create names for assets. --- x-pack/legacy/plugins/epm/common/types.ts | 1 + .../__snapshots__/template.test.ts.snap | 9 -------- .../lib/elasticsearch/template/install.ts | 11 ++++------ .../lib/elasticsearch/template/template.ts | 21 ++++--------------- 4 files changed, 9 insertions(+), 33 deletions(-) diff --git a/x-pack/legacy/plugins/epm/common/types.ts b/x-pack/legacy/plugins/epm/common/types.ts index 6c1a34ae834a5..a8f10a4ad41db 100644 --- a/x-pack/legacy/plugins/epm/common/types.ts +++ b/x-pack/legacy/plugins/epm/common/types.ts @@ -122,6 +122,7 @@ export interface Dataset { ingeset_pipeline: string; vars: object[]; type: string; + // This is for convenience and not in the output from the registry. When creating a dataset, this info should be added. package: string; } diff --git a/x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/__snapshots__/template.test.ts.snap index a17981a60d304..e15c3900efa8c 100644 --- a/x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -41,15 +41,6 @@ exports[`tests loading fields.yml: base.yml 1`] = ` }, "match_mapping_type": "string" } - }, - { - "labels": { - "path_match": "labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } } ], "date_detection": false, diff --git a/x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/install.ts b/x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/install.ts index a4cc66326f9ca..d31d00e7f4d7c 100644 --- a/x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/install.ts +++ b/x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/install.ts @@ -23,9 +23,10 @@ const isFields = (path: string) => { * The template is currently loaded with the pkgey-package-dataset */ export async function installTemplates(pkg: RegistryPackage, callCluster: CallESAsCurrentUser) { + // If no datasets exist in this package, no templates have to be installed. if (!pkg.datasets) return; - const promises = pkg.datasets.map(async dataset => { + return pkg.datasets.map(async dataset => { // Fetch all assset entries for this dataset const assetEntries = await getAssetsData(pkg, isFields, dataset.name); @@ -38,25 +39,21 @@ export async function installTemplates(pkg: RegistryPackage, callCluster: CallES } } - return installTemplate({ callCluster, fields, pkg, dataset }); + return installTemplate({ callCluster, fields, dataset }); }); - - return Promise.all(promises); } async function installTemplate({ callCluster, fields, - pkg, dataset, }: { callCluster: CallESAsCurrentUser; fields: Field[]; - pkg: RegistryPackage; dataset: Dataset; }): Promise { const mappings = generateMappings(fields); - const templateName = generateTemplateName(pkg.name, dataset.name, dataset.type); + const templateName = generateTemplateName(dataset); const template = getTemplate(templateName + '-*', mappings); // TODO: Check return values for errors await callCluster('indices.putTemplate', { diff --git a/x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/template.ts b/x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/template.ts index 39735cc7e85fd..1923db9fe20f7 100644 --- a/x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/template.ts +++ b/x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/template.ts @@ -5,6 +5,8 @@ */ import { Field } from '../../fields/field'; +import { Dataset } from '../../../../common/types'; +import { getDatasetAssetBaseName } from '../index'; export interface Template { order: number; @@ -58,13 +60,8 @@ export function generateMappings(fields: Field[]): Mappings { /** * Generates the template name out of the given information */ -export function generateTemplateName( - packageName: string, - datasetName: string, - type: string -): string { - // TODO: This is only a temporary name. More info like dataset type is needed to create full name - return type + '-' + packageName + '-' + datasetName; +export function generateTemplateName(dataset: Dataset): string { + return getDatasetAssetBaseName(dataset); } function getBaseTemplate(mappings: Mappings): Template { @@ -118,16 +115,6 @@ function getBaseTemplate(mappings: Mappings): Template { match_mapping_type: 'string', }, }, - // Example of a dynamic template - { - labels: { - path_match: 'labels.*', - mapping: { - type: 'keyword', - }, - match_mapping_type: 'string', - }, - }, ], // As we define fields ahead, we don't need any automatic field detection // This makes sure all the fields are mapped to keyword by default to prevent mapping conflicts