From 3b4814ba48f0088ed62ded231d255a9bfe4fa40e Mon Sep 17 00:00:00 2001 From: Stacey Gammon Date: Mon, 18 May 2020 12:48:50 -0400 Subject: [PATCH 1/5] dont hide errors (#66764) Co-authored-by: Elastic Machine --- src/core/public/application/ui/app_container.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/public/application/ui/app_container.tsx b/src/core/public/application/ui/app_container.tsx index aad7e6dcf270a..4317ede547202 100644 --- a/src/core/public/application/ui/app_container.tsx +++ b/src/core/public/application/ui/app_container.tsx @@ -87,6 +87,8 @@ export const AppContainer: FunctionComponent = ({ })) || null; } catch (e) { // TODO: add error UI + // eslint-disable-next-line no-console + console.error(e); } finally { setShowSpinner(false); setIsMounting(false); From 1c61ca2c6d7321542ddf5513effb42f2beb7ead6 Mon Sep 17 00:00:00 2001 From: Pete Harverson Date: Mon, 18 May 2020 18:13:43 +0100 Subject: [PATCH 2/5] [ML] Enhances api docs for modules endpoints (#66738) * [ML] Enhances api docs for modules endpoints * [ML] Edits to modules schema following review Co-authored-by: Elastic Machine --- .../routes/apidoc_scripts/schema_worker.ts | 7 +- x-pack/plugins/ml/server/routes/modules.ts | 336 ++++++++++++++++-- .../ml/server/routes/schemas/modules.ts | 69 +++- 3 files changed, 385 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts index 7514e482783b3..0eaf3143eaac0 100644 --- a/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts +++ b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts @@ -44,9 +44,10 @@ export function postProcess(parsedFiles: any[]): void { */ function updateBlockParameters(docEntries: DocEntry[], block: Block, paramsGroup: string): void { if (!block.local.parameter) { - block.local.parameter = { - fields: {}, - }; + block.local.parameter = {}; + } + if (!block.local.parameter.fields) { + block.local.parameter.fields = {}; } if (!block.local.parameter.fields![paramsGroup]) { diff --git a/x-pack/plugins/ml/server/routes/modules.ts b/x-pack/plugins/ml/server/routes/modules.ts index 622ae66ede426..ade3d3eca90ea 100644 --- a/x-pack/plugins/ml/server/routes/modules.ts +++ b/x-pack/plugins/ml/server/routes/modules.ts @@ -4,13 +4,18 @@ * you may not use this file except in compliance with the Elastic License. */ -import { schema, TypeOf } from '@kbn/config-schema'; +import { TypeOf } from '@kbn/config-schema'; import { RequestHandlerContext } from 'kibana/server'; import { DatafeedOverride, JobOverride } from '../../common/types/modules'; import { wrapError } from '../client/error_wrapper'; import { DataRecognizer } from '../models/data_recognizer'; -import { getModuleIdParamSchema, setupModuleBodySchema } from './schemas/modules'; +import { + moduleIdParamSchema, + optionalModuleIdParamSchema, + modulesIndexPatternTitleSchema, + setupModuleBodySchema, +} from './schemas/modules'; import { RouteInitialization } from '../types'; function recognize(context: RequestHandlerContext, indexPatternTitle: string) { @@ -85,17 +90,33 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) { * * @api {get} /api/ml/modules/recognize/:indexPatternTitle Recognize index pattern * @apiName RecognizeIndex - * @apiDescription Returns the list of modules that matching the index pattern. - * - * @apiParam {String} indexPatternTitle Index pattern title. + * @apiDescription By supplying an index pattern, discover if any of the modules are a match for data in that index. + * @apiSchema (params) modulesIndexPatternTitleSchema + * @apiSuccess {object[]} modules Array of objects describing the modules which match the index pattern. + * @apiSuccessExample {json} Success-Response: + * [{ + * "id": "nginx_ecs", + * "query": { + * "bool": { + * "filter": [ + * { "term": { "event.dataset": "nginx.access" } }, + * { "exists": { "field": "source.address" } }, + * { "exists": { "field": "url.original" } }, + * { "exists": { "field": "http.response.status_code" } } + * ] + * } + * }, + * "description": "Find unusual activity in HTTP access logs from filebeat (ECS)", + * "logo": { + * "icon": "logoNginx" + * } + * }] */ router.get( { path: '/api/ml/modules/recognize/{indexPatternTitle}', validate: { - params: schema.object({ - indexPatternTitle: schema.string(), - }), + params: modulesIndexPatternTitleSchema, }, options: { tags: ['access:ml:canCreateJob'], @@ -118,17 +139,114 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) { * * @api {get} /api/ml/modules/get_module/:moduleId Get module * @apiName GetModule - * @apiDescription Returns module by id. - * - * @apiParam {String} [moduleId] Module id + * @apiDescription Retrieve a whole ML module, containing jobs, datafeeds and saved objects. If + * no module ID is supplied, returns all modules. + * @apiSchema (params) moduleIdParamSchema + * @apiSuccess {object} module When a module ID is specified, returns a module object containing + * all of the jobs, datafeeds and saved objects which will be created when the module is setup. + * @apiSuccess {object[]} modules If no module ID is supplied, an array of all modules will be returned. + * @apiSuccessExample {json} Success-Response: + * { + * "id":"sample_data_ecommerce", + * "title":"Kibana sample data eCommerce", + * "description":"Find anomalies in eCommerce total sales data", + * "type":"Sample Dataset", + * "logoFile":"logo.json", + * "defaultIndexPattern":"kibana_sample_data_ecommerce", + * "query":{ + * "bool":{ + * "filter":[ + * { + * "term":{ + * "_index":"kibana_sample_data_ecommerce" + * } + * } + * ] + * } + * }, + * "jobs":[ + * { + * "id":"high_sum_total_sales", + * "config":{ + * "groups":[ + * "kibana_sample_data", + * "kibana_sample_ecommerce" + * ], + * "description":"Find customers spending an unusually high amount in an hour", + * "analysis_config":{ + * "bucket_span":"1h", + * "detectors":[ + * { + * "detector_description":"High total sales", + * "function":"high_sum", + * "field_name":"taxful_total_price", + * "over_field_name":"customer_full_name.keyword" + * } + * ], + * "influencers":[ + * "customer_full_name.keyword", + * "category.keyword" + * ] + * }, + * "analysis_limits":{ + * "model_memory_limit":"10mb" + * }, + * "data_description":{ + * "time_field":"order_date" + * }, + * "model_plot_config":{ + * "enabled":true + * }, + * "custom_settings":{ + * "created_by":"ml-module-sample", + * "custom_urls":[ + * { + * "url_name":"Raw data", + * "url_value":"kibana#/discover?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a + * (index:ff959d40-b880-11e8-a6d9-e546fe2bba5f,query:(language:kuery,query:'customer_full_name + * keyword:\"$customer_full_name.keyword$\"'),sort:!('@timestamp',desc))" + * }, + * { + * "url_name":"Data dashboard", + * "url_value":"kibana#/dashboard/722b74f0-b882-11e8-a6d9-e546fe2bba5f?_g=(filters:!(),time:(from:'$earliest$', + * mode:absolute,to:'$latest$'))&_a=(filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f + * index:'INDEX_PATTERN_ID', key:customer_full_name.keyword,negate:!f,params:(query:'$customer_full_name.keyword$') + * type:phrase,value:'$customer_full_name.keyword$'),query:(match:(customer_full_name.keyword: + * (query:'$customer_full_name.keyword$',type:phrase))))),query:(language:kuery, query:''))" + * } + * ] + * } + * } + * } + * ], + * "datafeeds":[ + * { + * "id":"datafeed-high_sum_total_sales", + * "config":{ + * "job_id":"high_sum_total_sales", + * "indexes":[ + * "INDEX_PATTERN_NAME" + * ], + * "query":{ + * "bool":{ + * "filter":[ + * { + * "term":{ "_index":"kibana_sample_data_ecommerce" } + * } + * ] + * } + * } + * } + * } + * ], + * "kibana":{} + * } */ router.get( { path: '/api/ml/modules/get_module/{moduleId?}', validate: { - params: schema.object({ - ...getModuleIdParamSchema(true), - }), + params: optionalModuleIdParamSchema, }, options: { tags: ['access:ml:canGetJobs'], @@ -154,17 +272,148 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) { /** * @apiGroup Modules * - * @api {post} /api/ml/modules/setup/:moduleId Setup module + * @api {post} /api/ml/modules/setup/:moduleId Set up module * @apiName SetupModule - * @apiDescription Created module items. - * + * @apiDescription Runs the module setup process. + * This creates jobs, datafeeds and kibana saved objects. It allows for customization of the module, + * overriding the default configuration. It also allows the user to start the datafeed. + * @apiSchema (params) moduleIdParamSchema * @apiSchema (body) setupModuleBodySchema + * @apiParamExample {json} jobOverrides-no-job-ID: + * "jobOverrides": { + * "analysis_limits": { + * "model_memory_limit": "13mb" + * } + * } + * @apiParamExample {json} jobOverrides-with-job-ID: + * "jobOverrides": [ + * { + * "analysis_limits": { + * "job_id": "foo" + * "model_memory_limit": "13mb" + * } + * } + * ] + * @apiParamExample {json} datafeedOverrides: + * "datafeedOverrides": [ + * { + * "scroll_size": 1001 + * }, + * { + * "job_id": "visitor_rate_ecs", + * "frequency": "30m" + * } + * ] + * @apiParamExample {json} query-overrrides-datafeedOverrides-query: + * { + * "query": {"bool":{"must":[{"match_all":{}}]}} + * "datafeedOverrides": { + * "query": {} + * } + * } + * @apiSuccess {object} results An object containing the results of creating the items in a module, + * i.e. the jobs, datafeeds and saved objects. Each item is listed by id with a success flag + * signifying whether the creation was successful. If the item creation failed, an error object + * with also be supplied containing the error. + * @apiSuccessExample {json} Success-Response: + * { + * "jobs": [{ + * "id": "test-visitor_rate_ecs", + * "success": true + * }, { + * "id": "test-status_code_rate_ecs", + * "success": true + * }, { + * "id": "test-source_ip_url_count_ecs", + * "success": true + * }, { + * "id": "test-source_ip_request_rate_ecs", + * "success": true + * }, { + * "id": "test-low_request_rate_ecs", + * "success": true + * }], + * "datafeeds": [{ + * "id": "datafeed-test-visitor_rate_ecs", + * "success": true, + * "started": false + * }, { + * "id": "datafeed-test-status_code_rate_ecs", + * "success": true, + * "started": false + * }, { + * "id": "datafeed-test-source_ip_url_count_ecs", + * "success": true, + * "started": false + * }, { + * "id": "datafeed-test-low_request_rate_ecs", + * "success": true, + * "started": false + * }, { + * "id": "datafeed-test-source_ip_request_rate_ecs", + * "success": true, + * "started": false + * }], + * "kibana": { + * "dashboard": [{ + * "id": "ml_http_access_explorer_ecs", + * "success": true + * }], + * "search": [{ + * "id": "ml_http_access_filebeat_ecs", + * "success": true + * }], + * "visualization": [{ + * "id": "ml_http_access_map_ecs", + * "success": true + * }, { + * "id": "ml_http_access_source_ip_timechart_ecs", + * "success": true + * }, { + * "id": "ml_http_access_status_code_timechart_ecs", + * "success": true + * }, { + * "id": "ml_http_access_top_source_ips_table_ecs", + * "success": true + * }, { + * "id": "ml_http_access_top_urls_table_ecs", + * "success": true + * }, { + * "id": "ml_http_access_events_timechart_ecs", + * "success": true + * }, { + * "id": "ml_http_access_unique_count_url_timechart_ecs", + * "success": true + * }] + * } + * } + * @apiSuccessExample {json} Error-Response: + * { + * "jobs": [{ + * "id": "test-status_code_rate_ecs", + * "success": false, + * "error": { + * "msg": "[resource_already_exists_exception] The job cannot be created with the Id 'test-status_code_rate_ecs'. The Id is + * already used.", + * "path": "/_ml/anomaly_detectors/test-status_code_rate_ecs", + * "query": {}, + * "body": "{...}", + * "statusCode": 400, + * "response": "{\"error\":{\"root_cause\":[{\"type\":\"resource_already_exists_exception\",\"reason\":\"The job cannot be created + * with the Id 'test-status_code_rate_ecs'. The Id is already used.\"}],\"type\":\"resource_already_exists_exception\", + * \"reason\":\"The job cannot be created with the Id 'test-status_code_rate_ecs'. The Id is already used.\"},\"status\":400}" + * } + * }, + * }, + * ... + * }] + * } */ router.post( { path: '/api/ml/modules/setup/{moduleId}', validate: { - params: schema.object(getModuleIdParamSchema()), + params: moduleIdParamSchema, body: setupModuleBodySchema, }, options: { @@ -217,15 +466,58 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) { * * @api {post} /api/ml/modules/jobs_exist/:moduleId Check if module jobs exist * @apiName CheckExistingModuleJobs - * @apiDescription Checks if the jobs in the module have been created. - * - * @apiParam {String} moduleId Module id + * @apiDescription Check whether the jobs in the module with the specified ID exist in the + * current list of jobs. The check runs a test to see if any of the jobs in existence + * have an ID which ends with the ID of each job in the module. This is done as a prefix + * may be supplied in the setup endpoint which is added to the start of the ID of every job in the module. + * @apiSchema (params) moduleIdParamSchema + * @apiSuccess {boolean} jobsExist true if all the jobs in the module have a matching job with an + * ID which ends with the job ID specified in the module, false otherwise. + * @apiSuccess {Object[]} jobs present if the jobs do all exist, with each object having keys of id, + * and optionally earliestTimestampMs, latestTimestampMs, latestResultsTimestampMs + * properties if the job has processed any data. + * @apiSuccessExample {json} Success-Response: + * { + * "jobsExist":true, + * "jobs":[ + * { + * "id":"nginx_low_request_rate_ecs", + * "earliestTimestampMs":1547016291000, + * "latestTimestampMs":1548256497000 + * "latestResultsTimestampMs":1548255600000 + * }, + * { + * "id":"nginx_source_ip_request_rate_ecs", + * "earliestTimestampMs":1547015109000, + * "latestTimestampMs":1548257222000 + * "latestResultsTimestampMs":1548255600000 + * }, + * { + * "id":"nginx_source_ip_url_count_ecs", + * "earliestTimestampMs":1547015109000, + * "latestTimestampMs":1548257222000 + * "latestResultsTimestampMs":1548255600000 + * }, + * { + * "id":"nginx_status_code_rate_ecs", + * "earliestTimestampMs":1547015109000, + * "latestTimestampMs":1548257222000 + * "latestResultsTimestampMs":1548255600000 + * }, + * { + * "id":"nginx_visitor_rate_ecs", + * "earliestTimestampMs":1547016291000, + * "latestTimestampMs":1548256497000 + * "latestResultsTimestampMs":1548255600000 + * } + * ] + * } */ router.get( { path: '/api/ml/modules/jobs_exist/{moduleId}', validate: { - params: schema.object(getModuleIdParamSchema()), + params: moduleIdParamSchema, }, options: { tags: ['access:ml:canGetJobs'], diff --git a/x-pack/plugins/ml/server/routes/schemas/modules.ts b/x-pack/plugins/ml/server/routes/schemas/modules.ts index 98e3d80f0ff84..23148c14c734e 100644 --- a/x-pack/plugins/ml/server/routes/schemas/modules.ts +++ b/x-pack/plugins/ml/server/routes/schemas/modules.ts @@ -7,24 +7,89 @@ import { schema } from '@kbn/config-schema'; export const setupModuleBodySchema = schema.object({ + /** + * Job ID prefix. This will be added to the start of the ID every job created by the module (optional). + */ prefix: schema.maybe(schema.string()), + /** + * List of group IDs. This will override the groups assigned to each job created by the module (optional). + */ groups: schema.maybe(schema.arrayOf(schema.string())), + /** + * Name of kibana index pattern. Overrides the index used in each datafeed and each index pattern + * used in the custom urls and saved objects created by the module. A matching index pattern must + * exist in kibana if the module contains custom urls or saved objects which rely on an index pattern ID. + * If the module does not contain custom urls or saved objects which require an index pattern ID, the + * indexPatternName can be any index name or pattern that will match an ES index. It can also be a comma + * separated list of names. If no indexPatternName is supplied, the default index pattern specified in + * the manifest.json will be used (optional). + */ indexPatternName: schema.maybe(schema.string()), + /** + * ES Query DSL object. Overrides the query object for each datafeed created by the module (optional). + */ query: schema.maybe(schema.any()), + /** + * Flag to specify that each job created by the module uses a dedicated index (optional). + */ useDedicatedIndex: schema.maybe(schema.boolean()), + /** + * Flag to specify that each datafeed created by the module is started once saved. Defaults to false (optional). + */ startDatafeed: schema.maybe(schema.boolean()), + /** + * Start date for datafeed. Specified in epoch seconds. Only used if startDatafeed is true. + * If not specified, a value of 0 is used i.e. start at the beginning of the data (optional). + */ start: schema.maybe(schema.number()), + /** + * End date for datafeed. Specified in epoch seconds. Only used if startDatafeed is true. + * If not specified, the datafeed will continue to run in real time (optional). + */ end: schema.maybe(schema.number()), + /** + * Partial job configuration which will override jobs contained in the module. Can be an array of objects. + * If a job_id is specified, only that job in the module will be overridden. + * Applied before any of the existing + * overridable options (e.g. useDedicatedIndex, groups, indexPatternName etc) + * and so can be overridden themselves (optional). + */ jobOverrides: schema.maybe(schema.any()), + /** + * Partial datafeed configuration which will override datafeeds contained in the module. + * Can be an array of objects. + * If a datafeed_id or a job_id is specified, + * only that datafeed in the module will be overridden. Applied before any of the existing + * overridable options (e.g. useDedicatedIndex, groups, indexPatternName etc) + * and so can be overridden themselves (optional). + */ datafeedOverrides: schema.maybe(schema.any()), /** * Indicates whether an estimate of the model memory limit - * should be made by checking the cardinality of fields in the job configurations. + * should be made by checking the cardinality of fields in the job configurations (optional). */ estimateModelMemory: schema.maybe(schema.boolean()), }); export const getModuleIdParamSchema = (optional = false) => { const stringType = schema.string(); - return { moduleId: optional ? schema.maybe(stringType) : stringType }; + return schema.object({ + /** + * ID of the module. + */ + moduleId: optional ? schema.maybe(stringType) : stringType, + }); }; + +export const optionalModuleIdParamSchema = getModuleIdParamSchema(true); + +export const moduleIdParamSchema = getModuleIdParamSchema(false); + +export const modulesIndexPatternTitleSchema = schema.object({ + /** + * Index pattern to recognize. Note that this does not need to be a Kibana + * index pattern, and can be the name of a single Elasticsearch index, + * or include a wildcard (*) to match multiple indices. + */ + indexPatternTitle: schema.string(), +}); From 84e06afddc264c20349f357e62f0794d22de5db4 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Mon, 18 May 2020 19:18:25 +0200 Subject: [PATCH 3/5] [Ingest Manager] Better handling of package installation problems (#66541) * Better handle non-available package registry * Log errors in route handler only * Await installation of prebuilt component templates * Remove leftover import * Remove useless use of await. --- .../server/routes/setup/handlers.ts | 10 ++++++ .../epm/elasticsearch/template/install.ts | 36 +++++++++++-------- .../server/services/epm/packages/install.ts | 18 ++++------ .../server/services/epm/registry/requests.ts | 2 +- 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/routes/setup/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/setup/handlers.ts index 12781f2f77d17..8bc80c69ce9b2 100644 --- a/x-pack/plugins/ingest_manager/server/routes/setup/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/setup/handlers.ts @@ -74,12 +74,22 @@ export const createFleetSetupHandler: RequestHandler< export const ingestManagerSetupHandler: RequestHandler = async (context, request, response) => { const soClient = context.core.savedObjects.client; const callCluster = context.core.elasticsearch.adminClient.callAsCurrentUser; + const logger = appContextService.getLogger(); try { await setupIngestManager(soClient, callCluster); return response.ok({ body: { isInitialized: true }, }); } catch (e) { + if (e.isBoom) { + logger.error(e.output.payload.message); + return response.customError({ + statusCode: e.output.statusCode, + body: { message: e.output.payload.message }, + }); + } + logger.error(e.message); + logger.error(e.stack); return response.customError({ statusCode: 500, body: { message: e.message }, diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts index 6ef6f863753b5..2c452f16cc104 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import Boom from 'boom'; import { Dataset, RegistryPackage, ElasticsearchAssetType, TemplateRef } from '../../../../types'; import { CallESAsCurrentUser } from '../../../../types'; import { Field, loadFieldsFromYaml, processFields } from '../../fields/field'; @@ -20,8 +21,8 @@ export const installTemplates = async ( // install any pre-built index template assets, // atm, this is only the base package's global index templates // Install component templates first, as they are used by the index templates - installPreBuiltComponentTemplates(pkgName, pkgVersion, callCluster); - installPreBuiltTemplates(pkgName, pkgVersion, callCluster); + await installPreBuiltComponentTemplates(pkgName, pkgVersion, callCluster); + await installPreBuiltTemplates(pkgName, pkgVersion, callCluster); // build templates per dataset from yml files const datasets = registryPackage.datasets; @@ -53,16 +54,7 @@ const installPreBuiltTemplates = async ( pkgVersion, (entry: Registry.ArchiveEntry) => isTemplate(entry) ); - // templatePaths.forEach(async path => { - // const { file } = Registry.pathParts(path); - // const templateName = file.substr(0, file.lastIndexOf('.')); - // const content = JSON.parse(Registry.getAsset(path).toString('utf8')); - // await callCluster('indices.putTemplate', { - // name: templateName, - // body: content, - // }); - // }); - templatePaths.forEach(async path => { + const templateInstallPromises = templatePaths.map(async path => { const { file } = Registry.pathParts(path); const templateName = file.substr(0, file.lastIndexOf('.')); const content = JSON.parse(Registry.getAsset(path).toString('utf8')); @@ -91,8 +83,15 @@ const installPreBuiltTemplates = async ( // The existing convenience endpoint `indices.putTemplate` only sends to _template, // which does not support v2 templates. // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await callCluster('transport.request', callClusterParams); + return callCluster('transport.request', callClusterParams); }); + try { + return await Promise.all(templateInstallPromises); + } catch (e) { + throw new Boom(`Error installing prebuilt index templates ${e.message}`, { + statusCode: 400, + }); + } }; const installPreBuiltComponentTemplates = async ( @@ -105,7 +104,7 @@ const installPreBuiltComponentTemplates = async ( pkgVersion, (entry: Registry.ArchiveEntry) => isComponentTemplate(entry) ); - templatePaths.forEach(async path => { + const templateInstallPromises = templatePaths.map(async path => { const { file } = Registry.pathParts(path); const templateName = file.substr(0, file.lastIndexOf('.')); const content = JSON.parse(Registry.getAsset(path).toString('utf8')); @@ -124,8 +123,15 @@ const installPreBuiltComponentTemplates = async ( // This uses the catch-all endpoint 'transport.request' because there is no // convenience endpoint for component templates yet. // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await callCluster('transport.request', callClusterParams); + return callCluster('transport.request', callClusterParams); }); + try { + return await Promise.all(templateInstallPromises); + } catch (e) { + throw new Boom(`Error installing prebuilt component templates ${e.message}`, { + statusCode: 400, + }); + } }; const isTemplate = ({ path }: Registry.ArchiveEntry) => { diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index 632bc3ac9b69f..79a5e98b9507d 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -73,17 +73,13 @@ export async function ensureInstalledPackage(options: { if (installedPackage) { return installedPackage; } - // if the requested packaged was not found to be installed, try installing - try { - await installLatestPackage({ - savedObjectsClient, - pkgName, - callCluster, - }); - return await getInstallation({ savedObjectsClient, pkgName }); - } catch (err) { - throw new Error(err.message); - } + // if the requested packaged was not found to be installed, install + await installLatestPackage({ + savedObjectsClient, + pkgName, + callCluster, + }); + return await getInstallation({ savedObjectsClient, pkgName }); } export async function installPackage(options: { diff --git a/x-pack/plugins/ingest_manager/server/services/epm/registry/requests.ts b/x-pack/plugins/ingest_manager/server/services/epm/registry/requests.ts index 654aa8aae1355..93e475cbc5956 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/registry/requests.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/registry/requests.ts @@ -17,7 +17,7 @@ export async function getResponse(url: string): Promise { throw new Boom(response.statusText, { statusCode: response.status }); } } catch (e) { - throw Boom.boomify(e); + throw new Boom(`Error connecting to package registry: ${e.message}`, { statusCode: 502 }); } } From 4f1b9a234f556052d265ae8db6f06689e54d943e Mon Sep 17 00:00:00 2001 From: Spencer Date: Mon, 18 May 2020 10:20:53 -0700 Subject: [PATCH 4/5] [dev/cli] add support for --no-cache (#66837) --- src/cli/cluster/run_kbn_optimizer.ts | 1 + src/cli/serve/serve.js | 1 + 2 files changed, 2 insertions(+) diff --git a/src/cli/cluster/run_kbn_optimizer.ts b/src/cli/cluster/run_kbn_optimizer.ts index b811fc1f6b294..e4bd6b98dd2b9 100644 --- a/src/cli/cluster/run_kbn_optimizer.ts +++ b/src/cli/cluster/run_kbn_optimizer.ts @@ -35,6 +35,7 @@ export function runKbnOptimizer(opts: Record, config: LegacyConfig) repoRoot: REPO_ROOT, watch: true, includeCoreBundle: true, + cache: !!opts.cache, oss: !!opts.oss, examples: !!opts.runExamples, pluginPaths: config.get('plugins.paths'), diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js index 471939121143a..6b0daebd5a042 100644 --- a/src/cli/serve/serve.js +++ b/src/cli/serve/serve.js @@ -218,6 +218,7 @@ export default function(program) { "Don't put a proxy in front of the dev server, which adds a random basePath" ) .option('--no-watch', 'Prevents automatic restarts of the server in --dev mode') + .option('--no-cache', 'Disable the kbn/optimizer cache') .option('--no-dev-config', 'Prevents loading the kibana.dev.yml file in --dev mode'); } From 1006c315cb067d34ac973928f40c0f10d1e53273 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Mon, 18 May 2020 14:25:44 -0500 Subject: [PATCH 5/5] [APM] Lowercase agent names so icons work (#66824) * [APM] Lowercase agent names so icons work .NET agent name can be reported as "dotNet" instead of "dotnet". Lowercase the key so either one will work. * Extract getNormalizedAgentName --- x-pack/plugins/apm/common/agent_name.ts | 13 +++++++++++++ .../components/app/ServiceMap/Cytoscape.stories.tsx | 7 +++++++ .../apm/public/components/app/ServiceMap/icons.ts | 5 ++--- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/apm/common/agent_name.ts b/x-pack/plugins/apm/common/agent_name.ts index 085828b729ea5..dac29a4f50682 100644 --- a/x-pack/plugins/apm/common/agent_name.ts +++ b/x-pack/plugins/apm/common/agent_name.ts @@ -41,3 +41,16 @@ export function isJavaAgentName( ): agentName is 'java' { return agentName === 'java'; } + +/** + * "Normalizes" and agent name by: + * + * * Converting to lowercase + * * Converting "rum-js" to "js-base" + * + * This helps dealing with some older agent versions + */ +export function getNormalizedAgentName(agentName?: string) { + const lowercased = agentName && agentName.toLowerCase(); + return isRumAgentName(lowercased) ? 'js-base' : lowercased; +} diff --git a/x-pack/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx b/x-pack/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx index 340c299f52c0b..2d1e99096a44f 100644 --- a/x-pack/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx +++ b/x-pack/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx @@ -186,6 +186,13 @@ storiesOf('app/ServiceMap/Cytoscape', module) 'agent.name': 'dotnet' } }, + { + data: { + id: 'dotNet', + 'service.name': 'dotNet service', + 'agent.name': 'dotNet' + } + }, { data: { id: 'go', diff --git a/x-pack/plugins/apm/public/components/app/ServiceMap/icons.ts b/x-pack/plugins/apm/public/components/app/ServiceMap/icons.ts index 9fe5cbd23b07c..1b4bf1b77791c 100644 --- a/x-pack/plugins/apm/public/components/app/ServiceMap/icons.ts +++ b/x-pack/plugins/apm/public/components/app/ServiceMap/icons.ts @@ -5,7 +5,7 @@ */ import cytoscape from 'cytoscape'; -import { isRumAgentName } from '../../../../common/agent_name'; +import { getNormalizedAgentName } from '../../../../common/agent_name'; import { AGENT_NAME, SPAN_SUBTYPE, @@ -87,8 +87,7 @@ const agentIcons: { [key: string]: string } = { }; function getAgentIcon(agentName?: string) { - // RUM can have multiple names. Normalize it - const normalizedAgentName = isRumAgentName(agentName) ? 'js-base' : agentName; + const normalizedAgentName = getNormalizedAgentName(agentName); return normalizedAgentName && agentIcons[normalizedAgentName]; }