From 6242c7ed4bb59cfa1674f65057fa6cbe37fe0344 Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 17 Mar 2021 14:04:28 -0700 Subject: [PATCH 1/4] [kbn/optimizer] report timings when run from CLI or build (#94764) Co-authored-by: spalger Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- docs/developer/telemetry.asciidoc | 1 + packages/kbn-optimizer/src/cli.ts | 5 +- packages/kbn-optimizer/src/index.ts | 1 + .../src/report_optimizer_timings.ts | 73 +++++++++++++++++++ .../tasks/build_kibana_platform_plugins.ts | 11 ++- 5 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 packages/kbn-optimizer/src/report_optimizer_timings.ts diff --git a/docs/developer/telemetry.asciidoc b/docs/developer/telemetry.asciidoc index 75f42a860624b..45d2a140cf8b9 100644 --- a/docs/developer/telemetry.asciidoc +++ b/docs/developer/telemetry.asciidoc @@ -6,6 +6,7 @@ To help us provide a good developer experience, we track some straightforward me The operations we current report timing data for: * Total execution time of `yarn kbn bootstrap` +* Total execution time of `@kbn/optimizer` runs as well as the following metadata about the runs: The number of bundles created, the number of bundles which were cached, usage of `--watch`, `--dist`, `--workers` and `--no-cache` flags, and the count of themes being built. Along with the execution time of each execution, we ship the following information about your machine to the service: diff --git a/packages/kbn-optimizer/src/cli.ts b/packages/kbn-optimizer/src/cli.ts index 8f82f34646e60..6e3106dbc2af7 100644 --- a/packages/kbn-optimizer/src/cli.ts +++ b/packages/kbn-optimizer/src/cli.ts @@ -18,6 +18,7 @@ import { logOptimizerState } from './log_optimizer_state'; import { OptimizerConfig } from './optimizer'; import { runOptimizer } from './run_optimizer'; import { validateLimitsForAllBundles, updateBundleLimits } from './limits'; +import { reportOptimizerTimings } from './report_optimizer_timings'; function getLimitsPath(flags: Flags, defaultPath: string) { if (flags.limits) { @@ -144,7 +145,9 @@ export function runKbnOptimizerCli(options: { defaultLimitsPath: string }) { const update$ = runOptimizer(config); - await lastValueFrom(update$.pipe(logOptimizerState(log, config))); + await lastValueFrom( + update$.pipe(logOptimizerState(log, config), reportOptimizerTimings(log, config)) + ); if (updateLimits) { updateBundleLimits({ diff --git a/packages/kbn-optimizer/src/index.ts b/packages/kbn-optimizer/src/index.ts index 8d6e89008bc68..a5838a8a0fac8 100644 --- a/packages/kbn-optimizer/src/index.ts +++ b/packages/kbn-optimizer/src/index.ts @@ -12,3 +12,4 @@ export * from './log_optimizer_state'; export * from './node'; export * from './limits'; export * from './cli'; +export * from './report_optimizer_timings'; diff --git a/packages/kbn-optimizer/src/report_optimizer_timings.ts b/packages/kbn-optimizer/src/report_optimizer_timings.ts new file mode 100644 index 0000000000000..dcb3a0fba77b5 --- /dev/null +++ b/packages/kbn-optimizer/src/report_optimizer_timings.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { concatMap } from 'rxjs/operators'; +import { CiStatsReporter, ToolingLog } from '@kbn/dev-utils'; + +import { OptimizerConfig } from './optimizer'; +import { OptimizerUpdate$ } from './run_optimizer'; +import { pipeClosure } from './common'; + +export function reportOptimizerTimings(log: ToolingLog, config: OptimizerConfig) { + return pipeClosure((update$: OptimizerUpdate$) => { + let sent = false; + + const cachedBundles = new Set(); + const notCachedBundles = new Set(); + + return update$.pipe( + concatMap(async (update) => { + // if we've already sent timing data then move on + if (sent) { + return update; + } + + if (update.event?.type === 'bundle cached') { + cachedBundles.add(update.event.bundle.id); + } + if (update.event?.type === 'bundle not cached') { + notCachedBundles.add(update.event.bundle.id); + } + + // wait for the optimizer to complete, either with a success or failure + if (update.state.phase !== 'issue' && update.state.phase !== 'success') { + return update; + } + + sent = true; + const reporter = CiStatsReporter.fromEnv(log); + const time = Date.now() - update.state.startTime; + + await reporter.timings({ + timings: [ + { + group: '@kbn/optimizer', + id: 'overall time', + ms: time, + meta: { + optimizerBundleCount: config.bundles.length, + optimizerBundleCacheCount: cachedBundles.size, + optimizerBundleCachePct: Math.floor( + (cachedBundles.size / config.bundles.length) * 100 + ), + optimizerWatch: config.watch, + optimizerProduction: config.dist, + optimizerProfileWebpack: config.profileWebpack, + optimizerBundleThemeTagsCount: config.themeTags.length, + optimizerCache: config.cache, + optimizerMaxWorkerCount: config.maxWorkerCount, + }, + }, + ], + }); + + return update; + }) + ); + }); +} diff --git a/src/dev/build/tasks/build_kibana_platform_plugins.ts b/src/dev/build/tasks/build_kibana_platform_plugins.ts index 973d71043f028..edff77d458f0f 100644 --- a/src/dev/build/tasks/build_kibana_platform_plugins.ts +++ b/src/dev/build/tasks/build_kibana_platform_plugins.ts @@ -11,7 +11,12 @@ import Path from 'path'; import { REPO_ROOT } from '@kbn/utils'; import { lastValueFrom } from '@kbn/std'; import { CiStatsMetric } from '@kbn/dev-utils'; -import { runOptimizer, OptimizerConfig, logOptimizerState } from '@kbn/optimizer'; +import { + runOptimizer, + OptimizerConfig, + logOptimizerState, + reportOptimizerTimings, +} from '@kbn/optimizer'; import { Task, deleteAll, write, read } from '../lib'; @@ -30,7 +35,9 @@ export const BuildKibanaPlatformPlugins: Task = { limitsPath: Path.resolve(REPO_ROOT, 'packages/kbn-optimizer/limits.yml'), }); - await lastValueFrom(runOptimizer(config).pipe(logOptimizerState(log, config))); + await lastValueFrom( + runOptimizer(config).pipe(logOptimizerState(log, config), reportOptimizerTimings(log, config)) + ); const combinedMetrics: CiStatsMetric[] = []; const metricFilePaths: string[] = []; From 5669fac0303ffb400f3c38316813eae362836fec Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Wed, 17 Mar 2021 22:05:02 +0100 Subject: [PATCH 2/4] remove legacy doc links file (#94274) * remove legacy doc links file * remove entry from watch paths * remove unused invalid link * fix alerting.indexThreshold link * fix more links * update generated doc * update generated doc --- ...na-plugin-core-public.doclinksstart.links.md | 1 - .../kibana-plugin-core-public.doclinksstart.md | 2 +- .../core/server/kibana-plugin-core-server.md | 2 +- ...in-core-server.savedobjectsclientcontract.md | 2 +- src/core/public/doc_links/doc_links_service.ts | 11 +++++------ src/core/public/public.api.md | 1 - .../cli_dev_mode/get_server_watch_paths.test.ts | 1 - src/dev/cli_dev_mode/get_server_watch_paths.ts | 1 - .../documentation_links/documentation_links.ts | 17 ----------------- 9 files changed, 8 insertions(+), 30 deletions(-) delete mode 100644 src/legacy/ui/public/documentation_links/documentation_links.ts diff --git a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md index c95ba01981cfe..dc0e60b5986a4 100644 --- a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md +++ b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md @@ -99,7 +99,6 @@ readonly links: { readonly luceneExpressions: string; }; readonly indexPatterns: { - readonly loadingData: string; readonly introduction: string; }; readonly addData: string; diff --git a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md index 971e02cc2c195..8bcb1a8b6ca1a 100644 --- a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md +++ b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md @@ -17,5 +17,5 @@ export interface DocLinksStart | --- | --- | --- | | [DOC\_LINK\_VERSION](./kibana-plugin-core-public.doclinksstart.doc_link_version.md) | string | | | [ELASTIC\_WEBSITE\_URL](./kibana-plugin-core-public.doclinksstart.elastic_website_url.md) | string | | -| [links](./kibana-plugin-core-public.doclinksstart.links.md) | {
readonly dashboard: {
readonly guide: string;
readonly drilldowns: string;
readonly drilldownsTriggerPicker: string;
readonly urlDrilldownTemplateSyntax: string;
readonly urlDrilldownVariables: string;
};
readonly discover: Record<string, string>;
readonly filebeat: {
readonly base: string;
readonly installation: string;
readonly configuration: string;
readonly elasticsearchOutput: string;
readonly elasticsearchModule: string;
readonly startup: string;
readonly exportedFields: string;
};
readonly auditbeat: {
readonly base: string;
};
readonly metricbeat: {
readonly base: string;
readonly configure: string;
readonly httpEndpoint: string;
readonly install: string;
readonly start: string;
};
readonly enterpriseSearch: {
readonly base: string;
readonly appSearchBase: string;
readonly workplaceSearchBase: string;
};
readonly heartbeat: {
readonly base: string;
};
readonly logstash: {
readonly base: string;
};
readonly functionbeat: {
readonly base: string;
};
readonly winlogbeat: {
readonly base: string;
};
readonly aggs: {
readonly composite: string;
readonly composite_missing_bucket: string;
readonly date_histogram: string;
readonly date_range: string;
readonly date_format_pattern: string;
readonly filter: string;
readonly filters: string;
readonly geohash_grid: string;
readonly histogram: string;
readonly ip_range: string;
readonly range: string;
readonly significant_terms: string;
readonly terms: string;
readonly avg: string;
readonly avg_bucket: string;
readonly max_bucket: string;
readonly min_bucket: string;
readonly sum_bucket: string;
readonly cardinality: string;
readonly count: string;
readonly cumulative_sum: string;
readonly derivative: string;
readonly geo_bounds: string;
readonly geo_centroid: string;
readonly max: string;
readonly median: string;
readonly min: string;
readonly moving_avg: string;
readonly percentile_ranks: string;
readonly serial_diff: string;
readonly std_dev: string;
readonly sum: string;
readonly top_hits: string;
};
readonly runtimeFields: string;
readonly scriptedFields: {
readonly scriptFields: string;
readonly scriptAggs: string;
readonly painless: string;
readonly painlessApi: string;
readonly painlessLangSpec: string;
readonly painlessSyntax: string;
readonly painlessWalkthrough: string;
readonly luceneExpressions: string;
};
readonly indexPatterns: {
readonly loadingData: string;
readonly introduction: string;
};
readonly addData: string;
readonly kibana: string;
readonly elasticsearch: Record<string, string>;
readonly siem: {
readonly guide: string;
readonly gettingStarted: string;
};
readonly query: {
readonly eql: string;
readonly luceneQuerySyntax: string;
readonly queryDsl: string;
readonly kueryQuerySyntax: string;
};
readonly date: {
readonly dateMath: string;
readonly dateMathIndexNames: string;
};
readonly management: Record<string, string>;
readonly ml: Record<string, string>;
readonly transforms: Record<string, string>;
readonly visualize: Record<string, string>;
readonly apis: Readonly<{
createIndex: string;
createSnapshotLifecyclePolicy: string;
createRoleMapping: string;
createRoleMappingTemplates: string;
createApiKey: string;
createPipeline: string;
createTransformRequest: string;
cronExpressions: string;
executeWatchActionModes: string;
indexExists: string;
openIndex: string;
putComponentTemplate: string;
painlessExecute: string;
painlessExecuteAPIContexts: string;
putComponentTemplateMetadata: string;
putSnapshotLifecyclePolicy: string;
putWatch: string;
updateTransform: string;
}>;
readonly observability: Record<string, string>;
readonly alerting: Record<string, string>;
readonly maps: Record<string, string>;
readonly monitoring: Record<string, string>;
readonly security: Readonly<{
apiKeyServiceSettings: string;
clusterPrivileges: string;
elasticsearchSettings: string;
elasticsearchEnableSecurity: string;
indicesPrivileges: string;
kibanaTLS: string;
kibanaPrivileges: string;
mappingRoles: string;
mappingRolesFieldRules: string;
runAsPrivilege: string;
}>;
readonly watcher: Record<string, string>;
readonly ccs: Record<string, string>;
readonly plugins: Record<string, string>;
readonly snapshotRestore: Record<string, string>;
readonly ingest: Record<string, string>;
} | | +| [links](./kibana-plugin-core-public.doclinksstart.links.md) | {
readonly dashboard: {
readonly guide: string;
readonly drilldowns: string;
readonly drilldownsTriggerPicker: string;
readonly urlDrilldownTemplateSyntax: string;
readonly urlDrilldownVariables: string;
};
readonly discover: Record<string, string>;
readonly filebeat: {
readonly base: string;
readonly installation: string;
readonly configuration: string;
readonly elasticsearchOutput: string;
readonly elasticsearchModule: string;
readonly startup: string;
readonly exportedFields: string;
};
readonly auditbeat: {
readonly base: string;
};
readonly metricbeat: {
readonly base: string;
readonly configure: string;
readonly httpEndpoint: string;
readonly install: string;
readonly start: string;
};
readonly enterpriseSearch: {
readonly base: string;
readonly appSearchBase: string;
readonly workplaceSearchBase: string;
};
readonly heartbeat: {
readonly base: string;
};
readonly logstash: {
readonly base: string;
};
readonly functionbeat: {
readonly base: string;
};
readonly winlogbeat: {
readonly base: string;
};
readonly aggs: {
readonly composite: string;
readonly composite_missing_bucket: string;
readonly date_histogram: string;
readonly date_range: string;
readonly date_format_pattern: string;
readonly filter: string;
readonly filters: string;
readonly geohash_grid: string;
readonly histogram: string;
readonly ip_range: string;
readonly range: string;
readonly significant_terms: string;
readonly terms: string;
readonly avg: string;
readonly avg_bucket: string;
readonly max_bucket: string;
readonly min_bucket: string;
readonly sum_bucket: string;
readonly cardinality: string;
readonly count: string;
readonly cumulative_sum: string;
readonly derivative: string;
readonly geo_bounds: string;
readonly geo_centroid: string;
readonly max: string;
readonly median: string;
readonly min: string;
readonly moving_avg: string;
readonly percentile_ranks: string;
readonly serial_diff: string;
readonly std_dev: string;
readonly sum: string;
readonly top_hits: string;
};
readonly runtimeFields: string;
readonly scriptedFields: {
readonly scriptFields: string;
readonly scriptAggs: string;
readonly painless: string;
readonly painlessApi: string;
readonly painlessLangSpec: string;
readonly painlessSyntax: string;
readonly painlessWalkthrough: string;
readonly luceneExpressions: string;
};
readonly indexPatterns: {
readonly introduction: string;
};
readonly addData: string;
readonly kibana: string;
readonly elasticsearch: Record<string, string>;
readonly siem: {
readonly guide: string;
readonly gettingStarted: string;
};
readonly query: {
readonly eql: string;
readonly luceneQuerySyntax: string;
readonly queryDsl: string;
readonly kueryQuerySyntax: string;
};
readonly date: {
readonly dateMath: string;
readonly dateMathIndexNames: string;
};
readonly management: Record<string, string>;
readonly ml: Record<string, string>;
readonly transforms: Record<string, string>;
readonly visualize: Record<string, string>;
readonly apis: Readonly<{
createIndex: string;
createSnapshotLifecyclePolicy: string;
createRoleMapping: string;
createRoleMappingTemplates: string;
createApiKey: string;
createPipeline: string;
createTransformRequest: string;
cronExpressions: string;
executeWatchActionModes: string;
indexExists: string;
openIndex: string;
putComponentTemplate: string;
painlessExecute: string;
painlessExecuteAPIContexts: string;
putComponentTemplateMetadata: string;
putSnapshotLifecyclePolicy: string;
putWatch: string;
updateTransform: string;
}>;
readonly observability: Record<string, string>;
readonly alerting: Record<string, string>;
readonly maps: Record<string, string>;
readonly monitoring: Record<string, string>;
readonly security: Readonly<{
apiKeyServiceSettings: string;
clusterPrivileges: string;
elasticsearchSettings: string;
elasticsearchEnableSecurity: string;
indicesPrivileges: string;
kibanaTLS: string;
kibanaPrivileges: string;
mappingRoles: string;
mappingRolesFieldRules: string;
runAsPrivilege: string;
}>;
readonly watcher: Record<string, string>;
readonly ccs: Record<string, string>;
readonly plugins: Record<string, string>;
readonly snapshotRestore: Record<string, string>;
readonly ingest: Record<string, string>;
} | | diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index b4faa4299a929..8dd4667002ead 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -300,7 +300,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [SavedObjectAttributeSingle](./kibana-plugin-core-server.savedobjectattributesingle.md) | Don't use this type, it's simply a helper type for [SavedObjectAttribute](./kibana-plugin-core-server.savedobjectattribute.md) | | [SavedObjectMigrationFn](./kibana-plugin-core-server.savedobjectmigrationfn.md) | A migration function for a [saved object type](./kibana-plugin-core-server.savedobjectstype.md) used to migrate it to a given version | | [SavedObjectSanitizedDoc](./kibana-plugin-core-server.savedobjectsanitizeddoc.md) | Describes Saved Object documents that have passed through the migration framework and are guaranteed to have a references root property. | -| [SavedObjectsClientContract](./kibana-plugin-core-server.savedobjectsclientcontract.md) | Saved Objects is Kibana's data persisentence mechanism allowing plugins to use Elasticsearch for storing plugin state.\#\# SavedObjectsClient errorsSince the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to application code. Ideally, all errors will be either:1. Caused by bad implementation (ie. undefined is not a function) and as such unpredictable 2. An error that has been classified and decorated appropriately by the decorators in [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md)Type 1 errors are inevitable, but since all expected/handle-able errors should be Type 2 the isXYZError() helpers exposed at SavedObjectsErrorHelpers should be used to understand and manage error responses from the SavedObjectsClient.Type 2 errors are decorated versions of the source error, so if the elasticsearch client threw an error it will be decorated based on its type. That means that rather than looking for error.body.error.type or doing substring checks on error.body.error.reason, just use the helpers to understand the meaning of the error:\`\`\`js if (SavedObjectsErrorHelpers.isNotFoundError(error)) { // handle 404 }if (SavedObjectsErrorHelpers.isNotAuthorizedError(error)) { // 401 handling should be automatic, but in case you wanted to know }// always rethrow the error unless you handle it throw error; \`\`\`\#\#\# 404s from missing indexFrom the perspective of application code and APIs the SavedObjectsClient is a black box that persists objects. One of the internal details that users have no control over is that we use an elasticsearch index for persistance and that index might be missing.At the time of writing we are in the process of transitioning away from the operating assumption that the SavedObjects index is always available. Part of this transition is handling errors resulting from an index missing. These used to trigger a 500 error in most cases, and in others cause 404s with different error messages.From my (Spencer) perspective, a 404 from the SavedObjectsApi is a 404; The object the request/call was targeting could not be found. This is why \#14141 takes special care to ensure that 404 errors are generic and don't distinguish between index missing or document missing.See [SavedObjectsClient](./kibana-plugin-core-server.savedobjectsclient.md) See [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md) | +| [SavedObjectsClientContract](./kibana-plugin-core-server.savedobjectsclientcontract.md) | Saved Objects is Kibana's data persisentence mechanism allowing plugins to use Elasticsearch for storing plugin state.\#\# SavedObjectsClient errorsSince the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to to application code. Ideally, all errors will be either:1. Caused by bad implementation (ie. undefined is not a function) and as such unpredictable 2. An error that has been classified and decorated appropriately by the decorators in [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md)Type 1 errors are inevitable, but since all expected/handle-able errors should be Type 2 the isXYZError() helpers exposed at SavedObjectsErrorHelpers should be used to understand and manage error responses from the SavedObjectsClient.Type 2 errors are decorated versions of the source error, so if the elasticsearch client threw an error it will be decorated based on its type. That means that rather than looking for error.body.error.type or doing substring checks on error.body.error.reason, just use the helpers to understand the meaning of the error:\`\`\`js if (SavedObjectsErrorHelpers.isNotFoundError(error)) { // handle 404 }if (SavedObjectsErrorHelpers.isNotAuthorizedError(error)) { // 401 handling should be automatic, but in case you wanted to know }// always rethrow the error unless you handle it throw error; \`\`\`\#\#\# 404s from missing indexFrom the perspective of application code and APIs the SavedObjectsClient is a black box that persists objects. One of the internal details that users have no control over is that we use an elasticsearch index for persistance and that index might be missing.At the time of writing we are in the process of transitioning away from the operating assumption that the SavedObjects index is always available. Part of this transition is handling errors resulting from an index missing. These used to trigger a 500 error in most cases, and in others cause 404s with different error messages.From my (Spencer) perspective, a 404 from the SavedObjectsApi is a 404; The object the request/call was targeting could not be found. This is why \#14141 takes special care to ensure that 404 errors are generic and don't distinguish between index missing or document missing.See [SavedObjectsClient](./kibana-plugin-core-server.savedobjectsclient.md) See [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md) | | [SavedObjectsClientFactory](./kibana-plugin-core-server.savedobjectsclientfactory.md) | Describes the factory used to create instances of the Saved Objects Client. | | [SavedObjectsClientFactoryProvider](./kibana-plugin-core-server.savedobjectsclientfactoryprovider.md) | Provider to invoke to retrieve a [SavedObjectsClientFactory](./kibana-plugin-core-server.savedobjectsclientfactory.md). | | [SavedObjectsClientWrapperFactory](./kibana-plugin-core-server.savedobjectsclientwrapperfactory.md) | Describes the factory used to create instances of Saved Objects Client Wrappers. | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclientcontract.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclientcontract.md index 537cfbc175671..610356a733126 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclientcontract.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclientcontract.md @@ -8,7 +8,7 @@ Saved Objects is Kibana's data persisentence mechanism allowing plugins to use E \#\# SavedObjectsClient errors -Since the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to application code. Ideally, all errors will be either: +Since the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to to application code. Ideally, all errors will be either: 1. Caused by bad implementation (ie. undefined is not a function) and as such unpredictable 2. An error that has been classified and decorated appropriately by the decorators in [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md) diff --git a/src/core/public/doc_links/doc_links_service.ts b/src/core/public/doc_links/doc_links_service.ts index 6c715c681e6e8..fa7ff3b2d4293 100644 --- a/src/core/public/doc_links/doc_links_service.ts +++ b/src/core/public/doc_links/doc_links_service.ts @@ -16,6 +16,7 @@ interface StartDeps { /** @internal */ export class DocLinksService { public setup() {} + public start({ injectedMetadata }: StartDeps): DocLinksStart { const DOC_LINK_VERSION = injectedMetadata.getKibanaBranch(); const ELASTIC_WEBSITE_URL = 'https://www.elastic.co/'; @@ -110,7 +111,7 @@ export class DocLinksService { runtimeFields: `${ELASTICSEARCH_DOCS}runtime.html`, scriptedFields: { scriptFields: `${ELASTICSEARCH_DOCS}search-request-script-fields.html`, - scriptAggs: `${ELASTICSEARCH_DOCS}search-aggregations.html#_values_source`, + scriptAggs: `${ELASTICSEARCH_DOCS}search-aggregations.html`, painless: `${ELASTICSEARCH_DOCS}modules-scripting-painless.html`, painlessApi: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/painless/${DOC_LINK_VERSION}/painless-api-reference.html`, painlessLangSpec: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/painless/${DOC_LINK_VERSION}/painless-lang-spec.html`, @@ -120,7 +121,6 @@ export class DocLinksService { luceneExpressions: `${ELASTICSEARCH_DOCS}modules-scripting-expression.html`, }, indexPatterns: { - loadingData: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/tutorial-load-dataset.html`, introduction: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/index-patterns.html`, fieldFormattersString: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/field-formatters-string.html`, }, @@ -201,10 +201,10 @@ export class DocLinksService { emailActionConfig: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/email-action-type.html#configuring-email`, generalSettings: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/alert-action-settings-kb.html#general-alert-action-settings`, indexAction: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/index-action-type.html`, - esQuery: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/alert-type-es-query.html`, - indexThreshold: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/alert-type-index-threshold.html#index-action-configuration`, + esQuery: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/rule-type-es-query.html`, + indexThreshold: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/rule-type-index-threshold.html`, pagerDutyAction: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/pagerduty-action-type.html`, - preconfiguredConnectors: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/pre-configured-action-types-and-connectors.html`, + preconfiguredConnectors: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/pre-configured-connectors.html`, serviceNowAction: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/servicenow-action-type.html#configuring-servicenow`, setupPrerequisites: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/alerting-getting-started.html#alerting-setup-prerequisites`, slackAction: `${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/slack-action-type.html#configuring-slack`, @@ -390,7 +390,6 @@ export interface DocLinksStart { readonly luceneExpressions: string; }; readonly indexPatterns: { - readonly loadingData: string; readonly introduction: string; }; readonly addData: string; diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md index 4bab4ea54993b..bf25fcaa75acc 100644 --- a/src/core/public/public.api.md +++ b/src/core/public/public.api.md @@ -570,7 +570,6 @@ export interface DocLinksStart { readonly luceneExpressions: string; }; readonly indexPatterns: { - readonly loadingData: string; readonly introduction: string; }; readonly addData: string; diff --git a/src/dev/cli_dev_mode/get_server_watch_paths.test.ts b/src/dev/cli_dev_mode/get_server_watch_paths.test.ts index 7f84338b4efb8..ab113b96a5f03 100644 --- a/src/dev/cli_dev_mode/get_server_watch_paths.test.ts +++ b/src/dev/cli_dev_mode/get_server_watch_paths.test.ts @@ -28,7 +28,6 @@ it('produces the right watch and ignore list', () => { Array [ /src/core, /src/legacy/server, - /src/legacy/ui, /src/legacy/utils, /config, /x-pack/test/plugin_functional/plugins/resolver_test, diff --git a/src/dev/cli_dev_mode/get_server_watch_paths.ts b/src/dev/cli_dev_mode/get_server_watch_paths.ts index 4e00dd4ca98b9..46aa15659a513 100644 --- a/src/dev/cli_dev_mode/get_server_watch_paths.ts +++ b/src/dev/cli_dev_mode/get_server_watch_paths.ts @@ -41,7 +41,6 @@ export function getServerWatchPaths({ pluginPaths, pluginScanDirs }: Options) { [ fromRoot('src/core'), fromRoot('src/legacy/server'), - fromRoot('src/legacy/ui'), fromRoot('src/legacy/utils'), fromRoot('config'), ...pluginPaths, diff --git a/src/legacy/ui/public/documentation_links/documentation_links.ts b/src/legacy/ui/public/documentation_links/documentation_links.ts deleted file mode 100644 index 933812d29e9bb..0000000000000 --- a/src/legacy/ui/public/documentation_links/documentation_links.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -/* - WARNING: The links in this file are validated during the docs build. This is accomplished with some regex magic that - looks for these particular constants. As a result, we should not add new constants or change the existing ones. - If you absolutely must make a change, talk to Clinton Gormley first so he can update his Perl scripts. - */ -export const DOC_LINK_VERSION = 'stub'; -export const ELASTIC_WEBSITE_URL = 'stub'; - -export const documentationLinks = {}; From 631608e05ef61ffa2cbae0d2a151f3af8e4d7d19 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Wed, 17 Mar 2021 17:09:34 -0400 Subject: [PATCH 3/4] [DOCS] Migrate ingest pipeline docs to ES reference (#94625) --- .../ingest-pipelines.asciidoc | 170 ------------------ docs/redirects.asciidoc | 5 + docs/user/management.asciidoc | 9 +- 3 files changed, 8 insertions(+), 176 deletions(-) delete mode 100644 docs/management/ingest-pipelines/ingest-pipelines.asciidoc diff --git a/docs/management/ingest-pipelines/ingest-pipelines.asciidoc b/docs/management/ingest-pipelines/ingest-pipelines.asciidoc deleted file mode 100644 index d9745bfef524a..0000000000000 --- a/docs/management/ingest-pipelines/ingest-pipelines.asciidoc +++ /dev/null @@ -1,170 +0,0 @@ -[role="xpack"] -[[ingest-node-pipelines]] -== Ingest Node Pipelines - -*Ingest Node Pipelines* enables you to create and manage {es} -pipelines that perform common transformations and -enrichments on your data. For example, you might remove a field, -rename an existing field, or set a new field. - -To begin, open the main menu, then click *Stack Management > Ingest Node Pipelines*. With *Ingest Node Pipelines*, you can: - -* View a list of your pipelines and drill down into details. -* Create a pipeline that defines a series of tasks, known as processors. -* Test a pipeline before feeding it with real data to ensure the pipeline works as expected. -* Delete a pipeline that is no longer needed. - -[role="screenshot"] -image:management/ingest-pipelines/images/ingest-pipeline-list.png["Ingest node pipeline list"] - -[float] -=== Required permissions - -The minimum required permissions to access *Ingest Node Pipelines* are -the `manage_pipeline` and `cluster:monitor/nodes/info` cluster privileges. - -To add privileges, open the main menu, then click *Stack Management > Roles*. - -[role="screenshot"] -image:management/ingest-pipelines/images/ingest-pipeline-privileges.png["Privileges required for Ingest Node Pipelines"] - -[float] -[[ingest-node-pipelines-manage]] -=== Manage pipelines - -From the list view, you can to drill down into the details of a pipeline. -To -edit, clone, or delete a pipeline, use the *Actions* menu. - -If you don’t have any pipelines, you can create one using the -*Create pipeline* form. You’ll define processors to transform documents -in a specific way. To handle exceptions, you can optionally define -failure processors to execute immediately after a failed processor. -Before creating the pipeline, you can verify it provides the expected output. - -[float] -[[ingest-node-pipelines-example]] -==== Example: Create a pipeline - -In this example, you’ll create a pipeline to handle server logs in the -Common Log Format. The log looks similar to this: - -[source,js] ----------------------------------- -212.87.37.154 - - [05/May/2020:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" -200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) -AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\" ----------------------------------- - -The log contains an IP address, timestamp, and user agent. You want to give -these three items their own field in {es} for fast search and visualization. -You also want to know where the request is coming from. - -. In *Ingest Node Pipelines*, click *Create a pipeline*. -. Provide a name and description for the pipeline. -. Add a grok processor to parse the log message: - -.. Click *Add a processor* and select the *Grok* processor type. -.. Set the field input to `message` and enter the following grok pattern: -+ -[source,js] ----------------------------------- -%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent} ----------------------------------- -+ -.. Click *Update* to save the processor. - -. Add processors to map the date, IP, and user agent fields. - -.. Map the appropriate field to each processor type: -+ --- -* **Date**: `timestamp` -* **GeoIP**: `clientip` -* **User agent**: `agent` - -For the **Date** processor, you also need to specify the date format you want to use: `dd/MMM/YYYY:HH:mm:ss Z`. --- -Your form should look similar to this: -+ -[role="screenshot"] -image:management/ingest-pipelines/images/ingest-pipeline-processor.png["Processors for Ingest Node Pipelines"] -+ -Alternatively, you can click the **Import processors** link and define the processors as JSON: -+ -[source,js] ----------------------------------- -{ - "processors": [ - { - "grok": { - "field": "message", - "patterns": ["%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \\[%{HTTPDATE:timestamp}\\] \"%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}"] - } - }, - { - "date": { - "field": "timestamp", - "formats": [ "dd/MMM/YYYY:HH:mm:ss Z" ] - } - }, - { - "geoip": { - "field": "clientip" - } - }, - { - "user_agent": { - "field": "agent" - } - } - ] -} ----------------------------------- -+ -The four {ref}/ingest-processors.html[processors] will run sequentially: -{ref}/grok-processor.html[grok], {ref}/date-processor.html[date], -{ref}/geoip-processor.html[geoip], and {ref}/user-agent-processor.html[user_agent]. You can reorder processors using the arrow icon next to each processor. - -. To test the pipeline to verify that it produces the expected results, click *Add documents*. - -. In the *Documents* tab, provide a sample document for testing: -+ -[source,js] ----------------------------------- -[ - { - "_source": { - "message": "212.87.37.154 - - [05/May/2020:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"" - } - } -] ----------------------------------- - -. Click *Run the pipeline* and check if the pipeline worked as expected. -+ -You can also -view the verbose output and refresh the output from this view. - -. If everything looks correct, close the panel, and then click *Create pipeline*. -+ -At this point, you’re ready to use the Elasticsearch index API to load -the logs data. - -. In the Kibana Console, index a document with the pipeline -you created. -+ -[source,js] ----------------------------------- -PUT my-index/_doc/1?pipeline=access_logs -{ - "message": "212.87.37.154 - - [05/May/2020:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"" -} ----------------------------------- - -. To verify, run: -+ -[source,js] ----------------------------------- -GET my-index/_doc/1 ----------------------------------- diff --git a/docs/redirects.asciidoc b/docs/redirects.asciidoc index c7bdff800bb0b..a2ab1c10d9cd5 100644 --- a/docs/redirects.asciidoc +++ b/docs/redirects.asciidoc @@ -280,3 +280,8 @@ This content has moved. Refer to <>. [role="exclude",id="explore-dashboard-data"] This content has moved. Refer to <>. + +[role="exclude",id="ingest-node-pipelines"] +== Ingest Node Pipelines + +This content has moved. See {ref}/ingest.html[Ingest pipelines]. diff --git a/docs/user/management.asciidoc b/docs/user/management.asciidoc index f29718e6d588b..5644cdbfc45ec 100644 --- a/docs/user/management.asciidoc +++ b/docs/user/management.asciidoc @@ -17,10 +17,9 @@ Consult your administrator if you do not have the appropriate access. [cols="50, 50"] |=== -| <> -| Create and manage {es} -pipelines that enable you to perform common transformations and -enrichments on your data. +| {ref}/ingest.html[Ingest Node Pipelines] +| Create and manage ingest pipelines that let you perform common transformations +and enrichments on your data. | {logstash-ref}/logstash-centralized-pipeline-management.html[Logstash Pipelines] | Create, edit, and delete your Logstash pipeline configurations. @@ -187,8 +186,6 @@ include::{kib-repo-dir}/management/alerting/connector-management.asciidoc[] include::{kib-repo-dir}/management/managing-beats.asciidoc[] -include::{kib-repo-dir}/management/ingest-pipelines/ingest-pipelines.asciidoc[] - include::{kib-repo-dir}/management/managing-fields.asciidoc[] include::{kib-repo-dir}/management/managing-licenses.asciidoc[] From 3ec9a17e2eec74d91f1dc6406466dd609152324c Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 17 Mar 2021 14:16:12 -0700 Subject: [PATCH 4/4] skip failing suite (#94854) --- .../apps/ml/data_frame_analytics/outlier_detection_creation.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts index 8ce85c58ec1fb..0a00bb3cd757f 100644 --- a/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts +++ b/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts @@ -12,7 +12,8 @@ export default function ({ getService }: FtrProviderContext) { const ml = getService('ml'); const editedDescription = 'Edited description'; - describe('outlier detection creation', function () { + // FAILING: https://github.com/elastic/kibana/issues/94854 + describe.skip('outlier detection creation', function () { before(async () => { await esArchiver.loadIfNeeded('ml/ihp_outlier'); await ml.testResources.createIndexPatternIfNeeded('ft_ihp_outlier', '@timestamp');