Merge branch 'implement/restrict-theme-options' of github.com:spalger…

…/kibana into implement/restrict-theme-options

docs/developer/telemetry.asciidoc

@@ -6,6 +6,7 @@ To help us provide a good developer experience, we track some straightforward me
 The operations we current report timing data for:
 
 * Total execution time of `yarn kbn bootstrap`
+* Total execution time of `@kbn/optimizer` runs as well as the following metadata about the runs: The number of bundles created, the number of bundles which were cached, usage of `--watch`, `--dist`, `--workers` and `--no-cache` flags, and the count of themes being built.
 
 Along with the execution time of each execution, we ship the following information about your machine to the service:
 

docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md

@@ -99,7 +99,6 @@ readonly links: {
             readonly luceneExpressions: string;
         };
         readonly indexPatterns: {
-            readonly loadingData: string;
             readonly introduction: string;
         };
         readonly addData: string;

docs/development/core/server/kibana-plugin-core-server.md

@@ -300,7 +300,7 @@ The plugin integrates with the core system via lifecycle events: `setup`<!-- -->
 |  [SavedObjectAttributeSingle](./kibana-plugin-core-server.savedobjectattributesingle.md) | Don't use this type, it's simply a helper type for [SavedObjectAttribute](./kibana-plugin-core-server.savedobjectattribute.md) |
 |  [SavedObjectMigrationFn](./kibana-plugin-core-server.savedobjectmigrationfn.md) | A migration function for a [saved object type](./kibana-plugin-core-server.savedobjectstype.md) used to migrate it to a given version |
 |  [SavedObjectSanitizedDoc](./kibana-plugin-core-server.savedobjectsanitizeddoc.md) | Describes Saved Object documents that have passed through the migration framework and are guaranteed to have a <code>references</code> root property. |
-|  [SavedObjectsClientContract](./kibana-plugin-core-server.savedobjectsclientcontract.md) | Saved Objects is Kibana's data persisentence mechanism allowing plugins to use Elasticsearch for storing plugin state.<!-- -->\#\# SavedObjectsClient errors<!-- -->Since the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to application code. Ideally, all errors will be either:<!-- -->1. Caused by bad implementation (ie. undefined is not a function) and as such unpredictable 2. An error that has been classified and decorated appropriately by the decorators in [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md)<!-- -->Type 1 errors are inevitable, but since all expected/handle-able errors should be Type 2 the <code>isXYZError()</code> helpers exposed at <code>SavedObjectsErrorHelpers</code> should be used to understand and manage error responses from the <code>SavedObjectsClient</code>.<!-- -->Type 2 errors are decorated versions of the source error, so if the elasticsearch client threw an error it will be decorated based on its type. That means that rather than looking for <code>error.body.error.type</code> or doing substring checks on <code>error.body.error.reason</code>, just use the helpers to understand the meaning of the error:<!-- -->\`\`\`<!-- -->js if (SavedObjectsErrorHelpers.isNotFoundError(error)) { // handle 404 }<!-- -->if (SavedObjectsErrorHelpers.isNotAuthorizedError(error)) { // 401 handling should be automatic, but in case you wanted to know }<!-- -->// always rethrow the error unless you handle it throw error; \`\`\`<!-- -->\#\#\# 404s from missing index<!-- -->From the perspective of application code and APIs the SavedObjectsClient is a black box that persists objects. One of the internal details that users have no control over is that we use an elasticsearch index for persistance and that index might be missing.<!-- -->At the time of writing we are in the process of transitioning away from the operating assumption that the SavedObjects index is always available. Part of this transition is handling errors resulting from an index missing. These used to trigger a 500 error in most cases, and in others cause 404s with different error messages.<!-- -->From my (Spencer) perspective, a 404 from the SavedObjectsApi is a 404; The object the request/call was targeting could not be found. This is why \#14141 takes special care to ensure that 404 errors are generic and don't distinguish between index missing or document missing.<!-- -->See [SavedObjectsClient](./kibana-plugin-core-server.savedobjectsclient.md) See [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md) |
+|  [SavedObjectsClientContract](./kibana-plugin-core-server.savedobjectsclientcontract.md) | Saved Objects is Kibana's data persisentence mechanism allowing plugins to use Elasticsearch for storing plugin state.<!-- -->\#\# SavedObjectsClient errors<!-- -->Since the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to to application code. Ideally, all errors will be either:<!-- -->1. Caused by bad implementation (ie. undefined is not a function) and as such unpredictable 2. An error that has been classified and decorated appropriately by the decorators in [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md)<!-- -->Type 1 errors are inevitable, but since all expected/handle-able errors should be Type 2 the <code>isXYZError()</code> helpers exposed at <code>SavedObjectsErrorHelpers</code> should be used to understand and manage error responses from the <code>SavedObjectsClient</code>.<!-- -->Type 2 errors are decorated versions of the source error, so if the elasticsearch client threw an error it will be decorated based on its type. That means that rather than looking for <code>error.body.error.type</code> or doing substring checks on <code>error.body.error.reason</code>, just use the helpers to understand the meaning of the error:<!-- -->\`\`\`<!-- -->js if (SavedObjectsErrorHelpers.isNotFoundError(error)) { // handle 404 }<!-- -->if (SavedObjectsErrorHelpers.isNotAuthorizedError(error)) { // 401 handling should be automatic, but in case you wanted to know }<!-- -->// always rethrow the error unless you handle it throw error; \`\`\`<!-- -->\#\#\# 404s from missing index<!-- -->From the perspective of application code and APIs the SavedObjectsClient is a black box that persists objects. One of the internal details that users have no control over is that we use an elasticsearch index for persistance and that index might be missing.<!-- -->At the time of writing we are in the process of transitioning away from the operating assumption that the SavedObjects index is always available. Part of this transition is handling errors resulting from an index missing. These used to trigger a 500 error in most cases, and in others cause 404s with different error messages.<!-- -->From my (Spencer) perspective, a 404 from the SavedObjectsApi is a 404; The object the request/call was targeting could not be found. This is why \#14141 takes special care to ensure that 404 errors are generic and don't distinguish between index missing or document missing.<!-- -->See [SavedObjectsClient](./kibana-plugin-core-server.savedobjectsclient.md) See [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md) |
 |  [SavedObjectsClientFactory](./kibana-plugin-core-server.savedobjectsclientfactory.md) | Describes the factory used to create instances of the Saved Objects Client. |
 |  [SavedObjectsClientFactoryProvider](./kibana-plugin-core-server.savedobjectsclientfactoryprovider.md) | Provider to invoke to retrieve a [SavedObjectsClientFactory](./kibana-plugin-core-server.savedobjectsclientfactory.md)<!-- -->. |
 |  [SavedObjectsClientWrapperFactory](./kibana-plugin-core-server.savedobjectsclientwrapperfactory.md) | Describes the factory used to create instances of Saved Objects Client Wrappers. |

docs/development/core/server/kibana-plugin-core-server.savedobjectsclientcontract.md

@@ -8,7 +8,7 @@ Saved Objects is Kibana's data persisentence mechanism allowing plugins to use E
 
 \#\# SavedObjectsClient errors
 
-Since the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to application code. Ideally, all errors will be either:
+Since the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to to application code. Ideally, all errors will be either:
 
 1. Caused by bad implementation (ie. undefined is not a function) and as such unpredictable 2. An error that has been classified and decorated appropriately by the decorators in [SavedObjectsErrorHelpers](./kibana-plugin-core-server.savedobjectserrorhelpers.md)
 

docs/redirects.asciidoc

@@ -280,3 +280,8 @@ This content has moved. Refer to <<dashboard, **Dashboard**>>.
 
 [role="exclude",id="explore-dashboard-data"]
 This content has moved. Refer to <<dashboard, **Dashboard**>>.
+
+[role="exclude",id="ingest-node-pipelines"]
+== Ingest Node Pipelines
+
+This content has moved. See {ref}/ingest.html[Ingest pipelines].

docs/user/management.asciidoc

@@ -17,10 +17,9 @@ Consult your administrator if you do not have the appropriate access.
 [cols="50, 50"]
 |===
 
-| <<ingest-node-pipelines, Ingest Node Pipelines>>
-| Create and manage {es}
-pipelines that enable you to perform common transformations and
-enrichments on your data.
+| {ref}/ingest.html[Ingest Node Pipelines]
+| Create and manage ingest pipelines that let you perform common transformations
+and enrichments on your data.
 
 | {logstash-ref}/logstash-centralized-pipeline-management.html[Logstash Pipelines]
 | Create, edit, and delete your Logstash pipeline configurations.
@@ -187,8 +186,6 @@ include::{kib-repo-dir}/management/alerting/connector-management.asciidoc[]
 
 include::{kib-repo-dir}/management/managing-beats.asciidoc[]
 
-include::{kib-repo-dir}/management/ingest-pipelines/ingest-pipelines.asciidoc[]
-
 include::{kib-repo-dir}/management/managing-fields.asciidoc[]
 
 include::{kib-repo-dir}/management/managing-licenses.asciidoc[]

packages/kbn-optimizer/src/cli.ts

@@ -18,6 +18,7 @@ import { logOptimizerState } from './log_optimizer_state';
 import { OptimizerConfig } from './optimizer';
 import { runOptimizer } from './run_optimizer';
 import { validateLimitsForAllBundles, updateBundleLimits } from './limits';
+import { reportOptimizerTimings } from './report_optimizer_timings';
 
 function getLimitsPath(flags: Flags, defaultPath: string) {
   if (flags.limits) {
@@ -144,7 +145,9 @@ export function runKbnOptimizerCli(options: { defaultLimitsPath: string }) {
 
       const update$ = runOptimizer(config);
 
-      await lastValueFrom(update$.pipe(logOptimizerState(log, config)));
+      await lastValueFrom(
+        update$.pipe(logOptimizerState(log, config), reportOptimizerTimings(log, config))
+      );
 
       if (updateLimits) {
         updateBundleLimits({

packages/kbn-optimizer/src/index.ts

@@ -12,3 +12,4 @@ export * from './log_optimizer_state';
 export * from './node';
 export * from './limits';
 export * from './cli';
+export * from './report_optimizer_timings';

packages/kbn-optimizer/src/report_optimizer_timings.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { concatMap } from 'rxjs/operators';
+import { CiStatsReporter, ToolingLog } from '@kbn/dev-utils';
+
+import { OptimizerConfig } from './optimizer';
+import { OptimizerUpdate$ } from './run_optimizer';
+import { pipeClosure } from './common';
+
+export function reportOptimizerTimings(log: ToolingLog, config: OptimizerConfig) {
+  return pipeClosure((update$: OptimizerUpdate$) => {
+    let sent = false;
+
+    const cachedBundles = new Set<string>();
+    const notCachedBundles = new Set<string>();
+
+    return update$.pipe(
+      concatMap(async (update) => {
+        // if we've already sent timing data then move on
+        if (sent) {
+          return update;
+        }
+
+        if (update.event?.type === 'bundle cached') {
+          cachedBundles.add(update.event.bundle.id);
+        }
+        if (update.event?.type === 'bundle not cached') {
+          notCachedBundles.add(update.event.bundle.id);
+        }
+
+        // wait for the optimizer to complete, either with a success or failure
+        if (update.state.phase !== 'issue' && update.state.phase !== 'success') {
+          return update;
+        }
+
+        sent = true;
+        const reporter = CiStatsReporter.fromEnv(log);
+        const time = Date.now() - update.state.startTime;
+
+        await reporter.timings({
+          timings: [
+            {
+              group: '@kbn/optimizer',
+              id: 'overall time',
+              ms: time,
+              meta: {
+                optimizerBundleCount: config.bundles.length,
+                optimizerBundleCacheCount: cachedBundles.size,
+                optimizerBundleCachePct: Math.floor(
+                  (cachedBundles.size / config.bundles.length) * 100
+                ),
+                optimizerWatch: config.watch,
+                optimizerProduction: config.dist,
+                optimizerProfileWebpack: config.profileWebpack,
+                optimizerBundleThemeTagsCount: config.themeTags.length,
+                optimizerCache: config.cache,
+                optimizerMaxWorkerCount: config.maxWorkerCount,
+              },
+            },
+          ],
+        });
+
+        return update;
+      })
+    );
+  });
+}