Merge remote-tracking branch 'upstream/master' into 111842-error-hand…

…ling-react

.buildkite/scripts/steps/storybooks/build_and_upload.js

@@ -8,6 +8,7 @@ const STORYBOOKS = [
   'canvas',
   'codeeditor',
   'ci_composite',
+  'custom_integrations',
   'url_template_editor',
   'dashboard',
   'dashboard_enhanced',

.ci/.storybook/main.js

@@ -11,6 +11,12 @@ const aliases = require('../../src/dev/storybook/aliases.ts').storybookAliases;
 
 config.refs = {};
 
+// Required due to https://github.com/storybookjs/storybook/issues/13834
+config.babel = async (options) => ({
+  ...options,
+  plugins: ['@babel/plugin-transform-typescript', ...options.plugins],
+});
+
 for (const alias of Object.keys(aliases).filter((a) => a !== 'ci_composite')) {
   // snake_case -> Title Case
   const title = alias

.github/CODEOWNERS

@@ -243,7 +243,6 @@
 /packages/kbn-std/ @elastic/kibana-core
 /packages/kbn-config/ @elastic/kibana-core
 /packages/kbn-logging/ @elastic/kibana-core
-/packages/kbn-legacy-logging/ @elastic/kibana-core
 /packages/kbn-crypto/ @elastic/kibana-core
 /packages/kbn-http-tools/ @elastic/kibana-core
 /src/plugins/saved_objects_management/ @elastic/kibana-core

.i18nrc.json

@@ -5,6 +5,7 @@
     "kbnConfig": "packages/kbn-config/src",
     "console": "src/plugins/console",
     "core": "src/core",
+    "customIntegrations": "src/plugins/custom_integrations",
     "discover": "src/plugins/discover",
     "bfetch": "src/plugins/bfetch",
     "dashboard": "src/plugins/dashboard",
@@ -18,6 +19,7 @@
     "home": "src/plugins/home",
     "flot": "packages/kbn-ui-shared-deps-src/src/flot_charts",
     "charts": "src/plugins/charts",
+    "customIntegrations": "src/plugins/custom_integrations",
     "esUi": "src/plugins/es_ui_shared",
     "devTools": "src/plugins/dev_tools",
     "expressions": "src/plugins/expressions",

config/kibana.yml

@@ -84,24 +84,32 @@
 # Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
 #elasticsearch.shardTimeout: 30000
 
-# Logs queries sent to Elasticsearch. Requires logging.verbose set to true.
-#elasticsearch.logQueries: false
-
 # Specifies the path where Kibana creates the process ID file.
 #pid.file: /run/kibana/kibana.pid
 
+# Set the value of this setting to off to suppress all logging output, or to debug to log everything.
+# logging.root.level: debug
+
 # Enables you to specify a file where Kibana stores log output.
-#logging.dest: stdout
+# logging.appenders.default:
+#   type: file
+#   fileName: /var/logs/kibana.log
+
 
-# Set the value of this setting to true to suppress all logging output.
-#logging.silent: false
+# Logs queries sent to Elasticsearch.
+# logging.loggers:
+#   - name: elasticsearch.queries
+#     level: debug
 
-# Set the value of this setting to true to suppress all logging output other than error messages.
-#logging.quiet: false
+# Logs http responses.
+# logging.loggers:
+#   - name: http.server.response
+#     level: debug
 
-# Set the value of this setting to true to log all events, including system usage information
-# and all requests.
-#logging.verbose: false
+# Logs system usage information.
+# logging.loggers:
+#   - name: metrics.ops
+#     level: debug
 
 # Set the interval in milliseconds to sample system and process performance
 # metrics. Minimum is 100ms. Defaults to 5000.

dev_docs/key_concepts/kibana_platform_plugin_intro.mdx

@@ -1,7 +1,7 @@
 ---
 id: kibPlatformIntro
 slug: /kibana-dev-docs/key-concepts/platform-intro
-title: Plugins and the Kibana platform
+title: Plugins, packages, and the platform
 summary: An introduction to the Kibana platform and how to use it to build a plugin.
 date: 2021-01-06
 tags: ['kibana', 'onboarding', 'dev', 'architecture']
@@ -17,30 +17,47 @@ already existing applications. Did you know that almost everything you see in th
 Kibana UI is built inside a plugin? If you removed all plugins from Kibana, you'd be left with an empty navigation menu, and a set of
 developer tools. The Kibana platform is a blank canvas, just waiting for a developer to come along and create something!
 
-![Kibana personas](assets/kibana_platform_plugin_end_user.png)
+![Kibana personas](../assets/kibana_platform_plugin_end_user.png)
 
-## Platform services
+## 1,000 foot view
 
-Plugins have access to three kinds of public services:
+At a super high-level, Kibana is composed of **plugins**, **core**, and **Kibana packages**.
 
-- Platform services provided by `core` (<DocLink id="kibPlatformIntro" section="core-services" text="Core services"/>)
-- Platform services provided by plugins (<DocLink id="kibPlatformIntro" section="platform-plugins" text="Platform plugins"/>)
-- Shared services provided by plugins, that are only relevant for only a few, specific plugins (e.g. "presentation utils").
+![Kibana 1000 ft arch](../assets/1000_ft_arch.png)
 
-The first two items are what make up "Platform services".
+**Plugins** provide the majority of all functionality in Kibana. All applications and UIs are defined here.
 
-<DocCallOut title="What is the difference between services provided by platform plugins, and those by core?">
+**Core** provides the runtime and the most fundamental services.
 
-We try to put only the most stable and fundamental code into `Core`, while more application focused functionality goes in a plugin, but the heuristic isn't
-clear, and we haven't done a great job of sticking to it. For example, notifications and toasts are core services, but data and search are plugin services.
+**@kbn packages** provide static utilities that can be imported anywhere in Kibana.
+
+<DocAccordion buttonContent="(Internal only) FAQ: Should I put my code in a plugin or a package?" color="warning">
+<DocCallOut color="warning">
+If it's stateful, it has to go in a plugin, but packages are often a good choices for stateless utilities. Stateless code exported publicly from a plugin will increase the page load bundle size of _every single page_, even if none of those plugin's services are actually needed. With packages, however, only code that is needed for the current page is downloaded.
+
+The downside however is that the packages folder is far away from the plugins folder so having a part of your code in a plugin and the rest in a package may make it hard to find, leading to duplication.
+
+The Operations team hopes to resolve this conundrum by supporting co-located packages and plugins and automatically putting all stateless code inside a package. You can track this work by following [this issue](https://github.com/elastic/kibana/issues/112886).
+
+Until then, consider whether it makes sense to logically separate the code, and consider the size of the exports, when determining whether you should put stateless public exports in a package or a plugin.
+
+</DocCallOut>
+</DocAccordion>
+
+<DocAccordion buttonContent="FAQ: What is the difference between services provided by plugins and those by Core?" color="warning">
+<DocCallOut >
+
+We try to put only the most stable and fundamental code into `Core`, while optional add-ons, applications, and solution-oriented functionality goes in a plugin. Unfortunately, we haven't done a great job of sticking to that. For example, notifications and toasts are core services, but data and search are plugin services.
 
 Today it looks something like this.
 
-![Core vs platform plugins vs plugins](assets/platform_plugins_core.png)
+![Core vs platform plugins vs plugins](../assets/platform_plugins_core.png)
+
+"Platform plugins" provide core-like functionality, just outside of core, and their public APIs tend to be more volatile. Other plugins may still expose shared services, but they are intended only for usage by a small subset of specific plugins, and may not be generic or "platform-like".
 
-<DocAccordion buttonContent="A bit of history">
-When the Kibana platform and plugin infrastructure was built, we thought of two types of code: core services, and other plugin services. We planned to keep the most stable and fundamental
-code needed to build plugins inside core.
+**A bit of history**
+
+When the Kibana platform and plugin infrastructure was built, we thought of two types of code: core services, and other plugin services. We planned to keep the most stable and fundamental code needed to build plugins inside core.
 
 In reality, we ended up with many platform-like services living outside of core, with no (short term) intention of moving them. We highly encourage plugin developers to use
 them, so we consider them part of platform services.
@@ -49,36 +66,18 @@ When we built our platform system, we also thought we'd end up with only a handf
 footprint and speed up Kibana.
 
 In reality, our plugin model ended up being used like micro-services. Plugins are the only form of encapsulation we provide developers, and they liked it! However, we ended
-up with a ton of small plugins, that developers never intended to be uninstallable, nor tested in this manner. We are considering ways to provide developers the ability to build services
-with the encapsulation
-they desire, without the need to build a plugin.
+up with a ton of small plugins, that developers never intended to be uninstallable, nor tested in this manner. We are considering ways to provide developers the ability to build services with the encapsulation they desire, without the need to build a plugin.
 
 Another side effect of having many small plugins is that common code often ends up extracted into another plugin. Use case specific utilities are exported,
 that are not meant to be used in a general manner. This makes our definition of "platform code" a bit trickier to define. We'd like to say "The platform is made up of
-every publically exposed service", but in today's world, that wouldn't be a very accurate picture.
+every publicly exposed service", but in today's world, that wouldn't be a very accurate picture.
 
 We recognize the need to better clarify the relationship between core functionality, platform-like plugin functionality, and functionality exposed by other plugins.
 It's something we will be working on!
 
-</DocAccordion>
-
 We will continue to focus on adding clarity around these types of services and what developers can expect from each.
 
-</DocCallOut>
-
-### Core services
-
-Sometimes referred to just as <DocLink id="kibServerAndCoreComponents" text="Core, Core services"/> provide the most basic and fundamental tools neccessary for building a plugin, like creating saved objects,
-routing, application registration, notifications and <DocLink id="kibCoreLogging" text="logging"/>. The Core platform is not a plugin itself, although
-there are some plugins that provide platform functionality. We call these <DocLink id="kibPlatformIntro" section="platform-plugins" text="Platform plugins"/>.
-
-### Platform plugins
-
-Plugins that provide fundamental services and functionality to extend and customize Kibana, for example, the
-
-<DocLink id="kibDataPlugin" text="data" /> plugin. There is no official way to tell if a plugin is a
-platform plugin or not. Platform plugins are _usually_ plugins that are managed by the Platform Group,
-but we are starting to see some exceptions.
+</DocCallOut></DocAccordion>
 
 ## Plugins
 
@@ -92,7 +91,7 @@ A plugin may register many applications, or none.
 
 Applications are top level pages in the Kibana UI. Dashboard, Canvas, Maps, App Search, etc, are all examples of applications:
 
-![applications in kibana](./assets/applications.png)
+![applications in kibana](../assets/applications.png)
 
 A plugin can register an application by
 adding it to core's application <DocLink id="kibPlatformIntro" section="registry" text="registry"/>.

docs/apm/troubleshooting.asciidoc

@@ -72,8 +72,6 @@ then the index template will not be set up automatically. Instead, you'll need t
 
 *Using a custom index names*
 This problem can also occur if you've customized the index name that you write APM data to.
-The default index name that APM writes events to can be found
-{apm-server-ref}/elasticsearch-output.html#index-option-es[here].
 If you change the default, you must also configure the `setup.template.name` and `setup.template.pattern` options.
 See {apm-server-ref}/configuration-template.html[Load the Elasticsearch index template].
 If the Elasticsearch index template has already been successfully loaded to the index,

docs/dev-tools/grokdebugger/index.asciidoc

@@ -9,21 +9,22 @@ structure it. Grok is good for parsing syslog, apache, and other
 webserver logs, mysql logs, and in general, any log format that is
 written for human consumption.
 
-Grok patterns are supported in the ingest node
-{ref}/grok-processor.html[grok processor] and the Logstash
-{logstash-ref}/plugins-filters-grok.html[grok filter]. See
-{logstash-ref}/plugins-filters-grok.html#_grok_basics[grok basics]
-for more information on the syntax for a grok pattern.
-
-The Elastic Stack ships
-with more than 120 reusable grok patterns. See
-https://github.com/elastic/elasticsearch/tree/master/libs/grok/src/main/resources/patterns[Ingest node grok patterns] and https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns[Logstash grok patterns]
-for the complete list of patterns.
+Grok patterns are supported in {es} {ref}/runtime.html[runtime fields], the {es}
+{ref}/grok-processor.html[grok ingest processor], and the {ls}
+{logstash-ref}/plugins-filters-grok.html[grok filter]. For syntax, see
+{ref}/grok.html[Grokking grok].
+
+The {stack} ships with more than 120 reusable grok patterns. For a complete
+list of patterns, see
+https://github.com/elastic/elasticsearch/tree/master/libs/grok/src/main/resources/patterns[{es}
+grok patterns] and
+https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns[{ls}
+grok patterns].
 
 Because
-ingest node and Logstash share the same grok implementation and pattern
+{es} and {ls} share the same grok implementation and pattern
 libraries, any grok pattern that you create in the *Grok Debugger* will work
-in ingest node and Logstash.
+in both {es} and {ls}.
 
 [float]
 [[grokdebugger-getting-started]]

docs/developer/architecture/core/logging-configuration-migration.asciidoc

@@ -76,9 +76,5 @@ you can override the flags with:
 
 |--verbose| --logging.root.level=debug --logging.root.appenders[0]=default --logging.root.appenders[1]=custom | --verbose
 
-|--quiet| --logging.root.level=error --logging.root.appenders[0]=default --logging.root.appenders[1]=custom | not supported
-
 |--silent| --logging.root.level=off | --silent
 |===
-
-NOTE: To preserve backwards compatibility, you are required to pass the root `default` appender until the legacy logging system is removed in `v8.0`.

docs/developer/getting-started/monorepo-packages.asciidoc

@@ -74,7 +74,6 @@ yarn kbn watch
 - @kbn/i18n
 - @kbn/interpreter
 - @kbn/io-ts-utils
-- @kbn/legacy-logging
 - @kbn/logging
 - @kbn/mapbox-gl
 - @kbn/monaco

docs/developer/plugin-list.asciidoc

@@ -458,7 +458,7 @@ the infrastructure monitoring use-case within Kibana.
 
 
 |{kib-repo}blob/{branch}/x-pack/plugins/ingest_pipelines/README.md[ingestPipelines]
-|The ingest_pipelines plugin provides Kibana support for Elasticsearch's ingest nodes. Please refer to the Elasticsearch documentation for more details.
+|The ingest_pipelines plugin provides Kibana support for Elasticsearch's ingest pipelines.
 
 
 |{kib-repo}blob/{branch}/x-pack/plugins/lens/readme.md[lens]

docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md

@@ -234,5 +234,17 @@ readonly links: {
         readonly ecs: {
             readonly guide: string;
         };
+        readonly clients: {
+            readonly guide: string;
+            readonly goOverview: string;
+            readonly javaIndex: string;
+            readonly jsIntro: string;
+            readonly netGuide: string;
+            readonly perlGuide: string;
+            readonly phpGuide: string;
+            readonly pythonGuide: string;
+            readonly rubyOverview: string;
+            readonly rustGuide: string;
+        };
     };
 ```