Merge branch 'master' into short-urls-2

.buildkite/pipelines/es_snapshots/verify.yml

@@ -64,7 +64,7 @@ steps:
   - command: .buildkite/scripts/steps/test/jest_integration.sh
     label: 'Jest Integration Tests'
     agents:
-      queue: jest
+      queue: n2-4
     timeout_in_minutes: 120
     key: jest-integration
     retry:

.buildkite/pipelines/hourly.yml

@@ -1,3 +1,5 @@
+env:
+  REPORT_FAILED_TESTS_TO_GITHUB: 'true'
 steps:
   - command: .buildkite/scripts/lifecycle/pre_build.sh
     label: Pre-Build
@@ -118,14 +120,14 @@ steps:
   - command: .buildkite/scripts/steps/test/jest_integration.sh
     label: 'Jest Integration Tests'
     agents:
-      queue: jest
+      queue: n2-4
     timeout_in_minutes: 120
     key: jest-integration
 
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'
     agents:
-      queue: jest
+      queue: n2-2
     timeout_in_minutes: 120
     key: api-integration
 

.buildkite/scripts/lifecycle/post_command.sh

@@ -22,6 +22,5 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'x-pack/test/functional/failure_debug/html/*.html'
   buildkite-agent artifact upload '.es/**/*.hprof'
 
-  # TODO - re-enable when Jenkins is disabled
-  # node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" 'target/junit/**/*.xml'
+  node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" 'target/junit/**/*.xml'
 fi

.buildkite/scripts/steps/storybooks/build_and_upload.js

@@ -13,6 +13,7 @@ const STORYBOOKS = [
   'dashboard_enhanced',
   'data_enhanced',
   'embeddable',
+  'fleet',
   'infra',
   'security_solution',
   'ui_actions_enhanced',

.eslintrc.js

@@ -1502,7 +1502,7 @@ module.exports = {
      * TSVB overrides
      */
     {
-      files: ['src/plugins/vis_type_timeseries/**/*.{js,mjs,ts,tsx}'],
+      files: ['src/plugins/vis_types/timeseries/**/*.{js,mjs,ts,tsx}'],
       rules: {
         'import/no-default-export': 'error',
       },

.github/CODEOWNERS

@@ -30,7 +30,7 @@
 /src/plugins/vis_types/table/ @elastic/kibana-vis-editors
 /src/plugins/vis_types/tagcloud/ @elastic/kibana-vis-editors
 /src/plugins/vis_types/timelion/ @elastic/kibana-vis-editors
-/src/plugins/vis_type_timeseries/ @elastic/kibana-vis-editors
+/src/plugins/vis_types/timeseries/ @elastic/kibana-vis-editors
 /src/plugins/vis_types/vega/ @elastic/kibana-vis-editors
 /src/plugins/vis_types/vislib/ @elastic/kibana-vis-editors
 /src/plugins/vis_types/xy/ @elastic/kibana-vis-editors

.i18nrc.json

@@ -63,7 +63,7 @@
     "visTypeMetric": "src/plugins/vis_types/metric",
     "visTypeTable": "src/plugins/vis_types/table",
     "visTypeTagCloud": "src/plugins/vis_types/tagcloud",
-    "visTypeTimeseries": "src/plugins/vis_type_timeseries",
+    "visTypeTimeseries": "src/plugins/vis_types/timeseries",
     "visTypeVega": "src/plugins/vis_types/vega",
     "visTypeVislib": "src/plugins/vis_types/vislib",
     "visTypeXy": "src/plugins/vis_types/xy",

Jenkinsfile

@@ -1,5 +1,10 @@
 #!/bin/groovy
 
+if (!env.ghprbPullId) {
+  print "Non-PR builds are now in Buildkite."
+  return
+}
+
 library 'kibana-pipeline-library'
 kibanaLibrary.load()
 

api_docs/vis_type_timeseries.json

@@ -24,7 +24,7 @@
           ") => data is ",
           "SeriesData"
         ],
-        "path": "src/plugins/vis_type_timeseries/common/vis_data_utils.ts",
+        "path": "src/plugins/vis_types/timeseries/common/vis_data_utils.ts",
         "deprecated": false,
         "children": [
           {
@@ -37,7 +37,7 @@
             "signature": [
               "TimeseriesVisData"
             ],
-            "path": "src/plugins/vis_type_timeseries/common/vis_data_utils.ts",
+            "path": "src/plugins/vis_types/timeseries/common/vis_data_utils.ts",
             "deprecated": false,
             "isRequired": true
           }
@@ -58,7 +58,7 @@
           ") => data is ",
           "TableData"
         ],
-        "path": "src/plugins/vis_type_timeseries/common/vis_data_utils.ts",
+        "path": "src/plugins/vis_types/timeseries/common/vis_data_utils.ts",
         "deprecated": false,
         "children": [
           {
@@ -71,7 +71,7 @@
             "signature": [
               "TimeseriesVisData"
             ],
-            "path": "src/plugins/vis_type_timeseries/common/vis_data_utils.ts",
+            "path": "src/plugins/vis_types/timeseries/common/vis_data_utils.ts",
             "deprecated": false,
             "isRequired": true
           }
@@ -95,7 +95,7 @@
           " | ",
           "TableData"
         ],
-        "path": "src/plugins/vis_type_timeseries/common/types/vis_data.ts",
+        "path": "src/plugins/vis_types/timeseries/common/types/vis_data.ts",
         "deprecated": false,
         "initialIsOpen": false
       }
@@ -108,7 +108,7 @@
       "tags": [],
       "label": "VisTypeTimeseriesSetup",
       "description": [],
-      "path": "src/plugins/vis_type_timeseries/server/plugin.ts",
+      "path": "src/plugins/vis_types/timeseries/server/plugin.ts",
       "deprecated": false,
       "children": [
         {
@@ -133,7 +133,7 @@
             "TimeseriesVisData",
             ">"
           ],
-          "path": "src/plugins/vis_type_timeseries/server/plugin.ts",
+          "path": "src/plugins/vis_types/timeseries/server/plugin.ts",
           "deprecated": false,
           "children": [
             {
@@ -146,7 +146,7 @@
               "signature": [
                 "DataRequestHandlerContext"
               ],
-              "path": "src/plugins/vis_type_timeseries/server/plugin.ts",
+              "path": "src/plugins/vis_types/timeseries/server/plugin.ts",
               "deprecated": false,
               "isRequired": true
             },
@@ -167,7 +167,7 @@
                 },
                 "<unknown, unknown, unknown, any>"
               ],
-              "path": "src/plugins/vis_type_timeseries/server/plugin.ts",
+              "path": "src/plugins/vis_types/timeseries/server/plugin.ts",
               "deprecated": false,
               "isRequired": true
             },
@@ -181,7 +181,7 @@
               "signature": [
                 "any"
               ],
-              "path": "src/plugins/vis_type_timeseries/server/plugin.ts",
+              "path": "src/plugins/vis_types/timeseries/server/plugin.ts",
               "deprecated": false,
               "isRequired": true
             }

docs/api/machine-learning.asciidoc

@@ -0,0 +1,11 @@
+[[machine-learning-api]]
+== {ml-cap} APIs
+
+//Manage {kib} saved objects, including dashboards, visualizations, and more.
+
+The following {ml} API is available: 
+
+* <<machine-learning-api-sync, Sync API>>
+//to retrieve a single {kib} saved object by ID
+
+include::machine-learning/sync.asciidoc[]

docs/api/machine-learning/sync.asciidoc

@@ -0,0 +1,79 @@
+[[machine-learning-api-sync]]
+=== Sync {ml} saved objects API
+++++
+<titleabbrev>Sync {ml} saved objects</titleabbrev>
+++++
+
+Synchronizes {kib} saved objects for {ml} jobs.
+
+[[machine-learning-api-sync-request]]
+==== Request
+
+`GET <kibana host>:<port>/api/ml/saved_objects/sync`
+
+`GET <kibana host>:<port>/s/<space_id>/api/ml/saved_objects/sync`
+
+
+[[machine-learning-api-sync-path-params]]
+==== Path parameters
+
+`space_id`::
+(Optional, string) An identifier for the space. If `space_id` is not provided in
+the URL the default space is used.
+
+[[machine-learning-api-sync-query-params]]
+==== Query parameters
+
+`simulate`::
+(Optional, boolean) When `true`, simulates the synchronization by only returning
+the list actions that _would_ be performed.
+
+[[machine-learning-api-sync-response-body]]
+==== Response body
+
+`datafeedsAdded`::
+(array) If a saved object for an {anomaly-job} is missing a {dfeed} identifier,
+it is added. This list contains the {dfeed} identifiers and indicates whether
+the synchronization was successful.
+
+`datafeedsRemoved`::
+(array) If saved objects exist for {dfeeds} that no longer exist, they are
+deleted. This list contains the {dfeed} identifiers and indicates whether the
+synchronization was successful.
+
+`savedObjectsCreated`::
+(array) If saved objects are missing for {ml} jobs, they are created. This
+list contains the job identifiers and indicates whether the synchronization was
+successful.
+
+`savedObjectsDeleted`::
+(array) If saved objects exist for jobs that no longer exist, they are deleted. 
+This list contains the job identifiers and indicates whether the synchronization
+was successful.
+
+[[machine-learning-api-sync-codes]]
+==== Response code
+
+`200`::
+  Indicates a successful call.
+
+[[machine-learning-api-sync-example]]
+==== Example
+
+Retrieve the list of {ml} saved objects that require synchronization:
+
+[source,sh]
+--------------------------------------------------
+$ curl -X GET api/ml/saved_objects/sync?simulate=true
+--------------------------------------------------
+// KIBANA
+
+If there are two jobs and a {dfeed} that need to be synchronized, for example,
+the API returns the following:
+
+[source,sh]
+--------------------------------------------------
+{{"savedObjectsCreated":{"myjob1":{"success":true},"myjob2":{"success":true}},"savedObjectsDeleted":{},"datafeedsAdded":{},"datafeedsRemoved":{"myfeed3":{"success":true}}}
+--------------------------------------------------
+
+To perform the synchronization, re-run the API and omit the `simulate` parameter.

docs/developer/plugin-list.asciidoc

@@ -297,7 +297,7 @@ The plugin exposes the static DefaultEditorController class to consume.
 |Contains the timelion visualization and the timelion backend.
 
 
-|{kib-repo}blob/{branch}/src/plugins/vis_type_timeseries[visTypeTimeseries]
+|{kib-repo}blob/{branch}/src/plugins/vis_types/timeseries[visTypeTimeseries]
 |WARNING: Missing README.
 
 

docs/settings/fleet-settings.asciidoc

@@ -86,6 +86,8 @@ Optional properties are:
   be changed by updating the {kib} config.
   `is_default`:: If `true`, this policy is the default agent policy.
   `is_default_fleet_server`:: If `true`, this policy is the default {fleet-server} agent policy.
+  `data_output_id`:: ID of the output to send data (Need to be identical to `monitoring_output_id`)
+  `monitoring_output_id`:: ID of the output to send monitoring data. (Need to be identical to `data_output_id`)
   `package_policies`:: List of integration policies to add to this policy.
     `name`::: (required) Name of the integration policy.
     `package`::: (required) Integration that this policy configures
@@ -96,6 +98,20 @@ Optional properties are:
     integration. Follows the same schema as integration inputs, with the
     exception that any object in `vars` can be passed `frozen: true` in order to
     prevent that specific `var` from being edited by the user.
+
+| `xpack.fleet.outputs`
+  | List of ouputs that are configured when the {fleet} app starts.
+Required properties are:
+
+  `id`:: Unique ID for this output. The ID should be a string.
+  `name`:: Output name.
+  `type`:: Type of Output. Currently we only support "elasticsearch".
+  `hosts`:: Array that contains the list of host for that output.
+  `config`:: Extra config for that output.
+
+Optional properties are:
+
+  `is_default`:: If `true`, this output is the default output.
 |===
 
 Example configuration:

docs/user/api.asciidoc

@@ -96,6 +96,7 @@ include::{kib-repo-dir}/api/alerting.asciidoc[]
 include::{kib-repo-dir}/api/actions-and-connectors.asciidoc[]
 include::{kib-repo-dir}/api/dashboard-api.asciidoc[]
 include::{kib-repo-dir}/api/logstash-configuration-management.asciidoc[]
+include::{kib-repo-dir}/api/machine-learning.asciidoc[]
 include::{kib-repo-dir}/api/url-shortening.asciidoc[]
 include::{kib-repo-dir}/api/task-manager/health.asciidoc[]
 include::{kib-repo-dir}/api/upgrade-assistant.asciidoc[]

package.json

@@ -574,7 +574,7 @@
     "@types/nock": "^10.0.3",
     "@types/node": "14.14.44",
     "@types/node-fetch": "^2.5.7",
-    "@types/node-forge": "^0.10.4",
+    "@types/node-forge": "^0.10.5",
     "@types/nodemailer": "^6.4.0",
     "@types/normalize-path": "^3.0.0",
     "@types/object-hash": "^1.3.0",

packages/kbn-storybook/templates/index.ejs

@@ -16,12 +16,11 @@
 
     <!-- Added for Kibana shared dependencies -->
     <script>
-      window.__kbnPublicPath__ = { 'kbn-ui-shared-deps-npm': '.', 'kbn-ui-shared-deps-src': '.' };
+      window.__kbnPublicPath__ = { 'kbn-ui-shared-deps-npm': '', 'kbn-ui-shared-deps-src': '' };
     </script>
-    <script src="kbn-ui-shared-deps-npm.@elastic.js"></script>
-    <script src="kbn-ui-shared-deps-npm.js"></script>
+    <script src="kbn-ui-shared-deps-npm.dll.js"></script>
     <script src="kbn-ui-shared-deps-src.js"></script>
-    <link href="kbn-ui-shared-deps-npm.css" rel="stylesheet" />
+    <link href="kbn-ui-shared-deps-src.css" rel="stylesheet" />
     <link id="eui-theme-css" href="kbn-ui-shared-deps-npm.v8.light.css" rel="stylesheet" />
     <!--                                      -->
 

packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts

@@ -37,30 +37,31 @@ export function runFailedTestsReporterCli() {
       }
 
       if (updateGithub) {
-        let branch: string | undefined = '';
+        let branch: string = '';
         let isPr = false;
 
         if (process.env.BUILDKITE === 'true') {
-          branch = process.env.BUILDKITE_BRANCH;
+          branch = process.env.BUILDKITE_BRANCH || '';
           isPr = process.env.BUILDKITE_PULL_REQUEST === 'true';
+          updateGithub = process.env.REPORT_FAILED_TESTS_TO_GITHUB === 'true';
         } else {
           // JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others
           const jobNameSplit = (process.env.JOB_NAME || '').split(/\+|\//);
-          branch = jobNameSplit.length >= 3 ? jobNameSplit[2] : process.env.GIT_BRANCH;
+          branch = jobNameSplit.length >= 3 ? jobNameSplit[2] : process.env.GIT_BRANCH || '';
           isPr = !!process.env.ghprbPullId;
+
+          const isMasterOrVersion = branch === 'master' || branch.match(/^\d+\.(x|\d+)$/);
+          if (!isMasterOrVersion || isPr) {
+            log.info('Failure issues only created on master/version branch jobs');
+            updateGithub = false;
+          }
         }
 
         if (!branch) {
           throw createFailError(
             'Unable to determine originating branch from job name or other environment variables'
           );
         }
-
-        const isMasterOrVersion = branch === 'master' || branch.match(/^\d+\.(x|\d+)$/);
-        if (!isMasterOrVersion || isPr) {
-          log.info('Failure issues only created on master/version branch jobs');
-          updateGithub = false;
-        }
       }
 
       const githubApi = new GithubApi({

src/core/public/doc_links/doc_links_service.ts

@@ -241,6 +241,7 @@ export class DocLinksService {
           kibanaSearchSettings: `${KIBANA_DOCS}advanced-options.html#kibana-search-settings`,
           visualizationSettings: `${KIBANA_DOCS}advanced-options.html#kibana-visualization-settings`,
           timelionSettings: `${KIBANA_DOCS}advanced-options.html#kibana-timelion-settings`,
+          savedObjectsApiList: `${KIBANA_DOCS}saved-objects-api.html#saved-objects-api`,
         },
         ml: {
           guide: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/index.html`,
@@ -387,7 +388,7 @@ export class DocLinksService {
         },
         snapshotRestore: {
           guide: `${KIBANA_DOCS}snapshot-repositories.html`,
-          changeIndexSettings: `${ELASTICSEARCH_DOCS}snapshots-restore-snapshot.html#change-index-settings-during-restore`,
+          changeIndexSettings: `${ELASTICSEARCH_DOCS}index-modules.html`,
           createSnapshot: `${ELASTICSEARCH_DOCS}snapshots-take-snapshot.html`,
           getSnapshot: `${ELASTICSEARCH_DOCS}get-snapshot-api.html`,
           registerSharedFileSystem: `${ELASTICSEARCH_DOCS}snapshots-register-repository.html#snapshots-filesystem-repository`,