Merge branch '7.x' into backport/7.x/pr-58945

docs/developer/core/development-functional-tests.asciidoc

@@ -177,6 +177,25 @@ To run tests on Firefox locally, use `config.firefox.js`:
 node scripts/functional_test_runner --config test/functional/config.firefox.js
 -----------
 
+[float]
+===== Using the test_user service
+
+Tests should run at the positive security boundry condition, meaning that they should be run with the mimimum privileges required (and documented) and not as the superuser.
+ This prevents the type of regression where additional privleges accidentally become required to perform the same action. 
+
+The functional UI tests now default to logging in with a user named `test_user` and the roles of this user can be changed dynamically without logging in and out.  
+
+In order to achieve this a new service was introduced called `createTestUserService` (see `test/common/services/security/test_user.ts`). The purpose of this test user service is to create roles defined in the test config files and setRoles() or restoreDefaults().
+
+An example of how to set the role like how its defined below:
+
+`await security.testUser.setRoles(['kibana_user', 'kibana_date_nanos']);`
+
+Here we are setting the `test_user` to have the `kibana_user` role and also role access to a specific data index (`kibana_date_nanos`).
+
+Tests should normally setRoles() in the before() and restoreDefaults() in the after().
+
+
 [float]
 ===== Anatomy of a test file
 

docs/settings/monitoring-settings.asciidoc

@@ -20,9 +20,7 @@ which support the same values as <<settings,{kib} configuration settings>>.
 To control how data is collected from your {es} nodes, you configure
 {ref}/monitoring-settings.html[`xpack.monitoring.collection`
 settings] in `elasticsearch.yml`. To control how monitoring data is collected
-from Logstash, you configure
-{logstash-ref}/monitoring-internal-collection.html#monitoring-settings[`xpack.monitoring` settings]
-in `logstash.yml`.
+from Logstash, configure monitoring settings in `logstash.yml`.
 
 For more information, see
 {ref}/monitor-elasticsearch-cluster.html[Monitor a cluster].

package.json

@@ -116,7 +116,7 @@
     "@babel/register": "^7.7.0",
     "@elastic/charts": "^17.1.1",
     "@elastic/datemath": "5.0.2",
-    "@elastic/ems-client": "7.6.0",
+    "@elastic/ems-client": "7.7.0",
     "@elastic/eui": "20.0.2",
     "@elastic/filesaver": "1.1.2",
     "@elastic/good": "8.1.1-kibana2",
@@ -236,7 +236,7 @@
     "react-resize-detector": "^4.2.0",
     "react-router-dom": "^5.1.2",
     "react-sizeme": "^2.3.6",
-    "react-use": "^13.13.0",
+    "react-use": "^13.27.0",
     "reactcss": "1.2.3",
     "redux": "^4.0.5",
     "redux-actions": "^2.6.5",

packages/kbn-config-schema/src/internals/index.ts

@@ -314,7 +314,8 @@ export const internals = Joi.extend([
           for (const [entryKey, entryValue] of value) {
             const { value: validatedEntryKey, error: keyError } = Joi.validate(
               entryKey,
-              params.key
+              params.key,
+              { presence: 'required' }
             );
 
             if (keyError) {
@@ -323,7 +324,8 @@ export const internals = Joi.extend([
 
             const { value: validatedEntryValue, error: valueError } = Joi.validate(
               entryValue,
-              params.value
+              params.value,
+              { presence: 'required' }
             );
 
             if (valueError) {
@@ -374,7 +376,8 @@ export const internals = Joi.extend([
           for (const [entryKey, entryValue] of Object.entries(value)) {
             const { value: validatedEntryKey, error: keyError } = Joi.validate(
               entryKey,
-              params.key
+              params.key,
+              { presence: 'required' }
             );
 
             if (keyError) {
@@ -383,7 +386,8 @@ export const internals = Joi.extend([
 
             const { value: validatedEntryValue, error: valueError } = Joi.validate(
               entryValue,
-              params.value
+              params.value,
+              { presence: 'required' }
             );
 
             if (valueError) {

packages/kbn-config-schema/src/types/map_of_type.test.ts

@@ -159,6 +159,24 @@ test('object within mapOf', () => {
   expect(type.validate(value)).toEqual(expected);
 });
 
+test('enforces required object fields within mapOf', () => {
+  const type = schema.mapOf(
+    schema.string(),
+    schema.object({
+      bar: schema.object({
+        baz: schema.number(),
+      }),
+    })
+  );
+  const value = {
+    foo: {},
+  };
+
+  expect(() => type.validate(value)).toThrowErrorMatchingInlineSnapshot(
+    `"[foo.bar.baz]: expected value of type [number] but got [undefined]"`
+  );
+});
+
 test('error preserves full path', () => {
   const type = schema.object({
     grandParentKey: schema.object({

packages/kbn-config-schema/src/types/map_type.ts

@@ -57,7 +57,10 @@ export class MapOfType<K, V> extends Type<Map<K, V>> {
           path.length,
           0,
           // If `key` validation failed, let's stress that to make error more obvious.
-          type === 'map.key' ? `key("${entryKey}")` : entryKey.toString()
+          type === 'map.key' ? `key("${entryKey}")` : entryKey.toString(),
+          // Error could have happened deep inside value/key schema and error message should
+          // include full path.
+          ...(reason instanceof SchemaTypeError ? reason.path : [])
         );
 
         return reason instanceof SchemaTypesError

packages/kbn-config-schema/src/types/record_of_type.test.ts

@@ -159,6 +159,24 @@ test('object within recordOf', () => {
   expect(type.validate(value)).toEqual({ foo: { bar: 123 } });
 });
 
+test('enforces required object fields within recordOf', () => {
+  const type = schema.recordOf(
+    schema.string(),
+    schema.object({
+      bar: schema.object({
+        baz: schema.number(),
+      }),
+    })
+  );
+  const value = {
+    foo: {},
+  };
+
+  expect(() => type.validate(value)).toThrowErrorMatchingInlineSnapshot(
+    `"[foo.bar.baz]: expected value of type [number] but got [undefined]"`
+  );
+});
+
 test('error preserves full path', () => {
   const type = schema.object({
     grandParentKey: schema.object({

packages/kbn-config-schema/src/types/record_type.ts

@@ -49,7 +49,10 @@ export class RecordOfType<K extends string, V> extends Type<Record<K, V>> {
           path.length,
           0,
           // If `key` validation failed, let's stress that to make error more obvious.
-          type === 'record.key' ? `key("${entryKey}")` : entryKey.toString()
+          type === 'record.key' ? `key("${entryKey}")` : entryKey.toString(),
+          // Error could have happened deep inside value/key schema and error message should
+          // include full path.
+          ...(reason instanceof SchemaTypeError ? reason.path : [])
         );
 
         return reason instanceof SchemaTypesError

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -255,5 +255,20 @@ export const schema = Joi.object()
         fixedHeaderHeight: Joi.number().default(50),
       })
       .default(),
+
+    // settings for the security service if there is no defaultRole defined, then default to superuser role.
+    security: Joi.object()
+      .keys({
+        roles: Joi.object().default(),
+        defaultRoles: Joi.array()
+          .items(Joi.string())
+          .when('$primary', {
+            is: true,
+            then: Joi.array().min(1),
+          })
+          .default(['superuser']),
+        disableTestUser: Joi.boolean(),
+      })
+      .default(),
   })
   .default();

src/legacy/core_plugins/kibana/public/discover/kibana_services.ts

@@ -53,8 +53,6 @@ export { wrapInI18nContext } from 'ui/i18n';
 import { search } from '../../../../../plugins/data/public';
 export const { getRequestInspectorStats, getResponseInspectorStats, tabifyAggResponse } = search;
 // @ts-ignore
-export { shortenDottedString } from '../../common/utils/shorten_dotted_string';
-// @ts-ignore
 export { intervalOptions } from 'ui/agg_types';
 export { subscribeWithScope } from '../../../../../plugins/kibana_legacy/public';
 // @ts-ignore

src/legacy/core_plugins/kibana/public/discover/np_ready/angular/directives/field_name/field_name.tsx

@@ -21,7 +21,7 @@ import classNames from 'classnames';
 import { EuiFlexGroup, EuiFlexItem, EuiToolTip } from '@elastic/eui';
 
 import { FieldIcon, FieldIconProps } from '../../../../../../../../../plugins/kibana_react/public';
-import { shortenDottedString } from '../../../../kibana_services';
+import { shortenDottedString } from '../../../helpers';
 import { getFieldTypeName } from './field_type_name';
 
 // property field is provided at discover's field chooser

src/legacy/core_plugins/kibana/public/discover/np_ready/angular/doc_table/components/table_header/helpers.tsx

@@ -16,7 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import { IndexPattern, shortenDottedString } from '../../../../../kibana_services';
+import { IndexPattern } from '../../../../../kibana_services';
+import { shortenDottedString } from '../../../../helpers';
 
 export type SortOrder = [string, string];
 export interface ColumnProps {

src/legacy/core_plugins/kibana/public/discover/np_ready/helpers/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { shortenDottedString } from './shorten_dotted_string';

src/legacy/core_plugins/kibana/common/utils/shorten_dotted_string.js → src/legacy/core_plugins/kibana/public/discover/np_ready/helpers/shorten_dotted_string.ts

@@ -22,10 +22,5 @@ const DOT_PREFIX_RE = /(.).+?\./g;
 /**
  * Convert a dot.notated.string into a short
  * version (d.n.string)
- *
- * @param {string} str - the long string to convert
- * @return {string}
  */
-export function shortenDottedString(input) {
-  return typeof input !== 'string' ? input : input.replace(DOT_PREFIX_RE, '$1.');
-}
+export const shortenDottedString = (input: string) => input.replace(DOT_PREFIX_RE, '$1.');

src/legacy/ui/public/vis/map/service_settings.js

@@ -47,7 +47,8 @@ uiModules
         this._showZoomMessage = true;
         this._emsClient = new EMSClient({
           language: i18n.getLocale(),
-          kbnVersion: kbnVersion,
+          appVersion: kbnVersion,
+          appName: 'kibana',
           fileApiUrl: mapConfig.emsFileApiUrl,
           tileApiUrl: mapConfig.emsTileApiUrl,
           htmlSanitizer: $sanitize,

src/plugins/console/public/lib/kb/kb.js

@@ -147,13 +147,9 @@ function loadApisFromJson(
 }
 
 export function setActiveApi(api) {
-  if (_.isString(api)) {
+  if (!api) {
     $.ajax({
-      url:
-        '../api/console/api_server?sense_version=' +
-        encodeURIComponent('@@SENSE_VERSION') +
-        '&apis=' +
-        encodeURIComponent(api),
+      url: '../api/console/api_server',
       dataType: 'json', // disable automatic guessing
     }).then(
       function(data) {
@@ -169,7 +165,7 @@ export function setActiveApi(api) {
   ACTIVE_API = api;
 }
 
-setActiveApi('es_6_0');
+setActiveApi();
 
 export const _test = {
   loadApisFromJson: loadApisFromJson,

src/plugins/console/server/lib/index.ts

@@ -22,4 +22,4 @@ export { ProxyConfigCollection } from './proxy_config_collection';
 export { proxyRequest } from './proxy_request';
 export { getElasticsearchProxyConfig } from './elasticsearch_proxy_config';
 export { setHeaders } from './set_headers';
-export { addProcessorDefinition, addExtensionSpecFilePath } from './spec_definitions';
+export { addProcessorDefinition, addExtensionSpecFilePath, loadSpec } from './spec_definitions';

src/plugins/console/server/lib/spec_definitions/es_6_0.js → src/plugins/console/server/lib/spec_definitions/es.js

@@ -18,26 +18,30 @@
  */
 
 import Api from './api';
-import { getSpec } from './spec';
-import { register } from './es_6_0/ingest';
-const ES_6_0 = new Api('es_6_0');
-const spec = getSpec();
+import { getSpec } from './json';
+import { register } from './js/ingest';
+const ES = new Api('es');
 
-// adding generated specs
-Object.keys(spec).forEach(endpoint => {
-  ES_6_0.addEndpointDescription(endpoint, spec[endpoint]);
-});
+export const loadSpec = () => {
+  const spec = getSpec();
 
-//adding globals and custom API definitions
-require('./es_6_0/aliases')(ES_6_0);
-require('./es_6_0/aggregations')(ES_6_0);
-require('./es_6_0/document')(ES_6_0);
-require('./es_6_0/filter')(ES_6_0);
-require('./es_6_0/globals')(ES_6_0);
-register(ES_6_0);
-require('./es_6_0/mappings')(ES_6_0);
-require('./es_6_0/query')(ES_6_0);
-require('./es_6_0/reindex')(ES_6_0);
-require('./es_6_0/search')(ES_6_0);
+  // adding generated specs
+  Object.keys(spec).forEach(endpoint => {
+    ES.addEndpointDescription(endpoint, spec[endpoint]);
+  });
 
-export default ES_6_0;
+  // adding globals and custom API definitions
+  require('./js/aliases')(ES);
+  require('./js/aggregations')(ES);
+  require('./js/document')(ES);
+  require('./js/filter')(ES);
+  require('./js/globals')(ES);
+  register(ES);
+  require('./js/mappings')(ES);
+  require('./js/settings')(ES);
+  require('./js/query')(ES);
+  require('./js/reindex')(ES);
+  require('./js/search')(ES);
+};
+
+export default ES;

src/plugins/console/server/lib/spec_definitions/index.d.ts

@@ -19,6 +19,13 @@
 
 export declare function addProcessorDefinition(...args: any[]): any;
 
-export declare function resolveApi(senseVersion: string, apis: string[]): object;
+export declare function resolveApi(): object;
 
 export declare function addExtensionSpecFilePath(...args: any[]): any;
+
+/**
+ * A function that synchronously reads files JSON from disk and builds
+ * the autocomplete structures served to the client. This must be called
+ * after any extensions have been loaded.
+ */
+export declare function loadSpec(): any;

src/plugins/console/server/lib/spec_definitions/index.js

@@ -17,8 +17,10 @@
  * under the License.
  */
 
-export { addProcessorDefinition } from './es_6_0/ingest';
+export { addProcessorDefinition } from './js/ingest';
 
-export { addExtensionSpecFilePath } from './spec';
+export { addExtensionSpecFilePath } from './json';
+
+export { loadSpec } from './es';
 
 export { resolveApi } from './server';

src/plugins/console/server/lib/spec_definitions/es_6_0/mappings.js → src/plugins/console/server/lib/spec_definitions/js/mappings.js

@@ -19,9 +19,7 @@
 
 const _ = require('lodash');
 
-const BOOLEAN = {
-  __one_of: [true, false],
-};
+import { BOOLEAN } from './shared';
 
 export default function(api) {
   api.addEndpointDescription('put_mapping', {