Merge master into feature-ingest

.ci/end2end.groovy

@@ -0,0 +1,125 @@
+#!/usr/bin/env groovy
+
+library identifier: 'apm@current',
+retriever: modernSCM(
+  [$class: 'GitSCMSource',
+  credentialsId: 'f94e9298-83ae-417e-ba91-85c279771570',
+  id: '37cf2c00-2cc7-482e-8c62-7bbffef475e2',
+  remote: 'git@github.com:elastic/apm-pipeline-library.git'])
+
+pipeline {
+  agent { label 'linux && immutable' }
+  environment {
+    BASE_DIR = 'src/github.com/elastic/kibana'
+    HOME = "${env.WORKSPACE}"
+    APM_ITS = 'apm-integration-testing'
+    CYPRESS_DIR = 'x-pack/legacy/plugins/apm/cypress'
+    PIPELINE_LOG_LEVEL = 'DEBUG'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '40', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+  }
+  triggers {
+    issueCommentTrigger('(?i).*jenkins\\W+run\\W+(?:the\\W+)?e2e(?:\\W+please)?.*')
+  }
+  parameters {
+    booleanParam(name: 'FORCE', defaultValue: false, description: 'Whether to force the run.')
+  }
+  stages {
+    stage('Checkout') {
+      options { skipDefaultCheckout() }
+      steps {
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: false,
+                    shallow: false, reference: "/var/lib/jenkins/.git-references/kibana.git")
+        script {
+          dir("${BASE_DIR}"){
+            def regexps =[ "^x-pack/legacy/plugins/apm/.*" ]
+            env.APM_UPDATED = isGitRegionMatch(patterns: regexps)
+          }
+        }
+        dir("${APM_ITS}"){
+          git changelog: false,
+              credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
+              poll: false,
+              url: "git@github.com:elastic/${APM_ITS}.git"
+        }
+      }
+    }
+    stage('Start services') {
+      options { skipDefaultCheckout() }
+      when {
+        anyOf {
+          expression { return params.FORCE }
+          expression { return env.APM_UPDATED != "false" }
+        }
+      }
+      steps {
+        dir("${APM_ITS}"){
+          sh './scripts/compose.py start master --no-kibana --no-xpack-secure'
+        }
+      }
+    }
+    stage('Prepare Kibana') {
+      options { skipDefaultCheckout() }
+      when {
+        anyOf {
+          expression { return params.FORCE }
+          expression { return env.APM_UPDATED != "false" }
+        }
+      }
+      environment {
+        JENKINS_NODE_COOKIE = 'dontKillMe'
+      }
+      steps {
+        dir("${BASE_DIR}"){
+          sh script: "${CYPRESS_DIR}/ci/prepare-kibana.sh"
+        }
+      }
+    }
+    stage('Smoke Tests'){
+      options { skipDefaultCheckout() }
+      when {
+        anyOf {
+          expression { return params.FORCE }
+          expression { return env.APM_UPDATED != "false" }
+        }
+      }
+      steps{
+        dir("${BASE_DIR}"){
+          sh '''
+            jobs -l
+            docker build --tag cypress ${CYPRESS_DIR}/ci
+            docker run --rm -t --user "$(id -u):$(id -g)" \
+                    -v `pwd`:/app --network="host" \
+                    --name cypress cypress'''
+        }
+      }
+      post {
+        always {
+          dir("${BASE_DIR}"){
+            archiveArtifacts(allowEmptyArchive: false, artifacts: "${CYPRESS_DIR}/screenshots/**,${CYPRESS_DIR}/videos/**,${CYPRESS_DIR}/*e2e-tests.xml")
+            junit(allowEmptyResults: true, testResults: "${CYPRESS_DIR}/*e2e-tests.xml")
+          }
+          dir("${APM_ITS}"){
+            sh 'docker-compose logs > apm-its.log || true'
+            sh 'docker-compose down -v || true'
+            archiveArtifacts(allowEmptyArchive: false, artifacts: 'apm-its.log')
+          }
+        }
+      }
+    }
+  }
+  post {
+    always {
+      dir("${BASE_DIR}"){
+        archiveArtifacts(allowEmptyArchive: true, artifacts: "${CYPRESS_DIR}/ingest-data.log,kibana.log")
+      }
+    }
+  }
+}

.ci/packer_cache.sh

@@ -44,6 +44,7 @@ tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \
   x-pack/legacy/plugins/*/node_modules \
   x-pack/legacy/plugins/reporting/.chromium \
   test/plugin_functional/plugins/*/node_modules \
+  examples/*/node_modules \
   .es \
   .chromedriver \
   .geckodriver;

.eslintrc.js

@@ -163,26 +163,13 @@ module.exports = {
         'jsx-a11y/click-events-have-key-events': 'off',
       },
     },
-    {
-      files: ['x-pack/legacy/plugins/graph/**/*.{js,ts,tsx}'],
-      rules: {
-        'react-hooks/exhaustive-deps': 'off',
-      },
-    },
     {
       files: ['x-pack/legacy/plugins/index_management/**/*.{js,ts,tsx}'],
       rules: {
         'react-hooks/exhaustive-deps': 'off',
         'react-hooks/rules-of-hooks': 'off',
       },
     },
-    {
-      files: ['x-pack/legacy/plugins/infra/**/*.{js,ts,tsx}'],
-      rules: {
-        'react-hooks/exhaustive-deps': 'off',
-        'react-hooks/rules-of-hooks': 'off',
-      },
-    },
     {
       files: ['x-pack/legacy/plugins/lens/**/*.{js,ts,tsx}'],
       rules: {
@@ -215,13 +202,6 @@ module.exports = {
         'react-hooks/rules-of-hooks': 'off',
       },
     },
-    {
-      files: ['x-pack/legacy/plugins/watcher/**/*.{js,ts,tsx}'],
-      rules: {
-        'react-hooks/rules-of-hooks': 'off',
-        'react-hooks/exhaustive-deps': 'off',
-      },
-    },
 
     /**
      * Prettier
@@ -337,6 +317,7 @@ module.exports = {
                   '!src/core/server/index.ts',
                   '!src/core/server/mocks.ts',
                   '!src/core/server/types.ts',
+                  '!src/core/server/test_utils.ts',
                   // for absolute imports until fixed in
                   // https://github.com/elastic/kibana/issues/36096
                   '!src/core/server/types',
@@ -350,6 +331,21 @@ module.exports = {
                 ],
                 allowSameFolder: true,
               },
+              {
+                target: ['src/core/**/*'],
+                from: ['x-pack/**/*'],
+                errorMessage: 'OSS cannot import x-pack files.',
+              },
+              {
+                target: ['src/core/**/*'],
+                from: [
+                  'plugins/**/*',
+                  'src/plugins/**/*',
+                  'src/legacy/core_plugins/**/*',
+                  'src/legacy/ui/**/*',
+                ],
+                errorMessage: 'The core cannot depend on any plugins.',
+              },
               {
                 from: ['src/legacy/ui/**/*', 'ui/**/*'],
                 target: [

.github/CODEOWNERS

@@ -5,6 +5,9 @@
 # App
 /x-pack/legacy/plugins/lens/ @elastic/kibana-app
 /x-pack/legacy/plugins/graph/ @elastic/kibana-app
+/src/plugins/share/ @elastic/kibana-app
+/src/legacy/server/url_shortening/ @elastic/kibana-app
+/src/legacy/server/sample_data/ @elastic/kibana-app
 
 # App Architecture
 /src/plugins/data/ @elastic/kibana-app-arch
@@ -16,6 +19,16 @@
 /src/plugins/ui_actions/ @elastic/kibana-app-arch
 /src/plugins/visualizations/ @elastic/kibana-app-arch
 /x-pack/plugins/advanced_ui_actions/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/data/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/embeddable_api/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/interpreter/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/kibana_react/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/kibana/public/management/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/kibana/server/field_formats/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/kibana/server/routes/api/management/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/kibana/server/routes/api/suggestions/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/visualizations/ @elastic/kibana-app-arch
+/src/legacy/server/index_patterns/ @elastic/kibana-app-arch
 
 # APM
 /x-pack/legacy/plugins/apm/  @elastic/apm-ui
@@ -54,14 +67,25 @@
 /packages/kbn-es/ @elastic/kibana-operations
 /packages/kbn-pm/ @elastic/kibana-operations
 /packages/kbn-test/ @elastic/kibana-operations
+/src/legacy/server/keystore/ @elastic/kibana-operations
+/src/legacy/server/pid/ @elastic/kibana-operations
+/src/legacy/server/sass/ @elastic/kibana-operations
+/src/legacy/server/utils/ @elastic/kibana-operations
+/src/legacy/server/warnings/ @elastic/kibana-operations
 
 # Platform
 /src/core/  @elastic/kibana-platform
-/src/legacy/server/saved_objects/  @elastic/kibana-platform
 /config/kibana.yml @elastic/kibana-platform
 /x-pack/plugins/features/  @elastic/kibana-platform
 /x-pack/plugins/licensing/  @elastic/kibana-platform
 /packages/kbn-config-schema/ @elastic/kibana-platform
+/src/legacy/server/config/ @elastic/kibana-platform
+/src/legacy/server/csp/ @elastic/kibana-platform
+/src/legacy/server/http/ @elastic/kibana-platform
+/src/legacy/server/i18n/ @elastic/kibana-platform
+/src/legacy/server/logging/ @elastic/kibana-platform
+/src/legacy/server/saved_objects/  @elastic/kibana-platform
+/src/legacy/server/status/ @elastic/kibana-platform
 
 # Security
 /x-pack/legacy/plugins/security/  @elastic/kibana-security

.gitignore

@@ -29,6 +29,7 @@ disabledPlugins
 webpackstats.json
 /config/*
 !/config/kibana.yml
+!/config/apm.js
 coverage
 selenium
 .babel_register_cache.json

.i18nrc.json

@@ -17,8 +17,10 @@
     "kbn": "src/legacy/core_plugins/kibana",
     "kbnDocViews": "src/legacy/core_plugins/kbn_doc_views",
     "kbnVislibVisTypes": "src/legacy/core_plugins/kbn_vislib_vis_types",
+    "management": "src/legacy/core_plugins/management",
     "kibana_react": "src/legacy/core_plugins/kibana_react",
     "kibana-react": "src/plugins/kibana_react",
+    "kibana_utils": "src/plugins/kibana_utils",
     "navigation": "src/legacy/core_plugins/navigation",
     "newsfeed": "src/plugins/newsfeed",
     "regionMap": "src/legacy/core_plugins/region_map",

CONTRIBUTING.md

@@ -24,6 +24,7 @@ A high level overview of our contributing guidelines.
   - [Internationalization](#internationalization)
   - [Testing and Building](#testing-and-building)
     - [Debugging server code](#debugging-server-code)
+    - [Instrumenting with Elastic APM](#instrumenting-with-elastic-apm)
   - [Debugging Unit Tests](#debugging-unit-tests)
   - [Unit Testing Plugins](#unit-testing-plugins)
   - [Cross-browser compatibility](#cross-browser-compatibility)
@@ -374,6 +375,29 @@ macOS users on a machine with a discrete graphics card may see significant speed
 ### Debugging Server Code
 `yarn debug` will start the server with Node's inspect flag. Kibana's development mode will start three processes on ports `9229`, `9230`, and `9231`. Chrome's developer tools need to be configured to connect to all three connections. Add `localhost:<port>` for each Kibana process in Chrome's developer tools connection tab.
 
+### Instrumenting with Elastic APM
+Kibana ships with the [Elastic APM Node.js Agent](https://github.com/elastic/apm-agent-nodejs) built-in for debugging purposes.
+
+Its default configuration is meant to be used by core Kibana developers only, but it can easily be re-configured to your needs.
+In its default configuration it's disabled and will, once enabled, send APM data to a centrally managed Elasticsearch cluster accessible only to Elastic employees.
+
+To change the location where data is sent, use the [`serverUrl`](https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#server-url) APM config option.
+To activate the APM agent, use the [`active`](https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#active) APM config option.
+
+All config options can be set either via environment variables, or by creating an appropriate config file under `config/apm.dev.js`.
+For more information about configuring the APM agent, please refer to [the documentation](https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuring-the-agent.html).
+
+Example `config/apm.dev.js` file:
+
+```js
+module.exports = {
+  active: true,
+};
+```
+
+Once the agent is active, it will trace all incoming HTTP requests to Kibana, monitor for errors, and collect process-level metrics.
+The collected data will be sent to the APM Server and is viewable in the APM UI in Kibana.
+
 ### Unit testing frameworks
 Kibana is migrating unit testing from Mocha to Jest. Legacy unit tests still
 exist in Mocha but all new unit tests should be written in Jest. Mocha tests
@@ -389,7 +413,7 @@ The following table outlines possible test file locations and how to invoke them
 | Jest               | `src/**/*.test.js`<br>`src/**/*.test.ts`                                                                                                                | `node scripts/jest -t regexp [test path]`                                               |
 | Jest (integration) | `**/integration_tests/**/*.test.js`                                                                                                                     | `node scripts/jest_integration -t regexp [test path]`                                   |
 | Mocha              | `src/**/__tests__/**/*.js`<br>`!src/**/public/__tests__/*.js`<br>`packages/kbn-datemath/test/**/*.js`<br>`packages/kbn-dev-utils/src/**/__tests__/**/*.js`<br>`tasks/**/__tests__/**/*.js` | `node scripts/mocha --grep=regexp [test path]`       |
-| Functional         | `test/*integration/**/config.js`<br>`test/*functional/**/config.js`                                                                                     | `node scripts/functional_tests_server --config test/[directory]/config.js`<br>`node scripts/functional_test_runner --config test/[directory]/config.js --grep=regexp`       |
+| Functional         | `test/*integration/**/config.js`<br>`test/*functional/**/config.js`<br>`test/accessibility/config.js`                                                                                    | `node scripts/functional_tests_server --config test/[directory]/config.js`<br>`node scripts/functional_test_runner --config test/[directory]/config.js --grep=regexp`       |
 | Karma              | `src/**/public/__tests__/*.js`                                                                                                                          | `npm run test:dev`                                                                      |
 
 For X-Pack tests located in `x-pack/` see [X-Pack Testing](x-pack/README.md#testing)