From b566ba3a98eb020232a34edd3614819e0bb78e50 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 3 Feb 2020 17:25:17 +0100 Subject: [PATCH 01/11] Fix TSVB rollup integration (#56525) --- .../vis_type_timeseries/server/lib/get_vis_data.ts | 6 ++++-- .../core_plugins/vis_type_timeseries/server/routes/vis.ts | 2 +- .../call_with_request_factory.js | 8 ++++---- .../search_strategies/register_rollup_search_strategy.js | 7 ++++++- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/legacy/core_plugins/vis_type_timeseries/server/lib/get_vis_data.ts b/src/legacy/core_plugins/vis_type_timeseries/server/lib/get_vis_data.ts index 58e624fa13442..9a320174ad546 100644 --- a/src/legacy/core_plugins/vis_type_timeseries/server/lib/get_vis_data.ts +++ b/src/legacy/core_plugins/vis_type_timeseries/server/lib/get_vis_data.ts @@ -17,7 +17,7 @@ * under the License. */ -import { RequestHandlerContext } from 'src/core/server'; +import { KibanaRequest, RequestHandlerContext } from 'src/core/server'; import _ from 'lodash'; import { first, map } from 'rxjs/operators'; import { getPanelData } from './vis_data/get_panel_data'; @@ -57,13 +57,15 @@ export type GetVisData = ( export function getVisData( requestContext: RequestHandlerContext, options: GetVisDataOptions, - framework: Framework + framework: Framework, + request?: PublicMethodsOf ): Promise { // NOTE / TODO: This facade has been put in place to make migrating to the New Platform easier. It // removes the need to refactor many layers of dependencies on "req", and instead just augments the top // level object passed from here. The layers should be refactored fully at some point, but for now // this works and we are still using the New Platform services for these vis data portions. const reqFacade: any = { + ...(request || {}), payload: options, getUiSettingsService: () => requestContext.core.uiSettings.client, getSavedObjectsClient: () => requestContext.core.savedObjects.client, diff --git a/src/legacy/core_plugins/vis_type_timeseries/server/routes/vis.ts b/src/legacy/core_plugins/vis_type_timeseries/server/routes/vis.ts index 32e87f5a3f666..83c7b73b5b85f 100644 --- a/src/legacy/core_plugins/vis_type_timeseries/server/routes/vis.ts +++ b/src/legacy/core_plugins/vis_type_timeseries/server/routes/vis.ts @@ -52,7 +52,7 @@ export const visDataRoutes = ( ); } try { - const results = await getVisData(requestContext, request.body, framework); + const results = await getVisData(requestContext, request.body, framework, request); return response.ok({ body: results }); } catch (error) { return response.internalError({ diff --git a/x-pack/legacy/plugins/rollup/server/lib/call_with_request_factory/call_with_request_factory.js b/x-pack/legacy/plugins/rollup/server/lib/call_with_request_factory/call_with_request_factory.js index 537c5cf45ad61..284151d404a47 100644 --- a/x-pack/legacy/plugins/rollup/server/lib/call_with_request_factory/call_with_request_factory.js +++ b/x-pack/legacy/plugins/rollup/server/lib/call_with_request_factory/call_with_request_factory.js @@ -8,10 +8,10 @@ import { once } from 'lodash'; import { elasticsearchJsPlugin } from '../../client/elasticsearch_rollup'; const callWithRequest = once(server => { - const config = { plugins: [elasticsearchJsPlugin] }; - const cluster = server.plugins.elasticsearch.createCluster('rollup', config); - - return cluster.callWithRequest; + const client = server.newPlatform.setup.core.elasticsearch.createClient('rollup', { + plugins: [elasticsearchJsPlugin], + }); + return (request, ...args) => client.asScoped(request).callAsCurrentUser(...args); }); export const callWithRequestFactory = (server, request) => { diff --git a/x-pack/legacy/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js b/x-pack/legacy/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js index 5d732c8e7fbaa..b1369ed43c6de 100644 --- a/x-pack/legacy/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js +++ b/x-pack/legacy/plugins/rollup/server/lib/search_strategies/register_rollup_search_strategy.js @@ -28,5 +28,10 @@ export const registerRollupSearchStrategy = (kbnServer, server) => RollupSearchCapabilities ); - addSearchStrategy(new RollupSearchStrategy(server)); + addSearchStrategy( + new RollupSearchStrategy({ + ...server, + newPlatform: kbnServer.newPlatform, + }) + ); }); From 18a3cee1cff34e59733a51bd7f9fb2ecd7496e68 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Mon, 3 Feb 2020 18:30:51 +0100 Subject: [PATCH 02/11] Perform successful Elasticsearch version check before migrations (#51311) (#56629) * Convert parts of Elasticsearch version check to ts * Move ES version check to NP * Improve types * Wait till for compatible ES nodes before SO migrations * Don't wait for ES compatibility if skipMigrations=true * Legacy Elasticsearch plugin integration test * Make ES compatibility check and migrations logging more visible * Test for isCompatible=false when ES version check throws * Start pollEsNodesVersion immediately * Refactor pollEsNodesVersion --- .../elasticsearch/elasticsearch_config.ts | 14 +- .../elasticsearch_service.mock.ts | 7 + .../elasticsearch_service.test.ts | 11 +- .../elasticsearch/elasticsearch_service.ts | 46 ++- src/core/server/elasticsearch/types.ts | 2 + .../version_check/ensure_es_version.test.ts | 261 ++++++++++++++++++ .../version_check/ensure_es_version.ts | 164 +++++++++++ .../es_kibana_version_compatability.test.ts} | 18 +- .../es_kibana_version_compatability.ts} | 15 +- .../integration_tests/core_services.test.ts | 6 +- .../integration_tests/legacy_service.test.ts | 2 +- .../migrations/kibana/kibana_migrator.test.ts | 6 - .../migrations/kibana/kibana_migrator.ts | 15 +- .../saved_objects_service.test.ts | 61 +++- .../saved_objects/saved_objects_service.ts | 27 +- .../core_plugins/elasticsearch/index.d.ts | 1 + .../core_plugins/elasticsearch/index.js | 14 +- .../integration_tests/elasticsearch.test.ts | 89 ++++++ .../lib/__tests__/ensure_es_version.js | 223 --------------- .../lib/__tests__/health_check.js | 151 ---------- .../elasticsearch/lib/ensure_es_version.js | 126 --------- .../elasticsearch/lib/health_check.js | 75 ----- .../elasticsearch/lib/version_health_check.js | 39 +++ .../lib/version_health_check.test.js | 71 +++++ .../default_route_provider.test.ts | 2 +- .../default_route_provider_config.test.ts | 1 + .../max_payload_size.test.js | 2 +- .../on_post_auth_interceptor.test.ts | 5 +- .../on_request_interceptor.test.ts | 5 +- 29 files changed, 806 insertions(+), 653 deletions(-) create mode 100644 src/core/server/elasticsearch/version_check/ensure_es_version.test.ts create mode 100644 src/core/server/elasticsearch/version_check/ensure_es_version.ts rename src/{legacy/core_plugins/elasticsearch/lib/__tests__/is_es_compatible_with_kibana.js => core/server/elasticsearch/version_check/es_kibana_version_compatability.test.ts} (72%) rename src/{legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.js => core/server/elasticsearch/version_check/es_kibana_version_compatability.ts} (76%) create mode 100644 src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts delete mode 100644 src/legacy/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js delete mode 100644 src/legacy/core_plugins/elasticsearch/lib/__tests__/health_check.js delete mode 100644 src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js delete mode 100644 src/legacy/core_plugins/elasticsearch/lib/health_check.js create mode 100644 src/legacy/core_plugins/elasticsearch/lib/version_health_check.js create mode 100644 src/legacy/core_plugins/elasticsearch/lib/version_health_check.test.js diff --git a/src/core/server/elasticsearch/elasticsearch_config.ts b/src/core/server/elasticsearch/elasticsearch_config.ts index 41bbdadab9185..2f4160ad75e9d 100644 --- a/src/core/server/elasticsearch/elasticsearch_config.ts +++ b/src/core/server/elasticsearch/elasticsearch_config.ts @@ -103,7 +103,19 @@ const configSchema = schema.object({ ), apiVersion: schema.string({ defaultValue: DEFAULT_API_VERSION }), healthCheck: schema.object({ delay: schema.duration({ defaultValue: 2500 }) }), - ignoreVersionMismatch: schema.boolean({ defaultValue: false }), + ignoreVersionMismatch: schema.conditional( + schema.contextRef('dev'), + false, + schema.boolean({ + validate: rawValue => { + if (rawValue === true) { + return '"ignoreVersionMismatch" can only be set to true in development mode'; + } + }, + defaultValue: false, + }), + schema.boolean({ defaultValue: false }) + ), }); const deprecations: ConfigDeprecationProvider = () => [ diff --git a/src/core/server/elasticsearch/elasticsearch_service.mock.ts b/src/core/server/elasticsearch/elasticsearch_service.mock.ts index a4e51ca55b3e7..b8ad375496544 100644 --- a/src/core/server/elasticsearch/elasticsearch_service.mock.ts +++ b/src/core/server/elasticsearch/elasticsearch_service.mock.ts @@ -23,6 +23,7 @@ import { IScopedClusterClient } from './scoped_cluster_client'; import { ElasticsearchConfig } from './elasticsearch_config'; import { ElasticsearchService } from './elasticsearch_service'; import { InternalElasticsearchServiceSetup, ElasticsearchServiceSetup } from './types'; +import { NodesVersionCompatibility } from './version_check/ensure_es_version'; const createScopedClusterClientMock = (): jest.Mocked => ({ callAsInternalUser: jest.fn(), @@ -71,6 +72,12 @@ type MockedInternalElasticSearchServiceSetup = jest.Mocked< const createInternalSetupContractMock = () => { const setupContract: MockedInternalElasticSearchServiceSetup = { ...createSetupContractMock(), + esNodesCompatibility$: new BehaviorSubject({ + isCompatible: true, + incompatibleNodes: [], + warningNodes: [], + kibanaVersion: '8.0.0', + }), legacy: { config$: new BehaviorSubject({} as ElasticsearchConfig), }, diff --git a/src/core/server/elasticsearch/elasticsearch_service.test.ts b/src/core/server/elasticsearch/elasticsearch_service.test.ts index 5a7d223fec7ad..022a03e01d37d 100644 --- a/src/core/server/elasticsearch/elasticsearch_service.test.ts +++ b/src/core/server/elasticsearch/elasticsearch_service.test.ts @@ -31,6 +31,7 @@ import { httpServiceMock } from '../http/http_service.mock'; import { ElasticsearchConfig } from './elasticsearch_config'; import { ElasticsearchService } from './elasticsearch_service'; import { elasticsearchServiceMock } from './elasticsearch_service.mock'; +import { duration } from 'moment'; let elasticsearchService: ElasticsearchService; const configService = configServiceMock.create(); @@ -41,7 +42,7 @@ configService.atPath.mockReturnValue( new BehaviorSubject({ hosts: ['http://1.2.3.4'], healthCheck: { - delay: 2000, + delay: duration(2000), }, ssl: { verificationMode: 'none', @@ -125,7 +126,7 @@ describe('#setup', () => { const config = MockClusterClient.mock.calls[0][0]; expect(config).toMatchInlineSnapshot(` Object { - "healthCheckDelay": 2000, + "healthCheckDelay": "PT2S", "hosts": Array [ "http://8.8.8.8", ], @@ -150,7 +151,7 @@ Object { const config = MockClusterClient.mock.calls[0][0]; expect(config).toMatchInlineSnapshot(` Object { - "healthCheckDelay": 2000, + "healthCheckDelay": "PT2S", "hosts": Array [ "http://1.2.3.4", ], @@ -174,7 +175,7 @@ Object { new BehaviorSubject({ hosts: ['http://1.2.3.4', 'http://9.8.7.6'], healthCheck: { - delay: 2000, + delay: duration(2000), }, ssl: { verificationMode: 'none', @@ -196,7 +197,7 @@ Object { const config = MockClusterClient.mock.calls[0][0]; expect(config).toMatchInlineSnapshot(` Object { - "healthCheckDelay": 2000, + "healthCheckDelay": "PT2S", "hosts": Array [ "http://8.8.8.8", ], diff --git a/src/core/server/elasticsearch/elasticsearch_service.ts b/src/core/server/elasticsearch/elasticsearch_service.ts index aba246ce66fb5..4d897dbd8f7aa 100644 --- a/src/core/server/elasticsearch/elasticsearch_service.ts +++ b/src/core/server/elasticsearch/elasticsearch_service.ts @@ -30,6 +30,7 @@ import { ElasticsearchConfig, ElasticsearchConfigType } from './elasticsearch_co import { InternalHttpServiceSetup, GetAuthHeaders } from '../http/'; import { InternalElasticsearchServiceSetup } from './types'; import { CallAPIOptions } from './api_types'; +import { pollEsNodesVersion } from './version_check/ensure_es_version'; /** @internal */ interface CoreClusterClients { @@ -46,9 +47,17 @@ interface SetupDeps { export class ElasticsearchService implements CoreService { private readonly log: Logger; private readonly config$: Observable; - private subscription?: Subscription; + private subscriptions: { + client?: Subscription; + esNodesCompatibility?: Subscription; + } = { + client: undefined, + esNodesCompatibility: undefined, + }; + private kibanaVersion: string; constructor(private readonly coreContext: CoreContext) { + this.kibanaVersion = coreContext.env.packageInfo.version; this.log = coreContext.logger.get('elasticsearch-service'); this.config$ = coreContext.configService .atPath('elasticsearch') @@ -60,7 +69,7 @@ export class ElasticsearchService implements CoreService { - if (this.subscription !== undefined) { + if (this.subscriptions.client !== undefined) { this.log.error('Clients cannot be changed after they are created'); return false; } @@ -91,7 +100,7 @@ export class ElasticsearchService implements CoreService; - this.subscription = clients$.connect(); + this.subscriptions.client = clients$.connect(); const config = await this.config$.pipe(first()).toPromise(); @@ -149,11 +158,31 @@ export class ElasticsearchService implements CoreService).connect(); + + // TODO: Move to Status Service https://github.com/elastic/kibana/issues/41983 + esNodesCompatibility$.subscribe(({ isCompatible, message }) => { + if (!isCompatible && message) { + this.log.error(message); + } + }); + return { legacy: { config$: clients$.pipe(map(clients => clients.config)) }, adminClient, dataClient, + esNodesCompatibility$, createClient: (type: string, clientConfig: Partial = {}) => { const finalConfig = merge({}, config, clientConfig); @@ -166,11 +195,12 @@ export class ElasticsearchService implements CoreService; }; + esNodesCompatibility$: Observable; } diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts new file mode 100644 index 0000000000000..4989c4a31295c --- /dev/null +++ b/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts @@ -0,0 +1,261 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { mapNodesVersionCompatibility, pollEsNodesVersion, NodesInfo } from './ensure_es_version'; +import { loggingServiceMock } from '../../logging/logging_service.mock'; +import { take, delay } from 'rxjs/operators'; +import { TestScheduler } from 'rxjs/testing'; +import { of } from 'rxjs'; + +const mockLoggerFactory = loggingServiceMock.create(); +const mockLogger = mockLoggerFactory.get('mock logger'); + +const KIBANA_VERSION = '5.1.0'; + +function createNodes(...versions: string[]): NodesInfo { + const nodes = {} as any; + versions + .map(version => { + return { + version, + http: { + publish_address: 'http_address', + }, + ip: 'ip', + }; + }) + .forEach((node, i) => { + nodes[`node-${i}`] = node; + }); + + return { nodes }; +} + +describe('mapNodesVersionCompatibility', () => { + function createNodesInfoWithoutHTTP(version: string): NodesInfo { + return { nodes: { 'node-without-http': { version, ip: 'ip' } } } as any; + } + + it('returns isCompatible=true with a single node that matches', async () => { + const nodesInfo = createNodes('5.1.0'); + const result = await mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, false); + expect(result.isCompatible).toBe(true); + }); + + it('returns isCompatible=true with multiple nodes that satisfy', async () => { + const nodesInfo = createNodes('5.1.0', '5.2.0', '5.1.1-Beta1'); + const result = await mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, false); + expect(result.isCompatible).toBe(true); + }); + + it('returns isCompatible=false for a single node that is out of date', () => { + // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana. + const nodesInfo = createNodes('5.1.0', '5.2.0', '5.0.0'); + const result = mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, false); + expect(result.isCompatible).toBe(false); + expect(result.message).toMatchInlineSnapshot( + `"This version of Kibana (v5.1.0) is incompatible with the following Elasticsearch nodes in your cluster: v5.0.0 @ http_address (ip)"` + ); + }); + + it('returns isCompatible=false for an incompatible node without http publish address', async () => { + const nodesInfo = createNodesInfoWithoutHTTP('6.1.1'); + const result = mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, false); + expect(result.isCompatible).toBe(false); + expect(result.message).toMatchInlineSnapshot( + `"This version of Kibana (v5.1.0) is incompatible with the following Elasticsearch nodes in your cluster: v6.1.1 @ undefined (ip)"` + ); + }); + + it('returns isCompatible=true for outdated nodes when ignoreVersionMismatch=true', async () => { + // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana. + const nodesInfo = createNodes('5.1.0', '5.2.0', '5.0.0'); + const ignoreVersionMismatch = true; + const result = mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, ignoreVersionMismatch); + expect(result.isCompatible).toBe(true); + expect(result.message).toMatchInlineSnapshot( + `"Ignoring version incompatibility between Kibana v5.1.0 and the following Elasticsearch nodes: v5.0.0 @ http_address (ip)"` + ); + }); + + it('returns isCompatible=true with a message if a node is only off by a patch version', () => { + const result = mapNodesVersionCompatibility(createNodes('5.1.1'), KIBANA_VERSION, false); + expect(result.isCompatible).toBe(true); + expect(result.message).toMatchInlineSnapshot( + `"You're running Kibana 5.1.0 with some different versions of Elasticsearch. Update Kibana or Elasticsearch to the same version to prevent compatibility issues: v5.1.1 @ http_address (ip)"` + ); + }); + + it('returns isCompatible=true with a message if a node is only off by a patch version and without http publish address', async () => { + const result = mapNodesVersionCompatibility(createNodes('5.1.1'), KIBANA_VERSION, false); + expect(result.isCompatible).toBe(true); + expect(result.message).toMatchInlineSnapshot( + `"You're running Kibana 5.1.0 with some different versions of Elasticsearch. Update Kibana or Elasticsearch to the same version to prevent compatibility issues: v5.1.1 @ http_address (ip)"` + ); + }); +}); + +describe('pollEsNodesVersion', () => { + const callWithInternalUser = jest.fn(); + const getTestScheduler = () => + new TestScheduler((actual, expected) => { + expect(actual).toEqual(expected); + }); + + beforeEach(() => { + callWithInternalUser.mockClear(); + }); + + it('returns iscCompatible=false and keeps polling when a poll request throws', done => { + expect.assertions(3); + const expectedCompatibilityResults = [false, false, true]; + jest.clearAllMocks(); + callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.0.0')); + callWithInternalUser.mockRejectedValueOnce(new Error('mock request error')); + callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.1.1-Beta1')); + pollEsNodesVersion({ + callWithInternalUser, + esVersionCheckInterval: 1, + ignoreVersionMismatch: false, + kibanaVersion: KIBANA_VERSION, + log: mockLogger, + }) + .pipe(take(3)) + .subscribe({ + next: result => { + expect(result.isCompatible).toBe(expectedCompatibilityResults.shift()); + }, + complete: done, + error: done, + }); + }); + + it('returns compatibility results', done => { + expect.assertions(1); + const nodes = createNodes('5.1.0', '5.2.0', '5.0.0'); + callWithInternalUser.mockResolvedValueOnce(nodes); + pollEsNodesVersion({ + callWithInternalUser, + esVersionCheckInterval: 1, + ignoreVersionMismatch: false, + kibanaVersion: KIBANA_VERSION, + log: mockLogger, + }) + .pipe(take(1)) + .subscribe({ + next: result => { + expect(result).toEqual(mapNodesVersionCompatibility(nodes, KIBANA_VERSION, false)); + }, + complete: done, + error: done, + }); + }); + + it('only emits if the node versions changed since the previous poll', done => { + expect.assertions(4); + callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.0.0')); // emit + callWithInternalUser.mockResolvedValueOnce(createNodes('5.0.0', '5.1.0', '5.2.0')); // ignore, same versions, different ordering + callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.1', '5.2.0', '5.0.0')); // emit + callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.1', '5.1.2', '5.1.3')); // emit + callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.1', '5.1.2', '5.1.3')); // ignore + callWithInternalUser.mockResolvedValueOnce(createNodes('5.0.0', '5.1.0', '5.2.0')); // emit, different from previous version + + pollEsNodesVersion({ + callWithInternalUser, + esVersionCheckInterval: 1, + ignoreVersionMismatch: false, + kibanaVersion: KIBANA_VERSION, + log: mockLogger, + }) + .pipe(take(4)) + .subscribe({ + next: result => expect(result).toBeDefined(), + complete: done, + error: done, + }); + }); + + it('starts polling immediately and then every esVersionCheckInterval', () => { + expect.assertions(1); + callWithInternalUser.mockReturnValueOnce([createNodes('5.1.0', '5.2.0', '5.0.0')]); + callWithInternalUser.mockReturnValueOnce([createNodes('5.1.1', '5.2.0', '5.0.0')]); + + getTestScheduler().run(({ expectObservable }) => { + const expected = 'a 99ms (b|)'; + + const esNodesCompatibility$ = pollEsNodesVersion({ + callWithInternalUser, + esVersionCheckInterval: 100, + ignoreVersionMismatch: false, + kibanaVersion: KIBANA_VERSION, + log: mockLogger, + }).pipe(take(2)); + + expectObservable(esNodesCompatibility$).toBe(expected, { + a: mapNodesVersionCompatibility( + createNodes('5.1.0', '5.2.0', '5.0.0'), + KIBANA_VERSION, + false + ), + b: mapNodesVersionCompatibility( + createNodes('5.1.1', '5.2.0', '5.0.0'), + KIBANA_VERSION, + false + ), + }); + }); + }); + + it('waits for es version check requests to complete before scheduling the next one', () => { + expect.assertions(2); + + getTestScheduler().run(({ expectObservable }) => { + const expected = '100ms a 99ms (b|)'; + + callWithInternalUser.mockReturnValueOnce( + of(createNodes('5.1.0', '5.2.0', '5.0.0')).pipe(delay(100)) + ); + callWithInternalUser.mockReturnValueOnce( + of(createNodes('5.1.1', '5.2.0', '5.0.0')).pipe(delay(100)) + ); + + const esNodesCompatibility$ = pollEsNodesVersion({ + callWithInternalUser, + esVersionCheckInterval: 10, + ignoreVersionMismatch: false, + kibanaVersion: KIBANA_VERSION, + log: mockLogger, + }).pipe(take(2)); + + expectObservable(esNodesCompatibility$).toBe(expected, { + a: mapNodesVersionCompatibility( + createNodes('5.1.0', '5.2.0', '5.0.0'), + KIBANA_VERSION, + false + ), + b: mapNodesVersionCompatibility( + createNodes('5.1.1', '5.2.0', '5.0.0'), + KIBANA_VERSION, + false + ), + }); + }); + + expect(callWithInternalUser).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.ts new file mode 100644 index 0000000000000..3e760ec0efabd --- /dev/null +++ b/src/core/server/elasticsearch/version_check/ensure_es_version.ts @@ -0,0 +1,164 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * ES and Kibana versions are locked, so Kibana should require that ES has the same version as + * that defined in Kibana's package.json. + */ + +import { timer, of, from, Observable } from 'rxjs'; +import { map, distinctUntilChanged, catchError, exhaustMap } from 'rxjs/operators'; +import { + esVersionCompatibleWithKibana, + esVersionEqualsKibana, +} from './es_kibana_version_compatability'; +import { Logger } from '../../logging'; +import { APICaller } from '..'; + +export interface PollEsNodesVersionOptions { + callWithInternalUser: APICaller; + log: Logger; + kibanaVersion: string; + ignoreVersionMismatch: boolean; + esVersionCheckInterval: number; +} + +interface NodeInfo { + version: string; + ip: string; + http: { + publish_address: string; + }; + name: string; +} + +export interface NodesInfo { + nodes: { + [key: string]: NodeInfo; + }; +} + +export interface NodesVersionCompatibility { + isCompatible: boolean; + message?: string; + incompatibleNodes: NodeInfo[]; + warningNodes: NodeInfo[]; + kibanaVersion: string; +} + +function getHumanizedNodeName(node: NodeInfo) { + const publishAddress = node?.http?.publish_address + ' ' || ''; + return 'v' + node.version + ' @ ' + publishAddress + '(' + node.ip + ')'; +} + +export function mapNodesVersionCompatibility( + nodesInfo: NodesInfo, + kibanaVersion: string, + ignoreVersionMismatch: boolean +): NodesVersionCompatibility { + if (Object.keys(nodesInfo.nodes).length === 0) { + return { + isCompatible: false, + message: 'Unable to retrieve version information from Elasticsearch nodes.', + incompatibleNodes: [], + warningNodes: [], + kibanaVersion, + }; + } + const nodes = Object.keys(nodesInfo.nodes) + .sort() // Sorting ensures a stable node ordering for comparison + .map(key => nodesInfo.nodes[key]) + .map(node => Object.assign({}, node, { name: getHumanizedNodeName(node) })); + + // Aggregate incompatible ES nodes. + const incompatibleNodes = nodes.filter( + node => !esVersionCompatibleWithKibana(node.version, kibanaVersion) + ); + + // Aggregate ES nodes which should prompt a Kibana upgrade. It's acceptable + // if ES and Kibana versions are not the same as long as they are not + // incompatible, but we should warn about it. + // Ignore version qualifiers https://github.com/elastic/elasticsearch/issues/36859 + const warningNodes = nodes.filter(node => !esVersionEqualsKibana(node.version, kibanaVersion)); + + // Note: If incompatible and warning nodes are present `message` only contains + // an incompatibility notice. + let message; + if (incompatibleNodes.length > 0) { + const incompatibleNodeNames = incompatibleNodes.map(node => node.name).join(', '); + if (ignoreVersionMismatch) { + message = `Ignoring version incompatibility between Kibana v${kibanaVersion} and the following Elasticsearch nodes: ${incompatibleNodeNames}`; + } else { + message = `This version of Kibana (v${kibanaVersion}) is incompatible with the following Elasticsearch nodes in your cluster: ${incompatibleNodeNames}`; + } + } else if (warningNodes.length > 0) { + const warningNodeNames = warningNodes.map(node => node.name).join(', '); + message = + `You're running Kibana ${kibanaVersion} with some different versions of ` + + 'Elasticsearch. Update Kibana or Elasticsearch to the same ' + + `version to prevent compatibility issues: ${warningNodeNames}`; + } + + return { + isCompatible: ignoreVersionMismatch || incompatibleNodes.length === 0, + message, + incompatibleNodes, + warningNodes, + kibanaVersion, + }; +} + +// Returns true if two NodesVersionCompatibility entries match +function compareNodes(prev: NodesVersionCompatibility, curr: NodesVersionCompatibility) { + const nodesEqual = (n: NodeInfo, m: NodeInfo) => n.ip === m.ip && n.version === m.version; + return ( + curr.isCompatible === prev.isCompatible && + curr.incompatibleNodes.length === prev.incompatibleNodes.length && + curr.warningNodes.length === prev.warningNodes.length && + curr.incompatibleNodes.every((node, i) => nodesEqual(node, prev.incompatibleNodes[i])) && + curr.warningNodes.every((node, i) => nodesEqual(node, prev.warningNodes[i])) + ); +} + +export const pollEsNodesVersion = ({ + callWithInternalUser, + log, + kibanaVersion, + ignoreVersionMismatch, + esVersionCheckInterval: healthCheckInterval, +}: PollEsNodesVersionOptions): Observable => { + log.debug('Checking Elasticsearch version'); + return timer(0, healthCheckInterval).pipe( + exhaustMap(() => { + return from( + callWithInternalUser('nodes.info', { + filterPath: ['nodes.*.version', 'nodes.*.http.publish_address', 'nodes.*.ip'], + }) + ).pipe( + catchError(_err => { + return of({ nodes: {} }); + }) + ); + }), + map((nodesInfo: NodesInfo) => + mapNodesVersionCompatibility(nodesInfo, kibanaVersion, ignoreVersionMismatch) + ), + distinctUntilChanged(compareNodes) // Only emit if there are new nodes or versions + ); +}; diff --git a/src/legacy/core_plugins/elasticsearch/lib/__tests__/is_es_compatible_with_kibana.js b/src/core/server/elasticsearch/version_check/es_kibana_version_compatability.test.ts similarity index 72% rename from src/legacy/core_plugins/elasticsearch/lib/__tests__/is_es_compatible_with_kibana.js rename to src/core/server/elasticsearch/version_check/es_kibana_version_compatability.test.ts index 092c0ecf1071c..152f25c813881 100644 --- a/src/legacy/core_plugins/elasticsearch/lib/__tests__/is_es_compatible_with_kibana.js +++ b/src/core/server/elasticsearch/version_check/es_kibana_version_compatability.test.ts @@ -17,41 +17,39 @@ * under the License. */ -import expect from '@kbn/expect'; - -import isEsCompatibleWithKibana from '../is_es_compatible_with_kibana'; +import { esVersionCompatibleWithKibana } from './es_kibana_version_compatability'; describe('plugins/elasticsearch', () => { describe('lib/is_es_compatible_with_kibana', () => { describe('returns false', () => { it('when ES major is greater than Kibana major', () => { - expect(isEsCompatibleWithKibana('1.0.0', '0.0.0')).to.be(false); + expect(esVersionCompatibleWithKibana('1.0.0', '0.0.0')).toBe(false); }); it('when ES major is less than Kibana major', () => { - expect(isEsCompatibleWithKibana('0.0.0', '1.0.0')).to.be(false); + expect(esVersionCompatibleWithKibana('0.0.0', '1.0.0')).toBe(false); }); it('when majors are equal, but ES minor is less than Kibana minor', () => { - expect(isEsCompatibleWithKibana('1.0.0', '1.1.0')).to.be(false); + expect(esVersionCompatibleWithKibana('1.0.0', '1.1.0')).toBe(false); }); }); describe('returns true', () => { it('when version numbers are the same', () => { - expect(isEsCompatibleWithKibana('1.1.1', '1.1.1')).to.be(true); + expect(esVersionCompatibleWithKibana('1.1.1', '1.1.1')).toBe(true); }); it('when majors are equal, and ES minor is greater than Kibana minor', () => { - expect(isEsCompatibleWithKibana('1.1.0', '1.0.0')).to.be(true); + expect(esVersionCompatibleWithKibana('1.1.0', '1.0.0')).toBe(true); }); it('when majors and minors are equal, and ES patch is greater than Kibana patch', () => { - expect(isEsCompatibleWithKibana('1.1.1', '1.1.0')).to.be(true); + expect(esVersionCompatibleWithKibana('1.1.1', '1.1.0')).toBe(true); }); it('when majors and minors are equal, but ES patch is less than Kibana patch', () => { - expect(isEsCompatibleWithKibana('1.1.0', '1.1.1')).to.be(true); + expect(esVersionCompatibleWithKibana('1.1.0', '1.1.1')).toBe(true); }); }); }); diff --git a/src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.js b/src/core/server/elasticsearch/version_check/es_kibana_version_compatability.ts similarity index 76% rename from src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.js rename to src/core/server/elasticsearch/version_check/es_kibana_version_compatability.ts index 4afbd488d2946..28b9c0a23e672 100644 --- a/src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.js +++ b/src/core/server/elasticsearch/version_check/es_kibana_version_compatability.ts @@ -17,15 +17,14 @@ * under the License. */ +import semver, { coerce } from 'semver'; + /** - * Let's weed out the ES versions that won't work with a given Kibana version. + * Checks for the compatibilitiy between Elasticsearch and Kibana versions * 1. Major version differences will never work together. * 2. Older versions of ES won't work with newer versions of Kibana. */ - -import semver from 'semver'; - -export default function isEsCompatibleWithKibana(esVersion, kibanaVersion) { +export function esVersionCompatibleWithKibana(esVersion: string, kibanaVersion: string) { const esVersionNumbers = { major: semver.major(esVersion), minor: semver.minor(esVersion), @@ -50,3 +49,9 @@ export default function isEsCompatibleWithKibana(esVersion, kibanaVersion) { return true; } + +export function esVersionEqualsKibana(nodeVersion: string, kibanaVersion: string) { + const nodeSemVer = coerce(nodeVersion); + const kibanaSemver = coerce(kibanaVersion); + return nodeSemVer && kibanaSemver && nodeSemVer.version === kibanaSemver.version; +} diff --git a/src/core/server/http/integration_tests/core_services.test.ts b/src/core/server/http/integration_tests/core_services.test.ts index 65c4f1432721d..0d4570a34ca3c 100644 --- a/src/core/server/http/integration_tests/core_services.test.ts +++ b/src/core/server/http/integration_tests/core_services.test.ts @@ -46,7 +46,7 @@ describe('http service', () => { let root: ReturnType; beforeEach(async () => { - root = kbnTestServer.createRoot(); + root = kbnTestServer.createRoot({ migrations: { skip: true } }); }, 30000); afterEach(async () => { @@ -180,7 +180,7 @@ describe('http service', () => { describe('#basePath()', () => { let root: ReturnType; beforeEach(async () => { - root = kbnTestServer.createRoot(); + root = kbnTestServer.createRoot({ migrations: { skip: true } }); }, 30000); afterEach(async () => await root.shutdown()); @@ -209,7 +209,7 @@ describe('http service', () => { describe('elasticsearch', () => { let root: ReturnType; beforeEach(async () => { - root = kbnTestServer.createRoot(); + root = kbnTestServer.createRoot({ migrations: { skip: true } }); }, 30000); afterEach(async () => { diff --git a/src/core/server/legacy/integration_tests/legacy_service.test.ts b/src/core/server/legacy/integration_tests/legacy_service.test.ts index da2550f2ae799..e8bcf7a42d192 100644 --- a/src/core/server/legacy/integration_tests/legacy_service.test.ts +++ b/src/core/server/legacy/integration_tests/legacy_service.test.ts @@ -22,7 +22,7 @@ describe('legacy service', () => { describe('http server', () => { let root: ReturnType; beforeEach(() => { - root = kbnTestServer.createRoot(); + root = kbnTestServer.createRoot({ migrations: { skip: true } }); }, 30000); afterEach(async () => await root.shutdown()); diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts index b89abc596ad18..c6a72eb53d6c4 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts @@ -59,12 +59,6 @@ describe('KibanaMigrator', () => { }); describe('runMigrations', () => { - it('resolves isMigrated if migrations were skipped', async () => { - const skipMigrations = true; - const result = await new KibanaMigrator(mockOptions()).runMigrations(skipMigrations); - expect(result).toEqual([{ status: 'skipped' }, { status: 'skipped' }]); - }); - it('only runs migrations once if called multiple times', async () => { const options = mockOptions(); const clusterStub = jest.fn(() => ({ status: 404 })); diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts index c35e8dd90b5b1..747b48a540109 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts @@ -107,24 +107,15 @@ export class KibanaMigrator { * The promise resolves with an array of migration statuses, one for each * elasticsearch index which was migrated. */ - public runMigrations(skipMigrations: boolean = false): Promise> { + public runMigrations(): Promise> { if (this.migrationResult === undefined) { - this.migrationResult = this.runMigrationsInternal(skipMigrations); + this.migrationResult = this.runMigrationsInternal(); } return this.migrationResult; } - private runMigrationsInternal(skipMigrations: boolean) { - if (skipMigrations) { - this.log.warn( - 'Skipping Saved Object migrations on startup. Note: Individual documents will still be migrated when read or written.' - ); - return Promise.resolve( - Object.keys(this.mappingProperties).map(() => ({ status: 'skipped' })) - ); - } - + private runMigrationsInternal() { const kibanaIndexName = this.kibanaConfig.index; const indexMap = createIndexMap({ config: this.config, diff --git a/src/core/server/saved_objects/saved_objects_service.test.ts b/src/core/server/saved_objects/saved_objects_service.test.ts index deb6f98e17b7b..2bd1335e4d031 100644 --- a/src/core/server/saved_objects/saved_objects_service.test.ts +++ b/src/core/server/saved_objects/saved_objects_service.test.ts @@ -26,12 +26,24 @@ import { KibanaMigrator, mockKibanaMigratorInstance } from './migrations/kibana/ import * as legacyElasticsearch from 'elasticsearch'; import { Env } from '../config'; import { configServiceMock } from '../mocks'; - -afterEach(() => { - jest.clearAllMocks(); -}); +import { elasticsearchServiceMock } from '../elasticsearch/elasticsearch_service.mock'; +import { legacyServiceMock } from '../legacy/legacy_service.mock'; +import { BehaviorSubject } from 'rxjs'; +import { NodesVersionCompatibility } from '../elasticsearch/version_check/ensure_es_version'; describe('SavedObjectsService', () => { + const createSetupDeps = () => { + const elasticsearchMock = elasticsearchServiceMock.createInternalSetup(); + return { + elasticsearch: elasticsearchMock, + legacyPlugins: legacyServiceMock.createDiscoverPlugins(), + }; + }; + + afterEach(() => { + jest.clearAllMocks(); + }); + describe('#setup()', () => { it('creates a KibanaMigrator which retries NoConnections errors from callAsInternalUser', async () => { const coreContext = mockCoreContext.create(); @@ -73,7 +85,7 @@ describe('SavedObjectsService', () => { await soService.setup(coreSetup); await soService.start({}); - expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledWith(true); + expect(mockKibanaMigratorInstance.runMigrations).not.toHaveBeenCalled(); }); it('skips KibanaMigrator migrations when migrations.skip=true', async () => { @@ -87,24 +99,51 @@ describe('SavedObjectsService', () => { await soService.setup(coreSetup); await soService.start({}); - expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledWith(true); + expect(mockKibanaMigratorInstance.runMigrations).not.toHaveBeenCalled(); + }); + + it('waits for all es nodes to be compatible before running migrations', async done => { + expect.assertions(2); + const configService = configServiceMock.create({ atPath: { skip: false } }); + const coreContext = mockCoreContext.create({ configService }); + const soService = new SavedObjectsService(coreContext); + const setupDeps = createSetupDeps(); + // Create an new subject so that we can control when isCompatible=true + // is emitted. + setupDeps.elasticsearch.esNodesCompatibility$ = new BehaviorSubject({ + isCompatible: false, + incompatibleNodes: [], + warningNodes: [], + kibanaVersion: '8.0.0', + }); + await soService.setup(setupDeps); + soService.start({}); + expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledTimes(0); + ((setupDeps.elasticsearch.esNodesCompatibility$ as any) as BehaviorSubject< + NodesVersionCompatibility + >).next({ + isCompatible: true, + incompatibleNodes: [], + warningNodes: [], + kibanaVersion: '8.0.0', + }); + setImmediate(() => { + expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledTimes(1); + done(); + }); }); it('resolves with KibanaMigrator after waiting for migrations to complete', async () => { const configService = configServiceMock.create({ atPath: { skip: false } }); const coreContext = mockCoreContext.create({ configService }); const soService = new SavedObjectsService(coreContext); - const coreSetup = ({ - elasticsearch: { adminClient: { callAsInternalUser: jest.fn() } }, - legacyPlugins: { uiExports: {}, pluginExtendedConfig: {} }, - } as unknown) as SavedObjectsSetupDeps; + const coreSetup = createSetupDeps(); await soService.setup(coreSetup); expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledTimes(0); const startContract = await soService.start({}); expect(startContract.migrator).toBe(mockKibanaMigratorInstance); - expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledWith(false); expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledTimes(1); }); }); diff --git a/src/core/server/saved_objects/saved_objects_service.ts b/src/core/server/saved_objects/saved_objects_service.ts index 69ca8306ca4da..03b166c705126 100644 --- a/src/core/server/saved_objects/saved_objects_service.ts +++ b/src/core/server/saved_objects/saved_objects_service.ts @@ -18,7 +18,7 @@ */ import { CoreService } from 'src/core/types'; -import { first } from 'rxjs/operators'; +import { first, filter, take } from 'rxjs/operators'; import { SavedObjectsClient, SavedObjectsSchema, @@ -186,6 +186,7 @@ export class SavedObjectsService private migrator: KibanaMigrator | undefined; private logger: Logger; private clientProvider: ISavedObjectsClientProvider | undefined; + private setupDeps: SavedObjectsSetupDeps | undefined; constructor(private readonly coreContext: CoreContext) { this.logger = coreContext.logger.get('savedobjects-service'); @@ -197,6 +198,8 @@ export class SavedObjectsService ): Promise { this.logger.debug('Setting up SavedObjects service'); + this.setupDeps = setupDeps; + const { savedObjectSchemas: savedObjectsSchemasDefinition, savedObjectMappings, @@ -223,14 +226,14 @@ export class SavedObjectsService savedObjectMappings, savedObjectMigrations, savedObjectValidations, - logger: this.coreContext.logger.get('migrations'), + logger: this.logger, kibanaVersion: this.coreContext.env.packageInfo.version, config: setupDeps.legacyPlugins.pluginExtendedConfig, savedObjectsConfig, kibanaConfig, callCluster: migrationsRetryCallCluster( adminClient.callAsInternalUser, - this.coreContext.logger.get('migrations'), + this.logger, migrationsRetryDelay ), })); @@ -287,7 +290,23 @@ export class SavedObjectsService .pipe(first()) .toPromise(); const skipMigrations = cliArgs.optimize || savedObjectsConfig.skip; - await this.migrator!.runMigrations(skipMigrations); + + if (skipMigrations) { + this.logger.warn( + 'Skipping Saved Object migrations on startup. Note: Individual documents will still be migrated when read or written.' + ); + } else { + this.logger.info( + 'Waiting until all Elasticsearch nodes are compatible with Kibana before starting saved objects migrations...' + ); + await this.setupDeps!.elasticsearch.esNodesCompatibility$.pipe( + filter(nodes => nodes.isCompatible), + take(1) + ).toPromise(); + + this.logger.info('Starting saved objects migrations'); + await this.migrator!.runMigrations(); + } return { migrator: this.migrator!, diff --git a/src/legacy/core_plugins/elasticsearch/index.d.ts b/src/legacy/core_plugins/elasticsearch/index.d.ts index 4cbb1c82cc1e4..df713160137a6 100644 --- a/src/legacy/core_plugins/elasticsearch/index.d.ts +++ b/src/legacy/core_plugins/elasticsearch/index.d.ts @@ -523,6 +523,7 @@ export interface CallCluster { } export interface ElasticsearchPlugin { + status: { on: (status: string, cb: () => void) => void }; getCluster(name: string): Cluster; createCluster(name: string, config: ClusterConfig): Cluster; waitUntilReady(): Promise; diff --git a/src/legacy/core_plugins/elasticsearch/index.js b/src/legacy/core_plugins/elasticsearch/index.js index da7b557e7ea19..55e81e4144bb5 100644 --- a/src/legacy/core_plugins/elasticsearch/index.js +++ b/src/legacy/core_plugins/elasticsearch/index.js @@ -17,10 +17,10 @@ * under the License. */ import { first } from 'rxjs/operators'; -import healthCheck from './lib/health_check'; import { Cluster } from './lib/cluster'; import { createProxy } from './lib/create_proxy'; import { handleESError } from './lib/handle_es_error'; +import { versionHealthCheck } from './lib/version_health_check'; export default function(kibana) { let defaultVars; @@ -92,15 +92,13 @@ export default function(kibana) { createProxy(server); - // Set up the health check service and start it. - const { start, waitUntilReady } = healthCheck( + const waitUntilHealthy = versionHealthCheck( this, - server, - esConfig.healthCheckDelay.asMilliseconds(), - esConfig.ignoreVersionMismatch + server.logWithMetadata, + server.newPlatform.__internals.elasticsearch.esNodesCompatibility$ ); - server.expose('waitUntilReady', waitUntilReady); - start(); + + server.expose('waitUntilReady', () => waitUntilHealthy); }, }); } diff --git a/src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts b/src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts new file mode 100644 index 0000000000000..5806c31b78414 --- /dev/null +++ b/src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + createTestServers, + TestElasticsearchUtils, + TestKibanaUtils, + TestUtils, + createRootWithCorePlugins, + getKbnServer, +} from '../../../../test_utils/kbn_server'; + +import { BehaviorSubject } from 'rxjs'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { NodesVersionCompatibility } from 'src/core/server/elasticsearch/version_check/ensure_es_version'; + +describe('Elasticsearch plugin', () => { + let servers: TestUtils; + let esServer: TestElasticsearchUtils; + let root: TestKibanaUtils['root']; + let elasticsearch: TestKibanaUtils['kbnServer']['server']['plugins']['elasticsearch']; + + const esNodesCompatibility$ = new BehaviorSubject({ + isCompatible: true, + incompatibleNodes: [], + warningNodes: [], + kibanaVersion: '8.0.0', + }); + + beforeAll(async function() { + const settings = { + elasticsearch: {}, + adjustTimeout: (t: any) => { + jest.setTimeout(t); + }, + }; + servers = createTestServers(settings); + esServer = await servers.startES(); + + const elasticsearchSettings = { + hosts: esServer.hosts, + username: esServer.username, + password: esServer.password, + }; + root = createRootWithCorePlugins({ elasticsearch: elasticsearchSettings }); + + const setup = await root.setup(); + setup.elasticsearch.esNodesCompatibility$ = esNodesCompatibility$; + await root.start(); + + elasticsearch = getKbnServer(root).server.plugins.elasticsearch; + }); + + afterAll(async () => { + await esServer.stop(); + await root.shutdown(); + }, 30000); + + it("should set it's status to green when all nodes are compatible", done => { + jest.setTimeout(30000); + elasticsearch.status.on('green', () => done()); + }); + + it("should set it's status to red when some nodes aren't compatible", done => { + esNodesCompatibility$.next({ + isCompatible: false, + incompatibleNodes: [], + warningNodes: [], + kibanaVersion: '8.0.0', + }); + elasticsearch.status.on('red', () => done()); + }); +}); diff --git a/src/legacy/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js b/src/legacy/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js deleted file mode 100644 index 781d198c66236..0000000000000 --- a/src/legacy/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import _ from 'lodash'; -import Bluebird from 'bluebird'; -import sinon from 'sinon'; -import expect from '@kbn/expect'; - -import { esTestConfig } from '@kbn/test'; -import { ensureEsVersion } from '../ensure_es_version'; - -describe('plugins/elasticsearch', () => { - describe('lib/ensure_es_version', () => { - const KIBANA_VERSION = '5.1.0'; - - let server; - - beforeEach(function() { - server = { - log: sinon.stub(), - logWithMetadata: sinon.stub(), - plugins: { - elasticsearch: { - getCluster: sinon - .stub() - .withArgs('admin') - .returns({ callWithInternalUser: sinon.stub() }), - status: { - red: sinon.stub(), - }, - url: esTestConfig.getUrl(), - }, - }, - config() { - return { - get: sinon.stub(), - }; - }, - }; - }); - - function setNodes(/* ...versions */) { - const versions = _.shuffle(arguments); - const nodes = {}; - let i = 0; - - while (versions.length) { - const name = 'node-' + ++i; - const version = versions.shift(); - - const node = { - version: version, - http: { - publish_address: 'http_address', - }, - ip: 'ip', - }; - - if (!_.isString(version)) _.assign(node, version); - nodes[name] = node; - } - - const cluster = server.plugins.elasticsearch.getCluster('admin'); - cluster.callWithInternalUser - .withArgs('nodes.info', sinon.match.any) - .returns(Bluebird.resolve({ nodes: nodes })); - } - - function setNodeWithoutHTTP(version) { - const nodes = { 'node-without-http': { version, ip: 'ip' } }; - const cluster = server.plugins.elasticsearch.getCluster('admin'); - cluster.callWithInternalUser - .withArgs('nodes.info', sinon.match.any) - .returns(Bluebird.resolve({ nodes: nodes })); - } - - it('returns true with single a node that matches', async () => { - setNodes('5.1.0'); - const result = await ensureEsVersion(server, KIBANA_VERSION); - expect(result).to.be(true); - }); - - it('returns true with multiple nodes that satisfy', async () => { - setNodes('5.1.0', '5.2.0', '5.1.1-Beta1'); - const result = await ensureEsVersion(server, KIBANA_VERSION); - expect(result).to.be(true); - }); - - it('throws an error with a single node that is out of date', async () => { - // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana. - setNodes('5.1.0', '5.2.0', '5.0.0'); - try { - await ensureEsVersion(server, KIBANA_VERSION); - } catch (e) { - expect(e).to.be.a(Error); - } - }); - - it('does not throw on outdated nodes, if `ignoreVersionMismatch` is enabled in development mode', async () => { - // set config values - server.config = () => ({ - get: name => { - switch (name) { - case 'env.dev': - return true; - default: - throw new Error(`Unknown option "${name}"`); - } - }, - }); - - // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana. - setNodes('5.1.0', '5.2.0', '5.0.0'); - - const ignoreVersionMismatch = true; - const result = await ensureEsVersion(server, KIBANA_VERSION, ignoreVersionMismatch); - expect(result).to.be(true); - }); - - it('throws an error if `ignoreVersionMismatch` is enabled in production mode', async () => { - // set config values - server.config = () => ({ - get: name => { - switch (name) { - case 'env.dev': - return false; - default: - throw new Error(`Unknown option "${name}"`); - } - }, - }); - - // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana. - setNodes('5.1.0', '5.2.0', '5.0.0'); - - try { - const ignoreVersionMismatch = true; - await ensureEsVersion(server, KIBANA_VERSION, ignoreVersionMismatch); - } catch (e) { - expect(e).to.be.a(Error); - } - }); - - it('fails if that single node is a client node', async () => { - setNodes('5.1.0', '5.2.0', { version: '5.0.0', attributes: { client: 'true' } }); - try { - await ensureEsVersion(server, KIBANA_VERSION); - } catch (e) { - expect(e).to.be.a(Error); - } - }); - - it('warns if a node is only off by a patch version', async () => { - setNodes('5.1.1'); - await ensureEsVersion(server, KIBANA_VERSION); - sinon.assert.callCount(server.logWithMetadata, 2); - expect(server.logWithMetadata.getCall(0).args[0]).to.contain('debug'); - expect(server.logWithMetadata.getCall(1).args[0]).to.contain('warning'); - }); - - it('warns if a node is off by a patch version and without http publish address', async () => { - setNodeWithoutHTTP('5.1.1'); - await ensureEsVersion(server, KIBANA_VERSION); - sinon.assert.callCount(server.logWithMetadata, 2); - expect(server.logWithMetadata.getCall(0).args[0]).to.contain('debug'); - expect(server.logWithMetadata.getCall(1).args[0]).to.contain('warning'); - }); - - it('errors if a node incompatible and without http publish address', async () => { - setNodeWithoutHTTP('6.1.1'); - try { - await ensureEsVersion(server, KIBANA_VERSION); - } catch (e) { - expect(e.message).to.contain('incompatible nodes'); - expect(e).to.be.a(Error); - } - }); - - it('only warns once per node list', async () => { - setNodes('5.1.1'); - - await ensureEsVersion(server, KIBANA_VERSION); - sinon.assert.callCount(server.logWithMetadata, 2); - expect(server.logWithMetadata.getCall(0).args[0]).to.contain('debug'); - expect(server.logWithMetadata.getCall(1).args[0]).to.contain('warning'); - - await ensureEsVersion(server, KIBANA_VERSION); - sinon.assert.callCount(server.logWithMetadata, 3); - expect(server.logWithMetadata.getCall(2).args[0]).to.contain('debug'); - }); - - it('warns again if the node list changes', async () => { - setNodes('5.1.1'); - - await ensureEsVersion(server, KIBANA_VERSION); - sinon.assert.callCount(server.logWithMetadata, 2); - expect(server.logWithMetadata.getCall(0).args[0]).to.contain('debug'); - expect(server.logWithMetadata.getCall(1).args[0]).to.contain('warning'); - - setNodes('5.1.2'); - await ensureEsVersion(server, KIBANA_VERSION); - sinon.assert.callCount(server.logWithMetadata, 4); - expect(server.logWithMetadata.getCall(2).args[0]).to.contain('debug'); - expect(server.logWithMetadata.getCall(3).args[0]).to.contain('warning'); - }); - }); -}); diff --git a/src/legacy/core_plugins/elasticsearch/lib/__tests__/health_check.js b/src/legacy/core_plugins/elasticsearch/lib/__tests__/health_check.js deleted file mode 100644 index 3b593c6352394..0000000000000 --- a/src/legacy/core_plugins/elasticsearch/lib/__tests__/health_check.js +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import Bluebird from 'bluebird'; -import sinon from 'sinon'; -import expect from '@kbn/expect'; - -const NoConnections = require('elasticsearch').errors.NoConnections; - -import healthCheck from '../health_check'; -import kibanaVersion from '../kibana_version'; - -const esPort = 9220; - -describe('plugins/elasticsearch', () => { - describe('lib/health_check', function() { - let health; - let plugin; - let cluster; - let server; - const sandbox = sinon.createSandbox(); - - function getTimerCount() { - return Object.keys(sandbox.clock.timers || {}).length; - } - - beforeEach(() => { - sandbox.useFakeTimers(); - const COMPATIBLE_VERSION_NUMBER = '5.0.0'; - - // Stub the Kibana version instead of drawing from package.json. - sandbox.stub(kibanaVersion, 'get').returns(COMPATIBLE_VERSION_NUMBER); - - // setup the plugin stub - plugin = { - name: 'elasticsearch', - status: { - red: sinon.stub(), - green: sinon.stub(), - yellow: sinon.stub(), - }, - }; - - cluster = { callWithInternalUser: sinon.stub(), errors: { NoConnections } }; - cluster.callWithInternalUser.withArgs('index', sinon.match.any).returns(Bluebird.resolve()); - cluster.callWithInternalUser - .withArgs('mget', sinon.match.any) - .returns(Bluebird.resolve({ ok: true })); - cluster.callWithInternalUser - .withArgs('get', sinon.match.any) - .returns(Bluebird.resolve({ found: false })); - cluster.callWithInternalUser - .withArgs('search', sinon.match.any) - .returns(Bluebird.resolve({ hits: { hits: [] } })); - cluster.callWithInternalUser.withArgs('nodes.info', sinon.match.any).returns( - Bluebird.resolve({ - nodes: { - 'node-01': { - version: COMPATIBLE_VERSION_NUMBER, - http_address: `inet[/127.0.0.1:${esPort}]`, - ip: '127.0.0.1', - }, - }, - }) - ); - - // Setup the server mock - server = { - logWithMetadata: sinon.stub(), - info: { port: 5601 }, - config: () => ({ get: sinon.stub() }), - plugins: { - elasticsearch: { - getCluster: sinon.stub().returns(cluster), - }, - }, - ext: sinon.stub(), - }; - - health = healthCheck(plugin, server, 0); - }); - - afterEach(() => sandbox.restore()); - - it('should stop when cluster is shutdown', () => { - // ensure that health.start() is responsible for the timer we are observing - expect(getTimerCount()).to.be(0); - health.start(); - expect(getTimerCount()).to.be(1); - - // ensure that a server extension was registered - sinon.assert.calledOnce(server.ext); - sinon.assert.calledWithExactly(server.ext, sinon.match.string, sinon.match.func); - - const [, handler] = server.ext.firstCall.args; - handler(); // this should be health.stop - - // ensure that the handler unregistered the timer - expect(getTimerCount()).to.be(0); - }); - - it('should set the cluster green if everything is ready', function() { - cluster.callWithInternalUser.withArgs('ping').returns(Bluebird.resolve()); - - return health.run().then(function() { - sinon.assert.calledOnce(plugin.status.yellow); - sinon.assert.calledWithExactly(plugin.status.yellow, 'Waiting for Elasticsearch'); - - sinon.assert.calledOnce( - cluster.callWithInternalUser.withArgs('nodes.info', sinon.match.any) - ); - sinon.assert.notCalled(plugin.status.red); - sinon.assert.calledOnce(plugin.status.green); - sinon.assert.calledWithExactly(plugin.status.green, 'Ready'); - }); - }); - - describe('#waitUntilReady', function() { - it('waits for green status', function() { - plugin.status.once = sinon.spy(function(event, handler) { - expect(event).to.be('green'); - setImmediate(handler); - }); - - const waitUntilReadyPromise = health.waitUntilReady(); - - sandbox.clock.runAll(); - - return waitUntilReadyPromise.then(function() { - sinon.assert.calledOnce(plugin.status.once); - }); - }); - }); - }); -}); diff --git a/src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js b/src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js deleted file mode 100644 index 8d304cd558418..0000000000000 --- a/src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * ES and Kibana versions are locked, so Kibana should require that ES has the same version as - * that defined in Kibana's package.json. - */ - -import { forEach, get } from 'lodash'; -import { coerce } from 'semver'; -import isEsCompatibleWithKibana from './is_es_compatible_with_kibana'; - -/** - * tracks the node descriptions that get logged in warnings so - * that we don't spam the log with the same message over and over. - * - * There are situations, like in testing or multi-tenancy, where - * the server argument changes, so we must track the previous - * node warnings per server - */ -const lastWarnedNodesForServer = new WeakMap(); - -export function ensureEsVersion(server, kibanaVersion, ignoreVersionMismatch = false) { - const { callWithInternalUser } = server.plugins.elasticsearch.getCluster('admin'); - - server.logWithMetadata(['plugin', 'debug'], 'Checking Elasticsearch version'); - return callWithInternalUser('nodes.info', { - filterPath: ['nodes.*.version', 'nodes.*.http.publish_address', 'nodes.*.ip'], - }).then(function(info) { - // Aggregate incompatible ES nodes. - const incompatibleNodes = []; - - // Aggregate ES nodes which should prompt a Kibana upgrade. - const warningNodes = []; - - forEach(info.nodes, esNode => { - if (!isEsCompatibleWithKibana(esNode.version, kibanaVersion)) { - // Exit early to avoid collecting ES nodes with newer major versions in the `warningNodes`. - return incompatibleNodes.push(esNode); - } - - // It's acceptable if ES and Kibana versions are not the same so long as - // they are not incompatible, but we should warn about it - - // Ignore version qualifiers - // https://github.com/elastic/elasticsearch/issues/36859 - const looseMismatch = coerce(esNode.version).version !== coerce(kibanaVersion).version; - if (looseMismatch) { - warningNodes.push(esNode); - } - }); - - function getHumanizedNodeNames(nodes) { - return nodes.map(node => { - const publishAddress = get(node, 'http.publish_address') - ? get(node, 'http.publish_address') + ' ' - : ''; - return 'v' + node.version + ' @ ' + publishAddress + '(' + node.ip + ')'; - }); - } - - if (warningNodes.length) { - const simplifiedNodes = warningNodes.map(node => ({ - version: node.version, - http: { - publish_address: get(node, 'http.publish_address'), - }, - ip: node.ip, - })); - - // Don't show the same warning over and over again. - const warningNodeNames = getHumanizedNodeNames(simplifiedNodes).join(', '); - if (lastWarnedNodesForServer.get(server) !== warningNodeNames) { - lastWarnedNodesForServer.set(server, warningNodeNames); - server.logWithMetadata( - ['warning'], - `You're running Kibana ${kibanaVersion} with some different versions of ` + - 'Elasticsearch. Update Kibana or Elasticsearch to the same ' + - `version to prevent compatibility issues: ${warningNodeNames}`, - { - kibanaVersion, - nodes: simplifiedNodes, - } - ); - } - } - - if (incompatibleNodes.length && !shouldIgnoreVersionMismatch(server, ignoreVersionMismatch)) { - const incompatibleNodeNames = getHumanizedNodeNames(incompatibleNodes); - throw new Error( - `This version of Kibana requires Elasticsearch v` + - `${kibanaVersion} on all nodes. I found ` + - `the following incompatible nodes in your cluster: ${incompatibleNodeNames.join(', ')}` - ); - } - - return true; - }); -} - -function shouldIgnoreVersionMismatch(server, ignoreVersionMismatch) { - const isDevMode = server.config().get('env.dev'); - if (!isDevMode && ignoreVersionMismatch) { - throw new Error( - `Option "elasticsearch.ignoreVersionMismatch" can only be used in development mode` - ); - } - - return isDevMode && ignoreVersionMismatch; -} diff --git a/src/legacy/core_plugins/elasticsearch/lib/health_check.js b/src/legacy/core_plugins/elasticsearch/lib/health_check.js deleted file mode 100644 index 40053ec774542..0000000000000 --- a/src/legacy/core_plugins/elasticsearch/lib/health_check.js +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import Bluebird from 'bluebird'; -import kibanaVersion from './kibana_version'; -import { ensureEsVersion } from './ensure_es_version'; - -export default function(plugin, server, requestDelay, ignoreVersionMismatch) { - plugin.status.yellow('Waiting for Elasticsearch'); - - function waitUntilReady() { - return new Bluebird(resolve => { - plugin.status.once('green', resolve); - }); - } - - function check() { - return ensureEsVersion(server, kibanaVersion.get(), ignoreVersionMismatch) - .then(() => plugin.status.green('Ready')) - .catch(err => plugin.status.red(err)); - } - - let timeoutId = null; - - function scheduleCheck(ms) { - if (timeoutId) return; - - const myId = setTimeout(function() { - check().finally(function() { - if (timeoutId === myId) startorRestartChecking(); - }); - }, ms); - - timeoutId = myId; - } - - function startorRestartChecking() { - scheduleCheck(stopChecking() ? requestDelay : 1); - } - - function stopChecking() { - if (!timeoutId) return false; - clearTimeout(timeoutId); - timeoutId = null; - return true; - } - - server.ext('onPreStop', stopChecking); - - return { - waitUntilReady: waitUntilReady, - run: check, - start: startorRestartChecking, - stop: stopChecking, - isRunning: function() { - return !!timeoutId; - }, - }; -} diff --git a/src/legacy/core_plugins/elasticsearch/lib/version_health_check.js b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.js new file mode 100644 index 0000000000000..4ee8307f490eb --- /dev/null +++ b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.js @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export const versionHealthCheck = (esPlugin, logWithMetadata, esNodesCompatibility$) => { + esPlugin.status.yellow('Waiting for Elasticsearch'); + + return new Promise(resolve => { + esNodesCompatibility$.subscribe(({ isCompatible, message, kibanaVersion, warningNodes }) => { + if (!isCompatible) { + esPlugin.status.red(message); + } else { + if (message) { + logWithMetadata(['warning'], message, { + kibanaVersion, + nodes: warningNodes, + }); + } + esPlugin.status.green('Ready'); + resolve(); + } + }); + }); +}; diff --git a/src/legacy/core_plugins/elasticsearch/lib/version_health_check.test.js b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.test.js new file mode 100644 index 0000000000000..ba7c95bcdfec5 --- /dev/null +++ b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.test.js @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { versionHealthCheck } from './version_health_check'; +import { Subject } from 'rxjs'; + +describe('plugins/elasticsearch', () => { + describe('lib/health_version_check', function() { + let plugin; + let logWithMetadata; + + beforeEach(() => { + plugin = { + status: { + red: jest.fn(), + green: jest.fn(), + yellow: jest.fn(), + }, + }; + + logWithMetadata = jest.fn(); + jest.clearAllMocks(); + }); + + it('returned promise resolves when all nodes are compatible ', function() { + const esNodesCompatibility$ = new Subject(); + const versionHealthyPromise = versionHealthCheck( + plugin, + logWithMetadata, + esNodesCompatibility$ + ); + esNodesCompatibility$.next({ isCompatible: true, message: undefined }); + return expect(versionHealthyPromise).resolves.toBe(undefined); + }); + + it('should set elasticsearch plugin status to green when all nodes are compatible', function() { + const esNodesCompatibility$ = new Subject(); + versionHealthCheck(plugin, logWithMetadata, esNodesCompatibility$); + expect(plugin.status.yellow).toHaveBeenCalledWith('Waiting for Elasticsearch'); + expect(plugin.status.green).not.toHaveBeenCalled(); + esNodesCompatibility$.next({ isCompatible: true, message: undefined }); + expect(plugin.status.green).toHaveBeenCalledWith('Ready'); + expect(plugin.status.red).not.toHaveBeenCalled(); + }); + + it('should set elasticsearch plugin status to red when some nodes are incompatible', function() { + const esNodesCompatibility$ = new Subject(); + versionHealthCheck(plugin, logWithMetadata, esNodesCompatibility$); + expect(plugin.status.yellow).toHaveBeenCalledWith('Waiting for Elasticsearch'); + expect(plugin.status.red).not.toHaveBeenCalled(); + esNodesCompatibility$.next({ isCompatible: false, message: 'your nodes are incompatible' }); + expect(plugin.status.red).toHaveBeenCalledWith('your nodes are incompatible'); + expect(plugin.status.green).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/legacy/server/http/integration_tests/default_route_provider.test.ts b/src/legacy/server/http/integration_tests/default_route_provider.test.ts index 4898cb2b67852..d91438d904558 100644 --- a/src/legacy/server/http/integration_tests/default_route_provider.test.ts +++ b/src/legacy/server/http/integration_tests/default_route_provider.test.ts @@ -29,7 +29,7 @@ let mockDefaultRouteSetting: any = ''; describe('default route provider', () => { let root: Root; beforeAll(async () => { - root = kbnTestServer.createRoot(); + root = kbnTestServer.createRoot({ migrations: { skip: true } }); await root.setup(); await root.start(); diff --git a/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts b/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts index da785a59893ab..8365941cbeb10 100644 --- a/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts +++ b/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts @@ -30,6 +30,7 @@ describe('default route provider', () => { server: { defaultRoute: '/app/some/default/route', }, + migrations: { skip: true }, }); await root.setup(); diff --git a/src/legacy/server/http/integration_tests/max_payload_size.test.js b/src/legacy/server/http/integration_tests/max_payload_size.test.js index 4408f0141bb21..7f22f83c78f0e 100644 --- a/src/legacy/server/http/integration_tests/max_payload_size.test.js +++ b/src/legacy/server/http/integration_tests/max_payload_size.test.js @@ -21,7 +21,7 @@ import * as kbnTestServer from '../../../../test_utils/kbn_server'; let root; beforeAll(async () => { - root = kbnTestServer.createRoot({ server: { maxPayloadBytes: 100 } }); + root = kbnTestServer.createRoot({ server: { maxPayloadBytes: 100 }, migrations: { skip: true } }); await root.setup(); await root.start(); diff --git a/x-pack/plugins/spaces/server/lib/request_interceptors/on_post_auth_interceptor.test.ts b/x-pack/plugins/spaces/server/lib/request_interceptors/on_post_auth_interceptor.test.ts index c1f557f164ad6..b62211440bb7a 100644 --- a/x-pack/plugins/spaces/server/lib/request_interceptors/on_post_auth_interceptor.test.ts +++ b/x-pack/plugins/spaces/server/lib/request_interceptors/on_post_auth_interceptor.test.ts @@ -107,7 +107,10 @@ describe('onPostAuthInterceptor', () => { availableSpaces: any[], testOptions = { simulateGetSpacesFailure: false, simulateGetSingleSpaceFailure: false } ) { - const { http } = await root.setup(); + const { http, elasticsearch } = await root.setup(); + + // Mock esNodesCompatibility$ to prevent `root.start()` from blocking on ES version check + elasticsearch.esNodesCompatibility$ = elasticsearchServiceMock.createInternalSetup().esNodesCompatibility$; const loggingMock = loggingServiceMock .create() diff --git a/x-pack/plugins/spaces/server/lib/request_interceptors/on_request_interceptor.test.ts b/x-pack/plugins/spaces/server/lib/request_interceptors/on_request_interceptor.test.ts index d6ff4a20052e4..5e6cf67ee8c90 100644 --- a/x-pack/plugins/spaces/server/lib/request_interceptors/on_request_interceptor.test.ts +++ b/x-pack/plugins/spaces/server/lib/request_interceptors/on_request_interceptor.test.ts @@ -17,6 +17,7 @@ import { import * as kbnTestServer from '../../../../../../src/test_utils/kbn_server'; import { LegacyAPI } from '../../plugin'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; describe('onRequestInterceptor', () => { let root: ReturnType; @@ -104,7 +105,9 @@ describe('onRequestInterceptor', () => { routes: 'legacy' | 'new-platform'; } async function setup(opts: SetupOpts = { basePath: '/', routes: 'legacy' }) { - const { http } = await root.setup(); + const { http, elasticsearch } = await root.setup(); + // Mock esNodesCompatibility$ to prevent `root.start()` from blocking on ES version check + elasticsearch.esNodesCompatibility$ = elasticsearchServiceMock.createInternalSetup().esNodesCompatibility$; initSpacesOnRequestInterceptor({ getLegacyAPI: () => From e1e367e42f94b9f609d3af2cc22bee59fe1fb789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Mon, 3 Feb 2020 18:39:09 +0100 Subject: [PATCH 03/11] [Logs UI] Add smoke tests for log rate and categories tabs (#55088) (#56632) Backports the following commits to 7.6: - [Logs UI] Add smoke tests for log rate and categories tabs (#55088) --- x-pack/test/functional/apps/infra/index.ts | 2 + .../apps/infra/log_entry_categories_tab.ts | 28 +++++++++ .../apps/infra/log_entry_rate_tab.ts | 28 +++++++++ .../apps/infra/logs_source_configuration.ts | 62 ++++++++++++------- .../page_objects/infra_logs_page.ts | 10 +-- x-pack/test/functional/services/index.ts | 4 +- .../test/functional/services/logs_ui/index.ts | 18 ++++++ .../services/logs_ui/log_entry_categories.ts | 23 +++++++ .../services/logs_ui/log_entry_rate.ts | 23 +++++++ .../log_stream.ts} | 17 +++-- 10 files changed, 182 insertions(+), 33 deletions(-) create mode 100644 x-pack/test/functional/apps/infra/log_entry_categories_tab.ts create mode 100644 x-pack/test/functional/apps/infra/log_entry_rate_tab.ts create mode 100644 x-pack/test/functional/services/logs_ui/index.ts create mode 100644 x-pack/test/functional/services/logs_ui/log_entry_categories.ts create mode 100644 x-pack/test/functional/services/logs_ui/log_entry_rate.ts rename x-pack/test/functional/services/{infra_log_stream.ts => logs_ui/log_stream.ts} (70%) diff --git a/x-pack/test/functional/apps/infra/index.ts b/x-pack/test/functional/apps/infra/index.ts index b706dc8cce546..597b522a88c51 100644 --- a/x-pack/test/functional/apps/infra/index.ts +++ b/x-pack/test/functional/apps/infra/index.ts @@ -12,6 +12,8 @@ export default ({ loadTestFile }: FtrProviderContext) => { loadTestFile(require.resolve('./home_page')); loadTestFile(require.resolve('./feature_controls')); + loadTestFile(require.resolve('./log_entry_categories_tab')); + loadTestFile(require.resolve('./log_entry_rate_tab')); loadTestFile(require.resolve('./logs_source_configuration')); loadTestFile(require.resolve('./metrics_source_configuration')); loadTestFile(require.resolve('./link_to')); diff --git a/x-pack/test/functional/apps/infra/log_entry_categories_tab.ts b/x-pack/test/functional/apps/infra/log_entry_categories_tab.ts new file mode 100644 index 0000000000000..c703738e37228 --- /dev/null +++ b/x-pack/test/functional/apps/infra/log_entry_categories_tab.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const logsUi = getService('logsUi'); + const retry = getService('retry'); + + describe('Log Entry Categories Tab', function() { + this.tags('smoke'); + + describe('with a trial license', () => { + it('is visible', async () => { + await logsUi.logEntryCategoriesPage.navigateTo(); + + await retry.try(async () => { + expect(await logsUi.logEntryCategoriesPage.getSetupScreen()).to.be.ok(); + }); + }); + }); + }); +}; diff --git a/x-pack/test/functional/apps/infra/log_entry_rate_tab.ts b/x-pack/test/functional/apps/infra/log_entry_rate_tab.ts new file mode 100644 index 0000000000000..95228a520aaa2 --- /dev/null +++ b/x-pack/test/functional/apps/infra/log_entry_rate_tab.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const logsUi = getService('logsUi'); + const retry = getService('retry'); + + describe('Log Entry Rate Tab', function() { + this.tags('smoke'); + + describe('with a trial license', () => { + it('is visible', async () => { + await logsUi.logEntryRatePage.navigateTo(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getSetupScreen()).to.be.ok(); + }); + }); + }); + }); +}; diff --git a/x-pack/test/functional/apps/infra/logs_source_configuration.ts b/x-pack/test/functional/apps/infra/logs_source_configuration.ts index 183acf3a980ee..ecad5a40ec42e 100644 --- a/x-pack/test/functional/apps/infra/logs_source_configuration.ts +++ b/x-pack/test/functional/apps/infra/logs_source_configuration.ts @@ -10,12 +10,14 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default ({ getPageObjects, getService }: FtrProviderContext) => { const esArchiver = getService('esArchiver'); - const infraLogStream = getService('infraLogStream'); + const logsUi = getService('logsUi'); const infraSourceConfigurationForm = getService('infraSourceConfigurationForm'); const pageObjects = getPageObjects(['common', 'infraLogs']); + const retry = getService('retry'); describe('Logs Source Configuration', function() { this.tags('smoke'); + before(async () => { await esArchiver.load('empty_kibana'); }); @@ -32,8 +34,11 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { }); it('can change the log indices to a pattern that matches nothing', async () => { - await pageObjects.common.navigateToActualUrl('infraLogs', 'logs/settings'); - await infraSourceConfigurationForm.getForm(); + await pageObjects.infraLogs.navigateToTab('settings'); + + await retry.try(async () => { + await infraSourceConfigurationForm.getForm(); + }); const nameInput = await infraSourceConfigurationForm.getNameInput(); await nameInput.clearValueWithKeyboard({ charByChar: true }); @@ -47,13 +52,19 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { }); it('renders the no indices screen when no indices match the pattern', async () => { - await pageObjects.common.navigateToActualUrl('infraLogs', 'logs/stream'); - await pageObjects.infraLogs.getNoLogsIndicesPrompt(); + await logsUi.logStreamPage.navigateTo(); + + await retry.try(async () => { + await logsUi.logStreamPage.getNoLogsIndicesPrompt(); + }); }); it('can change the log indices back to a pattern that matches something', async () => { - await pageObjects.common.navigateToActualUrl('infraLogs', 'logs/settings'); - await infraSourceConfigurationForm.getForm(); + await pageObjects.infraLogs.navigateToTab('settings'); + + await retry.try(async () => { + await infraSourceConfigurationForm.getForm(); + }); const logIndicesInput = await infraSourceConfigurationForm.getLogIndicesInput(); await logIndicesInput.clearValueWithKeyboard({ charByChar: true }); @@ -63,16 +74,19 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { }); it('renders the default log columns with their headers', async () => { - await pageObjects.common.navigateToActualUrl('infraLogs', 'logs/stream'); - const columnHeaderLabels = await infraLogStream.getColumnHeaderLabels(); + await logsUi.logStreamPage.navigateTo(); - expect(columnHeaderLabels).to.eql(['Oct 17, 2018', 'event.dataset', 'Message']); + await retry.try(async () => { + const columnHeaderLabels = await logsUi.logStreamPage.getColumnHeaderLabels(); - const logStreamEntries = await infraLogStream.getStreamEntries(); + expect(columnHeaderLabels).to.eql(['Oct 17, 2018', 'event.dataset', 'Message']); + }); + + const logStreamEntries = await logsUi.logStreamPage.getStreamEntries(); expect(logStreamEntries.length).to.be.greaterThan(0); const firstLogStreamEntry = logStreamEntries[0]; - const logStreamEntryColumns = await infraLogStream.getLogColumnsOfStreamEntry( + const logStreamEntryColumns = await logsUi.logStreamPage.getLogColumnsOfStreamEntry( firstLogStreamEntry ); @@ -80,32 +94,34 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { }); it('can change the log columns', async () => { - await pageObjects.common.navigateToActualUrl('infraLogs', 'logs/settings'); - await infraSourceConfigurationForm.getForm(); + await pageObjects.infraLogs.navigateToTab('settings'); + + await retry.try(async () => { + await infraSourceConfigurationForm.getForm(); + }); await infraSourceConfigurationForm.removeAllLogColumns(); await infraSourceConfigurationForm.addTimestampLogColumn(); await infraSourceConfigurationForm.addFieldLogColumn('host.name'); - // await infraSourceConfigurationForm.moveLogColumn(0, 1); - await infraSourceConfigurationForm.saveConfiguration(); }); it('renders the changed log columns with their headers', async () => { - await pageObjects.common.navigateToActualUrl('infraLogs', 'logs/stream'); - const columnHeaderLabels = await infraLogStream.getColumnHeaderLabels(); + await logsUi.logStreamPage.navigateTo(); + + await retry.try(async () => { + const columnHeaderLabels = await logsUi.logStreamPage.getColumnHeaderLabels(); - // TODO: make test more robust - // expect(columnHeaderLabels).to.eql(['host.name', 'Timestamp']); - expect(columnHeaderLabels).to.eql(['Oct 17, 2018', 'host.name']); + expect(columnHeaderLabels).to.eql(['Oct 17, 2018', 'host.name']); + }); - const logStreamEntries = await infraLogStream.getStreamEntries(); + const logStreamEntries = await logsUi.logStreamPage.getStreamEntries(); expect(logStreamEntries.length).to.be.greaterThan(0); const firstLogStreamEntry = logStreamEntries[0]; - const logStreamEntryColumns = await infraLogStream.getLogColumnsOfStreamEntry( + const logStreamEntryColumns = await logsUi.logStreamPage.getLogColumnsOfStreamEntry( firstLogStreamEntry ); diff --git a/x-pack/test/functional/page_objects/infra_logs_page.ts b/x-pack/test/functional/page_objects/infra_logs_page.ts index 6eb1349210bae..1c58f8dc41eba 100644 --- a/x-pack/test/functional/page_objects/infra_logs_page.ts +++ b/x-pack/test/functional/page_objects/infra_logs_page.ts @@ -18,12 +18,14 @@ export function InfraLogsPageProvider({ getPageObjects, getService }: FtrProvide await pageObjects.common.navigateToApp('infraLogs'); }, - async getLogStream() { - return await testSubjects.find('logStream'); + async navigateToTab(logsUiTab: LogsUiTab) { + await pageObjects.common.navigateToActualUrl('infraLogs', `/logs/${logsUiTab}`); }, - async getNoLogsIndicesPrompt() { - return await testSubjects.find('noLogsIndicesPrompt'); + async getLogStream() { + return await testSubjects.find('logStream'); }, }; } + +type LogsUiTab = 'log-categories' | 'log-rate' | 'settings' | 'stream'; diff --git a/x-pack/test/functional/services/index.ts b/x-pack/test/functional/services/index.ts index 5ab8933a4804a..ee9a57e5d65d5 100644 --- a/x-pack/test/functional/services/index.ts +++ b/x-pack/test/functional/services/index.ts @@ -46,7 +46,7 @@ import { GrokDebuggerProvider } from './grok_debugger'; import { UserMenuProvider } from './user_menu'; import { UptimeProvider } from './uptime'; import { InfraSourceConfigurationFormProvider } from './infra_source_configuration_form'; -import { InfraLogStreamProvider } from './infra_log_stream'; +import { LogsUiProvider } from './logs_ui'; import { MachineLearningProvider } from './ml'; import { TransformProvider } from './transform'; @@ -87,7 +87,7 @@ export const services = { userMenu: UserMenuProvider, uptime: UptimeProvider, infraSourceConfigurationForm: InfraSourceConfigurationFormProvider, - infraLogStream: InfraLogStreamProvider, + logsUi: LogsUiProvider, ml: MachineLearningProvider, transform: TransformProvider, }; diff --git a/x-pack/test/functional/services/logs_ui/index.ts b/x-pack/test/functional/services/logs_ui/index.ts new file mode 100644 index 0000000000000..c70a8470aafce --- /dev/null +++ b/x-pack/test/functional/services/logs_ui/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../../ftr_provider_context'; +import { LogEntryCategoriesPageProvider } from './log_entry_categories'; +import { LogEntryRatePageProvider } from './log_entry_rate'; +import { LogStreamPageProvider } from './log_stream'; + +export function LogsUiProvider(context: FtrProviderContext) { + return { + logEntryCategoriesPage: LogEntryCategoriesPageProvider(context), + logEntryRatePage: LogEntryRatePageProvider(context), + logStreamPage: LogStreamPageProvider(context), + }; +} diff --git a/x-pack/test/functional/services/logs_ui/log_entry_categories.ts b/x-pack/test/functional/services/logs_ui/log_entry_categories.ts new file mode 100644 index 0000000000000..b9a400b155679 --- /dev/null +++ b/x-pack/test/functional/services/logs_ui/log_entry_categories.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { WebElementWrapper } from '../../../../../test/functional/services/lib/web_element_wrapper'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export function LogEntryCategoriesPageProvider({ getPageObjects, getService }: FtrProviderContext) { + const pageObjects = getPageObjects(['infraLogs']); + const testSubjects = getService('testSubjects'); + + return { + async navigateTo() { + pageObjects.infraLogs.navigateToTab('log-categories'); + }, + + async getSetupScreen(): Promise { + return await testSubjects.find('logEntryCategoriesSetupPage'); + }, + }; +} diff --git a/x-pack/test/functional/services/logs_ui/log_entry_rate.ts b/x-pack/test/functional/services/logs_ui/log_entry_rate.ts new file mode 100644 index 0000000000000..96c69e85aa0a4 --- /dev/null +++ b/x-pack/test/functional/services/logs_ui/log_entry_rate.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { WebElementWrapper } from '../../../../../test/functional/services/lib/web_element_wrapper'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export function LogEntryRatePageProvider({ getPageObjects, getService }: FtrProviderContext) { + const pageObjects = getPageObjects(['infraLogs']); + const testSubjects = getService('testSubjects'); + + return { + async navigateTo() { + pageObjects.infraLogs.navigateToTab('log-rate'); + }, + + async getSetupScreen(): Promise { + return await testSubjects.find('logEntryRateSetupPage'); + }, + }; +} diff --git a/x-pack/test/functional/services/infra_log_stream.ts b/x-pack/test/functional/services/logs_ui/log_stream.ts similarity index 70% rename from x-pack/test/functional/services/infra_log_stream.ts rename to x-pack/test/functional/services/logs_ui/log_stream.ts index af113d3afffb4..ce37d2d5a60da 100644 --- a/x-pack/test/functional/services/infra_log_stream.ts +++ b/x-pack/test/functional/services/logs_ui/log_stream.ts @@ -4,14 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ -import { FtrProviderContext } from '../ftr_provider_context'; -import { WebElementWrapper } from '../../../../test/functional/services/lib/web_element_wrapper'; +import { FtrProviderContext } from '../../ftr_provider_context'; +import { WebElementWrapper } from '../../../../../test/functional/services/lib/web_element_wrapper'; -export function InfraLogStreamProvider({ getService }: FtrProviderContext) { - const testSubjects = getService('testSubjects'); +export function LogStreamPageProvider({ getPageObjects, getService }: FtrProviderContext) { + const pageObjects = getPageObjects(['infraLogs']); const retry = getService('retry'); + const testSubjects = getService('testSubjects'); return { + async navigateTo() { + pageObjects.infraLogs.navigateToTab('stream'); + }, + async getColumnHeaderLabels(): Promise { const columnHeaderElements: WebElementWrapper[] = await testSubjects.findAll( '~logColumnHeader' @@ -35,5 +40,9 @@ export function InfraLogStreamProvider({ getService }: FtrProviderContext) { ): Promise { return await testSubjects.findAllDescendant('~logColumn', entryElement); }, + + async getNoLogsIndicesPrompt() { + return await testSubjects.find('noLogsIndicesPrompt'); + }, }; } From e1b78746ee249a065353c460efcdf68cc4622ed3 Mon Sep 17 00:00:00 2001 From: Alison Goryachev Date: Mon, 3 Feb 2020 15:53:38 -0500 Subject: [PATCH 04/11] [Index Management] Add try/catch when parsing index filter from URI (#56051) (#56668) --- .../home/index_list/index_table/index_table.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/x-pack/legacy/plugins/index_management/public/app/sections/home/index_list/index_table/index_table.js b/x-pack/legacy/plugins/index_management/public/app/sections/home/index_list/index_table/index_table.js index 9441207a0da20..e043323c5b290 100644 --- a/x-pack/legacy/plugins/index_management/public/app/sections/home/index_list/index_table/index_table.js +++ b/x-pack/legacy/plugins/index_management/public/app/sections/home/index_list/index_table/index_table.js @@ -104,7 +104,13 @@ export class IndexTable extends Component { const { filterChanged, filterFromURI } = this.props; if (filterFromURI) { const decodedFilter = decodeURIComponent(filterFromURI); - filterChanged(EuiSearchBar.Query.parse(decodedFilter)); + + try { + const filter = EuiSearchBar.Query.parse(decodedFilter); + filterChanged(filter); + } catch (e) { + this.setState({ filterError: e }); + } } } componentWillUnmount() { @@ -122,7 +128,8 @@ export class IndexTable extends Component { return; } return ( - + <> + - - + + ); } onFilterChanged = ({ query, error }) => { From 9f0f92e16406c51c913450c23ee9b89e851b1e22 Mon Sep 17 00:00:00 2001 From: Justin Kambic Date: Mon, 3 Feb 2020 14:01:52 -0700 Subject: [PATCH 05/11] [Uptime] Fix flaky functional test for #54541 (#56449) (#56656) * Add timeout block to race-prone functional test code. * Add timeout for pagination click functions. Co-authored-by: Elastic Machine Co-authored-by: Elastic Machine --- x-pack/test/functional/apps/uptime/overview.ts | 12 ++++++++---- x-pack/test/functional/services/uptime.ts | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/test/functional/apps/uptime/overview.ts b/x-pack/test/functional/apps/uptime/overview.ts index 73b91a61196bf..9a879032fadc1 100644 --- a/x-pack/test/functional/apps/uptime/overview.ts +++ b/x-pack/test/functional/apps/uptime/overview.ts @@ -53,8 +53,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('pagination is cleared when filter criteria changes', async () => { await pageObjects.uptime.goToUptimePageAndSetDateRange(DEFAULT_DATE_START, DEFAULT_DATE_END); await pageObjects.uptime.changePage('next'); - // there should now be pagination data in the URL - await pageObjects.uptime.pageUrlContains('pagination'); await pageObjects.uptime.pageHasExpectedIds([ '0010-down', '0011-up', @@ -67,9 +65,11 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { '0018-up', '0019-up', ]); + await retry.tryForTime(12000, async () => { + // there should now be pagination data in the URL + await pageObjects.uptime.pageUrlContains('pagination'); + }); await pageObjects.uptime.setStatusFilter('up'); - // ensure that pagination is removed from the URL - await pageObjects.uptime.pageUrlContains('pagination', false); await pageObjects.uptime.pageHasExpectedIds([ '0000-intermittent', '0001-up', @@ -82,6 +82,10 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { '0008-up', '0009-up', ]); + await retry.tryForTime(12000, async () => { + // ensure that pagination is removed from the URL + await pageObjects.uptime.pageUrlContains('pagination', false); + }); }); describe('snapshot counts', () => { diff --git a/x-pack/test/functional/services/uptime.ts b/x-pack/test/functional/services/uptime.ts index ed39f28aabbfa..1d8e0c97b99c4 100644 --- a/x-pack/test/functional/services/uptime.ts +++ b/x-pack/test/functional/services/uptime.ts @@ -38,10 +38,10 @@ export function UptimeProvider({ getService }: FtrProviderContext) { await browser.pressKeys(browser.keys.ENTER); }, async goToNextPage() { - await testSubjects.click('xpack.uptime.monitorList.nextButton'); + await testSubjects.click('xpack.uptime.monitorList.nextButton', 5000); }, async goToPreviousPage() { - await testSubjects.click('xpack.uptime.monitorList.prevButton'); + await testSubjects.click('xpack.uptime.monitorList.prevButton', 5000); }, async setStatusFilterUp() { await testSubjects.click('xpack.uptime.filterBar.filterStatusUp'); From de7a4b197029c065100b5fb1835b1d150ec01305 Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Mon, 3 Feb 2020 14:38:25 -0800 Subject: [PATCH 06/11] [APM] Fixes maxTraceItems in waterfall trace and related error queries (#56111) (#56681) * Addresses #55494 by combining waterfall trace and related error queries. Makes sure that maxTraceItems is only respected for Transactions and Spans. * addressed feedback back separating the combined query into 1 for transactions/spans and 1 for errors + errorCount Co-authored-by: Elastic Machine Co-authored-by: Elastic Machine --- .../Marks/__test__/get_error_marks.test.ts | 15 +- .../Marks/get_error_marks.ts | 9 +- .../WaterfallContainer/Waterfall/index.tsx | 6 +- .../waterfall_helpers.test.ts.snap | 141 +++++++++--------- .../waterfall_helpers.test.ts | 17 ++- .../waterfall_helpers/waterfall_helpers.ts | 91 +++++++++-- .../plugins/apm/public/hooks/useWaterfall.ts | 2 +- .../errors/__snapshots__/queries.test.ts.snap | 64 -------- .../get_trace_errors_per_transaction.ts | 65 -------- .../apm/server/lib/errors/queries.test.ts | 9 -- .../traces/__snapshots__/queries.test.ts.snap | 50 +++---- .../apm/server/lib/traces/get_trace.ts | 9 +- .../apm/server/lib/traces/get_trace_items.ts | 80 +++++++++- x-pack/legacy/plugins/apm/typings/common.d.ts | 4 + 14 files changed, 267 insertions(+), 295 deletions(-) delete mode 100644 x-pack/legacy/plugins/apm/server/lib/errors/get_trace_errors_per_transaction.ts diff --git a/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Marks/__test__/get_error_marks.test.ts b/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Marks/__test__/get_error_marks.test.ts index 8fd8edd7f8a72..b7e83073a205b 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Marks/__test__/get_error_marks.test.ts +++ b/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Marks/__test__/get_error_marks.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IWaterfallItem } from '../../Waterfall/waterfall_helpers/waterfall_helpers'; +import { IWaterfallError } from '../../Waterfall/waterfall_helpers/waterfall_helpers'; import { getErrorMarks } from '../get_error_marks'; describe('getErrorMarks', () => { @@ -12,13 +12,6 @@ describe('getErrorMarks', () => { it('when items are missing', () => { expect(getErrorMarks([], {})).toEqual([]); }); - it('when any error is available', () => { - const items = [ - { docType: 'span' }, - { docType: 'transaction' } - ] as IWaterfallItem[]; - expect(getErrorMarks(items, {})).toEqual([]); - }); }); it('returns error marks', () => { @@ -29,14 +22,13 @@ describe('getErrorMarks', () => { skew: 5, doc: { error: { id: 1 }, service: { name: 'opbeans-java' } } } as unknown, - { docType: 'transaction' }, { docType: 'error', offset: 50, skew: 0, doc: { error: { id: 2 }, service: { name: 'opbeans-node' } } } as unknown - ] as IWaterfallItem[]; + ] as IWaterfallError[]; expect( getErrorMarks(items, { 'opbeans-java': 'red', 'opbeans-node': 'blue' }) ).toEqual([ @@ -67,14 +59,13 @@ describe('getErrorMarks', () => { skew: 5, doc: { error: { id: 1 }, service: { name: 'opbeans-java' } } } as unknown, - { docType: 'transaction' }, { docType: 'error', offset: 50, skew: 0, doc: { error: { id: 2 }, service: { name: 'opbeans-node' } } } as unknown - ] as IWaterfallItem[]; + ] as IWaterfallError[]; expect(getErrorMarks(items, {})).toEqual([ { type: 'errorMark', diff --git a/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Marks/get_error_marks.ts b/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Marks/get_error_marks.ts index f1f0163a49d10..e2b00c13c5c1f 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Marks/get_error_marks.ts +++ b/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Marks/get_error_marks.ts @@ -6,7 +6,6 @@ import { isEmpty } from 'lodash'; import { ErrorRaw } from '../../../../../../../typings/es_schemas/raw/ErrorRaw'; import { - IWaterfallItem, IWaterfallError, IServiceColors } from '../Waterfall/waterfall_helpers/waterfall_helpers'; @@ -19,16 +18,14 @@ export interface ErrorMark extends Mark { } export const getErrorMarks = ( - items: IWaterfallItem[], + errorItems: IWaterfallError[], serviceColors: IServiceColors ): ErrorMark[] => { - if (isEmpty(items)) { + if (isEmpty(errorItems)) { return []; } - return (items.filter( - item => item.docType === 'error' - ) as IWaterfallError[]).map(error => ({ + return errorItems.map(error => ({ type: 'errorMark', offset: error.offset + error.skew, verticalLine: false, diff --git a/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/index.tsx b/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/index.tsx index b48fc1cf7ca27..4f584786f2f9a 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/index.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/index.tsx @@ -80,13 +80,9 @@ export const Waterfall: React.FC = ({ const { serviceColors, duration } = waterfall; const agentMarks = getAgentMarks(waterfall.entryTransaction); - const errorMarks = getErrorMarks(waterfall.items, serviceColors); + const errorMarks = getErrorMarks(waterfall.errorItems, serviceColors); const renderWaterfallItem = (item: IWaterfallItem) => { - if (item.docType === 'error') { - return null; - } - const errorCount = item.docType === 'transaction' ? waterfall.errorsPerTransaction[item.doc.transaction.id] diff --git a/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/waterfall_helpers/__snapshots__/waterfall_helpers.test.ts.snap b/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/waterfall_helpers/__snapshots__/waterfall_helpers.test.ts.snap index 5b1b9be33c375..c9b29e8692f44 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/waterfall_helpers/__snapshots__/waterfall_helpers.test.ts.snap +++ b/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/waterfall_helpers/__snapshots__/waterfall_helpers.test.ts.snap @@ -24,6 +24,77 @@ Object { "name": "GET /api", }, }, + "errorItems": Array [ + Object { + "doc": Object { + "agent": Object { + "name": "ruby", + "version": "2", + }, + "error": Object { + "grouping_key": "errorGroupingKey1", + "id": "error1", + "log": Object { + "message": "error message", + }, + }, + "parent": Object { + "id": "myTransactionId1", + }, + "processor": Object { + "event": "error", + }, + "service": Object { + "name": "opbeans-ruby", + }, + "timestamp": Object { + "us": 1549324795810000, + }, + "trace": Object { + "id": "myTraceId", + }, + "transaction": Object { + "id": "myTransactionId1", + }, + }, + "docType": "error", + "duration": 0, + "id": "error1", + "offset": 25994, + "parent": Object { + "doc": Object { + "processor": Object { + "event": "transaction", + }, + "service": Object { + "name": "opbeans-node", + }, + "timestamp": Object { + "us": 1549324795784006, + }, + "trace": Object { + "id": "myTraceId", + }, + "transaction": Object { + "duration": Object { + "us": 49660, + }, + "id": "myTransactionId1", + "name": "GET /api", + }, + }, + "docType": "transaction", + "duration": 49660, + "id": "myTransactionId1", + "offset": 0, + "parent": undefined, + "parentId": undefined, + "skew": 0, + }, + "parentId": "myTransactionId1", + "skew": 0, + }, + ], "errorsCount": 1, "errorsPerTransaction": Object { "myTransactionId1": 2, @@ -716,75 +787,6 @@ Object { "parentId": "mySpanIdA", "skew": 0, }, - Object { - "doc": Object { - "agent": Object { - "name": "ruby", - "version": "2", - }, - "error": Object { - "grouping_key": "errorGroupingKey1", - "id": "error1", - "log": Object { - "message": "error message", - }, - }, - "parent": Object { - "id": "myTransactionId1", - }, - "processor": Object { - "event": "error", - }, - "service": Object { - "name": "opbeans-ruby", - }, - "timestamp": Object { - "us": 1549324795810000, - }, - "trace": Object { - "id": "myTraceId", - }, - "transaction": Object { - "id": "myTransactionId1", - }, - }, - "docType": "error", - "duration": 0, - "id": "error1", - "offset": 25994, - "parent": Object { - "doc": Object { - "processor": Object { - "event": "transaction", - }, - "service": Object { - "name": "opbeans-node", - }, - "timestamp": Object { - "us": 1549324795784006, - }, - "trace": Object { - "id": "myTraceId", - }, - "transaction": Object { - "duration": Object { - "us": 49660, - }, - "id": "myTransactionId1", - "name": "GET /api", - }, - }, - "docType": "transaction", - "duration": 49660, - "id": "myTransactionId1", - "offset": 0, - "parent": undefined, - "parentId": undefined, - "skew": 0, - }, - "parentId": "myTransactionId1", - "skew": 0, - }, ], "rootTransaction": Object { "processor": Object { @@ -848,6 +850,7 @@ Object { "name": "Api::ProductsController#index", }, }, + "errorItems": Array [], "errorsCount": 0, "errorsPerTransaction": Object { "myTransactionId1": 2, diff --git a/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/waterfall_helpers/waterfall_helpers.test.ts b/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/waterfall_helpers/waterfall_helpers.test.ts index 426842bc02f51..6b13b93200c61 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/waterfall_helpers/waterfall_helpers.test.ts +++ b/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/waterfall_helpers/waterfall_helpers.test.ts @@ -12,7 +12,8 @@ import { getOrderedWaterfallItems, getWaterfall, IWaterfallItem, - IWaterfallTransaction + IWaterfallTransaction, + IWaterfallError } from './waterfall_helpers'; import { APMError } from '../../../../../../../../typings/es_schemas/ui/APMError'; @@ -100,7 +101,9 @@ describe('waterfall_helpers', () => { } }, timestamp: { us: 1549324795823304 } - } as unknown) as Transaction, + } as unknown) as Transaction + ]; + const errorDocs = [ ({ processor: { event: 'error' }, parent: { id: 'myTransactionId1' }, @@ -130,14 +133,15 @@ describe('waterfall_helpers', () => { }; const waterfall = getWaterfall( { - trace: { items: hits, exceedsMax: false }, + trace: { items: hits, errorDocs, exceedsMax: false }, errorsPerTransaction }, entryTransactionId ); - expect(waterfall.items.length).toBe(7); + expect(waterfall.items.length).toBe(6); expect(waterfall.items[0].id).toBe('myTransactionId1'); + expect(waterfall.errorItems.length).toBe(1); expect(waterfall.errorsCount).toEqual(1); expect(waterfall).toMatchSnapshot(); }); @@ -150,7 +154,7 @@ describe('waterfall_helpers', () => { }; const waterfall = getWaterfall( { - trace: { items: hits, exceedsMax: false }, + trace: { items: hits, errorDocs, exceedsMax: false }, errorsPerTransaction }, entryTransactionId @@ -158,6 +162,7 @@ describe('waterfall_helpers', () => { expect(waterfall.items.length).toBe(4); expect(waterfall.items[0].id).toBe('myTransactionId2'); + expect(waterfall.errorItems.length).toBe(0); expect(waterfall.errorsCount).toEqual(0); expect(waterfall).toMatchSnapshot(); }); @@ -386,7 +391,7 @@ describe('waterfall_helpers', () => { it('should return parent skew for errors', () => { const child = { docType: 'error' - } as IWaterfallItem; + } as IWaterfallError; const parent = { docType: 'transaction', diff --git a/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/waterfall_helpers/waterfall_helpers.ts b/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/waterfall_helpers/waterfall_helpers.ts index 1af6cddb3ba4a..3b52163aa7fa4 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/waterfall_helpers/waterfall_helpers.ts +++ b/x-pack/legacy/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/WaterfallContainer/Waterfall/waterfall_helpers/waterfall_helpers.ts @@ -24,6 +24,8 @@ interface IWaterfallGroup { [key: string]: IWaterfallItem[]; } +const ROOT_ID = 'root'; + export interface IWaterfall { entryTransaction?: Transaction; rootTransaction?: Transaction; @@ -36,6 +38,7 @@ export interface IWaterfall { errorsPerTransaction: TraceAPIResponse['errorsPerTransaction']; errorsCount: number; serviceColors: IServiceColors; + errorItems: IWaterfallError[]; } interface IWaterfallItemBase { @@ -70,10 +73,7 @@ export type IWaterfallTransaction = IWaterfallItemBase< export type IWaterfallSpan = IWaterfallItemBase; export type IWaterfallError = IWaterfallItemBase; -export type IWaterfallItem = - | IWaterfallTransaction - | IWaterfallSpan - | IWaterfallError; +export type IWaterfallItem = IWaterfallTransaction | IWaterfallSpan; function getTransactionItem(transaction: Transaction): IWaterfallTransaction { return { @@ -99,20 +99,34 @@ function getSpanItem(span: Span): IWaterfallSpan { }; } -function getErrorItem(error: APMError): IWaterfallError { - return { +function getErrorItem( + error: APMError, + items: IWaterfallItem[], + entryWaterfallTransaction?: IWaterfallTransaction +): IWaterfallError { + const entryTimestamp = entryWaterfallTransaction?.doc.timestamp.us ?? 0; + const parent = items.find( + waterfallItem => waterfallItem.id === error.parent?.id + ); + const errorItem: IWaterfallError = { docType: 'error', doc: error, id: error.error.id, - parentId: error.parent?.id, - offset: 0, + parent, + parentId: parent?.id, + offset: error.timestamp.us - entryTimestamp, skew: 0, duration: 0 }; + + return { + ...errorItem, + skew: getClockSkew(errorItem, parent) + }; } export function getClockSkew( - item: IWaterfallItem, + item: IWaterfallItem | IWaterfallError, parentItem?: IWaterfallItem ) { if (!parentItem) { @@ -218,13 +232,11 @@ const getWaterfallItems = (items: TraceAPIResponse['trace']['items']) => return getSpanItem(item as Span); case 'transaction': return getTransactionItem(item as Transaction); - case 'error': - return getErrorItem(item as APMError); } }); const getChildrenGroupedByParentId = (waterfallItems: IWaterfallItem[]) => - groupBy(waterfallItems, item => (item.parentId ? item.parentId : 'root')); + groupBy(waterfallItems, item => (item.parentId ? item.parentId : ROOT_ID)); const getEntryWaterfallTransaction = ( entryTransactionId: string, @@ -234,6 +246,48 @@ const getEntryWaterfallTransaction = ( item => item.docType === 'transaction' && item.id === entryTransactionId ) as IWaterfallTransaction; +function isInEntryTransaction( + parentIdLookup: Map, + entryTransactionId: string, + currentId: string +): boolean { + if (currentId === entryTransactionId) { + return true; + } + const parentId = parentIdLookup.get(currentId); + if (parentId) { + return isInEntryTransaction(parentIdLookup, entryTransactionId, parentId); + } + return false; +} + +function getWaterfallErrors( + errorDocs: TraceAPIResponse['trace']['errorDocs'], + items: IWaterfallItem[], + entryWaterfallTransaction?: IWaterfallTransaction +) { + const errorItems = errorDocs.map(errorDoc => + getErrorItem(errorDoc, items, entryWaterfallTransaction) + ); + if (!entryWaterfallTransaction) { + return errorItems; + } + const parentIdLookup = [...items, ...errorItems].reduce( + (map, { id, parentId }) => { + map.set(id, parentId ?? ROOT_ID); + return map; + }, + new Map() + ); + return errorItems.filter(errorItem => + isInEntryTransaction( + parentIdLookup, + entryWaterfallTransaction?.id, + errorItem.id + ) + ); +} + export function getWaterfall( { trace, errorsPerTransaction }: TraceAPIResponse, entryTransactionId?: Transaction['transaction']['id'] @@ -244,7 +298,8 @@ export function getWaterfall( items: [], errorsPerTransaction, errorsCount: sum(Object.values(errorsPerTransaction)), - serviceColors: {} + serviceColors: {}, + errorItems: [] }; } @@ -261,6 +316,11 @@ export function getWaterfall( childrenByParentId, entryWaterfallTransaction ); + const errorItems = getWaterfallErrors( + trace.errorDocs, + items, + entryWaterfallTransaction + ); const rootTransaction = getRootTransaction(childrenByParentId); const duration = getWaterfallDuration(items); @@ -274,7 +334,8 @@ export function getWaterfall( duration, items, errorsPerTransaction, - errorsCount: items.filter(item => item.docType === 'error').length, - serviceColors + errorsCount: errorItems.length, + serviceColors, + errorItems }; } diff --git a/x-pack/legacy/plugins/apm/public/hooks/useWaterfall.ts b/x-pack/legacy/plugins/apm/public/hooks/useWaterfall.ts index f3f10c0e46d9b..8d623a0268222 100644 --- a/x-pack/legacy/plugins/apm/public/hooks/useWaterfall.ts +++ b/x-pack/legacy/plugins/apm/public/hooks/useWaterfall.ts @@ -11,7 +11,7 @@ import { getWaterfall } from '../components/app/TransactionDetails/WaterfallWith const INITIAL_DATA = { root: undefined, - trace: { items: [], exceedsMax: false }, + trace: { items: [], exceedsMax: false, errorDocs: [] }, errorsPerTransaction: {} }; diff --git a/x-pack/legacy/plugins/apm/server/lib/errors/__snapshots__/queries.test.ts.snap b/x-pack/legacy/plugins/apm/server/lib/errors/__snapshots__/queries.test.ts.snap index 88d8edd17454a..a2629366dd6d9 100644 --- a/x-pack/legacy/plugins/apm/server/lib/errors/__snapshots__/queries.test.ts.snap +++ b/x-pack/legacy/plugins/apm/server/lib/errors/__snapshots__/queries.test.ts.snap @@ -205,67 +205,3 @@ Object { "index": "myIndex", } `; - -exports[`error queries fetches trace errors 1`] = ` -Object { - "body": Object { - "aggs": Object { - "transactions": Object { - "terms": Object { - "execution_hint": "map", - "field": "transaction.id", - }, - }, - }, - "query": Object { - "bool": Object { - "filter": Array [ - Object { - "term": Object { - "trace.id": "foo", - }, - }, - Object { - "term": Object { - "processor.event": "error", - }, - }, - Object { - "range": Object { - "@timestamp": Object { - "format": "epoch_millis", - "gte": 1528113600000, - "lte": 1528977600000, - }, - }, - }, - ], - "should": Array [ - Object { - "bool": Object { - "must_not": Array [ - Object { - "exists": Object { - "field": "error.log.level", - }, - }, - ], - }, - }, - Object { - "terms": Object { - "error.log.level": Array [ - "critical", - "error", - "fatal", - ], - }, - }, - ], - }, - }, - "size": 0, - }, - "index": "myIndex", -} -`; diff --git a/x-pack/legacy/plugins/apm/server/lib/errors/get_trace_errors_per_transaction.ts b/x-pack/legacy/plugins/apm/server/lib/errors/get_trace_errors_per_transaction.ts deleted file mode 100644 index 6027693be5180..0000000000000 --- a/x-pack/legacy/plugins/apm/server/lib/errors/get_trace_errors_per_transaction.ts +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { - ERROR_LOG_LEVEL, - PROCESSOR_EVENT, - TRACE_ID, - TRANSACTION_ID -} from '../../../common/elasticsearch_fieldnames'; -import { rangeFilter } from '../helpers/range_filter'; -import { Setup, SetupTimeRange } from '../helpers/setup_request'; - -export interface ErrorsPerTransaction { - [transactionId: string]: number; -} - -const includedLogLevels = ['critical', 'error', 'fatal']; - -export async function getTraceErrorsPerTransaction( - traceId: string, - setup: Setup & SetupTimeRange -): Promise { - const { start, end, client, indices } = setup; - - const params = { - index: indices['apm_oss.errorIndices'], - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [TRACE_ID]: traceId } }, - { term: { [PROCESSOR_EVENT]: 'error' } }, - { range: rangeFilter(start, end) } - ], - should: [ - { bool: { must_not: [{ exists: { field: ERROR_LOG_LEVEL } }] } }, - { terms: { [ERROR_LOG_LEVEL]: includedLogLevels } } - ] - } - }, - aggs: { - transactions: { - terms: { - field: TRANSACTION_ID, - // high cardinality - execution_hint: 'map' - } - } - } - } - } as const; - - const resp = await client.search(params); - return (resp.aggregations?.transactions.buckets || []).reduce( - (acc, bucket) => ({ - ...acc, - [bucket.key]: bucket.doc_count - }), - {} - ); -} diff --git a/x-pack/legacy/plugins/apm/server/lib/errors/queries.test.ts b/x-pack/legacy/plugins/apm/server/lib/errors/queries.test.ts index 2b1704d9424e4..f1e5d31efd4bd 100644 --- a/x-pack/legacy/plugins/apm/server/lib/errors/queries.test.ts +++ b/x-pack/legacy/plugins/apm/server/lib/errors/queries.test.ts @@ -6,7 +6,6 @@ import { getErrorGroup } from './get_error_group'; import { getErrorGroups } from './get_error_groups'; -import { getTraceErrorsPerTransaction } from './get_trace_errors_per_transaction'; import { SearchParamsMock, inspectSearchParams @@ -56,12 +55,4 @@ describe('error queries', () => { expect(mock.params).toMatchSnapshot(); }); - - it('fetches trace errors', async () => { - mock = await inspectSearchParams(setup => - getTraceErrorsPerTransaction('foo', setup) - ); - - expect(mock.params).toMatchSnapshot(); - }); }); diff --git a/x-pack/legacy/plugins/apm/server/lib/traces/__snapshots__/queries.test.ts.snap b/x-pack/legacy/plugins/apm/server/lib/traces/__snapshots__/queries.test.ts.snap index a2828e1d74920..0a9f9d38b2be7 100644 --- a/x-pack/legacy/plugins/apm/server/lib/traces/__snapshots__/queries.test.ts.snap +++ b/x-pack/legacy/plugins/apm/server/lib/traces/__snapshots__/queries.test.ts.snap @@ -3,6 +3,15 @@ exports[`trace queries fetches a trace 1`] = ` Object { "body": Object { + "aggs": Object { + "by_transaction_id": Object { + "terms": Object { + "execution_hint": "map", + "field": "transaction.id", + "size": "myIndex", + }, + }, + }, "query": Object { "bool": Object { "filter": Array [ @@ -12,12 +21,8 @@ Object { }, }, Object { - "terms": Object { - "processor.event": Array [ - "span", - "transaction", - "error", - ], + "term": Object { + "processor.event": "error", }, }, Object { @@ -30,36 +35,19 @@ Object { }, }, ], - "should": Object { - "exists": Object { - "field": "parent.id", + "must_not": Object { + "terms": Object { + "error.log.level": Array [ + "debug", + "info", + "warning", + ], }, }, }, }, "size": "myIndex", - "sort": Array [ - Object { - "_score": Object { - "order": "asc", - }, - }, - Object { - "transaction.duration.us": Object { - "order": "desc", - }, - }, - Object { - "span.duration.us": Object { - "order": "desc", - }, - }, - ], - "track_total_hits": true, }, - "index": Array [ - "myIndex", - "myIndex", - ], + "index": "myIndex", } `; diff --git a/x-pack/legacy/plugins/apm/server/lib/traces/get_trace.ts b/x-pack/legacy/plugins/apm/server/lib/traces/get_trace.ts index e38ce56edde80..a1b9270e0d7b3 100644 --- a/x-pack/legacy/plugins/apm/server/lib/traces/get_trace.ts +++ b/x-pack/legacy/plugins/apm/server/lib/traces/get_trace.ts @@ -5,16 +5,15 @@ */ import { PromiseReturnType } from '../../../typings/common'; -import { getTraceErrorsPerTransaction } from '../errors/get_trace_errors_per_transaction'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { getTraceItems } from './get_trace_items'; export type TraceAPIResponse = PromiseReturnType; export async function getTrace(traceId: string, setup: Setup & SetupTimeRange) { - const [trace, errorsPerTransaction] = await Promise.all([ - getTraceItems(traceId, setup), - getTraceErrorsPerTransaction(traceId, setup) - ]); + const { errorsPerTransaction, ...trace } = await getTraceItems( + traceId, + setup + ); return { trace, diff --git a/x-pack/legacy/plugins/apm/server/lib/traces/get_trace_items.ts b/x-pack/legacy/plugins/apm/server/lib/traces/get_trace_items.ts index 8118b6acaee39..9d3e0d6db7f16 100644 --- a/x-pack/legacy/plugins/apm/server/lib/traces/get_trace_items.ts +++ b/x-pack/legacy/plugins/apm/server/lib/traces/get_trace_items.ts @@ -9,13 +9,20 @@ import { TRACE_ID, PARENT_ID, TRANSACTION_DURATION, - SPAN_DURATION + SPAN_DURATION, + TRANSACTION_ID, + ERROR_LOG_LEVEL } from '../../../common/elasticsearch_fieldnames'; import { Span } from '../../../typings/es_schemas/ui/Span'; import { Transaction } from '../../../typings/es_schemas/ui/Transaction'; import { APMError } from '../../../typings/es_schemas/ui/APMError'; import { rangeFilter } from '../helpers/range_filter'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; +import { PromiseValueType } from '../../../typings/common'; + +interface ErrorsPerTransaction { + [transactionId: string]: number; +} export async function getTraceItems( traceId: string, @@ -23,8 +30,36 @@ export async function getTraceItems( ) { const { start, end, client, config, indices } = setup; const maxTraceItems = config['xpack.apm.ui.maxTraceItems']; + const excludedLogLevels = ['debug', 'info', 'warning']; - const params = { + const errorResponsePromise = client.search({ + index: indices['apm_oss.errorIndices'], + body: { + size: maxTraceItems, + query: { + bool: { + filter: [ + { term: { [TRACE_ID]: traceId } }, + { term: { [PROCESSOR_EVENT]: 'error' } }, + { range: rangeFilter(start, end) } + ], + must_not: { terms: { [ERROR_LOG_LEVEL]: excludedLogLevels } } + } + }, + aggs: { + by_transaction_id: { + terms: { + field: TRANSACTION_ID, + size: maxTraceItems, + // high cardinality + execution_hint: 'map' as const + } + } + } + } + }); + + const traceResponsePromise = client.search({ index: [ indices['apm_oss.spanIndices'], indices['apm_oss.transactionIndices'] @@ -35,7 +70,7 @@ export async function getTraceItems( bool: { filter: [ { term: { [TRACE_ID]: traceId } }, - { terms: { [PROCESSOR_EVENT]: ['span', 'transaction', 'error'] } }, + { terms: { [PROCESSOR_EVENT]: ['span', 'transaction'] } }, { range: rangeFilter(start, end) } ], should: { @@ -50,12 +85,43 @@ export async function getTraceItems( ], track_total_hits: true } - }; + }); + + const [errorResponse, traceResponse]: [ + // explicit intermediary types to avoid TS "excessively deep" error + PromiseValueType, + PromiseValueType + // @ts-ignore + ] = await Promise.all([errorResponsePromise, traceResponsePromise]); + + const exceedsMax = traceResponse.hits.total.value > maxTraceItems; - const resp = await client.search(params); + const items = (traceResponse.hits.hits as Array<{ + _source: Transaction | Span; + }>).map(hit => hit._source); + + const errorFrequencies: { + errorsPerTransaction: ErrorsPerTransaction; + errorDocs: APMError[]; + } = { + errorDocs: errorResponse.hits.hits.map( + ({ _source }) => _source as APMError + ), + errorsPerTransaction: + errorResponse.aggregations?.by_transaction_id.buckets.reduce( + (acc, current) => { + return { + ...acc, + [current.key]: current.doc_count + }; + }, + {} as ErrorsPerTransaction + ) ?? {} + }; return { - items: resp.hits.hits.map(hit => hit._source), - exceedsMax: resp.hits.total.value > maxTraceItems + items, + exceedsMax, + ...errorFrequencies }; } diff --git a/x-pack/legacy/plugins/apm/typings/common.d.ts b/x-pack/legacy/plugins/apm/typings/common.d.ts index b9064980bd657..1e718f818246c 100644 --- a/x-pack/legacy/plugins/apm/typings/common.d.ts +++ b/x-pack/legacy/plugins/apm/typings/common.d.ts @@ -22,6 +22,10 @@ type AllowUnknownObjectProperties = T extends object } : T; +export type PromiseValueType = Value extends Promise + ? Value + : Value; + export type PromiseReturnType = Func extends ( ...args: any[] ) => Promise From 84fc9573ec56d3f06f2fc9d2d52901f7cc2caadd Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Mon, 3 Feb 2020 14:41:19 -0800 Subject: [PATCH 07/11] [APM] Fix initial error sort field (#56577) (#56685) * Closes #52840. Changes the initial sorting field from `latestOccurrenceAt` -> `occurrenceCount` * update jest snapshots Co-authored-by: Elastic Machine Co-authored-by: Elastic Machine --- .../__test__/__snapshots__/List.test.tsx.snap | 68 +++++++++---------- .../app/ErrorGroupOverview/List/index.tsx | 2 +- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/x-pack/legacy/plugins/apm/public/components/app/ErrorGroupOverview/List/__test__/__snapshots__/List.test.tsx.snap b/x-pack/legacy/plugins/apm/public/components/app/ErrorGroupOverview/List/__test__/__snapshots__/List.test.tsx.snap index 492d28206f3dd..a45357121354f 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ErrorGroupOverview/List/__test__/__snapshots__/List.test.tsx.snap +++ b/x-pack/legacy/plugins/apm/public/components/app/ErrorGroupOverview/List/__test__/__snapshots__/List.test.tsx.snap @@ -43,7 +43,7 @@ exports[`ErrorGroupOverview -> List should render empty state 1`] = ` } initialPageSize={25} initialSortDirection="desc" - initialSortField="latestOccurrenceAt" + initialSortField="occurrenceCount" items={Array []} noItemsMessage="No errors were found" sortItems={false} @@ -190,7 +190,7 @@ exports[`ErrorGroupOverview -> List should render empty state 1`] = ` List should render empty state 1`] = ` } >