From d1498ac1fb3ea0e29632b540cf23ef1d092b35f4 Mon Sep 17 00:00:00 2001 From: Andrew Tate Date: Mon, 26 Sep 2022 14:16:31 -0500 Subject: [PATCH 1/6] [Lens] optimize duplicate formula functions (#140859) --- src/plugins/data/public/index.ts | 2 + .../dedupe_aggs.test.ts | 108 ++++++ .../indexpattern_datasource/dedupe_aggs.ts | 98 +++++ .../indexpattern.test.ts | 347 ++++++++++++------ .../definitions/cardinality.test.ts | 111 ++++++ .../operations/definitions/cardinality.tsx | 8 + .../operations/definitions/count.test.ts | 122 ++++++ .../operations/definitions/count.tsx | 9 + .../definitions/get_group_by_key.ts | 81 ++++ .../operations/definitions/index.ts | 23 ++ .../definitions/last_value.test.tsx | 105 ++++++ .../operations/definitions/last_value.tsx | 9 + .../operations/definitions/metrics.test.ts | 132 +++++++ .../operations/definitions/metrics.tsx | 9 + .../definitions/percentile.test.tsx | 109 +++++- .../operations/definitions/percentile.tsx | 15 + .../definitions/terms/helpers.test.ts | 134 ------- .../operations/definitions/terms/helpers.ts | 26 -- .../operations/definitions/terms/index.tsx | 11 +- .../definitions/terms/terms.test.tsx | 43 --- .../indexpattern_datasource/to_expression.ts | 21 +- 21 files changed, 1173 insertions(+), 350 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/dedupe_aggs.test.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/dedupe_aggs.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.test.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.test.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/get_group_by_key.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.test.ts diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index c8eefbdd92c6e..fa545d7648df1 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -139,6 +139,8 @@ export type { ParsedInterval, // expressions ExecutionContextSearch, + ExpressionFunctionKql, + ExpressionFunctionLucene, ExpressionFunctionKibana, ExpressionFunctionKibanaContext, ExpressionValueSearchContext, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dedupe_aggs.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/dedupe_aggs.test.ts new file mode 100644 index 0000000000000..eb98d7411491c --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dedupe_aggs.test.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + buildExpression, + ExpressionAstExpressionBuilder, + parseExpression, +} from '@kbn/expressions-plugin/common'; +import { dedupeAggs } from './dedupe_aggs'; +import { operationDefinitionMap } from './operations'; +import { OriginalColumn } from './to_expression'; + +describe('dedupeAggs', () => { + const buildMapsFromAggBuilders = (aggs: ExpressionAstExpressionBuilder[]) => { + const esAggsIdMap: Record = {}; + const aggsToIdsMap = new Map(); + aggs.forEach((builder, i) => { + const esAggsId = `col-${i}-${i}`; + esAggsIdMap[esAggsId] = [{ id: `original-${i}` } as OriginalColumn]; + aggsToIdsMap.set(builder, esAggsId); + }); + return { + esAggsIdMap, + aggsToIdsMap, + }; + }; + + it('removes duplicate aggregations', () => { + const aggs = [ + 'aggSum id="0" enabled=true schema="metric" field="bytes" emptyAsNull=false', + 'aggSum id="1" enabled=true schema="metric" field="bytes" emptyAsNull=false', + 'aggFilteredMetric id="2" enabled=true schema="metric" \n customBucket={aggFilter id="2-filter" enabled=true schema="bucket" filter={kql q="hour_of_day: *"}} \n customMetric={aggTopMetrics id="2-metric" enabled=true schema="metric" field="hour_of_day" size=1 sortOrder="desc" sortField="timestamp"}', + 'aggFilteredMetric id="3" enabled=true schema="metric" \n customBucket={aggFilter id="3-filter" enabled=true schema="bucket" filter={kql q="hour_of_day: *"}} \n customMetric={aggTopMetrics id="3-metric" enabled=true schema="metric" field="hour_of_day" size=1 sortOrder="desc" sortField="timestamp"}', + 'aggAvg id="4" enabled=true schema="metric" field="bytes"', + 'aggAvg id="5" enabled=true schema="metric" field="bytes"', + ].map((expression) => buildExpression(parseExpression(expression))); + + const { esAggsIdMap, aggsToIdsMap } = buildMapsFromAggBuilders(aggs); + + // eslint-disable-next-line @typescript-eslint/naming-convention + const { sum, last_value, average } = operationDefinitionMap; + + const operations = [sum, last_value, average]; + + operations.forEach((op) => expect(op.getGroupByKey).toBeDefined()); + + const { esAggsIdMap: newIdMap, aggs: newAggs } = dedupeAggs( + aggs, + esAggsIdMap, + aggsToIdsMap, + operations + ); + + expect(newAggs).toHaveLength(3); + + expect(newIdMap).toMatchInlineSnapshot(` + Object { + "col-0-0": Array [ + Object { + "id": "original-0", + }, + Object { + "id": "original-1", + }, + ], + "col-2-2": Array [ + Object { + "id": "original-2", + }, + Object { + "id": "original-3", + }, + ], + "col-4-4": Array [ + Object { + "id": "original-4", + }, + Object { + "id": "original-5", + }, + ], + } + `); + }); + + it('should update any terms order-by reference', () => { + const aggs = [ + 'aggTerms id="0" enabled=true schema="segment" field="clientip" orderBy="3" order="desc" size=5 includeIsRegex=false excludeIsRegex=false otherBucket=true otherBucketLabel="Other" missingBucket=false missingBucketLabel="(missing value)"', + 'aggMedian id="1" enabled=true schema="metric" field="bytes"', + 'aggMedian id="2" enabled=true schema="metric" field="bytes"', + 'aggMedian id="3" enabled=true schema="metric" field="bytes"', + ].map((expression) => buildExpression(parseExpression(expression))); + + const { esAggsIdMap, aggsToIdsMap } = buildMapsFromAggBuilders(aggs); + + const { aggs: newAggs } = dedupeAggs(aggs, esAggsIdMap, aggsToIdsMap, [ + operationDefinitionMap.median, + ]); + + expect(newAggs).toHaveLength(2); + + expect(newAggs[0].functions[0].getArgument('orderBy')?.[0]).toBe('1'); + }); +}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dedupe_aggs.ts b/x-pack/plugins/lens/public/indexpattern_datasource/dedupe_aggs.ts new file mode 100644 index 0000000000000..ce5b2b0f41f23 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dedupe_aggs.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AggFunctionsMapping } from '@kbn/data-plugin/public'; +import { + ExpressionAstExpressionBuilder, + ExpressionAstFunctionBuilder, +} from '@kbn/expressions-plugin/common'; +import { GenericOperationDefinition } from './operations'; +import { extractAggId, OriginalColumn } from './to_expression'; + +function groupByKey(items: T[], getKey: (item: T) => string | undefined): Record { + const groups: Record = {}; + + items.forEach((item) => { + const key = getKey(item); + if (key) { + if (!(key in groups)) { + groups[key] = []; + } + groups[key].push(item); + } + }); + + return groups; +} + +/** + * Consolidates duplicate agg expression builders to increase performance + */ +export function dedupeAggs( + _aggs: ExpressionAstExpressionBuilder[], + _esAggsIdMap: Record, + aggExpressionToEsAggsIdMap: Map, + allOperations: GenericOperationDefinition[] +): { + aggs: ExpressionAstExpressionBuilder[]; + esAggsIdMap: Record; +} { + let aggs = [..._aggs]; + const esAggsIdMap = { ..._esAggsIdMap }; + + const aggsByArgs = groupByKey(aggs, (expressionBuilder) => { + for (const operation of allOperations) { + const groupKey = operation.getGroupByKey?.(expressionBuilder); + if (groupKey) { + return `${operation.type}-${groupKey}`; + } + } + }); + + const termsFuncs = aggs + .map((agg) => agg.functions[0]) + .filter((func) => func.name === 'aggTerms') as Array< + ExpressionAstFunctionBuilder + >; + + // collapse each group into a single agg expression builder + Object.values(aggsByArgs).forEach((expressionBuilders) => { + if (expressionBuilders.length <= 1) { + // don't need to optimize if there aren't more than one + return; + } + + const [firstExpressionBuilder, ...restExpressionBuilders] = expressionBuilders; + + // throw away all but the first expression builder + aggs = aggs.filter((aggBuilder) => !restExpressionBuilders.includes(aggBuilder)); + + const firstEsAggsId = aggExpressionToEsAggsIdMap.get(firstExpressionBuilder); + if (firstEsAggsId === undefined) { + throw new Error('Could not find current column ID for expression builder'); + } + + restExpressionBuilders.forEach((expressionBuilder) => { + const currentEsAggsId = aggExpressionToEsAggsIdMap.get(expressionBuilder); + if (currentEsAggsId === undefined) { + throw new Error('Could not find current column ID for expression builder'); + } + + esAggsIdMap[firstEsAggsId].push(...esAggsIdMap[currentEsAggsId]); + + delete esAggsIdMap[currentEsAggsId]; + + termsFuncs.forEach((func) => { + if (func.getArgument('orderBy')?.[0] === extractAggId(currentEsAggsId)) { + func.replaceArgument('orderBy', [extractAggId(firstEsAggsId)]); + } + }); + }); + }); + + return { aggs, esAggsIdMap }; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index c19338dd156f0..481701b1e7824 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -1220,138 +1220,261 @@ describe('IndexPattern Data Source', () => { expect(ast.chain[1].arguments.timeFields).not.toContain('timefield'); }); - it('should call optimizeEsAggs once per operation for which it is available', () => { - const queryBaseState: IndexPatternPrivateState = { - currentIndexPatternId: '1', - layers: { - first: { - indexPatternId: '1', - columns: { - col1: { - label: 'timestamp', - dataType: 'date', - operationType: 'date_histogram', - sourceField: 'timestamp', - isBucketed: true, - scale: 'interval', - params: { - interval: 'auto', - includeEmptyRows: true, - dropPartials: false, - }, - } as DateHistogramIndexPatternColumn, - col2: { - label: '95th percentile of bytes', - dataType: 'number', - operationType: 'percentile', - sourceField: 'bytes', - isBucketed: false, - scale: 'ratio', - params: { - percentile: 95, + describe('optimizations', () => { + it('should call optimizeEsAggs once per operation for which it is available', () => { + const queryBaseState: IndexPatternPrivateState = { + currentIndexPatternId: '1', + layers: { + first: { + indexPatternId: '1', + columns: { + col1: { + label: 'timestamp', + dataType: 'date', + operationType: 'date_histogram', + sourceField: 'timestamp', + isBucketed: true, + scale: 'interval', + params: { + interval: 'auto', + includeEmptyRows: true, + dropPartials: false, + }, + } as DateHistogramIndexPatternColumn, + col2: { + label: '95th percentile of bytes', + dataType: 'number', + operationType: 'percentile', + sourceField: 'bytes', + isBucketed: false, + scale: 'ratio', + params: { + percentile: 95, + }, + } as PercentileIndexPatternColumn, + col3: { + label: '95th percentile of bytes', + dataType: 'number', + operationType: 'percentile', + sourceField: 'bytes', + isBucketed: false, + scale: 'ratio', + params: { + percentile: 95, + }, + } as PercentileIndexPatternColumn, + }, + columnOrder: ['col1', 'col2', 'col3'], + incompleteColumns: {}, + }, + }, + }; + + const optimizeMock = jest.spyOn(operationDefinitionMap.percentile, 'optimizeEsAggs'); + + indexPatternDatasource.toExpression(queryBaseState, 'first', indexPatterns); + + expect(operationDefinitionMap.percentile.optimizeEsAggs).toHaveBeenCalledTimes(1); + + optimizeMock.mockRestore(); + }); + + it('should update anticipated esAggs column IDs based on the order of the optimized agg expression builders', () => { + const queryBaseState: IndexPatternPrivateState = { + currentIndexPatternId: '1', + layers: { + first: { + indexPatternId: '1', + columns: { + col1: { + label: 'timestamp', + dataType: 'date', + operationType: 'date_histogram', + sourceField: 'timestamp', + isBucketed: true, + scale: 'interval', + params: { + interval: 'auto', + includeEmptyRows: true, + dropPartials: false, + }, + } as DateHistogramIndexPatternColumn, + col2: { + label: '95th percentile of bytes', + dataType: 'number', + operationType: 'percentile', + sourceField: 'bytes', + isBucketed: false, + scale: 'ratio', + params: { + percentile: 95, + }, + } as PercentileIndexPatternColumn, + col3: { + label: 'Count of records', + dataType: 'number', + isBucketed: false, + sourceField: '___records___', + operationType: 'count', + timeScale: 'h', }, - } as PercentileIndexPatternColumn, - col3: { - label: '95th percentile of bytes', - dataType: 'number', - operationType: 'percentile', - sourceField: 'bytes', - isBucketed: false, - scale: 'ratio', - params: { - percentile: 95, + col4: { + label: 'Count of records2', + dataType: 'number', + isBucketed: false, + sourceField: 'bytes', + operationType: 'count', + timeScale: 'h', }, - } as PercentileIndexPatternColumn, + }, + columnOrder: ['col1', 'col2', 'col3', 'col4'], + incompleteColumns: {}, }, - columnOrder: ['col1', 'col2', 'col3'], - incompleteColumns: {}, }, - }, - }; + }; - const optimizeMock = jest.spyOn(operationDefinitionMap.percentile, 'optimizeEsAggs'); + const optimizeMock = jest + .spyOn(operationDefinitionMap.percentile, 'optimizeEsAggs') + .mockImplementation((aggs, esAggsIdMap) => { + // change the order of the aggregations + return { aggs: aggs.reverse(), esAggsIdMap }; + }); - indexPatternDatasource.toExpression(queryBaseState, 'first', indexPatterns); + const ast = indexPatternDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns + ) as Ast; - expect(operationDefinitionMap.percentile.optimizeEsAggs).toHaveBeenCalledTimes(1); + expect(operationDefinitionMap.percentile.optimizeEsAggs).toHaveBeenCalledTimes(1); - optimizeMock.mockRestore(); - }); + const idMap = JSON.parse(ast.chain[2].arguments.idMap as unknown as string); - it('should update anticipated esAggs column IDs based on the order of the optimized agg expression builders', () => { - const queryBaseState: IndexPatternPrivateState = { - currentIndexPatternId: '1', - layers: { - first: { - indexPatternId: '1', - columns: { - col1: { - label: 'timestamp', - dataType: 'date', - operationType: 'date_histogram', - sourceField: 'timestamp', - isBucketed: true, - scale: 'interval', - params: { - interval: 'auto', - includeEmptyRows: true, - dropPartials: false, + expect(Object.keys(idMap)).toEqual(['col-0-3', 'col-1-2', 'col-2-1', 'col-3-0']); + + optimizeMock.mockRestore(); + }); + + it('should deduplicate aggs for supported operations', () => { + const queryBaseState: IndexPatternPrivateState = { + currentIndexPatternId: '1', + layers: { + first: { + indexPatternId: '1', + columns: { + col1: { + label: 'My Op', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + sourceField: 'op', + params: { + size: 5, + orderBy: { + type: 'column', + columnId: 'col4', // col4 will disappear + }, + orderDirection: 'asc', + }, + } as TermsIndexPatternColumn, + col2: { + label: 'Count of records', + dataType: 'number', + isBucketed: false, + sourceField: '___records___', + operationType: 'count', + timeScale: 'h', }, - } as DateHistogramIndexPatternColumn, - col2: { - label: '95th percentile of bytes', - dataType: 'number', - operationType: 'percentile', - sourceField: 'bytes', - isBucketed: false, - scale: 'ratio', - params: { - percentile: 95, + col3: { + label: 'Count of records', + dataType: 'number', + isBucketed: false, + sourceField: '___records___', + operationType: 'count', + timeScale: 'h', + }, + col4: { + label: 'Count of records', + dataType: 'number', + isBucketed: false, + sourceField: '___records___', + operationType: 'count', + timeScale: 'h', }, - } as PercentileIndexPatternColumn, - col3: { - label: 'Count of records', - dataType: 'number', - isBucketed: false, - sourceField: '___records___', - operationType: 'count', - timeScale: 'h', - }, - col4: { - label: 'Count of records2', - dataType: 'number', - isBucketed: false, - sourceField: '___records___', - operationType: 'count', - timeScale: 'h', }, + columnOrder: ['col1', 'col2', 'col3', 'col4'], + incompleteColumns: {}, }, - columnOrder: ['col1', 'col2', 'col3', 'col4'], - incompleteColumns: {}, }, - }, - }; + }; - const optimizeMock = jest - .spyOn(operationDefinitionMap.percentile, 'optimizeEsAggs') - .mockImplementation((aggs, esAggsIdMap) => { - // change the order of the aggregations - return { aggs: aggs.reverse(), esAggsIdMap }; - }); + const ast = indexPatternDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns + ) as Ast; - const ast = indexPatternDatasource.toExpression( - queryBaseState, - 'first', - indexPatterns - ) as Ast; + const idMap = JSON.parse(ast.chain[2].arguments.idMap as unknown as string); - expect(operationDefinitionMap.percentile.optimizeEsAggs).toHaveBeenCalledTimes(1); + const aggs = ast.chain[1].arguments.aggs; - const idMap = JSON.parse(ast.chain[2].arguments.idMap as unknown as string); + expect(aggs).toHaveLength(2); - expect(Object.keys(idMap)).toEqual(['col-0-3', 'col-1-2', 'col-2-1', 'col-3-0']); + // orderby reference updated + expect((aggs[0] as Ast).chain[0].arguments.orderBy[0]).toBe('1'); - optimizeMock.mockRestore(); + expect(idMap).toMatchInlineSnapshot(` + Object { + "col-0-0": Array [ + Object { + "dataType": "string", + "id": "col1", + "isBucketed": true, + "label": "My Op", + "operationType": "terms", + "params": Object { + "orderBy": Object { + "columnId": "col4", + "type": "column", + }, + "orderDirection": "asc", + "size": 5, + }, + "sourceField": "op", + }, + ], + "col-1-1": Array [ + Object { + "dataType": "number", + "id": "col2", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "sourceField": "___records___", + "timeScale": "h", + }, + Object { + "dataType": "number", + "id": "col3", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "sourceField": "___records___", + "timeScale": "h", + }, + Object { + "dataType": "number", + "id": "col4", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "sourceField": "___records___", + "timeScale": "h", + }, + ], + } + `); + }); }); describe('references', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.test.ts new file mode 100644 index 0000000000000..10d26cc3dbbd2 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.test.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildExpression, parseExpression } from '@kbn/expressions-plugin/common'; +import { operationDefinitionMap } from '.'; + +describe('unique values function', () => { + describe('getGroupByKey', () => { + const getKey = operationDefinitionMap.unique_count.getGroupByKey!; + const expressionToKey = (expression: string) => + getKey(buildExpression(parseExpression(expression))) as string; + describe('generates unique keys based on configuration', () => { + const keys = [ + // group 1 + [ + 'aggCardinality id="0" enabled=true schema="metric" field="bytes" emptyAsNull=false', + 'aggCardinality id="1" enabled=true schema="metric" field="bytes" emptyAsNull=false', + ], + // group 2 + [ + 'aggFilteredMetric id="2" enabled=true schema="metric" \n customBucket={aggFilter id="2-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggCardinality id="2-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + 'aggFilteredMetric id="3" enabled=true schema="metric" \n customBucket={aggFilter id="3-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggCardinality id="3-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + ], + // group 3 + [ + 'aggFilteredMetric id="4" enabled=true schema="metric" \n customBucket={aggFilter id="4-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "}} \n customMetric={aggCardinality id="4-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + 'aggFilteredMetric id="5" enabled=true schema="metric" \n customBucket={aggFilter id="5-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "}} \n customMetric={aggCardinality id="5-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + ], + // group 4 + [ + 'aggFilteredMetric id="6" enabled=true schema="metric" \n customBucket={aggFilter id="6-filter" enabled=true schema="bucket" filter={lucene q="\\"geo.dest: \\\\\\"AL\\\\\\" \\""}} \n customMetric={aggCardinality id="6-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + 'aggFilteredMetric id="7" enabled=true schema="metric" \n customBucket={aggFilter id="7-filter" enabled=true schema="bucket" filter={lucene q="\\"geo.dest: \\\\\\"AL\\\\\\" \\""}} \n customMetric={aggCardinality id="7-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + ], + // group 5 + [ + 'aggFilteredMetric id="8" enabled=true schema="metric" \n customBucket={aggFilter id="8-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "} timeWindow="1m"} \n customMetric={aggCardinality id="8-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + 'aggFilteredMetric id="9" enabled=true schema="metric" \n customBucket={aggFilter id="9-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "} timeWindow="1m"} \n customMetric={aggCardinality id="9-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + ], + // check emptyAsNull cases + [ + 'aggCardinality id="10" enabled=true schema="metric" field="bytes" emptyAsNull=true', + 'aggCardinality id="11" enabled=true schema="metric" field="bytes" emptyAsNull=true', + ], + [ + 'aggFilteredMetric id="12" enabled=true schema="metric" \n customBucket={aggFilter id="2-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggCardinality id="2-metric" enabled=true schema="metric" field="bytes" emptyAsNull=true}', + 'aggFilteredMetric id="13" enabled=true schema="metric" \n customBucket={aggFilter id="3-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggCardinality id="3-metric" enabled=true schema="metric" field="bytes" emptyAsNull=true}', + ], + ].map((group) => group.map(expressionToKey)); + + it.each(keys.map((group, i) => ({ group })))('%#', ({ group: thisGroup }) => { + expect(thisGroup[0]).toEqual(thisGroup[1]); + const otherGroups = keys.filter((group) => group !== thisGroup); + for (const otherGroup of otherGroups) { + expect(thisGroup[0]).not.toEqual(otherGroup[0]); + } + }); + + it('snapshot', () => { + expect(keys).toMatchInlineSnapshot(` + Array [ + Array [ + "aggCardinality-bytes-false-undefined", + "aggCardinality-bytes-false-undefined", + ], + Array [ + "aggCardinality-filtered-bytes-false-undefined-undefined-kql-geo.dest: \\"GA\\" ", + "aggCardinality-filtered-bytes-false-undefined-undefined-kql-geo.dest: \\"GA\\" ", + ], + Array [ + "aggCardinality-filtered-bytes-false-undefined-undefined-kql-geo.dest: \\"AL\\" ", + "aggCardinality-filtered-bytes-false-undefined-undefined-kql-geo.dest: \\"AL\\" ", + ], + Array [ + "aggCardinality-filtered-bytes-false-undefined-undefined-lucene-\\"geo.dest: \\\\\\"AL\\\\\\" \\"", + "aggCardinality-filtered-bytes-false-undefined-undefined-lucene-\\"geo.dest: \\\\\\"AL\\\\\\" \\"", + ], + Array [ + "aggCardinality-filtered-bytes-false-1m-undefined-kql-geo.dest: \\"AL\\" ", + "aggCardinality-filtered-bytes-false-1m-undefined-kql-geo.dest: \\"AL\\" ", + ], + Array [ + "aggCardinality-bytes-true-undefined", + "aggCardinality-bytes-true-undefined", + ], + Array [ + "aggCardinality-filtered-bytes-true-undefined-undefined-kql-geo.dest: \\"GA\\" ", + "aggCardinality-filtered-bytes-true-undefined-undefined-kql-geo.dest: \\"GA\\" ", + ], + ] + `); + }); + }); + + it('returns undefined for aggs from different operation classes', () => { + expect( + expressionToKey( + 'aggSum id="0" enabled=true schema="metric" field="bytes" emptyAsNull=false' + ) + ).toBeUndefined(); + expect( + expressionToKey( + 'aggFilteredMetric id="2" enabled=true schema="metric" \n customBucket={aggFilter id="2-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggSum id="2-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}' + ) + ).toBeUndefined(); + }); + }); +}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index 44474fba14dac..b08fbf8f6c89a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -26,6 +26,7 @@ import { adjustTimeScaleLabelSuffix } from '../time_scale_utils'; import { getDisallowedPreviousShiftMessage } from '../../time_shift_utils'; import { updateColumnParam } from '../layer_helpers'; import { getColumnReducedTimeRangeError } from '../../reduced_time_range_utils'; +import { getGroupByKey } from './get_group_by_key'; const supportedTypes = new Set([ 'string', @@ -186,6 +187,13 @@ export const cardinalityOperation: OperationDefinition< emptyAsNull: column.params?.emptyAsNull, }).toAst(); }, + getGroupByKey: (agg) => { + return getGroupByKey( + agg, + ['aggCardinality'], + [{ name: 'field' }, { name: 'emptyAsNull', transformer: (val) => String(Boolean(val)) }] + ); + }, onFieldChange: (oldColumn, field) => { return { ...oldColumn, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.test.ts new file mode 100644 index 0000000000000..9f89834548837 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildExpression, parseExpression } from '@kbn/expressions-plugin/common'; +import { operationDefinitionMap } from '.'; + +describe('count operation', () => { + describe('getGroupByKey', () => { + const getKey = operationDefinitionMap.count.getGroupByKey!; + const expressionToKey = (expression: string) => + getKey(buildExpression(parseExpression(expression))) as string; + + describe('generates unique keys based on configuration', () => { + const keys = [ + [ + 'aggValueCount id="0" enabled=true schema="metric" field="bytes" emptyAsNull=false', + 'aggValueCount id="1" enabled=true schema="metric" field="bytes" emptyAsNull=false', + ], + [ + 'aggFilteredMetric id="2" enabled=true schema="metric" \n customBucket={aggFilter id="2-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggValueCount id="2-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + 'aggFilteredMetric id="3" enabled=true schema="metric" \n customBucket={aggFilter id="3-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggValueCount id="3-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + ], + [ + 'aggFilteredMetric id="4" enabled=true schema="metric" \n customBucket={aggFilter id="4-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "}} \n customMetric={aggValueCount id="4-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + 'aggFilteredMetric id="5" enabled=true schema="metric" \n customBucket={aggFilter id="5-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "}} \n customMetric={aggValueCount id="5-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + ], + [ + 'aggFilteredMetric id="6" enabled=true schema="metric" \n customBucket={aggFilter id="6-filter" enabled=true schema="bucket" filter={lucene q="\\"geo.dest: \\\\\\"AL\\\\\\" \\""}} \n customMetric={aggValueCount id="6-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + 'aggFilteredMetric id="7" enabled=true schema="metric" \n customBucket={aggFilter id="7-filter" enabled=true schema="bucket" filter={lucene q="\\"geo.dest: \\\\\\"AL\\\\\\" \\""}} \n customMetric={aggValueCount id="7-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + ], + [ + 'aggFilteredMetric id="8" enabled=true schema="metric" \n customBucket={aggFilter id="8-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "} timeWindow="1m"} \n customMetric={aggValueCount id="8-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + 'aggFilteredMetric id="9" enabled=true schema="metric" \n customBucket={aggFilter id="9-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "} timeWindow="1m"} \n customMetric={aggValueCount id="9-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + ], + [ + 'aggCount id="10" enabled=true schema="metric" emptyAsNull=true', + 'aggCount id="11" enabled=true schema="metric" emptyAsNull=true', + ], + [ + 'aggCount id="10" enabled=true schema="metric" emptyAsNull=false', + 'aggCount id="11" enabled=true schema="metric" emptyAsNull=false', + ], + [ + 'aggValueCount id="12" enabled=true schema="metric" field="agent.keyword" emptyAsNull=false', + 'aggValueCount id="13" enabled=true schema="metric" field="agent.keyword" emptyAsNull=false', + ], + [ + 'aggValueCount id="12" enabled=true schema="metric" field="agent.keyword" emptyAsNull=true', + 'aggValueCount id="13" enabled=true schema="metric" field="agent.keyword" emptyAsNull=true', + ], + ].map((group) => group.map(expressionToKey)); + + it.each(keys.map((group, i) => ({ group })))('%#', ({ group: thisGroup }) => { + expect(thisGroup[0]).toEqual(thisGroup[1]); + const otherGroups = keys.filter((group) => group !== thisGroup); + for (const otherGroup of otherGroups) { + expect(thisGroup[0]).not.toEqual(otherGroup[0]); + } + }); + + it('snapshot', () => { + expect(keys).toMatchInlineSnapshot(` + Array [ + Array [ + "aggValueCount-bytes-false-undefined", + "aggValueCount-bytes-false-undefined", + ], + Array [ + "aggValueCount-filtered-bytes-false-undefined-undefined-kql-geo.dest: \\"GA\\" ", + "aggValueCount-filtered-bytes-false-undefined-undefined-kql-geo.dest: \\"GA\\" ", + ], + Array [ + "aggValueCount-filtered-bytes-false-undefined-undefined-kql-geo.dest: \\"AL\\" ", + "aggValueCount-filtered-bytes-false-undefined-undefined-kql-geo.dest: \\"AL\\" ", + ], + Array [ + "aggValueCount-filtered-bytes-false-undefined-undefined-lucene-\\"geo.dest: \\\\\\"AL\\\\\\" \\"", + "aggValueCount-filtered-bytes-false-undefined-undefined-lucene-\\"geo.dest: \\\\\\"AL\\\\\\" \\"", + ], + Array [ + "aggValueCount-filtered-bytes-false-1m-undefined-kql-geo.dest: \\"AL\\" ", + "aggValueCount-filtered-bytes-false-1m-undefined-kql-geo.dest: \\"AL\\" ", + ], + Array [ + "aggCount-undefined-true-undefined", + "aggCount-undefined-true-undefined", + ], + Array [ + "aggCount-undefined-false-undefined", + "aggCount-undefined-false-undefined", + ], + Array [ + "aggValueCount-agent.keyword-false-undefined", + "aggValueCount-agent.keyword-false-undefined", + ], + Array [ + "aggValueCount-agent.keyword-true-undefined", + "aggValueCount-agent.keyword-true-undefined", + ], + ] + `); + }); + }); + + it('returns undefined for aggs from different operation classes', () => { + expect( + expressionToKey( + 'aggSum id="0" enabled=true schema="metric" field="bytes" emptyAsNull=false' + ) + ).toBeUndefined(); + expect( + expressionToKey( + 'aggFilteredMetric id="2" enabled=true schema="metric" \n customBucket={aggFilter id="2-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggSum id="2-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}' + ) + ).toBeUndefined(); + }); + }); +}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx index e2eedc5a2d3c6..4c325e604f64c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx @@ -26,6 +26,7 @@ import { adjustTimeScaleLabelSuffix } from '../time_scale_utils'; import { getDisallowedPreviousShiftMessage } from '../../time_shift_utils'; import { updateColumnParam } from '../layer_helpers'; import { getColumnReducedTimeRangeError } from '../../reduced_time_range_utils'; +import { getGroupByKey } from './get_group_by_key'; const countLabel = i18n.translate('xpack.lens.indexPattern.countOf', { defaultMessage: 'Count of records', @@ -206,6 +207,14 @@ export const countOperation: OperationDefinition { + return getGroupByKey( + agg, + ['aggCount', 'aggValueCount'], + [{ name: 'field' }, { name: 'emptyAsNull', transformer: (val) => String(Boolean(val)) }] + ); + }, + isTransferable: (column, newIndexPattern) => { const newField = newIndexPattern.getFieldByName(column.sourceField); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/get_group_by_key.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/get_group_by_key.ts new file mode 100644 index 0000000000000..92bcfdb2e7450 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/get_group_by_key.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + AggFunctionsMapping, + ExpressionFunctionKql, + ExpressionFunctionLucene, +} from '@kbn/data-plugin/public'; +import { + AnyExpressionFunctionDefinition, + ExpressionAstExpressionBuilder, + ExpressionAstFunctionBuilder, +} from '@kbn/expressions-plugin/common'; +import { Primitive } from 'utility-types'; + +/** + * Computes a group-by key for an agg expression builder based on distinctive expression function arguments + */ +export const getGroupByKey = ( + agg: ExpressionAstExpressionBuilder, + aggNames: string[], + importantExpressionArgs: Array<{ name: string; transformer?: (value: Primitive) => string }> +) => { + const { + functions: [fnBuilder], + } = agg; + + const pieces = []; + + if (aggNames.includes(fnBuilder.name)) { + pieces.push(fnBuilder.name); + + importantExpressionArgs.forEach(({ name, transformer }) => { + const arg = fnBuilder.getArgument(name)?.[0]; + pieces.push(transformer ? transformer(arg) : arg); + }); + + pieces.push(fnBuilder.getArgument('timeShift')?.[0]); + } + + if (fnBuilder.name === 'aggFilteredMetric') { + const metricFnBuilder = fnBuilder.getArgument('customMetric')?.[0].functions[0] as + | ExpressionAstFunctionBuilder + | undefined; + + if (metricFnBuilder && aggNames.includes(metricFnBuilder.name)) { + pieces.push(metricFnBuilder.name); + pieces.push('filtered'); + + const aggFilterFnBuilder = ( + fnBuilder.getArgument('customBucket')?.[0] as ExpressionAstExpressionBuilder + ).functions[0] as ExpressionAstFunctionBuilder; + + importantExpressionArgs.forEach(({ name, transformer }) => { + const arg = metricFnBuilder.getArgument(name)?.[0]; + pieces.push(transformer ? transformer(arg) : arg); + }); + + pieces.push(aggFilterFnBuilder.getArgument('timeWindow')); + pieces.push(fnBuilder.getArgument('timeShift')); + + const filterExpression = aggFilterFnBuilder.getArgument('filter')?.[0] as + | ExpressionAstExpressionBuilder + | undefined; + + if (filterExpression) { + const filterFnBuilder = filterExpression.functions[0] as + | ExpressionAstFunctionBuilder + | undefined; + + pieces.push(filterFnBuilder?.name, filterFnBuilder?.getArgument('q')?.[0]); + } + } + } + + return pieces.length ? pieces.map(String).join('-') : undefined; +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 31292218078e2..5ce13adbf5ca8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -408,6 +408,29 @@ interface BaseOperationDefinitionProps< * Title for the help component */ helpComponentTitle?: string; + + /** + * Used to remove duplicate aggregations for performance reasons. + * + * This method should return a key only if the provided agg is generated by this particular operation. + * The key should represent the important configurations of the operation, such that + * + * const column1 = ...; + * const column2 = ...; // different configuration! + * + * const agg1 = operation.toEsAggsFn(column1); + * const agg2 = operation.toEsAggsFn(column1); + * const agg3 = operation.toEsAggsFn(column2); + * + * const key1 = operation.getGroupByKey(agg1); + * const key2 = operation.getGroupByKey(agg2); + * const key3 = operation.getGroupByKey(agg3); + * + * key1 === key2 + * key1 !== key3 + */ + getGroupByKey?: (agg: ExpressionAstExpressionBuilder) => string | undefined; + /** * Optimizes EsAggs expression. Invoked only once per operation type. */ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx index df219c1e851be..05386492544f4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx @@ -21,6 +21,7 @@ import type { IndexPatternLayer } from '../../types'; import type { IndexPattern } from '../../../types'; import { TermsIndexPatternColumn } from './terms'; import { EuiSwitch, EuiSwitchEvent } from '@elastic/eui'; +import { buildExpression, parseExpression } from '@kbn/expressions-plugin/common'; const uiSettingsMock = {} as IUiSettingsClient; @@ -917,4 +918,108 @@ describe('last_value', () => { ]); }); }); + + describe('getGroupByKey', () => { + const getKey = lastValueOperation.getGroupByKey!; + const expressionToKey = (expression: string) => + getKey(buildExpression(parseExpression(expression))); + + describe('collapses duplicate aggs', () => { + const keys = [ + [ + 'aggFilteredMetric id="0" enabled=true schema="metric" \n customBucket={aggFilter id="0-filter" enabled=true schema="bucket" filter={kql q="bytes: *"}} \n customMetric={aggTopMetrics id="0-metric" enabled=true schema="metric" field="bytes" size=1 sortOrder="desc" sortField="timestamp"}', + 'aggFilteredMetric id="1" enabled=true schema="metric" \n customBucket={aggFilter id="1-filter" enabled=true schema="bucket" filter={kql q="bytes: *"}} \n customMetric={aggTopMetrics id="1-metric" enabled=true schema="metric" field="bytes" size=1 sortOrder="desc" sortField="timestamp"}', + ], + [ + 'aggFilteredMetric id="2" enabled=true schema="metric" \n customBucket={aggFilter id="2-filter" enabled=true schema="bucket" filter={kql q="machine.ram: *"}} \n customMetric={aggTopMetrics id="2-metric" enabled=true schema="metric" field="machine.ram" size=1 sortOrder="desc" sortField="timestamp"}', + 'aggFilteredMetric id="3" enabled=true schema="metric" \n customBucket={aggFilter id="3-filter" enabled=true schema="bucket" filter={kql q="machine.ram: *"}} \n customMetric={aggTopMetrics id="3-metric" enabled=true schema="metric" field="machine.ram" size=1 sortOrder="desc" sortField="timestamp"}', + ], + [ + 'aggFilteredMetric id="4" enabled=true schema="metric" \n customBucket={aggFilter id="4-filter" enabled=true schema="bucket" filter={kql q="machine.ram: *"} timeShift="1h"} \n customMetric={aggTopMetrics id="4-metric" enabled=true schema="metric" field="machine.ram" size=1 sortOrder="desc" sortField="timestamp"} timeShift="1h"', + 'aggFilteredMetric id="5" enabled=true schema="metric" \n customBucket={aggFilter id="5-filter" enabled=true schema="bucket" filter={kql q="machine.ram: *"} timeShift="1h"} \n customMetric={aggTopMetrics id="5-metric" enabled=true schema="metric" field="machine.ram" size=1 sortOrder="desc" sortField="timestamp"} timeShift="1h"', + ], + [ + 'aggFilteredMetric id="6" enabled=true schema="metric" \n customBucket={aggFilter id="6-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggTopMetrics id="6-metric" enabled=true schema="metric" field="bytes" size=1 sortOrder="desc" sortField="timestamp"}', + 'aggFilteredMetric id="7" enabled=true schema="metric" \n customBucket={aggFilter id="7-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggTopMetrics id="7-metric" enabled=true schema="metric" field="bytes" size=1 sortOrder="desc" sortField="timestamp"}', + ], + [ + 'aggFilteredMetric id="8" enabled=true schema="metric" \n customBucket={aggFilter id="8-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "}} \n customMetric={aggTopMetrics id="8-metric" enabled=true schema="metric" field="bytes" size=1 sortOrder="desc" sortField="timestamp"}', + 'aggFilteredMetric id="9" enabled=true schema="metric" \n customBucket={aggFilter id="9-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "}} \n customMetric={aggTopMetrics id="9-metric" enabled=true schema="metric" field="bytes" size=1 sortOrder="desc" sortField="timestamp"}', + ], + [ + 'aggFilteredMetric id="10" enabled=true schema="metric" \n customBucket={aggFilter id="10-filter" enabled=true schema="bucket" filter={lucene q="\\"geo.dest: \\\\\\"AL\\\\\\" \\""}} \n customMetric={aggTopMetrics id="10-metric" enabled=true schema="metric" field="bytes" size=1 sortOrder="desc" sortField="timestamp"}', + 'aggFilteredMetric id="11" enabled=true schema="metric" \n customBucket={aggFilter id="11-filter" enabled=true schema="bucket" filter={lucene q="\\"geo.dest: \\\\\\"AL\\\\\\" \\""}} \n customMetric={aggTopMetrics id="11-metric" enabled=true schema="metric" field="bytes" size=1 sortOrder="desc" sortField="timestamp"}', + ], + [ + 'aggFilteredMetric id="12" enabled=true schema="metric" \n customBucket={aggFilter id="12-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "} timeWindow="1m"} \n customMetric={aggTopMetrics id="12-metric" enabled=true schema="metric" field="bytes" size=1 sortOrder="desc" sortField="timestamp"}', + 'aggFilteredMetric id="13" enabled=true schema="metric" \n customBucket={aggFilter id="13-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "} timeWindow="1m"} \n customMetric={aggTopMetrics id="13-metric" enabled=true schema="metric" field="bytes" size=1 sortOrder="desc" sortField="timestamp"}', + ], + // uses aggTopHit + [ + 'aggFilteredMetric id="14" enabled=true schema="metric" \n customBucket={aggFilter id="14-filter" enabled=true schema="bucket" filter={kql q="bytes: *"}} \n customMetric={aggTopHit id="14-metric" enabled=true schema="metric" field="bytes" size=1 sortOrder="desc" sortField="timestamp" aggregate="concat"}', + 'aggFilteredMetric id="15" enabled=true schema="metric" \n customBucket={aggFilter id="15-filter" enabled=true schema="bucket" filter={kql q="bytes: *"}} \n customMetric={aggTopHit id="15-metric" enabled=true schema="metric" field="bytes" size=1 sortOrder="desc" sortField="timestamp" aggregate="concat"}', + ], + ].map((group) => group.map(expressionToKey)); + + it.each(keys.map((group, i) => ({ group })))('%#', ({ group: thisGroup }) => { + expect(thisGroup[0]).toEqual(thisGroup[1]); + const otherGroups = keys.filter((group) => group !== thisGroup); + for (const otherGroup of otherGroups) { + expect(thisGroup[0]).not.toEqual(otherGroup[0]); + } + }); + + it('snapshot', () => { + expect(keys).toMatchInlineSnapshot(` + Array [ + Array [ + "aggTopMetrics-filtered-bytes-timestamp-undefined-undefined-kql-bytes: *", + "aggTopMetrics-filtered-bytes-timestamp-undefined-undefined-kql-bytes: *", + ], + Array [ + "aggTopMetrics-filtered-machine.ram-timestamp-undefined-undefined-kql-machine.ram: *", + "aggTopMetrics-filtered-machine.ram-timestamp-undefined-undefined-kql-machine.ram: *", + ], + Array [ + "aggTopMetrics-filtered-machine.ram-timestamp-undefined-1h-kql-machine.ram: *", + "aggTopMetrics-filtered-machine.ram-timestamp-undefined-1h-kql-machine.ram: *", + ], + Array [ + "aggTopMetrics-filtered-bytes-timestamp-undefined-undefined-kql-geo.dest: \\"GA\\" ", + "aggTopMetrics-filtered-bytes-timestamp-undefined-undefined-kql-geo.dest: \\"GA\\" ", + ], + Array [ + "aggTopMetrics-filtered-bytes-timestamp-undefined-undefined-kql-geo.dest: \\"AL\\" ", + "aggTopMetrics-filtered-bytes-timestamp-undefined-undefined-kql-geo.dest: \\"AL\\" ", + ], + Array [ + "aggTopMetrics-filtered-bytes-timestamp-undefined-undefined-lucene-\\"geo.dest: \\\\\\"AL\\\\\\" \\"", + "aggTopMetrics-filtered-bytes-timestamp-undefined-undefined-lucene-\\"geo.dest: \\\\\\"AL\\\\\\" \\"", + ], + Array [ + "aggTopMetrics-filtered-bytes-timestamp-1m-undefined-kql-geo.dest: \\"AL\\" ", + "aggTopMetrics-filtered-bytes-timestamp-1m-undefined-kql-geo.dest: \\"AL\\" ", + ], + Array [ + "aggTopHit-filtered-bytes-timestamp-undefined-undefined-kql-bytes: *", + "aggTopHit-filtered-bytes-timestamp-undefined-undefined-kql-bytes: *", + ], + ] + `); + }); + }); + + it('returns undefined for aggs from different operation classes', () => { + expect( + expressionToKey( + 'aggSum id="0" enabled=true schema="metric" field="bytes" emptyAsNull=false' + ) + ).toBeUndefined(); + expect( + expressionToKey( + 'aggFilteredMetric id="2" enabled=true schema="metric" \n customBucket={aggFilter id="2-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggSum id="2-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}' + ) + ).toBeUndefined(); + }); + }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index b7fd3a40fdbff..2709d22b2a323 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -33,6 +33,7 @@ import { getDisallowedPreviousShiftMessage } from '../../time_shift_utils'; import { isScriptedField } from './terms/helpers'; import { FormRow } from './shared_components/form_row'; import { getColumnReducedTimeRangeError } from '../../reduced_time_range_utils'; +import { getGroupByKey } from './get_group_by_key'; function ofName(name: string, timeShift: string | undefined, reducedTimeRange: string | undefined) { return adjustTimeScaleLabelSuffix( @@ -258,6 +259,14 @@ export const lastValueOperation: OperationDefinition< ).toAst(); }, + getGroupByKey: (agg) => { + return getGroupByKey( + agg, + ['aggTopHit', 'aggTopMetrics'], + [{ name: 'field' }, { name: 'sortField' }] + ); + }, + isTransferable: (column, newIndexPattern) => { const newField = newIndexPattern.getFieldByName(column.sourceField); const newTimeField = newIndexPattern.getFieldByName(column.params.sortField); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.test.ts new file mode 100644 index 0000000000000..b98125d0bbd2b --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.test.ts @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildExpression, parseExpression } from '@kbn/expressions-plugin/common'; +import { operationDefinitionMap } from '.'; + +const sumOperation = operationDefinitionMap.sum; + +describe('metrics', () => { + describe('getGroupByKey', () => { + const getKey = sumOperation.getGroupByKey!; + const expressionToKey = (expression: string) => + getKey(buildExpression(parseExpression(expression))); + + describe('should collapse filtered aggs with matching parameters', () => { + const keys = [ + [ + 'aggSum id="0" enabled=true schema="metric" field="bytes" emptyAsNull=false', + 'aggSum id="1" enabled=true schema="metric" field="bytes" emptyAsNull=false', + ], + [ + 'aggSum id="2" enabled=true schema="metric" field="bytes" emptyAsNull=true', + 'aggSum id="3" enabled=true schema="metric" field="bytes" emptyAsNull=true', + ], + [ + 'aggSum id="4" enabled=true schema="metric" field="hour_of_day" emptyAsNull=false', + 'aggSum id="5" enabled=true schema="metric" field="hour_of_day" emptyAsNull=false', + ], + [ + 'aggSum id="6" enabled=true schema="metric" field="machine.ram" timeShift="1h" emptyAsNull=false', + 'aggSum id="7" enabled=true schema="metric" field="machine.ram" timeShift="1h" emptyAsNull=false', + ], + [ + 'aggSum id="8" enabled=true schema="metric" field="machine.ram" timeShift="2h" emptyAsNull=false', + 'aggSum id="9" enabled=true schema="metric" field="machine.ram" timeShift="2h" emptyAsNull=false', + ], + [ + 'aggFilteredMetric id="10" enabled=true schema="metric" \n customBucket={aggFilter id="2-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggSum id="2-metric" enabled=true schema="metric" field="bytes" emptyAsNull=true}', + 'aggFilteredMetric id="11" enabled=true schema="metric" \n customBucket={aggFilter id="3-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggSum id="3-metric" enabled=true schema="metric" field="bytes" emptyAsNull=true}', + ], + [ + 'aggFilteredMetric id="12" enabled=true schema="metric" \n customBucket={aggFilter id="2-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggSum id="2-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + 'aggFilteredMetric id="13" enabled=true schema="metric" \n customBucket={aggFilter id="3-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggSum id="3-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + ], + [ + 'aggFilteredMetric id="14" enabled=true schema="metric" \n customBucket={aggFilter id="4-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "}} \n customMetric={aggSum id="4-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + 'aggFilteredMetric id="15" enabled=true schema="metric" \n customBucket={aggFilter id="5-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "}} \n customMetric={aggSum id="5-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + ], + [ + 'aggFilteredMetric id="16" enabled=true schema="metric" \n customBucket={aggFilter id="6-filter" enabled=true schema="bucket" filter={lucene q="\\"geo.dest: \\\\\\"AL\\\\\\" \\""}} \n customMetric={aggSum id="6-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + 'aggFilteredMetric id="17" enabled=true schema="metric" \n customBucket={aggFilter id="7-filter" enabled=true schema="bucket" filter={lucene q="\\"geo.dest: \\\\\\"AL\\\\\\" \\""}} \n customMetric={aggSum id="7-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + ], + [ + 'aggFilteredMetric id="18" enabled=true schema="metric" \n customBucket={aggFilter id="8-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "} timeWindow="1m"} \n customMetric={aggSum id="8-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + 'aggFilteredMetric id="19" enabled=true schema="metric" \n customBucket={aggFilter id="9-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"AL\\" "} timeWindow="1m"} \n customMetric={aggSum id="9-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}', + ], + ].map((group) => group.map(expressionToKey)); + + it.each(keys.map((group, i) => ({ group })))('%#', ({ group: thisGroup }) => { + expect(thisGroup[0]).toEqual(thisGroup[1]); + const otherGroups = keys.filter((group) => group !== thisGroup); + for (const otherGroup of otherGroups) { + expect(thisGroup[0]).not.toEqual(otherGroup[0]); + } + }); + + it('snapshot', () => { + expect(keys).toMatchInlineSnapshot(` + Array [ + Array [ + "aggSum-bytes-false-undefined", + "aggSum-bytes-false-undefined", + ], + Array [ + "aggSum-bytes-true-undefined", + "aggSum-bytes-true-undefined", + ], + Array [ + "aggSum-hour_of_day-false-undefined", + "aggSum-hour_of_day-false-undefined", + ], + Array [ + "aggSum-machine.ram-false-1h", + "aggSum-machine.ram-false-1h", + ], + Array [ + "aggSum-machine.ram-false-2h", + "aggSum-machine.ram-false-2h", + ], + Array [ + "aggSum-filtered-bytes-true-undefined-undefined-kql-geo.dest: \\"GA\\" ", + "aggSum-filtered-bytes-true-undefined-undefined-kql-geo.dest: \\"GA\\" ", + ], + Array [ + "aggSum-filtered-bytes-false-undefined-undefined-kql-geo.dest: \\"GA\\" ", + "aggSum-filtered-bytes-false-undefined-undefined-kql-geo.dest: \\"GA\\" ", + ], + Array [ + "aggSum-filtered-bytes-false-undefined-undefined-kql-geo.dest: \\"AL\\" ", + "aggSum-filtered-bytes-false-undefined-undefined-kql-geo.dest: \\"AL\\" ", + ], + Array [ + "aggSum-filtered-bytes-false-undefined-undefined-lucene-\\"geo.dest: \\\\\\"AL\\\\\\" \\"", + "aggSum-filtered-bytes-false-undefined-undefined-lucene-\\"geo.dest: \\\\\\"AL\\\\\\" \\"", + ], + Array [ + "aggSum-filtered-bytes-false-1m-undefined-kql-geo.dest: \\"AL\\" ", + "aggSum-filtered-bytes-false-1m-undefined-kql-geo.dest: \\"AL\\" ", + ], + ] + `); + }); + }); + + it('returns undefined for aggs from different operation classes', () => { + expect( + expressionToKey( + 'aggCardinality id="0" enabled=true schema="metric" field="bytes" emptyAsNull=false' + ) + ).toBeUndefined(); + expect( + expressionToKey( + 'aggFilteredMetric id="2" enabled=true schema="metric" \n customBucket={aggFilter id="2-filter" enabled=true schema="bucket" filter={kql q="geo.dest: \\"GA\\" "}} \n customMetric={aggCardinality id="2-metric" enabled=true schema="metric" field="bytes" emptyAsNull=false}' + ) + ).toBeUndefined(); + }); + }); +}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx index 9b8f790e51b90..09dc8576a042a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx @@ -28,6 +28,7 @@ import { adjustTimeScaleLabelSuffix } from '../time_scale_utils'; import { getDisallowedPreviousShiftMessage } from '../../time_shift_utils'; import { updateColumnParam } from '../layer_helpers'; import { getColumnReducedTimeRangeError } from '../../reduced_time_range_utils'; +import { getGroupByKey } from './get_group_by_key'; type MetricColumn = FieldBasedIndexPatternColumn & { operationType: T; @@ -198,6 +199,14 @@ function buildMetricOperation>({ ...aggConfigParams, }).toAst(); }, + getGroupByKey: (agg) => { + return getGroupByKey( + agg, + [typeToFn[type]], + [{ name: 'field' }, { name: 'emptyAsNull', transformer: (val) => String(Boolean(val)) }] + ); + }, + getErrorMessage: (layer, columnId, indexPattern) => combineErrorMessages([ getInvalidFieldMessage( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx index f72342f79bfc6..77434d7ecc7ad 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx @@ -28,6 +28,7 @@ import { } from '@kbn/expressions-plugin/public'; import type { OriginalColumn } from '../../to_expression'; import { IndexPattern } from '../../../types'; +import faker from 'faker'; jest.mock('lodash', () => { const original = jest.requireActual('lodash'); @@ -234,7 +235,7 @@ describe('percentile', () => { const aggs = [ // group 1 makeEsAggBuilder('aggSinglePercentile', { - id: 1, + id: '1', enabled: true, schema: 'metric', field: field1, @@ -242,7 +243,7 @@ describe('percentile', () => { timeShift: undefined, }), makeEsAggBuilder('aggSinglePercentile', { - id: 2, + id: '2', enabled: true, schema: 'metric', field: field1, @@ -250,7 +251,7 @@ describe('percentile', () => { timeShift: undefined, }), makeEsAggBuilder('aggSinglePercentile', { - id: 3, + id: '3', enabled: true, schema: 'metric', field: field1, @@ -259,7 +260,7 @@ describe('percentile', () => { }), // group 2 makeEsAggBuilder('aggSinglePercentile', { - id: 4, + id: '4', enabled: true, schema: 'metric', field: field2, @@ -267,7 +268,7 @@ describe('percentile', () => { timeShift: undefined, }), makeEsAggBuilder('aggSinglePercentile', { - id: 5, + id: '5', enabled: true, schema: 'metric', field: field2, @@ -276,7 +277,7 @@ describe('percentile', () => { }), // group 3 makeEsAggBuilder('aggSinglePercentile', { - id: 6, + id: '6', enabled: true, schema: 'metric', field: field2, @@ -284,7 +285,7 @@ describe('percentile', () => { timeShift: timeShift1, }), makeEsAggBuilder('aggSinglePercentile', { - id: 7, + id: '7', enabled: true, schema: 'metric', field: field2, @@ -293,7 +294,7 @@ describe('percentile', () => { }), // group 4 makeEsAggBuilder('aggSinglePercentile', { - id: 8, + id: '8', enabled: true, schema: 'metric', field: field2, @@ -301,7 +302,7 @@ describe('percentile', () => { timeShift: timeShift2, }), makeEsAggBuilder('aggSinglePercentile', { - id: 9, + id: '9', enabled: true, schema: 'metric', field: field2, @@ -344,7 +345,7 @@ describe('percentile', () => { "foo", ], "id": Array [ - 1, + "1", ], "percents": Array [ 10, @@ -381,7 +382,7 @@ describe('percentile', () => { "bar", ], "id": Array [ - 4, + "4", ], "percents": Array [ 10, @@ -417,7 +418,7 @@ describe('percentile', () => { "bar", ], "id": Array [ - 6, + "6", ], "percents": Array [ 50, @@ -456,7 +457,7 @@ describe('percentile', () => { "bar", ], "id": Array [ - 8, + "8", ], "percents": Array [ 70, @@ -544,7 +545,7 @@ describe('percentile', () => { const aggs = [ // group 1 makeEsAggBuilder('aggSinglePercentile', { - id: 1, + id: '1', enabled: true, schema: 'metric', field: field1, @@ -552,7 +553,7 @@ describe('percentile', () => { timeShift: undefined, }), makeEsAggBuilder('aggSinglePercentile', { - id: 2, + id: '2', enabled: true, schema: 'metric', field: field1, @@ -560,7 +561,7 @@ describe('percentile', () => { timeShift: undefined, }), makeEsAggBuilder('aggSinglePercentile', { - id: 4, + id: '4', enabled: true, schema: 'metric', field: field2, @@ -568,7 +569,7 @@ describe('percentile', () => { timeShift: undefined, }), makeEsAggBuilder('aggSinglePercentile', { - id: 3, + id: '3', enabled: true, schema: 'metric', field: field1, @@ -609,17 +610,87 @@ describe('percentile', () => { `); }); + it('should update order-by references for any terms columns', () => { + const field1 = 'foo'; + const field2 = 'bar'; + const percentile = faker.random.number(100); + + const aggs = [ + makeEsAggBuilder('aggTerms', { + id: '1', + enabled: true, + schema: 'metric', + field: field1, + orderBy: '4', + timeShift: undefined, + }), + makeEsAggBuilder('aggTerms', { + id: '2', + enabled: true, + schema: 'metric', + field: field1, + orderBy: '6', + timeShift: undefined, + }), + makeEsAggBuilder('aggSinglePercentile', { + id: '3', + enabled: true, + schema: 'metric', + field: field1, + percentile, + timeShift: undefined, + }), + makeEsAggBuilder('aggSinglePercentile', { + id: '4', + enabled: true, + schema: 'metric', + field: field1, + percentile, + timeShift: undefined, + }), + makeEsAggBuilder('aggSinglePercentile', { + id: '5', + enabled: true, + schema: 'metric', + field: field2, + percentile, + timeShift: undefined, + }), + makeEsAggBuilder('aggSinglePercentile', { + id: '6', + enabled: true, + schema: 'metric', + field: field2, + percentile, + timeShift: undefined, + }), + ]; + + const { esAggsIdMap, aggsToIdsMap } = buildMapsFromAggBuilders(aggs); + + const { aggs: newAggs } = percentileOperation.optimizeEsAggs!( + aggs, + esAggsIdMap, + aggsToIdsMap + ); + + expect(newAggs.length).toBe(4); + + expect(newAggs[0].functions[0].getArgument('orderBy')?.[0]).toBe(`3.${percentile}`); + expect(newAggs[1].functions[0].getArgument('orderBy')?.[0]).toBe(`5.${percentile}`); + }); + it("shouldn't touch non-percentile aggs or single percentiles with no siblings", () => { const aggs = [ makeEsAggBuilder('aggSinglePercentile', { - id: 1, + id: '1', enabled: true, schema: 'metric', field: 'foo', percentile: 30, }), makeEsAggBuilder('aggMax', { - id: 1, + id: '1', enabled: true, schema: 'metric', field: 'bar', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 88b941b2bb7c9..3bc91b91ed3d6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -13,6 +13,7 @@ import { buildExpression, buildExpressionFunction, ExpressionAstExpressionBuilder, + ExpressionAstFunctionBuilder, } from '@kbn/expressions-plugin/public'; import { AggExpressionFunctionArgs } from '@kbn/data-plugin/common'; import { OperationDefinition } from '.'; @@ -195,6 +196,12 @@ export const percentileOperation: OperationDefinition< } }); + const termsFuncs = aggs + .map((agg) => agg.functions[0]) + .filter((func) => func.name === 'aggTerms') as Array< + ExpressionAstFunctionBuilder + >; + // collapse them into a single esAggs expression builder Object.values(percentileExpressionsByArgs).forEach((expressionBuilders) => { if (expressionBuilders.length <= 1) { @@ -224,6 +231,7 @@ export const percentileOperation: OperationDefinition< const percentileToBuilder: Record = {}; for (const builder of expressionBuilders) { const percentile = builder.functions[0].getArgument('percentile')![0] as number; + if (percentile in percentileToBuilder) { // found a duplicate percentile so let's optimize @@ -248,6 +256,13 @@ export const percentileOperation: OperationDefinition< percentileToBuilder[percentile] = builder; aggPercentilesConfig.percents!.push(percentile); } + + // update any terms order-bys + termsFuncs.forEach((func) => { + if (func.getArgument('orderBy')?.[0] === builder.functions[0].getArgument('id')?.[0]) { + func.replaceArgument('orderBy', [`${esAggsColumnId}.${percentile}`]); + } + }); } const multiPercentilesAst = buildExpressionFunction( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.test.ts index 703156418b364..c544121a935f5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.test.ts @@ -16,11 +16,9 @@ import { getDisallowedTermsMessage, getMultiTermsScriptedFieldErrorMessage, isSortableByColumn, - computeOrderForMultiplePercentiles, } from './helpers'; import { ReferenceBasedIndexPatternColumn } from '../column_types'; import type { PercentileRanksIndexPatternColumn } from '../percentile_ranks'; -import type { PercentileIndexPatternColumn } from '../percentile'; import { MULTI_KEY_VISUAL_SEPARATOR } from './constants'; import { MovingAverageIndexPatternColumn } from '../calculations'; @@ -410,138 +408,6 @@ describe('getDisallowedTermsMessage()', () => { }); }); -describe('computeOrderForMultiplePercentiles()', () => { - it('should return null for no percentile orderColumn', () => { - expect( - computeOrderForMultiplePercentiles( - { - label: 'Percentile rank (1024.5) of bytes', - dataType: 'number', - operationType: 'percentile_rank', - sourceField: 'bytes', - isBucketed: false, - scale: 'ratio', - params: { value: 1024.5 }, - } as PercentileRanksIndexPatternColumn, - getLayer(getStringBasedOperationColumn(), [ - { - label: 'Percentile rank (1024.5) of bytes', - dataType: 'number', - operationType: 'percentile_rank', - sourceField: 'bytes', - isBucketed: false, - scale: 'ratio', - params: { value: 1024.5 }, - } as PercentileRanksIndexPatternColumn, - ]), - ['col1', 'col2'] - ) - ).toBeNull(); - }); - - it('should return null for single percentile', () => { - expect( - computeOrderForMultiplePercentiles( - { - label: 'Percentile 95 of bytes', - dataType: 'number', - operationType: 'percentile', - sourceField: 'bytes', - isBucketed: false, - scale: 'ratio', - params: { percentile: 95 }, - } as PercentileIndexPatternColumn, - getLayer(getStringBasedOperationColumn(), [ - { - label: 'Percentile 95 of bytes', - dataType: 'number', - operationType: 'percentile', - sourceField: 'bytes', - isBucketed: false, - scale: 'ratio', - params: { percentile: 95 }, - } as PercentileIndexPatternColumn, - ]), - ['col1', 'col2'] - ) - ).toBeNull(); - }); - - it('should return correct orderBy for multiple percentile on the same field', () => { - expect( - computeOrderForMultiplePercentiles( - { - label: 'Percentile 95 of bytes', - dataType: 'number', - operationType: 'percentile', - sourceField: 'bytes', - isBucketed: false, - scale: 'ratio', - params: { percentile: 95 }, - } as PercentileIndexPatternColumn, - getLayer(getStringBasedOperationColumn(), [ - { - label: 'Percentile 95 of bytes', - dataType: 'number', - operationType: 'percentile', - sourceField: 'bytes', - isBucketed: false, - scale: 'ratio', - params: { percentile: 95 }, - } as PercentileIndexPatternColumn, - { - label: 'Percentile 65 of bytes', - dataType: 'number', - operationType: 'percentile', - sourceField: 'bytes', - isBucketed: false, - scale: 'ratio', - params: { percentile: 65 }, - } as PercentileIndexPatternColumn, - ]), - ['col1', 'col2', 'col3'] - ) - ).toBe('1.95'); - }); - - it('should return null for multiple percentile on different field', () => { - expect( - computeOrderForMultiplePercentiles( - { - label: 'Percentile 95 of bytes', - dataType: 'number', - operationType: 'percentile', - sourceField: 'bytes', - isBucketed: false, - scale: 'ratio', - params: { percentile: 95 }, - } as PercentileIndexPatternColumn, - getLayer(getStringBasedOperationColumn(), [ - { - label: 'Percentile 95 of bytes', - dataType: 'number', - operationType: 'percentile', - sourceField: 'bytes', - isBucketed: false, - scale: 'ratio', - params: { percentile: 95 }, - } as PercentileIndexPatternColumn, - { - label: 'Percentile 65 of geo', - dataType: 'number', - operationType: 'percentile', - sourceField: 'geo', - isBucketed: false, - scale: 'ratio', - params: { percentile: 65 }, - } as PercentileIndexPatternColumn, - ]), - ['col1', 'col2', 'col3'] - ) - ).toBeNull(); - }); -}); - describe('isSortableByColumn()', () => { it('should sort by the given column', () => { expect( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.ts index 7139a8effd4ca..77d52664c0e54 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.ts @@ -21,7 +21,6 @@ import type { FiltersIndexPatternColumn } from '..'; import type { TermsIndexPatternColumn } from './types'; import type { LastValueIndexPatternColumn } from '../last_value'; import type { PercentileRanksIndexPatternColumn } from '../percentile_ranks'; -import type { PercentileIndexPatternColumn } from '../percentile'; import type { IndexPatternLayer } from '../../../types'; import { MULTI_KEY_VISUAL_SEPARATOR, supportedTypes } from './constants'; @@ -241,31 +240,6 @@ export function isPercentileRankSortable(column: GenericIndexPatternColumn) { ); } -export function computeOrderForMultiplePercentiles( - column: GenericIndexPatternColumn, - layer: IndexPatternLayer, - orderedColumnIds: string[] -) { - // compute the percentiles orderBy correctly for multiple percentiles - if (column.operationType === 'percentile') { - const percentileColumns = []; - for (const [key, value] of Object.entries(layer.columns)) { - if ( - value.operationType === 'percentile' && - (value as PercentileIndexPatternColumn).sourceField === - (column as PercentileIndexPatternColumn).sourceField - ) { - percentileColumns.push(key); - } - } - if (percentileColumns.length > 1) { - const parentColumn = String(orderedColumnIds.indexOf(percentileColumns[0])); - return `${parentColumn}.${(column as PercentileIndexPatternColumn).params?.percentile}`; - } - } - return null; -} - export function isSortableByColumn(layer: IndexPatternLayer, columnId: string) { const column = layer.columns[columnId]; return ( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 3f8496d31f678..9017e91bff089 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -49,7 +49,6 @@ import { getFieldsByValidationState, isSortableByColumn, isPercentileRankSortable, - computeOrderForMultiplePercentiles, } from './helpers'; import { DEFAULT_MAX_DOC_COUNT, @@ -266,7 +265,7 @@ export const termsOperation: OperationDefinition< max_doc_count: column.params.orderBy.maxDocCount, }).toAst(); } - let orderBy: string = '_key'; + let orderBy = '_key'; if (column.params?.orderBy.type === 'column') { const orderColumn = layer.columns[column.params.orderBy.columnId]; @@ -275,14 +274,6 @@ export const termsOperation: OperationDefinition< if (!isPercentileRankSortable(orderColumn)) { orderBy = '_key'; } - - const orderByMultiplePercentiles = computeOrderForMultiplePercentiles( - orderColumn, - layer, - orderedColumnIds - ); - - orderBy = orderByMultiplePercentiles ?? orderBy; } // To get more accurate results, we set shard_size to a minimum of 1000 diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index 8c3b7f90d57d3..584498189fd06 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -350,49 +350,6 @@ describe('terms', () => { ); }); - it('should reflect correct orderBy for multiple percentiles', () => { - const newLayer = { - ...layer, - columns: { - ...layer.columns, - col2: { - ...layer.columns.col2, - operationType: 'percentile', - params: { - percentile: 95, - }, - }, - col3: { - ...layer.columns.col2, - operationType: 'percentile', - params: { - percentile: 65, - }, - }, - }, - }; - const termsColumn = layer.columns.col1 as TermsIndexPatternColumn; - const esAggsFn = termsOperation.toEsAggsFn( - { - ...termsColumn, - params: { ...termsColumn.params, orderBy: { type: 'column', columnId: 'col3' } }, - }, - 'col1', - {} as IndexPattern, - newLayer, - uiSettingsMock, - ['col1', 'col2', 'col3'] - ); - expect(esAggsFn).toEqual( - expect.objectContaining({ - function: 'aggTerms', - arguments: expect.objectContaining({ - orderBy: ['1.65'], - }), - }) - ); - }); - it('should not enable missing bucket if other bucket is not set', () => { const termsColumn = layer.columns.col1 as TermsIndexPatternColumn; const esAggsFn = termsOperation.toEsAggsFn( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index bf816e5891486..72cb2a2ab729e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -27,6 +27,7 @@ import { DateHistogramIndexPatternColumn, RangeIndexPatternColumn } from './oper import { FormattedIndexPatternColumn } from './operations/definitions/column_types'; import { isColumnFormatted, isColumnOfType } from './operations/definitions/helpers'; import type { IndexPattern, IndexPatternMap } from '../types'; +import { dedupeAggs } from './dedupe_aggs'; export type OriginalColumn = { id: string } & GenericIndexPatternColumn; @@ -40,7 +41,7 @@ declare global { } // esAggs column ID manipulation functions -const extractEsAggId = (id: string) => id.split('.')[0].split('-')[2]; +export const extractAggId = (id: string) => id.split('.')[0].split('-')[2]; const updatePositionIndex = (currentId: string, newIndex: number) => { const [fullId, percentile] = currentId.split('.'); const idParts = fullId.split('-'); @@ -214,10 +215,18 @@ function getExpressionForLayer( ); } - uniq(esAggEntries.map(([_, column]) => column.operationType)).forEach((type) => { - const optimizeAggs = operationDefinitionMap[type].optimizeEsAggs?.bind( - operationDefinitionMap[type] - ); + const allOperations = uniq( + esAggEntries.map(([_, column]) => operationDefinitionMap[column.operationType]) + ); + + // De-duplicate aggs for supported operations + const dedupedResult = dedupeAggs(aggs, esAggsIdMap, aggExpressionToEsAggsIdMap, allOperations); + aggs = dedupedResult.aggs; + esAggsIdMap = dedupedResult.esAggsIdMap; + + // Apply any operation-specific custom optimizations + allOperations.forEach((operation) => { + const optimizeAggs = operation.optimizeEsAggs?.bind(operation); if (optimizeAggs) { const { aggs: newAggs, esAggsIdMap: newIdMap } = optimizeAggs( aggs, @@ -257,7 +266,7 @@ function getExpressionForLayer( const esAggsIds = Object.keys(esAggsIdMap); aggs.forEach((builder) => { const esAggId = builder.functions[0].getArgument('id')?.[0]; - const matchingEsAggColumnIds = esAggsIds.filter((id) => extractEsAggId(id) === esAggId); + const matchingEsAggColumnIds = esAggsIds.filter((id) => extractAggId(id) === esAggId); matchingEsAggColumnIds.forEach((currentId) => { const currentColumn = esAggsIdMap[currentId][0]; From 0edfb0b1f3407d9802069a2ddfddbed5f6597dfd Mon Sep 17 00:00:00 2001 From: Kaarina Tungseth Date: Mon, 26 Sep 2022 14:34:32 -0500 Subject: [PATCH 2/6] [DOCS] Moves advanced settings page (#141834) --- docs/user/management.asciidoc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/user/management.asciidoc b/docs/user/management.asciidoc index 908cdc792431c..02261d062e826 100644 --- a/docs/user/management.asciidoc +++ b/docs/user/management.asciidoc @@ -176,8 +176,6 @@ see the https://www.elastic.co/subscriptions[subscription page]. -- -include::{kib-repo-dir}/management/advanced-options.asciidoc[] - include::{kib-repo-dir}/management/cases/index.asciidoc[] include::{kib-repo-dir}/management/action-types.asciidoc[] @@ -196,6 +194,10 @@ include::security/index.asciidoc[] include::{kib-repo-dir}/spaces/index.asciidoc[] +include::{kib-repo-dir}/management/advanced-options.asciidoc[] + include::{kib-repo-dir}/management/managing-tags.asciidoc[] include::{kib-repo-dir}/management/watcher-ui/index.asciidoc[] + + From c494c0402f6bbd2f8b12fcca572ba96ce541a2f8 Mon Sep 17 00:00:00 2001 From: Spencer Date: Mon, 26 Sep 2022 14:50:28 -0500 Subject: [PATCH 3/6] [stack_functional_integration] remove esArchiver service (#141842) --- .../services/es_archiver.ts | 1 + .../lib/config/schema.ts | 7 ++++ ...onfig.stack_functional_integration_base.js | 9 +++-- .../services/es_archiver.js | 40 ------------------- .../services/index.js | 14 ------- 5 files changed, 14 insertions(+), 57 deletions(-) delete mode 100644 x-pack/test/stack_functional_integration/services/es_archiver.js delete mode 100644 x-pack/test/stack_functional_integration/services/index.js diff --git a/packages/kbn-ftr-common-functional-services/services/es_archiver.ts b/packages/kbn-ftr-common-functional-services/services/es_archiver.ts index 8a81297bf1784..abb0b89544bc1 100644 --- a/packages/kbn-ftr-common-functional-services/services/es_archiver.ts +++ b/packages/kbn-ftr-common-functional-services/services/es_archiver.ts @@ -18,6 +18,7 @@ export function EsArchiverProvider({ getService }: FtrProviderContext): EsArchiv const retry = getService('retry'); const esArchiver = new EsArchiver({ + baseDir: config.get('esArchiver.baseDirectory'), client, log, kbnClient: kibanaServer, diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index ce44dd3cc0496..6d5dc75b8d969 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -260,6 +260,13 @@ export const schema = Joi.object() // definition of apps that work with `common.navigateToApp()` apps: Joi.object().pattern(ID_PATTERN, appUrlPartsSchema()).default(), + // settings for the saved objects svc + esArchiver: Joi.object() + .keys({ + baseDirectory: Joi.string().optional(), + }) + .default(), + // settings for the saved objects svc kbnArchiver: Joi.object() .keys({ diff --git a/x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js b/x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js index 1658bcbf6cd35..00dd89acbc9ef 100644 --- a/x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js +++ b/x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js @@ -12,7 +12,6 @@ import { REPO_ROOT } from '@kbn/utils'; import chalk from 'chalk'; import { esTestConfig, kbnTestConfig } from '@kbn/test'; import { TriggersActionsPageProvider } from '../../functional_with_es_ssl/page_objects/triggers_actions_ui_page'; -import { services } from '../services'; const log = new ToolingLog({ level: 'info', @@ -28,13 +27,14 @@ export default async ({ readConfigFile }) => { const xpackFunctionalConfig = await readConfigFile( require.resolve('../../functional/config.base.js') ); - const externalConf = consumeState(resolve(__dirname, stateFilePath)); + const externalConf = consumeState(resolve(__dirname, stateFilePath)) ?? { + TESTS_LIST: 'alerts', + }; process.env.stack_functional_integration = true; logAll(log); const settings = { ...xpackFunctionalConfig.getAll(), - services, pageObjects: { triggersActionsUI: TriggersActionsPageProvider, ...xpackFunctionalConfig.get('pageObjects'), @@ -53,6 +53,9 @@ export default async ({ readConfigFile }) => { ...xpackFunctionalConfig.get('kbnTestServer'), serverArgs: [...xpackFunctionalConfig.get('kbnTestServer.serverArgs')], }, + esArchiver: { + baseDirectory: INTEGRATION_TEST_ROOT, + }, testFiles: tests(externalConf.TESTS_LIST).map(prepend).map(logTest), // testFiles: ['alerts'].map(prepend).map(logTest), // If we need to do things like disable animations, we can do it in configure_start_kibana.sh, in the provisioner...which lives in the integration-test private repo diff --git a/x-pack/test/stack_functional_integration/services/es_archiver.js b/x-pack/test/stack_functional_integration/services/es_archiver.js deleted file mode 100644 index 821cf72e2c6bc..0000000000000 --- a/x-pack/test/stack_functional_integration/services/es_archiver.js +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import Path from 'path'; - -import { EsArchiver } from '@kbn/es-archiver'; -import { REPO_ROOT } from '@kbn/utils'; - -import { KibanaServer } from '@kbn/ftr-common-functional-services'; - -const INTEGRATION_TEST_ROOT = - process.env.WORKSPACE || Path.resolve(REPO_ROOT, '../integration-test'); - -export function EsArchiverProvider({ getService }) { - const config = getService('config'); - const client = getService('es'); - const log = getService('log'); - const kibanaServer = getService('kibanaServer'); - const retry = getService('retry'); - - const esArchiver = new EsArchiver({ - baseDir: INTEGRATION_TEST_ROOT, - client, - log, - kbnClient: kibanaServer, - }); - - KibanaServer.extendEsArchiver({ - esArchiver, - kibanaServer, - retry, - defaults: config.get('uiSettings.defaults'), - }); - - return esArchiver; -} diff --git a/x-pack/test/stack_functional_integration/services/index.js b/x-pack/test/stack_functional_integration/services/index.js deleted file mode 100644 index e311dd8b38f7e..0000000000000 --- a/x-pack/test/stack_functional_integration/services/index.js +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { services as xpackFunctionalServices } from '../../functional/services'; -import { EsArchiverProvider } from './es_archiver'; - -export const services = { - ...xpackFunctionalServices, - esArchiver: EsArchiverProvider, -}; From ab06783505956cca82ead6708bd00c2b078f341c Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Mon, 26 Sep 2022 12:52:17 -0700 Subject: [PATCH 4/6] Update CODEOWNERS for app-services test files (#141841) --- .github/CODEOWNERS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 51323cee4a112..480acad008f00 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -84,10 +84,12 @@ /x-pack/examples/ui_actions_enhanced_examples/ @elastic/kibana-app-services /x-pack/plugins/embeddable_enhanced/ @elastic/kibana-app-services /x-pack/plugins/runtime_fields @elastic/kibana-app-services -/x-pack/test/search_sessions_integration/ @elastic/kibana-app-services /src/plugins/dashboard/public/application/embeddable/viewport/print_media @elastic/kibana-app-services x-pack/plugins/files @elastic/kibana-app-services x-pack/examples/files_example @elastic/kibana-app-services +/x-pack/test/search_sessions_integration/ @elastic/kibana-app-services +/test/plugin_functional/test_suites/panel_actions @elastic/kibana-app-services +/test/plugin_functional/test_suites/data_plugin @elastic/kibana-app-services ### Observability Plugins From cedbf8076f570133a39b1469e346541de0a1d142 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau Date: Mon, 26 Sep 2022 17:00:46 -0400 Subject: [PATCH 5/6] [RAM] rmv public validation around our search strategy for alerts (#141850) * rmv public validation * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../search_strategy/search_strategy.test.ts | 48 +------------------ .../server/search_strategy/search_strategy.ts | 9 ---- .../tests/basic/search_strategy.ts | 18 ------- 3 files changed, 1 insertion(+), 74 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/search_strategy/search_strategy.test.ts b/x-pack/plugins/rule_registry/server/search_strategy/search_strategy.test.ts index 1b32f688ee8c0..ffcf973f028a2 100644 --- a/x-pack/plugins/rule_registry/server/search_strategy/search_strategy.test.ts +++ b/x-pack/plugins/rule_registry/server/search_strategy/search_strategy.test.ts @@ -8,11 +8,7 @@ import { of } from 'rxjs'; import { merge } from 'lodash'; import { loggerMock } from '@kbn/logging-mocks'; import { AlertConsumers } from '@kbn/rule-data-utils'; -import { - ruleRegistrySearchStrategyProvider, - EMPTY_RESPONSE, - RULE_SEARCH_STRATEGY_NAME, -} from './search_strategy'; +import { ruleRegistrySearchStrategyProvider, EMPTY_RESPONSE } from './search_strategy'; import { ruleDataServiceMock } from '../rule_data_plugin_service/rule_data_plugin_service.mock'; import { dataPluginMock } from '@kbn/data-plugin/server/mocks'; import { SearchStrategyDependencies } from '@kbn/data-plugin/server'; @@ -385,48 +381,6 @@ describe('ruleRegistrySearchStrategyProvider()', () => { ).toStrictEqual([{ test: { order: 'desc' } }]); }); - it('should reject, to the best of our ability, public requests', async () => { - (getIsKibanaRequest as jest.Mock).mockImplementation(() => { - return false; - }); - const request: RuleRegistrySearchRequest = { - featureIds: [AlertConsumers.LOGS], - sort: [ - { - test: { - order: 'desc', - }, - }, - ], - }; - const options = {}; - const deps = { - request: {}, - }; - - const strategy = ruleRegistrySearchStrategyProvider( - data, - ruleDataService, - alerting, - logger, - security, - spaces - ); - - let err = null; - try { - await strategy - .search(request, options, deps as unknown as SearchStrategyDependencies) - .toPromise(); - } catch (e) { - err = e; - } - expect(err).not.toBeNull(); - expect(err.message).toBe( - `The ${RULE_SEARCH_STRATEGY_NAME} search strategy is currently only available for internal use.` - ); - }); - it('passes the query ids if provided', async () => { const request: RuleRegistrySearchRequest = { featureIds: [AlertConsumers.SIEM], diff --git a/x-pack/plugins/rule_registry/server/search_strategy/search_strategy.ts b/x-pack/plugins/rule_registry/server/search_strategy/search_strategy.ts index 6b9ac15cd8d6a..49a6439ef9b59 100644 --- a/x-pack/plugins/rule_registry/server/search_strategy/search_strategy.ts +++ b/x-pack/plugins/rule_registry/server/search_strategy/search_strategy.ts @@ -5,7 +5,6 @@ * 2.0. */ import { map, mergeMap, catchError } from 'rxjs/operators'; -import Boom from '@hapi/boom'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { Logger } from '@kbn/core/server'; import { from, of } from 'rxjs'; @@ -25,7 +24,6 @@ import { Dataset } from '../rule_data_plugin_service/index_options'; import { MAX_ALERT_SEARCH_SIZE } from '../../common/constants'; import { AlertAuditAction, alertAuditEvent } from '..'; import { getSpacesFilter, getAuthzFilter } from '../lib'; -import { getIsKibanaRequest } from '../lib/get_is_kibana_request'; export const EMPTY_RESPONSE: RuleRegistrySearchResponse = { rawResponse: {} as RuleRegistrySearchResponse['rawResponse'], @@ -47,13 +45,6 @@ export const ruleRegistrySearchStrategyProvider = ( const requestUserEs = data.search.getSearchStrategy(ENHANCED_ES_SEARCH_STRATEGY); return { search: (request, options, deps) => { - // We want to ensure this request came from our UI. We can't really do this - // but we have a best effort we can try - if (!getIsKibanaRequest(deps.request.headers)) { - throw Boom.notFound( - `The ${RULE_SEARCH_STRATEGY_NAME} search strategy is currently only available for internal use.` - ); - } // SIEM uses RBAC fields in their alerts but also utilizes ES DLS which // is different than every other solution so we need to special case // those requests. diff --git a/x-pack/test/rule_registry/security_and_spaces/tests/basic/search_strategy.ts b/x-pack/test/rule_registry/security_and_spaces/tests/basic/search_strategy.ts index a04899d68d585..11982a8b51425 100644 --- a/x-pack/test/rule_registry/security_and_spaces/tests/basic/search_strategy.ts +++ b/x-pack/test/rule_registry/security_and_spaces/tests/basic/search_strategy.ts @@ -110,24 +110,6 @@ export default ({ getService }: FtrProviderContext) => { const second = result.rawResponse.hits.hits[1].fields?.['kibana.alert.evaluation.value']; expect(first > second).to.be(true); }); - - it('should reject public requests', async () => { - const result = await secureBsearch.send({ - supertestWithoutAuth, - auth: { - username: logsOnlySpacesAll.username, - password: logsOnlySpacesAll.password, - }, - options: { - featureIds: [AlertConsumers.LOGS], - }, - strategy: 'privateRuleRegistryAlertsSearchStrategy', - }); - expect(result.statusCode).to.be(500); - expect(result.message).to.be( - `The privateRuleRegistryAlertsSearchStrategy search strategy is currently only available for internal use.` - ); - }); }); describe('siem', () => { From 605c958297abe950588658650bd321804ac38acc Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Mon, 26 Sep 2022 14:06:00 -0700 Subject: [PATCH 6/6] [DOCS] Add v3 open API spec for ml sync (#141554) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: István Zoltán Szabó --- x-pack/plugins/ml/common/openapi/README.md | 2 + .../plugins/ml/common/openapi/ml_apis_v3.yaml | 169 ++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 x-pack/plugins/ml/common/openapi/ml_apis_v3.yaml diff --git a/x-pack/plugins/ml/common/openapi/README.md b/x-pack/plugins/ml/common/openapi/README.md index 256b3be6a8cc4..450f95cd52071 100644 --- a/x-pack/plugins/ml/common/openapi/README.md +++ b/x-pack/plugins/ml/common/openapi/README.md @@ -5,6 +5,7 @@ The current self-contained spec file can be used for online tools like those fou A guide about the openApi specification can be found at [https://swagger.io/docs/specification/about/](https://swagger.io/docs/specification/about/). The `ml_apis_v2.json` file uses OpenAPI Specification Version 2.0. +The `ml_apis_v3.yaml` file uses OpenAPI Specification Version 3.0.1. ## Tools @@ -12,4 +13,5 @@ It is possible to validate the docs before bundling them by running the followin ``` npx swagger-cli validate ml_apis_v2.json +npx swagger-cli validate ml_apis_v3.yaml ``` diff --git a/x-pack/plugins/ml/common/openapi/ml_apis_v3.yaml b/x-pack/plugins/ml/common/openapi/ml_apis_v3.yaml new file mode 100644 index 0000000000000..938b3c312c0c4 --- /dev/null +++ b/x-pack/plugins/ml/common/openapi/ml_apis_v3.yaml @@ -0,0 +1,169 @@ +openapi: 3.0.1 +info: + title: Machine learning APIs + description: Kibana APIs for the machine learning feature + version: "1.0.1" + license: + name: Elastic License 2.0 + url: https://www.elastic.co/licensing/elastic-license +tags: + - name: ml + description: Machine learning +servers: + - url: https://localhost:5601/ +paths: + /s/{spaceId}/api/ml/saved_objects/sync: + get: + summary: Synchronizes Kibana saved objects for machine learning jobs and trained models. + description: > + You must have `all` privileges for the **Machine Learning** feature in the **Analytics** section of the Kibana feature privileges. + This API runs automatically when you start Kibana and periodically thereafter. + operationId: ml-sync + tags: + - ml + parameters: + - $ref: '#/components/parameters/spaceParam' + - $ref: '#/components/parameters/simulateParam' + responses: + '200': + description: Indicates a successful call + content: + application/json: + schema: + $ref: '#/components/schemas/mlSyncResponse' + examples: + syncExample: + $ref: '#/components/examples/mlSyncExample' +components: + parameters: + spaceParam: + in: path + name: spaceId + description: An identifier for the space. If `/s/` and the identifier are omitted from the path, the default space is used. + required: true + schema: + type: string + simulateParam: + in: query + name: simulate + description: When true, simulates the synchronization by returning only the list of actions that would be performed. + required: false + schema: + type: boolean + example: 'true' + securitySchemes: + basicAuth: + type: http + scheme: basic + apiKeyAuth: + type: apiKey + in: header + name: ApiKey + schemas: + mlSyncResponseSuccess: + type: boolean + description: The success or failure of the synchronization. + mlSyncResponseAnomalyDetectors: + type: object + title: Sync API response for anomaly detection jobs + description: The sync machine learning saved objects API response contains this object when there are anomaly detection jobs affected by the synchronization. There is an object for each relevant job, which contains the synchronization status. + properties: + success: + $ref: '#/components/schemas/mlSyncResponseSuccess' + mlSyncResponseDatafeeds: + type: object + title: Sync API response for datafeeds + description: The sync machine learning saved objects API response contains this object when there are datafeeds affected by the synchronization. There is an object for each relevant datafeed, which contains the synchronization status. + properties: + success: + $ref: '#/components/schemas/mlSyncResponseSuccess' + mlSyncResponseDataFrameAnalytics: + type: object + title: Sync API response for data frame analytics jobs + description: The sync machine learning saved objects API response contains this object when there are data frame analytics jobs affected by the synchronization. There is an object for each relevant job, which contains the synchronization status. + properties: + success: + $ref: '#/components/schemas/mlSyncResponseSuccess' + mlSyncResponseSavedObjectsCreated: + type: object + title: Sync API response for created saved objects + description: If saved objects are missing for machine learning jobs or trained models, they are created when you run the sync machine learning saved objects API. + properties: + anomaly-detector: + type: object + description: If saved objects are missing for anomaly detection jobs, they are created. + additionalProperties: + $ref: '#/components/schemas/mlSyncResponseAnomalyDetectors' + data-frame-analytics: + type: object + description: If saved objects are missing for data frame analytics jobs, they are created. + additionalProperties: + $ref: '#/components/schemas/mlSyncResponseDataFrameAnalytics' + trained-model: + type: object + description: If saved objects are missing for trained models, they are created. + additionalProperties: + $ref: '#/components/schemas/mlSyncResponseTrainedModels' + mlSyncResponseSavedObjectsDeleted: + type: object + title: Sync API response for deleted saved objects + description: If saved objects exist for machine learning jobs or trained models that no longer exist, they are deleted when you run the sync machine learning saved objects API. + properties: + anomaly-detector: + type: object + description: If there are saved objects exist for nonexistent anomaly detection jobs, they are deleted. + additionalProperties: + $ref: '#/components/schemas/mlSyncResponseAnomalyDetectors' + data-frame-analytics: + type: object + description: If there are saved objects exist for nonexistent data frame analytics jobs, they are deleted. + additionalProperties: + $ref: '#/components/schemas/mlSyncResponseDataFrameAnalytics' + trained-model: + type: object + description: If there are saved objects exist for nonexistent trained models, they are deleted. + additionalProperties: + $ref: '#/components/schemas/mlSyncResponseTrainedModels' + mlSyncResponseTrainedModels: + type: object + title: Sync API response for trained models + description: The sync machine learning saved objects API response contains this object when there are trained models affected by the synchronization. There is an object for each relevant trained model, which contains the synchronization status. + properties: + success: + $ref: '#/components/schemas/mlSyncResponseSuccess' + mlSyncResponse: + type: object + title: Sync API response + properties: + datafeedsAdded: + type: object + description: If a saved object for an anomaly detection job is missing a datafeed identifier, it is added when you run the sync machine learning saved objects API. + additionalProperties: + $ref: '#/components/schemas/mlSyncResponseDatafeeds' + datafeedsRemoved: + type: object + description: If a saved object for an anomaly detection job references a datafeed that no longer exists, it is deleted when you run the sync machine learning saved objects API. + additionalProperties: + $ref: '#/components/schemas/mlSyncResponseDatafeeds' + savedObjectsCreated: + $ref: '#/components/schemas/mlSyncResponseSavedObjectsCreated' + savedObjectsDeleted: + $ref: '#/components/schemas/mlSyncResponseSavedObjectsDeleted' + examples: + mlSyncExample: + summary: Two anomaly detection jobs required synchronization in this example. + value: + { + "savedObjectsCreated": { + "anomaly_detector": { + "myjob1": { "success":true }, + "myjob2":{ "success":true } + } + }, + "savedObjectsDeleted": {}, + "datafeedsAdded":{}, + "datafeedsRemoved":{} + } +security: + - basicAuth: [ ] + - ApiKeyAuth: [ ] \ No newline at end of file