From 6b92c62b88a2ec3b7e2d6915402bee5eeaee5822 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 10 Nov 2023 09:22:56 +0100 Subject: [PATCH 01/10] should not add 'searchQuery' if it's just a match_all query --- .../queries/get_query_with_params.test.ts | 32 +++++++++++++++++++ .../queries/get_query_with_params.ts | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.test.ts index aa600065fd223..ba62d87f6edb7 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.test.ts @@ -84,4 +84,36 @@ describe('getQueryWithParams', () => { }, }); }); + + it("should not add `searchQuery` if it's just a match_all query", () => { + const query = getQueryWithParams({ + params: { + index: 'the-index', + timeFieldName: 'the-time-field-name', + start: 1577836800000, + end: 1609459200000, + baselineMin: 10, + baselineMax: 20, + deviationMin: 30, + deviationMax: 40, + includeFrozen: false, + searchQuery: '{"match_all":{}}', + }, + }); + expect(query).toEqual({ + bool: { + filter: [ + { + range: { + 'the-time-field-name': { + format: 'epoch_millis', + gte: 1577836800000, + lte: 1609459200000, + }, + }, + }, + ], + }, + }); + }); }); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts index b7e2e2619316a..00506125796e4 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts @@ -31,7 +31,7 @@ export const getQueryWithParams = ({ return { bool: { filter: [ - searchQuery, + ...(searchQuery.match_all === undefined ? [searchQuery] : []), ...getFilters(params), ...(Array.isArray(termFilters) ? termFilters.map(getTermsQuery) : []), ...(filter ? [filter] : []), From f65d6f9d41e6302bb114a0be4c36c63249075570 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 10 Nov 2023 10:25:40 +0100 Subject: [PATCH 02/10] should not add multiple equal overall time range filters --- .../queries/fetch_categories.test.ts | 122 ++++++++++++++++++ .../queries/fetch_categories.ts | 54 ++++++-- .../queries/fetch_significant_categories.ts | 29 ----- 3 files changed, 165 insertions(+), 40 deletions(-) create mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.test.ts diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.test.ts new file mode 100644 index 0000000000000..c6556962ad7d2 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils'; + +import { getBaselineOrDeviationFilter, getCategoryRequest } from './fetch_categories'; + +describe('getBaselineOrDeviationFilter', () => { + it('returns a filter that matches both baseline and deviation time range', () => { + const params = { + index: 'the-index', + timeFieldName: 'the-time-field-name', + start: 0, + end: 50, + baselineMin: 10, + baselineMax: 20, + deviationMin: 30, + deviationMax: 40, + includeFrozen: false, + searchQuery: '{ "match_all": {} }', + }; + + const baselineOrDeviationFilter = getBaselineOrDeviationFilter(params); + + expect(baselineOrDeviationFilter).toEqual({ + bool: { + should: [ + { + range: { + 'the-time-field-name': { gte: 10, lte: 20, format: 'epoch_millis' }, + }, + }, + { + range: { + 'the-time-field-name': { gte: 30, lte: 40, format: 'epoch_millis' }, + }, + }, + ], + }, + }); + }); +}); + +describe('getCategoryRequest', () => { + it('returns the category request', () => { + const params = { + index: 'the-index', + timeFieldName: 'the-time-field-name', + start: 0, + end: 50, + baselineMin: 10, + baselineMax: 20, + deviationMin: 30, + deviationMax: 40, + includeFrozen: false, + searchQuery: '{ "match_all": {} }', + }; + + const randomSamplerWrapper = createRandomSamplerWrapper({ + probability: 0.1, + seed: 1234, + }); + + const query = getCategoryRequest(params, 'the-field-name', randomSamplerWrapper); + + expect(query).toEqual({ + index: 'the-index', + size: 0, + body: { + query: { + bool: { + filter: [ + { + bool: { + should: [ + { + range: { + 'the-time-field-name': { + gte: 10, + lte: 20, + format: 'epoch_millis', + }, + }, + }, + { + range: { + 'the-time-field-name': { + gte: 30, + lte: 40, + format: 'epoch_millis', + }, + }, + }, + ], + }, + }, + ], + }, + }, + aggs: { + sample: { + random_sampler: { probability: 0.1, seed: 1234 }, + aggs: { + categories: { + categorize_text: { field: 'the-field-name', size: 1000 }, + aggs: { + hit: { + top_hits: { size: 1, sort: ['the-time-field-name'], _source: 'the-field-name' }, + }, + }, + }, + }, + }, + }, + }, + }); + }); +}); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts index e70255ababffd..fbcbd600e0616 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts @@ -28,30 +28,65 @@ import { isRequestAbortedError } from '../../../lib/is_request_aborted_error'; import { getQueryWithParams } from './get_query_with_params'; +// Filter that includes docs from both the baseline and deviation time range. +export const getBaselineOrDeviationFilter = ( + params: AiopsLogRateAnalysisSchema +): estypes.QueryDslQueryContainer => { + return { + bool: { + should: [ + { + range: { + [params.timeFieldName]: { + gte: params.baselineMin, + lte: params.baselineMax, + format: 'epoch_millis', + }, + }, + }, + { + range: { + [params.timeFieldName]: { + gte: params.deviationMin, + lte: params.deviationMax, + format: 'epoch_millis', + }, + }, + }, + ], + }, + }; +}; + export const getCategoryRequest = ( params: AiopsLogRateAnalysisSchema, fieldName: string, - from: number | undefined, - to: number | undefined, - filter: estypes.QueryDslQueryContainer, { wrap }: RandomSamplerWrapper ): estypes.SearchRequest => { const { index, timeFieldName } = params; + const query = getQueryWithParams({ - params, + // Passing in an empty string for the time field name will avoid + // adding any range queries. We're enforcing this here since this + // is covered by the filter which will matches docs in both + // baseline and deviation time range. + params: { ...params, timeFieldName: '' }, termFilters: undefined, - filter, + filter: getBaselineOrDeviationFilter(params), }); + const { params: request } = createCategoryRequest( index, fieldName, timeFieldName, - from, - to, + undefined, + undefined, query, wrap ); + request.body.query = query; + return request; }; @@ -64,9 +99,6 @@ export const fetchCategories = async ( esClient: ElasticsearchClient, params: AiopsLogRateAnalysisSchema, fieldNames: string[], - from: number | undefined, - to: number | undefined, - filter: estypes.QueryDslQueryContainer, logger: Logger, // The default value of 1 means no sampling will be used sampleProbability: number = 1, @@ -82,7 +114,7 @@ export const fetchCategories = async ( const settledPromises = await Promise.allSettled( fieldNames.map((fieldName) => { - const request = getCategoryRequest(params, fieldName, from, to, filter, randomSamplerWrapper); + const request = getCategoryRequest(params, fieldName, randomSamplerWrapper); return esClient.search(request, { signal: abortSignal, maxRetries: 0, diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_categories.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_categories.ts index 49617a1661ba6..2284622dc808c 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_categories.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_categories.ts @@ -42,39 +42,10 @@ export const fetchSignificantCategories = async ( emitError: (m: string) => void, abortSignal?: AbortSignal ) => { - // Filter that includes docs from both the baseline and deviation time range. - const baselineOrDeviationFilter = { - bool: { - should: [ - { - range: { - [params.timeFieldName]: { - gte: params.baselineMin, - lte: params.baselineMax, - format: 'epoch_millis', - }, - }, - }, - { - range: { - [params.timeFieldName]: { - gte: params.deviationMin, - lte: params.deviationMax, - format: 'epoch_millis', - }, - }, - }, - ], - }, - }; - const categoriesOverall = await fetchCategories( esClient, params, fieldNames, - undefined, - undefined, - baselineOrDeviationFilter, logger, sampleProbability, emitError, From a840327331b4d0b04b20588116f2990714f2661c Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 10 Nov 2023 13:59:44 +0100 Subject: [PATCH 03/10] adds a comment to tests --- .../routes/log_rate_analysis/queries/fetch_categories.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.test.ts index c6556962ad7d2..6c4825767c43f 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.test.ts @@ -67,6 +67,9 @@ describe('getCategoryRequest', () => { const query = getCategoryRequest(params, 'the-field-name', randomSamplerWrapper); + // Because the time range filter is covered by the should clauses that cover both + // baseline (10,20) and deviation (30,40), we expect that there is no other + // time range filter whatsoever, for example for start/end (0,50). expect(query).toEqual({ index: 'the-index', size: 0, From f0313513431403354a7809f4b623fab816b97803 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 10 Nov 2023 15:31:50 +0100 Subject: [PATCH 04/10] fix fetchCategoryCounts and add unit tests --- .../queries/fetch_categories.ts | 2 +- .../queries/fetch_category_counts.test.ts | 70 +++++++++++++++++++ .../queries/fetch_category_counts.ts | 26 +++---- 3 files changed, 81 insertions(+), 17 deletions(-) create mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.test.ts diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts index fbcbd600e0616..400e22f5c0e1b 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts @@ -68,7 +68,7 @@ export const getCategoryRequest = ( const query = getQueryWithParams({ // Passing in an empty string for the time field name will avoid // adding any range queries. We're enforcing this here since this - // is covered by the filter which will matches docs in both + // is covered by the filter which will match docs in both // baseline and deviation time range. params: { ...params, timeFieldName: '' }, termFilters: undefined, diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.test.ts new file mode 100644 index 0000000000000..4a97c6ef7bc8c --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.test.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getCategoryCountRequest } from './fetch_category_counts'; + +describe('getCategoryCountRequest', () => { + it('returns the category count request', () => { + const params = { + index: 'the-index', + timeFieldName: 'the-time-field-name', + start: 0, + end: 50, + baselineMin: 10, + baselineMax: 20, + deviationMin: 30, + deviationMax: 40, + includeFrozen: false, + searchQuery: '{ "match_all": {} }', + }; + + const category = { + key: 'runTask ended no files to process', + count: 667, + examples: ['[runTask()] ended: no files to process'], + }; + + const query = getCategoryCountRequest( + params, + 'the-field-name', + category, + params.baselineMin, + params.baselineMax + ); + + expect(query).toEqual({ + index: 'the-index', + body: { + query: { + bool: { + filter: [ + { range: { 'the-time-field-name': { gte: 10, lte: 20, format: 'epoch_millis' } } }, + { + bool: { + should: [ + { + match: { + 'the-field-name': { + auto_generate_synonyms_phrase_query: false, + fuzziness: 0, + operator: 'and', + query: 'runTask ended no files to process', + }, + }, + }, + ], + }, + }, + ], + }, + }, + size: 0, + track_total_hits: true, + }, + }); + }); +}); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts index 608734ae0e19a..00f797e31d485 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts @@ -33,24 +33,18 @@ export const getCategoryCountRequest = ( ): estypes.SearchRequest => { const { index } = params; - const query = getQueryWithParams({ - params, - }); - const categoryQuery = getCategoryQuery(fieldName, [category]); - if (Array.isArray(query.bool?.filter)) { - query.bool?.filter?.push(categoryQuery); - query.bool?.filter?.push({ - range: { - [params.timeFieldName]: { - gte: from, - lte: to, - format: 'epoch_millis', - }, - }, - }); - } + const query = getQueryWithParams({ + // This will override the original start/end params if + // the optional from/to args are provided. + params: { + ...params, + ...(from ? { start: from } : {}), + ...(to ? { end: to } : {}), + }, + filter: categoryQuery, + }); return { index, From cfc9c2a8da17b4b03305e5ee09c2b2d6508894de Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 10 Nov 2023 15:46:31 +0100 Subject: [PATCH 05/10] break out getCategoryCountMSearchRequest and create unit test for it --- .../queries/fetch_category_counts.test.ts | 108 +++++++++++++++--- .../queries/fetch_category_counts.ts | 25 +++- 2 files changed, 114 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.test.ts index 4a97c6ef7bc8c..be9488164f35e 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.test.ts @@ -5,23 +5,23 @@ * 2.0. */ -import { getCategoryCountRequest } from './fetch_category_counts'; +import { getCategoryCountRequest, getCategoryCountMSearchRequest } from './fetch_category_counts'; + +const params = { + index: 'the-index', + timeFieldName: 'the-time-field-name', + start: 0, + end: 50, + baselineMin: 10, + baselineMax: 20, + deviationMin: 30, + deviationMax: 40, + includeFrozen: false, + searchQuery: '{ "match_all": {} }', +}; describe('getCategoryCountRequest', () => { it('returns the category count request', () => { - const params = { - index: 'the-index', - timeFieldName: 'the-time-field-name', - start: 0, - end: 50, - baselineMin: 10, - baselineMax: 20, - deviationMin: 30, - deviationMax: 40, - includeFrozen: false, - searchQuery: '{ "match_all": {} }', - }; - const category = { key: 'runTask ended no files to process', count: 667, @@ -68,3 +68,83 @@ describe('getCategoryCountRequest', () => { }); }); }); + +describe('getCategoryCountMSearchRequest', () => { + it('returns the request body for the msearch request', () => { + const categories = [ + { + key: 'SLO summary transforms installed and started', + count: 500, + examples: ['SLO summary transforms installed and started'], + }, + { key: 'Trusted Apps', count: 500, examples: ['Trusted Apps: '] }, + ]; + + const query = getCategoryCountMSearchRequest( + params, + 'the-field-name', + categories, + params.baselineMin, + params.baselineMax + ); + + expect(query).toEqual([ + { index: 'the-index' }, + { + query: { + bool: { + filter: [ + { range: { 'the-time-field-name': { gte: 10, lte: 20, format: 'epoch_millis' } } }, + { + bool: { + should: [ + { + match: { + 'the-field-name': { + auto_generate_synonyms_phrase_query: false, + fuzziness: 0, + operator: 'and', + query: 'SLO summary transforms installed and started', + }, + }, + }, + ], + }, + }, + ], + }, + }, + size: 0, + track_total_hits: true, + }, + { index: 'the-index' }, + { + query: { + bool: { + filter: [ + { range: { 'the-time-field-name': { gte: 10, lte: 20, format: 'epoch_millis' } } }, + { + bool: { + should: [ + { + match: { + 'the-field-name': { + auto_generate_synonyms_phrase_query: false, + fuzziness: 0, + operator: 'and', + query: 'Trusted Apps', + }, + }, + }, + ], + }, + }, + ], + }, + }, + size: 0, + track_total_hits: true, + }, + ]); + }); +}); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts index 00f797e31d485..555d49b46dc19 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts @@ -56,6 +56,19 @@ export const getCategoryCountRequest = ( }; }; +export const getCategoryCountMSearchRequest = ( + params: AiopsLogRateAnalysisSchema, + fieldName: string, + categories: FetchCategoriesResponse['categories'], + from: number | undefined, + to: number | undefined +): estypes.MsearchRequestItem[] => + categories.flatMap((category) => [ + { index: params.index }, + getCategoryCountRequest(params, fieldName, category, from, to) + .body as estypes.MsearchMultisearchBody, + ]); + export const fetchCategoryCounts = async ( esClient: ElasticsearchClient, params: AiopsLogRateAnalysisSchema, @@ -69,11 +82,13 @@ export const fetchCategoryCounts = async ( ): Promise => { const updatedCategories = cloneDeep(categories); - const searches = categories.categories.flatMap((category) => [ - { index: params.index }, - getCategoryCountRequest(params, fieldName, category, from, to) - .body as estypes.MsearchMultisearchBody, - ]); + const searches = getCategoryCountMSearchRequest( + params, + fieldName, + categories.categories, + from, + to + ); let mSearchresponse; From ab4a63c985deba106a02aae0a3549e66d0d18558 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 10 Nov 2023 16:19:21 +0100 Subject: [PATCH 06/10] consolidate mocks --- .../queries/__mocks__/params_match_all.ts | 19 +++++ .../queries/__mocks__/params_search_query.ts | 16 +++++ .../queries/__mocks__/search_query.ts | 16 +++++ .../queries/fetch_categories.test.ts | 31 +------- .../queries/fetch_category_counts.test.ts | 27 +++---- .../queries/fetch_index_info.test.ts | 37 +++++----- .../queries/get_filters.test.ts | 27 ++----- .../queries/get_histogram_query.test.ts | 45 ++++++------ .../queries/get_query_with_params.test.ts | 72 ++++++++----------- .../queries/get_request_base.test.ts | 24 ++----- 10 files changed, 141 insertions(+), 173 deletions(-) create mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/__mocks__/params_match_all.ts create mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/__mocks__/params_search_query.ts create mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/__mocks__/search_query.ts diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/__mocks__/params_match_all.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/__mocks__/params_match_all.ts new file mode 100644 index 0000000000000..a81ba523caa43 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/__mocks__/params_match_all.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const paramsMock = { + index: 'the-index', + timeFieldName: 'the-time-field-name', + start: 0, + end: 50, + baselineMin: 10, + baselineMax: 20, + deviationMin: 30, + deviationMax: 40, + includeFrozen: false, + searchQuery: '{ "match_all": {} }', +}; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/__mocks__/params_search_query.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/__mocks__/params_search_query.ts new file mode 100644 index 0000000000000..dd0ac80585403 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/__mocks__/params_search_query.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AiopsLogRateAnalysisSchema } from '../../../../../common/api/log_rate_analysis/schema'; + +import { paramsMock } from './params_match_all'; +import { searchQueryMock } from './search_query'; + +export const paramsSearchQueryMock: AiopsLogRateAnalysisSchema = { + ...paramsMock, + searchQuery: searchQueryMock, +}; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/__mocks__/search_query.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/__mocks__/search_query.ts new file mode 100644 index 0000000000000..58be3bf372711 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/__mocks__/search_query.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// This is the format that gets passed on from the Kibana search bar. +export const searchQueryMock = JSON.stringify({ + bool: { + filter: [], + minimum_should_match: 1, + must_not: [], + should: [{ term: { 'the-term': { value: 'the-value' } } }], + }, +}); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.test.ts index 6c4825767c43f..d38dd57c0afd2 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.test.ts @@ -7,24 +7,12 @@ import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils'; +import { paramsMock } from './__mocks__/params_match_all'; import { getBaselineOrDeviationFilter, getCategoryRequest } from './fetch_categories'; describe('getBaselineOrDeviationFilter', () => { it('returns a filter that matches both baseline and deviation time range', () => { - const params = { - index: 'the-index', - timeFieldName: 'the-time-field-name', - start: 0, - end: 50, - baselineMin: 10, - baselineMax: 20, - deviationMin: 30, - deviationMax: 40, - includeFrozen: false, - searchQuery: '{ "match_all": {} }', - }; - - const baselineOrDeviationFilter = getBaselineOrDeviationFilter(params); + const baselineOrDeviationFilter = getBaselineOrDeviationFilter(paramsMock); expect(baselineOrDeviationFilter).toEqual({ bool: { @@ -47,25 +35,12 @@ describe('getBaselineOrDeviationFilter', () => { describe('getCategoryRequest', () => { it('returns the category request', () => { - const params = { - index: 'the-index', - timeFieldName: 'the-time-field-name', - start: 0, - end: 50, - baselineMin: 10, - baselineMax: 20, - deviationMin: 30, - deviationMax: 40, - includeFrozen: false, - searchQuery: '{ "match_all": {} }', - }; - const randomSamplerWrapper = createRandomSamplerWrapper({ probability: 0.1, seed: 1234, }); - const query = getCategoryRequest(params, 'the-field-name', randomSamplerWrapper); + const query = getCategoryRequest(paramsMock, 'the-field-name', randomSamplerWrapper); // Because the time range filter is covered by the should clauses that cover both // baseline (10,20) and deviation (30,40), we expect that there is no other diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.test.ts index be9488164f35e..b3fa6ff5f31a8 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.test.ts @@ -5,20 +5,9 @@ * 2.0. */ -import { getCategoryCountRequest, getCategoryCountMSearchRequest } from './fetch_category_counts'; +import { paramsMock } from './__mocks__/params_match_all'; -const params = { - index: 'the-index', - timeFieldName: 'the-time-field-name', - start: 0, - end: 50, - baselineMin: 10, - baselineMax: 20, - deviationMin: 30, - deviationMax: 40, - includeFrozen: false, - searchQuery: '{ "match_all": {} }', -}; +import { getCategoryCountRequest, getCategoryCountMSearchRequest } from './fetch_category_counts'; describe('getCategoryCountRequest', () => { it('returns the category count request', () => { @@ -29,11 +18,11 @@ describe('getCategoryCountRequest', () => { }; const query = getCategoryCountRequest( - params, + paramsMock, 'the-field-name', category, - params.baselineMin, - params.baselineMax + paramsMock.baselineMin, + paramsMock.baselineMax ); expect(query).toEqual({ @@ -81,11 +70,11 @@ describe('getCategoryCountMSearchRequest', () => { ]; const query = getCategoryCountMSearchRequest( - params, + paramsMock, 'the-field-name', categories, - params.baselineMin, - params.baselineMax + paramsMock.baselineMin, + paramsMock.baselineMax ); expect(query).toEqual([ diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts index db14c30caebdc..0344005c869f8 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts @@ -9,27 +9,14 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ElasticsearchClient } from '@kbn/core/server'; -import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; +import { paramsSearchQueryMock } from './__mocks__/params_search_query'; import { fetchIndexInfo, getRandomDocsRequest } from './fetch_index_info'; -const params: AiopsLogRateAnalysisSchema = { - index: 'the-index', - timeFieldName: 'the-time-field-name', - start: 1577836800000, - end: 1609459200000, - baselineMin: 10, - baselineMax: 20, - deviationMin: 30, - deviationMax: 40, - includeFrozen: false, - searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}', -}; - describe('fetch_index_info', () => { describe('getRandomDocsRequest', () => { it('returns the most basic request body for a sample of random documents', () => { - const req = getRandomDocsRequest(params); + const req = getRandomDocsRequest(paramsSearchQueryMock); expect(req).toEqual({ body: { @@ -40,13 +27,20 @@ describe('fetch_index_info', () => { query: { bool: { filter: [ - { bool: { filter: [], must: [{ match_all: {} }], must_not: [] } }, + { + bool: { + filter: [], + minimum_should_match: 1, + must_not: [], + should: [{ term: { 'the-term': { value: 'the-value' } } }], + }, + }, { range: { 'the-time-field-name': { format: 'epoch_millis', - gte: 1577836800000, - lte: 1609459200000, + gte: 0, + lte: 50, }, }, }, @@ -59,7 +53,7 @@ describe('fetch_index_info', () => { size: 1000, track_total_hits: true, }, - index: params.index, + index: paramsSearchQueryMock.index, ignore_throttled: undefined, ignore_unavailable: true, }); @@ -105,7 +99,10 @@ describe('fetch_index_info', () => { search: esClientSearchMock, } as unknown as ElasticsearchClient; - const { totalDocCount, fieldCandidates } = await fetchIndexInfo(esClientMock, params); + const { totalDocCount, fieldCandidates } = await fetchIndexInfo( + esClientMock, + paramsSearchQueryMock + ); expect(fieldCandidates).toEqual(['myIpFieldName', 'myKeywordFieldName']); expect(totalDocCount).toEqual(5000000); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.test.ts index c34d81fa91bdf..22107bc5942d0 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.test.ts @@ -5,43 +5,28 @@ * 2.0. */ +import { paramsSearchQueryMock } from './__mocks__/params_search_query'; + import { getFilters } from './get_filters'; describe('getFilters', () => { it('returns an empty array with no timeFieldName and searchQuery supplied', () => { const filters = getFilters({ - index: 'the-index', + ...paramsSearchQueryMock, timeFieldName: '', - searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}', - start: 1577836800000, - end: 1609459200000, - baselineMin: 10, - baselineMax: 20, - deviationMin: 30, - deviationMax: 40, }); expect(filters).toEqual([]); }); it('returns a range filter when timeFieldName is supplied', () => { - const filters = getFilters({ - index: 'the-index', - timeFieldName: 'the-time-field-name', - searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}', - start: 1577836800000, - end: 1609459200000, - baselineMin: 10, - baselineMax: 20, - deviationMin: 30, - deviationMax: 40, - }); + const filters = getFilters(paramsSearchQueryMock); expect(filters).toEqual([ { range: { 'the-time-field-name': { format: 'epoch_millis', - gte: 1577836800000, - lte: 1609459200000, + gte: 0, + lte: 50, }, }, }, diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_histogram_query.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_histogram_query.test.ts index 95113d39f41c8..39af620dd266a 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_histogram_query.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_histogram_query.test.ts @@ -5,34 +5,30 @@ * 2.0. */ -import { getHistogramQuery } from './get_histogram_query'; +import { paramsSearchQueryMock } from './__mocks__/params_search_query'; -const paramsMock = { - index: 'the-index', - timeFieldName: 'the-time-field-name', - start: 1577836800000, - end: 1609459200000, - baselineMin: 10, - baselineMax: 20, - deviationMin: 30, - deviationMax: 40, - includeFrozen: false, - searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}', -}; +import { getHistogramQuery } from './get_histogram_query'; describe('getHistogramQuery', () => { it('returns histogram query without additional filters', () => { - const query = getHistogramQuery(paramsMock); + const query = getHistogramQuery(paramsSearchQueryMock); expect(query).toEqual({ bool: { filter: [ - { bool: { filter: [], must: [{ match_all: {} }], must_not: [] } }, + { + bool: { + filter: [], + minimum_should_match: 1, + must_not: [], + should: [{ term: { 'the-term': { value: 'the-value' } } }], + }, + }, { range: { 'the-time-field-name': { format: 'epoch_millis', - gte: 1577836800000, - lte: 1609459200000, + gte: 0, + lte: 50, }, }, }, @@ -42,7 +38,7 @@ describe('getHistogramQuery', () => { }); it('returns histogram query with additional filters', () => { - const query = getHistogramQuery(paramsMock, [ + const query = getHistogramQuery(paramsSearchQueryMock, [ { term: { ['the-filter-fieldName']: 'the-filter-fieldValue' }, }, @@ -50,7 +46,14 @@ describe('getHistogramQuery', () => { expect(query).toEqual({ bool: { filter: [ - { bool: { filter: [], must: [{ match_all: {} }], must_not: [] } }, + { + bool: { + filter: [], + minimum_should_match: 1, + must_not: [], + should: [{ term: { 'the-term': { value: 'the-value' } } }], + }, + }, { term: { 'the-filter-fieldName': 'the-filter-fieldValue', @@ -60,8 +63,8 @@ describe('getHistogramQuery', () => { range: { 'the-time-field-name': { format: 'epoch_millis', - gte: 1577836800000, - lte: 1609459200000, + gte: 0, + lte: 50, }, }, }, diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.test.ts index ba62d87f6edb7..cd43e031e916a 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.test.ts @@ -5,34 +5,33 @@ * 2.0. */ +import { paramsMock } from './__mocks__/params_match_all'; +import { paramsSearchQueryMock } from './__mocks__/params_search_query'; + import { getQueryWithParams } from './get_query_with_params'; describe('getQueryWithParams', () => { it('returns the most basic query filtering', () => { const query = getQueryWithParams({ - params: { - index: 'the-index', - timeFieldName: 'the-time-field-name', - start: 1577836800000, - end: 1609459200000, - baselineMin: 10, - baselineMax: 20, - deviationMin: 30, - deviationMax: 40, - includeFrozen: false, - searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}', - }, + params: paramsSearchQueryMock, }); expect(query).toEqual({ bool: { filter: [ - { bool: { filter: [], must: [{ match_all: {} }], must_not: [] } }, + { + bool: { + filter: [], + minimum_should_match: 1, + must_not: [], + should: [{ term: { 'the-term': { value: 'the-value' } } }], + }, + }, { range: { 'the-time-field-name': { format: 'epoch_millis', - gte: 1577836800000, - lte: 1609459200000, + gte: 0, + lte: 50, }, }, }, @@ -43,18 +42,7 @@ describe('getQueryWithParams', () => { it('returns a query considering a custom field/value pair', () => { const query = getQueryWithParams({ - params: { - index: 'the-index', - timeFieldName: 'the-time-field-name', - start: 1577836800000, - end: 1609459200000, - baselineMin: 10, - baselineMax: 20, - deviationMin: 30, - deviationMax: 40, - includeFrozen: false, - searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}', - }, + params: paramsSearchQueryMock, termFilters: [ { fieldName: 'actualFieldName', @@ -65,13 +53,20 @@ describe('getQueryWithParams', () => { expect(query).toEqual({ bool: { filter: [ - { bool: { filter: [], must: [{ match_all: {} }], must_not: [] } }, + { + bool: { + filter: [], + minimum_should_match: 1, + must_not: [], + should: [{ term: { 'the-term': { value: 'the-value' } } }], + }, + }, { range: { 'the-time-field-name': { format: 'epoch_millis', - gte: 1577836800000, - lte: 1609459200000, + gte: 0, + lte: 50, }, }, }, @@ -87,18 +82,7 @@ describe('getQueryWithParams', () => { it("should not add `searchQuery` if it's just a match_all query", () => { const query = getQueryWithParams({ - params: { - index: 'the-index', - timeFieldName: 'the-time-field-name', - start: 1577836800000, - end: 1609459200000, - baselineMin: 10, - baselineMax: 20, - deviationMin: 30, - deviationMax: 40, - includeFrozen: false, - searchQuery: '{"match_all":{}}', - }, + params: paramsMock, }); expect(query).toEqual({ bool: { @@ -107,8 +91,8 @@ describe('getQueryWithParams', () => { range: { 'the-time-field-name': { format: 'epoch_millis', - gte: 1577836800000, - lte: 1609459200000, + gte: 0, + lte: 50, }, }, }, diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_request_base.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_request_base.test.ts index ee21cb985f5d6..33797e219fd37 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_request_base.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_request_base.test.ts @@ -5,36 +5,20 @@ * 2.0. */ +import { paramsMock } from './__mocks__/params_match_all'; + import { getRequestBase } from './get_request_base'; describe('getRequestBase', () => { it('defaults to not setting `ignore_throttled`', () => { - const requestBase = getRequestBase({ - index: 'the-index', - timeFieldName: 'the-time-field-name', - searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}', - start: 1577836800000, - end: 1609459200000, - baselineMin: 10, - baselineMax: 20, - deviationMin: 30, - deviationMax: 40, - }); + const requestBase = getRequestBase(paramsMock); expect(requestBase.ignore_throttled).toEqual(undefined); }); it('adds `ignore_throttled=false` when `includeFrozen=true`', () => { const requestBase = getRequestBase({ - index: 'the-index', - timeFieldName: 'the-time-field-name', + ...paramsMock, includeFrozen: true, - searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}', - start: 1577836800000, - end: 1609459200000, - baselineMin: 10, - baselineMax: 20, - deviationMin: 30, - deviationMax: 40, }); expect(requestBase.ignore_throttled).toEqual(false); }); From c6ec4c436488432bdb9a03b889b92e4edbcc3417 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 14 Nov 2023 08:36:20 +0100 Subject: [PATCH 07/10] comments --- .../routes/log_rate_analysis/queries/fetch_categories.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts index 400e22f5c0e1b..3a8232880d116 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts @@ -68,8 +68,8 @@ export const getCategoryRequest = ( const query = getQueryWithParams({ // Passing in an empty string for the time field name will avoid // adding any range queries. We're enforcing this here since this - // is covered by the filter which will match docs in both - // baseline and deviation time range. + // is covered by the filter which will match docs in both baseline + // and deviation time range via `getBaselineOrDeviationFilter`. params: { ...params, timeFieldName: '' }, termFilters: undefined, filter: getBaselineOrDeviationFilter(params), @@ -85,6 +85,9 @@ export const getCategoryRequest = ( wrap ); + // In this case we're only interested in the aggregation which + // `createCategoryRequest` returns, so we're re-applying the original + // query we create via `getQueryWithParams` here. request.body.query = query; return request; From cd8ecfb7b9f58cec4957bdae0cd8b04629bdbc87 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 14 Nov 2023 08:59:33 +0100 Subject: [PATCH 08/10] adds fallback to match_all when esSearchQuery is the search bar default --- .../log_rate_analysis_content.tsx | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx index 0770cfea83771..0ce422a0fb872 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx @@ -5,7 +5,8 @@ * 2.0. */ -import React, { useEffect, useState, type FC } from 'react'; +import { isEqual } from 'lodash'; +import React, { useEffect, useMemo, useState, type FC } from 'react'; import { EuiEmptyPrompt, EuiHorizontalRule, EuiPanel } from '@elastic/eui'; import type { Moment } from 'moment'; @@ -32,7 +33,18 @@ import { import type { GroupTableItem } from '../../log_rate_analysis_results_table/types'; import { useLogRateAnalysisResultsTableRowContext } from '../../log_rate_analysis_results_table/log_rate_analysis_results_table_row_provider'; -const DEFAULT_SEARCH_QUERY = { match_all: {} }; +const DEFAULT_SEARCH_QUERY: estypes.QueryDslQueryContainer = { match_all: {} }; +const DEFAULT_SEARCH_BAR_QUERY: estypes.QueryDslQueryContainer = { + bool: { + filter: [], + must: [ + { + match_all: {}, + }, + ], + must_not: [], + }, +}; export function getDocumentCountStatsSplitLabel( significantItem?: SignificantItem, @@ -93,6 +105,13 @@ export const LogRateAnalysisContent: FC = ({ setIsBrushCleared(windowParameters === undefined); }, [windowParameters]); + // Checks if `esSearchQuery` is the default empty query passed on from the search bar + // and if that's the case fall back to a simpler match all query. + const searchQuery = useMemo( + () => (isEqual(esSearchQuery, DEFAULT_SEARCH_BAR_QUERY) ? DEFAULT_SEARCH_QUERY : esSearchQuery), + [esSearchQuery] + ); + const { currentSelectedSignificantItem, currentSelectedGroup, @@ -105,7 +124,7 @@ export const LogRateAnalysisContent: FC = ({ const { documentStats, earliest, latest } = useData( dataView, 'log_rate_analysis', - esSearchQuery, + searchQuery, setGlobalState, currentSelectedSignificantItem, currentSelectedGroup, @@ -170,7 +189,7 @@ export const LogRateAnalysisContent: FC = ({ stickyHistogram={stickyHistogram} onReset={clearSelection} sampleProbability={sampleProbability} - searchQuery={esSearchQuery} + searchQuery={searchQuery} windowParameters={windowParameters} barColorOverride={barColorOverride} barHighlightColorOverride={barHighlightColorOverride} From 84647fa2fdb745219cda68114734254354251d79 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 16 Nov 2023 09:46:36 +0100 Subject: [PATCH 09/10] cleanup/refactor get_filters -> get_range_query --- .../queries/fetch_categories.ts | 7 ++-- .../queries/get_filters.test.ts | 35 ---------------- .../log_rate_analysis/queries/get_filters.ts | 40 ------------------- .../queries/get_query_with_params.ts | 15 +++++-- .../queries/get_range_query.test.ts | 36 +++++++++++++++++ .../queries/get_range_query.ts | 24 +++++++++++ 6 files changed, 75 insertions(+), 82 deletions(-) delete mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.test.ts delete mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.ts create mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_range_query.test.ts create mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_range_query.ts diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts index 3a8232880d116..703242b7e9a6a 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts @@ -66,12 +66,11 @@ export const getCategoryRequest = ( const { index, timeFieldName } = params; const query = getQueryWithParams({ - // Passing in an empty string for the time field name will avoid - // adding any range queries. We're enforcing this here since this + params, + // We're skipping the overall range query here since this // is covered by the filter which will match docs in both baseline // and deviation time range via `getBaselineOrDeviationFilter`. - params: { ...params, timeFieldName: '' }, - termFilters: undefined, + skipRangeQuery: true, filter: getBaselineOrDeviationFilter(params), }); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.test.ts deleted file mode 100644 index 22107bc5942d0..0000000000000 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { paramsSearchQueryMock } from './__mocks__/params_search_query'; - -import { getFilters } from './get_filters'; - -describe('getFilters', () => { - it('returns an empty array with no timeFieldName and searchQuery supplied', () => { - const filters = getFilters({ - ...paramsSearchQueryMock, - timeFieldName: '', - }); - expect(filters).toEqual([]); - }); - - it('returns a range filter when timeFieldName is supplied', () => { - const filters = getFilters(paramsSearchQueryMock); - expect(filters).toEqual([ - { - range: { - 'the-time-field-name': { - format: 'epoch_millis', - gte: 0, - lte: 50, - }, - }, - }, - ]); - }); -}); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.ts deleted file mode 100644 index e910d2e997441..0000000000000 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; - -import type { ESFilter } from '@kbn/es-types'; - -import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; - -export function rangeQuery( - start?: number, - end?: number, - field = '@timestamp' -): estypes.QueryDslQueryContainer[] { - return [ - { - range: { - [field]: { - gte: start, - lte: end, - format: 'epoch_millis', - }, - }, - }, - ]; -} - -export function getFilters({ start, end, timeFieldName }: AiopsLogRateAnalysisSchema): ESFilter[] { - const filters: ESFilter[] = []; - - if (timeFieldName !== '') { - filters.push(...rangeQuery(start, end, timeFieldName)); - } - - return filters; -} diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts index 00506125796e4..80d67fe73fa81 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts @@ -11,7 +11,7 @@ import type { FieldValuePair } from '@kbn/ml-agg-utils'; import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; -import { getFilters } from './get_filters'; +import { getRangeQuery } from './get_range_query'; export const getTermsQuery = ({ fieldName, fieldValue }: FieldValuePair) => { return { term: { [fieldName]: fieldValue } }; @@ -21,21 +21,30 @@ interface QueryParams { params: AiopsLogRateAnalysisSchema<'2'>; termFilters?: FieldValuePair[]; filter?: estypes.QueryDslQueryContainer; + skipRangeQuery?: boolean; } export const getQueryWithParams = ({ params, termFilters, filter, + skipRangeQuery = false, }: QueryParams): estypes.QueryDslQueryContainer => { const searchQuery = JSON.parse(params.searchQuery) as estypes.QueryDslQueryContainer; return { bool: { filter: [ + // Add `searchQuery` if it's not a `match_all` query ...(searchQuery.match_all === undefined ? [searchQuery] : []), - ...getFilters(params), + + // Add a range query based on `start/end` for the `timeFieldName`, check for skip flag. + ...(!skipRangeQuery ? [getRangeQuery(params.start, params.end, params.timeFieldName)] : []), + + // Add optional term filters ...(Array.isArray(termFilters) ? termFilters.map(getTermsQuery) : []), + + // Add other optional filters ...(filter ? [filter] : []), - ] as estypes.QueryDslQueryContainer[], + ], }, }; }; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_range_query.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_range_query.test.ts new file mode 100644 index 0000000000000..8d232df5d2c23 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_range_query.test.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getRangeQuery } from './get_range_query'; + +describe('getRangeQuery', () => { + it('returns a range filter with default time field', () => { + const query = getRangeQuery(0, 50); + expect(query).toEqual({ + range: { + '@timestamp': { + gte: 0, + lte: 50, + format: 'epoch_millis', + }, + }, + }); + }); + + it('returns a range filter with a custom time field', () => { + const query = getRangeQuery(0, 50, 'the-time-field'); + expect(query).toEqual({ + range: { + 'the-time-field': { + gte: 0, + lte: 50, + format: 'epoch_millis', + }, + }, + }); + }); +}); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_range_query.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_range_query.ts new file mode 100644 index 0000000000000..ca9ae7a299ee7 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_range_query.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +export function getRangeQuery( + start?: number, + end?: number, + field = '@timestamp' +): estypes.QueryDslQueryContainer { + return { + range: { + [field]: { + gte: start, + lte: end, + format: 'epoch_millis', + }, + }, + }; +} From b4d6a4ed14f6f6eee3ab9820f6e3df08451b21a0 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:03:23 +0000 Subject: [PATCH 10/10] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/aiops/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/aiops/tsconfig.json b/x-pack/plugins/aiops/tsconfig.json index 66d996fe0b002..31d68a5a4ab0f 100644 --- a/x-pack/plugins/aiops/tsconfig.json +++ b/x-pack/plugins/aiops/tsconfig.json @@ -23,7 +23,6 @@ "@kbn/data-views-plugin", "@kbn/datemath", "@kbn/es-query", - "@kbn/es-types", "@kbn/field-formats-plugin", "@kbn/field-types", "@kbn/i18n-react",