Skip to content

Commit

Permalink
Chunk exception list items in query to support up to 1000000
Browse files Browse the repository at this point in the history
  • Loading branch information
@marshallmain
marshallmain committed Jul 21, 2020
1 parent 99fa3c4 commit 93243bf
Show file tree
Hide file tree
Showing 2 changed files with 246 additions and 21 deletions.
179 changes: 177 additions & 2 deletions x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { getQueryFilter } from './get_query_filter';
import { Filter } from 'src/plugins/data/public';
import { getQueryFilter, buildExceptionFilter } from './get_query_filter';
import { Filter, EsQueryConfig } from 'src/plugins/data/public';
import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock';

describe('get_filter', () => {
Expand Down Expand Up @@ -910,4 +910,179 @@ describe('get_filter', () => {
});
});
});

describe('buildExceptionFilter', () => {
const config: EsQueryConfig = {
allowLeadingWildcards: true,
queryStringOptions: { analyze_wildcard: true },
ignoreFilterIfFieldNotInIndex: false,
dateFormatTZ: 'Zulu',
};
test('it should build a filter without chunking exception items', () => {
const exceptionFilter = buildExceptionFilter(
[
{ language: 'kuery', query: 'host.name: linux and some.field: value' },
{ language: 'kuery', query: 'user.name: name' },
],
{
fields: [],
title: 'auditbeat-*',
},
config,
true,
2
);
expect(exceptionFilter).toEqual({
meta: {
alias: null,
negate: true,
disabled: false,
},
query: {
bool: {
should: [
{
bool: {
filter: [
{
bool: {
minimum_should_match: 1,
should: [
{
match: {
'host.name': 'linux',
},
},
],
},
},
{
bool: {
minimum_should_match: 1,
should: [
{
match: {
'some.field': 'value',
},
},
],
},
},
],
},
},
{
bool: {
minimum_should_match: 1,
should: [
{
match: {
'user.name': 'name',
},
},
],
},
},
],
},
},
});
});

test('it should properly chunk exception items', () => {
const exceptionFilter = buildExceptionFilter(
[
{ language: 'kuery', query: 'host.name: linux and some.field: value' },
{ language: 'kuery', query: 'user.name: name' },
{ language: 'kuery', query: 'file.path: /safe/path' },
],
{
fields: [],
title: 'auditbeat-*',
},
config,
true,
2
);
expect(exceptionFilter).toEqual({
meta: {
alias: null,
negate: true,
disabled: false,
},
query: {
bool: {
should: [
{
bool: {
should: [
{
bool: {
filter: [
{
bool: {
minimum_should_match: 1,
should: [
{
match: {
'host.name': 'linux',
},
},
],
},
},
{
bool: {
minimum_should_match: 1,
should: [
{
match: {
'some.field': 'value',
},
},
],
},
},
],
},
},
{
bool: {
minimum_should_match: 1,
should: [
{
match: {
'user.name': 'name',
},
},
],
},
},
],
},
},
{
bool: {
should: [
{
bool: {
minimum_should_match: 1,
should: [
{
match: {
'file.path': '/safe/path',
},
},
],
},
},
],
},
},
],
},
},
});
});
});
});
88 changes: 69 additions & 19 deletions x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import {
Filter,
Query,
IIndexPattern,
isFilterDisabled,
buildEsQuery,
Expand All @@ -16,10 +17,10 @@ import {
CreateExceptionListItemSchema,
} from '../../../lists/common/schemas';
import { buildExceptionListQueries } from './build_exceptions_query';
import { Query, Language, Index } from './schemas/common/schemas';
import { Query as QueryString, Language, Index } from './schemas/common/schemas';

export const getQueryFilter = (
query: Query,
query: QueryString,
language: Language,
filters: Array<Partial<Filter>>,
index: Index,
Expand Down Expand Up @@ -48,27 +49,76 @@ export const getQueryFilter = (
*/
const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists });
if (exceptionQueries.length > 0) {
// Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
// allowing us to make 1024-item chunks of exception list items.
// Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
// very conservative value.
const exceptionFilter = buildExceptionFilter(
exceptionQueries,
indexPattern,
config,
excludeExceptions,
1024
);
enabledFilters.push(exceptionFilter);
}
const initialQuery = { query, language };

return buildEsQuery(indexPattern, initialQuery, enabledFilters, config);
};

export const buildExceptionFilter = (
exceptionQueries: Query[],
indexPattern: IIndexPattern,
config: EsQueryConfig,
excludeExceptions: boolean,
chunkSize: number
) => {
const exceptionFilter: Filter = {
meta: {
alias: null,
negate: excludeExceptions,
disabled: false,
},
query: {
bool: {
should: undefined,
},
},
};
if (exceptionQueries.length <= chunkSize) {
const query = buildEsQuery(indexPattern, exceptionQueries, [], config);
exceptionFilter.query.bool.should = query.bool.filter;
} else {
const chunkedFilters: Filter[] = [];
for (let index = 0; index < exceptionQueries.length; index += chunkSize) {
const exceptionQueriesChunk = exceptionQueries.slice(index, index + chunkSize);
const esQueryChunk = buildEsQuery(indexPattern, exceptionQueriesChunk, [], config);
const filterChunk: Filter = {
meta: {
alias: null,
negate: false,
disabled: false,
},
query: {
bool: {
should: esQueryChunk.bool.filter,
},
},
};
chunkedFilters.push(filterChunk);
}
// Here we build a query with only the exceptions: it will put them all in the `filter` array
// of the resulting object, which would AND the exceptions together. When creating exceptionFilter,
// we move the `filter` array to `should` so they are OR'd together instead.
// This gets around the problem with buildEsQuery not allowing callers to specify whether queries passed in
// should be ANDed or ORed together.
const exceptionQuery = buildEsQuery(indexPattern, exceptionQueries, [], config);
const exceptionFilter: Filter = {
meta: {
alias: null,
negate: excludeExceptions,
disabled: false,
},
query: {
bool: {
should: exceptionQuery.bool.filter,
},
},
};
enabledFilters.push(exceptionFilter);
exceptionFilter.query.bool.should = buildEsQuery(
indexPattern,
[],
chunkedFilters,
config
).bool.filter;
}
const initialQuery = { query, language };

return buildEsQuery(indexPattern, initialQuery, enabledFilters, config);
return exceptionFilter;
};

0 comments on commit 93243bf

Please sign in to comment.