Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for number, date and IP range data types #76971

Merged
merged 25 commits into from
Mar 8, 2021
Merged

Support for number, date and IP range data types #76971

Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
4d77749
Support for number, date and IP ranges
wylieconlon Sep 8, 2020
e2ad9de
Update tests
wylieconlon Sep 8, 2020
2c98e04
Merge remote-tracking branch 'origin/master' into range-types
wylieconlon Sep 15, 2020
7e1fefb
Ranges don't work with range agg
wylieconlon Sep 15, 2020
f8c1478
Merge remote-tracking branch 'origin/master' into range-types
wylieconlon Jan 29, 2021
43cf942
Fix test case
wylieconlon Jan 29, 2021
4a3d312
Merge remote-tracking branch 'origin/master' into range-types
wylieconlon Feb 1, 2021
5aff4e1
Allow Discover to create range filters
wylieconlon Feb 1, 2021
3b08bbb
Merge remote-tracking branch 'origin/master' into range-types
wylieconlon Feb 2, 2021
9330884
Supports ranges in Visualize, KQL, remove Lens support
wylieconlon Feb 2, 2021
d1e60eb
Fix test mappings
wylieconlon Feb 3, 2021
8492355
Merge remote-tracking branch 'origin/master' into range-types
wylieconlon Feb 25, 2021
0e15906
Bring back field cache to work around bug
wylieconlon Feb 25, 2021
6c9b3db
Merge remote-tracking branch 'origin/master' into range-types
wylieconlon Mar 1, 2021
5d70230
Fix some tests
wylieconlon Mar 1, 2021
e08520c
Fix test expectation
wylieconlon Mar 2, 2021
e665736
Merge remote-tracking branch 'origin/master' into range-types
wylieconlon Mar 2, 2021
e6e3dca
Respond to review comments
wylieconlon Mar 3, 2021
0582ce8
Fix type error
wylieconlon Mar 3, 2021
3196f5b
Merge remote-tracking branch 'origin/master' into range-types
wylieconlon Mar 3, 2021
1e7357c
Remove added sample data
wylieconlon Mar 3, 2021
466dbb0
Fix api_docs
wylieconlon Mar 3, 2021
4a44b9d
Merge remote-tracking branch 'origin/master' into range-types
wylieconlon Mar 3, 2021
24d4b20
Fix test
wylieconlon Mar 3, 2021
c2d965c
Merge remote-tracking branch 'origin/master' into range-types
wylieconlon Mar 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions ...lopment/plugins/data/public/kibana-plugin-plugins-data-public.es_field_types.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,22 @@ export declare enum ES_FIELD_TYPES
| BYTE | <code>&quot;byte&quot;</code> | |
| DATE | <code>&quot;date&quot;</code> | |
| DATE\_NANOS | <code>&quot;date_nanos&quot;</code> | |
| DATE\_RANGE | <code>&quot;date_range&quot;</code> | |
| DOUBLE | <code>&quot;double&quot;</code> | |
| DOUBLE\_RANGE | <code>&quot;double_range&quot;</code> | |
| FLOAT | <code>&quot;float&quot;</code> | |
| FLOAT\_RANGE | <code>&quot;float_range&quot;</code> | |
| GEO\_POINT | <code>&quot;geo_point&quot;</code> | |
| GEO\_SHAPE | <code>&quot;geo_shape&quot;</code> | |
| HALF\_FLOAT | <code>&quot;half_float&quot;</code> | |
| HISTOGRAM | <code>&quot;histogram&quot;</code> | |
| INTEGER | <code>&quot;integer&quot;</code> | |
| INTEGER\_RANGE | <code>&quot;integer_range&quot;</code> | |
| IP | <code>&quot;ip&quot;</code> | |
| IP\_RANGE | <code>&quot;ip_range&quot;</code> | |
| KEYWORD | <code>&quot;keyword&quot;</code> | |
| LONG | <code>&quot;long&quot;</code> | |
| LONG\_RANGE | <code>&quot;long_range&quot;</code> | |
| MURMUR3 | <code>&quot;murmur3&quot;</code> | |
| NESTED | <code>&quot;nested&quot;</code> | |
| OBJECT | <code>&quot;object&quot;</code> | |
Expand Down
3 changes: 3 additions & 0 deletions ...opment/plugins/data/public/kibana-plugin-plugins-data-public.kbn_field_types.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,16 @@ export declare enum KBN_FIELD_TYPES
| BOOLEAN | <code>&quot;boolean&quot;</code> | |
| CONFLICT | <code>&quot;conflict&quot;</code> | |
| DATE | <code>&quot;date&quot;</code> | |
| DATE\_RANGE | <code>&quot;date_range&quot;</code> | |
| GEO\_POINT | <code>&quot;geo_point&quot;</code> | |
| GEO\_SHAPE | <code>&quot;geo_shape&quot;</code> | |
| HISTOGRAM | <code>&quot;histogram&quot;</code> | |
| IP | <code>&quot;ip&quot;</code> | |
| IP\_RANGE | <code>&quot;ip_range&quot;</code> | |
| MURMUR3 | <code>&quot;murmur3&quot;</code> | |
| NESTED | <code>&quot;nested&quot;</code> | |
| NUMBER | <code>&quot;number&quot;</code> | |
| NUMBER\_RANGE | <code>&quot;number_range&quot;</code> | |
| OBJECT | <code>&quot;object&quot;</code> | |
| STRING | <code>&quot;string&quot;</code> | |
| UNKNOWN | <code>&quot;unknown&quot;</code> | |
Expand Down
6 changes: 6 additions & 0 deletions ...lopment/plugins/data/server/kibana-plugin-plugins-data-server.es_field_types.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,22 @@ export declare enum ES_FIELD_TYPES
| BYTE | <code>&quot;byte&quot;</code> | |
| DATE | <code>&quot;date&quot;</code> | |
| DATE\_NANOS | <code>&quot;date_nanos&quot;</code> | |
| DATE\_RANGE | <code>&quot;date_range&quot;</code> | |
| DOUBLE | <code>&quot;double&quot;</code> | |
| DOUBLE\_RANGE | <code>&quot;double_range&quot;</code> | |
| FLOAT | <code>&quot;float&quot;</code> | |
| FLOAT\_RANGE | <code>&quot;float_range&quot;</code> | |
| GEO\_POINT | <code>&quot;geo_point&quot;</code> | |
| GEO\_SHAPE | <code>&quot;geo_shape&quot;</code> | |
| HALF\_FLOAT | <code>&quot;half_float&quot;</code> | |
| HISTOGRAM | <code>&quot;histogram&quot;</code> | |
| INTEGER | <code>&quot;integer&quot;</code> | |
| INTEGER\_RANGE | <code>&quot;integer_range&quot;</code> | |
| IP | <code>&quot;ip&quot;</code> | |
| IP\_RANGE | <code>&quot;ip_range&quot;</code> | |
| KEYWORD | <code>&quot;keyword&quot;</code> | |
| LONG | <code>&quot;long&quot;</code> | |
| LONG\_RANGE | <code>&quot;long_range&quot;</code> | |
| MURMUR3 | <code>&quot;murmur3&quot;</code> | |
| NESTED | <code>&quot;nested&quot;</code> | |
| OBJECT | <code>&quot;object&quot;</code> | |
Expand Down
3 changes: 3 additions & 0 deletions ...opment/plugins/data/server/kibana-plugin-plugins-data-server.kbn_field_types.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,16 @@ export declare enum KBN_FIELD_TYPES
| BOOLEAN | <code>&quot;boolean&quot;</code> | |
| CONFLICT | <code>&quot;conflict&quot;</code> | |
| DATE | <code>&quot;date&quot;</code> | |
| DATE\_RANGE | <code>&quot;date_range&quot;</code> | |
| GEO\_POINT | <code>&quot;geo_point&quot;</code> | |
| GEO\_SHAPE | <code>&quot;geo_shape&quot;</code> | |
| HISTOGRAM | <code>&quot;histogram&quot;</code> | |
| IP | <code>&quot;ip&quot;</code> | |
| IP\_RANGE | <code>&quot;ip_range&quot;</code> | |
| MURMUR3 | <code>&quot;murmur3&quot;</code> | |
| NESTED | <code>&quot;nested&quot;</code> | |
| NUMBER | <code>&quot;number&quot;</code> | |
| NUMBER\_RANGE | <code>&quot;number_range&quot;</code> | |
| OBJECT | <code>&quot;object&quot;</code> | |
| STRING | <code>&quot;string&quot;</code> | |
| UNKNOWN | <code>&quot;unknown&quot;</code> | |
Expand Down
2 changes: 2 additions & 0 deletions src/plugins/data/common/es_query/filters/build_filters.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ function buildBaseFilter(
case 'range':
const newParams = { gte: params.from, lt: params.to };
return buildRangeFilter(field, newParams, indexPattern);
case 'range_from_value':
return buildRangeFilter(field, params, indexPattern);
case 'exists':
return buildExistsFilter(field, indexPattern);
default:
Expand Down
1 change: 1 addition & 0 deletions src/plugins/data/common/es_query/filters/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ export enum FILTERS {
MISSING = 'missing',
QUERY_STRING = 'query_string',
RANGE = 'range',
RANGE_FROM_VALUE = 'range_from_value',
GEO_BOUNDING_BOX = 'geo_bounding_box',
GEO_POLYGON = 'geo_polygon',
SPATIAL_FILTER = 'spatial_filter',
Expand Down
3 changes: 3 additions & 0 deletions src/plugins/data/common/field_formats/converters/string.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,12 @@ export class StringFormat extends FieldFormat {
});
static fieldType = [
KBN_FIELD_TYPES.NUMBER,
KBN_FIELD_TYPES.NUMBER_RANGE,
KBN_FIELD_TYPES.BOOLEAN,
KBN_FIELD_TYPES.DATE,
KBN_FIELD_TYPES.DATE_RANGE,
KBN_FIELD_TYPES.IP,
KBN_FIELD_TYPES.IP_RANGE,
KBN_FIELD_TYPES.ATTACHMENT,
KBN_FIELD_TYPES.GEO_POINT,
KBN_FIELD_TYPES.GEO_SHAPE,
Expand Down
8 changes: 8 additions & 0 deletions src/plugins/data/common/index_patterns/field.stub.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,12 @@ export const stubFields: IFieldType[] = [
searchable: true,
filterable: true,
},
{
name: 'bytes_range',
type: 'number_range',
esTypes: ['integer_range'],
aggregatable: true,
searchable: true,
filterable: true,
},
];
3 changes: 3 additions & 0 deletions src/plugins/data/common/kbn_field_types/kbn_field_types.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,16 @@ describe('utils/kbn_field_types', () => {
KBN_FIELD_TYPES.BOOLEAN,
KBN_FIELD_TYPES.CONFLICT,
KBN_FIELD_TYPES.DATE,
KBN_FIELD_TYPES.DATE_RANGE,
KBN_FIELD_TYPES.GEO_POINT,
KBN_FIELD_TYPES.GEO_SHAPE,
KBN_FIELD_TYPES.HISTOGRAM,
KBN_FIELD_TYPES.IP,
KBN_FIELD_TYPES.IP_RANGE,
KBN_FIELD_TYPES.MURMUR3,
KBN_FIELD_TYPES.NESTED,
KBN_FIELD_TYPES.NUMBER,
KBN_FIELD_TYPES.NUMBER_RANGE,
KBN_FIELD_TYPES.OBJECT,
KBN_FIELD_TYPES.STRING,
KBN_FIELD_TYPES.UNKNOWN,
Expand Down
23 changes: 23 additions & 0 deletions src/plugins/data/common/kbn_field_types/kbn_field_types_factory.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,18 +43,41 @@ export const createKbnFieldTypes = (): KbnFieldType[] => [
ES_FIELD_TYPES.TOKEN_COUNT,
],
}),
new KbnFieldType({
name: KBN_FIELD_TYPES.NUMBER_RANGE,
sortable: true,
filterable: true,
esTypes: [
ES_FIELD_TYPES.FLOAT_RANGE,
ES_FIELD_TYPES.DOUBLE_RANGE,
ES_FIELD_TYPES.INTEGER_RANGE,
ES_FIELD_TYPES.LONG_RANGE,
],
}),
new KbnFieldType({
name: KBN_FIELD_TYPES.DATE,
sortable: true,
filterable: true,
esTypes: [ES_FIELD_TYPES.DATE, ES_FIELD_TYPES.DATE_NANOS],
}),
new KbnFieldType({
name: KBN_FIELD_TYPES.DATE_RANGE,
sortable: true,
filterable: true,
esTypes: [ES_FIELD_TYPES.DATE_RANGE],
}),
new KbnFieldType({
name: KBN_FIELD_TYPES.IP,
sortable: true,
filterable: true,
esTypes: [ES_FIELD_TYPES.IP],
}),
new KbnFieldType({
name: KBN_FIELD_TYPES.IP_RANGE,
sortable: true,
filterable: true,
esTypes: [ES_FIELD_TYPES.IP_RANGE],
}),
new KbnFieldType({
name: KBN_FIELD_TYPES.BOOLEAN,
sortable: true,
Expand Down
10 changes: 10 additions & 0 deletions src/plugins/data/common/kbn_field_types/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ export enum ES_FIELD_TYPES {

DATE = 'date',
DATE_NANOS = 'date_nanos',
DATE_RANGE = 'date_range',

GEO_POINT = 'geo_point',
GEO_SHAPE = 'geo_shape',
Expand All @@ -43,9 +44,15 @@ export enum ES_FIELD_TYPES {
SHORT = 'short',
UNSIGNED_LONG = 'unsigned_long',

FLOAT_RANGE = 'float_range',
DOUBLE_RANGE = 'double_range',
INTEGER_RANGE = 'integer_range',
LONG_RANGE = 'long_range',

NESTED = 'nested',
BYTE = 'byte',
IP = 'ip',
IP_RANGE = 'ip_range',
ATTACHMENT = 'attachment',
TOKEN_COUNT = 'token_count',
MURMUR3 = 'murmur3',
Expand All @@ -59,11 +66,14 @@ export enum KBN_FIELD_TYPES {
ATTACHMENT = 'attachment',
BOOLEAN = 'boolean',
DATE = 'date',
DATE_RANGE = 'date_range',
GEO_POINT = 'geo_point',
GEO_SHAPE = 'geo_shape',
IP = 'ip',
IP_RANGE = 'ip_range',
MURMUR3 = 'murmur3',
NUMBER = 'number',
NUMBER_RANGE = 'number_range',
STRING = 'string',
UNKNOWN = 'unknown',
CONFLICT = 'conflict',
Expand Down
2 changes: 1 addition & 1 deletion src/plugins/data/common/search/aggs/buckets/date_histogram.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ export const getDateHistogramBucketAgg = ({
{
name: 'field',
type: 'field',
filterFieldTypes: KBN_FIELD_TYPES.DATE,
filterFieldTypes: [KBN_FIELD_TYPES.DATE, KBN_FIELD_TYPES.DATE_RANGE],
default(agg: IBucketDateHistogramAggConfig) {
return agg.getIndexPattern().getTimeField?.()?.name;
},
Expand Down
2 changes: 1 addition & 1 deletion src/plugins/data/common/search/aggs/buckets/date_range.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ export const getDateRangeBucketAgg = ({
{
name: 'field',
type: 'field',
filterFieldTypes: KBN_FIELD_TYPES.DATE,
filterFieldTypes: [KBN_FIELD_TYPES.DATE, KBN_FIELD_TYPES.DATE_RANGE],
default(agg: IBucketAggConfig) {
return agg.getIndexPattern().timeFieldName;
},
Expand Down
11 changes: 9 additions & 2 deletions src/plugins/data/common/search/aggs/buckets/histogram.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ export const getHistogramBucketAgg = ({
{
name: 'field',
type: 'field',
filterFieldTypes: KBN_FIELD_TYPES.NUMBER,
filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.NUMBER_RANGE],
},
{
/*
Expand All @@ -105,6 +105,11 @@ export const getHistogramBucketAgg = ({
options: any
) {
const field = aggConfig.getField();
if (field?.type === 'number_range') {
// Can't scale number_histogram requests
return;
}

const aggBody = field.scripted
? { script: { source: field.script, lang: field.lang } }
: { field: field.name };
Expand Down Expand Up @@ -163,7 +168,9 @@ export const getHistogramBucketAgg = ({
{
name: 'maxBars',
shouldShow(agg) {
return isAutoInterval(get(agg, 'params.interval'));
const field = agg.getField();
// Show this for empty field and number field, but not range
return field?.type !== 'number_range' && isAutoInterval(get(agg, 'params.interval'));
},
write: () => {},
},
Expand Down
1 change: 1 addition & 0 deletions src/plugins/data/common/search/aggs/buckets/range.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ export const getRangeBucketAgg = ({ getFieldFormatsStart }: RangeBucketAggDepend
{
name: 'field',
type: 'field',
// number_range is not supported by Elasticsearch
filterFieldTypes: [KBN_FIELD_TYPES.NUMBER],
},
{
Expand Down
18 changes: 18 additions & 0 deletions src/plugins/data/public/public.api.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -640,10 +640,16 @@ export enum ES_FIELD_TYPES {
// (undocumented)
DATE_NANOS = "date_nanos",
// (undocumented)
DATE_RANGE = "date_range",
// (undocumented)
DOUBLE = "double",
// (undocumented)
DOUBLE_RANGE = "double_range",
// (undocumented)
FLOAT = "float",
// (undocumented)
FLOAT_RANGE = "float_range",
// (undocumented)
GEO_POINT = "geo_point",
// (undocumented)
GEO_SHAPE = "geo_shape",
Expand All @@ -658,12 +664,18 @@ export enum ES_FIELD_TYPES {
// (undocumented)
INTEGER = "integer",
// (undocumented)
INTEGER_RANGE = "integer_range",
// (undocumented)
IP = "ip",
// (undocumented)
IP_RANGE = "ip_range",
// (undocumented)
KEYWORD = "keyword",
// (undocumented)
LONG = "long",
// (undocumented)
LONG_RANGE = "long_range",
// (undocumented)
MURMUR3 = "murmur3",
// (undocumented)
NESTED = "nested",
Expand Down Expand Up @@ -1744,6 +1756,8 @@ export enum KBN_FIELD_TYPES {
// (undocumented)
DATE = "date",
// (undocumented)
DATE_RANGE = "date_range",
// (undocumented)
GEO_POINT = "geo_point",
// (undocumented)
GEO_SHAPE = "geo_shape",
Expand All @@ -1752,12 +1766,16 @@ export enum KBN_FIELD_TYPES {
// (undocumented)
IP = "ip",
// (undocumented)
IP_RANGE = "ip_range",
// (undocumented)
MURMUR3 = "murmur3",
// (undocumented)
NESTED = "nested",
// (undocumented)
NUMBER = "number",
// (undocumented)
NUMBER_RANGE = "number_range",
// (undocumented)
OBJECT = "object",
// (undocumented)
_SOURCE = "_source",
Expand Down
Loading