From 1c2aa1ac03a4839bb096f5cc5a508251158c97b1 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 8 Sep 2022 16:24:17 -0400 Subject: [PATCH 01/10] Synthetic _source: support `field` in many cases This adds support for the `field` scripting API in many but not all cases. Before this change numbers, dates, and IPs supported the `field` API when running with _source in synthetic mode because they always have doc values. This change adds support for `match_only_text`, `store`d `keyword` fields, and `store`d `text` fields. Two remaining field configurations work with synthetic _source and do not work with `field`: * A `text` field with a sub-`keyword` field that has `doc_values` * A `text` field with a sub-`keyword` field that is `store`d --- .../test/painless/50_script_doc_values.yml | 164 +++++++----------- .../extras/MatchOnlyTextFieldMapper.java | 29 +++- .../extras/MatchOnlyTextFieldMapperTests.java | 10 ++ .../fielddata/StoredFieldIndexFieldData.java | 105 +++++++++++ ...StoredFieldSortedBinaryIndexFieldData.java | 79 +++++++++ .../index/mapper/KeywordFieldMapper.java | 49 ++++-- .../index/mapper/TextFieldMapper.java | 37 ++-- .../search/lookup/SourceLookup.java | 34 +++- .../index/mapper/KeywordFieldMapperTests.java | 17 ++ .../index/mapper/TextFieldMapperTests.java | 22 +++ .../index/mapper/MapperServiceTestCase.java | 9 + .../index/mapper/MapperTestCase.java | 22 +++ 12 files changed, 439 insertions(+), 138 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index d07a7b7296c66..d9d89d7fb5053 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -2382,7 +2382,6 @@ keyword_stored_synthetic: - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } - do: - catch: bad_request # TODO make this work in a follow up search: index: test_synthetic body: @@ -2391,13 +2390,11 @@ keyword_stored_synthetic: field: script: source: "field('keyword_stored').get('missing')" -# - match: { hits.hits.0.fields.field.0: "no doc values" } -# - match: { hits.hits.1.fields.field.0: "missing" } -# - match: { hits.hits.2.fields.field.0: "no doc values 0" } # doc values are sorted - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { hits.hits.1.fields.field.0: "missing" } + - match: { hits.hits.2.fields.field.0: "no doc values 0" } # doc values are sorted - do: - catch: bad_request # TODO make this work in a follow up search: index: test_synthetic body: @@ -2407,13 +2404,11 @@ keyword_stored_synthetic: script: source: "/* avoid yaml stash */ $('keyword_stored', 'missing')" # same as `field('keyword').get('missing')` -# - match: { hits.hits.0.fields.field.0: "no doc values" } -# - match: { hits.hits.1.fields.field.0: "missing" } -# - match: { hits.hits.2.fields.field.0: "no doc values 0" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { hits.hits.1.fields.field.0: "missing" } + - match: { hits.hits.2.fields.field.0: "no doc values 0" } - do: - catch: bad_request # TODO make this work in a follow up search: index: test_synthetic body: @@ -2422,13 +2417,11 @@ keyword_stored_synthetic: field: script: source: "field('keyword_stored').get(1, 'dne')" -# - match: { hits.hits.0.fields.field.0: "dne" } -# - match: { hits.hits.1.fields.field.0: "dne" } -# - match: { hits.hits.2.fields.field.0: "no doc values 1" } # doc values are sorted - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "dne" } + - match: { hits.hits.1.fields.field.0: "dne" } + - match: { hits.hits.2.fields.field.0: "no doc values 1" } # doc values are sorted - do: - catch: bad_request # TODO make this work in a follow up search: index: test_synthetic body: @@ -2437,10 +2430,9 @@ keyword_stored_synthetic: field: script: source: "String.join(', ', field('keyword_stored'))" -# - match: { hits.hits.0.fields.field.0: "no doc values" } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "no doc values 0, no doc values 1, no doc values 2" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "no doc values 0, no doc values 1, no doc values 2" } --- long: @@ -5188,7 +5180,7 @@ text_sub_stored_keyword_synthetic: script_fields: field: script: - source: "String cat = ''; for (String s : field('text_sub_stored_keyword')) { cat += s; } cat + field('text_no_field_data').size();" + source: "String cat = ''; for (String s : field('text_sub_stored_keyword')) { cat += s; } cat + field('text_sub_stored_keyword').size();" # - match: { hits.hits.0.fields.field.0: "Lots of text.1" } # - match: { hits.hits.1.fields.field.0: "0" } # - match: { hits.hits.2.fields.field.0: "Lots of text.SOOOOO much texteven more text3" } @@ -5221,7 +5213,6 @@ text_stored_synthetic: - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5230,13 +5221,11 @@ text_stored_synthetic: field: script: source: "field('text_stored').get('')" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5245,13 +5234,11 @@ text_stored_synthetic: field: script: source: "/* avoid yaml stash */ $('text_stored', '')" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5260,13 +5247,11 @@ text_stored_synthetic: field: script: source: "String defaultText = 'default text'; field('text_stored').get(defaultText)" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "default text" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "default text" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5275,13 +5260,11 @@ text_stored_synthetic: field: script: source: "String defaultText = 'default text'; $('text_stored', defaultText)" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "default text" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "default text" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5290,13 +5273,11 @@ text_stored_synthetic: field: script: source: "field('text_stored').get(1, '')" -# - match: { hits.hits.0.fields.field.0: "" } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "" } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5305,13 +5286,11 @@ text_stored_synthetic: field: script: source: "String defaultText = 'default text'; field('text_stored').get(1, defaultText)" -# - match: { hits.hits.0.fields.field.0: "default text" } -# - match: { hits.hits.1.fields.field.0: "default text" } -# - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "default text" } + - match: { hits.hits.1.fields.field.0: "default text" } + - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5320,13 +5299,11 @@ text_stored_synthetic: field: script: source: "field('text_stored').get(1, '')" -# - match: { hits.hits.0.fields.field.0: "" } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "" } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5334,11 +5311,10 @@ text_stored_synthetic: script_fields: field: script: - source: "String cat = ''; for (String s : field('text_stored')) { cat += s; } cat + field('text_no_field_data').size();" -# - match: { hits.hits.0.fields.field.0: "Lots of text.1" } -# - match: { hits.hits.1.fields.field.0: "0" } -# - match: { hits.hits.2.fields.field.0: "Lots of text.SOOOOO much texteven more text3" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + source: "String cat = ''; for (String s : field('text_stored')) { cat += s; } cat + field('text_stored').size();" + - match: { hits.hits.0.fields.field.0: "Lots of text.1" } + - match: { hits.hits.1.fields.field.0: "0" } + - match: { hits.hits.2.fields.field.0: "Lots of text.SOOOOO much texteven more text3" } --- match_only_text: @@ -5497,7 +5473,6 @@ match_only_text_synthetic: - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5506,13 +5481,11 @@ match_only_text_synthetic: field: script: source: "field('match_only_text').get('')" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5521,13 +5494,11 @@ match_only_text_synthetic: field: script: source: "/* avoid yaml stash */ $('match_only_text', '')" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5536,13 +5507,11 @@ match_only_text_synthetic: field: script: source: "String defaultText = 'default text'; field('match_only_text').get(defaultText)" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "default text" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "default text" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5551,13 +5520,11 @@ match_only_text_synthetic: field: script: source: "String defaultText = 'default text'; $('match_only_text', defaultText)" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "default text" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "default text" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5566,13 +5533,11 @@ match_only_text_synthetic: field: script: source: "field('match_only_text').get(1, '')" -# - match: { hits.hits.0.fields.field.0: "" } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "" } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5581,13 +5546,11 @@ match_only_text_synthetic: field: script: source: "String defaultText = 'default text'; field('match_only_text').get(1, defaultText)" -# - match: { hits.hits.0.fields.field.0: "default text" } -# - match: { hits.hits.1.fields.field.0: "default text" } -# - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "default text" } + - match: { hits.hits.1.fields.field.0: "default text" } + - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5596,13 +5559,11 @@ match_only_text_synthetic: field: script: source: "field('match_only_text').get(1, '')" -# - match: { hits.hits.0.fields.field.0: "" } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "" } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5611,10 +5572,9 @@ match_only_text_synthetic: field: script: source: "String cat = ''; for (String s : field('match_only_text')) { cat += s; } cat + field('match_only_text').size();" -# - match: { hits.hits.0.fields.field.0: "Lots of text.1" } -# - match: { hits.hits.1.fields.field.0: "0" } -# - match: { hits.hits.2.fields.field.0: "Lots of text.SOOOOO much texteven more text3" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text.1" } + - match: { hits.hits.1.fields.field.0: "0" } + - match: { hits.hits.2.fields.field.0: "Lots of text.SOOOOO much texteven more text3" } --- version and sequence number: diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 26929a62d9766..eddd808bdff11 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; +import org.elasticsearch.index.fielddata.StoredFieldSortedBinaryIndexFieldData; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; @@ -297,17 +298,29 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { - if (fieldDataContext.fielddataOperation() == FielddataOperation.SCRIPT) { - return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( - name(), + if (fieldDataContext.fielddataOperation() != FielddataOperation.SCRIPT) { + throw new IllegalArgumentException(CONTENT_TYPE + " fields do not support sorting and aggregations"); + } + SourceLookup sourceLookup = fieldDataContext.lookupSupplier().get().source(); + if (sourceLookup.alwaysEmpty()) { + return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( + storedFieldNameForSyntheticSource(), CoreValuesSourceType.KEYWORD, - SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), - fieldDataContext.lookupSupplier().get().source(), TextDocValuesField::new - ); + ) { + @Override + protected BytesRef storedToBytesRef(Object stored) { + return new BytesRef((String) stored); + } + }; } - - throw new IllegalArgumentException(CONTENT_TYPE + " fields do not support sorting and aggregations"); + return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), + sourceLookup, + TextDocValuesField::new + ); } private String storedFieldNameForSyntheticSource() { diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java index 1179c03a35ab3..ef83d08d5826a 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java @@ -241,6 +241,16 @@ public List invalidExample() throws IOException { } } + public void testDocValues() throws IOException { + MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "match_only_text"))); + assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + } + + public void testDocValuesLoadedFromSynthetic() throws IOException { + MapperService mapper = createMapperService(syntheticSourceFieldMapping(b -> b.field("type", "match_only_text"))); + assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + } + @Override protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java new file mode 100644 index 0000000000000..5e8b559efee99 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.SortField; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; +import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.ToScriptFieldFactory; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +import java.util.Set; + +public abstract class StoredFieldIndexFieldData implements IndexFieldData.StoredFieldLeafFieldData> { + private final String fieldName; + private final ValuesSourceType valuesSourceType; + protected final ToScriptFieldFactory toScriptFieldFactory; + protected final StoredFieldLoader loader; + + protected StoredFieldIndexFieldData(String fieldName, ValuesSourceType valuesSourceType, ToScriptFieldFactory toScriptFieldFactory) { + this.fieldName = fieldName; + this.valuesSourceType = valuesSourceType; + this.toScriptFieldFactory = toScriptFieldFactory; + this.loader = StoredFieldLoader.create(false, Set.of(fieldName)); + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return valuesSourceType; + } + + @Override + public final StoredFieldLeafFieldData load(LeafReaderContext context) { + return loadDirect(context); + } + + @Override + public final StoredFieldLeafFieldData loadDirect(LeafReaderContext context) { + return new StoredFieldLeafFieldData(loader.getLoader(context, null)); + } + + protected abstract T loadLeaf(LeafStoredFieldLoader leafStoredFieldLoader); + + @Override + public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + throw new IllegalArgumentException("not supported for stored field fallback"); + } + + @Override + public BucketedSort newBucketedSort( + BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra + ) { + throw new IllegalArgumentException("not supported for stored field fallback"); + } + + public class StoredFieldLeafFieldData implements LeafFieldData { + private final LeafStoredFieldLoader loader; + + protected StoredFieldLeafFieldData(LeafStoredFieldLoader loader) { + this.loader = loader; + } + + @Override + public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { + return toScriptFieldFactory.getScriptFieldFactory(loadLeaf(loader), fieldName); + } + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void close() {} + + @Override + public SortedBinaryDocValues getBytesValues() { + throw new IllegalArgumentException("not supported for source fallback"); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java new file mode 100644 index 0000000000000..764e875273ab7 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; +import org.elasticsearch.script.field.ToScriptFieldFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public abstract class StoredFieldSortedBinaryIndexFieldData extends StoredFieldIndexFieldData { + + protected StoredFieldSortedBinaryIndexFieldData( + String fieldName, + ValuesSourceType valuesSourceType, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, toScriptFieldFactory); + } + + @Override + protected SourceValueFetcherSortedBinaryDocValues loadLeaf(LeafStoredFieldLoader leafStoredFieldLoader) { + return new SourceValueFetcherSortedBinaryDocValues(leafStoredFieldLoader); + } + + protected abstract BytesRef storedToBytesRef(Object stored); + + class SourceValueFetcherSortedBinaryDocValues extends SortedBinaryDocValues { + private final LeafStoredFieldLoader loader; + private final List sorted = new ArrayList<>(); + + private int current; + private int docValueCount; + + SourceValueFetcherSortedBinaryDocValues(LeafStoredFieldLoader loader) { + this.loader = loader; + } + + @Override + public boolean advanceExact(int doc) throws IOException { + loader.advanceTo(doc); + List values = loader.storedFields().get(getFieldName()); + if (values == null || values.isEmpty()) { + current = 0; + docValueCount = 0; + return false; + } + sorted.clear(); + for (Object o : values) { + sorted.add(storedToBytesRef(o)); + } + Collections.sort(sorted); + current = 0; + docValueCount = sorted.size(); + return true; + } + + @Override + public int docValueCount() { + return docValueCount; + } + + @Override + public BytesRef nextValue() throws IOException { + assert current < docValueCount; + return sorted.get(current++); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 42b2654cad41b..efc505f590a2c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -47,6 +47,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; +import org.elasticsearch.index.fielddata.StoredFieldSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; @@ -58,6 +59,7 @@ import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.search.runtime.StringScriptFieldFuzzyQuery; import org.elasticsearch.search.runtime.StringScriptFieldPrefixQuery; import org.elasticsearch.search.runtime.StringScriptFieldRegexpQuery; @@ -685,30 +687,45 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (operation == FielddataOperation.SEARCH) { failIfNoDocValues(); + return fieldDataFromDocValues(); } - - if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) { - return new SortedSetOrdinalsIndexFieldData.Builder( - name(), - CoreValuesSourceType.KEYWORD, - (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n) - ); + if (operation != FielddataOperation.SCRIPT) { + throw new IllegalStateException("unknown operation [" + operation.name() + "]"); } - if (operation == FielddataOperation.SCRIPT) { - SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); - Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); - - return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( + if (hasDocValues()) { + return fieldDataFromDocValues(); + } + SourceLookup sourceLookup = fieldDataContext.lookupSupplier().get().source(); + if (sourceLookup.alwaysEmpty() && isStored()) { + return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( name(), CoreValuesSourceType.KEYWORD, - sourceValueFetcher(sourcePaths), - searchLookup.source(), KeywordDocValuesField::new - ); + ) { + @Override + protected BytesRef storedToBytesRef(Object stored) { + return (BytesRef) stored; + } + }; } - throw new IllegalStateException("unknown field data type [" + operation.name() + "]"); + Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); + return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + sourceValueFetcher(sourcePaths), + sourceLookup, + KeywordDocValuesField::new + ); + } + + private SortedSetOrdinalsIndexFieldData.Builder fieldDataFromDocValues() { + return new SortedSetOrdinalsIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n) + ); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 3595f2972a397..14513905ba91e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -61,12 +61,14 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; +import org.elasticsearch.index.fielddata.StoredFieldSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.script.field.DelegateDocValuesField; import org.elasticsearch.script.field.TextDocValuesField; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -897,16 +899,7 @@ public static boolean hasGaps(TokenStream stream) throws IOException { @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { FielddataOperation operation = fieldDataContext.fielddataOperation(); - - if (operation == FielddataOperation.SCRIPT) { - return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( - name(), - CoreValuesSourceType.KEYWORD, - SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), - fieldDataContext.lookupSupplier().get().source(), - TextDocValuesField::new - ); - } else if (operation == FielddataOperation.SEARCH) { + if (operation == FielddataOperation.SEARCH) { if (fielddata == false) { throw new IllegalArgumentException( "Text fields are not optimised for operations that require per-document " @@ -930,7 +923,29 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext ); } - throw new IllegalStateException("unknown field data operation [" + operation.name() + "]"); + if (operation != FielddataOperation.SCRIPT) { + throw new IllegalStateException("unknown field data operation [" + operation.name() + "]"); + } + SourceLookup sourceLookup = fieldDataContext.lookupSupplier().get().source(); + if (sourceLookup.alwaysEmpty() && isStored()) { + return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( + name(), + CoreValuesSourceType.KEYWORD, + TextDocValuesField::new + ) { + @Override + protected BytesRef storedToBytesRef(Object stored) { + return new BytesRef((String) stored); + } + }; + } + return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), + sourceLookup, + TextDocValuesField::new + ); } } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index 337c82fce935a..c08fabe2f67c7 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -126,6 +126,13 @@ private static Tuple> sourceAsMapAndType(Bytes return XContentHelper.convertToMap(source, false); } + /** + * Will there never be a {@code _source}? + */ + public boolean alwaysEmpty() { + return sourceProvider.alwaysEmpty(); + } + /** * Get the source as a {@link Map} of java objects. *

@@ -215,6 +222,11 @@ public interface SourceProvider { boolean hasSourceAsMap(); void setSegmentAndDocument(LeafReaderContext context, int docId); + + /** + * Will there never be a {@code _source}? + */ + boolean alwaysEmpty(); } /** @@ -249,7 +261,12 @@ public boolean hasSourceAsMap() { @Override public void setSegmentAndDocument(LeafReaderContext context, int docId) { - // + // nothing to do + } + + @Override + public boolean alwaysEmpty() { + return true; } } @@ -298,6 +315,11 @@ public boolean hasSourceAsMap() { public void setSegmentAndDocument(LeafReaderContext context, int docId) { // } + + @Override + public boolean alwaysEmpty() { + return false; + } } /** @@ -359,6 +381,11 @@ public boolean hasSourceAsMap() { public void setSegmentAndDocument(LeafReaderContext context, int docId) { // } + + @Override + public boolean alwaysEmpty() { + return false; + } } /** @@ -449,5 +476,10 @@ public List extractRawValuesWithoutCaching(String path) { public boolean hasSourceAsMap() { return sourceProvider != null && sourceProvider.hasSourceAsMap(); } + + @Override + public boolean alwaysEmpty() { + return false; + } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 413a79d8e2764..a0938184752bf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -754,4 +754,21 @@ public void testLegacyField() throws Exception { assertThat(service.fieldType("mykeyw"), instanceOf(KeywordFieldMapper.KeywordFieldType.class)); assertNotEquals(Lucene.KEYWORD_ANALYZER, ((KeywordFieldMapper.KeywordFieldType) service.fieldType("mykeyw")).normalizer()); } + + public void testDocValues() throws IOException { + MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "keyword"))); + assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + } + + public void testDocValuesLoadedFromSource() throws IOException { + MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "keyword").field("doc_values", false))); + assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + } + + public void testDocValuesLoadedFromStoredSynthetic() throws IOException { + MapperService mapper = createMapperService( + syntheticSourceFieldMapping(b -> b.field("type", "keyword").field("doc_values", false).field("store", true)) + ); + assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 25a19a0b763fb..8152e6ad3cf1b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1212,4 +1212,26 @@ public void testIgnoreEagerGlobalOrdinalsOnLegacyIndex() throws IOException { mapperService = createMapperService(Version.fromString("5.0.0"), mapping); assertFalse(((TextFieldMapper) mapperService.documentMapper().mappers().getMapper("field")).fieldType().eagerGlobalOrdinals()); } + + public void testDocValues() throws IOException { + MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "text"))); + assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + } + + public void testDocValuesLoadedFromStoredSynthetic() throws IOException { + MapperService mapper = createMapperService(syntheticSourceFieldMapping(b -> b.field("type", "text").field("store", true))); + assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + } + + public void testDocValuesLoadedFromSubKeywordSynthetic() throws IOException { + MapperService mapper = createMapperService(syntheticSourceFieldMapping(b -> { + b.field("type", "text"); + b.startObject("fields"); + { + b.startObject("raw").field("type", "keyword").endObject(); + } + b.endObject(); + })); + assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 5e83c68db1907..69bebef3a15e0 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -732,4 +732,13 @@ protected final XContentBuilder syntheticSourceMapping(CheckedConsumer buildField) + throws IOException { + return syntheticSourceMapping(b -> { + b.startObject("field"); + buildField.accept(b); + b.endObject(); + }); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 9fd9620c9b04c..dccc152b334fe 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -34,7 +34,9 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.FieldDataContext; +import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.query.SearchExecutionContext; @@ -349,6 +351,26 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi return result.get(); } + protected final void assertScriptDocValues(MapperService mapperService, Object sourceValue, Matcher> dvMatcher) + throws IOException { + withLuceneIndex( + mapperService, + iw -> { iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field("field", sourceValue))).rootDoc()); }, + iw -> { + IndexSearcher searcher = newSearcher(iw); + MappedFieldType ft = mapperService.fieldType("field"); + SearchLookup searchLookup = new SearchLookup(null, null, mapperService.mappingLookup().getSourceProvider()); + IndexFieldData sfd = ft.fielddataBuilder( + new FieldDataContext("", () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) + ).build(null, null); + LeafFieldData lfd = sfd.load(getOnlyLeafReader(searcher.getIndexReader()).getContext()); + DocValuesScriptFieldFactory sff = lfd.getScriptFieldFactory("field"); + sff.setNextDocId(0); + assertThat(sff.toScriptDocValues(), dvMatcher); + } + ); + } + private class UpdateCheck { final XContentBuilder init; final XContentBuilder update; From ab6c34be3b1a1c2b346cc90645fdc2781026a36e Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 9 Sep 2022 09:47:02 -0400 Subject: [PATCH 02/10] Update docs/changelog/89950.yaml --- docs/changelog/89950.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/89950.yaml diff --git a/docs/changelog/89950.yaml b/docs/changelog/89950.yaml new file mode 100644 index 0000000000000..2027209fcb2a2 --- /dev/null +++ b/docs/changelog/89950.yaml @@ -0,0 +1,5 @@ +pr: 89950 +summary: "Synthetic _source: support `field` in many cases" +area: TSDB +type: enhancement +issues: [] From 1c647c75733eca398db1b6170d3648709266fc83 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 9 Sep 2022 09:59:27 -0400 Subject: [PATCH 03/10] Fixup --- .../org/elasticsearch/index/mapper/TextFieldMapperTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 8152e6ad3cf1b..402b081db0984 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1223,6 +1223,7 @@ public void testDocValuesLoadedFromStoredSynthetic() throws IOException { assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/86603") public void testDocValuesLoadedFromSubKeywordSynthetic() throws IOException { MapperService mapper = createMapperService(syntheticSourceFieldMapping(b -> { b.field("type", "text"); From bdc3fde5d5b380b74a516fcd0aa97ec96af4317c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 14 Sep 2022 15:29:18 -0400 Subject: [PATCH 04/10] Oh boy thats a lot of files --- .../script/ScriptScoreBenchmark.java | 3 +- .../AggConstructionContentionBenchmark.java | 2 +- .../extras/MatchOnlyTextFieldMapper.java | 5 ++- .../extras/RankFeatureFieldTypeTests.java | 2 +- .../extras/RankFeaturesFieldTypeTests.java | 2 +- .../extras/ScaledFloatFieldTypeTests.java | 8 +++-- .../mapper/ParentJoinFieldMapperTests.java | 6 ++-- .../percolator/PercolateQueryBuilder.java | 3 +- .../fieldcaps/FieldCapabilitiesFetcher.java | 3 +- .../org/elasticsearch/index/IndexService.java | 5 ++- .../org/elasticsearch/index/IndexWarmer.java | 2 +- .../index/fielddata/FieldDataContext.java | 11 ++++-- .../index/mapper/AbstractScriptFieldType.java | 2 +- .../index/mapper/ConstantFieldType.java | 2 +- .../index/mapper/DocumentParser.java | 12 +++++-- .../index/mapper/KeywordFieldMapper.java | 6 ++-- .../index/mapper/MappedFieldType.java | 10 +++--- .../index/mapper/TextFieldMapper.java | 6 ++-- .../vectors/DenseVectorFieldMapper.java | 2 +- .../index/query/SearchExecutionContext.java | 11 ++++-- .../search/lookup/SourceLookup.java | 32 ----------------- .../index/IndexSortSettingsTests.java | 2 +- .../fielddata/IndexFieldDataServiceTests.java | 35 +++++++++++++------ .../AbstractScriptFieldTypeTestCase.java | 18 +++++++--- .../mapper/CompositeRuntimeFieldTests.java | 8 ++++- .../index/mapper/IdFieldTypeTests.java | 6 ++-- .../index/mapper/IgnoredFieldMapperTests.java | 2 +- .../index/mapper/IndexFieldMapperTests.java | 2 +- .../index/mapper/NumberFieldTypeTests.java | 5 +-- .../mapper/PlaceHolderFieldMapperTests.java | 2 +- .../mapper/ProvidedIdFieldMapperTests.java | 12 +++---- .../index/mapper/RoutingFieldMapperTests.java | 2 +- .../index/mapper/TestRuntimeField.java | 2 +- .../index/mapper/TextFieldMapperTests.java | 7 ++-- .../index/mapper/TextFieldTypeTests.java | 4 +-- .../index/mapper/VersionFieldMapperTests.java | 2 +- .../FlattenedIndexFieldDataTests.java | 4 +-- .../vectors/DenseVectorFieldTypeTests.java | 4 +-- .../vectors/SparseVectorFieldTypeTests.java | 7 ++-- .../index/shard/IndexShardTests.java | 5 ++- .../fetch/subphase/FieldFetcherTests.java | 18 ++++++++-- .../search/sort/AbstractSortTestCase.java | 7 ++-- .../index/mapper/MapperServiceTestCase.java | 16 ++++++--- .../index/mapper/MapperTestCase.java | 30 ++++++++-------- .../aggregations/AggregatorTestCase.java | 7 ++-- ...AggregateDoubleMetricFieldMapperTests.java | 2 +- .../AggregateDoubleMetricFieldTypeTests.java | 2 +- .../downsample/TimeseriesFieldTypeHelper.java | 6 ++-- 48 files changed, 204 insertions(+), 148 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index 47f1b4f413532..57d1897d236c6 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -89,7 +89,8 @@ public class ScriptScoreBenchmark { private final CircuitBreakerService breakerService = new NoneCircuitBreakerService(); private final SearchLookup lookup = new SearchLookup( fieldTypes::get, - (mft, lookup, fdo) -> mft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark")).build(fieldDataCache, breakerService), + (mft, lookup, fdo) -> mft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark", false)) + .build(fieldDataCache, breakerService), new SourceLookup.ReaderSourceProvider() ); diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java index 26a4a8c92f181..622818ea8bbb0 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java @@ -211,7 +211,7 @@ public Analyzer buildCustomAnalyzer( protected IndexFieldData buildFieldData(MappedFieldType ft) { IndexFieldDataCache indexFieldDataCache = indicesFieldDataCache.buildIndexFieldDataCache(new IndexFieldDataCache.Listener() { }, index, ft.name()); - return ft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark")).build(indexFieldDataCache, breakerService); + return ft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark", false)).build(indexFieldDataCache, breakerService); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index eddd808bdff11..60cc93fd84909 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -301,8 +301,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (fieldDataContext.fielddataOperation() != FielddataOperation.SCRIPT) { throw new IllegalArgumentException(CONTENT_TYPE + " fields do not support sorting and aggregations"); } - SourceLookup sourceLookup = fieldDataContext.lookupSupplier().get().source(); - if (sourceLookup.alwaysEmpty()) { + if (fieldDataContext.isSyntheticSource()) { return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( storedFieldNameForSyntheticSource(), CoreValuesSourceType.KEYWORD, @@ -318,7 +317,7 @@ protected BytesRef storedToBytesRef(Object stored) { name(), CoreValuesSourceType.KEYWORD, SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), - sourceLookup, + fieldDataContext.lookupSupplier().get().source(), TextDocValuesField::new ); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java index 57c931992f5db..ecc119068acdd 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java @@ -20,7 +20,7 @@ public class RankFeatureFieldTypeTests extends FieldTypeTestCase { public void testIsNotAggregatable() { MappedFieldType fieldType = new RankFeatureFieldMapper.RankFeatureFieldType("field", Collections.emptyMap(), true); - assertFalse(fieldType.isAggregatable()); + assertFalse(fieldType.isAggregatable(randomBoolean())); } public void testFetchSourceValue() throws IOException { diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldTypeTests.java index 48a31691ac1bc..f0f42253b4471 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldTypeTests.java @@ -17,6 +17,6 @@ public class RankFeaturesFieldTypeTests extends FieldTypeTestCase { public void testIsNotAggregatable() { MappedFieldType fieldType = new RankFeaturesFieldMapper.RankFeaturesFieldType("field", Collections.emptyMap(), true); - assertFalse(fieldType.isAggregatable()); + assertFalse(fieldType.isAggregatable(randomBoolean())); } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java index 0592432640b33..3b622b3a6a824 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java @@ -190,8 +190,9 @@ public void testFieldData() throws IOException { "scaled_float1", scalingFactor ); - IndexNumericFieldData fielddata = (IndexNumericFieldData) f1.fielddataBuilder(FieldDataContext.noRuntimeFields("test")) - .build(null, null); + IndexNumericFieldData fielddata = (IndexNumericFieldData) f1.fielddataBuilder( + FieldDataContext.noRuntimeFields("test", randomBoolean()) + ).build(null, null); assertEquals(fielddata.getNumericType(), IndexNumericFieldData.NumericType.DOUBLE); LeafNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0)); SortedNumericDoubleValues values = leafFieldData.getDoubleValues(); @@ -204,7 +205,8 @@ public void testFieldData() throws IOException { "scaled_float2", scalingFactor ); - fielddata = (IndexNumericFieldData) f2.fielddataBuilder(FieldDataContext.noRuntimeFields("test")).build(null, null); + fielddata = (IndexNumericFieldData) f2.fielddataBuilder(FieldDataContext.noRuntimeFields("test", randomBoolean())) + .build(null, null); leafFieldData = fielddata.load(reader.leaves().get(0)); values = leafFieldData.getDoubleValues(); assertTrue(values.advanceExact(0)); diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java index cb296f5ecbf3b..3a4d5c032b3b6 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java @@ -426,19 +426,19 @@ public void testSubFields() throws IOException { ); ParentJoinFieldMapper mapper = (ParentJoinFieldMapper) mapperService.mappingLookup().getMapper("join_field"); assertTrue(mapper.fieldType().isSearchable()); - assertTrue(mapper.fieldType().isAggregatable()); + assertTrue(mapper.fieldType().isAggregatable(randomBoolean())); Iterator it = mapper.iterator(); FieldMapper next = (FieldMapper) it.next(); assertThat(next.name(), equalTo("join_field#parent")); assertTrue(next.fieldType().isSearchable()); - assertTrue(next.fieldType().isAggregatable()); + assertTrue(next.fieldType().isAggregatable(randomBoolean())); assertTrue(it.hasNext()); next = (FieldMapper) it.next(); assertThat(next.name(), equalTo("join_field#child")); assertTrue(next.fieldType().isSearchable()); - assertTrue(next.fieldType().isAggregatable()); + assertTrue(next.fieldType().isAggregatable(randomBoolean())); assertFalse(it.hasNext()); } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 97fcc404d4e47..2e512dd47abe2 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -650,7 +650,8 @@ public > IFD getForField( delegate.getFullyQualifiedIndex().getName(), delegate::lookup, this::sourcePath, - fielddataOperation + fielddataOperation, + delegate.isSourceSynthetic() ) ); IndexFieldDataCache cache = new IndexFieldDataCache.None(); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java index fdac89eec7a4e..20f6d800c34a9 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java @@ -111,6 +111,7 @@ static Map retrieveFieldCaps( Predicate filter = buildFilter(indexFieldfilter, filters, types, context); boolean isTimeSeriesIndex = context.getIndexSettings().getTimestampBounds() != null; + boolean isSyntheticSource = context.isSourceSynthetic(); Map responseMap = new HashMap<>(); for (String field : fieldNames) { MappedFieldType ft = context.getFieldType(field); @@ -124,7 +125,7 @@ static Map retrieveFieldCaps( isTimeSeriesIndex && ft.getMetricType() != null ? ft.typeName() : ft.familyTypeName(), context.isMetadataField(field), ft.isSearchable(), - ft.isAggregatable(), + ft.isAggregatable(isSyntheticSource), isTimeSeriesIndex ? ft.isDimension() : false, isTimeSeriesIndex ? ft.getMetricType() : null, ft.meta() diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 1d6379c973cf3..99f42e2f92094 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -206,7 +206,10 @@ public IndexService( this.indexSortSupplier = () -> indexSettings.getIndexSortConfig() .buildIndexSort( mapperService::fieldType, - (fieldType, searchLookup) -> indexFieldData.getForField(fieldType, FieldDataContext.noRuntimeFields("index sort")) + (fieldType, searchLookup) -> indexFieldData.getForField( + fieldType, + FieldDataContext.noRuntimeFields("index sort", mapperService.mappingLookup().isSourceSynthetic()) + ) ); } else { this.indexSortSupplier = () -> null; diff --git a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java index ce30af187004c..53ed1820c344b 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java +++ b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java @@ -119,7 +119,7 @@ public TerminationHandle warmReader(final IndexShard indexShard, final Elasticse final long start = System.nanoTime(); IndexFieldData.Global ifd = indexFieldDataService.getForField( fieldType, - FieldDataContext.noRuntimeFields("index warming") + FieldDataContext.noRuntimeFields("index warming", mapperService.mappingLookup().isSourceSynthetic()) ); IndexFieldData global = ifd.loadGlobal(reader); if (reader.leaves().isEmpty() == false) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java b/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java index 3cfab4e599015..6551af98d426b 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java @@ -22,12 +22,14 @@ * @param lookupSupplier a supplier for a SearchLookup to be used by runtime scripts * @param sourcePathsLookup a function to get source paths for a specific field * @param fielddataOperation the operation used to determine data structures to generate fielddata from + * @param isSyntheticSource is the {@code _source} stored in synthetic {@code _source} */ public record FieldDataContext( String fullyQualifiedIndexName, Supplier lookupSupplier, Function> sourcePathsLookup, - MappedFieldType.FielddataOperation fielddataOperation + MappedFieldType.FielddataOperation fielddataOperation, + boolean isSyntheticSource ) { /** @@ -36,13 +38,16 @@ public record FieldDataContext( * Used for validating index sorts, eager global ordinal loading, etc * * @param reason the reason that runtime fields are not supported + * @param isSyntheticSource is the {@code _source} stored in synthetic + * {@code _source} */ - public static FieldDataContext noRuntimeFields(String reason) { + public static FieldDataContext noRuntimeFields(String reason, boolean isSyntheticSource) { return new FieldDataContext( "", () -> { throw new UnsupportedOperationException("Runtime fields not supported for [" + reason + "]"); }, Set::of, - MappedFieldType.FielddataOperation.SEARCH + MappedFieldType.FielddataOperation.SEARCH, + isSyntheticSource ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java index aa0e0c17a52b7..ba7d732ebd76a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java @@ -64,7 +64,7 @@ public final boolean isSearchable() { } @Override - public final boolean isAggregatable() { + public final boolean isAggregatable(boolean isSyntheticSource) { return true; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java index 1f8e7fc77b072..fa0eefd032cf7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java @@ -36,7 +36,7 @@ public ConstantFieldType(String name, Map meta) { } @Override - public final boolean isAggregatable() { + public final boolean isAggregatable(boolean isSyntheticSource) { return true; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index cdfd37f4bf266..d25220ae4b221 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -144,7 +144,13 @@ private static void executeIndexTimeScripts(DocumentParserContext context) { SearchLookup searchLookup = new SearchLookup( context.mappingLookup().indexTimeLookup()::get, (ft, lookup, fto) -> ft.fielddataBuilder( - new FieldDataContext(context.indexSettings().getIndex().getName(), lookup, context.mappingLookup()::sourcePaths, fto) + new FieldDataContext( + context.indexSettings().getIndex().getName(), + lookup, + context.mappingLookup()::sourcePaths, + fto, + context.isSyntheticSource() + ) ).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), new SourceLookup.ReaderSourceProvider() ); @@ -749,7 +755,9 @@ private static Mapper getLeafMapper(final DocumentParserContext context, ObjectM MappedFieldType fieldType = context.mappingLookup().getFieldType(fieldPath); if (fieldType != null) { // we haven't found a mapper with this name above, which means if a field type is found it is for sure a runtime field. - assert fieldType.hasDocValues() == false && fieldType.isAggregatable() && fieldType.isSearchable(); + assert fieldType.hasDocValues() == false + && fieldType.isAggregatable(context.mappingLookup().isSourceSynthetic()) + && fieldType.isSearchable(); return NO_OP_FIELDMAPPER; } return null; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index efc505f590a2c..5195b6151d175 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -59,7 +59,6 @@ import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.search.runtime.StringScriptFieldFuzzyQuery; import org.elasticsearch.search.runtime.StringScriptFieldPrefixQuery; import org.elasticsearch.search.runtime.StringScriptFieldRegexpQuery; @@ -696,8 +695,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (hasDocValues()) { return fieldDataFromDocValues(); } - SourceLookup sourceLookup = fieldDataContext.lookupSupplier().get().source(); - if (sourceLookup.alwaysEmpty() && isStored()) { + if (fieldDataContext.isSyntheticSource() && isStored()) { return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( name(), CoreValuesSourceType.KEYWORD, @@ -715,7 +713,7 @@ protected BytesRef storedToBytesRef(Object stored) { name(), CoreValuesSourceType.KEYWORD, sourceValueFetcher(sourcePaths), - sourceLookup, + fieldDataContext.lookupSupplier().get().source(), KeywordDocValuesField::new ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 06fa0a97fa881..d8d7eacaad9f5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -174,12 +174,14 @@ public Function pointReaderIfPossible() { return null; } - /** Returns true if the field is aggregatable. - * + /** + * Returns true if the field is aggregatable. + * @param isSyntheticSource is the {@code _source} stored in synthetic + * {@code _source} */ - public boolean isAggregatable() { + public boolean isAggregatable(boolean isSyntheticSource) { try { - fielddataBuilder(FieldDataContext.noRuntimeFields("aggregation_check")); + fielddataBuilder(FieldDataContext.noRuntimeFields("aggregation_check", isSyntheticSource)); return true; } catch (IllegalArgumentException e) { return false; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 376a3e74ace62..afeac0a723ae3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -68,7 +68,6 @@ import org.elasticsearch.script.field.DelegateDocValuesField; import org.elasticsearch.script.field.TextDocValuesField; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -930,8 +929,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (operation != FielddataOperation.SCRIPT) { throw new IllegalStateException("unknown field data operation [" + operation.name() + "]"); } - SourceLookup sourceLookup = fieldDataContext.lookupSupplier().get().source(); - if (sourceLookup.alwaysEmpty() && isStored()) { + if (fieldDataContext.isSyntheticSource() && isStored()) { return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( name(), CoreValuesSourceType.KEYWORD, @@ -947,7 +945,7 @@ protected BytesRef storedToBytesRef(Object stored) { name(), CoreValuesSourceType.KEYWORD, SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), - sourceLookup, + fieldDataContext.lookupSupplier().get().source(), TextDocValuesField::new ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 35525da88dec2..2baa4c0337e50 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -278,7 +278,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { } @Override - public boolean isAggregatable() { + public boolean isAggregatable(boolean isSyntheticSource) { return false; } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index c07ae53d910f5..38c31a62b50c9 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -287,7 +287,8 @@ public > IFD getForField(MappedFieldType fieldType fullyQualifiedIndex.getName(), () -> this.lookup().forkAndTrackFieldReferences(fieldType.name()), this::sourcePath, - fielddataOperation + fielddataOperation, + isSourceSynthetic() ) ); } @@ -499,7 +500,13 @@ public SearchLookup lookup() { this::getFieldType, (fieldType, searchLookup, fielddataOperation) -> indexFieldDataLookup.apply( fieldType, - new FieldDataContext(fullyQualifiedIndex.getName(), searchLookup, this::sourcePath, fielddataOperation) + new FieldDataContext( + fullyQualifiedIndex.getName(), + searchLookup, + this::sourcePath, + fielddataOperation, + isSourceSynthetic() + ) ), sourceProvider ); diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index c08fabe2f67c7..347026fee547f 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -126,13 +126,6 @@ private static Tuple> sourceAsMapAndType(Bytes return XContentHelper.convertToMap(source, false); } - /** - * Will there never be a {@code _source}? - */ - public boolean alwaysEmpty() { - return sourceProvider.alwaysEmpty(); - } - /** * Get the source as a {@link Map} of java objects. *

@@ -222,11 +215,6 @@ public interface SourceProvider { boolean hasSourceAsMap(); void setSegmentAndDocument(LeafReaderContext context, int docId); - - /** - * Will there never be a {@code _source}? - */ - boolean alwaysEmpty(); } /** @@ -263,11 +251,6 @@ public boolean hasSourceAsMap() { public void setSegmentAndDocument(LeafReaderContext context, int docId) { // nothing to do } - - @Override - public boolean alwaysEmpty() { - return true; - } } /** @@ -315,11 +298,6 @@ public boolean hasSourceAsMap() { public void setSegmentAndDocument(LeafReaderContext context, int docId) { // } - - @Override - public boolean alwaysEmpty() { - return false; - } } /** @@ -381,11 +359,6 @@ public boolean hasSourceAsMap() { public void setSegmentAndDocument(LeafReaderContext context, int docId) { // } - - @Override - public boolean alwaysEmpty() { - return false; - } } /** @@ -476,10 +449,5 @@ public List extractRawValuesWithoutCaching(String path) { public boolean hasSourceAsMap() { return sourceProvider != null && sourceProvider.hasSourceAsMap(); } - - @Override - public boolean alwaysEmpty() { - return false; - } } } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index 6cbfb53c3f584..d67e6f66d0f17 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -220,7 +220,7 @@ private Sort buildIndexSort(IndexSettings indexSettings, Map indexFieldDataService.getForField( ft, - new FieldDataContext("test", s, Set::of, MappedFieldType.FielddataOperation.SEARCH) + new FieldDataContext("test", s, Set::of, MappedFieldType.FielddataOperation.SEARCH, randomBoolean()) ) ); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index b56fb251a3f75..32df5320f3fae 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -72,6 +72,7 @@ protected Collection> getPlugins() { public void testGetForFieldDefaults() { final IndexService indexService = createIndex("test"); + final boolean isSyntheticSource = indexService.mapperService().mappingLookup().isSourceSynthetic(); final IndicesService indicesService = getInstanceFromNode(IndicesService.class); final IndexFieldDataService ifdService = new IndexFieldDataService( indexService.getIndexSettings(), @@ -81,7 +82,7 @@ public void testGetForFieldDefaults() { MapperBuilderContext context = MapperBuilderContext.ROOT; final MappedFieldType stringMapper = new KeywordFieldMapper.Builder("string", Version.CURRENT).build(context).fieldType(); ifdService.clear(); - IndexFieldData fd = ifdService.getForField(stringMapper, FieldDataContext.noRuntimeFields("test")); + IndexFieldData fd = ifdService.getForField(stringMapper, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); assertTrue(fd instanceof SortedSetOrdinalsIndexFieldData); for (MappedFieldType mapper : Arrays.asList( @@ -91,7 +92,7 @@ public void testGetForFieldDefaults() { new NumberFieldMapper.Builder("long", LONG, ScriptCompiler.NONE, false, true, Version.CURRENT).build(context).fieldType() )) { ifdService.clear(); - fd = ifdService.getForField(mapper, FieldDataContext.noRuntimeFields("test")); + fd = ifdService.getForField(mapper, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); assertTrue(fd instanceof SortedNumericIndexFieldData); } @@ -104,7 +105,7 @@ public void testGetForFieldDefaults() { Version.CURRENT ).build(context).fieldType(); ifdService.clear(); - fd = ifdService.getForField(floatMapper, FieldDataContext.noRuntimeFields("test")); + fd = ifdService.getForField(floatMapper, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); assertTrue(fd instanceof SortedDoublesIndexFieldData); final MappedFieldType doubleMapper = new NumberFieldMapper.Builder( @@ -116,7 +117,7 @@ public void testGetForFieldDefaults() { Version.CURRENT ).build(context).fieldType(); ifdService.clear(); - fd = ifdService.getForField(doubleMapper, FieldDataContext.noRuntimeFields("test")); + fd = ifdService.getForField(doubleMapper, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); assertTrue(fd instanceof SortedDoublesIndexFieldData); } @@ -137,7 +138,16 @@ public void testGetForFieldRuntimeField() { return (IndexFieldData.Builder) (cache, breakerService) -> null; }); SearchLookup searchLookup = new SearchLookup(null, null, new SourceLookup.ReaderSourceProvider()); - ifdService.getForField(ft, new FieldDataContext("qualified", () -> searchLookup, null, MappedFieldType.FielddataOperation.SEARCH)); + ifdService.getForField( + ft, + new FieldDataContext( + "qualified", + () -> searchLookup, + null, + MappedFieldType.FielddataOperation.SEARCH, + indexService.mapperService().mappingLookup().isSourceSynthetic() + ) + ); assertSame(searchLookup, searchLookupSetOnce.get().get()); } @@ -177,8 +187,9 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon onRemovalCalled.incrementAndGet(); } }); - IndexFieldData ifd1 = ifdService.getForField(mapper1, FieldDataContext.noRuntimeFields("test")); - IndexFieldData ifd2 = ifdService.getForField(mapper2, FieldDataContext.noRuntimeFields("test")); + boolean isSyntheticSource = indexService.mapperService().mappingLookup().isSourceSynthetic(); + IndexFieldData ifd1 = ifdService.getForField(mapper1, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); + IndexFieldData ifd2 = ifdService.getForField(mapper2, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); LeafFieldData loadField1 = ifd1.load(leafReaderContext); LeafFieldData loadField2 = ifd2.load(leafReaderContext); @@ -252,7 +263,10 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon onRemovalCalled.incrementAndGet(); } }); - IndexFieldData ifd = ifdService.getForField(mapper1, FieldDataContext.noRuntimeFields("test")); + IndexFieldData ifd = ifdService.getForField( + mapper1, + FieldDataContext.noRuntimeFields("test", indexService.mapperService().mappingLookup().isSourceSynthetic()) + ); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); LeafFieldData load = ifd.load(leafReaderContext); assertEquals(1, onCacheCalled.get()); @@ -313,12 +327,13 @@ private void doTestRequireDocValues(MappedFieldType ft) { cache, null ); + boolean isSyntheticSource = randomBoolean(); if (ft.hasDocValues()) { - ifds.getForField(ft, FieldDataContext.noRuntimeFields("test")); // no exception + ifds.getForField(ft, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); // no exception } else { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> ifds.getForField(ft, FieldDataContext.noRuntimeFields("test")) + () -> ifds.getForField(ft, FieldDataContext.noRuntimeFields("test", isSyntheticSource)) ); assertThat(e.getMessage(), containsString("doc values")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index 5f979c1cb58ce..8a80d5cb2aafb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -153,7 +153,7 @@ public void testFieldCaps() throws Exception { MappedFieldType concreteIndexType = concreteIndexMapping.fieldType("field"); assertEquals(concreteIndexType.familyTypeName(), scriptFieldType.familyTypeName()); assertEquals(concreteIndexType.isSearchable(), scriptFieldType.isSearchable()); - assertEquals(concreteIndexType.isAggregatable(), scriptFieldType.isAggregatable()); + assertEquals(concreteIndexType.isAggregatable(false), scriptFieldType.isAggregatable(false)); } @SuppressWarnings("unused") @@ -188,7 +188,14 @@ protected static SearchExecutionContext mockContext() { } protected static FieldDataContext mockFielddataContext() { - return new FieldDataContext("test", mockContext()::lookup, mockContext()::sourcePath, MappedFieldType.FielddataOperation.SCRIPT); + SearchExecutionContext searchExecutionContext = mockContext(); + return new FieldDataContext( + "test", + searchExecutionContext::lookup, + mockContext()::sourcePath, + MappedFieldType.FielddataOperation.SCRIPT, + searchExecutionContext.isSourceSynthetic() + ); } protected static SearchExecutionContext mockContext(boolean allowExpensiveQueries) { @@ -219,15 +226,16 @@ protected static SearchExecutionContext mockContext( when(context.allowExpensiveQueries()).thenReturn(allowExpensiveQueries); SearchLookup lookup = new SearchLookup( context::getFieldType, - (mft, lookupSupplier, fdo) -> mft.fielddataBuilder(new FieldDataContext("test", lookupSupplier, context::sourcePath, fdo)) - .build(null, null), + (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( + new FieldDataContext("test", lookupSupplier, context::sourcePath, fdo, context.isSourceSynthetic()) + ).build(null, null), sourceProvider ); when(context.lookup()).thenReturn(lookup); when(context.getForField(any(), any())).then(args -> { MappedFieldType ft = args.getArgument(0); MappedFieldType.FielddataOperation fdo = args.getArgument(1); - return ft.fielddataBuilder(new FieldDataContext("test", context::lookup, context::sourcePath, fdo)) + return ft.fielddataBuilder(new FieldDataContext("test", context::lookup, context::sourcePath, fdo, context.isSourceSynthetic())) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); }); return context; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java index b2ce1482ce02c..681251689f89b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java @@ -336,7 +336,13 @@ public void testParseDocumentSubFieldAccess() throws IOException { SearchLookup searchLookup = new SearchLookup( mapperService::fieldType, (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( - new FieldDataContext("test", lookupSupplier, mapperService.mappingLookup()::sourcePaths, fdo) + new FieldDataContext( + "test", + lookupSupplier, + mapperService.mappingLookup()::sourcePaths, + fdo, + mapperService.mappingLookup().isSourceSynthetic() + ) ).build(null, null), new SourceLookup.ReaderSourceProvider() ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java index 9320a6c5de2ea..07a40d0e77a42 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java @@ -54,12 +54,12 @@ public void testTermsQuery() { public void testIsAggregatable() { MappedFieldType ft = new ProvidedIdFieldMapper.IdFieldType(() -> false); - assertFalse(ft.isAggregatable()); + assertFalse(ft.isAggregatable(randomBoolean())); ft = new ProvidedIdFieldMapper.IdFieldType(() -> true); - assertTrue(ft.isAggregatable()); + assertTrue(ft.isAggregatable(randomBoolean())); ft = new TsidExtractingIdFieldMapper.IdFieldType(); - assertFalse(ft.isAggregatable()); + assertFalse(ft.isAggregatable(randomBoolean())); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java index 836f6fe3d8af1..c082840e688d8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java @@ -53,7 +53,7 @@ public void testFetchIgnoredFieldValue() throws IOException { iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java index 84ada85884f24..ef7df900a52fb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java @@ -50,7 +50,7 @@ public void testFetchFieldValue() throws IOException { IndexFieldMapper.IndexFieldType ft = (IndexFieldMapper.IndexFieldType) mapperService.fieldType("_index"); SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 51e9b9713a15d..fa2b2311a90c2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -583,8 +583,9 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS // Create an index writer configured with the same index sort. NumberFieldType fieldType = new NumberFieldType("field", type); - IndexNumericFieldData fielddata = (IndexNumericFieldData) fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test")) - .build(null, null); + IndexNumericFieldData fielddata = (IndexNumericFieldData) fieldType.fielddataBuilder( + FieldDataContext.noRuntimeFields("test", randomBoolean()) + ).build(null, null); SortField sortField = fielddata.sortField(null, MultiValueMode.MIN, null, randomBoolean()); IndexWriterConfig writerConfig = new IndexWriterConfig(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java index 9d726f46154b6..b7035f77aab2e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java @@ -64,7 +64,7 @@ public void testFetchValue() throws Exception { }, iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java index 4df0c9823ec87..cab5ec2f929bc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java @@ -50,23 +50,23 @@ public void testDefaults() throws IOException { } public void testEnableFieldData() throws IOException { - boolean[] enabled = new boolean[1]; MapperService mapperService = createMapperService(() -> enabled[0], mapping(b -> {})); + boolean isSyntheticSource = mapperService.mappingLookup().isSourceSynthetic(); ProvidedIdFieldMapper.IdFieldType ft = (ProvidedIdFieldMapper.IdFieldType) mapperService.fieldType("_id"); IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, - () -> ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test")).build(null, null) + () -> ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test", isSyntheticSource)).build(null, null) ); assertThat(exc.getMessage(), containsString(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())); - assertFalse(ft.isAggregatable()); + assertFalse(ft.isAggregatable(isSyntheticSource)); enabled[0] = true; - ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test")).build(null, null); + ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test", isSyntheticSource)).build(null, null); assertWarnings(ProvidedIdFieldMapper.ID_FIELD_DATA_DEPRECATION_MESSAGE); - assertTrue(ft.isAggregatable()); + assertTrue(ft.isAggregatable(isSyntheticSource)); } public void testFetchIdFieldValue() throws IOException { @@ -78,7 +78,7 @@ public void testFetchIdFieldValue() throws IOException { iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index 0ab5c7854c336..1533a5179584a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -71,7 +71,7 @@ public void testFetchRoutingFieldValue() throws IOException { iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java index 40e9f4490903e..aa96b4011dd9c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java @@ -80,7 +80,7 @@ public boolean isSearchable() { } @Override - public boolean isAggregatable() { + public boolean isAggregatable(boolean isSyntheticSource) { return true; } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index c9ab24cad83a8..3c92692d43f7a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -528,10 +528,11 @@ public void testEagerGlobalOrdinals() throws IOException { public void testFielddata() throws IOException { MapperService disabledMapper = createMapperService(fieldMapping(this::minimalMapping)); + boolean isSyntheticSource = disabledMapper.mappingLookup().isSourceSynthetic(); Exception e = expectThrows( IllegalArgumentException.class, () -> disabledMapper.fieldType("field") - .fielddataBuilder(new FieldDataContext("index", null, null, MappedFieldType.FielddataOperation.SEARCH)) + .fielddataBuilder(new FieldDataContext("index", null, null, MappedFieldType.FielddataOperation.SEARCH, isSyntheticSource)) ); assertThat( e.getMessage(), @@ -542,7 +543,7 @@ public void testFielddata() throws IOException { ); MapperService enabledMapper = createMapperService(fieldMapping(b -> b.field("type", "text").field("fielddata", true))); - enabledMapper.fieldType("field").fielddataBuilder(FieldDataContext.noRuntimeFields("test")); // no exception + enabledMapper.fieldType("field").fielddataBuilder(FieldDataContext.noRuntimeFields("test", isSyntheticSource)); // no exception e = expectThrows( MapperParsingException.class, () -> createMapperService(fieldMapping(b -> b.field("type", "text").field("index", false).field("fielddata", true))) @@ -1207,7 +1208,7 @@ public void testIgnoreFieldDataOnLegacyIndex() throws IOException { expectThrows( IllegalArgumentException.class, () -> ((TextFieldMapper) finalMapperService.documentMapper().mappers().getMapper("field")).fieldType() - .fielddataBuilder(FieldDataContext.noRuntimeFields("test")) + .fielddataBuilder(FieldDataContext.noRuntimeFields("test", finalMapperService.mappingLookup().isSourceSynthetic())) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 755dd3f668bde..1a7c224a26813 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -50,9 +50,9 @@ private static TextFieldType createFieldType() { public void testIsAggregatableDependsOnFieldData() { TextFieldType ft = createFieldType(); - assertFalse(ft.isAggregatable()); + assertFalse(ft.isAggregatable(randomBoolean())); ft.setFielddata(true); - assertTrue(ft.isAggregatable()); + assertTrue(ft.isAggregatable(randomBoolean())); } public void testTermQuery() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java index b05697088f3d0..1ae505c0a9029 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java @@ -53,7 +53,7 @@ public void testFetchFieldValue() throws IOException { VersionFieldMapper.VersionFieldType ft = (VersionFieldMapper.VersionFieldType) mapperService.fieldType("_version"); SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java index 41bcfa4f365ab..af8db87dba2d6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java @@ -71,7 +71,7 @@ public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) { DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("test", "_na_", 1)); // Load global field data for subfield 'key'. - IndexFieldData ifd1 = ifdService.getForField(fieldType1, FieldDataContext.noRuntimeFields("test")); + IndexFieldData ifd1 = ifdService.getForField(fieldType1, FieldDataContext.noRuntimeFields("test", false)); assertTrue(ifd1 instanceof KeyedFlattenedFieldData); KeyedFlattenedFieldData fieldData1 = (KeyedFlattenedFieldData) ifd1; @@ -81,7 +81,7 @@ public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) { // Load global field data for the subfield 'other_key'. MappedFieldType fieldType2 = fieldMapper.fieldType().getChildFieldType("other_key"); - IndexFieldData ifd2 = ifdService.getForField(fieldType2, FieldDataContext.noRuntimeFields("test")); + IndexFieldData ifd2 = ifdService.getForField(fieldType2, FieldDataContext.noRuntimeFields("test", false)); assertTrue(ifd2 instanceof KeyedFlattenedFieldData); KeyedFlattenedFieldData fieldData2 = (KeyedFlattenedFieldData) ifd2; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index e9e4c6f22d140..c55b9fae0bfaa 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -48,12 +48,12 @@ public void testIsSearchable() { public void testIsAggregatable() { DenseVectorFieldType ft = createFieldType(); - assertFalse(ft.isAggregatable()); + assertFalse(ft.isAggregatable(randomBoolean())); } public void testFielddataBuilder() { DenseVectorFieldType ft = createFieldType(); - assertNotNull(ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test"))); + assertNotNull(ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test", randomBoolean()))); } public void testDocValueFormat() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java index 574fb63cd3fb0..2d730e6b7f925 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java @@ -19,12 +19,15 @@ public class SparseVectorFieldTypeTests extends FieldTypeTestCase { public void testDocValuesDisabled() { MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType("field", Collections.emptyMap()); assertFalse(fieldType.hasDocValues()); - expectThrows(IllegalArgumentException.class, () -> fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test"))); + expectThrows( + IllegalArgumentException.class, + () -> fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test", randomBoolean())) + ); } public void testIsNotAggregatable() { MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType("field", Collections.emptyMap()); - assertFalse(fieldType.isAggregatable()); + assertFalse(fieldType.isAggregatable(randomBoolean())); } public void testDocValueFormatIsNotSupported() { diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index ad07c073ed9b1..4230ba7ae7701 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2669,7 +2669,10 @@ public void testReaderWrapperWorksWithGlobalOrdinals() throws IOException { indicesFieldDataCache, new NoneCircuitBreakerService() ); - IndexFieldData.Global ifd = indexFieldDataService.getForField(foo, FieldDataContext.noRuntimeFields("test")); + IndexFieldData.Global ifd = indexFieldDataService.getForField( + foo, + FieldDataContext.noRuntimeFields("test", shard.mapperService().mappingLookup().isSourceSynthetic()) + ); FieldDataStats before = shard.fieldData().stats("foo"); assertThat(before.getMemorySizeInBytes(), equalTo(0L)); FieldDataStats after = null; diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index bb51fd74c1c91..ee0ccdce422a5 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -236,7 +236,11 @@ public void testMetadataFields() throws IOException { FieldFetcher fieldFetcher = FieldFetcher.create( newSearchExecutionContext( mapperService, - (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup()).apply(ft, fdc.lookupSupplier(), fdc.fielddataOperation()) + (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup(), mapperService.mappingLookup().isSourceSynthetic()).apply( + ft, + fdc.lookupSupplier(), + fdc.fielddataOperation() + ) ), fieldList ); @@ -1124,7 +1128,11 @@ public void testFetchRuntimeFieldWithSourceDisabled() throws IOException { MapperService mapperService = createMapperService(mapping); SearchExecutionContext searchExecutionContext = newSearchExecutionContext( mapperService, - (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup()).apply(ft, fdc.lookupSupplier(), fdc.fielddataOperation()) + (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup(), mapperService.mappingLookup().isSourceSynthetic()).apply( + ft, + fdc.lookupSupplier(), + fdc.fielddataOperation() + ) ); withLuceneIndex(mapperService, iw -> iw.addDocument(new LuceneDocument()), iw -> { FieldFetcher fieldFetcher = FieldFetcher.create(searchExecutionContext, fieldAndFormatList("runtime_field", null, false)); @@ -1153,7 +1161,11 @@ public void testFetchMetadataFieldWithSourceDisabled() throws IOException { MapperService mapperService = createMapperService(mapping); SearchExecutionContext searchExecutionContext = newSearchExecutionContext( mapperService, - (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup()).apply(ft, fdc.lookupSupplier(), fdc.fielddataOperation()) + (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup(), mapperService.mappingLookup().isSourceSynthetic()).apply( + ft, + fdc.lookupSupplier(), + fdc.fielddataOperation() + ) ); withLuceneIndex(mapperService, iw -> { ParsedDocument parsedDocument = mapperService.documentMapper().parse(source("{}")); diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index bbcbd72e8db93..9a60adcb6d5a1 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.NestedObjectMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -48,7 +49,6 @@ import org.elasticsearch.xcontent.XContentType; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.mockito.Mockito; import java.io.IOException; import java.util.Collections; @@ -60,6 +60,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; +import static org.mockito.Mockito.mock; public abstract class AbstractSortTestCase> extends ESTestCase { @@ -187,7 +188,7 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea index, Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build() ); - BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, Mockito.mock(BitsetFilterCache.Listener.class)); + BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, mock(BitsetFilterCache.Listener.class)); BiFunction> indexFieldDataLookup = (fieldType, fdc) -> { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fdc); return builder.build(new IndexFieldDataCache.None(), null); @@ -202,7 +203,7 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea bitsetFilterCache, indexFieldDataLookup, null, - null, + MappingLookup.EMPTY, null, scriptService, parserConfig(), diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 69bebef3a15e0..eb270ae144fd6 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -454,7 +454,7 @@ public Query filterQuery(Query query) { @Override protected IndexFieldData buildFieldData(MappedFieldType ft) { - return ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test")) + return ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test", mapperService.mappingLookup().isSourceSynthetic())) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } @@ -648,10 +648,18 @@ public void onRemoval(ShardId shardId, Accountable accountable) { } protected TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup( - Function> sourcePathsLookup + MapperService mapperService ) { - return (mft, lookupSource, fdo) -> mft.fielddataBuilder(new FieldDataContext("test", lookupSource, sourcePathsLookup, fdo)) - .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + return fieldDataLookup(mapperService.mappingLookup()::sourcePaths, mapperService.mappingLookup().isSourceSynthetic()); + } + + protected TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup( + Function> sourcePathsLookup, + boolean isSyntheticSource + ) { + return (mft, lookupSource, fdo) -> mft.fielddataBuilder( + new FieldDataContext("test", lookupSource, sourcePathsLookup, fdo, isSyntheticSource) + ).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer build) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index dccc152b334fe..a6acfb6971f2a 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -337,7 +337,7 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft, MappedFieldType.FielddataOperation.SEARCH)); @@ -361,7 +361,13 @@ protected final void assertScriptDocValues(MapperService mapperService, Object s MappedFieldType ft = mapperService.fieldType("field"); SearchLookup searchLookup = new SearchLookup(null, null, mapperService.mappingLookup().getSourceProvider()); IndexFieldData sfd = ft.fielddataBuilder( - new FieldDataContext("", () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) + new FieldDataContext( + "", + () -> searchLookup, + Set::of, + MappedFieldType.FielddataOperation.SCRIPT, + mapperService.mappingLookup().isSourceSynthetic() + ) ).build(null, null); LeafFieldData lfd = sfd.load(getOnlyLeafReader(searcher.getIndexReader()).getContext()); DocValuesScriptFieldFactory sff = lfd.getScriptFieldFactory("field"); @@ -614,18 +620,14 @@ protected void assertFetch(MapperService mapperService, String field, Object val SourceToParse source = source(b -> b.field(ft.name(), value)); ValueFetcher docValueFetcher = new DocValueFetcher( ft.docValueFormat(format, null), - ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test")) + ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test", mapperService.mappingLookup().isSourceSynthetic())) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer( - inv -> fieldDataLookup(mapperService.mappingLookup()::sourcePaths).apply( - ft, - () -> { throw new UnsupportedOperationException(); }, - fdt - ) + inv -> fieldDataLookup(mapperService).apply(ft, () -> { throw new UnsupportedOperationException(); }, fdt) ); ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); @@ -687,7 +689,8 @@ public final void testIndexTimeFieldData() throws IOException { MapperService mapperService = createMapperService(fieldMapping(this::minimalMapping)); assertParseMinimalWarnings(); MappedFieldType fieldType = mapperService.fieldType("field"); - if (fieldType.isAggregatable() == false) { + boolean isSyntheticSource = mapperService.mappingLookup().isSourceSynthetic(); + if (fieldType.isAggregatable(isSyntheticSource) == false) { return; // No field data available, so we ignore } SourceToParse source = source(this::writeField); @@ -697,14 +700,13 @@ public final void testIndexTimeFieldData() throws IOException { LeafReaderContext ctx = ir.leaves().get(0); - DocValuesScriptFieldFactory docValuesFieldSource = fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test")) - .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) - .load(ctx) - .getScriptFieldFactory("test"); + DocValuesScriptFieldFactory docValuesFieldSource = fieldType.fielddataBuilder( + FieldDataContext.noRuntimeFields("test", isSyntheticSource) + ).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()).load(ctx).getScriptFieldFactory("test"); docValuesFieldSource.setNextDocId(0); DocumentLeafReader reader = new DocumentLeafReader(doc.rootDoc(), Collections.emptyMap()); - DocValuesScriptFieldFactory indexData = fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test")) + DocValuesScriptFieldFactory indexData = fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test", isSyntheticSource)) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) .load(reader.getContext()) .getScriptFieldFactory("test"); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 4b3010c282f95..13fec4d45873f 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -297,7 +297,8 @@ protected AggregationContext createAggregationContext( indexSettings.getIndex().getName(), context.lookupSupplier(), context.sourcePathsLookup(), - context.fielddataOperation() + context.fielddataOperation(), + context.isSyntheticSource() ) ).build(new IndexFieldDataCache.None(), breakerService); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() { @@ -992,7 +993,7 @@ public void testSupportedFieldTypes() throws IOException { MappedFieldType fieldType = mapper.fieldType(); // Non-aggregatable fields are not testable (they will throw an error on all aggs anyway), so skip - if (fieldType.isAggregatable() == false) { + if (fieldType.isAggregatable(false) == false) { continue; } @@ -1054,7 +1055,7 @@ public void testSupportedFieldTypes() throws IOException { } private ValuesSourceType fieldToVST(MappedFieldType fieldType) { - return fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test")).build(null, null).getValuesSourceType(); + return fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test", false)).build(null, null).getValuesSourceType(); } /** diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java index 3987f4b3bfb2e..5a47a7d098621 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java @@ -524,7 +524,7 @@ public void testFieldCaps() throws IOException { MappedFieldType fieldType = aggMetricMapperService.fieldType("field"); assertThat(fieldType.familyTypeName(), equalTo("double")); assertTrue(fieldType.isSearchable()); - assertTrue(fieldType.isAggregatable()); + assertTrue(fieldType.isAggregatable(aggMetricMapperService.mappingLookup().isSourceSynthetic())); } /* diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java index 95e7545f25525..20f766c15acf5 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java @@ -124,7 +124,7 @@ public void testUsedInScript() throws IOException { SearchLookup lookup = new SearchLookup( searchExecutionContext::getFieldType, (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( - new FieldDataContext("test", lookupSupplier, searchExecutionContext::sourcePath, fdo) + new FieldDataContext("test", lookupSupplier, searchExecutionContext::sourcePath, fdo, randomBoolean()) ).build(null, null), new SourceLookup.ReaderSourceProvider() ); diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java index 21cd5a27aa2b7..b4caef24135b2 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.indices.IndicesService; @@ -32,10 +33,11 @@ private TimeseriesFieldTypeHelper(final MapperService mapperService, final Strin } public boolean isTimeSeriesLabel(final String field, final Map unused) { - final MappedFieldType fieldType = mapperService.mappingLookup().getFieldType(field); + final MappingLookup lookup = mapperService.mappingLookup(); + final MappedFieldType fieldType = lookup.getFieldType(field); return fieldType != null && (timestampField.equals(field) == false) - && (fieldType.isAggregatable()) + && (fieldType.isAggregatable(lookup.isSourceSynthetic())) && (fieldType.isDimension() == false) && (mapperService.isMetadataField(field) == false); } From 16a99256be883d094e4c2012aedec22c5eb52da8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 21 Sep 2022 09:54:15 -0400 Subject: [PATCH 05/10] No param --- .../index/mapper/extras/RankFeatureFieldTypeTests.java | 2 +- .../index/mapper/extras/RankFeaturesFieldTypeTests.java | 2 +- .../join/mapper/ParentJoinFieldMapperTests.java | 6 +++--- .../action/fieldcaps/FieldCapabilitiesFetcher.java | 3 +-- .../index/mapper/AbstractScriptFieldType.java | 2 +- .../org/elasticsearch/index/mapper/ConstantFieldType.java | 2 +- .../org/elasticsearch/index/mapper/DocumentParser.java | 4 +--- .../org/elasticsearch/index/mapper/MappedFieldType.java | 7 +++---- .../index/mapper/vectors/DenseVectorFieldMapper.java | 2 +- .../index/mapper/AbstractScriptFieldTypeTestCase.java | 2 +- .../org/elasticsearch/index/mapper/IdFieldTypeTests.java | 6 +++--- .../index/mapper/ProvidedIdFieldMapperTests.java | 4 ++-- .../org/elasticsearch/index/mapper/TestRuntimeField.java | 2 +- .../org/elasticsearch/index/mapper/TextFieldTypeTests.java | 4 ++-- .../index/mapper/vectors/DenseVectorFieldTypeTests.java | 2 +- .../index/mapper/vectors/SparseVectorFieldTypeTests.java | 2 +- .../org/elasticsearch/index/mapper/MapperTestCase.java | 2 +- .../search/aggregations/AggregatorTestCase.java | 2 +- .../mapper/AggregateDoubleMetricFieldMapperTests.java | 2 +- .../xpack/downsample/TimeseriesFieldTypeHelper.java | 2 +- 20 files changed, 28 insertions(+), 32 deletions(-) diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java index ecc119068acdd..57c931992f5db 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java @@ -20,7 +20,7 @@ public class RankFeatureFieldTypeTests extends FieldTypeTestCase { public void testIsNotAggregatable() { MappedFieldType fieldType = new RankFeatureFieldMapper.RankFeatureFieldType("field", Collections.emptyMap(), true); - assertFalse(fieldType.isAggregatable(randomBoolean())); + assertFalse(fieldType.isAggregatable()); } public void testFetchSourceValue() throws IOException { diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldTypeTests.java index f0f42253b4471..48a31691ac1bc 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldTypeTests.java @@ -17,6 +17,6 @@ public class RankFeaturesFieldTypeTests extends FieldTypeTestCase { public void testIsNotAggregatable() { MappedFieldType fieldType = new RankFeaturesFieldMapper.RankFeaturesFieldType("field", Collections.emptyMap(), true); - assertFalse(fieldType.isAggregatable(randomBoolean())); + assertFalse(fieldType.isAggregatable()); } } diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java index 3a4d5c032b3b6..cb296f5ecbf3b 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java @@ -426,19 +426,19 @@ public void testSubFields() throws IOException { ); ParentJoinFieldMapper mapper = (ParentJoinFieldMapper) mapperService.mappingLookup().getMapper("join_field"); assertTrue(mapper.fieldType().isSearchable()); - assertTrue(mapper.fieldType().isAggregatable(randomBoolean())); + assertTrue(mapper.fieldType().isAggregatable()); Iterator it = mapper.iterator(); FieldMapper next = (FieldMapper) it.next(); assertThat(next.name(), equalTo("join_field#parent")); assertTrue(next.fieldType().isSearchable()); - assertTrue(next.fieldType().isAggregatable(randomBoolean())); + assertTrue(next.fieldType().isAggregatable()); assertTrue(it.hasNext()); next = (FieldMapper) it.next(); assertThat(next.name(), equalTo("join_field#child")); assertTrue(next.fieldType().isSearchable()); - assertTrue(next.fieldType().isAggregatable(randomBoolean())); + assertTrue(next.fieldType().isAggregatable()); assertFalse(it.hasNext()); } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java index e9ddac125b8aa..9a44a188b85b7 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java @@ -111,7 +111,6 @@ static Map retrieveFieldCaps( Predicate filter = buildFilter(indexFieldfilter, filters, types, context); boolean isTimeSeriesIndex = context.getIndexSettings().getTimestampBounds() != null; - boolean isSyntheticSource = context.isSourceSynthetic(); Map responseMap = new HashMap<>(); for (String field : fieldNames) { MappedFieldType ft = context.getFieldType(field); @@ -121,7 +120,7 @@ static Map retrieveFieldCaps( ft.familyTypeName(), context.isMetadataField(field), ft.isSearchable(), - ft.isAggregatable(isSyntheticSource), + ft.isAggregatable(), isTimeSeriesIndex ? ft.isDimension() : false, isTimeSeriesIndex ? ft.getMetricType() : null, ft.meta() diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java index ba7d732ebd76a..aa0e0c17a52b7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java @@ -64,7 +64,7 @@ public final boolean isSearchable() { } @Override - public final boolean isAggregatable(boolean isSyntheticSource) { + public final boolean isAggregatable() { return true; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java index fa0eefd032cf7..1f8e7fc77b072 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java @@ -36,7 +36,7 @@ public ConstantFieldType(String name, Map meta) { } @Override - public final boolean isAggregatable(boolean isSyntheticSource) { + public final boolean isAggregatable() { return true; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index d25220ae4b221..a53236ba41b91 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -755,9 +755,7 @@ private static Mapper getLeafMapper(final DocumentParserContext context, ObjectM MappedFieldType fieldType = context.mappingLookup().getFieldType(fieldPath); if (fieldType != null) { // we haven't found a mapper with this name above, which means if a field type is found it is for sure a runtime field. - assert fieldType.hasDocValues() == false - && fieldType.isAggregatable(context.mappingLookup().isSourceSynthetic()) - && fieldType.isSearchable(); + assert fieldType.hasDocValues() == false && fieldType.isAggregatable() && fieldType.isSearchable(); return NO_OP_FIELDMAPPER; } return null; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index d8d7eacaad9f5..7536ef6ba547f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -176,12 +176,11 @@ public Function pointReaderIfPossible() { /** * Returns true if the field is aggregatable. - * @param isSyntheticSource is the {@code _source} stored in synthetic - * {@code _source} */ - public boolean isAggregatable(boolean isSyntheticSource) { + public boolean isAggregatable() { + // TODO make this abstract and each type should have a "native" implementation try { - fielddataBuilder(FieldDataContext.noRuntimeFields("aggregation_check", isSyntheticSource)); + fielddataBuilder(FieldDataContext.noRuntimeFields("aggregation_check", false)); return true; } catch (IllegalArgumentException e) { return false; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 2baa4c0337e50..35525da88dec2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -278,7 +278,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { } @Override - public boolean isAggregatable(boolean isSyntheticSource) { + public boolean isAggregatable() { return false; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index 8a80d5cb2aafb..5860851215d0b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -153,7 +153,7 @@ public void testFieldCaps() throws Exception { MappedFieldType concreteIndexType = concreteIndexMapping.fieldType("field"); assertEquals(concreteIndexType.familyTypeName(), scriptFieldType.familyTypeName()); assertEquals(concreteIndexType.isSearchable(), scriptFieldType.isSearchable()); - assertEquals(concreteIndexType.isAggregatable(false), scriptFieldType.isAggregatable(false)); + assertEquals(concreteIndexType.isAggregatable(), scriptFieldType.isAggregatable()); } @SuppressWarnings("unused") diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java index 07a40d0e77a42..9320a6c5de2ea 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java @@ -54,12 +54,12 @@ public void testTermsQuery() { public void testIsAggregatable() { MappedFieldType ft = new ProvidedIdFieldMapper.IdFieldType(() -> false); - assertFalse(ft.isAggregatable(randomBoolean())); + assertFalse(ft.isAggregatable()); ft = new ProvidedIdFieldMapper.IdFieldType(() -> true); - assertTrue(ft.isAggregatable(randomBoolean())); + assertTrue(ft.isAggregatable()); ft = new TsidExtractingIdFieldMapper.IdFieldType(); - assertFalse(ft.isAggregatable(randomBoolean())); + assertFalse(ft.isAggregatable()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java index cab5ec2f929bc..674dec3f7fa66 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java @@ -61,12 +61,12 @@ public void testEnableFieldData() throws IOException { () -> ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test", isSyntheticSource)).build(null, null) ); assertThat(exc.getMessage(), containsString(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())); - assertFalse(ft.isAggregatable(isSyntheticSource)); + assertFalse(ft.isAggregatable()); enabled[0] = true; ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test", isSyntheticSource)).build(null, null); assertWarnings(ProvidedIdFieldMapper.ID_FIELD_DATA_DEPRECATION_MESSAGE); - assertTrue(ft.isAggregatable(isSyntheticSource)); + assertTrue(ft.isAggregatable()); } public void testFetchIdFieldValue() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java index aa96b4011dd9c..40e9f4490903e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java @@ -80,7 +80,7 @@ public boolean isSearchable() { } @Override - public boolean isAggregatable(boolean isSyntheticSource) { + public boolean isAggregatable() { return true; } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 1a7c224a26813..755dd3f668bde 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -50,9 +50,9 @@ private static TextFieldType createFieldType() { public void testIsAggregatableDependsOnFieldData() { TextFieldType ft = createFieldType(); - assertFalse(ft.isAggregatable(randomBoolean())); + assertFalse(ft.isAggregatable()); ft.setFielddata(true); - assertTrue(ft.isAggregatable(randomBoolean())); + assertTrue(ft.isAggregatable()); } public void testTermQuery() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index c55b9fae0bfaa..b5f30c9ea10eb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -48,7 +48,7 @@ public void testIsSearchable() { public void testIsAggregatable() { DenseVectorFieldType ft = createFieldType(); - assertFalse(ft.isAggregatable(randomBoolean())); + assertFalse(ft.isAggregatable()); } public void testFielddataBuilder() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java index 2d730e6b7f925..302202a8b3763 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java @@ -27,7 +27,7 @@ public void testDocValuesDisabled() { public void testIsNotAggregatable() { MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType("field", Collections.emptyMap()); - assertFalse(fieldType.isAggregatable(randomBoolean())); + assertFalse(fieldType.isAggregatable()); } public void testDocValueFormatIsNotSupported() { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index a6acfb6971f2a..3a427960d80a0 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -690,7 +690,7 @@ public final void testIndexTimeFieldData() throws IOException { assertParseMinimalWarnings(); MappedFieldType fieldType = mapperService.fieldType("field"); boolean isSyntheticSource = mapperService.mappingLookup().isSourceSynthetic(); - if (fieldType.isAggregatable(isSyntheticSource) == false) { + if (fieldType.isAggregatable() == false) { return; // No field data available, so we ignore } SourceToParse source = source(this::writeField); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 89ade087013ea..0bdaf903b0c72 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -937,7 +937,7 @@ public void testSupportedFieldTypes() throws IOException { MappedFieldType fieldType = mapper.fieldType(); // Non-aggregatable fields are not testable (they will throw an error on all aggs anyway), so skip - if (fieldType.isAggregatable(false) == false) { + if (fieldType.isAggregatable() == false) { continue; } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java index e7c77c2d38a17..7d58d8c818ebc 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java @@ -524,7 +524,7 @@ public void testFieldCaps() throws IOException { MappedFieldType fieldType = aggMetricMapperService.fieldType("field"); assertThat(fieldType.familyTypeName(), equalTo("aggregate_metric_double")); assertTrue(fieldType.isSearchable()); - assertTrue(fieldType.isAggregatable(aggMetricMapperService.mappingLookup().isSourceSynthetic())); + assertTrue(fieldType.isAggregatable()); } /* diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java index 5d885a7f30097..06bd2260198ba 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java @@ -34,7 +34,7 @@ public boolean isTimeSeriesLabel(final String field, final Map unused final MappedFieldType fieldType = lookup.getFieldType(field); return fieldType != null && (timestampField.equals(field) == false) - && (fieldType.isAggregatable(lookup.isSourceSynthetic())) + && (fieldType.isAggregatable()) && (fieldType.isDimension() == false) && (mapperService.isMetadataField(field) == false); } From 1e4e216f84524143589648b31bf1f017cd0fc921 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 21 Sep 2022 11:26:58 -0400 Subject: [PATCH 06/10] test pass plz k thx --- .../xpack/wildcard/mapper/WildcardFieldMapperTests.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 51c850672c701..0d47a4595b297 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -59,6 +59,8 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperTestCase; +import org.elasticsearch.index.mapper.Mapping; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.query.SearchExecutionContext; @@ -1084,6 +1086,7 @@ protected final SearchExecutionContext createMockContext() { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fdc); return builder.build(new IndexFieldDataCache.None(), null); }; + MappingLookup lookup = MappingLookup.fromMapping(Mapping.EMPTY); return new SearchExecutionContext( 0, 0, @@ -1091,7 +1094,7 @@ protected final SearchExecutionContext createMockContext() { bitsetFilterCache, indexFieldDataLookup, null, - null, + lookup, null, null, parserConfig(), From 1778585e0db3f471daab09c47cd213c1b79529e7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 4 Nov 2022 10:48:50 -0400 Subject: [PATCH 07/10] Docs --- .../index/fielddata/StoredFieldIndexFieldData.java | 3 +++ .../index/fielddata/StoredFieldSortedBinaryIndexFieldData.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java index 5e8b559efee99..e9814e8f5ea65 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java @@ -23,6 +23,9 @@ import java.util.Set; +/** + * Per segment values for a field loaded from stored fields. + */ public abstract class StoredFieldIndexFieldData implements IndexFieldData.StoredFieldLeafFieldData> { private final String fieldName; private final ValuesSourceType valuesSourceType; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java index 764e875273ab7..17648fbc43eca 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java @@ -18,6 +18,9 @@ import java.util.Collections; import java.util.List; +/** + * Per segment values for a field loaded from stored fields exposing {@link SortedBinaryDocValues}. + */ public abstract class StoredFieldSortedBinaryIndexFieldData extends StoredFieldIndexFieldData { protected StoredFieldSortedBinaryIndexFieldData( From b79b033749a349624cf2791fa1adcb49ce76c3ae Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 9 Nov 2022 15:37:43 -0500 Subject: [PATCH 08/10] Move --- .../script/ScriptScoreBenchmark.java | 3 +- .../AggConstructionContentionBenchmark.java | 2 +- .../extras/MatchOnlyTextFieldMapper.java | 21 +++++++-- .../extras/ScaledFloatFieldTypeTests.java | 8 ++-- .../percolator/PercolateQueryBuilder.java | 3 +- .../AnnotatedTextFieldMapper.java | 18 +++++-- .../org/elasticsearch/index/IndexService.java | 5 +- .../org/elasticsearch/index/IndexWarmer.java | 2 +- .../index/fielddata/FieldDataContext.java | 11 ++--- .../index/mapper/DocumentParser.java | 8 +--- .../index/mapper/KeywordFieldMapper.java | 20 ++++++-- .../index/mapper/TextFieldMapper.java | 47 +++++++++++++++---- .../index/query/SearchExecutionContext.java | 11 +---- .../index/IndexSortSettingsTests.java | 2 +- .../fielddata/IndexFieldDataServiceTests.java | 35 ++++---------- .../AbstractScriptFieldTypeTestCase.java | 10 ++-- .../mapper/CompositeRuntimeFieldTests.java | 8 +--- .../index/mapper/NumberFieldTypeTests.java | 5 +- .../mapper/ProvidedIdFieldMapperTests.java | 5 +- .../index/mapper/TextFieldMapperTests.java | 7 ++- .../index/mapper/TextFieldTypeTests.java | 2 +- .../FlattenedIndexFieldDataTests.java | 4 +- .../vectors/DenseVectorFieldTypeTests.java | 8 +--- .../vectors/SparseVectorFieldTypeTests.java | 5 +- .../query/SearchExecutionContextTests.java | 2 +- .../index/shard/IndexShardTests.java | 5 +- .../bucket/filter/FiltersAggregatorTests.java | 2 +- .../sampler/SamplerAggregatorTests.java | 4 +- .../SignificantTermsAggregatorTests.java | 10 ++-- .../terms/SignificantTextAggregatorTests.java | 12 ++--- .../fetch/subphase/FieldFetcherTests.java | 18 ++----- .../CategoryContextMappingTests.java | 2 +- .../index/mapper/MapperServiceTestCase.java | 12 ++--- .../index/mapper/MapperTestCase.java | 27 ++++------- .../aggregations/AggregatorTestCase.java | 5 +- .../StringStatsAggregatorTests.java | 16 +++---- .../topmetrics/TopMetricsAggregatorTests.java | 2 +- .../AggregateDoubleMetricFieldTypeTests.java | 2 +- .../CategorizeTextAggregatorTests.java | 34 ++++++++++++-- x-pack/test/idp-fixture/build.gradle | 4 ++ 40 files changed, 209 insertions(+), 198 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index 57d1897d236c6..47f1b4f413532 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -89,8 +89,7 @@ public class ScriptScoreBenchmark { private final CircuitBreakerService breakerService = new NoneCircuitBreakerService(); private final SearchLookup lookup = new SearchLookup( fieldTypes::get, - (mft, lookup, fdo) -> mft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark", false)) - .build(fieldDataCache, breakerService), + (mft, lookup, fdo) -> mft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark")).build(fieldDataCache, breakerService), new SourceLookup.ReaderSourceProvider() ); diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java index 622818ea8bbb0..26a4a8c92f181 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java @@ -211,7 +211,7 @@ public Analyzer buildCustomAnalyzer( protected IndexFieldData buildFieldData(MappedFieldType ft) { IndexFieldDataCache indexFieldDataCache = indicesFieldDataCache.buildIndexFieldDataCache(new IndexFieldDataCache.Listener() { }, index, ft.name()); - return ft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark", false)).build(indexFieldDataCache, breakerService); + return ft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark")).build(indexFieldDataCache, breakerService); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 4a47bf16a2862..08f39c583e2f7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -122,7 +122,13 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) { NamedAnalyzer searchQuoteAnalyzer = analyzers.getSearchQuoteAnalyzer(); NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer(); TextSearchInfo tsi = new TextSearchInfo(Defaults.FIELD_TYPE, null, searchAnalyzer, searchQuoteAnalyzer); - MatchOnlyTextFieldType ft = new MatchOnlyTextFieldType(context.buildFullName(name), tsi, indexAnalyzer, meta.getValue()); + MatchOnlyTextFieldType ft = new MatchOnlyTextFieldType( + context.buildFullName(name), + tsi, + indexAnalyzer, + context.isSourceSynthetic(), + meta.getValue() + ); return ft; } @@ -149,10 +155,16 @@ public static class MatchOnlyTextFieldType extends StringFieldType { private final Analyzer indexAnalyzer; private final TextFieldType textFieldType; - public MatchOnlyTextFieldType(String name, TextSearchInfo tsi, Analyzer indexAnalyzer, Map meta) { + public MatchOnlyTextFieldType( + String name, + TextSearchInfo tsi, + Analyzer indexAnalyzer, + boolean isSyntheticSource, + Map meta + ) { super(name, true, false, false, tsi, meta); this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer); - this.textFieldType = new TextFieldType(name); + this.textFieldType = new TextFieldType(name, isSyntheticSource); } public MatchOnlyTextFieldType(String name) { @@ -160,6 +172,7 @@ public MatchOnlyTextFieldType(String name) { name, new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), Lucene.STANDARD_ANALYZER, + false, Collections.emptyMap() ); } @@ -309,7 +322,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (fieldDataContext.fielddataOperation() != FielddataOperation.SCRIPT) { throw new IllegalArgumentException(CONTENT_TYPE + " fields do not support sorting and aggregations"); } - if (fieldDataContext.isSyntheticSource()) { + if (textFieldType.isSyntheticSource()) { return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( storedFieldNameForSyntheticSource(), CoreValuesSourceType.KEYWORD, diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java index c6634f3df0e03..b07ff9dafbf95 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java @@ -190,9 +190,8 @@ public void testFieldData() throws IOException { "scaled_float1", scalingFactor ); - IndexNumericFieldData fielddata = (IndexNumericFieldData) f1.fielddataBuilder( - FieldDataContext.noRuntimeFields("test", randomBoolean()) - ).build(null, null); + IndexNumericFieldData fielddata = (IndexNumericFieldData) f1.fielddataBuilder(FieldDataContext.noRuntimeFields("test")) + .build(null, null); assertEquals(fielddata.getNumericType(), IndexNumericFieldData.NumericType.DOUBLE); LeafNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0)); SortedNumericDoubleValues values = leafFieldData.getDoubleValues(); @@ -205,8 +204,7 @@ public void testFieldData() throws IOException { "scaled_float2", scalingFactor ); - fielddata = (IndexNumericFieldData) f2.fielddataBuilder(FieldDataContext.noRuntimeFields("test", randomBoolean())) - .build(null, null); + fielddata = (IndexNumericFieldData) f2.fielddataBuilder(FieldDataContext.noRuntimeFields("test")).build(null, null); leafFieldData = fielddata.load(reader.leaves().get(0)); values = leafFieldData.getDoubleValues(); assertTrue(values.advanceExact(0)); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 23f7ee9560f27..de319f67b780b 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -649,8 +649,7 @@ public > IFD getForField( delegate.getFullyQualifiedIndex().getName(), delegate::lookup, this::sourcePath, - fielddataOperation, - delegate.isSourceSynthetic() + fielddataOperation ) ); IndexFieldDataCache cache = new IndexFieldDataCache.None(); diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 02854917e4ae3..7dde32b794b80 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -121,7 +121,13 @@ private AnnotatedTextFieldType buildFieldType(FieldType fieldType, MapperBuilder wrapAnalyzer(analyzers.getSearchAnalyzer()), wrapAnalyzer(analyzers.getSearchQuoteAnalyzer()) ); - return new AnnotatedTextFieldType(context.buildFullName(name), store.getValue(), tsi, meta.getValue()); + return new AnnotatedTextFieldType( + context.buildFullName(name), + store.getValue(), + tsi, + context.isSourceSynthetic(), + meta.getValue() + ); } @Override @@ -467,8 +473,14 @@ private void emitAnnotation(int firstSpannedTextPosInc, int annotationPosLen) th public static final class AnnotatedTextFieldType extends TextFieldMapper.TextFieldType { - private AnnotatedTextFieldType(String name, boolean store, TextSearchInfo tsi, Map meta) { - super(name, true, store, tsi, meta); + private AnnotatedTextFieldType( + String name, + boolean store, + TextSearchInfo tsi, + boolean isSyntheticSource, + Map meta + ) { + super(name, true, store, tsi, isSyntheticSource, meta); } public AnnotatedTextFieldType(String name, Map meta) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index c6017cde7a009..a107efaa97640 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -207,10 +207,7 @@ public IndexService( this.indexSortSupplier = () -> indexSettings.getIndexSortConfig() .buildIndexSort( mapperService::fieldType, - (fieldType, searchLookup) -> indexFieldData.getForField( - fieldType, - FieldDataContext.noRuntimeFields("index sort", mapperService.mappingLookup().isSourceSynthetic()) - ) + (fieldType, searchLookup) -> indexFieldData.getForField(fieldType, FieldDataContext.noRuntimeFields("index sort")) ); } else { this.indexSortSupplier = () -> null; diff --git a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java index 53ed1820c344b..ce30af187004c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java +++ b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java @@ -119,7 +119,7 @@ public TerminationHandle warmReader(final IndexShard indexShard, final Elasticse final long start = System.nanoTime(); IndexFieldData.Global ifd = indexFieldDataService.getForField( fieldType, - FieldDataContext.noRuntimeFields("index warming", mapperService.mappingLookup().isSourceSynthetic()) + FieldDataContext.noRuntimeFields("index warming") ); IndexFieldData global = ifd.loadGlobal(reader); if (reader.leaves().isEmpty() == false) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java b/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java index 6551af98d426b..3cfab4e599015 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java @@ -22,14 +22,12 @@ * @param lookupSupplier a supplier for a SearchLookup to be used by runtime scripts * @param sourcePathsLookup a function to get source paths for a specific field * @param fielddataOperation the operation used to determine data structures to generate fielddata from - * @param isSyntheticSource is the {@code _source} stored in synthetic {@code _source} */ public record FieldDataContext( String fullyQualifiedIndexName, Supplier lookupSupplier, Function> sourcePathsLookup, - MappedFieldType.FielddataOperation fielddataOperation, - boolean isSyntheticSource + MappedFieldType.FielddataOperation fielddataOperation ) { /** @@ -38,16 +36,13 @@ public record FieldDataContext( * Used for validating index sorts, eager global ordinal loading, etc * * @param reason the reason that runtime fields are not supported - * @param isSyntheticSource is the {@code _source} stored in synthetic - * {@code _source} */ - public static FieldDataContext noRuntimeFields(String reason, boolean isSyntheticSource) { + public static FieldDataContext noRuntimeFields(String reason) { return new FieldDataContext( "", () -> { throw new UnsupportedOperationException("Runtime fields not supported for [" + reason + "]"); }, Set::of, - MappedFieldType.FielddataOperation.SEARCH, - isSyntheticSource + MappedFieldType.FielddataOperation.SEARCH ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 052fcfa664a40..a41298014a193 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -144,13 +144,7 @@ private static void executeIndexTimeScripts(DocumentParserContext context) { SearchLookup searchLookup = new SearchLookup( context.mappingLookup().indexTimeLookup()::get, (ft, lookup, fto) -> ft.fielddataBuilder( - new FieldDataContext( - context.indexSettings().getIndex().getName(), - lookup, - context.mappingLookup()::sourcePaths, - fto, - context.isSyntheticSource() - ) + new FieldDataContext(context.indexSettings().getIndex().getName(), lookup, context.mappingLookup()::sourcePaths, fto) ).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), new SourceLookup.ReaderSourceProvider() ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 7ddfb34ac5618..d3f1a765a7e53 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -295,7 +295,15 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType } else if (splitQueriesOnWhitespace.getValue()) { searchAnalyzer = Lucene.WHITESPACE_ANALYZER; } - return new KeywordFieldType(context.buildFullName(name), fieldType, normalizer, searchAnalyzer, quoteAnalyzer, this); + return new KeywordFieldType( + context.buildFullName(name), + fieldType, + normalizer, + searchAnalyzer, + quoteAnalyzer, + this, + context.isSourceSynthetic() + ); } @Override @@ -335,6 +343,7 @@ public static final class KeywordFieldType extends StringFieldType { private final boolean eagerGlobalOrdinals; private final FieldValues scriptValues; private final boolean isDimension; + private final boolean isSyntheticSource; public KeywordFieldType( String name, @@ -342,7 +351,8 @@ public KeywordFieldType( NamedAnalyzer normalizer, NamedAnalyzer searchAnalyzer, NamedAnalyzer quoteAnalyzer, - Builder builder + Builder builder, + boolean isSyntheticSource ) { super( name, @@ -358,6 +368,7 @@ public KeywordFieldType( this.nullValue = builder.nullValue.getValue(); this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); + this.isSyntheticSource = isSyntheticSource; } public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { @@ -368,6 +379,7 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; + this.isSyntheticSource = false; } public KeywordFieldType(String name) { @@ -389,6 +401,7 @@ public KeywordFieldType(String name, FieldType fieldType) { this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; + this.isSyntheticSource = false; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { @@ -399,6 +412,7 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; + this.isSyntheticSource = false; } @Override @@ -696,7 +710,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (hasDocValues()) { return fieldDataFromDocValues(); } - if (fieldDataContext.isSyntheticSource() && isStored()) { + if (isSyntheticSource && isStored()) { return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( name(), CoreValuesSourceType.KEYWORD, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index c35e1f2c14d97..767640f03b0bf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -352,7 +352,14 @@ private TextFieldType buildFieldType(FieldType fieldType, MapperBuilderContext c ft = new LegacyTextFieldType(context.buildFullName(name), index.getValue(), store.getValue(), tsi, meta.getValue()); // ignore fieldData and eagerGlobalOrdinals } else { - ft = new TextFieldType(context.buildFullName(name), index.getValue(), store.getValue(), tsi, meta.getValue()); + ft = new TextFieldType( + context.buildFullName(name), + index.getValue(), + store.getValue(), + tsi, + context.isSourceSynthetic(), + meta.getValue() + ); ft.eagerGlobalOrdinals = eagerGlobalOrdinals.getValue(); if (fieldData.getValue()) { ft.setFielddata(true, freqFilter.getValue()); @@ -648,10 +655,19 @@ public static class TextFieldType extends StringFieldType { private PrefixFieldType prefixFieldType; private boolean indexPhrases = false; private boolean eagerGlobalOrdinals = false; - - public TextFieldType(String name, boolean indexed, boolean stored, TextSearchInfo tsi, Map meta) { + private final boolean isSyntheticSource; + + public TextFieldType( + String name, + boolean indexed, + boolean stored, + TextSearchInfo tsi, + boolean isSyntheticSource, + Map meta + ) { super(name, indexed, stored, false, tsi, meta); fielddata = false; + this.isSyntheticSource = isSyntheticSource; } public TextFieldType(String name, boolean indexed, boolean stored, Map meta) { @@ -664,14 +680,16 @@ public TextFieldType(String name, boolean indexed, boolean stored, Map new StoredFieldSortedBinaryIndexFieldData( name(), CoreValuesSourceType.KEYWORD, @@ -954,12 +972,23 @@ protected BytesRef storedToBytesRef(Object stored) { TextDocValuesField::new ); } + + public boolean isSyntheticSource() { + return isSyntheticSource; + } } public static class ConstantScoreTextFieldType extends TextFieldType { - public ConstantScoreTextFieldType(String name, boolean indexed, boolean stored, TextSearchInfo tsi, Map meta) { - super(name, indexed, stored, tsi, meta); + public ConstantScoreTextFieldType( + String name, + boolean indexed, + boolean stored, + TextSearchInfo tsi, + boolean isSyntheticSource, + Map meta + ) { + super(name, indexed, stored, tsi, isSyntheticSource, meta); } public ConstantScoreTextFieldType(String name) { @@ -968,6 +997,7 @@ public ConstantScoreTextFieldType(String name) { true, false, new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + false, Collections.emptyMap() ); } @@ -978,6 +1008,7 @@ public ConstantScoreTextFieldType(String name, boolean indexed, boolean stored, indexed, stored, new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + false, meta ); } @@ -1033,7 +1064,7 @@ static class LegacyTextFieldType extends ConstantScoreTextFieldType { private final MappedFieldType existQueryFieldType; LegacyTextFieldType(String name, boolean indexed, boolean stored, TextSearchInfo tsi, Map meta) { - super(name, indexed, stored, tsi, meta); + super(name, indexed, stored, tsi, false, meta); // norms are not available, neither are doc-values, so fall back to _source to run exists query existQueryFieldType = KeywordScriptFieldType.sourceOnly(name()).asMappedFieldTypes().findFirst().get(); } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 8a153264795ca..0aa390b13fe9c 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -286,8 +286,7 @@ public > IFD getForField(MappedFieldType fieldType fullyQualifiedIndex.getName(), () -> this.lookup().forkAndTrackFieldReferences(fieldType.name()), this::sourcePath, - fielddataOperation, - isSourceSynthetic() + fielddataOperation ) ); } @@ -499,13 +498,7 @@ public SearchLookup lookup() { this::getFieldType, (fieldType, searchLookup, fielddataOperation) -> indexFieldDataLookup.apply( fieldType, - new FieldDataContext( - fullyQualifiedIndex.getName(), - searchLookup, - this::sourcePath, - fielddataOperation, - isSourceSynthetic() - ) + new FieldDataContext(fullyQualifiedIndex.getName(), searchLookup, this::sourcePath, fielddataOperation) ), sourceProvider ); diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index d67e6f66d0f17..6cbfb53c3f584 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -220,7 +220,7 @@ private Sort buildIndexSort(IndexSettings indexSettings, Map indexFieldDataService.getForField( ft, - new FieldDataContext("test", s, Set::of, MappedFieldType.FielddataOperation.SEARCH, randomBoolean()) + new FieldDataContext("test", s, Set::of, MappedFieldType.FielddataOperation.SEARCH) ) ); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 08c3f452b062d..02a673d1c7833 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -72,7 +72,6 @@ protected Collection> getPlugins() { public void testGetForFieldDefaults() { final IndexService indexService = createIndex("test"); - final boolean isSyntheticSource = indexService.mapperService().mappingLookup().isSourceSynthetic(); final IndicesService indicesService = getInstanceFromNode(IndicesService.class); final IndexFieldDataService ifdService = new IndexFieldDataService( indexService.getIndexSettings(), @@ -82,7 +81,7 @@ public void testGetForFieldDefaults() { MapperBuilderContext context = MapperBuilderContext.root(false); final MappedFieldType stringMapper = new KeywordFieldMapper.Builder("string", Version.CURRENT).build(context).fieldType(); ifdService.clear(); - IndexFieldData fd = ifdService.getForField(stringMapper, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); + IndexFieldData fd = ifdService.getForField(stringMapper, FieldDataContext.noRuntimeFields("test")); assertTrue(fd instanceof SortedSetOrdinalsIndexFieldData); for (MappedFieldType mapper : Arrays.asList( @@ -92,7 +91,7 @@ public void testGetForFieldDefaults() { new NumberFieldMapper.Builder("long", LONG, ScriptCompiler.NONE, false, true, Version.CURRENT).build(context).fieldType() )) { ifdService.clear(); - fd = ifdService.getForField(mapper, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); + fd = ifdService.getForField(mapper, FieldDataContext.noRuntimeFields("test")); assertTrue(fd instanceof SortedNumericIndexFieldData); } @@ -105,7 +104,7 @@ public void testGetForFieldDefaults() { Version.CURRENT ).build(context).fieldType(); ifdService.clear(); - fd = ifdService.getForField(floatMapper, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); + fd = ifdService.getForField(floatMapper, FieldDataContext.noRuntimeFields("test")); assertTrue(fd instanceof SortedDoublesIndexFieldData); final MappedFieldType doubleMapper = new NumberFieldMapper.Builder( @@ -117,7 +116,7 @@ public void testGetForFieldDefaults() { Version.CURRENT ).build(context).fieldType(); ifdService.clear(); - fd = ifdService.getForField(doubleMapper, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); + fd = ifdService.getForField(doubleMapper, FieldDataContext.noRuntimeFields("test")); assertTrue(fd instanceof SortedDoublesIndexFieldData); } @@ -138,16 +137,7 @@ public void testGetForFieldRuntimeField() { return (IndexFieldData.Builder) (cache, breakerService) -> null; }); SearchLookup searchLookup = new SearchLookup(null, null, new SourceLookup.ReaderSourceProvider()); - ifdService.getForField( - ft, - new FieldDataContext( - "qualified", - () -> searchLookup, - null, - MappedFieldType.FielddataOperation.SEARCH, - indexService.mapperService().mappingLookup().isSourceSynthetic() - ) - ); + ifdService.getForField(ft, new FieldDataContext("qualified", () -> searchLookup, null, MappedFieldType.FielddataOperation.SEARCH)); assertSame(searchLookup, searchLookupSetOnce.get().get()); } @@ -187,9 +177,8 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon onRemovalCalled.incrementAndGet(); } }); - boolean isSyntheticSource = indexService.mapperService().mappingLookup().isSourceSynthetic(); - IndexFieldData ifd1 = ifdService.getForField(mapper1, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); - IndexFieldData ifd2 = ifdService.getForField(mapper2, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); + IndexFieldData ifd1 = ifdService.getForField(mapper1, FieldDataContext.noRuntimeFields("test")); + IndexFieldData ifd2 = ifdService.getForField(mapper2, FieldDataContext.noRuntimeFields("test")); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); LeafFieldData loadField1 = ifd1.load(leafReaderContext); LeafFieldData loadField2 = ifd2.load(leafReaderContext); @@ -263,10 +252,7 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon onRemovalCalled.incrementAndGet(); } }); - IndexFieldData ifd = ifdService.getForField( - mapper1, - FieldDataContext.noRuntimeFields("test", indexService.mapperService().mappingLookup().isSourceSynthetic()) - ); + IndexFieldData ifd = ifdService.getForField(mapper1, FieldDataContext.noRuntimeFields("test")); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); LeafFieldData load = ifd.load(leafReaderContext); assertEquals(1, onCacheCalled.get()); @@ -327,13 +313,12 @@ private void doTestRequireDocValues(MappedFieldType ft) { cache, null ); - boolean isSyntheticSource = randomBoolean(); if (ft.hasDocValues()) { - ifds.getForField(ft, FieldDataContext.noRuntimeFields("test", isSyntheticSource)); // no exception + ifds.getForField(ft, FieldDataContext.noRuntimeFields("test")); // no exception } else { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> ifds.getForField(ft, FieldDataContext.noRuntimeFields("test", isSyntheticSource)) + () -> ifds.getForField(ft, FieldDataContext.noRuntimeFields("test")) ); assertThat(e.getMessage(), containsString("doc values")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index dfdb7d7c244ae..04d9846ad2929 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -194,8 +194,7 @@ protected static FieldDataContext mockFielddataContext() { "test", searchExecutionContext::lookup, mockContext()::sourcePath, - MappedFieldType.FielddataOperation.SCRIPT, - searchExecutionContext.isSourceSynthetic() + MappedFieldType.FielddataOperation.SCRIPT ); } @@ -227,16 +226,15 @@ protected static SearchExecutionContext mockContext( when(context.allowExpensiveQueries()).thenReturn(allowExpensiveQueries); SearchLookup lookup = new SearchLookup( context::getFieldType, - (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( - new FieldDataContext("test", lookupSupplier, context::sourcePath, fdo, context.isSourceSynthetic()) - ).build(null, null), + (mft, lookupSupplier, fdo) -> mft.fielddataBuilder(new FieldDataContext("test", lookupSupplier, context::sourcePath, fdo)) + .build(null, null), sourceProvider ); when(context.lookup()).thenReturn(lookup); when(context.getForField(any(), any())).then(args -> { MappedFieldType ft = args.getArgument(0); MappedFieldType.FielddataOperation fdo = args.getArgument(1); - return ft.fielddataBuilder(new FieldDataContext("test", context::lookup, context::sourcePath, fdo, context.isSourceSynthetic())) + return ft.fielddataBuilder(new FieldDataContext("test", context::lookup, context::sourcePath, fdo)) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); }); return context; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java index 681251689f89b..b2ce1482ce02c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java @@ -336,13 +336,7 @@ public void testParseDocumentSubFieldAccess() throws IOException { SearchLookup searchLookup = new SearchLookup( mapperService::fieldType, (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( - new FieldDataContext( - "test", - lookupSupplier, - mapperService.mappingLookup()::sourcePaths, - fdo, - mapperService.mappingLookup().isSourceSynthetic() - ) + new FieldDataContext("test", lookupSupplier, mapperService.mappingLookup()::sourcePaths, fdo) ).build(null, null), new SourceLookup.ReaderSourceProvider() ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index c885a53841d19..e2f2e4a9ad8eb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -583,9 +583,8 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS // Create an index writer configured with the same index sort. NumberFieldType fieldType = new NumberFieldType("field", type); - IndexNumericFieldData fielddata = (IndexNumericFieldData) fieldType.fielddataBuilder( - FieldDataContext.noRuntimeFields("test", randomBoolean()) - ).build(null, null); + IndexNumericFieldData fielddata = (IndexNumericFieldData) fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test")) + .build(null, null); SortField sortField = fielddata.sortField(null, MultiValueMode.MIN, null, randomBoolean()); IndexWriterConfig writerConfig = new IndexWriterConfig(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java index 36b9e253d27f1..99daa3770c55f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java @@ -53,18 +53,17 @@ public void testEnableFieldData() throws IOException { boolean[] enabled = new boolean[1]; MapperService mapperService = createMapperService(() -> enabled[0], mapping(b -> {})); - boolean isSyntheticSource = mapperService.mappingLookup().isSourceSynthetic(); ProvidedIdFieldMapper.IdFieldType ft = (ProvidedIdFieldMapper.IdFieldType) mapperService.fieldType("_id"); IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, - () -> ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test", isSyntheticSource)).build(null, null) + () -> ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test")).build(null, null) ); assertThat(exc.getMessage(), containsString(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())); assertFalse(ft.isAggregatable()); enabled[0] = true; - ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test", isSyntheticSource)).build(null, null); + ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test")).build(null, null); assertWarnings(ProvidedIdFieldMapper.ID_FIELD_DATA_DEPRECATION_MESSAGE); assertTrue(ft.isAggregatable()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 506010856c842..635989cc5159e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -528,12 +528,11 @@ public void testEagerGlobalOrdinals() throws IOException { public void testFielddata() throws IOException { MapperService disabledMapper = createMapperService(fieldMapping(this::minimalMapping)); - boolean isSyntheticSource = disabledMapper.mappingLookup().isSourceSynthetic(); assertFalse(disabledMapper.fieldType("field").isAggregatable()); Exception e = expectThrows( IllegalArgumentException.class, () -> disabledMapper.fieldType("field") - .fielddataBuilder(new FieldDataContext("index", null, null, MappedFieldType.FielddataOperation.SEARCH, isSyntheticSource)) + .fielddataBuilder(new FieldDataContext("index", null, null, MappedFieldType.FielddataOperation.SEARCH)) ); assertThat( e.getMessage(), @@ -544,7 +543,7 @@ public void testFielddata() throws IOException { ); MapperService enabledMapper = createMapperService(fieldMapping(b -> b.field("type", "text").field("fielddata", true))); - enabledMapper.fieldType("field").fielddataBuilder(FieldDataContext.noRuntimeFields("test", isSyntheticSource)); // no exception + enabledMapper.fieldType("field").fielddataBuilder(FieldDataContext.noRuntimeFields("test")); // no exception assertTrue(enabledMapper.fieldType("field").isAggregatable()); e = expectThrows( MapperParsingException.class, @@ -1216,7 +1215,7 @@ public void testIgnoreFieldDataOnLegacyIndex() throws IOException { expectThrows( IllegalArgumentException.class, () -> ((TextFieldMapper) finalMapperService.documentMapper().mappers().getMapper("field")).fieldType() - .fielddataBuilder(FieldDataContext.noRuntimeFields("test", finalMapperService.mappingLookup().isSourceSynthetic())) + .fielddataBuilder(FieldDataContext.noRuntimeFields("test")) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 755dd3f668bde..0c54a1be33384 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -45,7 +45,7 @@ public class TextFieldTypeTests extends FieldTypeTestCase { private static TextFieldType createFieldType() { - return new TextFieldType("field"); + return new TextFieldType("field", randomBoolean()); } public void testIsAggregatableDependsOnFieldData() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java index bf426f30ec1db..7b6bd2f43fe65 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java @@ -71,7 +71,7 @@ public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) { DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("test", "_na_", 1)); // Load global field data for subfield 'key'. - IndexFieldData ifd1 = ifdService.getForField(fieldType1, FieldDataContext.noRuntimeFields("test", false)); + IndexFieldData ifd1 = ifdService.getForField(fieldType1, FieldDataContext.noRuntimeFields("test")); assertTrue(ifd1 instanceof KeyedFlattenedFieldData); KeyedFlattenedFieldData fieldData1 = (KeyedFlattenedFieldData) ifd1; @@ -81,7 +81,7 @@ public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) { // Load global field data for the subfield 'other_key'. MappedFieldType fieldType2 = fieldMapper.fieldType().getChildFieldType("other_key"); - IndexFieldData ifd2 = ifdService.getForField(fieldType2, FieldDataContext.noRuntimeFields("test", false)); + IndexFieldData ifd2 = ifdService.getForField(fieldType2, FieldDataContext.noRuntimeFields("test")); assertTrue(ifd2 instanceof KeyedFlattenedFieldData); KeyedFlattenedFieldData fieldData2 = (KeyedFlattenedFieldData) ifd2; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index c1b23d20728ac..5fd668e72fc1c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -83,13 +83,7 @@ public void testIsAggregatable() { public void testFielddataBuilder() { DenseVectorFieldType fft = createFloatFieldType(); - FieldDataContext fdc = new FieldDataContext( - "test", - () -> null, - Set::of, - MappedFieldType.FielddataOperation.SCRIPT, - randomBoolean() - ); + FieldDataContext fdc = new FieldDataContext("test", () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); assertNotNull(fft.fielddataBuilder(fdc)); DenseVectorFieldType bft = createByteFieldType(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java index 302202a8b3763..574fb63cd3fb0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java @@ -19,10 +19,7 @@ public class SparseVectorFieldTypeTests extends FieldTypeTestCase { public void testDocValuesDisabled() { MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType("field", Collections.emptyMap()); assertFalse(fieldType.hasDocValues()); - expectThrows( - IllegalArgumentException.class, - () -> fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test", randomBoolean())) - ); + expectThrows(IllegalArgumentException.class, () -> fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test"))); } public void testIsNotAggregatable() { diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index b2c1e12bae5fe..7bd6cc2abf083 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -104,7 +104,7 @@ public class SearchExecutionContextTests extends ESTestCase { public void testFailIfFieldMappingNotFound() { SearchExecutionContext context = createSearchExecutionContext(IndexMetadata.INDEX_UUID_NA_VALUE, null); context.setAllowUnmappedFields(false); - MappedFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + MappedFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); MappedFieldType result = context.failIfFieldMappingNotFound("name", fieldType); assertThat(result, sameInstance(fieldType)); QueryShardException e = expectThrows(QueryShardException.class, () -> context.failIfFieldMappingNotFound("name", null)); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 009bfb082c4b9..9d49cca516da3 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2687,10 +2687,7 @@ public void testReaderWrapperWorksWithGlobalOrdinals() throws IOException { indicesFieldDataCache, new NoneCircuitBreakerService() ); - IndexFieldData.Global ifd = indexFieldDataService.getForField( - foo, - FieldDataContext.noRuntimeFields("test", shard.mapperService().mappingLookup().isSourceSynthetic()) - ); + IndexFieldData.Global ifd = indexFieldDataService.getForField(foo, FieldDataContext.noRuntimeFields("test")); FieldDataStats before = shard.fieldData().stats("foo"); assertThat(before.getMemorySizeInBytes(), equalTo(0L)); FieldDataStats after = null; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java index 7915d213dbac1..a3c1ed5d7793a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java @@ -431,7 +431,7 @@ public void testRangeFilter() throws IOException { * Tests a filter that needs the cache to be fast. */ public void testPhraseFilter() throws IOException { - MappedFieldType ft = new TextFieldMapper.TextFieldType("test"); + MappedFieldType ft = new TextFieldMapper.TextFieldType("test", randomBoolean()); AggregationBuilder builder = new FiltersAggregationBuilder( "test", new KeyedFilter("q1", new MatchPhraseQueryBuilder("test", "will find me").slop(0)) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java index ed20e556a56fb..0212ceac737b3 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java @@ -41,7 +41,7 @@ public class SamplerAggregatorTests extends AggregatorTestCase { * Uses the sampler aggregation to find the minimum value of a field out of the top 3 scoring documents in a search. */ public void testSampler() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType("int", NumberFieldMapper.NumberType.LONG); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); @@ -76,7 +76,7 @@ public void testSampler() throws IOException { } public void testRidiculousSize() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType("int", NumberFieldMapper.NumberType.LONG); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java index edc6fd5dcf76c..8574f402fb92a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java @@ -103,7 +103,7 @@ protected List unsupportedMappedFieldTypes() { } public void testSignificance(SignificanceHeuristic heuristic) throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); @@ -211,7 +211,7 @@ public void testSignificance() throws IOException { * @throws IOException on test setup failure */ public void testSamplingConsistency() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); @@ -317,7 +317,7 @@ public void testNumericSignificance() throws IOException { * Uses the significant terms aggregation on an index with unmapped field */ public void testUnmapped() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); @@ -384,7 +384,7 @@ public void testRangeField() throws IOException { } public void testFieldAlias() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); @@ -440,7 +440,7 @@ public void testFieldAlias() throws IOException { } public void testFieldBackground() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java index 74b9b232d852e..c56b084e1511f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java @@ -71,7 +71,7 @@ protected List unsupportedMappedFieldTypes() { * Uses the significant text aggregation to find the keywords in text fields */ public void testSignificance() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); @@ -126,7 +126,7 @@ public void testSignificance() throws IOException { * Uses the significant text aggregation to find the keywords in text fields and include/exclude selected terms */ public void testIncludeExcludes() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); @@ -187,7 +187,7 @@ public void testIncludeExcludes() throws IOException { } public void testMissingField() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); indexWriterConfig.setMaxBufferedDocs(100); @@ -215,7 +215,7 @@ public void testMissingField() throws IOException { } public void testFieldAlias() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); @@ -274,7 +274,7 @@ public void testFieldAlias() throws IOException { } public void testInsideTermsAgg() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); @@ -331,7 +331,7 @@ private void indexDocuments(IndexWriter writer) throws IOException { * Test documents with arrays of text */ public void testSignificanceOnTextArrays() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index f440af13bbc8c..e37c7570b8986 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -236,11 +236,7 @@ public void testMetadataFields() throws IOException { FieldFetcher fieldFetcher = FieldFetcher.create( newSearchExecutionContext( mapperService, - (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup(), mapperService.mappingLookup().isSourceSynthetic()).apply( - ft, - fdc.lookupSupplier(), - fdc.fielddataOperation() - ) + (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup()).apply(ft, fdc.lookupSupplier(), fdc.fielddataOperation()) ), fieldList ); @@ -1128,11 +1124,7 @@ public void testFetchRuntimeFieldWithSourceDisabled() throws IOException { MapperService mapperService = createMapperService(mapping); SearchExecutionContext searchExecutionContext = newSearchExecutionContext( mapperService, - (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup(), mapperService.mappingLookup().isSourceSynthetic()).apply( - ft, - fdc.lookupSupplier(), - fdc.fielddataOperation() - ) + (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup()).apply(ft, fdc.lookupSupplier(), fdc.fielddataOperation()) ); withLuceneIndex(mapperService, iw -> iw.addDocument(new LuceneDocument()), iw -> { FieldFetcher fieldFetcher = FieldFetcher.create(searchExecutionContext, fieldAndFormatList("runtime_field", null, false)); @@ -1161,11 +1153,7 @@ public void testFetchMetadataFieldWithSourceDisabled() throws IOException { MapperService mapperService = createMapperService(mapping); SearchExecutionContext searchExecutionContext = newSearchExecutionContext( mapperService, - (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup(), mapperService.mappingLookup().isSourceSynthetic()).apply( - ft, - fdc.lookupSupplier(), - fdc.fielddataOperation() - ) + (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup()).apply(ft, fdc.lookupSupplier(), fdc.fielddataOperation()) ); withLuceneIndex(mapperService, iw -> { ParsedDocument parsedDocument = mapperService.documentMapper().parse(source("{}")); diff --git a/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java b/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java index 26f3e1d2e24b7..fa97b1c1d9536 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java @@ -796,7 +796,7 @@ public void testParsingContextFromDocument() throws Exception { assertTrue(context.contains("category1")); document = new LuceneDocument(); - TextFieldMapper.TextFieldType text = new TextFieldMapper.TextFieldType("category"); + TextFieldMapper.TextFieldType text = new TextFieldMapper.TextFieldType("category", randomBoolean()); document.add(new Field(text.name(), "category1", TextFieldMapper.Defaults.FIELD_TYPE)); // Ignore stored field document.add(new StoredField(text.name(), "category1", TextFieldMapper.Defaults.FIELD_TYPE)); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 614e656968e62..9454a6a8555a4 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -466,7 +466,7 @@ public Query filterQuery(Query query) { @Override protected IndexFieldData buildFieldData(MappedFieldType ft) { - return ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test", mapperService.mappingLookup().isSourceSynthetic())) + return ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test")) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } @@ -662,16 +662,14 @@ public void onRemoval(ShardId shardId, Accountable accountable) { protected TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup( MapperService mapperService ) { - return fieldDataLookup(mapperService.mappingLookup()::sourcePaths, mapperService.mappingLookup().isSourceSynthetic()); + return fieldDataLookup(mapperService.mappingLookup()::sourcePaths); } protected TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup( - Function> sourcePathsLookup, - boolean isSyntheticSource + Function> sourcePathsLookup ) { - return (mft, lookupSource, fdo) -> mft.fielddataBuilder( - new FieldDataContext("test", lookupSource, sourcePathsLookup, fdo, isSyntheticSource) - ).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + return (mft, lookupSource, fdo) -> mft.fielddataBuilder(new FieldDataContext("test", lookupSource, sourcePathsLookup, fdo)) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer build) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 9e544b8dc98b1..57e1d8731c181 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -131,15 +131,12 @@ public void testAggregatableConsistency() throws IOException { protected void assertAggregatableConsistency(MappedFieldType ft) { if (ft.isAggregatable()) { try { - ft.fielddataBuilder(FieldDataContext.noRuntimeFields("aggregation_test", randomBoolean())); + ft.fielddataBuilder(FieldDataContext.noRuntimeFields("aggregation_test")); } catch (Exception e) { fail("Unexpected exception when fetching field data from aggregatable field type"); } } else { - expectThrows( - IllegalArgumentException.class, - () -> ft.fielddataBuilder(FieldDataContext.noRuntimeFields("aggregation_test", randomBoolean())) - ); + expectThrows(IllegalArgumentException.class, () -> ft.fielddataBuilder(FieldDataContext.noRuntimeFields("aggregation_test"))); } } @@ -536,13 +533,7 @@ protected final void assertScriptDocValues(MapperService mapperService, Object s MappedFieldType ft = mapperService.fieldType("field"); SearchLookup searchLookup = new SearchLookup(null, null, mapperService.mappingLookup().getSourceProvider()); IndexFieldData sfd = ft.fielddataBuilder( - new FieldDataContext( - "", - () -> searchLookup, - Set::of, - MappedFieldType.FielddataOperation.SCRIPT, - mapperService.mappingLookup().isSourceSynthetic() - ) + new FieldDataContext("", () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) ).build(null, null); LeafFieldData lfd = sfd.load(getOnlyLeafReader(searcher.getIndexReader()).getContext()); DocValuesScriptFieldFactory sff = lfd.getScriptFieldFactory("field"); @@ -809,7 +800,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val SourceToParse source = source(b -> b.field(ft.name(), value)); ValueFetcher docValueFetcher = new DocValueFetcher( ft.docValueFormat(format, null), - ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test", mapperService.mappingLookup().isSourceSynthetic())) + ft.fielddataBuilder(FieldDataContext.noRuntimeFields("test")) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); @@ -878,7 +869,6 @@ public final void testIndexTimeFieldData() throws IOException { MapperService mapperService = createMapperService(fieldMapping(this::minimalMapping)); assertParseMinimalWarnings(); MappedFieldType fieldType = mapperService.fieldType("field"); - boolean isSyntheticSource = mapperService.mappingLookup().isSourceSynthetic(); if (fieldType.isAggregatable() == false) { return; // No field data available, so we ignore } @@ -889,13 +879,14 @@ public final void testIndexTimeFieldData() throws IOException { LeafReaderContext ctx = ir.leaves().get(0); - DocValuesScriptFieldFactory docValuesFieldSource = fieldType.fielddataBuilder( - FieldDataContext.noRuntimeFields("test", isSyntheticSource) - ).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()).load(ctx).getScriptFieldFactory("test"); + DocValuesScriptFieldFactory docValuesFieldSource = fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test")) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) + .load(ctx) + .getScriptFieldFactory("test"); docValuesFieldSource.setNextDocId(0); DocumentLeafReader reader = new DocumentLeafReader(doc.rootDoc(), Collections.emptyMap()); - DocValuesScriptFieldFactory indexData = fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test", isSyntheticSource)) + DocValuesScriptFieldFactory indexData = fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test")) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) .load(reader.getContext()) .getScriptFieldFactory("test"); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 5fb23166490da..b0befd0911cc7 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -314,8 +314,7 @@ protected AggregationContext createAggregationContext( indexSettings.getIndex().getName(), context.lookupSupplier(), context.sourcePathsLookup(), - context.fielddataOperation(), - context.isSyntheticSource() + context.fielddataOperation() ) ).build(new IndexFieldDataCache.None(), breakerService); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() { @@ -1103,7 +1102,7 @@ public void testSupportedFieldTypes() throws IOException { } private ValuesSourceType fieldToVST(MappedFieldType fieldType) { - return fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test", false)).build(null, null).getValuesSourceType(); + return fieldType.fielddataBuilder(FieldDataContext.noRuntimeFields("test")).build(null, null).getValuesSourceType(); } /** diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java index db8ccff4fa12e..ea53443bd27a3 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java @@ -116,7 +116,7 @@ public void testUnmappedWithMissingField() throws IOException { } public void testMissing() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field(fieldType.name()) @@ -192,7 +192,7 @@ public void testQueryFiltering() throws IOException { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/47469") public void testSingleValuedFieldWithFormatter() throws IOException { - TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field("text") @@ -218,7 +218,7 @@ public void testSingleValuedFieldWithFormatter() throws IOException { public void testNestedAggregation() throws IOException { MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType("value", NumberFieldMapper.NumberType.INTEGER); - TextFieldMapper.TextFieldType textFieldType = new TextFieldMapper.TextFieldType("text"); + TextFieldMapper.TextFieldType textFieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); textFieldType.setFielddata(true); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.NUMERIC) @@ -273,7 +273,7 @@ public void testNestedAggregation() throws IOException { } public void testValueScriptSingleValuedField() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field(fieldType.name()) @@ -295,7 +295,7 @@ public void testValueScriptSingleValuedField() throws IOException { } public void testValueScriptMultiValuedField() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field(fieldType.name()) @@ -322,7 +322,7 @@ public void testValueScriptMultiValuedField() throws IOException { } public void testFieldScriptSingleValuedField() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").script( @@ -345,7 +345,7 @@ public void testFieldScriptSingleValuedField() throws IOException { } public void testFieldScriptMultiValuedField() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").script( @@ -377,7 +377,7 @@ private void testAggregation( CheckedConsumer buildIndex, Consumer verify ) throws IOException { - TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); AggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field("text"); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java index 388821a41f51b..2e2a64d426a8d 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java @@ -537,7 +537,7 @@ private MappedFieldType numberFieldType(NumberType numberType, String name) { } private MappedFieldType textFieldType(String name) { - return new TextFieldMapper.TextFieldType(name); + return new TextFieldMapper.TextFieldType(name, randomBoolean()); } private MappedFieldType geoPointFieldType(String name) { diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java index eb0388f022fab..140f6ecc922a3 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java @@ -125,7 +125,7 @@ public void testUsedInScript() throws IOException { SearchLookup lookup = new SearchLookup( searchExecutionContext::getFieldType, (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( - new FieldDataContext("test", lookupSupplier, searchExecutionContext::sourcePath, fdo, randomBoolean()) + new FieldDataContext("test", lookupSupplier, searchExecutionContext::sourcePath, fdo) ).build(null, null), new SourceLookup.ReaderSourceProvider() ); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java index 83c4dd2513bfc..196584e4a7ce2 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java @@ -79,7 +79,7 @@ public void testCategorizationWithoutSubAggs() throws Exception { }, new AggTestConfig( new CategorizeTextAggregationBuilder("my_agg", TEXT_FIELD_NAME), - new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME), + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), longField(NUMERIC_FIELD_NAME) ) ); @@ -114,7 +114,13 @@ public void testCategorizationWithSubAggs() throws Exception { assertThat(((Max) result.getBuckets().get(1).getAggregations().get("max")).value(), equalTo(4.0)); assertThat(((Min) result.getBuckets().get(1).getAggregations().get("min")).value(), equalTo(0.0)); assertThat(((Avg) result.getBuckets().get(1).getAggregations().get("avg")).getValue(), equalTo(2.0)); - }, new AggTestConfig(aggBuilder, new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME), longField(NUMERIC_FIELD_NAME))); + }, + new AggTestConfig( + aggBuilder, + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + longField(NUMERIC_FIELD_NAME) + ) + ); } public void testCategorizationWithMultiBucketSubAggs() throws Exception { @@ -163,7 +169,13 @@ public void testCategorizationWithMultiBucketSubAggs() throws Exception { assertThat(histo.getBuckets().get(2).getDocCount(), equalTo(1L)); assertThat(((Avg) histo.getBuckets().get(0).getAggregations().get("avg")).getValue(), equalTo(0.0)); assertThat(((Avg) histo.getBuckets().get(2).getAggregations().get("avg")).getValue(), equalTo(4.0)); - }, new AggTestConfig(aggBuilder, new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME), longField(NUMERIC_FIELD_NAME))); + }, + new AggTestConfig( + aggBuilder, + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + longField(NUMERIC_FIELD_NAME) + ) + ); } public void testCategorizationAsSubAgg() throws Exception { @@ -245,7 +257,13 @@ public void testCategorizationAsSubAgg() throws Exception { assertThat(((Max) categorizationAggregation.getBuckets().get(1).getAggregations().get("max")).value(), equalTo(4.0)); assertThat(((Min) categorizationAggregation.getBuckets().get(1).getAggregations().get("min")).value(), equalTo(4.0)); assertThat(((Avg) categorizationAggregation.getBuckets().get(1).getAggregations().get("avg")).getValue(), equalTo(4.0)); - }, new AggTestConfig(aggBuilder, new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME), longField(NUMERIC_FIELD_NAME))); + }, + new AggTestConfig( + aggBuilder, + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + longField(NUMERIC_FIELD_NAME) + ) + ); } public void testCategorizationWithSubAggsManyDocs() throws Exception { @@ -294,7 +312,13 @@ public void testCategorizationWithSubAggsManyDocs() throws Exception { assertThat(histo.getBuckets().get(2).getDocCount(), equalTo(5000L)); assertThat(((Avg) histo.getBuckets().get(0).getAggregations().get("avg")).getValue(), equalTo(0.0)); assertThat(((Avg) histo.getBuckets().get(2).getAggregations().get("avg")).getValue(), equalTo(4.0)); - }, new AggTestConfig(aggBuilder, new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME), longField(NUMERIC_FIELD_NAME))); + }, + new AggTestConfig( + aggBuilder, + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + longField(NUMERIC_FIELD_NAME) + ) + ); } private static void writeTestDocs(RandomIndexWriter w) throws IOException { diff --git a/x-pack/test/idp-fixture/build.gradle b/x-pack/test/idp-fixture/build.gradle index 75e152bafae2e..5a9c387a9521b 100644 --- a/x-pack/test/idp-fixture/build.gradle +++ b/x-pack/test/idp-fixture/build.gradle @@ -5,6 +5,10 @@ import static org.elasticsearch.gradle.internal.distribution.InternalElasticsear apply plugin: 'elasticsearch.test.fixtures' apply plugin: 'elasticsearch.internal-distribution-download' +dockerCompose { + composeAdditionalArgs = ['--compatibility'] +} + tasks.register("copyKeystore", Sync) { from project(':x-pack:plugin:core') .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks') From 6ca4596d82676fb5f6daf2e7c04a089ee7f6bcb6 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 10 Nov 2022 08:10:00 -0500 Subject: [PATCH 09/10] fixup --- .../index/mapper/KeywordFieldMapper.java | 10 ++++++- .../index/mapper/TextFieldMapper.java | 29 ++++++++++++------- .../index/mapper/TextFieldMapperTests.java | 23 ++++++++++++--- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index d3f1a765a7e53..8ebf6c5c5fc65 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -710,7 +710,14 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (hasDocValues()) { return fieldDataFromDocValues(); } - if (isSyntheticSource && isStored()) { + if (isSyntheticSource) { + if (false == isStored()) { + throw new IllegalStateException( + "keyword field [" + + name() + + "] is only supported in synthetic _source index if it creates doc values or stored fields" + ); + } return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( name(), CoreValuesSourceType.KEYWORD, @@ -1141,6 +1148,7 @@ protected void write(XContentBuilder b, Object value) throws IOException { fieldType().ignoreAbove == Defaults.IGNORE_ABOVE ? null : originalName(), false ) { + @Override protected BytesRef convert(BytesRef value) { return value; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 767640f03b0bf..43bd6d220f188 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -952,17 +952,24 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (operation != FielddataOperation.SCRIPT) { throw new IllegalStateException("unknown field data operation [" + operation.name() + "]"); } - if (isSyntheticSource && isStored()) { - return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( - name(), - CoreValuesSourceType.KEYWORD, - TextDocValuesField::new - ) { - @Override - protected BytesRef storedToBytesRef(Object stored) { - return new BytesRef((String) stored); - } - }; + if (isSyntheticSource) { + if (isStored()) { + return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( + name(), + CoreValuesSourceType.KEYWORD, + TextDocValuesField::new + ) { + @Override + protected BytesRef storedToBytesRef(Object stored) { + return new BytesRef((String) stored); + } + }; + } + throw new IllegalArgumentException( + "fetching values from a text field [" + + name() + + "] is not yet supported because synthetic _source is enabled and the field doesn't create stored fields" + ); } return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 635989cc5159e..68bf0bd008107 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1230,15 +1230,24 @@ public void testIgnoreEagerGlobalOrdinalsOnLegacyIndex() throws IOException { public void testDocValues() throws IOException { MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "text"))); - assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + for (String input : new String[] { + "foo", // Won't be tokenized + "foo bar", // Will be tokenized. But script doc values still returns the whole field. + }) { + assertScriptDocValues(mapper, input, equalTo(List.of(input))); + } } public void testDocValuesLoadedFromStoredSynthetic() throws IOException { MapperService mapper = createMapperService(syntheticSourceFieldMapping(b -> b.field("type", "text").field("store", true))); - assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + for (String input : new String[] { + "foo", // Won't be tokenized + "foo bar", // Will be tokenized. But script doc values still returns the whole field. + }) { + assertScriptDocValues(mapper, input, equalTo(List.of(input))); + } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/86603") public void testDocValuesLoadedFromSubKeywordSynthetic() throws IOException { MapperService mapper = createMapperService(syntheticSourceFieldMapping(b -> { b.field("type", "text"); @@ -1248,6 +1257,12 @@ public void testDocValuesLoadedFromSubKeywordSynthetic() throws IOException { } b.endObject(); })); - assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + Exception e = expectThrows(IllegalArgumentException.class, () -> assertScriptDocValues(mapper, "foo", equalTo(List.of("foo")))); + assertThat( + e.getMessage(), + equalTo( + "fetching values from a text field [field] is not yet supported because synthetic _source is enabled and the field doesn't create stored fields" + ) + ); } } From ac7f8a11bcb003837f12183ba1320a72d320b11a Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 10 Nov 2022 10:05:51 -0500 Subject: [PATCH 10/10] format --- .../org/elasticsearch/index/mapper/TextFieldMapperTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 68bf0bd008107..02ad6eff329b5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1261,7 +1261,8 @@ public void testDocValuesLoadedFromSubKeywordSynthetic() throws IOException { assertThat( e.getMessage(), equalTo( - "fetching values from a text field [field] is not yet supported because synthetic _source is enabled and the field doesn't create stored fields" + "fetching values from a text field [field] is not yet supported because synthetic _source is " + + "enabled and the field doesn't create stored fields" ) ); }