diff --git a/docs/changelog/89950.yaml b/docs/changelog/89950.yaml new file mode 100644 index 0000000000000..2027209fcb2a2 --- /dev/null +++ b/docs/changelog/89950.yaml @@ -0,0 +1,5 @@ +pr: 89950 +summary: "Synthetic _source: support `field` in many cases" +area: TSDB +type: enhancement +issues: [] diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index d07a7b7296c66..d9d89d7fb5053 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -2382,7 +2382,6 @@ keyword_stored_synthetic: - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } - do: - catch: bad_request # TODO make this work in a follow up search: index: test_synthetic body: @@ -2391,13 +2390,11 @@ keyword_stored_synthetic: field: script: source: "field('keyword_stored').get('missing')" -# - match: { hits.hits.0.fields.field.0: "no doc values" } -# - match: { hits.hits.1.fields.field.0: "missing" } -# - match: { hits.hits.2.fields.field.0: "no doc values 0" } # doc values are sorted - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { hits.hits.1.fields.field.0: "missing" } + - match: { hits.hits.2.fields.field.0: "no doc values 0" } # doc values are sorted - do: - catch: bad_request # TODO make this work in a follow up search: index: test_synthetic body: @@ -2407,13 +2404,11 @@ keyword_stored_synthetic: script: source: "/* avoid yaml stash */ $('keyword_stored', 'missing')" # same as `field('keyword').get('missing')` -# - match: { hits.hits.0.fields.field.0: "no doc values" } -# - match: { hits.hits.1.fields.field.0: "missing" } -# - match: { hits.hits.2.fields.field.0: "no doc values 0" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { hits.hits.1.fields.field.0: "missing" } + - match: { hits.hits.2.fields.field.0: "no doc values 0" } - do: - catch: bad_request # TODO make this work in a follow up search: index: test_synthetic body: @@ -2422,13 +2417,11 @@ keyword_stored_synthetic: field: script: source: "field('keyword_stored').get(1, 'dne')" -# - match: { hits.hits.0.fields.field.0: "dne" } -# - match: { hits.hits.1.fields.field.0: "dne" } -# - match: { hits.hits.2.fields.field.0: "no doc values 1" } # doc values are sorted - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "dne" } + - match: { hits.hits.1.fields.field.0: "dne" } + - match: { hits.hits.2.fields.field.0: "no doc values 1" } # doc values are sorted - do: - catch: bad_request # TODO make this work in a follow up search: index: test_synthetic body: @@ -2437,10 +2430,9 @@ keyword_stored_synthetic: field: script: source: "String.join(', ', field('keyword_stored'))" -# - match: { hits.hits.0.fields.field.0: "no doc values" } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "no doc values 0, no doc values 1, no doc values 2" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "no doc values 0, no doc values 1, no doc values 2" } --- long: @@ -5188,7 +5180,7 @@ text_sub_stored_keyword_synthetic: script_fields: field: script: - source: "String cat = ''; for (String s : field('text_sub_stored_keyword')) { cat += s; } cat + field('text_no_field_data').size();" + source: "String cat = ''; for (String s : field('text_sub_stored_keyword')) { cat += s; } cat + field('text_sub_stored_keyword').size();" # - match: { hits.hits.0.fields.field.0: "Lots of text.1" } # - match: { hits.hits.1.fields.field.0: "0" } # - match: { hits.hits.2.fields.field.0: "Lots of text.SOOOOO much texteven more text3" } @@ -5221,7 +5213,6 @@ text_stored_synthetic: - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5230,13 +5221,11 @@ text_stored_synthetic: field: script: source: "field('text_stored').get('')" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5245,13 +5234,11 @@ text_stored_synthetic: field: script: source: "/* avoid yaml stash */ $('text_stored', '')" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5260,13 +5247,11 @@ text_stored_synthetic: field: script: source: "String defaultText = 'default text'; field('text_stored').get(defaultText)" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "default text" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "default text" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5275,13 +5260,11 @@ text_stored_synthetic: field: script: source: "String defaultText = 'default text'; $('text_stored', defaultText)" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "default text" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "default text" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5290,13 +5273,11 @@ text_stored_synthetic: field: script: source: "field('text_stored').get(1, '')" -# - match: { hits.hits.0.fields.field.0: "" } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "" } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5305,13 +5286,11 @@ text_stored_synthetic: field: script: source: "String defaultText = 'default text'; field('text_stored').get(1, defaultText)" -# - match: { hits.hits.0.fields.field.0: "default text" } -# - match: { hits.hits.1.fields.field.0: "default text" } -# - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "default text" } + - match: { hits.hits.1.fields.field.0: "default text" } + - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5320,13 +5299,11 @@ text_stored_synthetic: field: script: source: "field('text_stored').get(1, '')" -# - match: { hits.hits.0.fields.field.0: "" } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "" } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5334,11 +5311,10 @@ text_stored_synthetic: script_fields: field: script: - source: "String cat = ''; for (String s : field('text_stored')) { cat += s; } cat + field('text_no_field_data').size();" -# - match: { hits.hits.0.fields.field.0: "Lots of text.1" } -# - match: { hits.hits.1.fields.field.0: "0" } -# - match: { hits.hits.2.fields.field.0: "Lots of text.SOOOOO much texteven more text3" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + source: "String cat = ''; for (String s : field('text_stored')) { cat += s; } cat + field('text_stored').size();" + - match: { hits.hits.0.fields.field.0: "Lots of text.1" } + - match: { hits.hits.1.fields.field.0: "0" } + - match: { hits.hits.2.fields.field.0: "Lots of text.SOOOOO much texteven more text3" } --- match_only_text: @@ -5497,7 +5473,6 @@ match_only_text_synthetic: - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5506,13 +5481,11 @@ match_only_text_synthetic: field: script: source: "field('match_only_text').get('')" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5521,13 +5494,11 @@ match_only_text_synthetic: field: script: source: "/* avoid yaml stash */ $('match_only_text', '')" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5536,13 +5507,11 @@ match_only_text_synthetic: field: script: source: "String defaultText = 'default text'; field('match_only_text').get(defaultText)" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "default text" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "default text" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5551,13 +5520,11 @@ match_only_text_synthetic: field: script: source: "String defaultText = 'default text'; $('match_only_text', defaultText)" -# - match: { hits.hits.0.fields.field.0: "Lots of text." } -# - match: { hits.hits.1.fields.field.0: "default text" } -# - match: { hits.hits.2.fields.field.0: "Lots of text." } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text." } + - match: { hits.hits.1.fields.field.0: "default text" } + - match: { hits.hits.2.fields.field.0: "Lots of text." } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5566,13 +5533,11 @@ match_only_text_synthetic: field: script: source: "field('match_only_text').get(1, '')" -# - match: { hits.hits.0.fields.field.0: "" } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "" } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5581,13 +5546,11 @@ match_only_text_synthetic: field: script: source: "String defaultText = 'default text'; field('match_only_text').get(1, defaultText)" -# - match: { hits.hits.0.fields.field.0: "default text" } -# - match: { hits.hits.1.fields.field.0: "default text" } -# - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "default text" } + - match: { hits.hits.1.fields.field.0: "default text" } + - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5596,13 +5559,11 @@ match_only_text_synthetic: field: script: source: "field('match_only_text').get(1, '')" -# - match: { hits.hits.0.fields.field.0: "" } -# - match: { hits.hits.1.fields.field.0: "" } -# - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "" } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "SOOOOO much text" } - do: - catch: bad_request # TODO fix in a followup search: index: test_synthetic body: @@ -5611,10 +5572,9 @@ match_only_text_synthetic: field: script: source: "String cat = ''; for (String s : field('match_only_text')) { cat += s; } cat + field('match_only_text').size();" -# - match: { hits.hits.0.fields.field.0: "Lots of text.1" } -# - match: { hits.hits.1.fields.field.0: "0" } -# - match: { hits.hits.2.fields.field.0: "Lots of text.SOOOOO much texteven more text3" } - - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + - match: { hits.hits.0.fields.field.0: "Lots of text.1" } + - match: { hits.hits.1.fields.field.0: "0" } + - match: { hits.hits.2.fields.field.0: "Lots of text.SOOOOO much texteven more text3" } --- version and sequence number: diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index be9ddc69bd444..08f39c583e2f7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; +import org.elasticsearch.index.fielddata.StoredFieldSortedBinaryIndexFieldData; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; @@ -121,7 +122,13 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) { NamedAnalyzer searchQuoteAnalyzer = analyzers.getSearchQuoteAnalyzer(); NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer(); TextSearchInfo tsi = new TextSearchInfo(Defaults.FIELD_TYPE, null, searchAnalyzer, searchQuoteAnalyzer); - MatchOnlyTextFieldType ft = new MatchOnlyTextFieldType(context.buildFullName(name), tsi, indexAnalyzer, meta.getValue()); + MatchOnlyTextFieldType ft = new MatchOnlyTextFieldType( + context.buildFullName(name), + tsi, + indexAnalyzer, + context.isSourceSynthetic(), + meta.getValue() + ); return ft; } @@ -148,10 +155,16 @@ public static class MatchOnlyTextFieldType extends StringFieldType { private final Analyzer indexAnalyzer; private final TextFieldType textFieldType; - public MatchOnlyTextFieldType(String name, TextSearchInfo tsi, Analyzer indexAnalyzer, Map meta) { + public MatchOnlyTextFieldType( + String name, + TextSearchInfo tsi, + Analyzer indexAnalyzer, + boolean isSyntheticSource, + Map meta + ) { super(name, true, false, false, tsi, meta); this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer); - this.textFieldType = new TextFieldType(name); + this.textFieldType = new TextFieldType(name, isSyntheticSource); } public MatchOnlyTextFieldType(String name) { @@ -159,6 +172,7 @@ public MatchOnlyTextFieldType(String name) { name, new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), Lucene.STANDARD_ANALYZER, + false, Collections.emptyMap() ); } @@ -305,17 +319,28 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { - if (fieldDataContext.fielddataOperation() == FielddataOperation.SCRIPT) { - return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( - name(), + if (fieldDataContext.fielddataOperation() != FielddataOperation.SCRIPT) { + throw new IllegalArgumentException(CONTENT_TYPE + " fields do not support sorting and aggregations"); + } + if (textFieldType.isSyntheticSource()) { + return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( + storedFieldNameForSyntheticSource(), CoreValuesSourceType.KEYWORD, - SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), - fieldDataContext.lookupSupplier().get().source(), TextDocValuesField::new - ); + ) { + @Override + protected BytesRef storedToBytesRef(Object stored) { + return new BytesRef((String) stored); + } + }; } - - throw new IllegalArgumentException(CONTENT_TYPE + " fields do not support sorting and aggregations"); + return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), + fieldDataContext.lookupSupplier().get().source(), + TextDocValuesField::new + ); } private String storedFieldNameForSyntheticSource() { diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java index 5dc72f0822e31..9c9e8e7955a85 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java @@ -247,6 +247,16 @@ public List invalidExample() throws IOException { } } + public void testDocValues() throws IOException { + MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "match_only_text"))); + assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + } + + public void testDocValuesLoadedFromSynthetic() throws IOException { + MapperService mapper = createMapperService(syntheticSourceFieldMapping(b -> b.field("type", "match_only_text"))); + assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + } + @Override protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 02854917e4ae3..7dde32b794b80 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -121,7 +121,13 @@ private AnnotatedTextFieldType buildFieldType(FieldType fieldType, MapperBuilder wrapAnalyzer(analyzers.getSearchAnalyzer()), wrapAnalyzer(analyzers.getSearchQuoteAnalyzer()) ); - return new AnnotatedTextFieldType(context.buildFullName(name), store.getValue(), tsi, meta.getValue()); + return new AnnotatedTextFieldType( + context.buildFullName(name), + store.getValue(), + tsi, + context.isSourceSynthetic(), + meta.getValue() + ); } @Override @@ -467,8 +473,14 @@ private void emitAnnotation(int firstSpannedTextPosInc, int annotationPosLen) th public static final class AnnotatedTextFieldType extends TextFieldMapper.TextFieldType { - private AnnotatedTextFieldType(String name, boolean store, TextSearchInfo tsi, Map meta) { - super(name, true, store, tsi, meta); + private AnnotatedTextFieldType( + String name, + boolean store, + TextSearchInfo tsi, + boolean isSyntheticSource, + Map meta + ) { + super(name, true, store, tsi, isSyntheticSource, meta); } public AnnotatedTextFieldType(String name, Map meta) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java new file mode 100644 index 0000000000000..e9814e8f5ea65 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.SortField; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; +import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.ToScriptFieldFactory; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +import java.util.Set; + +/** + * Per segment values for a field loaded from stored fields. + */ +public abstract class StoredFieldIndexFieldData implements IndexFieldData.StoredFieldLeafFieldData> { + private final String fieldName; + private final ValuesSourceType valuesSourceType; + protected final ToScriptFieldFactory toScriptFieldFactory; + protected final StoredFieldLoader loader; + + protected StoredFieldIndexFieldData(String fieldName, ValuesSourceType valuesSourceType, ToScriptFieldFactory toScriptFieldFactory) { + this.fieldName = fieldName; + this.valuesSourceType = valuesSourceType; + this.toScriptFieldFactory = toScriptFieldFactory; + this.loader = StoredFieldLoader.create(false, Set.of(fieldName)); + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return valuesSourceType; + } + + @Override + public final StoredFieldLeafFieldData load(LeafReaderContext context) { + return loadDirect(context); + } + + @Override + public final StoredFieldLeafFieldData loadDirect(LeafReaderContext context) { + return new StoredFieldLeafFieldData(loader.getLoader(context, null)); + } + + protected abstract T loadLeaf(LeafStoredFieldLoader leafStoredFieldLoader); + + @Override + public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + throw new IllegalArgumentException("not supported for stored field fallback"); + } + + @Override + public BucketedSort newBucketedSort( + BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra + ) { + throw new IllegalArgumentException("not supported for stored field fallback"); + } + + public class StoredFieldLeafFieldData implements LeafFieldData { + private final LeafStoredFieldLoader loader; + + protected StoredFieldLeafFieldData(LeafStoredFieldLoader loader) { + this.loader = loader; + } + + @Override + public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { + return toScriptFieldFactory.getScriptFieldFactory(loadLeaf(loader), fieldName); + } + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void close() {} + + @Override + public SortedBinaryDocValues getBytesValues() { + throw new IllegalArgumentException("not supported for source fallback"); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java new file mode 100644 index 0000000000000..17648fbc43eca --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; +import org.elasticsearch.script.field.ToScriptFieldFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Per segment values for a field loaded from stored fields exposing {@link SortedBinaryDocValues}. + */ +public abstract class StoredFieldSortedBinaryIndexFieldData extends StoredFieldIndexFieldData { + + protected StoredFieldSortedBinaryIndexFieldData( + String fieldName, + ValuesSourceType valuesSourceType, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, toScriptFieldFactory); + } + + @Override + protected SourceValueFetcherSortedBinaryDocValues loadLeaf(LeafStoredFieldLoader leafStoredFieldLoader) { + return new SourceValueFetcherSortedBinaryDocValues(leafStoredFieldLoader); + } + + protected abstract BytesRef storedToBytesRef(Object stored); + + class SourceValueFetcherSortedBinaryDocValues extends SortedBinaryDocValues { + private final LeafStoredFieldLoader loader; + private final List sorted = new ArrayList<>(); + + private int current; + private int docValueCount; + + SourceValueFetcherSortedBinaryDocValues(LeafStoredFieldLoader loader) { + this.loader = loader; + } + + @Override + public boolean advanceExact(int doc) throws IOException { + loader.advanceTo(doc); + List values = loader.storedFields().get(getFieldName()); + if (values == null || values.isEmpty()) { + current = 0; + docValueCount = 0; + return false; + } + sorted.clear(); + for (Object o : values) { + sorted.add(storedToBytesRef(o)); + } + Collections.sort(sorted); + current = 0; + docValueCount = sorted.size(); + return true; + } + + @Override + public int docValueCount() { + return docValueCount; + } + + @Override + public BytesRef nextValue() throws IOException { + assert current < docValueCount; + return sorted.get(current++); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 7188e47db43ce..8ebf6c5c5fc65 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -47,6 +47,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; +import org.elasticsearch.index.fielddata.StoredFieldSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; @@ -294,7 +295,15 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType } else if (splitQueriesOnWhitespace.getValue()) { searchAnalyzer = Lucene.WHITESPACE_ANALYZER; } - return new KeywordFieldType(context.buildFullName(name), fieldType, normalizer, searchAnalyzer, quoteAnalyzer, this); + return new KeywordFieldType( + context.buildFullName(name), + fieldType, + normalizer, + searchAnalyzer, + quoteAnalyzer, + this, + context.isSourceSynthetic() + ); } @Override @@ -334,6 +343,7 @@ public static final class KeywordFieldType extends StringFieldType { private final boolean eagerGlobalOrdinals; private final FieldValues scriptValues; private final boolean isDimension; + private final boolean isSyntheticSource; public KeywordFieldType( String name, @@ -341,7 +351,8 @@ public KeywordFieldType( NamedAnalyzer normalizer, NamedAnalyzer searchAnalyzer, NamedAnalyzer quoteAnalyzer, - Builder builder + Builder builder, + boolean isSyntheticSource ) { super( name, @@ -357,6 +368,7 @@ public KeywordFieldType( this.nullValue = builder.nullValue.getValue(); this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); + this.isSyntheticSource = isSyntheticSource; } public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { @@ -367,6 +379,7 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; + this.isSyntheticSource = false; } public KeywordFieldType(String name) { @@ -388,6 +401,7 @@ public KeywordFieldType(String name, FieldType fieldType) { this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; + this.isSyntheticSource = false; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { @@ -398,6 +412,7 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; + this.isSyntheticSource = false; } @Override @@ -686,30 +701,51 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (operation == FielddataOperation.SEARCH) { failIfNoDocValues(); + return fieldDataFromDocValues(); } - - if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) { - return new SortedSetOrdinalsIndexFieldData.Builder( - name(), - CoreValuesSourceType.KEYWORD, - (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n) - ); + if (operation != FielddataOperation.SCRIPT) { + throw new IllegalStateException("unknown operation [" + operation.name() + "]"); } - if (operation == FielddataOperation.SCRIPT) { - SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); - Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); - - return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( + if (hasDocValues()) { + return fieldDataFromDocValues(); + } + if (isSyntheticSource) { + if (false == isStored()) { + throw new IllegalStateException( + "keyword field [" + + name() + + "] is only supported in synthetic _source index if it creates doc values or stored fields" + ); + } + return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( name(), CoreValuesSourceType.KEYWORD, - sourceValueFetcher(sourcePaths), - searchLookup.source(), KeywordDocValuesField::new - ); + ) { + @Override + protected BytesRef storedToBytesRef(Object stored) { + return (BytesRef) stored; + } + }; } - throw new IllegalStateException("unknown field data type [" + operation.name() + "]"); + Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); + return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + sourceValueFetcher(sourcePaths), + fieldDataContext.lookupSupplier().get().source(), + KeywordDocValuesField::new + ); + } + + private SortedSetOrdinalsIndexFieldData.Builder fieldDataFromDocValues() { + return new SortedSetOrdinalsIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n) + ); } @Override @@ -1112,6 +1148,7 @@ protected void write(XContentBuilder b, Object value) throws IOException { fieldType().ignoreAbove == Defaults.IGNORE_ABOVE ? null : originalName(), false ) { + @Override protected BytesRef convert(BytesRef value) { return value; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 0ad797cc34405..695b0627292e6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -174,8 +174,8 @@ public Function pointReaderIfPossible() { return null; } - /** Returns true if the field is aggregatable. - * + /** + * Returns true if the field is aggregatable. */ public boolean isAggregatable() { return hasDocValues(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 8e2dd973dd1da..43bd6d220f188 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -61,6 +61,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; +import org.elasticsearch.index.fielddata.StoredFieldSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; @@ -351,7 +352,14 @@ private TextFieldType buildFieldType(FieldType fieldType, MapperBuilderContext c ft = new LegacyTextFieldType(context.buildFullName(name), index.getValue(), store.getValue(), tsi, meta.getValue()); // ignore fieldData and eagerGlobalOrdinals } else { - ft = new TextFieldType(context.buildFullName(name), index.getValue(), store.getValue(), tsi, meta.getValue()); + ft = new TextFieldType( + context.buildFullName(name), + index.getValue(), + store.getValue(), + tsi, + context.isSourceSynthetic(), + meta.getValue() + ); ft.eagerGlobalOrdinals = eagerGlobalOrdinals.getValue(); if (fieldData.getValue()) { ft.setFielddata(true, freqFilter.getValue()); @@ -647,10 +655,19 @@ public static class TextFieldType extends StringFieldType { private PrefixFieldType prefixFieldType; private boolean indexPhrases = false; private boolean eagerGlobalOrdinals = false; - - public TextFieldType(String name, boolean indexed, boolean stored, TextSearchInfo tsi, Map meta) { + private final boolean isSyntheticSource; + + public TextFieldType( + String name, + boolean indexed, + boolean stored, + TextSearchInfo tsi, + boolean isSyntheticSource, + Map meta + ) { super(name, indexed, stored, false, tsi, meta); fielddata = false; + this.isSyntheticSource = isSyntheticSource; } public TextFieldType(String name, boolean indexed, boolean stored, Map meta) { @@ -663,14 +680,16 @@ public TextFieldType(String name, boolean indexed, boolean stored, Map new StoredFieldSortedBinaryIndexFieldData( + name(), + CoreValuesSourceType.KEYWORD, + TextDocValuesField::new + ) { + @Override + protected BytesRef storedToBytesRef(Object stored) { + return new BytesRef((String) stored); + } + }; + } + throw new IllegalArgumentException( + "fetching values from a text field [" + + name() + + "] is not yet supported because synthetic _source is enabled and the field doesn't create stored fields" + ); + } + return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), + fieldDataContext.lookupSupplier().get().source(), + TextDocValuesField::new + ); + } + + public boolean isSyntheticSource() { + return isSyntheticSource; } } public static class ConstantScoreTextFieldType extends TextFieldType { - public ConstantScoreTextFieldType(String name, boolean indexed, boolean stored, TextSearchInfo tsi, Map meta) { - super(name, indexed, stored, tsi, meta); + public ConstantScoreTextFieldType( + String name, + boolean indexed, + boolean stored, + TextSearchInfo tsi, + boolean isSyntheticSource, + Map meta + ) { + super(name, indexed, stored, tsi, isSyntheticSource, meta); } public ConstantScoreTextFieldType(String name) { @@ -955,6 +1004,7 @@ public ConstantScoreTextFieldType(String name) { true, false, new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + false, Collections.emptyMap() ); } @@ -965,6 +1015,7 @@ public ConstantScoreTextFieldType(String name, boolean indexed, boolean stored, indexed, stored, new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + false, meta ); } @@ -1020,7 +1071,7 @@ static class LegacyTextFieldType extends ConstantScoreTextFieldType { private final MappedFieldType existQueryFieldType; LegacyTextFieldType(String name, boolean indexed, boolean stored, TextSearchInfo tsi, Map meta) { - super(name, indexed, stored, tsi, meta); + super(name, indexed, stored, tsi, false, meta); // norms are not available, neither are doc-values, so fall back to _source to run exists query existQueryFieldType = KeywordScriptFieldType.sourceOnly(name()).asMappedFieldTypes().findFirst().get(); } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index 17cdafc4e6b8e..118670e8ca99e 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -227,7 +227,7 @@ public boolean hasSourceAsMap() { @Override public void setSegmentAndDocument(LeafReaderContext context, int docId) { - // + // nothing to do } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index 447562bb74767..04d9846ad2929 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -189,7 +189,13 @@ protected static SearchExecutionContext mockContext() { } protected static FieldDataContext mockFielddataContext() { - return new FieldDataContext("test", mockContext()::lookup, mockContext()::sourcePath, MappedFieldType.FielddataOperation.SCRIPT); + SearchExecutionContext searchExecutionContext = mockContext(); + return new FieldDataContext( + "test", + searchExecutionContext::lookup, + mockContext()::sourcePath, + MappedFieldType.FielddataOperation.SCRIPT + ); } protected static SearchExecutionContext mockContext(boolean allowExpensiveQueries) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java index cd0b228d42598..5b7bc25d0a009 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java @@ -53,7 +53,7 @@ public void testFetchIgnoredFieldValue() throws IOException { iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java index 7512249deb24f..1536e9e4e46db 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java @@ -49,7 +49,7 @@ public void testFetchFieldValue() throws IOException { IndexFieldMapper.IndexFieldType ft = (IndexFieldMapper.IndexFieldType) mapperService.fieldType("_index"); SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 3f6acbd9a1284..877d38b4b1f7a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -768,4 +768,21 @@ public void testLegacyField() throws Exception { assertThat(service.fieldType("mykeyw"), instanceOf(KeywordFieldMapper.KeywordFieldType.class)); assertNotEquals(Lucene.KEYWORD_ANALYZER, ((KeywordFieldMapper.KeywordFieldType) service.fieldType("mykeyw")).normalizer()); } + + public void testDocValues() throws IOException { + MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "keyword"))); + assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + } + + public void testDocValuesLoadedFromSource() throws IOException { + MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "keyword").field("doc_values", false))); + assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + } + + public void testDocValuesLoadedFromStoredSynthetic() throws IOException { + MapperService mapper = createMapperService( + syntheticSourceFieldMapping(b -> b.field("type", "keyword").field("doc_values", false).field("store", true)) + ); + assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java index d44889a261255..b20358348712e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java @@ -64,7 +64,7 @@ public void testFetchValue() throws Exception { }, iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java index 9de12bae4be82..99daa3770c55f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java @@ -50,7 +50,6 @@ public void testDefaults() throws IOException { } public void testEnableFieldData() throws IOException { - boolean[] enabled = new boolean[1]; MapperService mapperService = createMapperService(() -> enabled[0], mapping(b -> {})); @@ -78,7 +77,7 @@ public void testFetchIdFieldValue() throws IOException { iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index 597e28d6575e5..7342e729b8872 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -71,7 +71,7 @@ public void testFetchRoutingFieldValue() throws IOException { iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index d06a6f428d81a..02ad6eff329b5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1227,4 +1227,43 @@ public void testIgnoreEagerGlobalOrdinalsOnLegacyIndex() throws IOException { mapperService = createMapperService(Version.fromString("5.0.0"), mapping); assertFalse(((TextFieldMapper) mapperService.documentMapper().mappers().getMapper("field")).fieldType().eagerGlobalOrdinals()); } + + public void testDocValues() throws IOException { + MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "text"))); + for (String input : new String[] { + "foo", // Won't be tokenized + "foo bar", // Will be tokenized. But script doc values still returns the whole field. + }) { + assertScriptDocValues(mapper, input, equalTo(List.of(input))); + } + } + + public void testDocValuesLoadedFromStoredSynthetic() throws IOException { + MapperService mapper = createMapperService(syntheticSourceFieldMapping(b -> b.field("type", "text").field("store", true))); + for (String input : new String[] { + "foo", // Won't be tokenized + "foo bar", // Will be tokenized. But script doc values still returns the whole field. + }) { + assertScriptDocValues(mapper, input, equalTo(List.of(input))); + } + } + + public void testDocValuesLoadedFromSubKeywordSynthetic() throws IOException { + MapperService mapper = createMapperService(syntheticSourceFieldMapping(b -> { + b.field("type", "text"); + b.startObject("fields"); + { + b.startObject("raw").field("type", "keyword").endObject(); + } + b.endObject(); + })); + Exception e = expectThrows(IllegalArgumentException.class, () -> assertScriptDocValues(mapper, "foo", equalTo(List.of("foo")))); + assertThat( + e.getMessage(), + equalTo( + "fetching values from a text field [field] is not yet supported because synthetic _source is " + + "enabled and the field doesn't create stored fields" + ) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 755dd3f668bde..0c54a1be33384 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -45,7 +45,7 @@ public class TextFieldTypeTests extends FieldTypeTestCase { private static TextFieldType createFieldType() { - return new TextFieldType("field"); + return new TextFieldType("field", randomBoolean()); } public void testIsAggregatableDependsOnFieldData() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java index afeff3fc94447..8cd212130a916 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java @@ -53,7 +53,7 @@ public void testFetchFieldValue() throws IOException { VersionFieldMapper.VersionFieldType ft = (VersionFieldMapper.VersionFieldType) mapperService.fieldType("_version"); SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index b2c1e12bae5fe..7bd6cc2abf083 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -104,7 +104,7 @@ public class SearchExecutionContextTests extends ESTestCase { public void testFailIfFieldMappingNotFound() { SearchExecutionContext context = createSearchExecutionContext(IndexMetadata.INDEX_UUID_NA_VALUE, null); context.setAllowUnmappedFields(false); - MappedFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + MappedFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); MappedFieldType result = context.failIfFieldMappingNotFound("name", fieldType); assertThat(result, sameInstance(fieldType)); QueryShardException e = expectThrows(QueryShardException.class, () -> context.failIfFieldMappingNotFound("name", null)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java index 7915d213dbac1..a3c1ed5d7793a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java @@ -431,7 +431,7 @@ public void testRangeFilter() throws IOException { * Tests a filter that needs the cache to be fast. */ public void testPhraseFilter() throws IOException { - MappedFieldType ft = new TextFieldMapper.TextFieldType("test"); + MappedFieldType ft = new TextFieldMapper.TextFieldType("test", randomBoolean()); AggregationBuilder builder = new FiltersAggregationBuilder( "test", new KeyedFilter("q1", new MatchPhraseQueryBuilder("test", "will find me").slop(0)) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java index ed20e556a56fb..0212ceac737b3 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java @@ -41,7 +41,7 @@ public class SamplerAggregatorTests extends AggregatorTestCase { * Uses the sampler aggregation to find the minimum value of a field out of the top 3 scoring documents in a search. */ public void testSampler() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType("int", NumberFieldMapper.NumberType.LONG); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); @@ -76,7 +76,7 @@ public void testSampler() throws IOException { } public void testRidiculousSize() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType("int", NumberFieldMapper.NumberType.LONG); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java index edc6fd5dcf76c..8574f402fb92a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java @@ -103,7 +103,7 @@ protected List unsupportedMappedFieldTypes() { } public void testSignificance(SignificanceHeuristic heuristic) throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); @@ -211,7 +211,7 @@ public void testSignificance() throws IOException { * @throws IOException on test setup failure */ public void testSamplingConsistency() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); @@ -317,7 +317,7 @@ public void testNumericSignificance() throws IOException { * Uses the significant terms aggregation on an index with unmapped field */ public void testUnmapped() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); @@ -384,7 +384,7 @@ public void testRangeField() throws IOException { } public void testFieldAlias() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); @@ -440,7 +440,7 @@ public void testFieldAlias() throws IOException { } public void testFieldBackground() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java index 74b9b232d852e..c56b084e1511f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java @@ -71,7 +71,7 @@ protected List unsupportedMappedFieldTypes() { * Uses the significant text aggregation to find the keywords in text fields */ public void testSignificance() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); @@ -126,7 +126,7 @@ public void testSignificance() throws IOException { * Uses the significant text aggregation to find the keywords in text fields and include/exclude selected terms */ public void testIncludeExcludes() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); @@ -187,7 +187,7 @@ public void testIncludeExcludes() throws IOException { } public void testMissingField() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); indexWriterConfig.setMaxBufferedDocs(100); @@ -215,7 +215,7 @@ public void testMissingField() throws IOException { } public void testFieldAlias() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); @@ -274,7 +274,7 @@ public void testFieldAlias() throws IOException { } public void testInsideTermsAgg() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); @@ -331,7 +331,7 @@ private void indexDocuments(IndexWriter writer) throws IOException { * Test documents with arrays of text */ public void testSignificanceOnTextArrays() throws IOException { - TextFieldType textFieldType = new TextFieldType("text"); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 1da6d61ec797b..6811179644c09 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.NestedObjectMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -48,7 +49,6 @@ import org.elasticsearch.xcontent.XContentType; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.mockito.Mockito; import java.io.IOException; import java.util.Collections; @@ -60,6 +60,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; +import static org.mockito.Mockito.mock; public abstract class AbstractSortTestCase> extends ESTestCase { @@ -187,7 +188,7 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea index, Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build() ); - BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, Mockito.mock(BitsetFilterCache.Listener.class)); + BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, mock(BitsetFilterCache.Listener.class)); BiFunction> indexFieldDataLookup = (fieldType, fdc) -> { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fdc); return builder.build(new IndexFieldDataCache.None(), null); @@ -202,7 +203,7 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea bitsetFilterCache, indexFieldDataLookup, null, - null, + MappingLookup.EMPTY, null, scriptService, parserConfig(), diff --git a/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java b/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java index 26f3e1d2e24b7..fa97b1c1d9536 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java @@ -796,7 +796,7 @@ public void testParsingContextFromDocument() throws Exception { assertTrue(context.contains("category1")); document = new LuceneDocument(); - TextFieldMapper.TextFieldType text = new TextFieldMapper.TextFieldType("category"); + TextFieldMapper.TextFieldType text = new TextFieldMapper.TextFieldType("category", randomBoolean()); document.add(new Field(text.name(), "category1", TextFieldMapper.Defaults.FIELD_TYPE)); // Ignore stored field document.add(new StoredField(text.name(), "category1", TextFieldMapper.Defaults.FIELD_TYPE)); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 4ab96284963aa..9454a6a8555a4 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -659,6 +659,12 @@ public void onRemoval(ShardId shardId, Accountable accountable) { ); } + protected TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup( + MapperService mapperService + ) { + return fieldDataLookup(mapperService.mappingLookup()::sourcePaths); + } + protected TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup( Function> sourcePathsLookup ) { @@ -744,4 +750,13 @@ protected final XContentBuilder syntheticSourceMapping(CheckedConsumer buildField) + throws IOException { + return syntheticSourceMapping(b -> { + b.startObject("field"); + buildField.accept(b); + b.endObject(); + }); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index aa9fdb0d8c9b7..57e1d8731c181 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -34,7 +34,9 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.FieldDataContext; +import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.query.SearchExecutionContext; @@ -507,7 +509,7 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi iw -> { SearchLookup lookup = new SearchLookup( mapperService::fieldType, - fieldDataLookup(mapperService.mappingLookup()::sourcePaths), + fieldDataLookup(mapperService), new SourceLookup.ReaderSourceProvider() ); ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft, MappedFieldType.FielddataOperation.SEARCH)); @@ -521,6 +523,26 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi return result.get(); } + protected final void assertScriptDocValues(MapperService mapperService, Object sourceValue, Matcher> dvMatcher) + throws IOException { + withLuceneIndex( + mapperService, + iw -> { iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field("field", sourceValue))).rootDoc()); }, + iw -> { + IndexSearcher searcher = newSearcher(iw); + MappedFieldType ft = mapperService.fieldType("field"); + SearchLookup searchLookup = new SearchLookup(null, null, mapperService.mappingLookup().getSourceProvider()); + IndexFieldData sfd = ft.fielddataBuilder( + new FieldDataContext("", () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) + ).build(null, null); + LeafFieldData lfd = sfd.load(getOnlyLeafReader(searcher.getIndexReader()).getContext()); + DocValuesScriptFieldFactory sff = lfd.getScriptFieldFactory("field"); + sff.setNextDocId(0); + assertThat(sff.toScriptDocValues(), dvMatcher); + } + ); + } + private class UpdateCheck { final XContentBuilder init; final XContentBuilder update; @@ -785,11 +807,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer( - inv -> fieldDataLookup(mapperService.mappingLookup()::sourcePaths).apply( - ft, - () -> { throw new UnsupportedOperationException(); }, - fdt - ) + inv -> fieldDataLookup(mapperService).apply(ft, () -> { throw new UnsupportedOperationException(); }, fdt) ); ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java index db8ccff4fa12e..ea53443bd27a3 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java @@ -116,7 +116,7 @@ public void testUnmappedWithMissingField() throws IOException { } public void testMissing() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field(fieldType.name()) @@ -192,7 +192,7 @@ public void testQueryFiltering() throws IOException { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/47469") public void testSingleValuedFieldWithFormatter() throws IOException { - TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field("text") @@ -218,7 +218,7 @@ public void testSingleValuedFieldWithFormatter() throws IOException { public void testNestedAggregation() throws IOException { MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType("value", NumberFieldMapper.NumberType.INTEGER); - TextFieldMapper.TextFieldType textFieldType = new TextFieldMapper.TextFieldType("text"); + TextFieldMapper.TextFieldType textFieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); textFieldType.setFielddata(true); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.NUMERIC) @@ -273,7 +273,7 @@ public void testNestedAggregation() throws IOException { } public void testValueScriptSingleValuedField() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field(fieldType.name()) @@ -295,7 +295,7 @@ public void testValueScriptSingleValuedField() throws IOException { } public void testValueScriptMultiValuedField() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field(fieldType.name()) @@ -322,7 +322,7 @@ public void testValueScriptMultiValuedField() throws IOException { } public void testFieldScriptSingleValuedField() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").script( @@ -345,7 +345,7 @@ public void testFieldScriptSingleValuedField() throws IOException { } public void testFieldScriptMultiValuedField() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").script( @@ -377,7 +377,7 @@ private void testAggregation( CheckedConsumer buildIndex, Consumer verify ) throws IOException { - TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text"); + TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); fieldType.setFielddata(true); AggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field("text"); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java index 388821a41f51b..2e2a64d426a8d 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java @@ -537,7 +537,7 @@ private MappedFieldType numberFieldType(NumberType numberType, String name) { } private MappedFieldType textFieldType(String name) { - return new TextFieldMapper.TextFieldType(name); + return new TextFieldMapper.TextFieldType(name, randomBoolean()); } private MappedFieldType geoPointFieldType(String name) { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java index 83c4dd2513bfc..196584e4a7ce2 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java @@ -79,7 +79,7 @@ public void testCategorizationWithoutSubAggs() throws Exception { }, new AggTestConfig( new CategorizeTextAggregationBuilder("my_agg", TEXT_FIELD_NAME), - new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME), + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), longField(NUMERIC_FIELD_NAME) ) ); @@ -114,7 +114,13 @@ public void testCategorizationWithSubAggs() throws Exception { assertThat(((Max) result.getBuckets().get(1).getAggregations().get("max")).value(), equalTo(4.0)); assertThat(((Min) result.getBuckets().get(1).getAggregations().get("min")).value(), equalTo(0.0)); assertThat(((Avg) result.getBuckets().get(1).getAggregations().get("avg")).getValue(), equalTo(2.0)); - }, new AggTestConfig(aggBuilder, new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME), longField(NUMERIC_FIELD_NAME))); + }, + new AggTestConfig( + aggBuilder, + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + longField(NUMERIC_FIELD_NAME) + ) + ); } public void testCategorizationWithMultiBucketSubAggs() throws Exception { @@ -163,7 +169,13 @@ public void testCategorizationWithMultiBucketSubAggs() throws Exception { assertThat(histo.getBuckets().get(2).getDocCount(), equalTo(1L)); assertThat(((Avg) histo.getBuckets().get(0).getAggregations().get("avg")).getValue(), equalTo(0.0)); assertThat(((Avg) histo.getBuckets().get(2).getAggregations().get("avg")).getValue(), equalTo(4.0)); - }, new AggTestConfig(aggBuilder, new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME), longField(NUMERIC_FIELD_NAME))); + }, + new AggTestConfig( + aggBuilder, + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + longField(NUMERIC_FIELD_NAME) + ) + ); } public void testCategorizationAsSubAgg() throws Exception { @@ -245,7 +257,13 @@ public void testCategorizationAsSubAgg() throws Exception { assertThat(((Max) categorizationAggregation.getBuckets().get(1).getAggregations().get("max")).value(), equalTo(4.0)); assertThat(((Min) categorizationAggregation.getBuckets().get(1).getAggregations().get("min")).value(), equalTo(4.0)); assertThat(((Avg) categorizationAggregation.getBuckets().get(1).getAggregations().get("avg")).getValue(), equalTo(4.0)); - }, new AggTestConfig(aggBuilder, new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME), longField(NUMERIC_FIELD_NAME))); + }, + new AggTestConfig( + aggBuilder, + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + longField(NUMERIC_FIELD_NAME) + ) + ); } public void testCategorizationWithSubAggsManyDocs() throws Exception { @@ -294,7 +312,13 @@ public void testCategorizationWithSubAggsManyDocs() throws Exception { assertThat(histo.getBuckets().get(2).getDocCount(), equalTo(5000L)); assertThat(((Avg) histo.getBuckets().get(0).getAggregations().get("avg")).getValue(), equalTo(0.0)); assertThat(((Avg) histo.getBuckets().get(2).getAggregations().get("avg")).getValue(), equalTo(4.0)); - }, new AggTestConfig(aggBuilder, new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME), longField(NUMERIC_FIELD_NAME))); + }, + new AggTestConfig( + aggBuilder, + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + longField(NUMERIC_FIELD_NAME) + ) + ); } private static void writeTestDocs(RandomIndexWriter w) throws IOException { diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java index d3f0874f5abd1..06bd2260198ba 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java @@ -9,6 +9,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.TimeSeriesParams; import java.io.IOException; @@ -29,7 +30,8 @@ private TimeseriesFieldTypeHelper(final MapperService mapperService, final Strin } public boolean isTimeSeriesLabel(final String field, final Map unused) { - final MappedFieldType fieldType = mapperService.mappingLookup().getFieldType(field); + final MappingLookup lookup = mapperService.mappingLookup(); + final MappedFieldType fieldType = lookup.getFieldType(field); return fieldType != null && (timestampField.equals(field) == false) && (fieldType.isAggregatable()) diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 18b4fdfcdfa52..a1502d7ac8c33 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -60,6 +60,8 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperTestCase; +import org.elasticsearch.index.mapper.Mapping; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.query.SearchExecutionContext; @@ -1086,6 +1088,7 @@ protected final SearchExecutionContext createMockContext() { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fdc); return builder.build(new IndexFieldDataCache.None(), null); }; + MappingLookup lookup = MappingLookup.fromMapping(Mapping.EMPTY); return new SearchExecutionContext( 0, 0, @@ -1093,7 +1096,7 @@ protected final SearchExecutionContext createMockContext() { bitsetFilterCache, indexFieldDataLookup, null, - null, + lookup, null, null, parserConfig(), diff --git a/x-pack/test/idp-fixture/build.gradle b/x-pack/test/idp-fixture/build.gradle index 75e152bafae2e..5a9c387a9521b 100644 --- a/x-pack/test/idp-fixture/build.gradle +++ b/x-pack/test/idp-fixture/build.gradle @@ -5,6 +5,10 @@ import static org.elasticsearch.gradle.internal.distribution.InternalElasticsear apply plugin: 'elasticsearch.test.fixtures' apply plugin: 'elasticsearch.internal-distribution-download' +dockerCompose { + composeAdditionalArgs = ['--compatibility'] +} + tasks.register("copyKeystore", Sync) { from project(':x-pack:plugin:core') .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')