-
Notifications
You must be signed in to change notification settings - Fork 25.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Synthetic _source: support
field in many cases (#89950)
This adds support for the `field` scripting API in many but not all cases. Before this change numbers, dates, and IPs supported the `field` API when running with _source in synthetic mode because they always have doc values. This change adds support for `match_only_text`, `store`d `keyword` fields, and `store`d `text` fields. Two remaining field configurations work with synthetic _source and do not work with `field`: * A `text` field with a sub-`keyword` field that has `doc_values` * A `text` field with a sub-`keyword` field that is `store`d 
- Loading branch information
Showing
36 changed files
with
623 additions
and
205 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| pr: 89950 | ||
| summary: "Synthetic _source: support `field` in many cases" | ||
| area: TSDB | ||
| type: enhancement | ||
| issues: [] |
164 changes: 62 additions & 102 deletions
164
...-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
108 changes: 108 additions & 0 deletions
108
server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0 and the Server Side Public License, v 1; you may not use this file except | ||
| * in compliance with, at your election, the Elastic License 2.0 or the Server | ||
| * Side Public License, v 1. | ||
| */ | ||
|
|
||
| package org.elasticsearch.index.fielddata; | ||
|
|
||
| import org.apache.lucene.index.LeafReaderContext; | ||
| import org.apache.lucene.search.SortField; | ||
| import org.elasticsearch.common.util.BigArrays; | ||
| import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; | ||
| import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; | ||
| import org.elasticsearch.script.field.DocValuesScriptFieldFactory; | ||
| import org.elasticsearch.script.field.ToScriptFieldFactory; | ||
| import org.elasticsearch.search.DocValueFormat; | ||
| import org.elasticsearch.search.MultiValueMode; | ||
| import org.elasticsearch.search.aggregations.support.ValuesSourceType; | ||
| import org.elasticsearch.search.sort.BucketedSort; | ||
| import org.elasticsearch.search.sort.SortOrder; | ||
|
|
||
| import java.util.Set; | ||
|
|
||
| /** | ||
| * Per segment values for a field loaded from stored fields. | ||
| */ | ||
| public abstract class StoredFieldIndexFieldData<T> implements IndexFieldData<StoredFieldIndexFieldData<T>.StoredFieldLeafFieldData> { | ||
| private final String fieldName; | ||
| private final ValuesSourceType valuesSourceType; | ||
| protected final ToScriptFieldFactory<T> toScriptFieldFactory; | ||
| protected final StoredFieldLoader loader; | ||
|
|
||
| protected StoredFieldIndexFieldData(String fieldName, ValuesSourceType valuesSourceType, ToScriptFieldFactory<T> toScriptFieldFactory) { | ||
| this.fieldName = fieldName; | ||
| this.valuesSourceType = valuesSourceType; | ||
| this.toScriptFieldFactory = toScriptFieldFactory; | ||
| this.loader = StoredFieldLoader.create(false, Set.of(fieldName)); | ||
| } | ||
|
|
||
| @Override | ||
| public String getFieldName() { | ||
| return fieldName; | ||
| } | ||
|
|
||
| @Override | ||
| public ValuesSourceType getValuesSourceType() { | ||
| return valuesSourceType; | ||
| } | ||
|
|
||
| @Override | ||
| public final StoredFieldLeafFieldData load(LeafReaderContext context) { | ||
| return loadDirect(context); | ||
| } | ||
|
|
||
| @Override | ||
| public final StoredFieldLeafFieldData loadDirect(LeafReaderContext context) { | ||
| return new StoredFieldLeafFieldData(loader.getLoader(context, null)); | ||
| } | ||
|
|
||
| protected abstract T loadLeaf(LeafStoredFieldLoader leafStoredFieldLoader); | ||
|
|
||
| @Override | ||
| public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { | ||
| throw new IllegalArgumentException("not supported for stored field fallback"); | ||
| } | ||
|
|
||
| @Override | ||
| public BucketedSort newBucketedSort( | ||
| BigArrays bigArrays, | ||
| Object missingValue, | ||
| MultiValueMode sortMode, | ||
| XFieldComparatorSource.Nested nested, | ||
| SortOrder sortOrder, | ||
| DocValueFormat format, | ||
| int bucketSize, | ||
| BucketedSort.ExtraData extra | ||
| ) { | ||
| throw new IllegalArgumentException("not supported for stored field fallback"); | ||
| } | ||
|
|
||
| public class StoredFieldLeafFieldData implements LeafFieldData { | ||
| private final LeafStoredFieldLoader loader; | ||
|
|
||
| protected StoredFieldLeafFieldData(LeafStoredFieldLoader loader) { | ||
| this.loader = loader; | ||
| } | ||
|
|
||
| @Override | ||
| public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { | ||
| return toScriptFieldFactory.getScriptFieldFactory(loadLeaf(loader), fieldName); | ||
| } | ||
|
|
||
| @Override | ||
| public long ramBytesUsed() { | ||
| return 0; | ||
| } | ||
|
|
||
| @Override | ||
| public void close() {} | ||
|
|
||
| @Override | ||
| public SortedBinaryDocValues getBytesValues() { | ||
| throw new IllegalArgumentException("not supported for source fallback"); | ||
| } | ||
| } | ||
| } |
82 changes: 82 additions & 0 deletions
82
...rc/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0 and the Server Side Public License, v 1; you may not use this file except | ||
| * in compliance with, at your election, the Elastic License 2.0 or the Server | ||
| * Side Public License, v 1. | ||
| */ | ||
|
|
||
| package org.elasticsearch.index.fielddata; | ||
|
|
||
| import org.apache.lucene.util.BytesRef; | ||
| import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; | ||
| import org.elasticsearch.script.field.ToScriptFieldFactory; | ||
| import org.elasticsearch.search.aggregations.support.ValuesSourceType; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.Collections; | ||
| import java.util.List; | ||
|
|
||
| /** | ||
| * Per segment values for a field loaded from stored fields exposing {@link SortedBinaryDocValues}. | ||
| */ | ||
| public abstract class StoredFieldSortedBinaryIndexFieldData extends StoredFieldIndexFieldData<SortedBinaryDocValues> { | ||
|
|
||
| protected StoredFieldSortedBinaryIndexFieldData( | ||
| String fieldName, | ||
| ValuesSourceType valuesSourceType, | ||
| ToScriptFieldFactory<SortedBinaryDocValues> toScriptFieldFactory | ||
| ) { | ||
| super(fieldName, valuesSourceType, toScriptFieldFactory); | ||
| } | ||
|
|
||
| @Override | ||
| protected SourceValueFetcherSortedBinaryDocValues loadLeaf(LeafStoredFieldLoader leafStoredFieldLoader) { | ||
| return new SourceValueFetcherSortedBinaryDocValues(leafStoredFieldLoader); | ||
| } | ||
|
|
||
| protected abstract BytesRef storedToBytesRef(Object stored); | ||
|
|
||
| class SourceValueFetcherSortedBinaryDocValues extends SortedBinaryDocValues { | ||
| private final LeafStoredFieldLoader loader; | ||
| private final List<BytesRef> sorted = new ArrayList<>(); | ||
|
|
||
| private int current; | ||
| private int docValueCount; | ||
|
|
||
| SourceValueFetcherSortedBinaryDocValues(LeafStoredFieldLoader loader) { | ||
| this.loader = loader; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean advanceExact(int doc) throws IOException { | ||
| loader.advanceTo(doc); | ||
| List<Object> values = loader.storedFields().get(getFieldName()); | ||
| if (values == null || values.isEmpty()) { | ||
| current = 0; | ||
| docValueCount = 0; | ||
| return false; | ||
| } | ||
| sorted.clear(); | ||
| for (Object o : values) { | ||
| sorted.add(storedToBytesRef(o)); | ||
| } | ||
| Collections.sort(sorted); | ||
| current = 0; | ||
| docValueCount = sorted.size(); | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| public int docValueCount() { | ||
| return docValueCount; | ||
| } | ||
|
|
||
| @Override | ||
| public BytesRef nextValue() throws IOException { | ||
| assert current < docValueCount; | ||
| return sorted.get(current++); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.