Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support malformed numbers in synthetic _source #90428

Merged
merged 11 commits into from
Oct 4, 2022
Merged

Support malformed numbers in synthetic _source #90428

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
929512d
Update docs/changelog/90428.yaml
nik9000 Sep 27, 2022
3f30050
Support malformed numbers in synthetic _source
nik9000 Sep 27, 2022
483984e
WIP
nik9000 Sep 28, 2022
ee1436e
Merge branch 'main' into synthetic_source_ignore_malformed_numbers
nik9000 Sep 30, 2022
f7b2dc4
Move
nik9000 Oct 3, 2022
1477b2f
Merge branch 'main' into synthetic_source_ignore_malformed_numbers
nik9000 Oct 3, 2022
10ede36
All the tests
nik9000 Oct 3, 2022
45c12f4
Merge remote-tracking branch 'nik9000/synthetic_source_ignore_malform…
nik9000 Oct 3, 2022
9626603
Use previous
nik9000 Oct 3, 2022
b7eb5dc
Words
nik9000 Oct 4, 2022
c76d995
Merge branch 'main' into synthetic_source_ignore_malformed_numbers
nik9000 Oct 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/90428.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 90428
summary: Support malformed numbers in synthetic `_source`
area: TSDB
type: enhancement
issues: []
2 changes: 1 addition & 1 deletion ...-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -663,7 +663,7 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
}

@Override
protected SyntheticSourceSupport syntheticSourceSupport() {
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
throw new AssumptionViolatedException("not supported");
}

Expand Down
2 changes: 1 addition & 1 deletion ...er-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -709,7 +709,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
);
}
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName()) {
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed.value()) {
@Override
protected void writeValue(XContentBuilder b, long value) throws IOException {
b.value(decodeForSyntheticSource(value, scalingFactor));
Expand Down
3 changes: 2 additions & 1 deletion ...as/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,8 @@ protected boolean supportsIgnoreMalformed() {
}

@Override
protected SyntheticSourceSupport syntheticSourceSupport() {
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
assertFalse("match_only_text doesn't support ignoreMalformed", ignoreMalformed);
return new MatchOnlyTextSyntheticSourceSupport();
}

Expand Down
2 changes: 1 addition & 1 deletion ...tras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
}

@Override
protected SyntheticSourceSupport syntheticSourceSupport() {
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
throw new AssumptionViolatedException("not supported");
}

Expand Down
2 changes: 1 addition & 1 deletion ...ras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ protected boolean allowsNullValues() {
}

@Override
protected SyntheticSourceSupport syntheticSourceSupport() {
protected SyntheticSourceSupport syntheticSourceSupport(boolean syntheticSource) {
throw new AssumptionViolatedException("not supported");
}

Expand Down
123 changes: 62 additions & 61 deletions ...tras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,75 +337,76 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
}

@Override
protected SyntheticSourceSupport syntheticSourceSupport() {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pulled into a class so it's easier to read. sorry about the big diff.

return new SyntheticSourceSupport() {
private final double scalingFactor = randomDoubleBetween(0, Double.MAX_VALUE, false);
private final Double nullValue = usually() ? null : round(randomValue());

@Override
public SyntheticSourceExample example(int maxValues) {
if (randomBoolean()) {
Tuple<Double, Double> v = generateValue();
return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
}
List<Tuple<Double, Double>> values = randomList(1, maxValues, this::generateValue);
List<Double> in = values.stream().map(Tuple::v1).toList();
List<Double> outList = values.stream().map(Tuple::v2).sorted().toList();
Object out = outList.size() == 1 ? outList.get(0) : outList;
return new SyntheticSourceExample(in, out, this::mapping);
}
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
assumeFalse("scaled_float doesn't support ignore_malformed with synthetic _source", ignoreMalformed);
return new ScaledFloatSyntheticSourceSupport();
}

private Tuple<Double, Double> generateValue() {
if (nullValue != null && randomBoolean()) {
return Tuple.tuple(null, nullValue);
}
double d = randomValue();
return Tuple.tuple(d, round(d));
private static class ScaledFloatSyntheticSourceSupport implements SyntheticSourceSupport {
private final double scalingFactor = randomDoubleBetween(0, Double.MAX_VALUE, false);
private final Double nullValue = usually() ? null : round(randomValue());

@Override
public SyntheticSourceExample example(int maxValues) {
if (randomBoolean()) {
Tuple<Double, Double> v = generateValue();
return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
}
List<Tuple<Double, Double>> values = randomList(1, maxValues, this::generateValue);
List<Double> in = values.stream().map(Tuple::v1).toList();
List<Double> outList = values.stream().map(Tuple::v2).sorted().toList();
Object out = outList.size() == 1 ? outList.get(0) : outList;
return new SyntheticSourceExample(in, out, this::mapping);
}

private double round(double d) {
long encoded = Math.round(d * scalingFactor);
double decoded = encoded / scalingFactor;
long reencoded = Math.round(decoded * scalingFactor);
if (encoded != reencoded) {
if (encoded > reencoded) {
return decoded + Math.ulp(decoded);
}
return decoded - Math.ulp(decoded);
}
return decoded;
private Tuple<Double, Double> generateValue() {
if (nullValue != null && randomBoolean()) {
return Tuple.tuple(null, nullValue);
}
double d = randomValue();
return Tuple.tuple(d, round(d));
}

private void mapping(XContentBuilder b) throws IOException {
b.field("type", "scaled_float");
b.field("scaling_factor", scalingFactor);
if (nullValue != null) {
b.field("null_value", nullValue);
}
if (rarely()) {
b.field("index", false);
}
if (rarely()) {
b.field("store", false);
private double round(double d) {
long encoded = Math.round(d * scalingFactor);
double decoded = encoded / scalingFactor;
long reencoded = Math.round(decoded * scalingFactor);
if (encoded != reencoded) {
if (encoded > reencoded) {
return decoded + Math.ulp(decoded);
}
return decoded - Math.ulp(decoded);
}
return decoded;
}

@Override
public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
return List.of(
new SyntheticSourceInvalidExample(
equalTo("field [field] of type [scaled_float] doesn't support synthetic source because it doesn't have doc values"),
b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("doc_values", false)
),
new SyntheticSourceInvalidExample(
equalTo(
"field [field] of type [scaled_float] doesn't support synthetic source because it ignores malformed numbers"
),
b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("ignore_malformed", true)
)
);
private void mapping(XContentBuilder b) throws IOException {
b.field("type", "scaled_float");
b.field("scaling_factor", scalingFactor);
if (nullValue != null) {
b.field("null_value", nullValue);
}
};
if (rarely()) {
b.field("index", false);
}
if (rarely()) {
b.field("store", false);
}
}

@Override
public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
return List.of(
new SyntheticSourceInvalidExample(
equalTo("field [field] of type [scaled_float] doesn't support synthetic source because it doesn't have doc values"),
b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("doc_values", false)
),
new SyntheticSourceInvalidExample(
equalTo("field [field] of type [scaled_float] doesn't support synthetic source because it ignores malformed numbers"),
b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("ignore_malformed", true)
)
);
}
}

@Override
Expand Down Expand Up @@ -504,7 +505,7 @@ private double encodeDecode(double value, double scalingFactor) {
return ScaledFloatFieldMapper.decodeForSyntheticSource(ScaledFloatFieldMapper.encode(value, scalingFactor), scalingFactor);
}

private double randomValue() {
private static double randomValue() {
return randomBoolean() ? randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true) : randomFloat();
}
}
2 changes: 1 addition & 1 deletion .../src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -801,7 +801,7 @@ protected boolean supportsIgnoreMalformed() {
}

@Override
protected SyntheticSourceSupport syntheticSourceSupport() {
protected SyntheticSourceSupport syntheticSourceSupport(boolean syntheticSource) {
throw new AssumptionViolatedException("not supported");
}

Expand Down
2 changes: 1 addition & 1 deletion ...xtras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ protected boolean supportsIgnoreMalformed() {
}

@Override
protected SyntheticSourceSupport syntheticSourceSupport() {
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
throw new AssumptionViolatedException("not supported");
}

Expand Down
2 changes: 1 addition & 1 deletion .../test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ protected boolean supportsIgnoreMalformed() {
}

@Override
protected SyntheticSourceSupport syntheticSourceSupport() {
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
throw new AssumptionViolatedException("not supported");
}

Expand Down
2 changes: 1 addition & 1 deletion ...test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -595,7 +595,7 @@ protected boolean supportsIgnoreMalformed() {
}

@Override
protected SyntheticSourceSupport syntheticSourceSupport() {
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
throw new AssumptionViolatedException("not supported");
}

Expand Down
2 changes: 1 addition & 1 deletion ...murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ protected boolean supportsIgnoreMalformed() {
}

@Override
protected SyntheticSourceSupport syntheticSourceSupport() {
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
throw new AssumptionViolatedException("not supported");
}

Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -466,7 +466,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
);
}
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName()) {
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), false) {
@Override
protected void writeValue(XContentBuilder b, long value) throws IOException {
b.value(value == 1);
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -955,7 +955,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
);
}
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName()) {
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed) {
@Override
protected void writeValue(XContentBuilder b, long value) throws IOException {
b.value(fieldType().format(value, fieldType().dateTimeFormatter()));
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,7 +497,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
);
}
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName()) {
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed()) {
final GeoPoint point = new GeoPoint();

@Override
Expand Down
49 changes: 26 additions & 23 deletions server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -399,8 +399,8 @@ private static void validateParsed(float value) {
}

@Override
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName) {
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
@Override
protected void writeValue(XContentBuilder b, long value) throws IOException {
b.value(HalfFloatPoint.sortableShortToHalfFloat((short) value));
Expand Down Expand Up @@ -549,8 +549,8 @@ private static void validateParsed(float value) {
}

@Override
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName) {
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
@Override
protected void writeValue(XContentBuilder b, long value) throws IOException {
b.value(NumericUtils.sortableIntToFloat((int) value));
Expand Down Expand Up @@ -677,8 +677,8 @@ private static void validateParsed(double value) {
}

@Override
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName) {
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
@Override
protected void writeValue(XContentBuilder b, long value) throws IOException {
b.value(NumericUtils.sortableLongToDouble(value));
Expand Down Expand Up @@ -774,8 +774,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
}

@Override
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName);
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
}
},
SHORT("short", NumericType.SHORT) {
Expand Down Expand Up @@ -862,8 +862,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
}

@Override
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName);
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
}
},
INTEGER("integer", NumericType.INT) {
Expand Down Expand Up @@ -1017,8 +1017,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
}

@Override
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName);
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
}
},
LONG("long", NumericType.LONG) {
Expand Down Expand Up @@ -1142,8 +1142,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
}

@Override
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
return syntheticLongFieldLoader(fieldName, fieldSimpleName);
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
return syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
}
};

Expand Down Expand Up @@ -1382,10 +1382,14 @@ public double reduceToStoredPrecision(double value) {
return ((Number) value).doubleValue();
}

abstract SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName);
abstract SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed);

private static SourceLoader.SyntheticFieldLoader syntheticLongFieldLoader(String fieldName, String fieldSimpleName) {
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName) {
private static SourceLoader.SyntheticFieldLoader syntheticLongFieldLoader(
String fieldName,
String fieldSimpleName,
boolean ignoreMalformed
) {
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
@Override
protected void writeValue(XContentBuilder b, long value) throws IOException {
b.value(value);
Expand Down Expand Up @@ -1670,6 +1674,10 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
} catch (IllegalArgumentException e) {
if (ignoreMalformed.value() && context.parser().currentToken().isValue()) {
context.addIgnoredField(mappedFieldType.name());
if (context.isSyntheticSource()) {
// Save a copy of the field so synthetic source can load it
context.doc().add(IgnoreMalformedStoredValues.storedField(name(), context.parser()));
}
return;
} else {
throw e;
Expand Down Expand Up @@ -1757,17 +1765,12 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it doesn't have doc values"
);
}
if (ignoreMalformed.value()) {
throw new IllegalArgumentException(
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it ignores malformed numbers"
);
}
if (copyTo.copyToFields().isEmpty() != true) {
throw new IllegalArgumentException(
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
);
}
return type.syntheticFieldLoader(name(), simpleName());
return type.syntheticFieldLoader(name(), simpleName(), ignoreMalformed.value());
}

// For testing only:
Expand Down
Loading