From 343310c129ff67a4cb6d2e034ca538b56bf9ae9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 14 Jul 2020 19:40:30 +0200 Subject: [PATCH 01/28] Add field type for version strings This PR adds a new 'version' field type that allows indexing string values representing software versions similar to the ones defined in the Semantic Versioning definition (semver.org). The field behaves very similar to a 'keyword' field but allows efficient sorting and range queries that take into accound the special ordering needed for version strings. For example, the main version parts are sorted numerically (ie 2.0.0 < 11.0.0) whereas this wouldn't be possible with 'keyword' fields today. Valid version values are similar to the Semantic Versioning definition, with the notable exception that in addition to the "main" version consiting of major.minor.patch, we allow less or more than three numeric identifiers, i.e. "1.2" or "1.4.6.123.12" are treated as valid too. Relates to #48878 --- .../index/mapper/TermBasedFieldType.java | 5 +- x-pack/plugin/versionfield/build.gradle | 20 + .../xpack/versionfield/VersionEncoder.java | 232 +++++++++ .../VersionFieldDocValuesExtension.java | 42 ++ .../versionfield/VersionFieldPlugin.java | 28 ++ .../VersionFieldWildcardQuery.java | 114 +++++ .../versionfield/VersionScriptDocValues.java | 57 +++ .../VersionStringFieldMapper.java | 445 +++++++++++++++++ ...asticsearch.painless.spi.PainlessExtension | 1 + .../xpack/versionfield/whitelist.txt | 5 + .../versionfield/VersionEncoderTests.java | 228 +++++++++ .../VersionStringFieldMapperTests.java | 453 ++++++++++++++++++ 12 files changed, 1628 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/versionfield/build.gradle create mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java create mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldDocValuesExtension.java create mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java create mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java create mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java create mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java create mode 100644 x-pack/plugin/versionfield/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt create mode 100644 x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java create mode 100644 x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java index 0101b68fb88c8..91df9b4a9a306 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java @@ -33,9 +33,10 @@ /** Base {@link MappedFieldType} implementation for a field that is indexed * with the inverted index. */ -abstract class TermBasedFieldType extends SimpleMappedFieldType { +public abstract class TermBasedFieldType extends SimpleMappedFieldType { - TermBasedFieldType(String name, boolean isSearchable, boolean hasDocValues, TextSearchInfo textSearchInfo, Map meta) { + public TermBasedFieldType(String name, boolean isSearchable, boolean hasDocValues, TextSearchInfo textSearchInfo, + Map meta) { super(name, isSearchable, hasDocValues, textSearchInfo, meta); } diff --git a/x-pack/plugin/versionfield/build.gradle b/x-pack/plugin/versionfield/build.gradle new file mode 100644 index 0000000000000..b3423de6e0d54 --- /dev/null +++ b/x-pack/plugin/versionfield/build.gradle @@ -0,0 +1,20 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'versionfield' + description 'A plugin for a field type to store sofware versions' + classname 'org.elasticsearch.xpack.versionfield.VersionFieldPlugin' + extendedPlugins = ['x-pack-core', 'lang-painless'] +} +archivesBaseName = 'x-pack-versionfield' + +dependencies { + compileOnly project(path: xpackModule('core'), configuration: 'default') + compileOnly project(':modules:lang-painless:spi') + compileOnly project(':modules:lang-painless') + testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') +} + +integTest.enabled = false diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java new file mode 100644 index 0000000000000..fc0d0b64c1e9b --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java @@ -0,0 +1,232 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; + +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +/** + * Encodes a version string to a {@link BytesRef} that correctly sorts according to software version precedence rules like + * the ones described in Semantiv Versioning (https://semver.org/) + * + * Version strings are considered to consist of three parts: + *
    + *
  • a numeric major.minor.patch part starting the version string (e.g. 1.2.3) + *
  • an optional "pre-release" part that starts with a `-` character and can consist of several alpha-numerical sections + * separated by dots (e.g. "-alpha.2.3") + *
  • an optional "build" part that starts with a `+` character. This will simply be treated as a prefix with no guaranteed ordering, + * (although the ordering should be alphabetical in most cases). + *
+ * + * The version string is encoded such that the ordering works like the following: + *
    + *
  • Major, minor, and patch versions are always compared numerically + *
  • pre-release version have lower precedence than a normal version. (e.g 1.0.0-alpha < 1.0.0) + *
  • the precedence for pre-release versions with same main version is calculated comparing each dot separated identifier from + * left to right. Identifiers consisting of only digits are compared numerically and identifiers with letters or hyphens are compared + * lexically in ASCII sort order. Numeric identifiers always have lower precedence than non-numeric identifiers. + *
+ */ +class VersionEncoder { + + public static final byte NUMERIC_MARKER_BYTE = (byte) 0x01; + public static final byte PRERELESE_SEPARATOR_BYTE = (byte) 0x02; + public static final byte NO_PRERELESE_SEPARATOR_BYTE = (byte) 0x03; + + private static final char PRERELESE_SEPARATOR = '-'; + private static final char DOT_SEPARATOR = '.'; + private static final char BUILD_SEPARATOR = '+'; + + // Regex to test version validity: \d+(\.\d+)*(-[\-\dA-Za-z]+){0,1}(\.[-\dA-Za-z]+)*(\+[\.\-\dA-Za-z]+)? + // private static Pattern LEGAL_VERSION_PATTERN = Pattern.compile( + // "\\d+(\\.\\d+)*(-[\\-\\dA-Za-z]+){0,1}(\\.[\\-\\dA-Za-z]+)*(\\+[\\.\\-\\dA-Za-z]+)?" + // ); + + // Regex to test strict Semver Main Version validity: + // private static Pattern LEGAL_MAIN_VERSION_SEMVER = Pattern.compile("(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)"); + + // Regex to test relaxed Semver Main Version validity. Allows for more or less than three main version parts + private static Pattern LEGAL_MAIN_VERSION_SEMVER = Pattern.compile("(0|[1-9]\\d*)(\\.(0|[1-9]\\d*))*"); + + private static Pattern LEGAL_PRERELEASE_VERSION_SEMVER = Pattern.compile( + "(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))" + ); + + private static Pattern LEGAL_BUILDSUFFIX_SEMVER = Pattern.compile("(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?"); + + /** + * Encodes a version string. + */ + public static EncodedVersion encodeVersion(String versionString) { + // System.out.println("encoding: " + versionString); + VersionParts versionParts = VersionParts.ofVersion(versionString); + + // don't treat non-legal versions further, just mark them as illegal and return + if (legalVersionString(versionParts) == false) { + return new EncodedVersion(new BytesRef(versionString), false, true, 0, 0, 0); + } + + BytesRefBuilder encodedBytes = new BytesRefBuilder(); + Integer[] mainVersionParts = prefixDigitGroupsWithLength(versionParts.mainVersion, encodedBytes); + + if (versionParts.preRelease != null) { + encodedBytes.append(PRERELESE_SEPARATOR_BYTE); // versions with pre-release part sort before ones without + encodedBytes.append((byte) PRERELESE_SEPARATOR); + String[] preReleaseParts = versionParts.preRelease.substring(1).split("\\."); + boolean first = true; + for (String preReleasePart : preReleaseParts) { + if (first == false) { + encodedBytes.append((byte) DOT_SEPARATOR); + } + boolean isNumeric = preReleasePart.chars().allMatch(x -> Character.isDigit(x)); + if (isNumeric) { + prefixDigitGroupsWithLength(preReleasePart, encodedBytes); + } else { + encodedBytes.append(new BytesRef(preReleasePart)); + } + first = false; + } + } else { + encodedBytes.append(NO_PRERELESE_SEPARATOR_BYTE); + } + + if (versionParts.buildSuffix != null) { + encodedBytes.append(new BytesRef(versionParts.buildSuffix)); + } + // System.out.println("encoded: " + encodedBytes.toBytesRef()); + return new EncodedVersion( + encodedBytes.toBytesRef(), + true, + versionParts.preRelease != null, + mainVersionParts[0], + mainVersionParts[1], + mainVersionParts[2] + ); + } + + private static Integer[] prefixDigitGroupsWithLength(String input, BytesRefBuilder result) { + int pos = 0; + int mainVersionCounter = 0; + Integer[] mainVersionComponents = new Integer[3]; + while (pos < input.length()) { + if (Character.isDigit(input.charAt(pos))) { + // found beginning of number block, so get its length + int start = pos; + BytesRefBuilder number = new BytesRefBuilder(); + while (pos < input.length() && Character.isDigit(input.charAt(pos))) { + number.append((byte) input.charAt(pos)); + pos++; + } + int length = pos - start; + if (length >= 128) { + throw new IllegalArgumentException("Groups of digits cannot be longer than 127, but found: " + length); + } + result.append(NUMERIC_MARKER_BYTE); // ensure length byte does cause higher sort order comparing to other byte[] + result.append((byte) (length | 0x80)); // add upper bit to mark as length + result.append(number); + + // if present, parse out three leftmost version parts + if (mainVersionCounter < 3) { + mainVersionComponents[mainVersionCounter] = Integer.valueOf(number.toBytesRef().utf8ToString()); + mainVersionCounter++; + } + } else { + result.append((byte) input.charAt(pos)); + pos++; + } + } + return mainVersionComponents; + } + + public static String decodeVersion(BytesRef version) { + // System.out.println("decoding: " + version); + int inputPos = version.offset; + int resultPos = 0; + byte[] result = new byte[version.length]; + while (inputPos < version.offset + version.length) { + byte inputByte = version.bytes[inputPos]; + if (inputByte == NUMERIC_MARKER_BYTE) { + // need to skip this byte + inputPos++; + // this should always be a length encoding, which is skipped by increasing inputPos at the end of the loop + assert version.bytes[inputPos] < 0; + } else if (inputByte != PRERELESE_SEPARATOR_BYTE && inputByte != NO_PRERELESE_SEPARATOR_BYTE) { + result[resultPos] = inputByte; + resultPos++; + } + inputPos++; + } + // System.out.println("decoded to: " + new String(result, 0, resultPos)); + return new String(result, 0, resultPos, StandardCharsets.UTF_8); + } + + private static boolean legalVersionString(VersionParts versionParts) { + boolean legalMainVersion = LEGAL_MAIN_VERSION_SEMVER.matcher(versionParts.mainVersion).matches(); + boolean legalPreRelease = true; + if (versionParts.preRelease != null) { + legalPreRelease = LEGAL_PRERELEASE_VERSION_SEMVER.matcher(versionParts.preRelease).matches(); + } + boolean legalBuildSuffix = true; + if (versionParts.buildSuffix != null) { + legalBuildSuffix = LEGAL_BUILDSUFFIX_SEMVER.matcher(versionParts.buildSuffix).matches(); + } + return legalMainVersion && legalPreRelease && legalBuildSuffix; + } + + public static class EncodedVersion { + + public final boolean isLegal; + public final boolean isPreRelease; + public final BytesRef bytesRef; + public final Integer major; + public final Integer minor; + public final Integer patch; + + EncodedVersion(BytesRef bytesRef, boolean isLegal, boolean isPreRelease, Integer major, Integer minor, Integer patch) { + super(); + this.bytesRef = bytesRef; + this.isLegal = isLegal; + this.isPreRelease = isPreRelease; + this.major = major; + this.minor = minor; + this.patch = patch; + } + } + + private static class VersionParts { + final String mainVersion; + final String preRelease; + final String buildSuffix; + + private VersionParts(String mainVersion, String preRelease, String buildSuffix) { + this.mainVersion = mainVersion; + this.preRelease = preRelease; + this.buildSuffix = buildSuffix; + } + + private static VersionParts ofVersion(String versionString) { + String buildSuffix = extractSuffix(versionString, BUILD_SEPARATOR); + if (buildSuffix != null) { + versionString = versionString.substring(0, versionString.length() - buildSuffix.length()); + } + + String preRelease = extractSuffix(versionString, PRERELESE_SEPARATOR); + if (preRelease != null) { + versionString = versionString.substring(0, versionString.length() - preRelease.length()); + } + return new VersionParts(versionString, preRelease, buildSuffix); + } + + private static String extractSuffix(String input, char separator) { + int start = input.indexOf(separator); + return start > 0 ? input.substring(start) : null; + } + } +} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldDocValuesExtension.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldDocValuesExtension.java new file mode 100644 index 0000000000000..ac27125fa8610 --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldDocValuesExtension.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.script.FieldScript; +import org.elasticsearch.script.FilterScript; +import org.elasticsearch.script.NumberSortScript; +import org.elasticsearch.script.ScoreScript; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.StringSortScript; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.singletonList; + +public class VersionFieldDocValuesExtension implements PainlessExtension { + + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(VersionFieldDocValuesExtension.class, "whitelist.txt"); + + @Override + public Map, List> getContextWhitelists() { + Map, List> whitelist = new HashMap<>(); + List list = singletonList(WHITELIST); + whitelist.put(AggregationScript.CONTEXT, list); + whitelist.put(ScoreScript.CONTEXT, list); + whitelist.put(FilterScript.CONTEXT, list); + whitelist.put(FieldScript.CONTEXT, list); + whitelist.put(NumberSortScript.CONTEXT, list); + whitelist.put(StringSortScript.CONTEXT, list); + return whitelist; + } +} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java new file mode 100644 index 0000000000000..dd9352f754cc5 --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +public class VersionFieldPlugin extends Plugin implements MapperPlugin { + + public VersionFieldPlugin(Settings settings) {} + + @Override + public Map getMappers() { + Map mappers = new LinkedHashMap<>(); + mappers.put(VersionStringFieldMapper.CONTENT_TYPE, new VersionStringFieldMapper.TypeParser()); + return Collections.unmodifiableMap(mappers); + } +} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java new file mode 100644 index 0000000000000..03c6cf780a07d --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.AutomatonQuery; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Automata; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.Operations; + +import java.util.ArrayList; +import java.util.List; + +/** + * A variation of the {@link WildcardQuery} than skips over meta characters introduced using {@link VersionEncoder}. + */ +class VersionFieldWildcardQuery extends AutomatonQuery { + + private static final byte WILDCARD_STRING = '*'; + + private static final byte WILDCARD_CHAR = '?'; + + VersionFieldWildcardQuery(Term term) { + super(term, toAutomaton(term), Integer.MAX_VALUE, true); + } + + private static Automaton toAutomaton(Term wildcardquery) { + List automata = new ArrayList<>(); + + BytesRef wildcardText = wildcardquery.bytes(); + boolean containsPreReleaseSeparator = false; + + for (int i = 0; i < wildcardText.length;) { + final byte c = wildcardText.bytes[wildcardText.offset + i]; + int length = Character.charCount(c); + + switch (c) { + case WILDCARD_STRING: + automata.add(Automata.makeAnyString()); + break; + case WILDCARD_CHAR: + // this should also match leading digits, which have optional leading numeric marker and length bytes + automata.add(optionalNumericCharPrefix()); + automata.add(Automata.makeAnyChar()); + break; + + case '-': + // this should potentially match the first prerelease-dash, so we need an optional marker byte here + automata.add(Operations.optional(Automata.makeChar(VersionEncoder.PRERELESE_SEPARATOR_BYTE))); + containsPreReleaseSeparator = true; + automata.add(Automata.makeChar(c)); + break; + case '+': + // this can potentially appear after major version, optionally match the no-prerelease marker + automata.add(Operations.optional(Automata.makeChar(VersionEncoder.NO_PRERELESE_SEPARATOR_BYTE))); + containsPreReleaseSeparator = true; + automata.add(Automata.makeChar(c)); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + boolean firstDigitInGroup = true; + if (i > 0 + && wildcardText.bytes[wildcardText.offset + i - 1] >= (byte) '0' + && wildcardText.bytes[wildcardText.offset + i - 1] <= (byte) '9') { + firstDigitInGroup = false; + } + if (firstDigitInGroup) { + automata.add(optionalNumericCharPrefix()); + } + automata.add(Automata.makeChar(c)); + break; + default: + automata.add(Automata.makeChar(c)); + } + i += length; + } + // when we only have main version part, we need to add an optional NO_PRERELESE_SEPARATOR_BYTE + if (containsPreReleaseSeparator == false) { + automata.add(Operations.optional(Automata.makeChar(VersionEncoder.NO_PRERELESE_SEPARATOR_BYTE))); + } + return Operations.concatenate(automata); + } + + private static Automaton optionalNumericCharPrefix() { + return Operations.optional( + Operations.concatenate(Automata.makeChar(VersionEncoder.NUMERIC_MARKER_BYTE), Automata.makeCharRange(0x80, 0xFF)) + ); + } + + @Override + public String toString(String field) { + StringBuilder buffer = new StringBuilder(); + if (!getField().equals(field)) { + buffer.append(getField()); + buffer.append(":"); + } + buffer.append(term.text()); + return buffer.toString(); + } +} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java new file mode 100644 index 0000000000000..f74027db634cc --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.index.fielddata.ScriptDocValues; + +import java.io.IOException; + +public final class VersionScriptDocValues extends ScriptDocValues { + + private final SortedSetDocValues in; + private long[] ords = new long[0]; + private int count; + + public VersionScriptDocValues(SortedSetDocValues in) { + this.in = in; + } + + @Override + public void setNextDocId(int docId) throws IOException { + count = 0; + if (in.advanceExact(docId)) { + for (long ord = in.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = in.nextOrd()) { + ords = ArrayUtil.grow(ords, count + 1); + ords[count++] = ord; + } + } + } + + public String getValue() { + if (count == 0) { + return null; + } else { + return get(0); + } + } + + @Override + public String get(int index) { + try { + return VersionEncoder.decodeVersion(in.lookupOrd(ords[index])); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public int size() { + return count; + } +} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java new file mode 100644 index 0000000000000..6a3f736e52b72 --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -0,0 +1,445 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.apache.lucene.document.BinaryPoint; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.TermBasedFieldType; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.TypeParsers; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.support.QueryParsers; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.xpack.versionfield.VersionEncoder.EncodedVersion; + +import java.io.IOException; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; +import static org.elasticsearch.xpack.versionfield.VersionEncoder.encodeVersion; + +/** + * A {@link FieldMapper} for indexing fields with version strings. + */ +public class VersionStringFieldMapper extends FieldMapper { + + private static byte[] MIN_VALUE = new byte[16]; + private static byte[] MAX_VALUE = new byte[16]; + static { + Arrays.fill(MIN_VALUE, (byte) 0); + Arrays.fill(MAX_VALUE, (byte) -1); + } + + public static final String CONTENT_TYPE = "version"; + + public static class Defaults { + public static final FieldType FIELD_TYPE = new FieldType(); + + static { + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.freeze(); + } + + public static final String NULL_VALUE = null; + } + + static class Builder extends FieldMapper.Builder { + + protected String nullValue = Defaults.NULL_VALUE; + private boolean storeMalformed = false; + + Builder(String name) { + super(name, Defaults.FIELD_TYPE); + builder = this; + } + + Builder nullValue(String nullValue) { + this.nullValue = nullValue; + return builder; + } + + Builder storeMalformed(boolean storeMalformed) { + this.storeMalformed = storeMalformed; + return builder; + } + + private VersionStringFieldType buildFieldType(BuilderContext context) { + boolean validateVersion = storeMalformed == false; + return new VersionStringFieldType(buildFullName(context), indexed, validateVersion, meta, boost, fieldType); + } + + @Override + public VersionStringFieldMapper build(BuilderContext context) { + BooleanFieldMapper.Builder preReleaseSubfield = new BooleanFieldMapper.Builder(name + ".isPreRelease"); + NumberType type = NumberType.INTEGER; + NumberFieldMapper.Builder majorVersionSubField = new NumberFieldMapper.Builder(name + ".major", type).nullValue(0); + NumberFieldMapper.Builder minorVersionSubField = new NumberFieldMapper.Builder(name + ".minor", type).nullValue(0); + NumberFieldMapper.Builder patchVersionSubField = new NumberFieldMapper.Builder(name + ".patch", type).nullValue(0); + + return new VersionStringFieldMapper( + name, + fieldType, + buildFieldType(context), + storeMalformed, + nullValue, + multiFieldsBuilder.build(this, context), + copyTo, + preReleaseSubfield.build(context), + majorVersionSubField.build(context), + minorVersionSubField.build(context), + patchVersionSubField.build(context) + ); + } + } + + public static class TypeParser implements Mapper.TypeParser { + + public TypeParser() {} + + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + Builder builder = new Builder(name); + TypeParsers.parseField(builder, name, node, parserContext); + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String propName = entry.getKey(); + Object propNode = entry.getValue(); + if (propName.equals("null_value")) { + if (propNode == null) { + throw new MapperParsingException("Property [null_value] cannot be null."); + } + builder.nullValue(propNode.toString()); + iterator.remove(); + } else if (propName.equals("store_malformed")) { + builder.storeMalformed(XContentMapValues.nodeBooleanValue(propNode, name + ".store_malformed")); + iterator.remove(); + } else if (TypeParsers.parseMultiField(builder::addMultiField, name, parserContext, propName, propNode)) { + iterator.remove(); + } + } + return builder; + } + } + + public static final class VersionStringFieldType extends TermBasedFieldType { + + // if true, we want to throw errors on illegal versions at index and query time + private boolean validateVersion = false; + + public VersionStringFieldType( + String name, + boolean isSearchable, + boolean validateVersion, + Map meta, + float boost, + FieldType fieldType + ) { + super(name, isSearchable, true, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta); + setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + setBoost(boost); + this.validateVersion = validateVersion; + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public Query existsQuery(QueryShardContext context) { + return new DocValuesFieldExistsQuery(name()); + } + + @Override + public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { + if (context.allowExpensiveQueries() == false) { + throw new ElasticsearchException( + "[prefix] queries cannot be executed when '" + + ALLOW_EXPENSIVE_QUERIES.getKey() + + "' is set to false. For optimised prefix queries on text " + + "fields please enable [index_prefixes]." + ); + } + failIfNotIndexed(); + return wildcardQuery(value + "*", method, context); + } + + @Override + public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { + if (context.allowExpensiveQueries() == false) { + throw new ElasticsearchException( + "[wildcard] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false." + ); + } + failIfNotIndexed(); + + VersionFieldWildcardQuery query = new VersionFieldWildcardQuery(new Term(name(), value)); + QueryParsers.setRewriteMethod(query, method); + return query; + } + + @Override + protected BytesRef indexedValueForSearch(Object value) { + String valueAsString; + if (value instanceof String) { + valueAsString = (String) value; + } else if (value instanceof BytesRef) { + // encoded string, need to re-encode + valueAsString = ((BytesRef) value).utf8ToString(); + } else { + throw new IllegalArgumentException("Illegal value type: " + value.getClass() + ", value: " + value); + } + EncodedVersion encodedVersion = encodeVersion(valueAsString); + if (encodedVersion.isLegal == false && validateVersion) { + throw new IllegalArgumentException("Illegal version string: " + valueAsString); + } + return encodedVersion.bytesRef; + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + failIfNoDocValues(); + return new SortedSetOrdinalsIndexFieldData.Builder(VersionScriptDocValues::new, CoreValuesSourceType.BYTES); + } + + @Override + public Object valueForDisplay(Object value) { + if (value == null) { + return null; + } + return VERSION_DOCVALUE.format((BytesRef) value); + } + + @Override + public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support custom formats"); + } + if (timeZone != null) { + throw new IllegalArgumentException( + "Field [" + name() + "] of type [" + typeName() + "] does not support custom time zones" + ); + } + return VERSION_DOCVALUE; + } + + @Override + public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { + if (context.allowExpensiveQueries() == false) { + throw new ElasticsearchException( + "[range] queries on [version] fields cannot be executed when '" + + ALLOW_EXPENSIVE_QUERIES.getKey() + + "' is set to false." + ); + } + failIfNotIndexed(); + BytesRef lower = lowerTerm == null ? null : indexedValueForSearch(lowerTerm); + BytesRef upper = upperTerm == null ? null : indexedValueForSearch(upperTerm); + byte[] lowerBytes = lower == null ? MIN_VALUE : Arrays.copyOfRange(lower.bytes, lower.offset, 16); + byte[] upperBytes = upper == null ? MAX_VALUE : Arrays.copyOfRange(upper.bytes, upper.offset, 16); + + // point query on the 16byte prefix + Query pointPrefixQuery = BinaryPoint.newRangeQuery(name(), lowerBytes, upperBytes); + + Query termQuery = new TermRangeQuery( + name(), + lowerTerm == null ? null : lower, + upperTerm == null ? null : upper, + includeLower, + includeUpper + ); + + return new BooleanQuery.Builder().add(new BooleanClause(pointPrefixQuery, Occur.MUST)) + .add(new BooleanClause(termQuery, Occur.MUST)) + .build(); + } + } + + private boolean storeMalformed; + private String nullValue; + private BooleanFieldMapper prereleaseSubField; + private NumberFieldMapper majorVersionSubField; + private NumberFieldMapper minorVersionSubField; + private NumberFieldMapper patchVersionSubField; + + private VersionStringFieldMapper( + String simpleName, + FieldType fieldType, + MappedFieldType mappedFieldType, + boolean storeMalformed, + String nullValue, + MultiFields multiFields, + CopyTo copyTo, + BooleanFieldMapper preReleaseMapper, + NumberFieldMapper majorVersionMapper, + NumberFieldMapper minorVersionMapper, + NumberFieldMapper patchVersionMapper + ) { + super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); + this.storeMalformed = storeMalformed; + this.nullValue = nullValue; + this.prereleaseSubField = preReleaseMapper; + this.majorVersionSubField = majorVersionMapper; + this.minorVersionSubField = minorVersionMapper; + this.patchVersionSubField = patchVersionMapper; + } + + @Override + public VersionStringFieldType fieldType() { + return (VersionStringFieldType) super.fieldType(); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + protected VersionStringFieldMapper clone() { + return (VersionStringFieldMapper) super.clone(); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + String versionString; + if (context.externalValueSet()) { + versionString = context.externalValue().toString(); + } else { + XContentParser parser = context.parser(); + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { + versionString = nullValue; + } else { + versionString = parser.textOrNull(); + } + } + + if (versionString == null) { + return; + } + + EncodedVersion encoding = encodeVersion(versionString); + if (encoding.isLegal == false && storeMalformed == false) { + throw new IllegalArgumentException("Illegal version string: " + versionString); + } + BytesRef encodedVersion = encoding.bytesRef; + if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { + Field field = new Field(fieldType().name(), encodedVersion, fieldType); + context.doc().add(field); + // encode the first 16 bytes as points for efficient range query + byte[] first16bytes = Arrays.copyOfRange(encodedVersion.bytes, encodedVersion.offset, 16); + context.doc().add(new BinaryPoint(fieldType().name(), first16bytes)); + } + context.doc().add(new SortedSetDocValuesField(fieldType().name(), encodedVersion)); + + // add additional information extracted from version string + context.doc().add(new Field(prereleaseSubField.name(), encoding.isPreRelease ? "T" : "F", BooleanFieldMapper.Defaults.FIELD_TYPE)); + context.doc().add(new SortedNumericDocValuesField(prereleaseSubField.name(), encoding.isPreRelease ? 1 : 0)); + + addVersionPartSubfield(context, majorVersionSubField.name(), encoding.major); + addVersionPartSubfield(context, minorVersionSubField.name(), encoding.minor); + addVersionPartSubfield(context, patchVersionSubField.name(), encoding.patch); + } + + private void addVersionPartSubfield(ParseContext context, String fieldName, Integer versionPart) { + if (versionPart != null) { + context.doc().addAll(NumberType.INTEGER.createFields(fieldName, versionPart, true, true, false)); + } + } + + @Override + protected void mergeOptions(FieldMapper other, List conflicts) { + VersionStringFieldMapper mergeWith = (VersionStringFieldMapper) other; + this.storeMalformed = mergeWith.storeMalformed; + this.nullValue = mergeWith.nullValue; + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + if (nullValue != null) { + builder.field("null_value", nullValue); + } + builder.field("store_malformed", storeMalformed); + } + + @Override + public Iterator iterator() { + List subIterators = new ArrayList<>(); + subIterators.add(prereleaseSubField); + subIterators.add(majorVersionSubField); + subIterators.add(minorVersionSubField); + subIterators.add(patchVersionSubField); + @SuppressWarnings("unchecked") + Iterator concat = Iterators.concat(super.iterator(), subIterators.iterator()); + return concat; + } + + private static DocValueFormat VERSION_DOCVALUE = new DocValueFormat() { + + @Override + public String getWriteableName() { + return "version_semver"; + } + + @Override + public void writeTo(StreamOutput out) {} + + @Override + public String format(BytesRef value) { + return VersionEncoder.decodeVersion(value); + } + + @Override + public BytesRef parseBytesRef(String value) { + return VersionEncoder.encodeVersion(value).bytesRef; + } + + @Override + public String toString() { + return getWriteableName(); + } + }; +} diff --git a/x-pack/plugin/versionfield/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/versionfield/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..41ff870b75aa2 --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.versionfield.VersionFieldDocValuesExtension diff --git a/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt b/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt new file mode 100644 index 0000000000000..eb47eae53a158 --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt @@ -0,0 +1,5 @@ + +class org.elasticsearch.xpack.versionfield.VersionScriptDocValues { + String get(int) + String getValue() +} \ No newline at end of file diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java new file mode 100644 index 0000000000000..825146e315918 --- /dev/null +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.versionfield.VersionEncoder.EncodedVersion; + +import java.util.Arrays; + +import static org.elasticsearch.xpack.versionfield.VersionEncoder.decodeVersion; + +public class VersionEncoderTests extends ESTestCase { + + public void testEncodingOrderingSemver() { + assertTrue(encodeVersion("1").compareTo(encodeVersion("1.0")) < 0); + assertTrue(encodeVersion("1.0").compareTo(encodeVersion("1.0.0.0.0.0.0.0.0.1")) < 0); + assertTrue(encodeVersion("1.0.0").compareTo(encodeVersion("1.0.0.0.0.0.0.0.0.1")) < 0); + assertTrue(encodeVersion("1.0.0").compareTo(encodeVersion("2.0.0")) < 0); + assertTrue(encodeVersion("2.0.0").compareTo(encodeVersion("11.0.0")) < 0); + assertTrue(encodeVersion("2.0.0").compareTo(encodeVersion("2.1.0")) < 0); + assertTrue(encodeVersion("2.1.0").compareTo(encodeVersion("2.1.1")) < 0); + assertTrue(encodeVersion("2.1.1").compareTo(encodeVersion("2.1.1.0")) < 0); + assertTrue(encodeVersion("2.0.0").compareTo(encodeVersion("11.0.0")) < 0); + assertTrue(encodeVersion("1.0.0").compareTo(encodeVersion("2.0")) < 0); + assertTrue(encodeVersion("1.0.0-a").compareTo(encodeVersion("1.0.0-b")) < 0); + assertTrue(encodeVersion("1.0.0-1.0.0").compareTo(encodeVersion("1.0.0-2.0")) < 0); + assertTrue(encodeVersion("1.0.0-alpha").compareTo(encodeVersion("1.0.0-alpha.1")) < 0); + assertTrue(encodeVersion("1.0.0-alpha.1").compareTo(encodeVersion("1.0.0-alpha.beta")) < 0); + assertTrue(encodeVersion("1.0.0-alpha.beta").compareTo(encodeVersion("1.0.0-beta")) < 0); + assertTrue(encodeVersion("1.0.0-beta").compareTo(encodeVersion("1.0.0-beta.2")) < 0); + assertTrue(encodeVersion("1.0.0-beta.2").compareTo(encodeVersion("1.0.0-beta.11")) < 0); + assertTrue(encodeVersion("1.0.0-beta11").compareTo(encodeVersion("1.0.0-beta2")) < 0); // correct according to Semver specs + assertTrue(encodeVersion("1.0.0-beta.11").compareTo(encodeVersion("1.0.0-rc.1")) < 0); + assertTrue(encodeVersion("1.0.0-rc.1").compareTo(encodeVersion("1.0.0")) < 0); + assertTrue(encodeVersion("1.0.0").compareTo(encodeVersion("2.0.0-pre127")) < 0); + assertTrue(encodeVersion("2.0.0-pre127").compareTo(encodeVersion("2.0.0-pre128")) < 0); + assertTrue(encodeVersion("2.0.0-pre128").compareTo(encodeVersion("2.0.0-pre128-somethingelse")) < 0); + assertTrue(encodeVersion("2.0.0-pre20201231z110026").compareTo(encodeVersion("2.0.0-pre227")) < 0); + } + + private static BytesRef encodeVersion(String version) { + return VersionEncoder.encodeVersion(version).bytesRef; + } + + public void testPreReleaseFlag() { + assertTrue(VersionEncoder.encodeVersion("1.2-alpha.beta").isPreRelease); + assertTrue(VersionEncoder.encodeVersion("1.2.3-someOtherPreRelease").isPreRelease); + assertTrue(VersionEncoder.encodeVersion("1.2.3-some-Other-Pre.123").isPreRelease); + assertTrue(VersionEncoder.encodeVersion("1.2.3-some-Other-Pre.123+withBuild").isPreRelease); + + assertFalse(VersionEncoder.encodeVersion("1").isPreRelease); + assertFalse(VersionEncoder.encodeVersion("1.2").isPreRelease); + assertFalse(VersionEncoder.encodeVersion("1.2.3").isPreRelease); + assertFalse(VersionEncoder.encodeVersion("1.2.3+buildSufix").isPreRelease); + assertFalse(VersionEncoder.encodeVersion("1.2.3+buildSufix-withDash").isPreRelease); + } + + public void testVersionPartExtraction() { + int numParts = randomIntBetween(1, 6); + String[] parts = new String[numParts]; + for (int i = 0; i < numParts; i++) { + parts[i] = String.valueOf(randomIntBetween(1, 1000)); + } + EncodedVersion encodedVersion = VersionEncoder.encodeVersion(String.join(".", parts)); + assertEquals(parts[0], encodedVersion.major.toString()); + if (numParts > 1) { + assertEquals(parts[1], encodedVersion.minor.toString()); + } else { + assertNull(encodedVersion.minor); + } + if (numParts > 2) { + assertEquals(parts[2], encodedVersion.patch.toString()); + } else { + assertNull(encodedVersion.patch); + } + } + + public void testMaxDigitGroupLength() { + String versionString = "1.0." + "1".repeat(128); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> decodeVersion(encodeVersion(versionString))); + assertEquals("Groups of digits cannot be longer than 127, but found: 128", ex.getMessage()); + } + + /** + * test that encoding and decoding leads back to the same version string + */ + public void testRandomRoundtrip() { + String versionString = randomVersionString(); + assertEquals(versionString, decodeVersion(encodeVersion(versionString))); + } + + private String randomVersionString() { + StringBuilder sb = new StringBuilder(); + sb.append(randomIntBetween(0, 1000)); + int releaseNumerals = randomIntBetween(0, 4); + for (int i = 0; i < releaseNumerals; i++) { + sb.append("."); + sb.append(randomIntBetween(0, 10000)); + } + // optional pre-release part + if (randomBoolean()) { + sb.append("-"); + int preReleaseParts = randomIntBetween(1, 5); + for (int i = 0; i < preReleaseParts; i++) { + if (randomBoolean()) { + sb.append(randomIntBetween(0, 1000)); + } else { + int alphanumParts = 3; + for (int j = 0; j < alphanumParts; j++) { + if (randomBoolean()) { + sb.append(randomAlphaOfLengthBetween(1, 2)); + } else { + sb.append(randomIntBetween(1, 99)); + } + if (rarely()) { + sb.append(randomFrom(Arrays.asList("-"))); + } + } + } + sb.append("."); + } + sb.deleteCharAt(sb.length() - 1); // remove trailing dot + } + // optional build part + if (randomBoolean()) { + sb.append("+").append(randomAlphaOfLengthBetween(1, 15)); + } + return sb.toString(); + } + + /** + * taken from https://regex101.com/r/vkijKf/1/ via https://semver.org/ + */ + public void testSemVerValidation() { + String[] validSemverVersions = new String[] { + "0.0.4", + "1.2.3", + "10.20.30", + "1.1.2-prerelease+meta", + "1.1.2+meta", + "1.1.2+meta-valid", + "1.0.0-alpha", + "1.0.0-beta", + "1.0.0-alpha.beta", + "1.0.0-alpha.beta.1", + "1.0.0-alpha.1", + "1.0.0-alpha0.valid", + "1.0.0-alpha.0valid", + "1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay", + "1.0.0-rc.1+build.1", + "2.0.0-rc.1+build.123", + "1.2.3-beta", + "10.2.3-DEV-SNAPSHOT", + "1.2.3-SNAPSHOT-123", + "1.0.0", + "2.0.0", + "1.1.7", + "2.0.0+build.1848", + "2.0.1-alpha.1227", + "1.0.0-alpha+beta", + "1.2.3----RC-SNAPSHOT.12.9.1--.12+788", + "1.2.3----R-S.12.9.1--.12+meta", + "1.2.3----RC-SNAPSHOT.12.9.1--.12", + "1.0.0+0.build.1-rc.10000aaa-kk-0.1", + + "999999999.999999999.999999999", + "1.0.0-0A.is.legal", + // the following are not strict semver but we allow them + "1.2-SNAPSHOT", + "1.2-RC-SNAPSHOT", + "1", + "1.2.3.4" }; + for (String version : validSemverVersions) { + assertTrue("should be valid: " + version, VersionEncoder.encodeVersion(version).isLegal); + // since we're here, also check encoding / decoding rountrip + assertEquals(version, decodeVersion(encodeVersion(version))); + } + + String[] invalidSemverVersions = new String[] { + "1.2.3-0123", + "1.2.3-0123.0123", + "1.1.2+.123", + "+invalid", + "-invalid", + "-invalid+invalid", + "-invalid.01", + "alpha", + "alpha.beta", + "alpha.beta.1", + "alpha.1", + "alpha+beta", + "alpha_beta", + "alpha.", + "alpha..", + "beta", + "1.0.0-alpha_beta", + "-alpha.", + "1.0.0-alpha..", + "1.0.0-alpha..1", + "1.0.0-alpha...1", + "1.0.0-alpha....1", + "1.0.0-alpha.....1", + "1.0.0-alpha......1", + "1.0.0-alpha.......1", + "01.1.1", + "1.01.1", + "1.1.01", + "1.2.3.DEV", + "1.2.31.2.3----RC-SNAPSHOT.12.09.1--..12+788", + "-1.0.3-gamma+b7718", + "+justmeta", + "9.8.7+meta+meta", + "9.8.7-whatever+meta+meta", + "999999999.999999999.999999999.----RC-SNAPSHOT.12.09.1--------------------------------..12", + "12.el2", + "12.el2-1.0-rc5" }; + for (String version : invalidSemverVersions) { + assertFalse("should be invalid: " + version, VersionEncoder.encodeVersion(version).isLegal); + // since we're here, also check encoding / decoding rountrip + assertEquals(version, decodeVersion(encodeVersion(version))); + } + } +} diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java new file mode 100644 index 0000000000000..21536cdbbba8d --- /dev/null +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -0,0 +1,453 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +public class VersionStringFieldMapperTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + // return pluginList(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class, PainlessPlugin.class); + // TODO PainlessPlugin loading doesn't work when test is run through "gradle check" + return pluginList(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class); + } + + public String setUpIndex(String indexName) throws IOException { + createIndex( + indexName, + Settings.builder().put("index.number_of_shards", 1).build(), + "_doc", + "version", + "type=version", + "foo", + "type=keyword" + ); + ensureGreen(indexName); + + client().prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "11.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "1.0.0").endObject()).get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("4") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) + .get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("6").setSource(jsonBuilder().startObject().field("version", "21.11.0").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + return indexName; + } + + public void testExactQueries() throws Exception { + String indexName = "test"; + setUpIndex(indexName); + + // match + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", ("1.0.0"))).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.4.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.3.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.3.0+build.1234567")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + // term + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.0.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.4.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.3.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.3.0+build.1234567")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + // terms + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termsQuery("version", "1.0.0", "1.3.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termsQuery("version", "1.4.0", "1.3.0+build.1234567")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + // phrase query (just for keyword compatibility) + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchPhraseQuery("version", "2.1.0-alpha.beta")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + } + + public void testRangeQueries() throws Exception { + String indexName = setUpIndex("test"); + SearchResponse response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.rangeQuery("version").from("1.0.0").to("3.0.0")) + .get(); + assertEquals(4, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.1.0").to("3.0.0")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.rangeQuery("version").from("0.1.0").to("2.1.0-alpha.beta")) + .get(); + assertEquals(3, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("2.1.0").to("3.0.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("3.0.0").to("4.0.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + + // ranges excluding edges + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.0.0", false).to("3.0.0")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.0.0").to("2.1.0", false)).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + // open ranges + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.4.0")).get(); + assertEquals(4, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("1.4.0")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + } + + public void testPrefixQuery() throws IOException { + String indexName = setUpIndex("test"); + // prefix + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "1")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2.1")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2.1.0-")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "1.3.0+b")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.1")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.11")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + } + + public void testSort() throws IOException { + String indexName = setUpIndex("test"); + + // sort based on version field + SearchResponse response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.matchAllQuery()) + .addSort("version", SortOrder.DESC) + .get(); + assertEquals(6, response.getHits().getTotalHits().value); + SearchHit[] hits = response.getHits().getHits(); + assertEquals("21.11.0", hits[0].getSortValues()[0]); + assertEquals("11.1.0", hits[1].getSortValues()[0]); + assertEquals("2.1.0", hits[2].getSortValues()[0]); + assertEquals("2.1.0-alpha.beta", hits[3].getSortValues()[0]); + assertEquals("1.3.0+build.1234567", hits[4].getSortValues()[0]); + assertEquals("1.0.0", hits[5].getSortValues()[0]); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC).get(); + assertEquals(6, response.getHits().getTotalHits().value); + hits = response.getHits().getHits(); + assertEquals("1.0.0", hits[0].getSortValues()[0]); + assertEquals("1.3.0+build.1234567", hits[1].getSortValues()[0]); + assertEquals("2.1.0-alpha.beta", hits[2].getSortValues()[0]); + assertEquals("2.1.0", hits[3].getSortValues()[0]); + assertEquals("11.1.0", hits[4].getSortValues()[0]); + assertEquals("21.11.0", hits[5].getSortValues()[0]); + } + + public void testWildcardQuery() throws Exception { + String indexName = "test_wildcard"; + createIndex( + indexName, + Settings.builder().put("index.number_of_shards", 1).build(), + "_doc", + "version", + "type=version", + "foo", + "type=keyword" + ); + ensureGreen(indexName); + + client().prepareIndex(indexName) + .setId("1") + .setSource(jsonBuilder().startObject().field("version", "1.0.0-alpha.2.1.0-rc.1").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("2") + .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) + .get(); + client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + + // wildcard + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*alpha*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*b*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*bet*")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "2.1*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "2.1.0-*")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.1.0-*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.1.0*")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.?.0*")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.??.0*")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "?.1.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*-*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "1.3.0+b*")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + } + + public void testPreReleaseFlag() throws IOException { + String indexName = setUpIndex("test"); + SearchResponse response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.isPreRelease", true))) + .get(); + assertEquals(1, response.getHits().getTotalHits().value); + } + + public void testMainVersionParts() throws IOException { + String indexName = setUpIndex("test"); + SearchResponse response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.major", 11))) + .get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.minor", 1))) + .get(); + assertEquals(3, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.patch", 0))) + .get(); + assertEquals(6, response.getHits().getTotalHits().value); + } + + public void testStoreMalformed() throws Exception { + String indexName = "test_malformed"; + createIndex( + indexName, + Settings.builder().put("index.number_of_shards", 1).build(), + "_doc", + "version", + "type=version,store_malformed=true" + ); + ensureGreen(indexName); + + client().prepareIndex(indexName) + .setId("1") + .setSource(jsonBuilder().startObject().field("version", "1.invalid.0").endObject()) + .get(); + client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "2.2.0").endObject()).get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.2.0-badchar!").endObject()) + .get(); + client().admin().indices().prepareRefresh(indexName).get(); + + SearchResponse response = client().prepareSearch(indexName).addDocValueField("version").get(); + assertEquals(3, response.getHits().getTotalHits().value); + assertEquals("1", response.getHits().getAt(0).getId()); + assertEquals("1.invalid.0", response.getHits().getAt(0).field("version").getValue()); + + assertEquals("2", response.getHits().getAt(1).getId()); + assertEquals("2.2.0", response.getHits().getAt(1).field("version").getValue()); + + assertEquals("3", response.getHits().getAt(2).getId()); + assertEquals("2.2.0-badchar!", response.getHits().getAt(2).field("version").getValue()); + + // exact match for malformed term + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.invalid.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "2.2.0-badchar!")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + // also should appear in terms aggs + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version")).get(); + Terms terms = response.getAggregations().get("myterms"); + List buckets = terms.getBuckets(); + + assertEquals(3, buckets.size()); + assertEquals("2.2.0", buckets.get(0).getKey()); + assertEquals("1.invalid.0", buckets.get(1).getKey()); + assertEquals("2.2.0-badchar!", buckets.get(2).getKey()); + + // invalid values should sort after all valid ones + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC).get(); + assertEquals(3, response.getHits().getTotalHits().value); + SearchHit[] hits = response.getHits().getHits(); + assertEquals("2.2.0", hits[0].getSortValues()[0]); + assertEquals("1.invalid.0", hits[1].getSortValues()[0]); + assertEquals("2.2.0-badchar!", hits[2].getSortValues()[0]); + + // ranges can include them, but they are sorted last + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("3.0.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("3.0.0")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + } + + public void testAggs() throws Exception { + String indexName = "test_aggs"; + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); + ensureGreen(indexName); + + client().prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "1.0").endObject()).get(); + client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "1.3.0").endObject()).get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha").endObject()) + .get(); + client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "3.11.5").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + + // terms aggs + SearchResponse response = client().prepareSearch(indexName) + .addAggregation(AggregationBuilders.terms("myterms").field("version")) + .get(); + Terms terms = response.getAggregations().get("myterms"); + List buckets = terms.getBuckets(); + + assertEquals(5, buckets.size()); + assertEquals("1.0", buckets.get(0).getKey()); + assertEquals("1.3.0", buckets.get(1).getKey()); + assertEquals("2.1.0-alpha", buckets.get(2).getKey()); + assertEquals("2.1.0", buckets.get(3).getKey()); + assertEquals("3.11.5", buckets.get(4).getKey()); + + // test terms aggs on version parts + + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.major")).get(); + terms = response.getAggregations().get("myterms"); + buckets = terms.getBuckets(); + assertEquals(3, buckets.size()); + assertEquals(1L, buckets.get(0).getKey()); + assertEquals(2L, buckets.get(0).getDocCount()); + assertEquals(2L, buckets.get(1).getKey()); + assertEquals(2L, buckets.get(1).getDocCount()); + assertEquals(3L, buckets.get(2).getKey()); + assertEquals(1L, buckets.get(2).getDocCount()); + + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.minor")).get(); + terms = response.getAggregations().get("myterms"); + buckets = terms.getBuckets(); + assertEquals(4, buckets.size()); + assertEquals(1L, buckets.get(0).getKey()); + assertEquals(2L, buckets.get(0).getDocCount()); + assertEquals(0L, buckets.get(1).getKey()); + assertEquals(1L, buckets.get(1).getDocCount()); + assertEquals(3L, buckets.get(2).getKey()); + assertEquals(1L, buckets.get(2).getDocCount()); + assertEquals(11L, buckets.get(3).getKey()); + assertEquals(1L, buckets.get(3).getDocCount()); + + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.patch")).get(); + terms = response.getAggregations().get("myterms"); + buckets = terms.getBuckets(); + assertEquals(2, buckets.size()); + assertEquals(0L, buckets.get(0).getKey()); + assertEquals(3L, buckets.get(0).getDocCount()); + assertEquals(5L, buckets.get(1).getKey()); + assertEquals(1L, buckets.get(1).getDocCount()); + } + + public void testNullValue() throws Exception { + String indexName = "test_nullvalue"; + createIndex( + indexName, + Settings.builder().put("index.number_of_shards", 1).build(), + "_doc", + "version", + "type=version,null_value=0.0.0" + ); + ensureGreen(indexName); + + client().prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "2.2.0").endObject()).get(); + client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().nullField("version").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + + SearchResponse response = client().prepareSearch(indexName).addDocValueField("version").addSort("version", SortOrder.ASC).get(); + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("2", response.getHits().getAt(0).getId()); + assertEquals("0.0.0", response.getHits().getAt(0).field("version").getValue()); + + assertEquals("1", response.getHits().getAt(1).getId()); + assertEquals("2.2.0", response.getHits().getAt(1).field("version").getValue()); + + // range + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("3.0.0")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + } + + // TODO how to test scripts in ESSingleNodeTestCase? + // Currently fails with "java.lang.IllegalArgumentException: Illegal list shortcut value [value]". + + // public void testScripting() throws Exception { + // String indexName = setUpIndex("test"); + // + // ScriptQueryBuilder query = QueryBuilders.scriptQuery(new Script("doc['version'].value.length() <= 5")); + // SearchResponse response = client().prepareSearch(indexName).addDocValueField("version").setQuery(query).get(); + // assertEquals(2, response.getHits().getTotalHits().value); + // assertEquals("2", response.getHits().getAt(0).getId()); + // assertEquals("1.0.0", response.getHits().getAt(0).field("version").getValue()); + // + // assertEquals("5", response.getHits().getAt(1).getId()); + // assertEquals("2.1.0", response.getHits().getAt(1).field("version").getValue()); + // } +} From 397a7c036827cbae0a7460da250d2df5e177d769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 20 Jul 2020 19:26:59 +0200 Subject: [PATCH 02/28] Add yaml for search, ranges and scripting --- .../test/versionfield/10_basic.yml | 103 ++++++++++++++++++ .../test/versionfield/20_scripts.yml | 49 +++++++++ 2 files changed, 152 insertions(+) create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml new file mode 100644 index 0000000000000..38c3209bed046 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml @@ -0,0 +1,103 @@ +# Integration tests for the version field +# +--- +setup: + + - do: + indices.create: + index: test_index + body: + mappings: + properties: + version: + type: version + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "test_index", "_id" : "1" } }' + - '{"version": "1.1.0" }' + - '{ "index" : { "_index" : "test_index", "_id" : "2" } }' + - '{"version": "2.0.0-beta" }' + - '{ "index" : { "_index" : "test_index", "_id" : "3" } }' + - '{"version": "3.1.0" }' + +--- +"Store malformed": + - do: + indices.create: + index: test_malformed + body: + mappings: + properties: + version: + type: version + store_malformed: true + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "test_malformed", "_id" : "1" } }' + - '{"version": "1.1.0" }' + - '{ "index" : { "_index" : "test_malformed", "_id" : "2" } }' + - '{"version": "2.0.0-beta" }' + - '{ "index" : { "_index" : "test_malformed", "_id" : "3" } }' + - '{"version": "3.1.0" }' + - '{ "index" : { "_index" : "test_malformed", "_id" : "4" } }' + - '{"version": "1.el6" }' + + - do: + search: + index: test_malformed + body: + query: { "match" : { "version" : "1.el6" } } + + - do: + search: + index: test_malformed + body: + query: { "match_all" : { } } + sort: + version: asc + + - match: { hits.total.value: 4 } + - match: { hits.hits.0._source.version: "1.1.0" } + - match: { hits.hits.1._source.version: "2.0.0-beta" } + - match: { hits.hits.2._source.version: "3.1.0" } + - match: { hits.hits.3._source.version: "1.el6" } + +--- +"Basic ranges": + - do: + search: + index: test_index + body: + query: { "range" : { "version" : { "gt" : "1.1.0", "lt" : "9999" } } } + + - match: { hits.total.value: 2 } + + - do: + search: + index: test_index + body: + query: { "range" : { "version" : { "gte" : "1.1.0", "lt" : "9999" } } } + + - match: { hits.total.value: 3 } + + - do: + search: + index: test_index + body: + query: { "range" : { "version" : { "gte" : "2.0.0", "lt" : "9999" } } } + + - match: { hits.total.value: 1 } + + - do: + search: + index: test_index + body: + query: { "range" : { "version" : { "gte" : "2.0.0-alpha", "lt" : "9999" } } } + + - match: { hits.total.value: 2 } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml new file mode 100644 index 0000000000000..ff81f6d82d2e6 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml @@ -0,0 +1,49 @@ +# Integration tests for the version field +# +--- +setup: + + - do: + indices.create: + index: test_index + body: + mappings: + properties: + version: + type: version + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "test_index", "_id" : "1" } }' + - '{"version": "1.1.12" }' + - '{ "index" : { "_index" : "test_index", "_id" : "2" } }' + - '{"version": "2.0.0-beta" }' + - '{ "index" : { "_index" : "test_index", "_id" : "3" } }' + - '{"version": "3.1.0" }' + +--- +"Filter script": + - do: + search: + index: test_index + body: + query: { "script" : { "script" : { "source": "doc['version'].value.length() > 5"} } } + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.version: "1.1.12" } + - match: { hits.hits.1._source.version: "2.0.0-beta" } + +--- +"Sort script": + - do: + search: + index: test_index + body: + sort: { "_script" : { "type" : "number", "script" : { "source": "doc['version'].value.length()" } } } + + - match: { hits.total.value: 3 } + - match: { hits.hits.0._source.version: "3.1.0" } + - match: { hits.hits.1._source.version: "1.1.12" } + - match: { hits.hits.2._source.version: "2.0.0-beta" } From 08989e7c345da41fa7d75a93e56dcd17be413aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 22 Jul 2020 19:10:44 +0200 Subject: [PATCH 03/28] Add brute force regex query support --- .../VersionStringFieldMapper.java | 45 +++++++++++++++++ .../VersionStringFieldMapperTests.java | 50 +++++++++++++++---- 2 files changed, 84 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index 6a3f736e52b72..a03d13d359404 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -11,15 +11,20 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Nullable; @@ -207,6 +212,46 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer return wildcardQuery(value + "*", method, context); } + @Override + public Query regexpQuery( + String value, + int flags, + int maxDeterminizedStates, + @Nullable MultiTermQuery.RewriteMethod method, + QueryShardContext context + ) { + if (context.allowExpensiveQueries() == false) { + throw new ElasticsearchException( + "[regexp] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false." + ); + } + failIfNotIndexed(); + RegexpQuery query = new RegexpQuery(new Term(name(), new BytesRef(value)), flags, maxDeterminizedStates) { + + @Override + protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException { + return new FilteredTermsEnum(terms.iterator(), false) { + + @Override + protected AcceptStatus accept(BytesRef term) throws IOException { + byte[] decoded = VersionEncoder.decodeVersion(term).getBytes(); + boolean accepted = compiled.runAutomaton.run(decoded, 0, decoded.length); + // System.out.println(accepted + " : " + VersionEncoder.decodeVersion(term)); + if (accepted) { + return AcceptStatus.YES; + } + return AcceptStatus.NO; + } + }; + } + }; + + if (method != null) { + query.setRewriteMethod(method); + } + return query; + } + @Override public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { if (context.allowExpensiveQueries() == false) { diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 21536cdbbba8d..3baba88731a44 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -28,21 +28,11 @@ public class VersionStringFieldMapperTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - // return pluginList(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class, PainlessPlugin.class); - // TODO PainlessPlugin loading doesn't work when test is run through "gradle check" return pluginList(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class); } public String setUpIndex(String indexName) throws IOException { - createIndex( - indexName, - Settings.builder().put("index.number_of_shards", 1).build(), - "_doc", - "version", - "type=version", - "foo", - "type=keyword" - ); + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); ensureGreen(indexName); client().prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "11.1.0").endObject()).get(); @@ -186,6 +176,44 @@ public void testSort() throws IOException { assertEquals("21.11.0", hits[5].getSortValues()[0]); } + public void testRegexQuery() throws Exception { + String indexName = "test_regex"; + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); + ensureGreen(indexName); + + client().prepareIndex(indexName) + .setId("1") + .setSource(jsonBuilder().startObject().field("version", "1.0.0-alpha.2.1.0-rc.1").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("2") + .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) + .get(); + client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + + // regex + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "2.*0")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("2.1.0", response.getHits().getHits()[0].getSourceAsMap().get("version")); + assertEquals("2.33.0", response.getHits().getHits()[1].getSourceAsMap().get("version")); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "<0-10>.<0-10>.*al.*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("1.0.0-alpha.2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version")); + assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[1].getSourceAsMap().get("version")); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "1.[0-9].[0-9].*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("1.0.0-alpha.2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version")); + assertEquals("1.3.0+build.1234567", response.getHits().getHits()[1].getSourceAsMap().get("version")); + } + public void testWildcardQuery() throws Exception { String indexName = "test_wildcard"; createIndex( From 8a1acc28939452ec3fc6b6c9ee7165403dd944bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 23 Jul 2020 15:13:14 +0200 Subject: [PATCH 04/28] Add fuzzy query support --- .../test/versionfield/10_basic.yml | 5 ++ .../test/versionfield/20_scripts.yml | 5 ++ .../VersionStringFieldMapper.java | 64 ++++++++++++++++++- .../VersionStringFieldMapperTests.java | 50 +++++++++++++-- 4 files changed, 117 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml index 38c3209bed046..f678153fdd55a 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml @@ -3,6 +3,11 @@ --- setup: + - skip: + features: headers + version: " - 7.99.99" + reason: "version field is added to 8.0 first" + - do: indices.create: index: test_index diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml index ff81f6d82d2e6..60de6a8c8da65 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml @@ -3,6 +3,11 @@ --- setup: + - skip: + features: headers + version: " - 7.99.99" + reason: "version field is added to 8.0 first" + - do: indices.create: index: test_index diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index a03d13d359404..65b9b90031c2c 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -20,17 +20,21 @@ import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.ByteRunAutomaton; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; @@ -54,6 +58,7 @@ import org.elasticsearch.xpack.versionfield.VersionEncoder.EncodedVersion; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; @@ -212,6 +217,13 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer return wildcardQuery(value + "*", method, context); } + /** + * We cannot simply use RegexpQuery directly since we use the encoded terms from the dictionary, but the + * automaton in the query will assume unencoded terms. We are running through all terms, decode them and + * then run them through the automaton manually instead. This is not as efficient as intersecting the original + * Terms with the compiled automaton, but we expect the number of distinct version terms indexed into this field + * to be low enough and the use of "rexexp" queries on this field rare enough to brute-force this + */ @Override public Query regexpQuery( String value, @@ -234,9 +246,8 @@ protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOExc @Override protected AcceptStatus accept(BytesRef term) throws IOException { - byte[] decoded = VersionEncoder.decodeVersion(term).getBytes(); + byte[] decoded = VersionEncoder.decodeVersion(term).getBytes(StandardCharsets.UTF_8); boolean accepted = compiled.runAutomaton.run(decoded, 0, decoded.length); - // System.out.println(accepted + " : " + VersionEncoder.decodeVersion(term)); if (accepted) { return AcceptStatus.YES; } @@ -252,6 +263,55 @@ protected AcceptStatus accept(BytesRef term) throws IOException { return query; } + /** + * We cannot simply use FuzzyQuery directly since we use the encoded terms from the dictionary, but the + * automaton in the query will assume unencoded terms. We are running through all terms, decode them and + * then run them through the automaton manually instead. This is not as efficient as intersecting the original + * Terms with the compiled automaton, but we expect the number of distinct version terms indexed into this field + * to be low enough and the use of "fuzzy" queries on this field rare enough to brute-force this + */ + @Override + public Query fuzzyQuery( + Object value, + Fuzziness fuzziness, + int prefixLength, + int maxExpansions, + boolean transpositions, + QueryShardContext context + ) { + if (context.allowExpensiveQueries() == false) { + throw new ElasticsearchException( + "[fuzzy] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false." + ); + } + failIfNotIndexed(); + return new FuzzyQuery( + new Term(name(), (BytesRef) value), + fuzziness.asDistance(BytesRefs.toString(value)), + prefixLength, + maxExpansions, + transpositions + ) { + @Override + protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException { + ByteRunAutomaton runAutomaton = getAutomata().runAutomaton; + + return new FilteredTermsEnum(terms.iterator(), false) { + + @Override + protected AcceptStatus accept(BytesRef term) throws IOException { + byte[] decoded = VersionEncoder.decodeVersion(term).getBytes(StandardCharsets.UTF_8); + boolean accepted = runAutomaton.run(decoded, 0, decoded.length); + if (accepted) { + return AcceptStatus.YES; + } + return AcceptStatus.NO; + } + }; + } + }; + } + @Override public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { if (context.allowExpensiveQueries() == false) { diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 3baba88731a44..80d4598145fc5 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -178,12 +178,18 @@ public void testSort() throws IOException { public void testRegexQuery() throws Exception { String indexName = "test_regex"; - createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); + createIndex( + indexName, + Settings.builder().put("index.number_of_shards", 1).build(), + "_doc", + "version", + "type=version,store_malformed=true" + ); ensureGreen(indexName); client().prepareIndex(indexName) .setId("1") - .setSource(jsonBuilder().startObject().field("version", "1.0.0-alpha.2.1.0-rc.1").endObject()) + .setSource(jsonBuilder().startObject().field("version", "1.0.0alpha2.1.0-rc.1").endObject()) .get(); client().prepareIndex(indexName) .setId("2") @@ -197,7 +203,6 @@ public void testRegexQuery() throws Exception { client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); client().admin().indices().prepareRefresh(indexName).get(); - // regex SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "2.*0")).get(); assertEquals(2, response.getHits().getTotalHits().value); assertEquals("2.1.0", response.getHits().getHits()[0].getSourceAsMap().get("version")); @@ -205,15 +210,50 @@ public void testRegexQuery() throws Exception { response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "<0-10>.<0-10>.*al.*")).get(); assertEquals(2, response.getHits().getTotalHits().value); - assertEquals("1.0.0-alpha.2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version")); + assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version")); assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[1].getSourceAsMap().get("version")); response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "1.[0-9].[0-9].*")).get(); assertEquals(2, response.getHits().getTotalHits().value); - assertEquals("1.0.0-alpha.2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version")); + assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version")); assertEquals("1.3.0+build.1234567", response.getHits().getHits()[1].getSourceAsMap().get("version")); } + public void testFuzzyQuery() throws Exception { + String indexName = "test_fuzzy"; + createIndex( + indexName, + Settings.builder().put("index.number_of_shards", 1).build(), + "_doc", + "version", + "type=version,store_malformed=true" + ); + ensureGreen(indexName); + + client().prepareIndex(indexName) + .setId("1") + .setSource(jsonBuilder().startObject().field("version", "1.0.0-alpha.2.1.0-rc.1").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("2") + .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) + .get(); + client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); + client().prepareIndex(indexName).setId("6").setSource(jsonBuilder().startObject().field("version", "2.a3.0").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.fuzzyQuery("version", "2.3.0")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + assertEquals("2.1.0", response.getHits().getHits()[0].getSourceAsMap().get("version")); + assertEquals("2.33.0", response.getHits().getHits()[1].getSourceAsMap().get("version")); + assertEquals("2.a3.0", response.getHits().getHits()[2].getSourceAsMap().get("version")); + } + public void testWildcardQuery() throws Exception { String indexName = "test_wildcard"; createIndex( From 5bac78553dff91c4d2c8e4c5f9d1a92ff6453ceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 23 Jul 2020 18:04:04 +0200 Subject: [PATCH 05/28] Remove some left-over commented code --- .../xpack/versionfield/VersionEncoder.java | 12 ------------ .../VersionStringFieldMapperTests.java | 16 ---------------- 2 files changed, 28 deletions(-) diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java index fc0d0b64c1e9b..8cbcdf51f31a1 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java @@ -44,14 +44,6 @@ class VersionEncoder { private static final char DOT_SEPARATOR = '.'; private static final char BUILD_SEPARATOR = '+'; - // Regex to test version validity: \d+(\.\d+)*(-[\-\dA-Za-z]+){0,1}(\.[-\dA-Za-z]+)*(\+[\.\-\dA-Za-z]+)? - // private static Pattern LEGAL_VERSION_PATTERN = Pattern.compile( - // "\\d+(\\.\\d+)*(-[\\-\\dA-Za-z]+){0,1}(\\.[\\-\\dA-Za-z]+)*(\\+[\\.\\-\\dA-Za-z]+)?" - // ); - - // Regex to test strict Semver Main Version validity: - // private static Pattern LEGAL_MAIN_VERSION_SEMVER = Pattern.compile("(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)"); - // Regex to test relaxed Semver Main Version validity. Allows for more or less than three main version parts private static Pattern LEGAL_MAIN_VERSION_SEMVER = Pattern.compile("(0|[1-9]\\d*)(\\.(0|[1-9]\\d*))*"); @@ -65,7 +57,6 @@ class VersionEncoder { * Encodes a version string. */ public static EncodedVersion encodeVersion(String versionString) { - // System.out.println("encoding: " + versionString); VersionParts versionParts = VersionParts.ofVersion(versionString); // don't treat non-legal versions further, just mark them as illegal and return @@ -100,7 +91,6 @@ public static EncodedVersion encodeVersion(String versionString) { if (versionParts.buildSuffix != null) { encodedBytes.append(new BytesRef(versionParts.buildSuffix)); } - // System.out.println("encoded: " + encodedBytes.toBytesRef()); return new EncodedVersion( encodedBytes.toBytesRef(), true, @@ -146,7 +136,6 @@ private static Integer[] prefixDigitGroupsWithLength(String input, BytesRefBuild } public static String decodeVersion(BytesRef version) { - // System.out.println("decoding: " + version); int inputPos = version.offset; int resultPos = 0; byte[] result = new byte[version.length]; @@ -163,7 +152,6 @@ public static String decodeVersion(BytesRef version) { } inputPos++; } - // System.out.println("decoded to: " + new String(result, 0, resultPos)); return new String(result, 0, resultPos, StandardCharsets.UTF_8); } diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 80d4598145fc5..3771444050536 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -502,20 +502,4 @@ public void testNullValue() throws Exception { response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("3.0.0")).get(); assertEquals(2, response.getHits().getTotalHits().value); } - - // TODO how to test scripts in ESSingleNodeTestCase? - // Currently fails with "java.lang.IllegalArgumentException: Illegal list shortcut value [value]". - - // public void testScripting() throws Exception { - // String indexName = setUpIndex("test"); - // - // ScriptQueryBuilder query = QueryBuilders.scriptQuery(new Script("doc['version'].value.length() <= 5")); - // SearchResponse response = client().prepareSearch(indexName).addDocValueField("version").setQuery(query).get(); - // assertEquals(2, response.getHits().getTotalHits().value); - // assertEquals("2", response.getHits().getAt(0).getId()); - // assertEquals("1.0.0", response.getHits().getAt(0).field("version").getValue()); - // - // assertEquals("5", response.getHits().getAt(1).getId()); - // assertEquals("2.1.0", response.getHits().getAt(1).field("version").getValue()); - // } } From 8a7b43cadd5bf89d3e6069057290139f17a5bd3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 23 Jul 2020 18:48:19 +0200 Subject: [PATCH 06/28] Adding tests --- .../VersionStringFieldMapperTests.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 3771444050536..698102a20fd17 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; +import org.elasticsearch.search.aggregations.metrics.Cardinality; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; @@ -473,6 +474,11 @@ public void testAggs() throws Exception { assertEquals(3L, buckets.get(0).getDocCount()); assertEquals(5L, buckets.get(1).getKey()); assertEquals(1L, buckets.get(1).getDocCount()); + + // cardinality + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.cardinality("myterms").field("version")).get(); + Cardinality card = response.getAggregations().get("myterms"); + assertEquals(5, card.getValue()); } public void testNullValue() throws Exception { @@ -502,4 +508,46 @@ public void testNullValue() throws Exception { response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("3.0.0")).get(); assertEquals(2, response.getHits().getTotalHits().value); } + + public void testMultiValues() throws Exception { + String indexName = "test_multi"; + createIndex( + indexName, + Settings.builder().put("index.number_of_shards", 1).build(), + "_doc", + "version", + "type=version,store_malformed=true" + ); + ensureGreen(indexName); + + client().prepareIndex(indexName) + .setId("1") + .setSource(jsonBuilder().startObject().array("version", "1.0.0", "3.0.0").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("2") + .setSource(jsonBuilder().startObject().array("version", "2.0.0", "4.alpha.0").endObject()) + .get(); + client().admin().indices().prepareRefresh(indexName).get(); + + SearchResponse response = client().prepareSearch(indexName).addSort("version", SortOrder.ASC).get(); + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("1", response.getHits().getAt(0).getId()); + assertEquals("2", response.getHits().getAt(1).getId()); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "3.0.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("1", response.getHits().getAt(0).getId()); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "4.alpha.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("2", response.getHits().getAt(0).getId()); + + // range + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("1.5.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.5.0")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + } } From 910616e5ebf560c60e29d42d8b10e53c10781df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 28 Jul 2020 14:55:05 +0200 Subject: [PATCH 07/28] Make string_stats work --- .../stringstats/StringStatsAggregator.java | 2 +- x-pack/plugin/versionfield/build.gradle | 1 + .../versionfield/VersionStringFieldMapperTests.java | 13 ++++++++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregator.java index 02c0a2b934daf..25a7a06dca319 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregator.java @@ -101,7 +101,7 @@ public void collect(int doc, long bucket) throws IOException { for (int i = 0; i < valuesCount; i++) { BytesRef value = values.nextValue(); if (value.length > 0) { - String valueStr = value.utf8ToString(); + String valueStr = (String) format.format(value); int length = valueStr.length(); totalLength.increment(bucket, length); diff --git a/x-pack/plugin/versionfield/build.gradle b/x-pack/plugin/versionfield/build.gradle index b3423de6e0d54..854126c72b79c 100644 --- a/x-pack/plugin/versionfield/build.gradle +++ b/x-pack/plugin/versionfield/build.gradle @@ -15,6 +15,7 @@ dependencies { compileOnly project(':modules:lang-painless:spi') compileOnly project(':modules:lang-painless') testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') + testImplementation project(path: xpackModule('analytics'), configuration: 'default') } integTest.enabled = false diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 698102a20fd17..2c2b36f20a920 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -17,6 +17,9 @@ import org.elasticsearch.search.aggregations.metrics.Cardinality; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.analytics.AnalyticsAggregationBuilders; +import org.elasticsearch.xpack.analytics.AnalyticsPlugin; +import org.elasticsearch.xpack.analytics.stringstats.InternalStringStats; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import java.io.IOException; @@ -29,7 +32,7 @@ public class VersionStringFieldMapperTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return pluginList(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class); + return pluginList(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class, AnalyticsPlugin.class); } public String setUpIndex(String indexName) throws IOException { @@ -479,6 +482,14 @@ public void testAggs() throws Exception { response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.cardinality("myterms").field("version")).get(); Cardinality card = response.getAggregations().get("myterms"); assertEquals(5, card.getValue()); + + // string stats + response = client().prepareSearch(indexName) + .addAggregation(AnalyticsAggregationBuilders.stringStats("stats").field("version")) + .get(); + InternalStringStats stats = response.getAggregations().get("stats"); + assertEquals(3, stats.getMinLength()); + assertEquals(11, stats.getMaxLength()); } public void testNullValue() throws Exception { From 970085acc0c3cc275c91b66d9881d094456b280e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 28 Jul 2020 18:27:39 +0200 Subject: [PATCH 08/28] Adressing review comments --- .../xpack/versionfield/VersionEncoder.java | 27 +++++++++---------- .../versionfield/VersionFieldPlugin.java | 6 +---- .../VersionFieldWildcardQuery.java | 6 ++--- .../VersionStringFieldMapper.java | 7 +++-- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java index 8cbcdf51f31a1..71504ee4f068b 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java @@ -14,15 +14,14 @@ /** * Encodes a version string to a {@link BytesRef} that correctly sorts according to software version precedence rules like - * the ones described in Semantiv Versioning (https://semver.org/) + * the ones described in Semantic Versioning (https://semver.org/) * * Version strings are considered to consist of three parts: *
    *
  • a numeric major.minor.patch part starting the version string (e.g. 1.2.3) - *
  • an optional "pre-release" part that starts with a `-` character and can consist of several alpha-numerical sections + *
  • an optional "pre-release" part that starts with a `-` character and can consist of several alphanumerical sections * separated by dots (e.g. "-alpha.2.3") - *
  • an optional "build" part that starts with a `+` character. This will simply be treated as a prefix with no guaranteed ordering, - * (although the ordering should be alphabetical in most cases). + *
  • an optional "build" part that starts with a `+` character. This will simply be treated as a suffix with ASCII sort order. *
* * The version string is encoded such that the ordering works like the following: @@ -37,10 +36,10 @@ class VersionEncoder { public static final byte NUMERIC_MARKER_BYTE = (byte) 0x01; - public static final byte PRERELESE_SEPARATOR_BYTE = (byte) 0x02; - public static final byte NO_PRERELESE_SEPARATOR_BYTE = (byte) 0x03; + public static final byte PRERELEASE_SEPARATOR_BYTE = (byte) 0x02; + public static final byte NO_PRERELEASE_SEPARATOR_BYTE = (byte) 0x03; - private static final char PRERELESE_SEPARATOR = '-'; + private static final char PRERELEASE_SEPARATOR = '-'; private static final char DOT_SEPARATOR = '.'; private static final char BUILD_SEPARATOR = '+'; @@ -68,8 +67,8 @@ public static EncodedVersion encodeVersion(String versionString) { Integer[] mainVersionParts = prefixDigitGroupsWithLength(versionParts.mainVersion, encodedBytes); if (versionParts.preRelease != null) { - encodedBytes.append(PRERELESE_SEPARATOR_BYTE); // versions with pre-release part sort before ones without - encodedBytes.append((byte) PRERELESE_SEPARATOR); + encodedBytes.append(PRERELEASE_SEPARATOR_BYTE); // versions with pre-release part sort before ones without + encodedBytes.append((byte) PRERELEASE_SEPARATOR); String[] preReleaseParts = versionParts.preRelease.substring(1).split("\\."); boolean first = true; for (String preReleasePart : preReleaseParts) { @@ -85,7 +84,7 @@ public static EncodedVersion encodeVersion(String versionString) { first = false; } } else { - encodedBytes.append(NO_PRERELESE_SEPARATOR_BYTE); + encodedBytes.append(NO_PRERELEASE_SEPARATOR_BYTE); } if (versionParts.buildSuffix != null) { @@ -146,7 +145,7 @@ public static String decodeVersion(BytesRef version) { inputPos++; // this should always be a length encoding, which is skipped by increasing inputPos at the end of the loop assert version.bytes[inputPos] < 0; - } else if (inputByte != PRERELESE_SEPARATOR_BYTE && inputByte != NO_PRERELESE_SEPARATOR_BYTE) { + } else if (inputByte != PRERELEASE_SEPARATOR_BYTE && inputByte != NO_PRERELEASE_SEPARATOR_BYTE) { result[resultPos] = inputByte; resultPos++; } @@ -168,7 +167,7 @@ private static boolean legalVersionString(VersionParts versionParts) { return legalMainVersion && legalPreRelease && legalBuildSuffix; } - public static class EncodedVersion { + static class EncodedVersion { public final boolean isLegal; public final boolean isPreRelease; @@ -177,7 +176,7 @@ public static class EncodedVersion { public final Integer minor; public final Integer patch; - EncodedVersion(BytesRef bytesRef, boolean isLegal, boolean isPreRelease, Integer major, Integer minor, Integer patch) { + private EncodedVersion(BytesRef bytesRef, boolean isLegal, boolean isPreRelease, Integer major, Integer minor, Integer patch) { super(); this.bytesRef = bytesRef; this.isLegal = isLegal; @@ -205,7 +204,7 @@ private static VersionParts ofVersion(String versionString) { versionString = versionString.substring(0, versionString.length() - buildSuffix.length()); } - String preRelease = extractSuffix(versionString, PRERELESE_SEPARATOR); + String preRelease = extractSuffix(versionString, PRERELEASE_SEPARATOR); if (preRelease != null) { versionString = versionString.substring(0, versionString.length() - preRelease.length()); } diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java index dd9352f754cc5..44bd284baf8a5 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java @@ -11,8 +11,6 @@ import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; -import java.util.Collections; -import java.util.LinkedHashMap; import java.util.Map; public class VersionFieldPlugin extends Plugin implements MapperPlugin { @@ -21,8 +19,6 @@ public VersionFieldPlugin(Settings settings) {} @Override public Map getMappers() { - Map mappers = new LinkedHashMap<>(); - mappers.put(VersionStringFieldMapper.CONTENT_TYPE, new VersionStringFieldMapper.TypeParser()); - return Collections.unmodifiableMap(mappers); + return Map.of(VersionStringFieldMapper.CONTENT_TYPE, new VersionStringFieldMapper.TypeParser()); } } diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java index 03c6cf780a07d..ba0dc5a26976f 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java @@ -52,13 +52,13 @@ private static Automaton toAutomaton(Term wildcardquery) { case '-': // this should potentially match the first prerelease-dash, so we need an optional marker byte here - automata.add(Operations.optional(Automata.makeChar(VersionEncoder.PRERELESE_SEPARATOR_BYTE))); + automata.add(Operations.optional(Automata.makeChar(VersionEncoder.PRERELEASE_SEPARATOR_BYTE))); containsPreReleaseSeparator = true; automata.add(Automata.makeChar(c)); break; case '+': // this can potentially appear after major version, optionally match the no-prerelease marker - automata.add(Operations.optional(Automata.makeChar(VersionEncoder.NO_PRERELESE_SEPARATOR_BYTE))); + automata.add(Operations.optional(Automata.makeChar(VersionEncoder.NO_PRERELEASE_SEPARATOR_BYTE))); containsPreReleaseSeparator = true; automata.add(Automata.makeChar(c)); break; @@ -90,7 +90,7 @@ private static Automaton toAutomaton(Term wildcardquery) { } // when we only have main version part, we need to add an optional NO_PRERELESE_SEPARATOR_BYTE if (containsPreReleaseSeparator == false) { - automata.add(Operations.optional(Automata.makeChar(VersionEncoder.NO_PRERELESE_SEPARATOR_BYTE))); + automata.add(Operations.optional(Automata.makeChar(VersionEncoder.NO_PRERELEASE_SEPARATOR_BYTE))); } return Operations.concatenate(automata); } diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index 0739133c0b205..f456f63c71196 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -116,6 +116,11 @@ Builder storeMalformed(boolean storeMalformed) { return builder; } + @Override + public Builder indexOptions(IndexOptions indexOptions) { + throw new MapperParsingException("index_options not allowed in field [" + name + "] of type [version]"); + } + private VersionStringFieldType buildFieldType(BuilderContext context) { boolean validateVersion = storeMalformed == false; return new VersionStringFieldType(buildFullName(context), indexed, validateVersion, meta, boost, fieldType); @@ -166,8 +171,6 @@ public Mapper.Builder parse(String name, Map node, ParserCont } else if (propName.equals("store_malformed")) { builder.storeMalformed(XContentMapValues.nodeBooleanValue(propNode, name + ".store_malformed")); iterator.remove(); - } else if (TypeParsers.parseMultiField(builder::addMultiField, name, parserContext, propName, propNode)) { - iterator.remove(); } } return builder; From bdb89a2b1fa58da41732c5876494b3da23f54eee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 29 Jul 2020 18:11:21 +0200 Subject: [PATCH 09/28] Remove 'store_malformed' option --- .../VersionStringFieldMapper.java | 33 ++----------------- .../VersionStringFieldMapperTests.java | 11 ++++--- 2 files changed, 9 insertions(+), 35 deletions(-) diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index f456f63c71196..b2cfbfb140956 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -37,7 +37,6 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.BooleanFieldMapper; @@ -99,7 +98,6 @@ public static class Defaults { static class Builder extends FieldMapper.Builder { protected String nullValue = Defaults.NULL_VALUE; - private boolean storeMalformed = false; Builder(String name) { super(name, Defaults.FIELD_TYPE); @@ -111,19 +109,13 @@ Builder nullValue(String nullValue) { return builder; } - Builder storeMalformed(boolean storeMalformed) { - this.storeMalformed = storeMalformed; - return builder; - } - @Override public Builder indexOptions(IndexOptions indexOptions) { throw new MapperParsingException("index_options not allowed in field [" + name + "] of type [version]"); } private VersionStringFieldType buildFieldType(BuilderContext context) { - boolean validateVersion = storeMalformed == false; - return new VersionStringFieldType(buildFullName(context), indexed, validateVersion, meta, boost, fieldType); + return new VersionStringFieldType(buildFullName(context), indexed, meta, boost, fieldType); } @Override @@ -138,7 +130,6 @@ public VersionStringFieldMapper build(BuilderContext context) { name, fieldType, buildFieldType(context), - storeMalformed, nullValue, multiFieldsBuilder.build(this, context), copyTo, @@ -168,9 +159,6 @@ public Mapper.Builder parse(String name, Map node, ParserCont } builder.nullValue(propNode.toString()); iterator.remove(); - } else if (propName.equals("store_malformed")) { - builder.storeMalformed(XContentMapValues.nodeBooleanValue(propNode, name + ".store_malformed")); - iterator.remove(); } } return builder; @@ -179,13 +167,9 @@ public Mapper.Builder parse(String name, Map node, ParserCont public static final class VersionStringFieldType extends TermBasedFieldType { - // if true, we want to throw errors on illegal versions at index and query time - private boolean validateVersion = false; - public VersionStringFieldType( String name, boolean isSearchable, - boolean validateVersion, Map meta, float boost, FieldType fieldType @@ -193,7 +177,6 @@ public VersionStringFieldType( super(name, isSearchable, true, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta); setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); setBoost(boost); - this.validateVersion = validateVersion; } @Override @@ -340,11 +323,7 @@ protected BytesRef indexedValueForSearch(Object value) { } else { throw new IllegalArgumentException("Illegal value type: " + value.getClass() + ", value: " + value); } - EncodedVersion encodedVersion = encodeVersion(valueAsString); - if (encodedVersion.isLegal == false && validateVersion) { - throw new IllegalArgumentException("Illegal version string: " + valueAsString); - } - return encodedVersion.bytesRef; + return encodeVersion(valueAsString).bytesRef; } @Override @@ -406,7 +385,6 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower } } - private boolean storeMalformed; private String nullValue; private BooleanFieldMapper prereleaseSubField; private NumberFieldMapper majorVersionSubField; @@ -417,7 +395,6 @@ private VersionStringFieldMapper( String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, - boolean storeMalformed, String nullValue, MultiFields multiFields, CopyTo copyTo, @@ -427,7 +404,6 @@ private VersionStringFieldMapper( NumberFieldMapper patchVersionMapper ) { super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); - this.storeMalformed = storeMalformed; this.nullValue = nullValue; this.prereleaseSubField = preReleaseMapper; this.majorVersionSubField = majorVersionMapper; @@ -469,9 +445,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } EncodedVersion encoding = encodeVersion(versionString); - if (encoding.isLegal == false && storeMalformed == false) { - throw new IllegalArgumentException("Illegal version string: " + versionString); - } BytesRef encodedVersion = encoding.bytesRef; if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { Field field = new Field(fieldType().name(), encodedVersion, fieldType); @@ -500,7 +473,6 @@ private void addVersionPartSubfield(ParseContext context, String fieldName, Inte @Override protected void mergeOptions(FieldMapper other, List conflicts) { VersionStringFieldMapper mergeWith = (VersionStringFieldMapper) other; - this.storeMalformed = mergeWith.storeMalformed; this.nullValue = mergeWith.nullValue; } @@ -510,7 +482,6 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, if (nullValue != null) { builder.field("null_value", nullValue); } - builder.field("store_malformed", storeMalformed); } @Override diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 2c2b36f20a920..f617e51c04cc2 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -187,7 +187,7 @@ public void testRegexQuery() throws Exception { Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", - "type=version,store_malformed=true" + "type=version" ); ensureGreen(indexName); @@ -230,7 +230,7 @@ public void testFuzzyQuery() throws Exception { Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", - "type=version,store_malformed=true" + "type=version" ); ensureGreen(indexName); @@ -349,6 +349,9 @@ public void testMainVersionParts() throws IOException { assertEquals(6, response.getHits().getTotalHits().value); } + /** + * test that versions that are invalid under semver are still indexed and retrieveable, though they sort differently + */ public void testStoreMalformed() throws Exception { String indexName = "test_malformed"; createIndex( @@ -356,7 +359,7 @@ public void testStoreMalformed() throws Exception { Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", - "type=version,store_malformed=true" + "type=version" ); ensureGreen(indexName); @@ -527,7 +530,7 @@ public void testMultiValues() throws Exception { Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", - "type=version,store_malformed=true" + "type=version" ); ensureGreen(indexName); From 81bb1146b555f27f8890ee876600e760dc51c963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 29 Jul 2020 19:16:29 +0200 Subject: [PATCH 10/28] Add tests for handling empty string --- .../xpack/versionfield/VersionEncoder.java | 7 +++ .../VersionStringFieldMapper.java | 8 +-- .../versionfield/VersionEncoderTests.java | 5 +- .../VersionStringFieldMapperTests.java | 61 ++++++++----------- 4 files changed, 37 insertions(+), 44 deletions(-) diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java index 71504ee4f068b..8ec27305d5589 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.versionfield; +import org.apache.commons.codec.Charsets; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; @@ -42,6 +43,7 @@ class VersionEncoder { private static final char PRERELEASE_SEPARATOR = '-'; private static final char DOT_SEPARATOR = '.'; private static final char BUILD_SEPARATOR = '+'; + private static final String ENCODED_EMPTY_STRING = new String(new String(new byte[] { NO_PRERELEASE_SEPARATOR_BYTE }, Charsets.UTF_8)); // Regex to test relaxed Semver Main Version validity. Allows for more or less than three main version parts private static Pattern LEGAL_MAIN_VERSION_SEMVER = Pattern.compile("(0|[1-9]\\d*)(\\.(0|[1-9]\\d*))*"); @@ -60,6 +62,11 @@ public static EncodedVersion encodeVersion(String versionString) { // don't treat non-legal versions further, just mark them as illegal and return if (legalVersionString(versionParts) == false) { + if (versionString.length() == 0) { + // special case, we want empty string to sort after valid strings, which all start with 0x01, add a higher char that + // we are sure to remove when decoding + versionString = ENCODED_EMPTY_STRING; + } return new EncodedVersion(new BytesRef(versionString), false, true, 0, 0, 0); } diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index b2cfbfb140956..c70ea8916f2f6 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -167,13 +167,7 @@ public Mapper.Builder parse(String name, Map node, ParserCont public static final class VersionStringFieldType extends TermBasedFieldType { - public VersionStringFieldType( - String name, - boolean isSearchable, - Map meta, - float boost, - FieldType fieldType - ) { + public VersionStringFieldType(String name, boolean isSearchable, Map meta, float boost, FieldType fieldType) { super(name, isSearchable, true, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta); setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); setBoost(boost); diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java index 825146e315918..aea79b1c59adb 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java @@ -182,6 +182,7 @@ public void testSemVerValidation() { } String[] invalidSemverVersions = new String[] { + "", "1.2.3-0123", "1.2.3-0123.0123", "1.1.2+.123", @@ -218,7 +219,9 @@ public void testSemVerValidation() { "9.8.7-whatever+meta+meta", "999999999.999999999.999999999.----RC-SNAPSHOT.12.09.1--------------------------------..12", "12.el2", - "12.el2-1.0-rc5" }; + "12.el2-1.0-rc5", + "6.nüll.7" // make sure extended ascii-range (128-255) in invalid versions is decoded correctly + }; for (String version : invalidSemverVersions) { assertFalse("should be invalid: " + version, VersionEncoder.encodeVersion(version).isLegal); // since we're here, also check encoding / decoding rountrip diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index f617e51c04cc2..ec38b2e7ff2a2 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -182,13 +182,7 @@ public void testSort() throws IOException { public void testRegexQuery() throws Exception { String indexName = "test_regex"; - createIndex( - indexName, - Settings.builder().put("index.number_of_shards", 1).build(), - "_doc", - "version", - "type=version" - ); + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); ensureGreen(indexName); client().prepareIndex(indexName) @@ -225,13 +219,7 @@ public void testRegexQuery() throws Exception { public void testFuzzyQuery() throws Exception { String indexName = "test_fuzzy"; - createIndex( - indexName, - Settings.builder().put("index.number_of_shards", 1).build(), - "_doc", - "version", - "type=version" - ); + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); ensureGreen(indexName); client().prepareIndex(indexName) @@ -354,13 +342,7 @@ public void testMainVersionParts() throws IOException { */ public void testStoreMalformed() throws Exception { String indexName = "test_malformed"; - createIndex( - indexName, - Settings.builder().put("index.number_of_shards", 1).build(), - "_doc", - "version", - "type=version" - ); + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); ensureGreen(indexName); client().prepareIndex(indexName) @@ -372,10 +354,11 @@ public void testStoreMalformed() throws Exception { .setId("3") .setSource(jsonBuilder().startObject().field("version", "2.2.0-badchar!").endObject()) .get(); + client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "").endObject()).get(); client().admin().indices().prepareRefresh(indexName).get(); SearchResponse response = client().prepareSearch(indexName).addDocValueField("version").get(); - assertEquals(3, response.getHits().getTotalHits().value); + assertEquals(4, response.getHits().getTotalHits().value); assertEquals("1", response.getHits().getAt(0).getId()); assertEquals("1.invalid.0", response.getHits().getAt(0).field("version").getValue()); @@ -385,6 +368,9 @@ public void testStoreMalformed() throws Exception { assertEquals("3", response.getHits().getAt(2).getId()); assertEquals("2.2.0-badchar!", response.getHits().getAt(2).field("version").getValue()); + assertEquals("4", response.getHits().getAt(3).getId()); + assertEquals("", response.getHits().getAt(3).field("version").getValue()); + // exact match for malformed term response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.invalid.0")).get(); assertEquals(1, response.getHits().getTotalHits().value); @@ -392,29 +378,38 @@ public void testStoreMalformed() throws Exception { response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "2.2.0-badchar!")).get(); assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + // also should appear in terms aggs response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version")).get(); Terms terms = response.getAggregations().get("myterms"); List buckets = terms.getBuckets(); - assertEquals(3, buckets.size()); + assertEquals(4, buckets.size()); assertEquals("2.2.0", buckets.get(0).getKey()); - assertEquals("1.invalid.0", buckets.get(1).getKey()); - assertEquals("2.2.0-badchar!", buckets.get(2).getKey()); + assertEquals("", buckets.get(1).getKey()); + assertEquals("1.invalid.0", buckets.get(2).getKey()); + assertEquals("2.2.0-badchar!", buckets.get(3).getKey()); // invalid values should sort after all valid ones response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC).get(); - assertEquals(3, response.getHits().getTotalHits().value); + assertEquals(4, response.getHits().getTotalHits().value); SearchHit[] hits = response.getHits().getHits(); assertEquals("2.2.0", hits[0].getSortValues()[0]); - assertEquals("1.invalid.0", hits[1].getSortValues()[0]); - assertEquals("2.2.0-badchar!", hits[2].getSortValues()[0]); + assertEquals("", hits[1].getSortValues()[0]); + assertEquals("1.invalid.0", hits[2].getSortValues()[0]); + assertEquals("2.2.0-badchar!", hits[3].getSortValues()[0]); // ranges can include them, but they are sorted last response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("3.0.0")).get(); assertEquals(1, response.getHits().getTotalHits().value); response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("3.0.0")).get(); - assertEquals(2, response.getHits().getTotalHits().value); + assertEquals(3, response.getHits().getTotalHits().value); + + // using the empty string as lower bound should return all "invalid" versions + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("")).get(); + assertEquals(3, response.getHits().getTotalHits().value); } public void testAggs() throws Exception { @@ -525,13 +520,7 @@ public void testNullValue() throws Exception { public void testMultiValues() throws Exception { String indexName = "test_multi"; - createIndex( - indexName, - Settings.builder().put("index.number_of_shards", 1).build(), - "_doc", - "version", - "type=version" - ); + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); ensureGreen(indexName); client().prepareIndex(indexName) From 5fcf8a73916f2aa635d5fe9b2a8ea557ad62c9ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 30 Jul 2020 12:19:09 +0200 Subject: [PATCH 11/28] fix yaml test --- .../resources/rest-api-spec/test/versionfield/10_basic.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml index f678153fdd55a..f76e513f37ac9 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/10_basic.yml @@ -38,7 +38,6 @@ setup: properties: version: type: version - store_malformed: true - do: bulk: @@ -49,7 +48,7 @@ setup: - '{ "index" : { "_index" : "test_malformed", "_id" : "2" } }' - '{"version": "2.0.0-beta" }' - '{ "index" : { "_index" : "test_malformed", "_id" : "3" } }' - - '{"version": "3.1.0" }' + - '{"version": "v3.1.0" }' - '{ "index" : { "_index" : "test_malformed", "_id" : "4" } }' - '{"version": "1.el6" }' @@ -70,8 +69,8 @@ setup: - match: { hits.total.value: 4 } - match: { hits.hits.0._source.version: "1.1.0" } - match: { hits.hits.1._source.version: "2.0.0-beta" } - - match: { hits.hits.2._source.version: "3.1.0" } - - match: { hits.hits.3._source.version: "1.el6" } + - match: { hits.hits.2._source.version: "1.el6" } + - match: { hits.hits.3._source.version: "v3.1.0" } --- "Basic ranges": From cfe2dbab69de9fd5e84c46de8a274f69c8514cf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 31 Jul 2020 16:23:04 +0200 Subject: [PATCH 12/28] Change range query to points approximation and dv validation --- .../versionfield/ValidationOnSortedDv.java | 130 ++++++++++++++++++ .../VersionStringFieldMapper.java | 17 +-- .../VersionStringFieldMapperTests.java | 12 +- 3 files changed, 145 insertions(+), 14 deletions(-) create mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/ValidationOnSortedDv.java diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/ValidationOnSortedDv.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/ValidationOnSortedDv.java new file mode 100644 index 0000000000000..28c006de0d569 --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/ValidationOnSortedDv.java @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import java.util.Objects; + +/** + * Query that runs a validation for version ranges across sorted doc values. + * Used in conjunction with more selective query clauses. + */ +class ValidationOnSortedDv extends Query { + + private final String field; + private final BytesRef lower; + private final BytesRef upper; + private final boolean includeLower; + private final boolean includeUpper; + + ValidationOnSortedDv(String field, BytesRef lower, BytesRef upper, boolean includeLower, boolean includeUpper) { + this.field = field; + this.lower = lower; + this.upper = upper; + this.includeLower = includeLower; + this.includeUpper = includeUpper; + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + + return new ConstantScoreWeight(this, boost) { + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + final SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field); + + TwoPhaseIterator twoPhase = new TwoPhaseIterator(values) { + @Override + public boolean matches() throws IOException { + long ord = values.nextOrd(); + // multi-value document can have more than one value, iterate over ords + while (ord != SortedSetDocValues.NO_MORE_ORDS) { + BytesRef value = values.lookupOrd(ord); + boolean inRange = true; + if (lower != null) { + if (includeLower) { + inRange = lower.compareTo(value) <= 0; + } else { + inRange = lower.compareTo(value) < 0; + } + } + if (inRange && upper != null) { + if (includeUpper) { + inRange = upper.compareTo(value) >= 0; + } else { + inRange = upper.compareTo(value) > 0; + } + } + if (inRange) { + return true; // found at least one matching value + } + ord = values.nextOrd(); + } + return false; + } + + @Override + public float matchCost() { + // TODO: how can we compute this? + return 1000f; + } + }; + return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return true; + } + }; + } + + @Override + public String toString(String field) { + StringBuilder sb = new StringBuilder(field + ":"); + sb.append(includeLower ? "[" : "("); + sb.append(lower == null ? "" : VersionEncoder.decodeVersion(lower)); + sb.append("-"); + sb.append(upper == null ? "" : VersionEncoder.decodeVersion(upper)); + sb.append(includeUpper ? "]" : ")"); + return sb.toString(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + ValidationOnSortedDv other = (ValidationOnSortedDv) obj; + return Objects.equals(field, other.field) + && Objects.equals(lower, other.lower) + && Objects.equals(lower, other.upper) + && includeLower == other.includeLower + && includeUpper == other.includeUpper; + + } + + @Override + public int hashCode() { + return Objects.hash(field, lower, upper, includeLower, includeUpper); + } + +} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index c70ea8916f2f6..dd6bd20fe49ec 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -16,7 +16,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; @@ -24,7 +23,6 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.RegexpQuery; -import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.ByteRunAutomaton; @@ -365,17 +363,12 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower // point query on the 16byte prefix Query pointPrefixQuery = BinaryPoint.newRangeQuery(name(), lowerBytes, upperBytes); - Query termQuery = new TermRangeQuery( - name(), - lowerTerm == null ? null : lower, - upperTerm == null ? null : upper, - includeLower, - includeUpper - ); + ValidationOnSortedDv validationQuery = new ValidationOnSortedDv(name(), lower, upper, includeLower, includeUpper); - return new BooleanQuery.Builder().add(new BooleanClause(pointPrefixQuery, Occur.MUST)) - .add(new BooleanClause(termQuery, Occur.MUST)) - .build(); + BooleanQuery.Builder qBuilder = new BooleanQuery.Builder(); + qBuilder.add(pointPrefixQuery, Occur.MUST); + qBuilder.add(validationQuery, Occur.MUST); + return qBuilder.build(); } } diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index ec38b2e7ff2a2..51911067e37c6 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -531,12 +531,17 @@ public void testMultiValues() throws Exception { .setId("2") .setSource(jsonBuilder().startObject().array("version", "2.0.0", "4.alpha.0").endObject()) .get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().array("version", "2.1.0", "2.2.0", "5.99.0").endObject()) + .get(); client().admin().indices().prepareRefresh(indexName).get(); SearchResponse response = client().prepareSearch(indexName).addSort("version", SortOrder.ASC).get(); - assertEquals(2, response.getHits().getTotalHits().value); + assertEquals(3, response.getHits().getTotalHits().value); assertEquals("1", response.getHits().getAt(0).getId()); assertEquals("2", response.getHits().getAt(1).getId()); + assertEquals("3", response.getHits().getAt(2).getId()); response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "3.0.0")).get(); assertEquals(1, response.getHits().getTotalHits().value); @@ -551,6 +556,9 @@ public void testMultiValues() throws Exception { assertEquals(1, response.getHits().getTotalHits().value); response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.5.0")).get(); - assertEquals(2, response.getHits().getTotalHits().value); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("5.0.0").to("6.0.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); } } From c70e7c1d0d98915ec25d97cba02fb790dbd57c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 31 Jul 2020 18:19:38 +0200 Subject: [PATCH 13/28] Register DocValueFormat via plugin --- x-pack/plugin/versionfield/build.gradle | 1 + .../xpack/versionfield/VersionFieldIT.java | 76 +++++++++++++++++++ .../versionfield/VersionFieldPlugin.java | 17 ++++- .../VersionStringFieldMapper.java | 4 +- 4 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/versionfield/src/internalClusterTest/java/org/elasticsearch/xpack/versionfield/VersionFieldIT.java diff --git a/x-pack/plugin/versionfield/build.gradle b/x-pack/plugin/versionfield/build.gradle index 854126c72b79c..282c6bc7f6c43 100644 --- a/x-pack/plugin/versionfield/build.gradle +++ b/x-pack/plugin/versionfield/build.gradle @@ -1,6 +1,7 @@ evaluationDependsOn(xpackModule('core')) apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { name 'versionfield' diff --git a/x-pack/plugin/versionfield/src/internalClusterTest/java/org/elasticsearch/xpack/versionfield/VersionFieldIT.java b/x-pack/plugin/versionfield/src/internalClusterTest/java/org/elasticsearch/xpack/versionfield/VersionFieldIT.java new file mode 100644 index 0000000000000..dc461db7df2b8 --- /dev/null +++ b/x-pack/plugin/versionfield/src/internalClusterTest/java/org/elasticsearch/xpack/versionfield/VersionFieldIT.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; + +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +public class VersionFieldIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return List.of(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class); + } + + public void testTermsAggregation() throws Exception { + String indexName = "test"; + createIndex(indexName); + + client().admin() + .indices() + .preparePutMapping(indexName) + .setSource( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("version") + .field("type", "version") + .endObject() + .endObject() + .endObject() + .endObject() + ) + .get(); + ensureGreen(); + + client().prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "1.0").endObject()).get(); + client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "1.3.0").endObject()).get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha").endObject()) + .get(); + client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "3.11.5").endObject()).get(); + refresh(); + + // terms aggs + SearchResponse response = client().prepareSearch(indexName) + .addAggregation(AggregationBuilders.terms("myterms").field("version")) + .get(); + Terms terms = response.getAggregations().get("myterms"); + List buckets = terms.getBuckets(); + + assertEquals(5, buckets.size()); + assertEquals("1.0", buckets.get(0).getKey()); + assertEquals("1.3.0", buckets.get(1).getKey()); + assertEquals("2.1.0-alpha", buckets.get(2).getKey()); + assertEquals("2.1.0", buckets.get(3).getKey()); + assertEquals("3.11.5", buckets.get(4).getKey()); + } +} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java index 44bd284baf8a5..4e4cc71f1782b 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java @@ -6,14 +6,18 @@ package org.elasticsearch.xpack.versionfield; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.DocValueFormat; +import java.util.List; import java.util.Map; -public class VersionFieldPlugin extends Plugin implements MapperPlugin { +public class VersionFieldPlugin extends Plugin implements ActionPlugin, MapperPlugin { public VersionFieldPlugin(Settings settings) {} @@ -21,4 +25,15 @@ public VersionFieldPlugin(Settings settings) {} public Map getMappers() { return Map.of(VersionStringFieldMapper.CONTENT_TYPE, new VersionStringFieldMapper.TypeParser()); } + + @Override + public List getNamedWriteables() { + return List.of( + new NamedWriteableRegistry.Entry( + DocValueFormat.class, + VersionStringFieldMapper.VERSION_DOCVALUE.getWriteableName(), + in -> VersionStringFieldMapper.VERSION_DOCVALUE + ) + ); + } } diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index dd6bd20fe49ec..3a557c1c21b9c 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -491,11 +491,11 @@ protected Object parseSourceValue(Object value, String format) { return value.toString(); } - private static DocValueFormat VERSION_DOCVALUE = new DocValueFormat() { + public static DocValueFormat VERSION_DOCVALUE = new DocValueFormat() { @Override public String getWriteableName() { - return "version_semver"; + return "version"; } @Override From 57b040f04cace55f54f3ecf40e37dbbbdb17ac05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 20 Aug 2020 16:36:07 +0200 Subject: [PATCH 14/28] Add valueFetcher() method --- .../VersionStringFieldMapper.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index 3a557c1c21b9c..61c60c95b0672 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -42,12 +42,15 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TermBasedFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.TypeParsers; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.support.QueryParsers; import org.elasticsearch.search.DocValueFormat; @@ -403,6 +406,20 @@ public VersionStringFieldType fieldType() { return (VersionStringFieldType) super.fieldType(); } + @Override + public ValueFetcher valueFetcher(MapperService mapperService, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + + return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) { + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + }; + } + @Override protected String contentType() { return CONTENT_TYPE; @@ -483,14 +500,6 @@ public Iterator iterator() { return concat; } - @Override - protected Object parseSourceValue(Object value, String format) { - if (format != null) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); - } - return value.toString(); - } - public static DocValueFormat VERSION_DOCVALUE = new DocValueFormat() { @Override From 5b902fb308a8e4c8144891bdb6983fe2b44dabeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 21 Aug 2020 11:32:50 +0200 Subject: [PATCH 15/28] Restrict field options by moving to ParametrizedFieldMapper --- .../index/mapper/FieldMapper.java | 1 + .../versionfield/VersionFieldPlugin.java | 2 +- .../VersionStringFieldMapper.java | 114 +--- .../VersionStringFieldMapperTests.java | 643 ++++-------------- .../versionfield/VersionStringFieldTests.java | 536 +++++++++++++++ 5 files changed, 688 insertions(+), 608 deletions(-) create mode 100644 x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index d5e1bf5386fca..d349fd74a14c2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java index 4e4cc71f1782b..227b47c5611a5 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java @@ -23,7 +23,7 @@ public VersionFieldPlugin(Settings settings) {} @Override public Map getMappers() { - return Map.of(VersionStringFieldMapper.CONTENT_TYPE, new VersionStringFieldMapper.TypeParser()); + return Map.of(VersionStringFieldMapper.CONTENT_TYPE, VersionStringFieldMapper.PARSER); } @Override diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index 61c60c95b0672..619074fff3f1a 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; @@ -41,15 +40,14 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TermBasedFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; -import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.support.QueryParsers; @@ -72,7 +70,7 @@ /** * A {@link FieldMapper} for indexing fields with version strings. */ -public class VersionStringFieldMapper extends FieldMapper { +public class VersionStringFieldMapper extends ParametrizedFieldMapper { private static byte[] MIN_VALUE = new byte[16]; private static byte[] MAX_VALUE = new byte[16]; @@ -92,86 +90,55 @@ public static class Defaults { FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); FIELD_TYPE.freeze(); } - - public static final String NULL_VALUE = null; } - static class Builder extends FieldMapper.Builder { + static class Builder extends ParametrizedFieldMapper.Builder { - protected String nullValue = Defaults.NULL_VALUE; + private final Parameter> meta = Parameter.metaParam(); Builder(String name) { - super(name, Defaults.FIELD_TYPE); - builder = this; - } - - Builder nullValue(String nullValue) { - this.nullValue = nullValue; - return builder; - } - - @Override - public Builder indexOptions(IndexOptions indexOptions) { - throw new MapperParsingException("index_options not allowed in field [" + name + "] of type [version]"); + super(name); } - private VersionStringFieldType buildFieldType(BuilderContext context) { - return new VersionStringFieldType(buildFullName(context), indexed, meta, boost, fieldType); + private VersionStringFieldType buildFieldType(BuilderContext context, FieldType fieldtype) { + return new VersionStringFieldType(buildFullName(context), fieldtype, meta.getValue()); } @Override public VersionStringFieldMapper build(BuilderContext context) { + FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); BooleanFieldMapper.Builder preReleaseSubfield = new BooleanFieldMapper.Builder(name + ".isPreRelease"); NumberType type = NumberType.INTEGER; - NumberFieldMapper.Builder majorVersionSubField = new NumberFieldMapper.Builder(name + ".major", type).nullValue(0); - NumberFieldMapper.Builder minorVersionSubField = new NumberFieldMapper.Builder(name + ".minor", type).nullValue(0); - NumberFieldMapper.Builder patchVersionSubField = new NumberFieldMapper.Builder(name + ".patch", type).nullValue(0); + NumberFieldMapper.Builder majorVersionSubField = new NumberFieldMapper.Builder(name + ".major", type, false, false); + NumberFieldMapper.Builder minorVersionSubField = new NumberFieldMapper.Builder(name + ".minor", type, false, false); + NumberFieldMapper.Builder patchVersionSubField = new NumberFieldMapper.Builder(name + ".patch", type, false, false); return new VersionStringFieldMapper( name, - fieldType, - buildFieldType(context), - nullValue, + fieldtype, + buildFieldType(context, fieldtype), multiFieldsBuilder.build(this, context), - copyTo, + copyTo.build(), preReleaseSubfield.build(context), majorVersionSubField.build(context), minorVersionSubField.build(context), patchVersionSubField.build(context) ); } - } - - public static class TypeParser implements Mapper.TypeParser { - - public TypeParser() {} @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - Builder builder = new Builder(name); - TypeParsers.parseField(builder, name, node, parserContext); - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String propName = entry.getKey(); - Object propNode = entry.getValue(); - if (propName.equals("null_value")) { - if (propNode == null) { - throw new MapperParsingException("Property [null_value] cannot be null."); - } - builder.nullValue(propNode.toString()); - iterator.remove(); - } - } - return builder; + protected List> getParameters() { + return List.of(meta); } } + public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n)); + public static final class VersionStringFieldType extends TermBasedFieldType { - public VersionStringFieldType(String name, boolean isSearchable, Map meta, float boost, FieldType fieldType) { - super(name, isSearchable, true, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta); + public VersionStringFieldType(String name, FieldType fieldType, Map meta) { + super(name, true, true, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta); setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); - setBoost(boost); } @Override @@ -375,7 +342,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower } } - private String nullValue; + private final FieldType fieldType; private BooleanFieldMapper prereleaseSubField; private NumberFieldMapper majorVersionSubField; private NumberFieldMapper minorVersionSubField; @@ -385,7 +352,6 @@ private VersionStringFieldMapper( String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, - String nullValue, MultiFields multiFields, CopyTo copyTo, BooleanFieldMapper preReleaseMapper, @@ -393,8 +359,8 @@ private VersionStringFieldMapper( NumberFieldMapper minorVersionMapper, NumberFieldMapper patchVersionMapper ) { - super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); - this.nullValue = nullValue; + super(simpleName, mappedFieldType, multiFields, copyTo); + this.fieldType = fieldType; this.prereleaseSubField = preReleaseMapper; this.majorVersionSubField = majorVersionMapper; this.minorVersionSubField = minorVersionMapper; @@ -412,7 +378,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, String format) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) { + return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), null) { @Override protected String parseSourceValue(Object value) { return value.toString(); @@ -438,7 +404,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } else { XContentParser parser = context.parser(); if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { - versionString = nullValue; + return; } else { versionString = parser.textOrNull(); } @@ -450,13 +416,10 @@ protected void parseCreateField(ParseContext context) throws IOException { EncodedVersion encoding = encodeVersion(versionString); BytesRef encodedVersion = encoding.bytesRef; - if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { - Field field = new Field(fieldType().name(), encodedVersion, fieldType); - context.doc().add(field); - // encode the first 16 bytes as points for efficient range query - byte[] first16bytes = Arrays.copyOfRange(encodedVersion.bytes, encodedVersion.offset, 16); - context.doc().add(new BinaryPoint(fieldType().name(), first16bytes)); - } + context.doc().add(new Field(fieldType().name(), encodedVersion, fieldType)); + // encode the first 16 bytes as points for efficient range query + byte[] first16bytes = Arrays.copyOfRange(encodedVersion.bytes, encodedVersion.offset, 16); + context.doc().add(new BinaryPoint(fieldType().name(), first16bytes)); context.doc().add(new SortedSetDocValuesField(fieldType().name(), encodedVersion)); // add additional information extracted from version string @@ -474,20 +437,6 @@ private void addVersionPartSubfield(ParseContext context, String fieldName, Inte } } - @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - VersionStringFieldMapper mergeWith = (VersionStringFieldMapper) other; - this.nullValue = mergeWith.nullValue; - } - - @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - super.doXContentBody(builder, includeDefaults, params); - if (nullValue != null) { - builder.field("null_value", nullValue); - } - } - @Override public Iterator iterator() { List subIterators = new ArrayList<>(); @@ -525,4 +474,9 @@ public String toString() { return getWriteableName(); } }; + + @Override + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new Builder(simpleName()).init(this); + } } diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 51911067e37c6..caa44b645e3b3 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -6,559 +6,148 @@ package org.elasticsearch.xpack.versionfield; -import org.elasticsearch.action.search.SearchResponse; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperTestCase; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; -import org.elasticsearch.search.aggregations.metrics.Cardinality; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.xpack.analytics.AnalyticsAggregationBuilders; -import org.elasticsearch.xpack.analytics.AnalyticsPlugin; -import org.elasticsearch.xpack.analytics.stringstats.InternalStringStats; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.mapper.FieldMapperTestCase.fetchSourceValue; +import static org.elasticsearch.xpack.versionfield.VersionEncoder.encodeVersion; +import static org.hamcrest.Matchers.equalTo; -public class VersionStringFieldMapperTests extends ESSingleNodeTestCase { +public class VersionStringFieldMapperTests extends MapperTestCase { @Override - protected Collection> getPlugins() { - return pluginList(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class, AnalyticsPlugin.class); + protected Collection getPlugins() { + return Collections.singletonList(new VersionFieldPlugin(getIndexSettings())); } - public String setUpIndex(String indexName) throws IOException { - createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); - ensureGreen(indexName); - - client().prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "11.1.0").endObject()).get(); - client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "1.0.0").endObject()).get(); - client().prepareIndex(indexName) - .setId("3") - .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) - .get(); - client().prepareIndex(indexName) - .setId("4") - .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) - .get(); - client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); - client().prepareIndex(indexName).setId("6").setSource(jsonBuilder().startObject().field("version", "21.11.0").endObject()).get(); - client().admin().indices().prepareRefresh(indexName).get(); - return indexName; - } - - public void testExactQueries() throws Exception { - String indexName = "test"; - setUpIndex(indexName); - - // match - SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", ("1.0.0"))).get(); - assertEquals(1, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.4.0")).get(); - assertEquals(0, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.3.0")).get(); - assertEquals(0, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.3.0+build.1234567")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - // term - response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.0.0")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.4.0")).get(); - assertEquals(0, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.3.0")).get(); - assertEquals(0, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.3.0+build.1234567")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - // terms - response = client().prepareSearch(indexName).setQuery(QueryBuilders.termsQuery("version", "1.0.0", "1.3.0")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.termsQuery("version", "1.4.0", "1.3.0+build.1234567")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - // phrase query (just for keyword compatibility) - response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchPhraseQuery("version", "2.1.0-alpha.beta")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - } - - public void testRangeQueries() throws Exception { - String indexName = setUpIndex("test"); - SearchResponse response = client().prepareSearch(indexName) - .setQuery(QueryBuilders.rangeQuery("version").from("1.0.0").to("3.0.0")) - .get(); - assertEquals(4, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.1.0").to("3.0.0")).get(); - assertEquals(3, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName) - .setQuery(QueryBuilders.rangeQuery("version").from("0.1.0").to("2.1.0-alpha.beta")) - .get(); - assertEquals(3, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("2.1.0").to("3.0.0")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("3.0.0").to("4.0.0")).get(); - assertEquals(0, response.getHits().getTotalHits().value); - - // ranges excluding edges - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.0.0", false).to("3.0.0")).get(); - assertEquals(3, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.0.0").to("2.1.0", false)).get(); - assertEquals(3, response.getHits().getTotalHits().value); - - // open ranges - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.4.0")).get(); - assertEquals(4, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("1.4.0")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - } - - public void testPrefixQuery() throws IOException { - String indexName = setUpIndex("test"); - // prefix - SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "1")).get(); - assertEquals(3, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2.1")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2.1.0-")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "1.3.0+b")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2")).get(); - assertEquals(3, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.1")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.11")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - } - - public void testSort() throws IOException { - String indexName = setUpIndex("test"); - - // sort based on version field - SearchResponse response = client().prepareSearch(indexName) - .setQuery(QueryBuilders.matchAllQuery()) - .addSort("version", SortOrder.DESC) - .get(); - assertEquals(6, response.getHits().getTotalHits().value); - SearchHit[] hits = response.getHits().getHits(); - assertEquals("21.11.0", hits[0].getSortValues()[0]); - assertEquals("11.1.0", hits[1].getSortValues()[0]); - assertEquals("2.1.0", hits[2].getSortValues()[0]); - assertEquals("2.1.0-alpha.beta", hits[3].getSortValues()[0]); - assertEquals("1.3.0+build.1234567", hits[4].getSortValues()[0]); - assertEquals("1.0.0", hits[5].getSortValues()[0]); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC).get(); - assertEquals(6, response.getHits().getTotalHits().value); - hits = response.getHits().getHits(); - assertEquals("1.0.0", hits[0].getSortValues()[0]); - assertEquals("1.3.0+build.1234567", hits[1].getSortValues()[0]); - assertEquals("2.1.0-alpha.beta", hits[2].getSortValues()[0]); - assertEquals("2.1.0", hits[3].getSortValues()[0]); - assertEquals("11.1.0", hits[4].getSortValues()[0]); - assertEquals("21.11.0", hits[5].getSortValues()[0]); - } - - public void testRegexQuery() throws Exception { - String indexName = "test_regex"; - createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); - ensureGreen(indexName); - - client().prepareIndex(indexName) - .setId("1") - .setSource(jsonBuilder().startObject().field("version", "1.0.0alpha2.1.0-rc.1").endObject()) - .get(); - client().prepareIndex(indexName) - .setId("2") - .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) - .get(); - client().prepareIndex(indexName) - .setId("3") - .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) - .get(); - client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); - client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); - client().admin().indices().prepareRefresh(indexName).get(); - - SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "2.*0")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - assertEquals("2.1.0", response.getHits().getHits()[0].getSourceAsMap().get("version")); - assertEquals("2.33.0", response.getHits().getHits()[1].getSourceAsMap().get("version")); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "<0-10>.<0-10>.*al.*")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version")); - assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[1].getSourceAsMap().get("version")); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "1.[0-9].[0-9].*")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version")); - assertEquals("1.3.0+build.1234567", response.getHits().getHits()[1].getSourceAsMap().get("version")); - } - - public void testFuzzyQuery() throws Exception { - String indexName = "test_fuzzy"; - createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); - ensureGreen(indexName); - - client().prepareIndex(indexName) - .setId("1") - .setSource(jsonBuilder().startObject().field("version", "1.0.0-alpha.2.1.0-rc.1").endObject()) - .get(); - client().prepareIndex(indexName) - .setId("2") - .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) - .get(); - client().prepareIndex(indexName) - .setId("3") - .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) - .get(); - client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); - client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); - client().prepareIndex(indexName).setId("6").setSource(jsonBuilder().startObject().field("version", "2.a3.0").endObject()).get(); - client().admin().indices().prepareRefresh(indexName).get(); - - SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.fuzzyQuery("version", "2.3.0")).get(); - assertEquals(3, response.getHits().getTotalHits().value); - assertEquals("2.1.0", response.getHits().getHits()[0].getSourceAsMap().get("version")); - assertEquals("2.33.0", response.getHits().getHits()[1].getSourceAsMap().get("version")); - assertEquals("2.a3.0", response.getHits().getHits()[2].getSourceAsMap().get("version")); + @Override + protected void minimalMapping(XContentBuilder b) throws IOException { + b.field("type", "version"); } - public void testWildcardQuery() throws Exception { - String indexName = "test_wildcard"; - createIndex( - indexName, - Settings.builder().put("index.number_of_shards", 1).build(), - "_doc", - "version", - "type=version", - "foo", - "type=keyword" + public void testDefaults() throws Exception { + XContentBuilder mapping = fieldMapping(this::minimalMapping); + DocumentMapper mapper = createDocumentMapper(mapping); + assertEquals(Strings.toString(mapping), mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "1.2.3").endObject()), + XContentType.JSON + ) ); - ensureGreen(indexName); - - client().prepareIndex(indexName) - .setId("1") - .setSource(jsonBuilder().startObject().field("version", "1.0.0-alpha.2.1.0-rc.1").endObject()) - .get(); - client().prepareIndex(indexName) - .setId("2") - .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) - .get(); - client().prepareIndex(indexName) - .setId("3") - .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) - .get(); - client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); - client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); - client().admin().indices().prepareRefresh(indexName).get(); - - // wildcard - SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*alpha*")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*b*")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*bet*")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "2.1*")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "2.1.0-*")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.1.0-*")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.1.0*")).get(); - assertEquals(3, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.?.0*")).get(); - assertEquals(3, response.getHits().getTotalHits().value); + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(3, fields.length); + + assertEquals("1.2.3", VersionEncoder.decodeVersion(fields[0].binaryValue())); + IndexableFieldType fieldType = fields[0].fieldType(); + assertThat(fieldType.omitNorms(), equalTo(true)); + assertFalse(fieldType.tokenized()); + assertFalse(fieldType.stored()); + assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS)); + assertThat(fieldType.storeTermVectors(), equalTo(false)); + assertThat(fieldType.storeTermVectorOffsets(), equalTo(false)); + assertThat(fieldType.storeTermVectorPositions(), equalTo(false)); + assertThat(fieldType.storeTermVectorPayloads(), equalTo(false)); + assertEquals(DocValuesType.NONE, fieldType.docValuesType()); + + BytesRef encodedVersion = encodeVersion("1.2.3").bytesRef; + byte[] first16bytes = Arrays.copyOfRange(encodedVersion.bytes, encodedVersion.offset, 16); + assertEquals(new BytesRef(first16bytes), fields[1].binaryValue()); + fieldType = fields[1].fieldType(); + assertEquals(1, fieldType.pointDimensionCount()); + assertEquals(1, fieldType.pointIndexDimensionCount()); + assertEquals(16, fieldType.pointNumBytes()); + assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE)); + assertEquals(DocValuesType.NONE, fieldType.docValuesType()); + + assertEquals("1.2.3", VersionEncoder.decodeVersion(fields[2].binaryValue())); + fieldType = fields[2].fieldType(); + assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE)); + assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType()); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.??.0*")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "?.1.0")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*-*")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "1.3.0+b*")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - } - - public void testPreReleaseFlag() throws IOException { - String indexName = setUpIndex("test"); - SearchResponse response = client().prepareSearch(indexName) - .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.isPreRelease", true))) - .get(); - assertEquals(1, response.getHits().getTotalHits().value); - } - - public void testMainVersionParts() throws IOException { - String indexName = setUpIndex("test"); - SearchResponse response = client().prepareSearch(indexName) - .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.major", 11))) - .get(); - assertEquals(1, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName) - .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.minor", 1))) - .get(); - assertEquals(3, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName) - .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.patch", 0))) - .get(); - assertEquals(6, response.getHits().getTotalHits().value); - } - - /** - * test that versions that are invalid under semver are still indexed and retrieveable, though they sort differently - */ - public void testStoreMalformed() throws Exception { - String indexName = "test_malformed"; - createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); - ensureGreen(indexName); - - client().prepareIndex(indexName) - .setId("1") - .setSource(jsonBuilder().startObject().field("version", "1.invalid.0").endObject()) - .get(); - client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "2.2.0").endObject()).get(); - client().prepareIndex(indexName) - .setId("3") - .setSource(jsonBuilder().startObject().field("version", "2.2.0-badchar!").endObject()) - .get(); - client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "").endObject()).get(); - client().admin().indices().prepareRefresh(indexName).get(); - - SearchResponse response = client().prepareSearch(indexName).addDocValueField("version").get(); - assertEquals(4, response.getHits().getTotalHits().value); - assertEquals("1", response.getHits().getAt(0).getId()); - assertEquals("1.invalid.0", response.getHits().getAt(0).field("version").getValue()); - - assertEquals("2", response.getHits().getAt(1).getId()); - assertEquals("2.2.0", response.getHits().getAt(1).field("version").getValue()); - - assertEquals("3", response.getHits().getAt(2).getId()); - assertEquals("2.2.0-badchar!", response.getHits().getAt(2).field("version").getValue()); - - assertEquals("4", response.getHits().getAt(3).getId()); - assertEquals("", response.getHits().getAt(3).field("version").getValue()); - - // exact match for malformed term - response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.invalid.0")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "2.2.0-badchar!")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - // also should appear in terms aggs - response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version")).get(); - Terms terms = response.getAggregations().get("myterms"); - List buckets = terms.getBuckets(); - - assertEquals(4, buckets.size()); - assertEquals("2.2.0", buckets.get(0).getKey()); - assertEquals("", buckets.get(1).getKey()); - assertEquals("1.invalid.0", buckets.get(2).getKey()); - assertEquals("2.2.0-badchar!", buckets.get(3).getKey()); - - // invalid values should sort after all valid ones - response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC).get(); - assertEquals(4, response.getHits().getTotalHits().value); - SearchHit[] hits = response.getHits().getHits(); - assertEquals("2.2.0", hits[0].getSortValues()[0]); - assertEquals("", hits[1].getSortValues()[0]); - assertEquals("1.invalid.0", hits[2].getSortValues()[0]); - assertEquals("2.2.0-badchar!", hits[3].getSortValues()[0]); - - // ranges can include them, but they are sorted last - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("3.0.0")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("3.0.0")).get(); - assertEquals(3, response.getHits().getTotalHits().value); - - // using the empty string as lower bound should return all "invalid" versions - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("")).get(); - assertEquals(3, response.getHits().getTotalHits().value); } - public void testAggs() throws Exception { - String indexName = "test_aggs"; - createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); - ensureGreen(indexName); + public void testParsesNestedEmptyObjectStrict() throws IOException { + DocumentMapper defaultMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); - client().prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "1.0").endObject()).get(); - client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "1.3.0").endObject()).get(); - client().prepareIndex(indexName) - .setId("3") - .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha").endObject()) - .get(); - client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); - client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "3.11.5").endObject()).get(); - client().admin().indices().prepareRefresh(indexName).get(); - - // terms aggs - SearchResponse response = client().prepareSearch(indexName) - .addAggregation(AggregationBuilders.terms("myterms").field("version")) - .get(); - Terms terms = response.getAggregations().get("myterms"); - List buckets = terms.getBuckets(); - - assertEquals(5, buckets.size()); - assertEquals("1.0", buckets.get(0).getKey()); - assertEquals("1.3.0", buckets.get(1).getKey()); - assertEquals("2.1.0-alpha", buckets.get(2).getKey()); - assertEquals("2.1.0", buckets.get(3).getKey()); - assertEquals("3.11.5", buckets.get(4).getKey()); - - // test terms aggs on version parts - - response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.major")).get(); - terms = response.getAggregations().get("myterms"); - buckets = terms.getBuckets(); - assertEquals(3, buckets.size()); - assertEquals(1L, buckets.get(0).getKey()); - assertEquals(2L, buckets.get(0).getDocCount()); - assertEquals(2L, buckets.get(1).getKey()); - assertEquals(2L, buckets.get(1).getDocCount()); - assertEquals(3L, buckets.get(2).getKey()); - assertEquals(1L, buckets.get(2).getDocCount()); - - response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.minor")).get(); - terms = response.getAggregations().get("myterms"); - buckets = terms.getBuckets(); - assertEquals(4, buckets.size()); - assertEquals(1L, buckets.get(0).getKey()); - assertEquals(2L, buckets.get(0).getDocCount()); - assertEquals(0L, buckets.get(1).getKey()); - assertEquals(1L, buckets.get(1).getDocCount()); - assertEquals(3L, buckets.get(2).getKey()); - assertEquals(1L, buckets.get(2).getDocCount()); - assertEquals(11L, buckets.get(3).getKey()); - assertEquals(1L, buckets.get(3).getDocCount()); - - response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.patch")).get(); - terms = response.getAggregations().get("myterms"); - buckets = terms.getBuckets(); - assertEquals(2, buckets.size()); - assertEquals(0L, buckets.get(0).getKey()); - assertEquals(3L, buckets.get(0).getDocCount()); - assertEquals(5L, buckets.get(1).getKey()); - assertEquals(1L, buckets.get(1).getDocCount()); - - // cardinality - response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.cardinality("myterms").field("version")).get(); - Cardinality card = response.getAggregations().get("myterms"); - assertEquals(5, card.getValue()); - - // string stats - response = client().prepareSearch(indexName) - .addAggregation(AnalyticsAggregationBuilders.stringStats("stats").field("version")) - .get(); - InternalStringStats stats = response.getAggregations().get("stats"); - assertEquals(3, stats.getMinLength()); - assertEquals(11, stats.getMaxLength()); + BytesReference source = BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().startObject("field").endObject().endObject() + ); + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> defaultMapper.parse(new SourceToParse("test", "1", source, XContentType.JSON)) + ); + assertEquals( + "failed to parse field [field] of type [version] in document with id '1'. " + "Preview of field's value: '{}'", + ex.getMessage() + ); } - public void testNullValue() throws Exception { - String indexName = "test_nullvalue"; - createIndex( - indexName, - Settings.builder().put("index.number_of_shards", 1).build(), - "_doc", - "version", - "type=version,null_value=0.0.0" + public void testFailsParsingNestedList() throws IOException { + DocumentMapper defaultMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + BytesReference source = BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startArray("field") + .startObject() + .startArray("array_name") + .value("inner_field_first") + .value("inner_field_second") + .endArray() + .endObject() + .endArray() + .endObject() + ); + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> defaultMapper.parse(new SourceToParse("test", "1", source, XContentType.JSON)) + ); + assertEquals( + "failed to parse field [field] of type [version] in document with id '1'. " + + "Preview of field's value: '{array_name=[inner_field_first, inner_field_second]}'", + ex.getMessage() ); - ensureGreen(indexName); - - client().prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "2.2.0").endObject()).get(); - client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().nullField("version").endObject()).get(); - client().admin().indices().prepareRefresh(indexName).get(); - - SearchResponse response = client().prepareSearch(indexName).addDocValueField("version").addSort("version", SortOrder.ASC).get(); - assertEquals(2, response.getHits().getTotalHits().value); - assertEquals("2", response.getHits().getAt(0).getId()); - assertEquals("0.0.0", response.getHits().getAt(0).field("version").getValue()); - - assertEquals("1", response.getHits().getAt(1).getId()); - assertEquals("2.2.0", response.getHits().getAt(1).field("version").getValue()); - - // range - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("3.0.0")).get(); - assertEquals(2, response.getHits().getTotalHits().value); } - public void testMultiValues() throws Exception { - String indexName = "test_multi"; - createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); - ensureGreen(indexName); - - client().prepareIndex(indexName) - .setId("1") - .setSource(jsonBuilder().startObject().array("version", "1.0.0", "3.0.0").endObject()) - .get(); - client().prepareIndex(indexName) - .setId("2") - .setSource(jsonBuilder().startObject().array("version", "2.0.0", "4.alpha.0").endObject()) - .get(); - client().prepareIndex(indexName) - .setId("3") - .setSource(jsonBuilder().startObject().array("version", "2.1.0", "2.2.0", "5.99.0").endObject()) - .get(); - client().admin().indices().prepareRefresh(indexName).get(); - - SearchResponse response = client().prepareSearch(indexName).addSort("version", SortOrder.ASC).get(); - assertEquals(3, response.getHits().getTotalHits().value); - assertEquals("1", response.getHits().getAt(0).getId()); - assertEquals("2", response.getHits().getAt(1).getId()); - assertEquals("3", response.getHits().getAt(2).getId()); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "3.0.0")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - assertEquals("1", response.getHits().getAt(0).getId()); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "4.alpha.0")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - assertEquals("2", response.getHits().getAt(0).getId()); - - // range - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("1.5.0")).get(); - assertEquals(1, response.getHits().getTotalHits().value); + public void testFetchSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.5.0")).get(); - assertEquals(3, response.getHits().getTotalHits().value); + VersionStringFieldMapper mapper = new VersionStringFieldMapper.Builder("field").build(context); + assertEquals(List.of("value"), fetchSourceValue(mapper, "value")); + assertEquals(List.of("42"), fetchSourceValue(mapper, 42L)); + assertEquals(List.of("true"), fetchSourceValue(mapper, true)); - response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("5.0.0").to("6.0.0")).get(); - assertEquals(1, response.getHits().getTotalHits().value); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> fetchSourceValue(mapper, "value", "format")); + assertEquals("Field [field] of type [version] doesn't support formats.", e.getMessage()); } } diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java new file mode 100644 index 0000000000000..cf03e4930556e --- /dev/null +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java @@ -0,0 +1,536 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; +import org.elasticsearch.search.aggregations.metrics.Cardinality; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.analytics.AnalyticsAggregationBuilders; +import org.elasticsearch.xpack.analytics.AnalyticsPlugin; +import org.elasticsearch.xpack.analytics.stringstats.InternalStringStats; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +public class VersionStringFieldTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return pluginList(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class, AnalyticsPlugin.class); + } + + public String setUpIndex(String indexName) throws IOException { + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); + ensureGreen(indexName); + + client().prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "11.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "1.0.0").endObject()).get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("4") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) + .get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("6").setSource(jsonBuilder().startObject().field("version", "21.11.0").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + return indexName; + } + + public void testExactQueries() throws Exception { + String indexName = "test"; + setUpIndex(indexName); + + // match + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", ("1.0.0"))).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.4.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.3.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.3.0+build.1234567")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + // term + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.0.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.4.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.3.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.3.0+build.1234567")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + // terms + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termsQuery("version", "1.0.0", "1.3.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termsQuery("version", "1.4.0", "1.3.0+build.1234567")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + // phrase query (just for keyword compatibility) + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchPhraseQuery("version", "2.1.0-alpha.beta")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + } + + public void testRangeQueries() throws Exception { + String indexName = setUpIndex("test"); + SearchResponse response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.rangeQuery("version").from("1.0.0").to("3.0.0")) + .get(); + assertEquals(4, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.1.0").to("3.0.0")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.rangeQuery("version").from("0.1.0").to("2.1.0-alpha.beta")) + .get(); + assertEquals(3, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("2.1.0").to("3.0.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("3.0.0").to("4.0.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + + // ranges excluding edges + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.0.0", false).to("3.0.0")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.0.0").to("2.1.0", false)).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + // open ranges + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.4.0")).get(); + assertEquals(4, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("1.4.0")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + } + + public void testPrefixQuery() throws IOException { + String indexName = setUpIndex("test"); + // prefix + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "1")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2.1")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2.1.0-")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "1.3.0+b")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.1")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.11")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + } + + public void testSort() throws IOException { + String indexName = setUpIndex("test"); + + // sort based on version field + SearchResponse response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.matchAllQuery()) + .addSort("version", SortOrder.DESC) + .get(); + assertEquals(6, response.getHits().getTotalHits().value); + SearchHit[] hits = response.getHits().getHits(); + assertEquals("21.11.0", hits[0].getSortValues()[0]); + assertEquals("11.1.0", hits[1].getSortValues()[0]); + assertEquals("2.1.0", hits[2].getSortValues()[0]); + assertEquals("2.1.0-alpha.beta", hits[3].getSortValues()[0]); + assertEquals("1.3.0+build.1234567", hits[4].getSortValues()[0]); + assertEquals("1.0.0", hits[5].getSortValues()[0]); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC).get(); + assertEquals(6, response.getHits().getTotalHits().value); + hits = response.getHits().getHits(); + assertEquals("1.0.0", hits[0].getSortValues()[0]); + assertEquals("1.3.0+build.1234567", hits[1].getSortValues()[0]); + assertEquals("2.1.0-alpha.beta", hits[2].getSortValues()[0]); + assertEquals("2.1.0", hits[3].getSortValues()[0]); + assertEquals("11.1.0", hits[4].getSortValues()[0]); + assertEquals("21.11.0", hits[5].getSortValues()[0]); + } + + public void testRegexQuery() throws Exception { + String indexName = "test_regex"; + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); + ensureGreen(indexName); + + client().prepareIndex(indexName) + .setId("1") + .setSource(jsonBuilder().startObject().field("version", "1.0.0alpha2.1.0-rc.1").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("2") + .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) + .get(); + client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "2.*0")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("2.1.0", response.getHits().getHits()[0].getSourceAsMap().get("version")); + assertEquals("2.33.0", response.getHits().getHits()[1].getSourceAsMap().get("version")); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "<0-10>.<0-10>.*al.*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version")); + assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[1].getSourceAsMap().get("version")); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", "1.[0-9].[0-9].*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version")); + assertEquals("1.3.0+build.1234567", response.getHits().getHits()[1].getSourceAsMap().get("version")); + } + + public void testFuzzyQuery() throws Exception { + String indexName = "test_fuzzy"; + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); + ensureGreen(indexName); + + client().prepareIndex(indexName) + .setId("1") + .setSource(jsonBuilder().startObject().field("version", "1.0.0-alpha.2.1.0-rc.1").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("2") + .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) + .get(); + client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); + client().prepareIndex(indexName).setId("6").setSource(jsonBuilder().startObject().field("version", "2.a3.0").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.fuzzyQuery("version", "2.3.0")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + assertEquals("2.1.0", response.getHits().getHits()[0].getSourceAsMap().get("version")); + assertEquals("2.33.0", response.getHits().getHits()[1].getSourceAsMap().get("version")); + assertEquals("2.a3.0", response.getHits().getHits()[2].getSourceAsMap().get("version")); + } + + public void testWildcardQuery() throws Exception { + String indexName = "test_wildcard"; + createIndex( + indexName, + Settings.builder().put("index.number_of_shards", 1).build(), + "_doc", + "version", + "type=version", + "foo", + "type=keyword" + ); + ensureGreen(indexName); + + client().prepareIndex(indexName) + .setId("1") + .setSource(jsonBuilder().startObject().field("version", "1.0.0-alpha.2.1.0-rc.1").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("2") + .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) + .get(); + client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + + // wildcard + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*alpha*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*b*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*bet*")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "2.1*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "2.1.0-*")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.1.0-*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.1.0*")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.?.0*")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.??.0*")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "?.1.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*-*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "1.3.0+b*")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + } + + public void testPreReleaseFlag() throws IOException { + String indexName = setUpIndex("test"); + SearchResponse response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.isPreRelease", true))) + .get(); + assertEquals(1, response.getHits().getTotalHits().value); + } + + public void testMainVersionParts() throws IOException { + String indexName = setUpIndex("test"); + SearchResponse response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.major", 11))) + .get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.minor", 1))) + .get(); + assertEquals(3, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.patch", 0))) + .get(); + assertEquals(6, response.getHits().getTotalHits().value); + } + + /** + * test that versions that are invalid under semver are still indexed and retrieveable, though they sort differently + */ + public void testStoreMalformed() throws Exception { + String indexName = "test_malformed"; + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); + ensureGreen(indexName); + + client().prepareIndex(indexName) + .setId("1") + .setSource(jsonBuilder().startObject().field("version", "1.invalid.0").endObject()) + .get(); + client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "2.2.0").endObject()).get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.2.0-badchar!").endObject()) + .get(); + client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + + SearchResponse response = client().prepareSearch(indexName).addDocValueField("version").get(); + assertEquals(4, response.getHits().getTotalHits().value); + assertEquals("1", response.getHits().getAt(0).getId()); + assertEquals("1.invalid.0", response.getHits().getAt(0).field("version").getValue()); + + assertEquals("2", response.getHits().getAt(1).getId()); + assertEquals("2.2.0", response.getHits().getAt(1).field("version").getValue()); + + assertEquals("3", response.getHits().getAt(2).getId()); + assertEquals("2.2.0-badchar!", response.getHits().getAt(2).field("version").getValue()); + + assertEquals("4", response.getHits().getAt(3).getId()); + assertEquals("", response.getHits().getAt(3).field("version").getValue()); + + // exact match for malformed term + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.invalid.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "2.2.0-badchar!")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + // also should appear in terms aggs + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version")).get(); + Terms terms = response.getAggregations().get("myterms"); + List buckets = terms.getBuckets(); + + assertEquals(4, buckets.size()); + assertEquals("2.2.0", buckets.get(0).getKey()); + assertEquals("", buckets.get(1).getKey()); + assertEquals("1.invalid.0", buckets.get(2).getKey()); + assertEquals("2.2.0-badchar!", buckets.get(3).getKey()); + + // invalid values should sort after all valid ones + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC).get(); + assertEquals(4, response.getHits().getTotalHits().value); + SearchHit[] hits = response.getHits().getHits(); + assertEquals("2.2.0", hits[0].getSortValues()[0]); + assertEquals("", hits[1].getSortValues()[0]); + assertEquals("1.invalid.0", hits[2].getSortValues()[0]); + assertEquals("2.2.0-badchar!", hits[3].getSortValues()[0]); + + // ranges can include them, but they are sorted last + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("3.0.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("3.0.0")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + // using the empty string as lower bound should return all "invalid" versions + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + } + + public void testAggs() throws Exception { + String indexName = "test_aggs"; + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); + ensureGreen(indexName); + + client().prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "1.0").endObject()).get(); + client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "1.3.0").endObject()).get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha").endObject()) + .get(); + client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "3.11.5").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + + // terms aggs + SearchResponse response = client().prepareSearch(indexName) + .addAggregation(AggregationBuilders.terms("myterms").field("version")) + .get(); + Terms terms = response.getAggregations().get("myterms"); + List buckets = terms.getBuckets(); + + assertEquals(5, buckets.size()); + assertEquals("1.0", buckets.get(0).getKey()); + assertEquals("1.3.0", buckets.get(1).getKey()); + assertEquals("2.1.0-alpha", buckets.get(2).getKey()); + assertEquals("2.1.0", buckets.get(3).getKey()); + assertEquals("3.11.5", buckets.get(4).getKey()); + + // test terms aggs on version parts + + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.major")).get(); + terms = response.getAggregations().get("myterms"); + buckets = terms.getBuckets(); + assertEquals(3, buckets.size()); + assertEquals(1L, buckets.get(0).getKey()); + assertEquals(2L, buckets.get(0).getDocCount()); + assertEquals(2L, buckets.get(1).getKey()); + assertEquals(2L, buckets.get(1).getDocCount()); + assertEquals(3L, buckets.get(2).getKey()); + assertEquals(1L, buckets.get(2).getDocCount()); + + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.minor")).get(); + terms = response.getAggregations().get("myterms"); + buckets = terms.getBuckets(); + assertEquals(4, buckets.size()); + assertEquals(1L, buckets.get(0).getKey()); + assertEquals(2L, buckets.get(0).getDocCount()); + assertEquals(0L, buckets.get(1).getKey()); + assertEquals(1L, buckets.get(1).getDocCount()); + assertEquals(3L, buckets.get(2).getKey()); + assertEquals(1L, buckets.get(2).getDocCount()); + assertEquals(11L, buckets.get(3).getKey()); + assertEquals(1L, buckets.get(3).getDocCount()); + + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.patch")).get(); + terms = response.getAggregations().get("myterms"); + buckets = terms.getBuckets(); + assertEquals(2, buckets.size()); + assertEquals(0L, buckets.get(0).getKey()); + assertEquals(3L, buckets.get(0).getDocCount()); + assertEquals(5L, buckets.get(1).getKey()); + assertEquals(1L, buckets.get(1).getDocCount()); + + // cardinality + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.cardinality("myterms").field("version")).get(); + Cardinality card = response.getAggregations().get("myterms"); + assertEquals(5, card.getValue()); + + // string stats + response = client().prepareSearch(indexName) + .addAggregation(AnalyticsAggregationBuilders.stringStats("stats").field("version")) + .get(); + InternalStringStats stats = response.getAggregations().get("stats"); + assertEquals(3, stats.getMinLength()); + assertEquals(11, stats.getMaxLength()); + } + + public void testMultiValues() throws Exception { + String indexName = "test_multi"; + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); + ensureGreen(indexName); + + client().prepareIndex(indexName) + .setId("1") + .setSource(jsonBuilder().startObject().array("version", "1.0.0", "3.0.0").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("2") + .setSource(jsonBuilder().startObject().array("version", "2.0.0", "4.alpha.0").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().array("version", "2.1.0", "2.2.0", "5.99.0").endObject()) + .get(); + client().admin().indices().prepareRefresh(indexName).get(); + + SearchResponse response = client().prepareSearch(indexName).addSort("version", SortOrder.ASC).get(); + assertEquals(3, response.getHits().getTotalHits().value); + assertEquals("1", response.getHits().getAt(0).getId()); + assertEquals("2", response.getHits().getAt(1).getId()); + assertEquals("3", response.getHits().getAt(2).getId()); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "3.0.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("1", response.getHits().getAt(0).getId()); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "4.alpha.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("2", response.getHits().getAt(0).getId()); + + // range + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("1.5.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.5.0")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("5.0.0").to("6.0.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + } +} From eaa1ef8dd3ba0a8f4cd7ec646bf3ebecd726c801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 26 Aug 2020 12:26:11 +0200 Subject: [PATCH 16/28] Change range query validation part --- .../xpack/versionfield/VersionStringFieldMapper.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index d73ecb840c529..c5d4cdc9f0a67 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -18,6 +18,7 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.MultiTermQuery; @@ -334,12 +335,11 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower // point query on the 16byte prefix Query pointPrefixQuery = BinaryPoint.newRangeQuery(name(), lowerBytes, upperBytes); - ValidationOnSortedDv validationQuery = new ValidationOnSortedDv(name(), lower, upper, includeLower, includeUpper); - + Query validationQuery = SortedSetDocValuesField.newSlowRangeQuery(name(), lower, upper, includeLower, includeUpper); BooleanQuery.Builder qBuilder = new BooleanQuery.Builder(); - qBuilder.add(pointPrefixQuery, Occur.MUST); - qBuilder.add(validationQuery, Occur.MUST); - return qBuilder.build(); + qBuilder.add(pointPrefixQuery, Occur.FILTER); + qBuilder.add(validationQuery, Occur.FILTER); + return new ConstantScoreQuery(qBuilder.build()); } } From 82a40abaaade4f96bf5bc84b5b37591fa544f770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 31 Aug 2020 16:06:35 +0200 Subject: [PATCH 17/28] Small improvement to range query using points --- .../versionfield/VersionStringFieldMapper.java | 13 +++++++++---- .../xpack/versionfield/VersionStringFieldTests.java | 4 ++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index c5d4cdc9f0a67..5b2dfa7f5d1e5 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -329,16 +329,21 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower failIfNotIndexed(); BytesRef lower = lowerTerm == null ? null : indexedValueForSearch(lowerTerm); BytesRef upper = upperTerm == null ? null : indexedValueForSearch(upperTerm); - byte[] lowerBytes = lower == null ? MIN_VALUE : Arrays.copyOfRange(lower.bytes, lower.offset, 16); - byte[] upperBytes = upper == null ? MAX_VALUE : Arrays.copyOfRange(upper.bytes, upper.offset, 16); + byte[] lowerBytes = lower == null ? MIN_VALUE : Arrays.copyOfRange(lower.bytes, lower.offset, lower.offset + 16); + byte[] upperBytes = upper == null ? MAX_VALUE : Arrays.copyOfRange(upper.bytes, upper.offset, upper.offset + 16); // point query on the 16byte prefix Query pointPrefixQuery = BinaryPoint.newRangeQuery(name(), lowerBytes, upperBytes); - Query validationQuery = SortedSetDocValuesField.newSlowRangeQuery(name(), lower, upper, includeLower, includeUpper); BooleanQuery.Builder qBuilder = new BooleanQuery.Builder(); qBuilder.add(pointPrefixQuery, Occur.FILTER); - qBuilder.add(validationQuery, Occur.FILTER); + if (includeUpper == false + || includeLower == false + || (lower != null && lower.length > 16) + || (upper != null && upper.length > 16)) { + Query validationQuery = SortedSetDocValuesField.newSlowRangeQuery(name(), lower, upper, includeLower, includeUpper); + qBuilder.add(validationQuery, Occur.FILTER); + } return new ConstantScoreQuery(qBuilder.build()); } } diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java index cf03e4930556e..b46e117514bbe 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java @@ -106,6 +106,10 @@ public void testRangeQueries() throws Exception { assertEquals(1, response.getHits().getTotalHits().value); response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("3.0.0").to("4.0.0")).get(); assertEquals(0, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.rangeQuery("version").from("1.3.0+build.1234569").to("3.0.0")) + .get(); + assertEquals(2, response.getHits().getTotalHits().value); // ranges excluding edges response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.0.0", false).to("3.0.0")).get(); From e4b0728c0bae90c936bb1dd926d2ec4c6f247b81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 2 Sep 2020 18:22:31 +0200 Subject: [PATCH 18/28] Add helper methods to versions ScriptDocValues to help with version part extraction --- .../test/versionfield/20_scripts.yml | 102 ++++++++++++++++++ .../xpack/versionfield/VersionEncoder.java | 8 +- .../versionfield/VersionScriptDocValues.java | 60 +++++++++++ .../xpack/versionfield/whitelist.txt | 5 + 4 files changed, 171 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml index 60de6a8c8da65..efce0c065b02e 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml @@ -52,3 +52,105 @@ setup: - match: { hits.hits.0._source.version: "3.1.0" } - match: { hits.hits.1._source.version: "1.1.12" } - match: { hits.hits.2._source.version: "2.0.0-beta" } + +--- +"Filter script for illegal versions": + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "test_index", "_id" : "4" } }' + - '{"version": "a123.2.2-beta" }' + + - do: + search: + index: test_index + body: + query: { "bool" : { "filter" : [{ "script" : { "script" : {"source": "doc['version'].isLegalSemver() == false"}}}] }} + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._source.version: "a123.2.2-beta" } + + - do: + search: + index: test_index + body: + query: { "bool" : { "filter" : [{ "script" : { "script" : {"source": "doc['version'].isLegalSemver()"}}}] }} + + - match: { hits.total.value: 3 } + - match: { hits.hits.0._source.version: "1.1.12" } + - match: { hits.hits.1._source.version: "2.0.0-beta" } + - match: { hits.hits.2._source.version: "3.1.0" } + +--- +"Filter script for pre-release versions": + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "test_index", "_id" : "4" } }' + - '{"version": "a123.2.2-beta" }' + + - do: + search: + index: test_index + body: + query: { "bool" : { "filter" : [{ "script" : { "script" : {"source": "doc['version'].isPreRelease()"}}}] }} + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._source.version: "2.0.0-beta" } + +--- +"Aggregate using script on major, minor, patch versions": + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "test_index", "_id" : "4" } }' + - '{"version": "3" }' + - '{ "index" : { "_index" : "test_index", "_id" : "5" } }' + - '{"version": "3.1" }' + - '{ "index" : { "_index" : "test_index", "_id" : "6" } }' + - '{"version": "illegal3.1" }' + + - do: + search: + index: test_index + body: + size: 0 + aggs: { "majors": { "terms": { "script": { "source": "doc['version'].getMajor()", "lang": "painless"}}}} + + - length: { aggregations.majors.buckets: 3 } + - match: { aggregations.majors.buckets.0.key: "3" } + - match: { aggregations.majors.buckets.0.doc_count: 3 } + - match: { aggregations.majors.buckets.1.key: "1" } + - match: { aggregations.majors.buckets.1.doc_count: 1 } + - match: { aggregations.majors.buckets.2.key: "2" } + - match: { aggregations.majors.buckets.2.doc_count: 1 } + + - do: + search: + index: test_index + body: + size: 0 + aggs: { "majors": { "terms": { "script": { "source": "doc['version'].getMinor()", "lang": "painless"}}}} + + - length: { aggregations.majors.buckets: 2 } + - match: { aggregations.majors.buckets.0.key: "1" } + - match: { aggregations.majors.buckets.0.doc_count: 3 } + - match: { aggregations.majors.buckets.1.key: "0" } + - match: { aggregations.majors.buckets.1.doc_count: 1 } + + - do: + search: + index: test_index + body: + size: 0 + aggs: { "majors": { "terms": { "script": { "source": "doc['version'].getPatch()", "lang": "painless"}}}} + + - length: { aggregations.majors.buckets: 2 } + - match: { aggregations.majors.buckets.0.key: "0" } + - match: { aggregations.majors.buckets.0.doc_count: 2 } + - match: { aggregations.majors.buckets.1.key: "12" } + - match: { aggregations.majors.buckets.1.doc_count: 1 } diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java index 8ec27305d5589..03fd534571a0b 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java @@ -67,7 +67,7 @@ public static EncodedVersion encodeVersion(String versionString) { // we are sure to remove when decoding versionString = ENCODED_EMPTY_STRING; } - return new EncodedVersion(new BytesRef(versionString), false, true, 0, 0, 0); + return new EncodedVersion(new BytesRef(versionString), false, true, null, null, null); } BytesRefBuilder encodedBytes = new BytesRefBuilder(); @@ -161,7 +161,7 @@ public static String decodeVersion(BytesRef version) { return new String(result, 0, resultPos, StandardCharsets.UTF_8); } - private static boolean legalVersionString(VersionParts versionParts) { + static boolean legalVersionString(VersionParts versionParts) { boolean legalMainVersion = LEGAL_MAIN_VERSION_SEMVER.matcher(versionParts.mainVersion).matches(); boolean legalPreRelease = true; if (versionParts.preRelease != null) { @@ -194,7 +194,7 @@ private EncodedVersion(BytesRef bytesRef, boolean isLegal, boolean isPreRelease, } } - private static class VersionParts { + static class VersionParts { final String mainVersion; final String preRelease; final String buildSuffix; @@ -205,7 +205,7 @@ private VersionParts(String mainVersion, String preRelease, String buildSuffix) this.buildSuffix = buildSuffix; } - private static VersionParts ofVersion(String versionString) { + static VersionParts ofVersion(String versionString) { String buildSuffix = extractSuffix(versionString, BUILD_SEPARATOR); if (buildSuffix != null) { versionString = versionString.substring(0, versionString.length() - buildSuffix.length()); diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java index f74027db634cc..faaddcc414786 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java @@ -9,6 +9,7 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.xpack.versionfield.VersionEncoder.VersionParts; import java.io.IOException; @@ -54,4 +55,63 @@ public String get(int index) { public int size() { return count; } + + public boolean isLegalSemver() { + return VersionEncoder.legalVersionString(VersionParts.ofVersion(getValue())); + } + + public boolean isPreRelease() { + VersionParts parts = VersionParts.ofVersion(getValue()); + return (parts.preRelease != null) && VersionEncoder.legalVersionString(parts); + } + + public Integer getMajor() { + VersionParts parts = VersionParts.ofVersion(getValue()); + if (VersionEncoder.legalVersionString(parts) && parts.mainVersion != null) { + int firstDot = parts.mainVersion.indexOf("."); + if (firstDot > 0) { + return Integer.valueOf(parts.mainVersion.substring(0, firstDot)); + } else { + return Integer.valueOf(parts.mainVersion); + } + } + return null; + } + + public Integer getMinor() { + VersionParts parts = VersionParts.ofVersion(getValue()); + Integer rc = null; + if (VersionEncoder.legalVersionString(parts) && parts.mainVersion != null) { + int firstDot = parts.mainVersion.indexOf("."); + if (firstDot > 0) { + int secondDot = parts.mainVersion.indexOf(".", firstDot + 1); + if (secondDot > 0) { + rc = Integer.valueOf(parts.mainVersion.substring(firstDot + 1, secondDot)); + } else { + rc = Integer.valueOf(parts.mainVersion.substring(firstDot + 1)); + } + } + } + return rc; + } + + public Integer getPatch() { + VersionParts parts = VersionParts.ofVersion(getValue()); + Integer rc = null; + if (VersionEncoder.legalVersionString(parts) && parts.mainVersion != null) { + int firstDot = parts.mainVersion.indexOf("."); + if (firstDot > 0) { + int secondDot = parts.mainVersion.indexOf(".", firstDot + 1); + if (secondDot > 0) { + int thirdDot = parts.mainVersion.indexOf(".", secondDot + 1); + if (thirdDot > 0) { + rc = Integer.valueOf(parts.mainVersion.substring(secondDot + 1, thirdDot)); + } else { + rc = Integer.valueOf(parts.mainVersion.substring(secondDot + 1)); + } + } + } + } + return rc; + } } diff --git a/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt b/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt index eb47eae53a158..33c1f842eb74d 100644 --- a/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt +++ b/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt @@ -2,4 +2,9 @@ class org.elasticsearch.xpack.versionfield.VersionScriptDocValues { String get(int) String getValue() + boolean isLegalSemver() + boolean isPreRelease() + Integer getMajor() + Integer getMinor() + Integer getPatch() } \ No newline at end of file From 6c3266e055fd91ba16d22ecb72b9a5604b1724d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 3 Sep 2020 16:59:58 +0200 Subject: [PATCH 19/28] Removing points and subfields --- .../VersionStringFieldMapper.java | 84 +++---------------- .../VersionStringFieldMapperTests.java | 18 +--- .../versionfield/VersionStringFieldTests.java | 59 ------------- 3 files changed, 15 insertions(+), 146 deletions(-) diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index ea4950e9503c5..d4e5f5e0d4433 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -6,24 +6,20 @@ package org.elasticsearch.xpack.versionfield; -import org.apache.lucene.document.BinaryPoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.RegexpQuery87; +import org.apache.lucene.search.RegexpQuery; +import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.ByteRunAutomaton; @@ -37,13 +33,10 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; -import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.NumberFieldMapper; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SourceValueFetcher; @@ -110,22 +103,12 @@ private VersionStringFieldType buildFieldType(BuilderContext context, FieldType @Override public VersionStringFieldMapper build(BuilderContext context) { FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); - BooleanFieldMapper.Builder preReleaseSubfield = new BooleanFieldMapper.Builder(name + ".isPreRelease"); - NumberType type = NumberType.INTEGER; - NumberFieldMapper.Builder majorVersionSubField = new NumberFieldMapper.Builder(name + ".major", type, false, false); - NumberFieldMapper.Builder minorVersionSubField = new NumberFieldMapper.Builder(name + ".minor", type, false, false); - NumberFieldMapper.Builder patchVersionSubField = new NumberFieldMapper.Builder(name + ".patch", type, false, false); - return new VersionStringFieldMapper( name, fieldtype, buildFieldType(context, fieldtype), multiFieldsBuilder.build(this, context), - copyTo.build(), - preReleaseSubfield.build(context), - majorVersionSubField.build(context), - minorVersionSubField.build(context), - patchVersionSubField.build(context) + copyTo.build() ); } @@ -190,7 +173,7 @@ public Query regexpQuery( ); } failIfNotIndexed(); - RegexpQuery87 query = new RegexpQuery87(new Term(name(), new BytesRef(value)), syntaxFlags, matchFlags, maxDeterminizedStates) { + RegexpQuery query = new RegexpQuery(new Term(name(), new BytesRef(value)), syntaxFlags, matchFlags, maxDeterminizedStates) { @Override protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException { @@ -331,48 +314,28 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower failIfNotIndexed(); BytesRef lower = lowerTerm == null ? null : indexedValueForSearch(lowerTerm); BytesRef upper = upperTerm == null ? null : indexedValueForSearch(upperTerm); - byte[] lowerBytes = lower == null ? MIN_VALUE : Arrays.copyOfRange(lower.bytes, lower.offset, lower.offset + 16); - byte[] upperBytes = upper == null ? MAX_VALUE : Arrays.copyOfRange(upper.bytes, upper.offset, upper.offset + 16); - - // point query on the 16byte prefix - Query pointPrefixQuery = BinaryPoint.newRangeQuery(name(), lowerBytes, upperBytes); - - BooleanQuery.Builder qBuilder = new BooleanQuery.Builder(); - qBuilder.add(pointPrefixQuery, Occur.FILTER); - if (includeUpper == false - || includeLower == false - || (lower != null && lower.length > 16) - || (upper != null && upper.length > 16)) { - Query validationQuery = SortedSetDocValuesField.newSlowRangeQuery(name(), lower, upper, includeLower, includeUpper); - qBuilder.add(validationQuery, Occur.FILTER); - } - return new ConstantScoreQuery(qBuilder.build()); + + return new TermRangeQuery( + name(), + lowerTerm == null ? null : lower, + upperTerm == null ? null : upper, + includeLower, + includeUpper + ); } } private final FieldType fieldType; - private BooleanFieldMapper prereleaseSubField; - private NumberFieldMapper majorVersionSubField; - private NumberFieldMapper minorVersionSubField; - private NumberFieldMapper patchVersionSubField; private VersionStringFieldMapper( String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, MultiFields multiFields, - CopyTo copyTo, - BooleanFieldMapper preReleaseMapper, - NumberFieldMapper majorVersionMapper, - NumberFieldMapper minorVersionMapper, - NumberFieldMapper patchVersionMapper + CopyTo copyTo ) { super(simpleName, mappedFieldType, multiFields, copyTo); this.fieldType = fieldType; - this.prereleaseSubField = preReleaseMapper; - this.majorVersionSubField = majorVersionMapper; - this.minorVersionSubField = minorVersionMapper; - this.patchVersionSubField = patchVersionMapper; } @Override @@ -425,33 +388,12 @@ protected void parseCreateField(ParseContext context) throws IOException { EncodedVersion encoding = encodeVersion(versionString); BytesRef encodedVersion = encoding.bytesRef; context.doc().add(new Field(fieldType().name(), encodedVersion, fieldType)); - // encode the first 16 bytes as points for efficient range query - byte[] first16bytes = Arrays.copyOfRange(encodedVersion.bytes, encodedVersion.offset, 16); - context.doc().add(new BinaryPoint(fieldType().name(), first16bytes)); context.doc().add(new SortedSetDocValuesField(fieldType().name(), encodedVersion)); - - // add additional information extracted from version string - context.doc().add(new Field(prereleaseSubField.name(), encoding.isPreRelease ? "T" : "F", BooleanFieldMapper.Defaults.FIELD_TYPE)); - context.doc().add(new SortedNumericDocValuesField(prereleaseSubField.name(), encoding.isPreRelease ? 1 : 0)); - - addVersionPartSubfield(context, majorVersionSubField.name(), encoding.major); - addVersionPartSubfield(context, minorVersionSubField.name(), encoding.minor); - addVersionPartSubfield(context, patchVersionSubField.name(), encoding.patch); - } - - private void addVersionPartSubfield(ParseContext context, String fieldName, Integer versionPart) { - if (versionPart != null) { - context.doc().addAll(NumberType.INTEGER.createFields(fieldName, versionPart, true, true, false)); - } } @Override public Iterator iterator() { List subIterators = new ArrayList<>(); - subIterators.add(prereleaseSubField); - subIterators.add(majorVersionSubField); - subIterators.add(minorVersionSubField); - subIterators.add(patchVersionSubField); @SuppressWarnings("unchecked") Iterator concat = Iterators.concat(super.iterator(), subIterators.iterator()); return concat; diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index caa44b645e3b3..73c121b83f8fd 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -10,7 +10,6 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; @@ -29,13 +28,10 @@ import org.elasticsearch.plugins.Plugin; import java.io.IOException; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; -import static org.elasticsearch.index.mapper.FieldMapperTestCase.fetchSourceValue; -import static org.elasticsearch.xpack.versionfield.VersionEncoder.encodeVersion; import static org.hamcrest.Matchers.equalTo; public class VersionStringFieldMapperTests extends MapperTestCase { @@ -65,7 +61,7 @@ public void testDefaults() throws Exception { ); IndexableField[] fields = doc.rootDoc().getFields("field"); - assertEquals(3, fields.length); + assertEquals(2, fields.length); assertEquals("1.2.3", VersionEncoder.decodeVersion(fields[0].binaryValue())); IndexableFieldType fieldType = fields[0].fieldType(); @@ -79,18 +75,8 @@ public void testDefaults() throws Exception { assertThat(fieldType.storeTermVectorPayloads(), equalTo(false)); assertEquals(DocValuesType.NONE, fieldType.docValuesType()); - BytesRef encodedVersion = encodeVersion("1.2.3").bytesRef; - byte[] first16bytes = Arrays.copyOfRange(encodedVersion.bytes, encodedVersion.offset, 16); - assertEquals(new BytesRef(first16bytes), fields[1].binaryValue()); + assertEquals("1.2.3", VersionEncoder.decodeVersion(fields[1].binaryValue())); fieldType = fields[1].fieldType(); - assertEquals(1, fieldType.pointDimensionCount()); - assertEquals(1, fieldType.pointIndexDimensionCount()); - assertEquals(16, fieldType.pointNumBytes()); - assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE)); - assertEquals(DocValuesType.NONE, fieldType.docValuesType()); - - assertEquals("1.2.3", VersionEncoder.decodeVersion(fields[2].binaryValue())); - fieldType = fields[2].fieldType(); assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE)); assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType()); diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java index b46e117514bbe..24ee554903df9 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java @@ -317,30 +317,6 @@ public void testWildcardQuery() throws Exception { assertEquals(1, response.getHits().getTotalHits().value); } - public void testPreReleaseFlag() throws IOException { - String indexName = setUpIndex("test"); - SearchResponse response = client().prepareSearch(indexName) - .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.isPreRelease", true))) - .get(); - assertEquals(1, response.getHits().getTotalHits().value); - } - - public void testMainVersionParts() throws IOException { - String indexName = setUpIndex("test"); - SearchResponse response = client().prepareSearch(indexName) - .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.major", 11))) - .get(); - assertEquals(1, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName) - .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.minor", 1))) - .get(); - assertEquals(3, response.getHits().getTotalHits().value); - response = client().prepareSearch(indexName) - .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.patch", 0))) - .get(); - assertEquals(6, response.getHits().getTotalHits().value); - } - /** * test that versions that are invalid under semver are still indexed and retrieveable, though they sort differently */ @@ -445,41 +421,6 @@ public void testAggs() throws Exception { assertEquals("2.1.0", buckets.get(3).getKey()); assertEquals("3.11.5", buckets.get(4).getKey()); - // test terms aggs on version parts - - response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.major")).get(); - terms = response.getAggregations().get("myterms"); - buckets = terms.getBuckets(); - assertEquals(3, buckets.size()); - assertEquals(1L, buckets.get(0).getKey()); - assertEquals(2L, buckets.get(0).getDocCount()); - assertEquals(2L, buckets.get(1).getKey()); - assertEquals(2L, buckets.get(1).getDocCount()); - assertEquals(3L, buckets.get(2).getKey()); - assertEquals(1L, buckets.get(2).getDocCount()); - - response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.minor")).get(); - terms = response.getAggregations().get("myterms"); - buckets = terms.getBuckets(); - assertEquals(4, buckets.size()); - assertEquals(1L, buckets.get(0).getKey()); - assertEquals(2L, buckets.get(0).getDocCount()); - assertEquals(0L, buckets.get(1).getKey()); - assertEquals(1L, buckets.get(1).getDocCount()); - assertEquals(3L, buckets.get(2).getKey()); - assertEquals(1L, buckets.get(2).getDocCount()); - assertEquals(11L, buckets.get(3).getKey()); - assertEquals(1L, buckets.get(3).getDocCount()); - - response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.patch")).get(); - terms = response.getAggregations().get("myterms"); - buckets = terms.getBuckets(); - assertEquals(2, buckets.size()); - assertEquals(0L, buckets.get(0).getKey()); - assertEquals(3L, buckets.get(0).getDocCount()); - assertEquals(5L, buckets.get(1).getKey()); - assertEquals(1L, buckets.get(1).getDocCount()); - // cardinality response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.cardinality("myterms").field("version")).get(); Cardinality card = response.getAggregations().get("myterms"); From b26a3f1864d7f1fa2e9af3e9fced4beb941ab2d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 8 Sep 2020 16:21:05 +0200 Subject: [PATCH 20/28] Add tests for new regex case-insensitivity option --- .../versionfield/VersionStringFieldTests.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java index 24ee554903df9..0d88276bbfa5d 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java @@ -219,6 +219,23 @@ public void testRegexQuery() throws Exception { assertEquals(2, response.getHits().getTotalHits().value); assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version")); assertEquals("1.3.0+build.1234567", response.getHits().getHits()[1].getSourceAsMap().get("version")); + + // test case sensitivity / insensitivity + response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", ".*alpha.*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version")); + assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[1].getSourceAsMap().get("version")); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.regexpQuery("version", ".*Alpha.*")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.regexpQuery("version", ".*Alpha.*").caseInsensitive(true)) + .get(); + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("1.0.0alpha2.1.0-rc.1", response.getHits().getHits()[0].getSourceAsMap().get("version")); + assertEquals("2.1.0-alpha.beta", response.getHits().getHits()[1].getSourceAsMap().get("version")); + } public void testFuzzyQuery() throws Exception { From c0c39bb74bb7af73c3a9eb102eb20b9f0f8778bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 9 Sep 2020 15:08:43 +0200 Subject: [PATCH 21/28] Adding docs --- docs/reference/mapping/types/keyword.asciidoc | 2 + docs/reference/mapping/types/version.asciidoc | 125 ++++++++++++++++++ .../test/versionfield/20_scripts.yml | 4 +- .../versionfield/VersionScriptDocValues.java | 2 +- .../xpack/versionfield/whitelist.txt | 2 +- 5 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 docs/reference/mapping/types/version.asciidoc diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc index ee539d19c03ca..4a63361e94437 100644 --- a/docs/reference/mapping/types/keyword.asciidoc +++ b/docs/reference/mapping/types/keyword.asciidoc @@ -129,3 +129,5 @@ The following parameters are accepted by `keyword` fields: include::constant-keyword.asciidoc[] include::wildcard.asciidoc[] + +include::version.asciidoc[] diff --git a/docs/reference/mapping/types/version.asciidoc b/docs/reference/mapping/types/version.asciidoc new file mode 100644 index 0000000000000..34b2b69add7ef --- /dev/null +++ b/docs/reference/mapping/types/version.asciidoc @@ -0,0 +1,125 @@ +[role="xpack"] +[testenv="basic"] +[discrete] +[[version-field-type]] +=== Version field type + +The `version` field type is a specialization of the `keyword` field for +handling software version values and to support specialized precedence +rules for them. Precedence is defined following the rules outlined by +https://semver.org/[Semantic Versioning], which for example means that +major, minor and patch version parts are sorted numerically (i.e. +"2.1.0" < "2.4.1" < "2.11.2") and pre-release versions are sorted before +release versiond (i.e. "1.0.0-alpha" < "1.0.0"). + +You index a `version` field as follows + +[source,console] +-------------------------------------------------- +PUT my-index-000001 +{ + "mappings": { + "properties": { + "my_version": { + "type": "version" + } + } + } +} + +-------------------------------------------------- + +The field offers the same search capabilities as a regular keyword field. It +can e.g. be searched for exact matches using `match` or `term` queries and +supports prefix and wildcard searches. The main benefit is that `range` queries +will honour Semver ordering, so a `range` query between "1.0.0" and "1.5.0" +will include versions of "1.2.3" but not "1.11.2" for example. Not that this +would be different when using a regular `keyword` field for indexing where ordering +is alphabetical. + +Software versions are expected to follow the +https://semver.org/[Semantic Versioning rules] schema and precedence rules with +the notable exception that more or less than three main version identifiers are +allowed (i.e. "1.2" or "1.2.3.4" qualify as valid versions while they wouldn't under +strict Semver rules). Version strings that are not valid under the Semver definition +(e.g. "1.2.alpha.4") can still be indexed and retrieved as exact matches, however they +will all appear _after_ any valid version with regular alphabetical ordering. + +[discrete] +[[version-params]] +==== Parameters for wildcard fields + +The following parameters are accepted by `version` fields: + +[horizontal] + +<>:: + + Metadata about the field. + +[discrete] +==== Limitations + +This field type isn't optimized for heavy wildcard, regex or fuzzy searches. While those +type of queries work in this field, you should consider using a regular `keyword` field if +you strongly rely on these kind of queries. + +==== Script support + +TBD: The `version` fields offers some specialized access to detailed information derived from +valid version strings like the Major, Minor or Patch release number, whether the version value +is valid according to Semver or if it is a pre-release version. This can be helpful when e.g. +filtering for only released versions or running aggregations on parts of the version. +The following query, for example, filters for released versions and groups them by Major version +using a `terms` aggregation: + +[source,console] +-------------------------------------------------- +POST my-index-000001/_search +{ + "query": { + "bool": { + "filter": [ + { + "script": { + "script": { + "source": "doc['my_version'].isPreRelease() == false" + } + } + } + ] + } + }, + "aggs": { + "group_major": { + "terms": { + "script": { "source": "doc['my_version'].getMajor()"}, + "order": { + "_key": "asc" + } + } + } + } +} + +-------------------------------------------------- +// TEST[continued] + +Functions available on via doc values in scripting are: + +[horizontal] + +isValid():: + Returns `true` if the field contains a version thats legal according to the Semantic Versioning rules + +isPreRelease():: + Returns `true` if the field contains a pre-release version + +getMajor():: + Returns the Major version for valid versions. + +getMinor():: + Returns the Minor version for valid versions. + +getPatch():: + Returns the Patch version for valid versions. \ No newline at end of file diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml index efce0c065b02e..9e3432ba1b67d 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml @@ -66,7 +66,7 @@ setup: search: index: test_index body: - query: { "bool" : { "filter" : [{ "script" : { "script" : {"source": "doc['version'].isLegalSemver() == false"}}}] }} + query: { "bool" : { "filter" : [{ "script" : { "script" : {"source": "doc['version'].isValid() == false"}}}] }} - match: { hits.total.value: 1 } - match: { hits.hits.0._source.version: "a123.2.2-beta" } @@ -75,7 +75,7 @@ setup: search: index: test_index body: - query: { "bool" : { "filter" : [{ "script" : { "script" : {"source": "doc['version'].isLegalSemver()"}}}] }} + query: { "bool" : { "filter" : [{ "script" : { "script" : {"source": "doc['version'].isValid()"}}}] }} - match: { hits.total.value: 3 } - match: { hits.hits.0._source.version: "1.1.12" } diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java index faaddcc414786..a55c301f8036c 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java @@ -56,7 +56,7 @@ public int size() { return count; } - public boolean isLegalSemver() { + public boolean isValid() { return VersionEncoder.legalVersionString(VersionParts.ofVersion(getValue())); } diff --git a/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt b/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt index 33c1f842eb74d..43aa5a4165bd8 100644 --- a/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt +++ b/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt @@ -2,7 +2,7 @@ class org.elasticsearch.xpack.versionfield.VersionScriptDocValues { String get(int) String getValue() - boolean isLegalSemver() + boolean isValid() boolean isPreRelease() Integer getMajor() Integer getMinor() From d56499e2dc25ccfd36ca04bcf2b2b41deac96f6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 14 Sep 2020 14:59:17 +0200 Subject: [PATCH 22/28] Adressing review comments --- docs/reference/mapping/types/version.asciidoc | 18 ++++++++++-------- .../xpack/versionfield/VersionFieldPlugin.java | 3 +-- .../versionfield/VersionScriptDocValues.java | 7 ++++--- .../versionfield/VersionStringFieldMapper.java | 15 +-------------- .../versionfield/VersionEncoderTests.java | 3 +++ 5 files changed, 19 insertions(+), 27 deletions(-) diff --git a/docs/reference/mapping/types/version.asciidoc b/docs/reference/mapping/types/version.asciidoc index 34b2b69add7ef..53c2bded7073f 100644 --- a/docs/reference/mapping/types/version.asciidoc +++ b/docs/reference/mapping/types/version.asciidoc @@ -33,7 +33,7 @@ The field offers the same search capabilities as a regular keyword field. It can e.g. be searched for exact matches using `match` or `term` queries and supports prefix and wildcard searches. The main benefit is that `range` queries will honour Semver ordering, so a `range` query between "1.0.0" and "1.5.0" -will include versions of "1.2.3" but not "1.11.2" for example. Not that this +will include versions of "1.2.3" but not "1.11.2" for example. Note that this would be different when using a regular `keyword` field for indexing where ordering is alphabetical. @@ -43,11 +43,13 @@ the notable exception that more or less than three main version identifiers are allowed (i.e. "1.2" or "1.2.3.4" qualify as valid versions while they wouldn't under strict Semver rules). Version strings that are not valid under the Semver definition (e.g. "1.2.alpha.4") can still be indexed and retrieved as exact matches, however they -will all appear _after_ any valid version with regular alphabetical ordering. +will all appear _after_ any valid version with regular alphabetical ordering. The empty +String "" is considered invalid and sorted after all valid versions, but before other +invalid ones. [discrete] [[version-params]] -==== Parameters for wildcard fields +==== Parameters for version fields The following parameters are accepted by `version` fields: @@ -66,7 +68,7 @@ you strongly rely on these kind of queries. ==== Script support -TBD: The `version` fields offers some specialized access to detailed information derived from +The `version` fields offers some specialized access to detailed information derived from valid version strings like the Major, Minor or Patch release number, whether the version value is valid according to Semver or if it is a pre-release version. This can be helpful when e.g. filtering for only released versions or running aggregations on parts of the version. @@ -114,12 +116,12 @@ isValid():: isPreRelease():: Returns `true` if the field contains a pre-release version - + getMajor():: - Returns the Major version for valid versions. + Returns an Integer value of the Major version if the version is valid, or null otherwise getMinor():: - Returns the Minor version for valid versions. + Returns an Integer value of the Minor version if the version is valid, or null otherwise. getPatch():: - Returns the Patch version for valid versions. \ No newline at end of file + Returns an Integer value of the Patch version if the version is valid, or null otherwise. \ No newline at end of file diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java index 227b47c5611a5..80506a920ef33 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java @@ -9,7 +9,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.DocValueFormat; @@ -17,7 +16,7 @@ import java.util.List; import java.util.Map; -public class VersionFieldPlugin extends Plugin implements ActionPlugin, MapperPlugin { +public class VersionFieldPlugin extends Plugin implements MapperPlugin { public VersionFieldPlugin(Settings settings) {} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java index a55c301f8036c..75a1abb2b1f0b 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java @@ -36,10 +36,11 @@ public void setNextDocId(int docId) throws IOException { public String getValue() { if (count == 0) { - return null; - } else { - return get(0); + throw new IllegalStateException( + "A document doesn't have a value for a field! " + "Use doc[].size()==0 to check if a document is missing a field!" + ); } + return get(0); } @Override diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index d4e5f5e0d4433..3d5550da1fc91 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -147,7 +147,6 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer + "fields please enable [index_prefixes]." ); } - failIfNotIndexed(); return wildcardQuery(value + "*", method, context); } @@ -156,7 +155,7 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer * automaton in the query will assume unencoded terms. We are running through all terms, decode them and * then run them through the automaton manually instead. This is not as efficient as intersecting the original * Terms with the compiled automaton, but we expect the number of distinct version terms indexed into this field - * to be low enough and the use of "rexexp" queries on this field rare enough to brute-force this + * to be low enough and the use of "regexp" queries on this field rare enough to brute-force this */ @Override public Query regexpQuery( @@ -172,7 +171,6 @@ public Query regexpQuery( "[regexp] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false." ); } - failIfNotIndexed(); RegexpQuery query = new RegexpQuery(new Term(name(), new BytesRef(value)), syntaxFlags, matchFlags, maxDeterminizedStates) { @Override @@ -219,7 +217,6 @@ public Query fuzzyQuery( "[fuzzy] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false." ); } - failIfNotIndexed(); return new FuzzyQuery( new Term(name(), (BytesRef) value), fuzziness.asDistance(BytesRefs.toString(value)), @@ -254,7 +251,6 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, Qu "[wildcard] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false." ); } - failIfNotIndexed(); VersionFieldWildcardQuery query = new VersionFieldWildcardQuery(new Term(name(), value)); QueryParsers.setRewriteMethod(query, method); @@ -277,7 +273,6 @@ protected BytesRef indexedValueForSearch(Object value) { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), VersionScriptDocValues::new, CoreValuesSourceType.BYTES); } @@ -304,14 +299,6 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { - if (context.allowExpensiveQueries() == false) { - throw new ElasticsearchException( - "[range] queries on [version] fields cannot be executed when '" - + ALLOW_EXPENSIVE_QUERIES.getKey() - + "' is set to false." - ); - } - failIfNotIndexed(); BytesRef lower = lowerTerm == null ? null : indexedValueForSearch(lowerTerm); BytesRef upper = upperTerm == null ? null : indexedValueForSearch(upperTerm); diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java index aea79b1c59adb..95c91bd005c4b 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java @@ -41,6 +41,9 @@ public void testEncodingOrderingSemver() { assertTrue(encodeVersion("2.0.0-pre127").compareTo(encodeVersion("2.0.0-pre128")) < 0); assertTrue(encodeVersion("2.0.0-pre128").compareTo(encodeVersion("2.0.0-pre128-somethingelse")) < 0); assertTrue(encodeVersion("2.0.0-pre20201231z110026").compareTo(encodeVersion("2.0.0-pre227")) < 0); + // invalid versions sort after valid ones + assertTrue(encodeVersion("99999.99999.99999").compareTo(encodeVersion("1.invalid")) < 0); + assertTrue(encodeVersion("").compareTo(encodeVersion("a")) < 0); } private static BytesRef encodeVersion(String version) { From 94def2071ce68c386051a03cda6584892226bb14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 15 Sep 2020 13:28:19 +0200 Subject: [PATCH 23/28] Switch script method from isPreRelease to isRelease --- docs/reference/mapping/types/version.asciidoc | 6 +++--- .../rest-api-spec/test/versionfield/20_scripts.yml | 7 ++++--- .../xpack/versionfield/VersionScriptDocValues.java | 4 ++-- .../org/elasticsearch/xpack/versionfield/whitelist.txt | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/reference/mapping/types/version.asciidoc b/docs/reference/mapping/types/version.asciidoc index 53c2bded7073f..8bbfd033cac04 100644 --- a/docs/reference/mapping/types/version.asciidoc +++ b/docs/reference/mapping/types/version.asciidoc @@ -85,7 +85,7 @@ POST my-index-000001/_search { "script": { "script": { - "source": "doc['my_version'].isPreRelease() == false" + "source": "doc['my_version'].isRelease() == true" } } } @@ -114,8 +114,8 @@ Functions available on via doc values in scripting are: isValid():: Returns `true` if the field contains a version thats legal according to the Semantic Versioning rules -isPreRelease():: - Returns `true` if the field contains a pre-release version +isRelease():: + Returns `true` if the field contains a valid release version, `false` if it is a pre-release version or invalid. getMajor():: Returns an Integer value of the Major version if the version is valid, or null otherwise diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml index 9e3432ba1b67d..11b9ebdd55f80 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml @@ -95,10 +95,11 @@ setup: search: index: test_index body: - query: { "bool" : { "filter" : [{ "script" : { "script" : {"source": "doc['version'].isPreRelease()"}}}] }} + query: { "bool" : { "filter" : [{ "script" : { "script" : {"source": "doc['version'].isRelease()"}}}] }} - - match: { hits.total.value: 1 } - - match: { hits.hits.0._source.version: "2.0.0-beta" } + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.version: "1.1.12" } + - match: { hits.hits.1._source.version: "3.1.0" } --- "Aggregate using script on major, minor, patch versions": diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java index 75a1abb2b1f0b..355571a080539 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java @@ -61,9 +61,9 @@ public boolean isValid() { return VersionEncoder.legalVersionString(VersionParts.ofVersion(getValue())); } - public boolean isPreRelease() { + public boolean isRelease() { VersionParts parts = VersionParts.ofVersion(getValue()); - return (parts.preRelease != null) && VersionEncoder.legalVersionString(parts); + return VersionEncoder.legalVersionString(parts) && parts.preRelease == null; } public Integer getMajor() { diff --git a/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt b/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt index 43aa5a4165bd8..70ad94b6862fe 100644 --- a/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt +++ b/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt @@ -3,7 +3,7 @@ class org.elasticsearch.xpack.versionfield.VersionScriptDocValues { String get(int) String getValue() boolean isValid() - boolean isPreRelease() + boolean isRelease() Integer getMajor() Integer getMinor() Integer getPatch() From ce5148478d965fc4f3e4a85585301417348b2f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 15 Sep 2020 16:05:38 +0200 Subject: [PATCH 24/28] Another review iteration --- .../xpack/versionfield/VersionEncoder.java | 2 +- .../VersionFieldWildcardQuery.java | 22 ++-- .../versionfield/VersionStringFieldTests.java | 107 ++++++++---------- 3 files changed, 65 insertions(+), 66 deletions(-) diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java index 03fd534571a0b..dfd63271fb4bd 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java @@ -43,7 +43,7 @@ class VersionEncoder { private static final char PRERELEASE_SEPARATOR = '-'; private static final char DOT_SEPARATOR = '.'; private static final char BUILD_SEPARATOR = '+'; - private static final String ENCODED_EMPTY_STRING = new String(new String(new byte[] { NO_PRERELEASE_SEPARATOR_BYTE }, Charsets.UTF_8)); + private static final String ENCODED_EMPTY_STRING = new String(new byte[] { NO_PRERELEASE_SEPARATOR_BYTE }, Charsets.UTF_8); // Regex to test relaxed Semver Main Version validity. Allows for more or less than three main version parts private static Pattern LEGAL_MAIN_VERSION_SEMVER = Pattern.compile("(0|[1-9]\\d*)(\\.(0|[1-9]\\d*))*"); diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java index ba0dc5a26976f..07ab92877c9c1 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java @@ -22,6 +22,17 @@ */ class VersionFieldWildcardQuery extends AutomatonQuery { + private static final Automaton OPTIONAL_NUMERIC_CHARPREFIX = Operations.optional( + Operations.concatenate(Automata.makeChar(VersionEncoder.NUMERIC_MARKER_BYTE), Automata.makeCharRange(0x80, 0xFF)) + ); + + private static final Automaton OPTIONAL_RELEASE_SEPARATOR = Operations.optional( + Operations.union( + Automata.makeChar(VersionEncoder.PRERELEASE_SEPARATOR_BYTE), + Automata.makeChar(VersionEncoder.NO_PRERELEASE_SEPARATOR_BYTE) + ) + ); + private static final byte WILDCARD_STRING = '*'; private static final byte WILDCARD_CHAR = '?'; @@ -46,7 +57,8 @@ private static Automaton toAutomaton(Term wildcardquery) { break; case WILDCARD_CHAR: // this should also match leading digits, which have optional leading numeric marker and length bytes - automata.add(optionalNumericCharPrefix()); + automata.add(OPTIONAL_NUMERIC_CHARPREFIX); + automata.add(OPTIONAL_RELEASE_SEPARATOR); automata.add(Automata.makeAnyChar()); break; @@ -79,7 +91,7 @@ private static Automaton toAutomaton(Term wildcardquery) { firstDigitInGroup = false; } if (firstDigitInGroup) { - automata.add(optionalNumericCharPrefix()); + automata.add(OPTIONAL_NUMERIC_CHARPREFIX); } automata.add(Automata.makeChar(c)); break; @@ -95,12 +107,6 @@ private static Automaton toAutomaton(Term wildcardquery) { return Operations.concatenate(automata); } - private static Automaton optionalNumericCharPrefix() { - return Operations.optional( - Operations.concatenate(Automata.makeChar(VersionEncoder.NUMERIC_MARKER_BYTE), Automata.makeCharRange(0x80, 0xFF)) - ); - } - @Override public String toString(String field) { StringBuilder buffer = new StringBuilder(); diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java index 0d88276bbfa5d..2278ccdb8ba8c 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldTests.java @@ -158,23 +158,29 @@ public void testPrefixQuery() throws IOException { public void testSort() throws IOException { String indexName = setUpIndex("test"); + // also adding some invalid versions that should be sorted after legal ones + client().prepareIndex(indexName).setSource(jsonBuilder().startObject().field("version", "1.2.3alpha").endObject()).get(); + client().prepareIndex(indexName).setSource(jsonBuilder().startObject().field("version", "1.3.567#12").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); // sort based on version field SearchResponse response = client().prepareSearch(indexName) .setQuery(QueryBuilders.matchAllQuery()) .addSort("version", SortOrder.DESC) .get(); - assertEquals(6, response.getHits().getTotalHits().value); + assertEquals(8, response.getHits().getTotalHits().value); SearchHit[] hits = response.getHits().getHits(); - assertEquals("21.11.0", hits[0].getSortValues()[0]); - assertEquals("11.1.0", hits[1].getSortValues()[0]); - assertEquals("2.1.0", hits[2].getSortValues()[0]); - assertEquals("2.1.0-alpha.beta", hits[3].getSortValues()[0]); - assertEquals("1.3.0+build.1234567", hits[4].getSortValues()[0]); - assertEquals("1.0.0", hits[5].getSortValues()[0]); + assertEquals("1.3.567#12", hits[0].getSortValues()[0]); + assertEquals("1.2.3alpha", hits[1].getSortValues()[0]); + assertEquals("21.11.0", hits[2].getSortValues()[0]); + assertEquals("11.1.0", hits[3].getSortValues()[0]); + assertEquals("2.1.0", hits[4].getSortValues()[0]); + assertEquals("2.1.0-alpha.beta", hits[5].getSortValues()[0]); + assertEquals("1.3.0+build.1234567", hits[6].getSortValues()[0]); + assertEquals("1.0.0", hits[7].getSortValues()[0]); response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC).get(); - assertEquals(6, response.getHits().getTotalHits().value); + assertEquals(8, response.getHits().getTotalHits().value); hits = response.getHits().getHits(); assertEquals("1.0.0", hits[0].getSortValues()[0]); assertEquals("1.3.0+build.1234567", hits[1].getSortValues()[0]); @@ -182,6 +188,8 @@ public void testSort() throws IOException { assertEquals("2.1.0", hits[3].getSortValues()[0]); assertEquals("11.1.0", hits[4].getSortValues()[0]); assertEquals("21.11.0", hits[5].getSortValues()[0]); + assertEquals("1.2.3alpha", hits[6].getSortValues()[0]); + assertEquals("1.3.567#12", hits[7].getSortValues()[0]); } public void testRegexQuery() throws Exception { @@ -280,58 +288,43 @@ public void testWildcardQuery() throws Exception { ); ensureGreen(indexName); - client().prepareIndex(indexName) - .setId("1") - .setSource(jsonBuilder().startObject().field("version", "1.0.0-alpha.2.1.0-rc.1").endObject()) - .get(); - client().prepareIndex(indexName) - .setId("2") - .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) - .get(); - client().prepareIndex(indexName) - .setId("3") - .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) - .get(); - client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); - client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); + for (String version : List.of( + "1.0.0-alpha.2.1.0-rc.1", + "1.3.0+build.1234567", + "2.1.0-alpha.beta", + "2.1.0", + "2.33.0", + "3.1.1-a", + "3.1.1+b", + "3.1.123" + )) { + client().prepareIndex(indexName).setSource(jsonBuilder().startObject().field("version", version).endObject()).get(); + } client().admin().indices().prepareRefresh(indexName).get(); - // wildcard - SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*alpha*")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*b*")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*bet*")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "2.1*")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "2.1.0-*")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.1.0-*")).get(); - assertEquals(2, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.1.0*")).get(); - assertEquals(3, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.?.0*")).get(); - assertEquals(3, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.??.0*")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "?.1.0")).get(); - assertEquals(1, response.getHits().getTotalHits().value); - - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*-*")).get(); - assertEquals(2, response.getHits().getTotalHits().value); + checkWildcardQuery(indexName, "*alpha*", new String[] { "1.0.0-alpha.2.1.0-rc.1", "2.1.0-alpha.beta" }); + checkWildcardQuery(indexName, "*b*", new String[] { "1.3.0+build.1234567", "2.1.0-alpha.beta", "3.1.1+b" }); + checkWildcardQuery(indexName, "*bet*", new String[] { "2.1.0-alpha.beta" }); + checkWildcardQuery(indexName, "2.1*", new String[] { "2.1.0-alpha.beta", "2.1.0" }); + checkWildcardQuery(indexName, "2.1.0-*", new String[] { "2.1.0-alpha.beta" }); + checkWildcardQuery(indexName, "*2.1.0-*", new String[] { "1.0.0-alpha.2.1.0-rc.1", "2.1.0-alpha.beta" }); + checkWildcardQuery(indexName, "*2.1.0*", new String[] { "1.0.0-alpha.2.1.0-rc.1", "2.1.0-alpha.beta", "2.1.0" }); + checkWildcardQuery(indexName, "*2.?.0*", new String[] { "1.0.0-alpha.2.1.0-rc.1", "2.1.0-alpha.beta", "2.1.0" }); + checkWildcardQuery(indexName, "*2.??.0*", new String[] { "2.33.0" }); + checkWildcardQuery(indexName, "?.1.0", new String[] { "2.1.0" }); + checkWildcardQuery(indexName, "*-*", new String[] { "1.0.0-alpha.2.1.0-rc.1", "2.1.0-alpha.beta", "3.1.1-a" }); + checkWildcardQuery(indexName, "1.3.0+b*", new String[] { "1.3.0+build.1234567" }); + checkWildcardQuery(indexName, "3.1.1??", new String[] { "3.1.1-a", "3.1.1+b", "3.1.123" }); + } - response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "1.3.0+b*")).get(); - assertEquals(1, response.getHits().getTotalHits().value); + private void checkWildcardQuery(String indexName, String query, String... expectedResults) { + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", query)).get(); + assertEquals(expectedResults.length, response.getHits().getTotalHits().value); + for (int i = 0; i < expectedResults.length; i++) { + String expected = expectedResults[i]; + Object actual = response.getHits().getHits()[i].getSourceAsMap().get("version"); + assertEquals("expected " + expected + " in position " + i + " but found " + actual, expected, actual); + } } /** From 04ac11d32a63295e9ce6d7bc5dc7afb4a2a70f09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 17 Sep 2020 18:12:41 +0200 Subject: [PATCH 25/28] Another round of reviews --- .../xpack/versionfield/VersionScriptDocValues.java | 10 +++++----- .../xpack/versionfield/VersionStringFieldMapper.java | 9 +-------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java index 355571a080539..5de4b99a5c917 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java @@ -35,16 +35,16 @@ public void setNextDocId(int docId) throws IOException { } public String getValue() { - if (count == 0) { - throw new IllegalStateException( - "A document doesn't have a value for a field! " + "Use doc[].size()==0 to check if a document is missing a field!" - ); - } return get(0); } @Override public String get(int index) { + if (count == 0) { + throw new IllegalStateException( + "A document doesn't have a value for a field! " + "Use doc[].size()==0 to check if a document is missing a field!" + ); + } try { return VersionEncoder.decodeVersion(in.lookupOrd(ords[index])); } catch (IOException e) { diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index 3d5550da1fc91..a78a47b9283f3 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -301,14 +301,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { BytesRef lower = lowerTerm == null ? null : indexedValueForSearch(lowerTerm); BytesRef upper = upperTerm == null ? null : indexedValueForSearch(upperTerm); - - return new TermRangeQuery( - name(), - lowerTerm == null ? null : lower, - upperTerm == null ? null : upper, - includeLower, - includeUpper - ); + return new TermRangeQuery(name(), lower, upper, includeLower, includeUpper); } } From ccebd6f470c99c824a84211ed122ba10f1f9629c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 17 Sep 2020 19:34:11 +0200 Subject: [PATCH 26/28] Moving docs under 'structured' --- docs/reference/mapping/types.asciidoc | 4 ++++ docs/reference/mapping/types/keyword.asciidoc | 1 - docs/reference/mapping/types/version.asciidoc | 6 ++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index e35298946d7fb..1108559177c9a 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -51,6 +51,8 @@ Dates:: Date types, including <> and <>:: Range types, such as `long_range`, `double_range`, `date_range`, and `ip_range`. <>:: IPv4 and IPv6 addresses. +<>:: Software versions. Supports https://semver.org/[Semantic Versioning] +precedence rules. {plugins}/mapper-murmur3.html[`murmur3`]:: Compute and stores hashes of values. @@ -148,6 +150,8 @@ include::types/geo-shape.asciidoc[] include::types/ip.asciidoc[] +include::types/version.asciidoc[] + include::types/parent-join.asciidoc[] include::types/keyword.asciidoc[] diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc index 4a63361e94437..855f76d15b269 100644 --- a/docs/reference/mapping/types/keyword.asciidoc +++ b/docs/reference/mapping/types/keyword.asciidoc @@ -130,4 +130,3 @@ include::constant-keyword.asciidoc[] include::wildcard.asciidoc[] -include::version.asciidoc[] diff --git a/docs/reference/mapping/types/version.asciidoc b/docs/reference/mapping/types/version.asciidoc index 8bbfd033cac04..718ec7a14b8b9 100644 --- a/docs/reference/mapping/types/version.asciidoc +++ b/docs/reference/mapping/types/version.asciidoc @@ -1,8 +1,10 @@ [role="xpack"] [testenv="basic"] -[discrete] -[[version-field-type]] +[[version]] === Version field type +++++ +Version +++++ The `version` field type is a specialization of the `keyword` field for handling software version values and to support specialized precedence From 4c0b1e910c1c9a15b5d17e30b068c802f81bbf21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 18 Sep 2020 16:31:38 +0200 Subject: [PATCH 27/28] iter --- .../versionfield/ValidationOnSortedDv.java | 130 ------------------ .../VersionStringFieldMapper.java | 10 +- 2 files changed, 1 insertion(+), 139 deletions(-) delete mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/ValidationOnSortedDv.java diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/ValidationOnSortedDv.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/ValidationOnSortedDv.java deleted file mode 100644 index 28c006de0d569..0000000000000 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/ValidationOnSortedDv.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.versionfield; - -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.search.ConstantScoreScorer; -import org.apache.lucene.search.ConstantScoreWeight; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.TwoPhaseIterator; -import org.apache.lucene.search.Weight; -import org.apache.lucene.util.BytesRef; - -import java.io.IOException; -import java.util.Objects; - -/** - * Query that runs a validation for version ranges across sorted doc values. - * Used in conjunction with more selective query clauses. - */ -class ValidationOnSortedDv extends Query { - - private final String field; - private final BytesRef lower; - private final BytesRef upper; - private final boolean includeLower; - private final boolean includeUpper; - - ValidationOnSortedDv(String field, BytesRef lower, BytesRef upper, boolean includeLower, boolean includeUpper) { - this.field = field; - this.lower = lower; - this.upper = upper; - this.includeLower = includeLower; - this.includeUpper = includeUpper; - } - - @Override - public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { - - return new ConstantScoreWeight(this, boost) { - - @Override - public Scorer scorer(LeafReaderContext context) throws IOException { - final SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field); - - TwoPhaseIterator twoPhase = new TwoPhaseIterator(values) { - @Override - public boolean matches() throws IOException { - long ord = values.nextOrd(); - // multi-value document can have more than one value, iterate over ords - while (ord != SortedSetDocValues.NO_MORE_ORDS) { - BytesRef value = values.lookupOrd(ord); - boolean inRange = true; - if (lower != null) { - if (includeLower) { - inRange = lower.compareTo(value) <= 0; - } else { - inRange = lower.compareTo(value) < 0; - } - } - if (inRange && upper != null) { - if (includeUpper) { - inRange = upper.compareTo(value) >= 0; - } else { - inRange = upper.compareTo(value) > 0; - } - } - if (inRange) { - return true; // found at least one matching value - } - ord = values.nextOrd(); - } - return false; - } - - @Override - public float matchCost() { - // TODO: how can we compute this? - return 1000f; - } - }; - return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); - } - - @Override - public boolean isCacheable(LeafReaderContext ctx) { - return true; - } - }; - } - - @Override - public String toString(String field) { - StringBuilder sb = new StringBuilder(field + ":"); - sb.append(includeLower ? "[" : "("); - sb.append(lower == null ? "" : VersionEncoder.decodeVersion(lower)); - sb.append("-"); - sb.append(upper == null ? "" : VersionEncoder.decodeVersion(upper)); - sb.append(includeUpper ? "]" : ")"); - return sb.toString(); - } - - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != getClass()) { - return false; - } - ValidationOnSortedDv other = (ValidationOnSortedDv) obj; - return Objects.equals(field, other.field) - && Objects.equals(lower, other.lower) - && Objects.equals(lower, other.upper) - && includeLower == other.includeLower - && includeUpper == other.includeUpper; - - } - - @Override - public int hashCode() { - return Objects.hash(field, lower, upper, includeLower, includeUpper); - } - -} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index a78a47b9283f3..482f0bf69da32 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -139,14 +139,6 @@ public Query existsQuery(QueryShardContext context) { @Override public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { - if (context.allowExpensiveQueries() == false) { - throw new ElasticsearchException( - "[prefix] queries cannot be executed when '" - + ALLOW_EXPENSIVE_QUERIES.getKey() - + "' is set to false. For optimised prefix queries on text " - + "fields please enable [index_prefixes]." - ); - } return wildcardQuery(value + "*", method, context); } @@ -324,7 +316,7 @@ public VersionStringFieldType fieldType() { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } From 6849550894da2bc41fcd9300fc459ce2f985de4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 18 Sep 2020 16:59:24 +0200 Subject: [PATCH 28/28] Removing specialized script doc values functions --- docs/reference/mapping/types/version.asciidoc | 63 +---------- .../test/versionfield/20_scripts.yml | 103 ------------------ .../versionfield/VersionScriptDocValues.java | 60 ---------- .../xpack/versionfield/whitelist.txt | 5 - .../VersionStringFieldMapperTests.java | 2 +- 5 files changed, 3 insertions(+), 230 deletions(-) diff --git a/docs/reference/mapping/types/version.asciidoc b/docs/reference/mapping/types/version.asciidoc index 718ec7a14b8b9..a1dfc693955c9 100644 --- a/docs/reference/mapping/types/version.asciidoc +++ b/docs/reference/mapping/types/version.asciidoc @@ -12,7 +12,7 @@ rules for them. Precedence is defined following the rules outlined by https://semver.org/[Semantic Versioning], which for example means that major, minor and patch version parts are sorted numerically (i.e. "2.1.0" < "2.4.1" < "2.11.2") and pre-release versions are sorted before -release versiond (i.e. "1.0.0-alpha" < "1.0.0"). +release versions (i.e. "1.0.0-alpha" < "1.0.0"). You index a `version` field as follows @@ -34,7 +34,7 @@ PUT my-index-000001 The field offers the same search capabilities as a regular keyword field. It can e.g. be searched for exact matches using `match` or `term` queries and supports prefix and wildcard searches. The main benefit is that `range` queries -will honour Semver ordering, so a `range` query between "1.0.0" and "1.5.0" +will honor Semver ordering, so a `range` query between "1.0.0" and "1.5.0" will include versions of "1.2.3" but not "1.11.2" for example. Note that this would be different when using a regular `keyword` field for indexing where ordering is alphabetical. @@ -68,62 +68,3 @@ This field type isn't optimized for heavy wildcard, regex or fuzzy searches. Whi type of queries work in this field, you should consider using a regular `keyword` field if you strongly rely on these kind of queries. -==== Script support - -The `version` fields offers some specialized access to detailed information derived from -valid version strings like the Major, Minor or Patch release number, whether the version value -is valid according to Semver or if it is a pre-release version. This can be helpful when e.g. -filtering for only released versions or running aggregations on parts of the version. -The following query, for example, filters for released versions and groups them by Major version -using a `terms` aggregation: - -[source,console] --------------------------------------------------- -POST my-index-000001/_search -{ - "query": { - "bool": { - "filter": [ - { - "script": { - "script": { - "source": "doc['my_version'].isRelease() == true" - } - } - } - ] - } - }, - "aggs": { - "group_major": { - "terms": { - "script": { "source": "doc['my_version'].getMajor()"}, - "order": { - "_key": "asc" - } - } - } - } -} - --------------------------------------------------- -// TEST[continued] - -Functions available on via doc values in scripting are: - -[horizontal] - -isValid():: - Returns `true` if the field contains a version thats legal according to the Semantic Versioning rules - -isRelease():: - Returns `true` if the field contains a valid release version, `false` if it is a pre-release version or invalid. - -getMajor():: - Returns an Integer value of the Major version if the version is valid, or null otherwise - -getMinor():: - Returns an Integer value of the Minor version if the version is valid, or null otherwise. - -getPatch():: - Returns an Integer value of the Patch version if the version is valid, or null otherwise. \ No newline at end of file diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml index 11b9ebdd55f80..60de6a8c8da65 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/versionfield/20_scripts.yml @@ -52,106 +52,3 @@ setup: - match: { hits.hits.0._source.version: "3.1.0" } - match: { hits.hits.1._source.version: "1.1.12" } - match: { hits.hits.2._source.version: "2.0.0-beta" } - ---- -"Filter script for illegal versions": - - do: - bulk: - refresh: true - body: - - '{ "index" : { "_index" : "test_index", "_id" : "4" } }' - - '{"version": "a123.2.2-beta" }' - - - do: - search: - index: test_index - body: - query: { "bool" : { "filter" : [{ "script" : { "script" : {"source": "doc['version'].isValid() == false"}}}] }} - - - match: { hits.total.value: 1 } - - match: { hits.hits.0._source.version: "a123.2.2-beta" } - - - do: - search: - index: test_index - body: - query: { "bool" : { "filter" : [{ "script" : { "script" : {"source": "doc['version'].isValid()"}}}] }} - - - match: { hits.total.value: 3 } - - match: { hits.hits.0._source.version: "1.1.12" } - - match: { hits.hits.1._source.version: "2.0.0-beta" } - - match: { hits.hits.2._source.version: "3.1.0" } - ---- -"Filter script for pre-release versions": - - do: - bulk: - refresh: true - body: - - '{ "index" : { "_index" : "test_index", "_id" : "4" } }' - - '{"version": "a123.2.2-beta" }' - - - do: - search: - index: test_index - body: - query: { "bool" : { "filter" : [{ "script" : { "script" : {"source": "doc['version'].isRelease()"}}}] }} - - - match: { hits.total.value: 2 } - - match: { hits.hits.0._source.version: "1.1.12" } - - match: { hits.hits.1._source.version: "3.1.0" } - ---- -"Aggregate using script on major, minor, patch versions": - - - do: - bulk: - refresh: true - body: - - '{ "index" : { "_index" : "test_index", "_id" : "4" } }' - - '{"version": "3" }' - - '{ "index" : { "_index" : "test_index", "_id" : "5" } }' - - '{"version": "3.1" }' - - '{ "index" : { "_index" : "test_index", "_id" : "6" } }' - - '{"version": "illegal3.1" }' - - - do: - search: - index: test_index - body: - size: 0 - aggs: { "majors": { "terms": { "script": { "source": "doc['version'].getMajor()", "lang": "painless"}}}} - - - length: { aggregations.majors.buckets: 3 } - - match: { aggregations.majors.buckets.0.key: "3" } - - match: { aggregations.majors.buckets.0.doc_count: 3 } - - match: { aggregations.majors.buckets.1.key: "1" } - - match: { aggregations.majors.buckets.1.doc_count: 1 } - - match: { aggregations.majors.buckets.2.key: "2" } - - match: { aggregations.majors.buckets.2.doc_count: 1 } - - - do: - search: - index: test_index - body: - size: 0 - aggs: { "majors": { "terms": { "script": { "source": "doc['version'].getMinor()", "lang": "painless"}}}} - - - length: { aggregations.majors.buckets: 2 } - - match: { aggregations.majors.buckets.0.key: "1" } - - match: { aggregations.majors.buckets.0.doc_count: 3 } - - match: { aggregations.majors.buckets.1.key: "0" } - - match: { aggregations.majors.buckets.1.doc_count: 1 } - - - do: - search: - index: test_index - body: - size: 0 - aggs: { "majors": { "terms": { "script": { "source": "doc['version'].getPatch()", "lang": "painless"}}}} - - - length: { aggregations.majors.buckets: 2 } - - match: { aggregations.majors.buckets.0.key: "0" } - - match: { aggregations.majors.buckets.0.doc_count: 2 } - - match: { aggregations.majors.buckets.1.key: "12" } - - match: { aggregations.majors.buckets.1.doc_count: 1 } diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java index 5de4b99a5c917..fc07dd940a2a0 100644 --- a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java @@ -9,7 +9,6 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.xpack.versionfield.VersionEncoder.VersionParts; import java.io.IOException; @@ -56,63 +55,4 @@ public String get(int index) { public int size() { return count; } - - public boolean isValid() { - return VersionEncoder.legalVersionString(VersionParts.ofVersion(getValue())); - } - - public boolean isRelease() { - VersionParts parts = VersionParts.ofVersion(getValue()); - return VersionEncoder.legalVersionString(parts) && parts.preRelease == null; - } - - public Integer getMajor() { - VersionParts parts = VersionParts.ofVersion(getValue()); - if (VersionEncoder.legalVersionString(parts) && parts.mainVersion != null) { - int firstDot = parts.mainVersion.indexOf("."); - if (firstDot > 0) { - return Integer.valueOf(parts.mainVersion.substring(0, firstDot)); - } else { - return Integer.valueOf(parts.mainVersion); - } - } - return null; - } - - public Integer getMinor() { - VersionParts parts = VersionParts.ofVersion(getValue()); - Integer rc = null; - if (VersionEncoder.legalVersionString(parts) && parts.mainVersion != null) { - int firstDot = parts.mainVersion.indexOf("."); - if (firstDot > 0) { - int secondDot = parts.mainVersion.indexOf(".", firstDot + 1); - if (secondDot > 0) { - rc = Integer.valueOf(parts.mainVersion.substring(firstDot + 1, secondDot)); - } else { - rc = Integer.valueOf(parts.mainVersion.substring(firstDot + 1)); - } - } - } - return rc; - } - - public Integer getPatch() { - VersionParts parts = VersionParts.ofVersion(getValue()); - Integer rc = null; - if (VersionEncoder.legalVersionString(parts) && parts.mainVersion != null) { - int firstDot = parts.mainVersion.indexOf("."); - if (firstDot > 0) { - int secondDot = parts.mainVersion.indexOf(".", firstDot + 1); - if (secondDot > 0) { - int thirdDot = parts.mainVersion.indexOf(".", secondDot + 1); - if (thirdDot > 0) { - rc = Integer.valueOf(parts.mainVersion.substring(secondDot + 1, thirdDot)); - } else { - rc = Integer.valueOf(parts.mainVersion.substring(secondDot + 1)); - } - } - } - } - return rc; - } } diff --git a/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt b/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt index 70ad94b6862fe..eb47eae53a158 100644 --- a/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt +++ b/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt @@ -2,9 +2,4 @@ class org.elasticsearch.xpack.versionfield.VersionScriptDocValues { String get(int) String getValue() - boolean isValid() - boolean isRelease() - Integer getMajor() - Integer getMinor() - Integer getPatch() } \ No newline at end of file diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 73c121b83f8fd..a3e42a19337fe 100644 --- a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -124,7 +124,7 @@ public void testFailsParsingNestedList() throws IOException { ); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());