From 343310c129ff67a4cb6d2e034ca538b56bf9ae9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 14 Jul 2020 19:40:30 +0200 Subject: [PATCH] Add field type for version strings This PR adds a new 'version' field type that allows indexing string values representing software versions similar to the ones defined in the Semantic Versioning definition (semver.org). The field behaves very similar to a 'keyword' field but allows efficient sorting and range queries that take into accound the special ordering needed for version strings. For example, the main version parts are sorted numerically (ie 2.0.0 < 11.0.0) whereas this wouldn't be possible with 'keyword' fields today. Valid version values are similar to the Semantic Versioning definition, with the notable exception that in addition to the "main" version consiting of major.minor.patch, we allow less or more than three numeric identifiers, i.e. "1.2" or "1.4.6.123.12" are treated as valid too. Relates to #48878 --- .../index/mapper/TermBasedFieldType.java | 5 +- x-pack/plugin/versionfield/build.gradle | 20 + .../xpack/versionfield/VersionEncoder.java | 232 +++++++++ .../VersionFieldDocValuesExtension.java | 42 ++ .../versionfield/VersionFieldPlugin.java | 28 ++ .../VersionFieldWildcardQuery.java | 114 +++++ .../versionfield/VersionScriptDocValues.java | 57 +++ .../VersionStringFieldMapper.java | 445 +++++++++++++++++ ...asticsearch.painless.spi.PainlessExtension | 1 + .../xpack/versionfield/whitelist.txt | 5 + .../versionfield/VersionEncoderTests.java | 228 +++++++++ .../VersionStringFieldMapperTests.java | 453 ++++++++++++++++++ 12 files changed, 1628 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/versionfield/build.gradle create mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java create mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldDocValuesExtension.java create mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java create mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java create mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java create mode 100644 x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java create mode 100644 x-pack/plugin/versionfield/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt create mode 100644 x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java create mode 100644 x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java index 0101b68fb88c8..91df9b4a9a306 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java @@ -33,9 +33,10 @@ /** Base {@link MappedFieldType} implementation for a field that is indexed * with the inverted index. */ -abstract class TermBasedFieldType extends SimpleMappedFieldType { +public abstract class TermBasedFieldType extends SimpleMappedFieldType { - TermBasedFieldType(String name, boolean isSearchable, boolean hasDocValues, TextSearchInfo textSearchInfo, Map meta) { + public TermBasedFieldType(String name, boolean isSearchable, boolean hasDocValues, TextSearchInfo textSearchInfo, + Map meta) { super(name, isSearchable, hasDocValues, textSearchInfo, meta); } diff --git a/x-pack/plugin/versionfield/build.gradle b/x-pack/plugin/versionfield/build.gradle new file mode 100644 index 0000000000000..b3423de6e0d54 --- /dev/null +++ b/x-pack/plugin/versionfield/build.gradle @@ -0,0 +1,20 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'versionfield' + description 'A plugin for a field type to store sofware versions' + classname 'org.elasticsearch.xpack.versionfield.VersionFieldPlugin' + extendedPlugins = ['x-pack-core', 'lang-painless'] +} +archivesBaseName = 'x-pack-versionfield' + +dependencies { + compileOnly project(path: xpackModule('core'), configuration: 'default') + compileOnly project(':modules:lang-painless:spi') + compileOnly project(':modules:lang-painless') + testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') +} + +integTest.enabled = false diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java new file mode 100644 index 0000000000000..fc0d0b64c1e9b --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java @@ -0,0 +1,232 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; + +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +/** + * Encodes a version string to a {@link BytesRef} that correctly sorts according to software version precedence rules like + * the ones described in Semantiv Versioning (https://semver.org/) + * + * Version strings are considered to consist of three parts: + *
    + *
  • a numeric major.minor.patch part starting the version string (e.g. 1.2.3) + *
  • an optional "pre-release" part that starts with a `-` character and can consist of several alpha-numerical sections + * separated by dots (e.g. "-alpha.2.3") + *
  • an optional "build" part that starts with a `+` character. This will simply be treated as a prefix with no guaranteed ordering, + * (although the ordering should be alphabetical in most cases). + *
+ * + * The version string is encoded such that the ordering works like the following: + *
    + *
  • Major, minor, and patch versions are always compared numerically + *
  • pre-release version have lower precedence than a normal version. (e.g 1.0.0-alpha < 1.0.0) + *
  • the precedence for pre-release versions with same main version is calculated comparing each dot separated identifier from + * left to right. Identifiers consisting of only digits are compared numerically and identifiers with letters or hyphens are compared + * lexically in ASCII sort order. Numeric identifiers always have lower precedence than non-numeric identifiers. + *
+ */ +class VersionEncoder { + + public static final byte NUMERIC_MARKER_BYTE = (byte) 0x01; + public static final byte PRERELESE_SEPARATOR_BYTE = (byte) 0x02; + public static final byte NO_PRERELESE_SEPARATOR_BYTE = (byte) 0x03; + + private static final char PRERELESE_SEPARATOR = '-'; + private static final char DOT_SEPARATOR = '.'; + private static final char BUILD_SEPARATOR = '+'; + + // Regex to test version validity: \d+(\.\d+)*(-[\-\dA-Za-z]+){0,1}(\.[-\dA-Za-z]+)*(\+[\.\-\dA-Za-z]+)? + // private static Pattern LEGAL_VERSION_PATTERN = Pattern.compile( + // "\\d+(\\.\\d+)*(-[\\-\\dA-Za-z]+){0,1}(\\.[\\-\\dA-Za-z]+)*(\\+[\\.\\-\\dA-Za-z]+)?" + // ); + + // Regex to test strict Semver Main Version validity: + // private static Pattern LEGAL_MAIN_VERSION_SEMVER = Pattern.compile("(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)"); + + // Regex to test relaxed Semver Main Version validity. Allows for more or less than three main version parts + private static Pattern LEGAL_MAIN_VERSION_SEMVER = Pattern.compile("(0|[1-9]\\d*)(\\.(0|[1-9]\\d*))*"); + + private static Pattern LEGAL_PRERELEASE_VERSION_SEMVER = Pattern.compile( + "(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))" + ); + + private static Pattern LEGAL_BUILDSUFFIX_SEMVER = Pattern.compile("(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?"); + + /** + * Encodes a version string. + */ + public static EncodedVersion encodeVersion(String versionString) { + // System.out.println("encoding: " + versionString); + VersionParts versionParts = VersionParts.ofVersion(versionString); + + // don't treat non-legal versions further, just mark them as illegal and return + if (legalVersionString(versionParts) == false) { + return new EncodedVersion(new BytesRef(versionString), false, true, 0, 0, 0); + } + + BytesRefBuilder encodedBytes = new BytesRefBuilder(); + Integer[] mainVersionParts = prefixDigitGroupsWithLength(versionParts.mainVersion, encodedBytes); + + if (versionParts.preRelease != null) { + encodedBytes.append(PRERELESE_SEPARATOR_BYTE); // versions with pre-release part sort before ones without + encodedBytes.append((byte) PRERELESE_SEPARATOR); + String[] preReleaseParts = versionParts.preRelease.substring(1).split("\\."); + boolean first = true; + for (String preReleasePart : preReleaseParts) { + if (first == false) { + encodedBytes.append((byte) DOT_SEPARATOR); + } + boolean isNumeric = preReleasePart.chars().allMatch(x -> Character.isDigit(x)); + if (isNumeric) { + prefixDigitGroupsWithLength(preReleasePart, encodedBytes); + } else { + encodedBytes.append(new BytesRef(preReleasePart)); + } + first = false; + } + } else { + encodedBytes.append(NO_PRERELESE_SEPARATOR_BYTE); + } + + if (versionParts.buildSuffix != null) { + encodedBytes.append(new BytesRef(versionParts.buildSuffix)); + } + // System.out.println("encoded: " + encodedBytes.toBytesRef()); + return new EncodedVersion( + encodedBytes.toBytesRef(), + true, + versionParts.preRelease != null, + mainVersionParts[0], + mainVersionParts[1], + mainVersionParts[2] + ); + } + + private static Integer[] prefixDigitGroupsWithLength(String input, BytesRefBuilder result) { + int pos = 0; + int mainVersionCounter = 0; + Integer[] mainVersionComponents = new Integer[3]; + while (pos < input.length()) { + if (Character.isDigit(input.charAt(pos))) { + // found beginning of number block, so get its length + int start = pos; + BytesRefBuilder number = new BytesRefBuilder(); + while (pos < input.length() && Character.isDigit(input.charAt(pos))) { + number.append((byte) input.charAt(pos)); + pos++; + } + int length = pos - start; + if (length >= 128) { + throw new IllegalArgumentException("Groups of digits cannot be longer than 127, but found: " + length); + } + result.append(NUMERIC_MARKER_BYTE); // ensure length byte does cause higher sort order comparing to other byte[] + result.append((byte) (length | 0x80)); // add upper bit to mark as length + result.append(number); + + // if present, parse out three leftmost version parts + if (mainVersionCounter < 3) { + mainVersionComponents[mainVersionCounter] = Integer.valueOf(number.toBytesRef().utf8ToString()); + mainVersionCounter++; + } + } else { + result.append((byte) input.charAt(pos)); + pos++; + } + } + return mainVersionComponents; + } + + public static String decodeVersion(BytesRef version) { + // System.out.println("decoding: " + version); + int inputPos = version.offset; + int resultPos = 0; + byte[] result = new byte[version.length]; + while (inputPos < version.offset + version.length) { + byte inputByte = version.bytes[inputPos]; + if (inputByte == NUMERIC_MARKER_BYTE) { + // need to skip this byte + inputPos++; + // this should always be a length encoding, which is skipped by increasing inputPos at the end of the loop + assert version.bytes[inputPos] < 0; + } else if (inputByte != PRERELESE_SEPARATOR_BYTE && inputByte != NO_PRERELESE_SEPARATOR_BYTE) { + result[resultPos] = inputByte; + resultPos++; + } + inputPos++; + } + // System.out.println("decoded to: " + new String(result, 0, resultPos)); + return new String(result, 0, resultPos, StandardCharsets.UTF_8); + } + + private static boolean legalVersionString(VersionParts versionParts) { + boolean legalMainVersion = LEGAL_MAIN_VERSION_SEMVER.matcher(versionParts.mainVersion).matches(); + boolean legalPreRelease = true; + if (versionParts.preRelease != null) { + legalPreRelease = LEGAL_PRERELEASE_VERSION_SEMVER.matcher(versionParts.preRelease).matches(); + } + boolean legalBuildSuffix = true; + if (versionParts.buildSuffix != null) { + legalBuildSuffix = LEGAL_BUILDSUFFIX_SEMVER.matcher(versionParts.buildSuffix).matches(); + } + return legalMainVersion && legalPreRelease && legalBuildSuffix; + } + + public static class EncodedVersion { + + public final boolean isLegal; + public final boolean isPreRelease; + public final BytesRef bytesRef; + public final Integer major; + public final Integer minor; + public final Integer patch; + + EncodedVersion(BytesRef bytesRef, boolean isLegal, boolean isPreRelease, Integer major, Integer minor, Integer patch) { + super(); + this.bytesRef = bytesRef; + this.isLegal = isLegal; + this.isPreRelease = isPreRelease; + this.major = major; + this.minor = minor; + this.patch = patch; + } + } + + private static class VersionParts { + final String mainVersion; + final String preRelease; + final String buildSuffix; + + private VersionParts(String mainVersion, String preRelease, String buildSuffix) { + this.mainVersion = mainVersion; + this.preRelease = preRelease; + this.buildSuffix = buildSuffix; + } + + private static VersionParts ofVersion(String versionString) { + String buildSuffix = extractSuffix(versionString, BUILD_SEPARATOR); + if (buildSuffix != null) { + versionString = versionString.substring(0, versionString.length() - buildSuffix.length()); + } + + String preRelease = extractSuffix(versionString, PRERELESE_SEPARATOR); + if (preRelease != null) { + versionString = versionString.substring(0, versionString.length() - preRelease.length()); + } + return new VersionParts(versionString, preRelease, buildSuffix); + } + + private static String extractSuffix(String input, char separator) { + int start = input.indexOf(separator); + return start > 0 ? input.substring(start) : null; + } + } +} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldDocValuesExtension.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldDocValuesExtension.java new file mode 100644 index 0000000000000..ac27125fa8610 --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldDocValuesExtension.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.script.FieldScript; +import org.elasticsearch.script.FilterScript; +import org.elasticsearch.script.NumberSortScript; +import org.elasticsearch.script.ScoreScript; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.StringSortScript; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.singletonList; + +public class VersionFieldDocValuesExtension implements PainlessExtension { + + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(VersionFieldDocValuesExtension.class, "whitelist.txt"); + + @Override + public Map, List> getContextWhitelists() { + Map, List> whitelist = new HashMap<>(); + List list = singletonList(WHITELIST); + whitelist.put(AggregationScript.CONTEXT, list); + whitelist.put(ScoreScript.CONTEXT, list); + whitelist.put(FilterScript.CONTEXT, list); + whitelist.put(FieldScript.CONTEXT, list); + whitelist.put(NumberSortScript.CONTEXT, list); + whitelist.put(StringSortScript.CONTEXT, list); + return whitelist; + } +} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java new file mode 100644 index 0000000000000..dd9352f754cc5 --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldPlugin.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +public class VersionFieldPlugin extends Plugin implements MapperPlugin { + + public VersionFieldPlugin(Settings settings) {} + + @Override + public Map getMappers() { + Map mappers = new LinkedHashMap<>(); + mappers.put(VersionStringFieldMapper.CONTENT_TYPE, new VersionStringFieldMapper.TypeParser()); + return Collections.unmodifiableMap(mappers); + } +} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java new file mode 100644 index 0000000000000..03c6cf780a07d --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldWildcardQuery.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.AutomatonQuery; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Automata; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.Operations; + +import java.util.ArrayList; +import java.util.List; + +/** + * A variation of the {@link WildcardQuery} than skips over meta characters introduced using {@link VersionEncoder}. + */ +class VersionFieldWildcardQuery extends AutomatonQuery { + + private static final byte WILDCARD_STRING = '*'; + + private static final byte WILDCARD_CHAR = '?'; + + VersionFieldWildcardQuery(Term term) { + super(term, toAutomaton(term), Integer.MAX_VALUE, true); + } + + private static Automaton toAutomaton(Term wildcardquery) { + List automata = new ArrayList<>(); + + BytesRef wildcardText = wildcardquery.bytes(); + boolean containsPreReleaseSeparator = false; + + for (int i = 0; i < wildcardText.length;) { + final byte c = wildcardText.bytes[wildcardText.offset + i]; + int length = Character.charCount(c); + + switch (c) { + case WILDCARD_STRING: + automata.add(Automata.makeAnyString()); + break; + case WILDCARD_CHAR: + // this should also match leading digits, which have optional leading numeric marker and length bytes + automata.add(optionalNumericCharPrefix()); + automata.add(Automata.makeAnyChar()); + break; + + case '-': + // this should potentially match the first prerelease-dash, so we need an optional marker byte here + automata.add(Operations.optional(Automata.makeChar(VersionEncoder.PRERELESE_SEPARATOR_BYTE))); + containsPreReleaseSeparator = true; + automata.add(Automata.makeChar(c)); + break; + case '+': + // this can potentially appear after major version, optionally match the no-prerelease marker + automata.add(Operations.optional(Automata.makeChar(VersionEncoder.NO_PRERELESE_SEPARATOR_BYTE))); + containsPreReleaseSeparator = true; + automata.add(Automata.makeChar(c)); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + boolean firstDigitInGroup = true; + if (i > 0 + && wildcardText.bytes[wildcardText.offset + i - 1] >= (byte) '0' + && wildcardText.bytes[wildcardText.offset + i - 1] <= (byte) '9') { + firstDigitInGroup = false; + } + if (firstDigitInGroup) { + automata.add(optionalNumericCharPrefix()); + } + automata.add(Automata.makeChar(c)); + break; + default: + automata.add(Automata.makeChar(c)); + } + i += length; + } + // when we only have main version part, we need to add an optional NO_PRERELESE_SEPARATOR_BYTE + if (containsPreReleaseSeparator == false) { + automata.add(Operations.optional(Automata.makeChar(VersionEncoder.NO_PRERELESE_SEPARATOR_BYTE))); + } + return Operations.concatenate(automata); + } + + private static Automaton optionalNumericCharPrefix() { + return Operations.optional( + Operations.concatenate(Automata.makeChar(VersionEncoder.NUMERIC_MARKER_BYTE), Automata.makeCharRange(0x80, 0xFF)) + ); + } + + @Override + public String toString(String field) { + StringBuilder buffer = new StringBuilder(); + if (!getField().equals(field)) { + buffer.append(getField()); + buffer.append(":"); + } + buffer.append(term.text()); + return buffer.toString(); + } +} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java new file mode 100644 index 0000000000000..f74027db634cc --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.index.fielddata.ScriptDocValues; + +import java.io.IOException; + +public final class VersionScriptDocValues extends ScriptDocValues { + + private final SortedSetDocValues in; + private long[] ords = new long[0]; + private int count; + + public VersionScriptDocValues(SortedSetDocValues in) { + this.in = in; + } + + @Override + public void setNextDocId(int docId) throws IOException { + count = 0; + if (in.advanceExact(docId)) { + for (long ord = in.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = in.nextOrd()) { + ords = ArrayUtil.grow(ords, count + 1); + ords[count++] = ord; + } + } + } + + public String getValue() { + if (count == 0) { + return null; + } else { + return get(0); + } + } + + @Override + public String get(int index) { + try { + return VersionEncoder.decodeVersion(in.lookupOrd(ords[index])); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public int size() { + return count; + } +} diff --git a/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java new file mode 100644 index 0000000000000..6a3f736e52b72 --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -0,0 +1,445 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.apache.lucene.document.BinaryPoint; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.TermBasedFieldType; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.TypeParsers; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.support.QueryParsers; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.xpack.versionfield.VersionEncoder.EncodedVersion; + +import java.io.IOException; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; +import static org.elasticsearch.xpack.versionfield.VersionEncoder.encodeVersion; + +/** + * A {@link FieldMapper} for indexing fields with version strings. + */ +public class VersionStringFieldMapper extends FieldMapper { + + private static byte[] MIN_VALUE = new byte[16]; + private static byte[] MAX_VALUE = new byte[16]; + static { + Arrays.fill(MIN_VALUE, (byte) 0); + Arrays.fill(MAX_VALUE, (byte) -1); + } + + public static final String CONTENT_TYPE = "version"; + + public static class Defaults { + public static final FieldType FIELD_TYPE = new FieldType(); + + static { + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.freeze(); + } + + public static final String NULL_VALUE = null; + } + + static class Builder extends FieldMapper.Builder { + + protected String nullValue = Defaults.NULL_VALUE; + private boolean storeMalformed = false; + + Builder(String name) { + super(name, Defaults.FIELD_TYPE); + builder = this; + } + + Builder nullValue(String nullValue) { + this.nullValue = nullValue; + return builder; + } + + Builder storeMalformed(boolean storeMalformed) { + this.storeMalformed = storeMalformed; + return builder; + } + + private VersionStringFieldType buildFieldType(BuilderContext context) { + boolean validateVersion = storeMalformed == false; + return new VersionStringFieldType(buildFullName(context), indexed, validateVersion, meta, boost, fieldType); + } + + @Override + public VersionStringFieldMapper build(BuilderContext context) { + BooleanFieldMapper.Builder preReleaseSubfield = new BooleanFieldMapper.Builder(name + ".isPreRelease"); + NumberType type = NumberType.INTEGER; + NumberFieldMapper.Builder majorVersionSubField = new NumberFieldMapper.Builder(name + ".major", type).nullValue(0); + NumberFieldMapper.Builder minorVersionSubField = new NumberFieldMapper.Builder(name + ".minor", type).nullValue(0); + NumberFieldMapper.Builder patchVersionSubField = new NumberFieldMapper.Builder(name + ".patch", type).nullValue(0); + + return new VersionStringFieldMapper( + name, + fieldType, + buildFieldType(context), + storeMalformed, + nullValue, + multiFieldsBuilder.build(this, context), + copyTo, + preReleaseSubfield.build(context), + majorVersionSubField.build(context), + minorVersionSubField.build(context), + patchVersionSubField.build(context) + ); + } + } + + public static class TypeParser implements Mapper.TypeParser { + + public TypeParser() {} + + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + Builder builder = new Builder(name); + TypeParsers.parseField(builder, name, node, parserContext); + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String propName = entry.getKey(); + Object propNode = entry.getValue(); + if (propName.equals("null_value")) { + if (propNode == null) { + throw new MapperParsingException("Property [null_value] cannot be null."); + } + builder.nullValue(propNode.toString()); + iterator.remove(); + } else if (propName.equals("store_malformed")) { + builder.storeMalformed(XContentMapValues.nodeBooleanValue(propNode, name + ".store_malformed")); + iterator.remove(); + } else if (TypeParsers.parseMultiField(builder::addMultiField, name, parserContext, propName, propNode)) { + iterator.remove(); + } + } + return builder; + } + } + + public static final class VersionStringFieldType extends TermBasedFieldType { + + // if true, we want to throw errors on illegal versions at index and query time + private boolean validateVersion = false; + + public VersionStringFieldType( + String name, + boolean isSearchable, + boolean validateVersion, + Map meta, + float boost, + FieldType fieldType + ) { + super(name, isSearchable, true, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta); + setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + setBoost(boost); + this.validateVersion = validateVersion; + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public Query existsQuery(QueryShardContext context) { + return new DocValuesFieldExistsQuery(name()); + } + + @Override + public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { + if (context.allowExpensiveQueries() == false) { + throw new ElasticsearchException( + "[prefix] queries cannot be executed when '" + + ALLOW_EXPENSIVE_QUERIES.getKey() + + "' is set to false. For optimised prefix queries on text " + + "fields please enable [index_prefixes]." + ); + } + failIfNotIndexed(); + return wildcardQuery(value + "*", method, context); + } + + @Override + public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { + if (context.allowExpensiveQueries() == false) { + throw new ElasticsearchException( + "[wildcard] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false." + ); + } + failIfNotIndexed(); + + VersionFieldWildcardQuery query = new VersionFieldWildcardQuery(new Term(name(), value)); + QueryParsers.setRewriteMethod(query, method); + return query; + } + + @Override + protected BytesRef indexedValueForSearch(Object value) { + String valueAsString; + if (value instanceof String) { + valueAsString = (String) value; + } else if (value instanceof BytesRef) { + // encoded string, need to re-encode + valueAsString = ((BytesRef) value).utf8ToString(); + } else { + throw new IllegalArgumentException("Illegal value type: " + value.getClass() + ", value: " + value); + } + EncodedVersion encodedVersion = encodeVersion(valueAsString); + if (encodedVersion.isLegal == false && validateVersion) { + throw new IllegalArgumentException("Illegal version string: " + valueAsString); + } + return encodedVersion.bytesRef; + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + failIfNoDocValues(); + return new SortedSetOrdinalsIndexFieldData.Builder(VersionScriptDocValues::new, CoreValuesSourceType.BYTES); + } + + @Override + public Object valueForDisplay(Object value) { + if (value == null) { + return null; + } + return VERSION_DOCVALUE.format((BytesRef) value); + } + + @Override + public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support custom formats"); + } + if (timeZone != null) { + throw new IllegalArgumentException( + "Field [" + name() + "] of type [" + typeName() + "] does not support custom time zones" + ); + } + return VERSION_DOCVALUE; + } + + @Override + public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { + if (context.allowExpensiveQueries() == false) { + throw new ElasticsearchException( + "[range] queries on [version] fields cannot be executed when '" + + ALLOW_EXPENSIVE_QUERIES.getKey() + + "' is set to false." + ); + } + failIfNotIndexed(); + BytesRef lower = lowerTerm == null ? null : indexedValueForSearch(lowerTerm); + BytesRef upper = upperTerm == null ? null : indexedValueForSearch(upperTerm); + byte[] lowerBytes = lower == null ? MIN_VALUE : Arrays.copyOfRange(lower.bytes, lower.offset, 16); + byte[] upperBytes = upper == null ? MAX_VALUE : Arrays.copyOfRange(upper.bytes, upper.offset, 16); + + // point query on the 16byte prefix + Query pointPrefixQuery = BinaryPoint.newRangeQuery(name(), lowerBytes, upperBytes); + + Query termQuery = new TermRangeQuery( + name(), + lowerTerm == null ? null : lower, + upperTerm == null ? null : upper, + includeLower, + includeUpper + ); + + return new BooleanQuery.Builder().add(new BooleanClause(pointPrefixQuery, Occur.MUST)) + .add(new BooleanClause(termQuery, Occur.MUST)) + .build(); + } + } + + private boolean storeMalformed; + private String nullValue; + private BooleanFieldMapper prereleaseSubField; + private NumberFieldMapper majorVersionSubField; + private NumberFieldMapper minorVersionSubField; + private NumberFieldMapper patchVersionSubField; + + private VersionStringFieldMapper( + String simpleName, + FieldType fieldType, + MappedFieldType mappedFieldType, + boolean storeMalformed, + String nullValue, + MultiFields multiFields, + CopyTo copyTo, + BooleanFieldMapper preReleaseMapper, + NumberFieldMapper majorVersionMapper, + NumberFieldMapper minorVersionMapper, + NumberFieldMapper patchVersionMapper + ) { + super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); + this.storeMalformed = storeMalformed; + this.nullValue = nullValue; + this.prereleaseSubField = preReleaseMapper; + this.majorVersionSubField = majorVersionMapper; + this.minorVersionSubField = minorVersionMapper; + this.patchVersionSubField = patchVersionMapper; + } + + @Override + public VersionStringFieldType fieldType() { + return (VersionStringFieldType) super.fieldType(); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + protected VersionStringFieldMapper clone() { + return (VersionStringFieldMapper) super.clone(); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + String versionString; + if (context.externalValueSet()) { + versionString = context.externalValue().toString(); + } else { + XContentParser parser = context.parser(); + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { + versionString = nullValue; + } else { + versionString = parser.textOrNull(); + } + } + + if (versionString == null) { + return; + } + + EncodedVersion encoding = encodeVersion(versionString); + if (encoding.isLegal == false && storeMalformed == false) { + throw new IllegalArgumentException("Illegal version string: " + versionString); + } + BytesRef encodedVersion = encoding.bytesRef; + if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { + Field field = new Field(fieldType().name(), encodedVersion, fieldType); + context.doc().add(field); + // encode the first 16 bytes as points for efficient range query + byte[] first16bytes = Arrays.copyOfRange(encodedVersion.bytes, encodedVersion.offset, 16); + context.doc().add(new BinaryPoint(fieldType().name(), first16bytes)); + } + context.doc().add(new SortedSetDocValuesField(fieldType().name(), encodedVersion)); + + // add additional information extracted from version string + context.doc().add(new Field(prereleaseSubField.name(), encoding.isPreRelease ? "T" : "F", BooleanFieldMapper.Defaults.FIELD_TYPE)); + context.doc().add(new SortedNumericDocValuesField(prereleaseSubField.name(), encoding.isPreRelease ? 1 : 0)); + + addVersionPartSubfield(context, majorVersionSubField.name(), encoding.major); + addVersionPartSubfield(context, minorVersionSubField.name(), encoding.minor); + addVersionPartSubfield(context, patchVersionSubField.name(), encoding.patch); + } + + private void addVersionPartSubfield(ParseContext context, String fieldName, Integer versionPart) { + if (versionPart != null) { + context.doc().addAll(NumberType.INTEGER.createFields(fieldName, versionPart, true, true, false)); + } + } + + @Override + protected void mergeOptions(FieldMapper other, List conflicts) { + VersionStringFieldMapper mergeWith = (VersionStringFieldMapper) other; + this.storeMalformed = mergeWith.storeMalformed; + this.nullValue = mergeWith.nullValue; + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + if (nullValue != null) { + builder.field("null_value", nullValue); + } + builder.field("store_malformed", storeMalformed); + } + + @Override + public Iterator iterator() { + List subIterators = new ArrayList<>(); + subIterators.add(prereleaseSubField); + subIterators.add(majorVersionSubField); + subIterators.add(minorVersionSubField); + subIterators.add(patchVersionSubField); + @SuppressWarnings("unchecked") + Iterator concat = Iterators.concat(super.iterator(), subIterators.iterator()); + return concat; + } + + private static DocValueFormat VERSION_DOCVALUE = new DocValueFormat() { + + @Override + public String getWriteableName() { + return "version_semver"; + } + + @Override + public void writeTo(StreamOutput out) {} + + @Override + public String format(BytesRef value) { + return VersionEncoder.decodeVersion(value); + } + + @Override + public BytesRef parseBytesRef(String value) { + return VersionEncoder.encodeVersion(value).bytesRef; + } + + @Override + public String toString() { + return getWriteableName(); + } + }; +} diff --git a/x-pack/plugin/versionfield/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/versionfield/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..41ff870b75aa2 --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.versionfield.VersionFieldDocValuesExtension diff --git a/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt b/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt new file mode 100644 index 0000000000000..eb47eae53a158 --- /dev/null +++ b/x-pack/plugin/versionfield/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt @@ -0,0 +1,5 @@ + +class org.elasticsearch.xpack.versionfield.VersionScriptDocValues { + String get(int) + String getValue() +} \ No newline at end of file diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java new file mode 100644 index 0000000000000..825146e315918 --- /dev/null +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionEncoderTests.java @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.versionfield.VersionEncoder.EncodedVersion; + +import java.util.Arrays; + +import static org.elasticsearch.xpack.versionfield.VersionEncoder.decodeVersion; + +public class VersionEncoderTests extends ESTestCase { + + public void testEncodingOrderingSemver() { + assertTrue(encodeVersion("1").compareTo(encodeVersion("1.0")) < 0); + assertTrue(encodeVersion("1.0").compareTo(encodeVersion("1.0.0.0.0.0.0.0.0.1")) < 0); + assertTrue(encodeVersion("1.0.0").compareTo(encodeVersion("1.0.0.0.0.0.0.0.0.1")) < 0); + assertTrue(encodeVersion("1.0.0").compareTo(encodeVersion("2.0.0")) < 0); + assertTrue(encodeVersion("2.0.0").compareTo(encodeVersion("11.0.0")) < 0); + assertTrue(encodeVersion("2.0.0").compareTo(encodeVersion("2.1.0")) < 0); + assertTrue(encodeVersion("2.1.0").compareTo(encodeVersion("2.1.1")) < 0); + assertTrue(encodeVersion("2.1.1").compareTo(encodeVersion("2.1.1.0")) < 0); + assertTrue(encodeVersion("2.0.0").compareTo(encodeVersion("11.0.0")) < 0); + assertTrue(encodeVersion("1.0.0").compareTo(encodeVersion("2.0")) < 0); + assertTrue(encodeVersion("1.0.0-a").compareTo(encodeVersion("1.0.0-b")) < 0); + assertTrue(encodeVersion("1.0.0-1.0.0").compareTo(encodeVersion("1.0.0-2.0")) < 0); + assertTrue(encodeVersion("1.0.0-alpha").compareTo(encodeVersion("1.0.0-alpha.1")) < 0); + assertTrue(encodeVersion("1.0.0-alpha.1").compareTo(encodeVersion("1.0.0-alpha.beta")) < 0); + assertTrue(encodeVersion("1.0.0-alpha.beta").compareTo(encodeVersion("1.0.0-beta")) < 0); + assertTrue(encodeVersion("1.0.0-beta").compareTo(encodeVersion("1.0.0-beta.2")) < 0); + assertTrue(encodeVersion("1.0.0-beta.2").compareTo(encodeVersion("1.0.0-beta.11")) < 0); + assertTrue(encodeVersion("1.0.0-beta11").compareTo(encodeVersion("1.0.0-beta2")) < 0); // correct according to Semver specs + assertTrue(encodeVersion("1.0.0-beta.11").compareTo(encodeVersion("1.0.0-rc.1")) < 0); + assertTrue(encodeVersion("1.0.0-rc.1").compareTo(encodeVersion("1.0.0")) < 0); + assertTrue(encodeVersion("1.0.0").compareTo(encodeVersion("2.0.0-pre127")) < 0); + assertTrue(encodeVersion("2.0.0-pre127").compareTo(encodeVersion("2.0.0-pre128")) < 0); + assertTrue(encodeVersion("2.0.0-pre128").compareTo(encodeVersion("2.0.0-pre128-somethingelse")) < 0); + assertTrue(encodeVersion("2.0.0-pre20201231z110026").compareTo(encodeVersion("2.0.0-pre227")) < 0); + } + + private static BytesRef encodeVersion(String version) { + return VersionEncoder.encodeVersion(version).bytesRef; + } + + public void testPreReleaseFlag() { + assertTrue(VersionEncoder.encodeVersion("1.2-alpha.beta").isPreRelease); + assertTrue(VersionEncoder.encodeVersion("1.2.3-someOtherPreRelease").isPreRelease); + assertTrue(VersionEncoder.encodeVersion("1.2.3-some-Other-Pre.123").isPreRelease); + assertTrue(VersionEncoder.encodeVersion("1.2.3-some-Other-Pre.123+withBuild").isPreRelease); + + assertFalse(VersionEncoder.encodeVersion("1").isPreRelease); + assertFalse(VersionEncoder.encodeVersion("1.2").isPreRelease); + assertFalse(VersionEncoder.encodeVersion("1.2.3").isPreRelease); + assertFalse(VersionEncoder.encodeVersion("1.2.3+buildSufix").isPreRelease); + assertFalse(VersionEncoder.encodeVersion("1.2.3+buildSufix-withDash").isPreRelease); + } + + public void testVersionPartExtraction() { + int numParts = randomIntBetween(1, 6); + String[] parts = new String[numParts]; + for (int i = 0; i < numParts; i++) { + parts[i] = String.valueOf(randomIntBetween(1, 1000)); + } + EncodedVersion encodedVersion = VersionEncoder.encodeVersion(String.join(".", parts)); + assertEquals(parts[0], encodedVersion.major.toString()); + if (numParts > 1) { + assertEquals(parts[1], encodedVersion.minor.toString()); + } else { + assertNull(encodedVersion.minor); + } + if (numParts > 2) { + assertEquals(parts[2], encodedVersion.patch.toString()); + } else { + assertNull(encodedVersion.patch); + } + } + + public void testMaxDigitGroupLength() { + String versionString = "1.0." + "1".repeat(128); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> decodeVersion(encodeVersion(versionString))); + assertEquals("Groups of digits cannot be longer than 127, but found: 128", ex.getMessage()); + } + + /** + * test that encoding and decoding leads back to the same version string + */ + public void testRandomRoundtrip() { + String versionString = randomVersionString(); + assertEquals(versionString, decodeVersion(encodeVersion(versionString))); + } + + private String randomVersionString() { + StringBuilder sb = new StringBuilder(); + sb.append(randomIntBetween(0, 1000)); + int releaseNumerals = randomIntBetween(0, 4); + for (int i = 0; i < releaseNumerals; i++) { + sb.append("."); + sb.append(randomIntBetween(0, 10000)); + } + // optional pre-release part + if (randomBoolean()) { + sb.append("-"); + int preReleaseParts = randomIntBetween(1, 5); + for (int i = 0; i < preReleaseParts; i++) { + if (randomBoolean()) { + sb.append(randomIntBetween(0, 1000)); + } else { + int alphanumParts = 3; + for (int j = 0; j < alphanumParts; j++) { + if (randomBoolean()) { + sb.append(randomAlphaOfLengthBetween(1, 2)); + } else { + sb.append(randomIntBetween(1, 99)); + } + if (rarely()) { + sb.append(randomFrom(Arrays.asList("-"))); + } + } + } + sb.append("."); + } + sb.deleteCharAt(sb.length() - 1); // remove trailing dot + } + // optional build part + if (randomBoolean()) { + sb.append("+").append(randomAlphaOfLengthBetween(1, 15)); + } + return sb.toString(); + } + + /** + * taken from https://regex101.com/r/vkijKf/1/ via https://semver.org/ + */ + public void testSemVerValidation() { + String[] validSemverVersions = new String[] { + "0.0.4", + "1.2.3", + "10.20.30", + "1.1.2-prerelease+meta", + "1.1.2+meta", + "1.1.2+meta-valid", + "1.0.0-alpha", + "1.0.0-beta", + "1.0.0-alpha.beta", + "1.0.0-alpha.beta.1", + "1.0.0-alpha.1", + "1.0.0-alpha0.valid", + "1.0.0-alpha.0valid", + "1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay", + "1.0.0-rc.1+build.1", + "2.0.0-rc.1+build.123", + "1.2.3-beta", + "10.2.3-DEV-SNAPSHOT", + "1.2.3-SNAPSHOT-123", + "1.0.0", + "2.0.0", + "1.1.7", + "2.0.0+build.1848", + "2.0.1-alpha.1227", + "1.0.0-alpha+beta", + "1.2.3----RC-SNAPSHOT.12.9.1--.12+788", + "1.2.3----R-S.12.9.1--.12+meta", + "1.2.3----RC-SNAPSHOT.12.9.1--.12", + "1.0.0+0.build.1-rc.10000aaa-kk-0.1", + + "999999999.999999999.999999999", + "1.0.0-0A.is.legal", + // the following are not strict semver but we allow them + "1.2-SNAPSHOT", + "1.2-RC-SNAPSHOT", + "1", + "1.2.3.4" }; + for (String version : validSemverVersions) { + assertTrue("should be valid: " + version, VersionEncoder.encodeVersion(version).isLegal); + // since we're here, also check encoding / decoding rountrip + assertEquals(version, decodeVersion(encodeVersion(version))); + } + + String[] invalidSemverVersions = new String[] { + "1.2.3-0123", + "1.2.3-0123.0123", + "1.1.2+.123", + "+invalid", + "-invalid", + "-invalid+invalid", + "-invalid.01", + "alpha", + "alpha.beta", + "alpha.beta.1", + "alpha.1", + "alpha+beta", + "alpha_beta", + "alpha.", + "alpha..", + "beta", + "1.0.0-alpha_beta", + "-alpha.", + "1.0.0-alpha..", + "1.0.0-alpha..1", + "1.0.0-alpha...1", + "1.0.0-alpha....1", + "1.0.0-alpha.....1", + "1.0.0-alpha......1", + "1.0.0-alpha.......1", + "01.1.1", + "1.01.1", + "1.1.01", + "1.2.3.DEV", + "1.2.31.2.3----RC-SNAPSHOT.12.09.1--..12+788", + "-1.0.3-gamma+b7718", + "+justmeta", + "9.8.7+meta+meta", + "9.8.7-whatever+meta+meta", + "999999999.999999999.999999999.----RC-SNAPSHOT.12.09.1--------------------------------..12", + "12.el2", + "12.el2-1.0-rc5" }; + for (String version : invalidSemverVersions) { + assertFalse("should be invalid: " + version, VersionEncoder.encodeVersion(version).isLegal); + // since we're here, also check encoding / decoding rountrip + assertEquals(version, decodeVersion(encodeVersion(version))); + } + } +} diff --git a/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java new file mode 100644 index 0000000000000..21536cdbbba8d --- /dev/null +++ b/x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -0,0 +1,453 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +public class VersionStringFieldMapperTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + // return pluginList(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class, PainlessPlugin.class); + // TODO PainlessPlugin loading doesn't work when test is run through "gradle check" + return pluginList(VersionFieldPlugin.class, LocalStateCompositeXPackPlugin.class); + } + + public String setUpIndex(String indexName) throws IOException { + createIndex( + indexName, + Settings.builder().put("index.number_of_shards", 1).build(), + "_doc", + "version", + "type=version", + "foo", + "type=keyword" + ); + ensureGreen(indexName); + + client().prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "11.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "1.0.0").endObject()).get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("4") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) + .get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("6").setSource(jsonBuilder().startObject().field("version", "21.11.0").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + return indexName; + } + + public void testExactQueries() throws Exception { + String indexName = "test"; + setUpIndex(indexName); + + // match + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", ("1.0.0"))).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.4.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.3.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.3.0+build.1234567")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + // term + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.0.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.4.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.3.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termQuery("version", "1.3.0+build.1234567")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + // terms + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termsQuery("version", "1.0.0", "1.3.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.termsQuery("version", "1.4.0", "1.3.0+build.1234567")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + // phrase query (just for keyword compatibility) + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchPhraseQuery("version", "2.1.0-alpha.beta")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + } + + public void testRangeQueries() throws Exception { + String indexName = setUpIndex("test"); + SearchResponse response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.rangeQuery("version").from("1.0.0").to("3.0.0")) + .get(); + assertEquals(4, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.1.0").to("3.0.0")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.rangeQuery("version").from("0.1.0").to("2.1.0-alpha.beta")) + .get(); + assertEquals(3, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("2.1.0").to("3.0.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("3.0.0").to("4.0.0")).get(); + assertEquals(0, response.getHits().getTotalHits().value); + + // ranges excluding edges + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.0.0", false).to("3.0.0")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.0.0").to("2.1.0", false)).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + // open ranges + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("1.4.0")).get(); + assertEquals(4, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("1.4.0")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + } + + public void testPrefixQuery() throws IOException { + String indexName = setUpIndex("test"); + // prefix + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "1")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2.1")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2.1.0-")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "1.3.0+b")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "2")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.1")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.prefixQuery("version", "21.11")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + } + + public void testSort() throws IOException { + String indexName = setUpIndex("test"); + + // sort based on version field + SearchResponse response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.matchAllQuery()) + .addSort("version", SortOrder.DESC) + .get(); + assertEquals(6, response.getHits().getTotalHits().value); + SearchHit[] hits = response.getHits().getHits(); + assertEquals("21.11.0", hits[0].getSortValues()[0]); + assertEquals("11.1.0", hits[1].getSortValues()[0]); + assertEquals("2.1.0", hits[2].getSortValues()[0]); + assertEquals("2.1.0-alpha.beta", hits[3].getSortValues()[0]); + assertEquals("1.3.0+build.1234567", hits[4].getSortValues()[0]); + assertEquals("1.0.0", hits[5].getSortValues()[0]); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC).get(); + assertEquals(6, response.getHits().getTotalHits().value); + hits = response.getHits().getHits(); + assertEquals("1.0.0", hits[0].getSortValues()[0]); + assertEquals("1.3.0+build.1234567", hits[1].getSortValues()[0]); + assertEquals("2.1.0-alpha.beta", hits[2].getSortValues()[0]); + assertEquals("2.1.0", hits[3].getSortValues()[0]); + assertEquals("11.1.0", hits[4].getSortValues()[0]); + assertEquals("21.11.0", hits[5].getSortValues()[0]); + } + + public void testWildcardQuery() throws Exception { + String indexName = "test_wildcard"; + createIndex( + indexName, + Settings.builder().put("index.number_of_shards", 1).build(), + "_doc", + "version", + "type=version", + "foo", + "type=keyword" + ); + ensureGreen(indexName); + + client().prepareIndex(indexName) + .setId("1") + .setSource(jsonBuilder().startObject().field("version", "1.0.0-alpha.2.1.0-rc.1").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("2") + .setSource(jsonBuilder().startObject().field("version", "1.3.0+build.1234567").endObject()) + .get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha.beta").endObject()) + .get(); + client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "2.33.0").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + + // wildcard + SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*alpha*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*b*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*bet*")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "2.1*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "2.1.0-*")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.1.0-*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.1.0*")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.?.0*")).get(); + assertEquals(3, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*2.??.0*")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "?.1.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "*-*")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.wildcardQuery("version", "1.3.0+b*")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + } + + public void testPreReleaseFlag() throws IOException { + String indexName = setUpIndex("test"); + SearchResponse response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.isPreRelease", true))) + .get(); + assertEquals(1, response.getHits().getTotalHits().value); + } + + public void testMainVersionParts() throws IOException { + String indexName = setUpIndex("test"); + SearchResponse response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.major", 11))) + .get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.minor", 1))) + .get(); + assertEquals(3, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName) + .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.matchQuery("version.patch", 0))) + .get(); + assertEquals(6, response.getHits().getTotalHits().value); + } + + public void testStoreMalformed() throws Exception { + String indexName = "test_malformed"; + createIndex( + indexName, + Settings.builder().put("index.number_of_shards", 1).build(), + "_doc", + "version", + "type=version,store_malformed=true" + ); + ensureGreen(indexName); + + client().prepareIndex(indexName) + .setId("1") + .setSource(jsonBuilder().startObject().field("version", "1.invalid.0").endObject()) + .get(); + client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "2.2.0").endObject()).get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.2.0-badchar!").endObject()) + .get(); + client().admin().indices().prepareRefresh(indexName).get(); + + SearchResponse response = client().prepareSearch(indexName).addDocValueField("version").get(); + assertEquals(3, response.getHits().getTotalHits().value); + assertEquals("1", response.getHits().getAt(0).getId()); + assertEquals("1.invalid.0", response.getHits().getAt(0).field("version").getValue()); + + assertEquals("2", response.getHits().getAt(1).getId()); + assertEquals("2.2.0", response.getHits().getAt(1).field("version").getValue()); + + assertEquals("3", response.getHits().getAt(2).getId()); + assertEquals("2.2.0-badchar!", response.getHits().getAt(2).field("version").getValue()); + + // exact match for malformed term + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "1.invalid.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("version", "2.2.0-badchar!")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + + // also should appear in terms aggs + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version")).get(); + Terms terms = response.getAggregations().get("myterms"); + List buckets = terms.getBuckets(); + + assertEquals(3, buckets.size()); + assertEquals("2.2.0", buckets.get(0).getKey()); + assertEquals("1.invalid.0", buckets.get(1).getKey()); + assertEquals("2.2.0-badchar!", buckets.get(2).getKey()); + + // invalid values should sort after all valid ones + response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery()).addSort("version", SortOrder.ASC).get(); + assertEquals(3, response.getHits().getTotalHits().value); + SearchHit[] hits = response.getHits().getHits(); + assertEquals("2.2.0", hits[0].getSortValues()[0]); + assertEquals("1.invalid.0", hits[1].getSortValues()[0]); + assertEquals("2.2.0-badchar!", hits[2].getSortValues()[0]); + + // ranges can include them, but they are sorted last + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("3.0.0")).get(); + assertEquals(1, response.getHits().getTotalHits().value); + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").from("3.0.0")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + } + + public void testAggs() throws Exception { + String indexName = "test_aggs"; + createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).build(), "_doc", "version", "type=version"); + ensureGreen(indexName); + + client().prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "1.0").endObject()).get(); + client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().field("version", "1.3.0").endObject()).get(); + client().prepareIndex(indexName) + .setId("3") + .setSource(jsonBuilder().startObject().field("version", "2.1.0-alpha").endObject()) + .get(); + client().prepareIndex(indexName).setId("4").setSource(jsonBuilder().startObject().field("version", "2.1.0").endObject()).get(); + client().prepareIndex(indexName).setId("5").setSource(jsonBuilder().startObject().field("version", "3.11.5").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + + // terms aggs + SearchResponse response = client().prepareSearch(indexName) + .addAggregation(AggregationBuilders.terms("myterms").field("version")) + .get(); + Terms terms = response.getAggregations().get("myterms"); + List buckets = terms.getBuckets(); + + assertEquals(5, buckets.size()); + assertEquals("1.0", buckets.get(0).getKey()); + assertEquals("1.3.0", buckets.get(1).getKey()); + assertEquals("2.1.0-alpha", buckets.get(2).getKey()); + assertEquals("2.1.0", buckets.get(3).getKey()); + assertEquals("3.11.5", buckets.get(4).getKey()); + + // test terms aggs on version parts + + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.major")).get(); + terms = response.getAggregations().get("myterms"); + buckets = terms.getBuckets(); + assertEquals(3, buckets.size()); + assertEquals(1L, buckets.get(0).getKey()); + assertEquals(2L, buckets.get(0).getDocCount()); + assertEquals(2L, buckets.get(1).getKey()); + assertEquals(2L, buckets.get(1).getDocCount()); + assertEquals(3L, buckets.get(2).getKey()); + assertEquals(1L, buckets.get(2).getDocCount()); + + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.minor")).get(); + terms = response.getAggregations().get("myterms"); + buckets = terms.getBuckets(); + assertEquals(4, buckets.size()); + assertEquals(1L, buckets.get(0).getKey()); + assertEquals(2L, buckets.get(0).getDocCount()); + assertEquals(0L, buckets.get(1).getKey()); + assertEquals(1L, buckets.get(1).getDocCount()); + assertEquals(3L, buckets.get(2).getKey()); + assertEquals(1L, buckets.get(2).getDocCount()); + assertEquals(11L, buckets.get(3).getKey()); + assertEquals(1L, buckets.get(3).getDocCount()); + + response = client().prepareSearch(indexName).addAggregation(AggregationBuilders.terms("myterms").field("version.patch")).get(); + terms = response.getAggregations().get("myterms"); + buckets = terms.getBuckets(); + assertEquals(2, buckets.size()); + assertEquals(0L, buckets.get(0).getKey()); + assertEquals(3L, buckets.get(0).getDocCount()); + assertEquals(5L, buckets.get(1).getKey()); + assertEquals(1L, buckets.get(1).getDocCount()); + } + + public void testNullValue() throws Exception { + String indexName = "test_nullvalue"; + createIndex( + indexName, + Settings.builder().put("index.number_of_shards", 1).build(), + "_doc", + "version", + "type=version,null_value=0.0.0" + ); + ensureGreen(indexName); + + client().prepareIndex(indexName).setId("1").setSource(jsonBuilder().startObject().field("version", "2.2.0").endObject()).get(); + client().prepareIndex(indexName).setId("2").setSource(jsonBuilder().startObject().nullField("version").endObject()).get(); + client().admin().indices().prepareRefresh(indexName).get(); + + SearchResponse response = client().prepareSearch(indexName).addDocValueField("version").addSort("version", SortOrder.ASC).get(); + assertEquals(2, response.getHits().getTotalHits().value); + assertEquals("2", response.getHits().getAt(0).getId()); + assertEquals("0.0.0", response.getHits().getAt(0).field("version").getValue()); + + assertEquals("1", response.getHits().getAt(1).getId()); + assertEquals("2.2.0", response.getHits().getAt(1).field("version").getValue()); + + // range + response = client().prepareSearch(indexName).setQuery(QueryBuilders.rangeQuery("version").to("3.0.0")).get(); + assertEquals(2, response.getHits().getTotalHits().value); + } + + // TODO how to test scripts in ESSingleNodeTestCase? + // Currently fails with "java.lang.IllegalArgumentException: Illegal list shortcut value [value]". + + // public void testScripting() throws Exception { + // String indexName = setUpIndex("test"); + // + // ScriptQueryBuilder query = QueryBuilders.scriptQuery(new Script("doc['version'].value.length() <= 5")); + // SearchResponse response = client().prepareSearch(indexName).addDocValueField("version").setQuery(query).get(); + // assertEquals(2, response.getHits().getTotalHits().value); + // assertEquals("2", response.getHits().getAt(0).getId()); + // assertEquals("1.0.0", response.getHits().getAt(0).field("version").getValue()); + // + // assertEquals("5", response.getHits().getAt(1).getId()); + // assertEquals("2.1.0", response.getHits().getAt(1).field("version").getValue()); + // } +}