From 5b2ae4451ecf70defd92d87697c0e1d15bb7ea2d Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Tue, 17 Nov 2020 16:56:40 +0100 Subject: [PATCH 01/28] Add unsigned_long type support to QL This commit introduces the UNSIGNED_LONG type to QL, following it's availabilty in ES. Just like in ES, the type is encoded as a BigInteger. JDBC is now also UNSIGNED_LONG enabled. In SQL, the type is exposed only to driver clients of a version equal or greater to 7.11.0 (and all non-driver clients, irrespective of their version). --- .../xpack/eql/parser/ExpressionBuilder.java | 28 +-- .../xpack/eql/session/EqlSession.java | 2 +- .../xpack/eql/analysis/VerifierTests.java | 1 + .../src/test/resources/mapping-numeric.json | 3 + .../extractor/AbstractFieldHitExtractor.java | 36 ++-- .../operator/arithmetic/Arithmetics.java | 53 +++++- .../xpack/ql/index/IndexResolver.java | 49 +++-- .../xpack/ql/type/DataTypeConverter.java | 123 ++++++++++--- .../xpack/ql/type/DataTypeRegistry.java | 7 + .../xpack/ql/type/DataTypes.java | 17 +- .../elasticsearch/xpack/ql/util/Check.java | 11 ++ .../xpack/ql/util/StringUtils.java | 18 ++ .../xpack/ql/util/VersionUtil.java | 26 +++ .../BinaryArithmeticProcessorTests.java | 95 +++++++++- .../ql/type/DataTypeConversionTests.java | 144 +++++++++++++++ .../mapping-multi-field-variation.json | 1 + .../mapping-multi-field-with-nested.json | 1 + .../src/test/resources/mapping-numeric.json | 3 + .../elasticsearch/xpack/sql/jdbc/EsType.java | 1 + .../xpack/sql/jdbc/JdbcPreparedStatement.java | 10 +- .../xpack/sql/jdbc/TypeConverter.java | 67 ++++++- .../xpack/sql/jdbc/TypeUtils.java | 20 +- .../sql/jdbc/JdbcPreparedStatementTests.java | 56 ++++++ .../xpack/sql/jdbc/TypeConverterTests.java | 44 ++++- .../xpack/sql/qa/jdbc/JdbcTestUtils.java | 39 ++++ .../qa/jdbc/PreparedStatementTestCase.java | 125 ++++++++----- .../qa/jdbc/ResultSetMetaDataTestCase.java | 10 +- .../xpack/sql/qa/jdbc/ResultSetTestCase.java | 171 ++++++++++++++++-- .../xpack/sql/qa/SqlProtocolTestCase.java | 2 + .../xpack/sql/proto/SqlVersion.java | 7 + .../xpack/sql/proto/SqlVersionTests.java | 5 + .../function/scalar/math/MathProcessor.java | 11 +- .../xpack/sql/parser/ExpressionBuilder.java | 25 ++- .../sql/plan/logical/command/ShowColumns.java | 5 +- .../plan/logical/command/sys/SysColumns.java | 7 +- .../plan/logical/command/sys/SysTypes.java | 2 +- .../plugin/TransportSqlClearCursorAction.java | 4 +- .../sql/plugin/TransportSqlQueryAction.java | 4 +- .../plugin/TransportSqlTranslateAction.java | 2 +- .../xpack/sql/session/SqlConfiguration.java | 13 +- .../xpack/sql/session/SqlSession.java | 7 +- .../xpack/sql/type/SqlDataTypeConverter.java | 53 +++--- .../xpack/sql/type/SqlDataTypeRegistry.java | 6 + .../xpack/sql/type/SqlDataTypes.java | 22 ++- .../elasticsearch/xpack/sql/SqlTestUtils.java | 4 +- .../analyzer/FieldAttributeTests.java | 2 +- .../analysis/index/IndexResolverTests.java | 62 ++++++- .../scalar/DatabaseFunctionTests.java | 4 +- .../function/scalar/UserFunctionTests.java | 4 +- .../math/MathFunctionProcessorTests.java | 21 ++- .../logical/command/sys/SysColumnsTests.java | 169 ++++++++++------- .../logical/command/sys/SysTablesTests.java | 2 +- .../logical/command/sys/SysTypesTests.java | 6 +- .../sql/type/SqlDataTypeConverterTests.java | 48 ++++- 54 files changed, 1357 insertions(+), 301 deletions(-) create mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/VersionUtil.java diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java index e051c4a18e96e..50e34ef167225 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java @@ -9,6 +9,7 @@ import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.TerminalNode; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.xpack.eql.expression.predicate.operator.comparison.InsensitiveEquals; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.ArithmeticUnaryContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.ComparisonContext; @@ -46,6 +47,7 @@ import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.util.StringUtils; +import java.math.BigInteger; import java.time.ZoneId; import java.util.List; @@ -199,10 +201,9 @@ public Literal visitIntegerLiteral(EqlBaseParser.IntegerLiteralContext ctx) { Source source = source(ctx); String text = ctx.getText(); - long value; - + Number value; try { - value = Long.valueOf(StringUtils.parseLong(text)); + value = StringUtils.parseInteger(text); } catch (QlIllegalArgumentException siae) { // if it's too large, then quietly try to parse as a float instead try { @@ -213,15 +214,20 @@ public Literal visitIntegerLiteral(EqlBaseParser.IntegerLiteralContext ctx) { throw new ParsingException(source, siae.getMessage()); } - Object val = Long.valueOf(value); - DataType type = DataTypes.LONG; - - // try to downsize to int if possible (since that's the most common type) - if ((int) value == value) { - type = DataTypes.INTEGER; - val = Integer.valueOf((int) value); + DataType type; + if (value instanceof BigInteger) { + type = DataTypes.UNSIGNED_LONG; + } else { + assert value instanceof Long : "Expected value [" + value + "] of type Long but got: " + value.getClass(); + // try to downsize to int if possible (since that's the most common type) + if (value.longValue() == value.intValue()) { + type = DataTypes.INTEGER; + value = Integer.valueOf(value.intValue()); + } else { + type = DataTypes.LONG; + } } - return new Literal(source, val, type); + return new Literal(source, value, type); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java index 4ba9fdff687e0..05e053e647f19 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java @@ -100,7 +100,7 @@ private void preAnalyze(LogicalPlan parsed, ActionListener list listener.onFailure(new TaskCancelledException("cancelled")); return; } - indexResolver.resolveAsMergedMapping(indexWildcard, null, configuration.indicesOptions(), + indexResolver.resolveAsMergedMapping(indexWildcard, null, configuration.indicesOptions(), null, map(listener, r -> preAnalyzer.preAnalyze(parsed, r)) ); } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java index a2fbe2ca986ce..fcf2f44238af6 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -176,6 +176,7 @@ public void testAliasErrors() { // Test all elasticsearch numeric field types public void testNumeric() { final IndexResolution idxr = loadIndexResolution("mapping-numeric.json"); + accept(idxr, "foo where unsigned_long_field == 0"); accept(idxr, "foo where long_field == 0"); accept(idxr, "foo where integer_field == 0"); accept(idxr, "foo where short_field == 0"); diff --git a/x-pack/plugin/eql/src/test/resources/mapping-numeric.json b/x-pack/plugin/eql/src/test/resources/mapping-numeric.json index 1bf72719a9ff3..e750446d9f149 100644 --- a/x-pack/plugin/eql/src/test/resources/mapping-numeric.json +++ b/x-pack/plugin/eql/src/test/resources/mapping-numeric.json @@ -10,6 +10,9 @@ "@timestamp" : { "type" : "date" }, + "unsigned_long_field" : { + "type": "unsigned_long" + }, "long_field" : { "type" : "long" }, diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java index 8adad589113d6..96e07b28cfd93 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.ql.type.DataTypes; import java.io.IOException; +import java.math.BigInteger; import java.time.ZoneId; import java.util.ArrayDeque; import java.util.Deque; @@ -27,10 +28,14 @@ import java.util.Map; import java.util.Objects; import java.util.StringJoiner; +import java.util.function.BiFunction; +import java.util.function.Function; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.SCALED_FLOAT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; + /** * Extractor for ES fields. Works for both 'normal' fields but also nested ones (which require hitName to be set). * The latter is used as metadata in assembling the results in the tabular response. @@ -182,25 +187,16 @@ protected Object unwrapMultiValue(Object values) { return values; } if (dataType.isNumeric() && isFromDocValuesOnly(dataType) == false) { - if (dataType == DataTypes.DOUBLE || dataType == DataTypes.FLOAT || dataType == DataTypes.HALF_FLOAT) { - Number result = null; - try { - result = numberType(dataType).parse(values, true); - } catch(IllegalArgumentException iae) { - return null; - } - // docvalue_fields is always returning a Double value even if the underlying floating point data type is not Double - // even if we don't extract from docvalue_fields anymore, the behavior should be consistent - return result.doubleValue(); - } else { - Number result = null; - try { - result = numberType(dataType).parse(values, true); - } catch(IllegalArgumentException iae) { - return null; - } - return result; + Number result = null; + try { + // TODO: don't mapper modules expose _source parsing methods? should the _source be (re)validated? + result = dataType == UNSIGNED_LONG ? new BigInteger(values.toString()) : numberType(dataType).parse(values, true); + } catch(IllegalArgumentException iae) { + return null; } + // docvalue_fields is always returning a Double value even if the underlying floating point data type is not Double + // even if we don't extract from docvalue_fields anymore, the behavior should be consistent + return dataType.isRational() ? result.doubleValue() : result; } else if (DataTypes.isString(dataType)) { return values.toString(); } else { @@ -215,7 +211,7 @@ protected boolean isFromDocValuesOnly(DataType dataType) { || dataType == DATETIME || dataType == SCALED_FLOAT; // because of scaling_factor } - + private static NumberType numberType(DataType dataType) { return NumberType.valueOf(dataType.esType().toUpperCase(Locale.ROOT)); } @@ -339,4 +335,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(fieldName, useDocValue, hitName, arrayLeniency); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java index b029ce96cf218..6d2061c75ff6f 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java @@ -7,8 +7,11 @@ import org.elasticsearch.xpack.ql.QlIllegalArgumentException; +import java.math.BigInteger; import java.util.function.BiFunction; +import static org.elasticsearch.xpack.ql.util.Check.isUnsignedLong; + /** * Arithmetic operation using the type widening rules of the JLS 5.6.2 namely * widen to double or float or long or int in this order. @@ -31,6 +34,22 @@ default Object wrap(Object l, Object r) { } } + private static BigInteger unsignedLongOperation(Number l, Number r, BiFunction op) { + BigInteger biLeft, biRight; + if (l instanceof BigInteger) { + biLeft = (BigInteger) l; + biRight = BigInteger.valueOf(r.longValue()); + } else if (r instanceof BigInteger) { + biLeft = BigInteger.valueOf(l.longValue()); + biRight = (BigInteger) r; + } else { + return null; + } + BigInteger bi = op.apply(biLeft, biRight); + isUnsignedLong(bi); + return bi; + } + public static Number add(Number l, Number r) { if (l == null || r == null) { return null; @@ -42,6 +61,10 @@ public static Number add(Number l, Number r) { if (l instanceof Float || r instanceof Float) { return Float.valueOf(l.floatValue() + r.floatValue()); } + BigInteger bi = unsignedLongOperation(l, r, BigInteger::add); + if (bi != null) { + return bi; + } if (l instanceof Long || r instanceof Long) { return Long.valueOf(Math.addExact(l.longValue(), r.longValue())); } @@ -60,6 +83,10 @@ public static Number sub(Number l, Number r) { if (l instanceof Float || r instanceof Float) { return Float.valueOf(l.floatValue() - r.floatValue()); } + BigInteger bi = unsignedLongOperation(l, r, BigInteger::subtract); + if (bi != null) { + return bi; + } if (l instanceof Long || r instanceof Long) { return Long.valueOf(Math.subtractExact(l.longValue(), r.longValue())); } @@ -78,6 +105,16 @@ public static Number mul(Number l, Number r) { if (l instanceof Float || r instanceof Float) { return Float.valueOf(l.floatValue() * r.floatValue()); } + // Note: in case of unsigned_long overflow (or underflow, with negative fixed point numbers), the exception is thrown. + // This is unlike the way some other traditional RDBMS that support unsigned types work, which simply promote the result to a + // floating point type, but in line with how our implementation treats other fixed point type operations (i.e. Math#xxExact()). + // The reason for our behavior is (prolly) the need to establish a schema based on an index mapping. Noteworthy however is also + // that we're not strictly consistent with the returned type: a `SUM(field)` aggregation can return a floating point, for example. + // TODO: document this last point or address it? + BigInteger bi = unsignedLongOperation(l, r, BigInteger::multiply); + if (bi != null) { + return bi; + } if (l instanceof Long || r instanceof Long) { return Long.valueOf(Math.multiplyExact(l.longValue(), r.longValue())); } @@ -96,6 +133,10 @@ public static Number div(Number l, Number r) { if (l instanceof Float || r instanceof Float) { return l.floatValue() / r.floatValue(); } + BigInteger bi = unsignedLongOperation(l, r, BigInteger::divide); + if (bi != null) { + return bi; + } if (l instanceof Long || r instanceof Long) { return l.longValue() / r.longValue(); } @@ -114,6 +155,10 @@ public static Number mod(Number l, Number r) { if (l instanceof Float || r instanceof Float) { return Float.valueOf(l.floatValue() % r.floatValue()); } + BigInteger bi = unsignedLongOperation(l, r, BigInteger::remainder); + if (bi != null) { + return bi; + } if (l instanceof Long || r instanceof Long) { return Long.valueOf(l.longValue() % r.longValue()); } @@ -140,10 +185,16 @@ static Number negate(Number n) { } return Float.valueOf(-n.floatValue()); } + if (n instanceof BigInteger) { + if (((BigInteger) n).signum() != 0) { + throw new ArithmeticException("unsigned_long overflow"); + } + return n; + } if (n instanceof Long) { return Long.valueOf(Math.negateExact(n.longValue())); } return Integer.valueOf(Math.negateExact(n.intValue())); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index 92dd8a7035d4b..8086eb56a30ed 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -8,6 +8,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; @@ -22,6 +23,7 @@ import org.elasticsearch.action.support.IndicesOptions.WildcardStates; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.util.set.Sets; @@ -278,29 +280,31 @@ private void filterResults(String javaRegex, GetAliasesResponse aliases, GetInde /** * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. */ - public void resolveAsMergedMapping(String indexWildcard, String javaRegex, IndicesOptions indicesOptions, + public void resolveAsMergedMapping(String indexWildcard, String javaRegex, IndicesOptions indicesOptions, @Nullable Version version, ActionListener listener) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, indicesOptions); client.fieldCaps(fieldRequest, ActionListener.wrap( - response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response.getIndices(), response.get())), + response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response.getIndices(), response.get(), + version)), listener::onFailure)); } /** * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. */ - public void resolveAsMergedMapping(String indexWildcard, String javaRegex, boolean includeFrozen, + public void resolveAsMergedMapping(String indexWildcard, String javaRegex, boolean includeFrozen, @Nullable Version version, ActionListener listener) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen); client.fieldCaps(fieldRequest, ActionListener.wrap( - response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response.getIndices(), response.get())), + response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response.getIndices(), response.get(), + version)), listener::onFailure)); } public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, String indexPattern, String[] indexNames, - Map> fieldCaps) { + Map> fieldCaps, @Nullable Version version) { if (indexNames.length == 0) { return IndexResolution.notFound(indexPattern); @@ -359,7 +363,8 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri // everything checks return null; - }); + }, + version); if (indices.size() > 1) { throw new QlIllegalArgumentException( @@ -374,7 +379,8 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa Map> globalCaps, Map hierarchicalMapping, Map flattedMapping, - Function field) { + Function field, + Version version) { Map parentProps = hierarchicalMapping; @@ -393,7 +399,7 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa // lack of parent implies the field is an alias if (map == null) { // as such, create the field manually, marking the field to also be an alias - fieldFunction = s -> createField(typeRegistry, s, OBJECT.esType(), new TreeMap<>(), false, true); + fieldFunction = s -> createField(typeRegistry, s, OBJECT.esType(), new TreeMap<>(), false, true, version); } else { Iterator iterator = map.values().iterator(); FieldCapabilities parentCap = iterator.next(); @@ -401,10 +407,11 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa parentCap = iterator.next(); } final FieldCapabilities parentC = parentCap; - fieldFunction = s -> createField(typeRegistry, s, parentC.getType(), new TreeMap<>(), parentC.isAggregatable(), false); + fieldFunction = s -> createField(typeRegistry, s, parentC.getType(), new TreeMap<>(), parentC.isAggregatable(), false, + version); } - parent = createField(typeRegistry, parentName, globalCaps, hierarchicalMapping, flattedMapping, fieldFunction); + parent = createField(typeRegistry, parentName, globalCaps, hierarchicalMapping, flattedMapping, fieldFunction, version); } parentProps = parent.getProperties(); } @@ -433,8 +440,8 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa } private static EsField createField(DataTypeRegistry typeRegistry, String fieldName, String typeName, Map props, - boolean isAggregateable, boolean isAlias) { - DataType esType = typeRegistry.fromEs(typeName); + boolean isAggregateable, boolean isAlias, Version version) { + DataType esType = typeRegistry.fromEs(typeName, version); if (esType == TEXT) { return new TextEsField(fieldName, props, false, isAlias); @@ -473,15 +480,17 @@ private static FieldCapabilitiesRequest createFieldCapsRequest(String index, boo /** * Resolves a pattern to multiple, separate indices. Doesn't perform validation. */ - public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, boolean includeFrozen, + public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, boolean includeFrozen, @Nullable Version version, ActionListener> listener) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen); client.fieldCaps(fieldRequest, wrap(response -> { client.admin().indices().getAliases(createGetAliasesRequest(response, includeFrozen), wrap(aliases -> - listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), aliases.getAliases())), + listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), aliases.getAliases(), + version)), ex -> { if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) { - listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), null)); + listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), null, + version)); } else { listener.onFailure(ex); } @@ -500,8 +509,9 @@ private GetAliasesRequest createGetAliasesRequest(FieldCapabilitiesResponse resp } public static List separateMappings(DataTypeRegistry typeRegistry, String javaRegex, String[] indexNames, - Map> fieldCaps, ImmutableOpenMap> aliases) { - return buildIndices(typeRegistry, indexNames, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null); + Map> fieldCaps, ImmutableOpenMap> aliases, + @Nullable Version version) { + return buildIndices(typeRegistry, indexNames, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null, version); } private static class Fields { @@ -516,7 +526,7 @@ private static class Fields { private static List buildIndices(DataTypeRegistry typeRegistry, String[] indexNames, String javaRegex, Map> fieldCaps, ImmutableOpenMap> aliases, Function indexNameProcessor, - BiFunction, InvalidMappedField> validityVerifier) { + BiFunction, InvalidMappedField> validityVerifier, @Nullable Version version) { if ((indexNames == null || indexNames.length == 0) && (aliases == null || aliases.isEmpty())) { return emptyList(); @@ -645,7 +655,8 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ createField(typeRegistry, fieldName, fieldCaps, indexFields.hierarchicalMapping, indexFields.flattedMapping, s -> invalidField != null ? invalidField : createField(typeRegistry, s, typeCap.getType(), emptyMap(), typeCap.isAggregatable(), - isAliasFieldType.get())); + isAliasFieldType.get(), version), + version); } } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java index 7e7b016fb9a3c..7c9e253098741 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java @@ -12,12 +12,13 @@ import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; import java.time.ZonedDateTime; import java.time.format.DateTimeParseException; import java.util.Locale; import java.util.function.DoubleFunction; import java.util.function.Function; -import java.util.function.LongFunction; import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE; @@ -31,8 +32,11 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.NULL; import static org.elasticsearch.xpack.ql.type.DataTypes.SHORT; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; import static org.elasticsearch.xpack.ql.type.DataTypes.isString; +import static org.elasticsearch.xpack.ql.util.Check.UNSIGNED_LONG_MAX; +import static org.elasticsearch.xpack.ql.util.Check.isUnsignedLong; /** * Conversion utility from one Elasticsearch data type to another Elasticsearch data types. @@ -72,6 +76,9 @@ public static DataType commonType(DataType left, DataType right) { if (left.isInteger()) { // promote the highest int if (right.isInteger()) { + if (left == UNSIGNED_LONG || right == UNSIGNED_LONG) { + return UNSIGNED_LONG; + } return left.size() > right.size() ? left : right; } // promote the rational @@ -129,6 +136,9 @@ public static Converter converterFor(DataType from, DataType to) { if (to == LONG) { return conversionToLong(from); } + if (to == UNSIGNED_LONG) { + return conversionToUnsignedLong(from); + } if (to == INTEGER) { return conversionToInt(from); } @@ -170,6 +180,25 @@ private static Converter conversionToIp(DataType from) { return null; } + private static Converter conversionToUnsignedLong(DataType from) { + if (from.isRational()) { + return DefaultConverter.RATIONAL_TO_UNSIGNED_LONG; + } + if (from.isInteger()) { + return DefaultConverter.INTEGER_TO_UNSIGNED_LONG; + } + if (from == BOOLEAN) { + return DefaultConverter.BOOL_TO_UNSIGNED_LONG; + } + if (isString(from)) { + return DefaultConverter.STRING_TO_UNSIGNED_LONG; + } + if (from == DATETIME) { + return DefaultConverter.DATETIME_TO_UNSIGNED_LONG; + } + return null; + } + private static Converter conversionToLong(DataType from) { if (from.isRational()) { return DefaultConverter.RATIONAL_TO_LONG; @@ -334,15 +363,61 @@ public static int safeToInt(long x) { return (int) x; } - public static long safeToLong(double x) { + public static long safeDoubleToLong(double x) { if (x > Long.MAX_VALUE || x < Long.MIN_VALUE) { throw new QlIllegalArgumentException("[" + x + "] out of [long] range"); } return Math.round(x); } + public static Long safeToLong(Number x) { + try { + if (x instanceof BigInteger) { + return ((BigInteger) x).longValueExact(); + } + // integer converters are also provided double values (aggs generated on integer fields) + if (x instanceof Double || x instanceof Float) { + return safeDoubleToLong(x.doubleValue()); + } + return x.longValue(); + } catch (ArithmeticException ae) { + throw new QlIllegalArgumentException("[" + x + "] out of [long] range", ae); + } + } + + // 18446744073709551615.0 + private static final double UNSIGNED_LONG_MAX_AS_DOUBLE = UNSIGNED_LONG_MAX.doubleValue(); + + public static BigInteger safeToUnsignedLong(Double x) { + // UNSIGNED_LONG_MAX can't be represented precisely enough on a double, being converted as a rounded up value. + // Converting it to a double and back will yield a larger unsigned long, so the double comparison is still preferred, but + // it'll require the equality check. (BigDecimal comparisons only make sense for string-recovered floating point numbers.) + // This also means that 18446744073709551615.0 is actually a double too high to be converted as an unsigned long. + if (x < 0 || x >= UNSIGNED_LONG_MAX_AS_DOUBLE) { + throw new QlIllegalArgumentException("[" + x + "] out of [unsigned_long] range"); + } + return BigDecimal.valueOf(x).toBigInteger(); + } + + public static BigInteger safeToUnsignedLong(Long x) { + if (x < 0) { + throw new QlIllegalArgumentException("[" + x + "] out of [unsigned_long] range"); + } + return BigInteger.valueOf(x); + } + + public static BigInteger safeToUnsignedLong(String x) { + BigInteger bi = new BigDecimal(x).toBigInteger(); + try { + isUnsignedLong(bi); + } catch (ArithmeticException ae) { + throw new QlIllegalArgumentException("[" + x + "] out of [unsigned_long] range", ae); + } + return bi; + } + public static Number toInteger(double x, DataType dataType) { - long l = safeToLong(x); + long l = safeDoubleToLong(x); if (dataType == BYTE) { return safeToByte(l); @@ -391,42 +466,47 @@ public static Object convert(Object value, DataType dataType) { public enum DefaultConverter implements Converter { IDENTITY(Function.identity()), TO_NULL(value -> null), - + DATETIME_TO_STRING(o -> DateUtils.toString((ZonedDateTime) o)), OTHER_TO_STRING(String::valueOf), - RATIONAL_TO_LONG(fromDouble(DataTypeConverter::safeToLong)), - INTEGER_TO_LONG(fromLong(value -> value)), + RATIONAL_TO_UNSIGNED_LONG(fromDouble(DataTypeConverter::safeToUnsignedLong)), + INTEGER_TO_UNSIGNED_LONG(fromNumber(value -> DataTypeConverter.safeToUnsignedLong(value.longValue()))), + STRING_TO_UNSIGNED_LONG(fromString(DataTypeConverter::safeToUnsignedLong,"unsigned_long")), + DATETIME_TO_UNSIGNED_LONG(fromDateTime(DataTypeConverter::safeToUnsignedLong)), + + RATIONAL_TO_LONG(fromDouble(DataTypeConverter::safeDoubleToLong)), + INTEGER_TO_LONG(fromNumber(DataTypeConverter::safeToLong)), STRING_TO_LONG(fromString(Long::valueOf, "long")), DATETIME_TO_LONG(fromDateTime(value -> value)), - RATIONAL_TO_INT(fromDouble(value -> safeToInt(safeToLong(value)))), - INTEGER_TO_INT(fromLong(DataTypeConverter::safeToInt)), + RATIONAL_TO_INT(fromDouble(value -> safeToInt(safeDoubleToLong(value)))), + INTEGER_TO_INT(fromNumber(value -> safeToInt(safeToLong(value)))), BOOL_TO_INT(fromBool(value -> value ? 1 : 0)), STRING_TO_INT(fromString(Integer::valueOf, "integer")), DATETIME_TO_INT(fromDateTime(DataTypeConverter::safeToInt)), - RATIONAL_TO_SHORT(fromDouble(value -> safeToShort(safeToLong(value)))), - INTEGER_TO_SHORT(fromLong(DataTypeConverter::safeToShort)), + RATIONAL_TO_SHORT(fromDouble(value -> safeToShort(safeDoubleToLong(value)))), + INTEGER_TO_SHORT(fromNumber(value -> safeToShort(safeToLong(value)))), BOOL_TO_SHORT(fromBool(value -> value ? (short) 1 : (short) 0)), STRING_TO_SHORT(fromString(Short::valueOf, "short")), DATETIME_TO_SHORT(fromDateTime(DataTypeConverter::safeToShort)), - RATIONAL_TO_BYTE(fromDouble(value -> safeToByte(safeToLong(value)))), - INTEGER_TO_BYTE(fromLong(DataTypeConverter::safeToByte)), + RATIONAL_TO_BYTE(fromDouble(value -> safeToByte(safeDoubleToLong(value)))), + INTEGER_TO_BYTE(fromNumber(value -> safeToByte(safeToLong(value)))), BOOL_TO_BYTE(fromBool(value -> value ? (byte) 1 : (byte) 0)), STRING_TO_BYTE(fromString(Byte::valueOf, "byte")), DATETIME_TO_BYTE(fromDateTime(DataTypeConverter::safeToByte)), - // TODO floating point conversions are lossy but conversions to integer conversions are not. Are we ok with that? + // TODO floating point conversions are lossy but conversions to integer are not. Are we ok with that? RATIONAL_TO_FLOAT(fromDouble(value -> (float) value)), - INTEGER_TO_FLOAT(fromLong(value -> (float) value)), + INTEGER_TO_FLOAT(fromNumber(Number::floatValue)), BOOL_TO_FLOAT(fromBool(value -> value ? 1f : 0f)), STRING_TO_FLOAT(fromString(Float::valueOf, "float")), DATETIME_TO_FLOAT(fromDateTime(value -> (float) value)), RATIONAL_TO_DOUBLE(fromDouble(Double::valueOf)), - INTEGER_TO_DOUBLE(fromLong(Double::valueOf)), + INTEGER_TO_DOUBLE(fromNumber(Number::doubleValue)), BOOL_TO_DOUBLE(fromBool(value -> value ? 1d : 0d)), STRING_TO_DOUBLE(fromString(Double::valueOf, "double")), DATETIME_TO_DOUBLE(fromDateTime(Double::valueOf)), @@ -436,10 +516,11 @@ public enum DefaultConverter implements Converter { BOOL_TO_DATETIME(toDateTime(BOOL_TO_INT)), STRING_TO_DATETIME(fromString(DateUtils::asDateTime, "datetime")), - NUMERIC_TO_BOOLEAN(fromLong(value -> value != 0)), + NUMERIC_TO_BOOLEAN(fromDouble(value -> value != 0)), STRING_TO_BOOLEAN(fromString(DataTypeConverter::convertToBoolean, "boolean")), DATETIME_TO_BOOLEAN(fromDateTime(value -> value != 0)), + BOOL_TO_UNSIGNED_LONG(fromBool(value -> value ? BigInteger.ONE : BigInteger.ZERO)), BOOL_TO_LONG(fromBool(value -> value ? 1L : 0L)), STRING_TO_IP(o -> { @@ -461,11 +542,11 @@ private static Function fromDouble(DoubleFunction conver return (Object l) -> converter.apply(((Number) l).doubleValue()); } - private static Function fromLong(LongFunction converter) { - return (Object l) -> converter.apply(((Number) l).longValue()); + private static Function fromNumber(Function converter) { + return l -> converter.apply((Number) l); } - - private static Function fromString(Function converter, String to) { + + public static Function fromString(Function converter, String to) { return (Object value) -> { try { return converter.apply(value.toString()); @@ -519,4 +600,4 @@ public static DataType asInteger(DataType dataType) { return dataType.isInteger() ? dataType : LONG; } -} \ No newline at end of file +} diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeRegistry.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeRegistry.java index 714eef3586362..5b90c0e35c3af 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeRegistry.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeRegistry.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.ql.type; +import org.elasticsearch.Version; + import java.util.Collection; /** @@ -20,6 +22,11 @@ public interface DataTypeRegistry { DataType fromEs(String typeName); + // version-dependent type resolution + default DataType fromEs(String typeName, Version version) { + return fromEs(typeName); + } + DataType fromJava(Object value); boolean isUnsupported(DataType type); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java index 8935ea5a2092f..bcc17c5f86c25 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.ql.type; +import java.math.BigInteger; import java.time.ZonedDateTime; import java.util.Arrays; import java.util.Collection; @@ -28,6 +29,7 @@ public final class DataTypes { public static final DataType SHORT = new DataType("short", Short.BYTES, true, false, true); public static final DataType INTEGER = new DataType("integer", Integer.BYTES, true, false, true); public static final DataType LONG = new DataType("long", Long.BYTES, true, false, true); + public static final DataType UNSIGNED_LONG = new DataType("unsigned_long", Long.BYTES, true, false, true); // decimal numeric public static final DataType DOUBLE = new DataType("double", Double.BYTES, false, true, true); public static final DataType FLOAT = new DataType("float", Float.BYTES, false, true, true); @@ -46,7 +48,7 @@ public final class DataTypes { public static final DataType OBJECT = new DataType("object", 0, false, false, false); public static final DataType NESTED = new DataType("nested", 0, false, false, false); //@formatter:on - + private static final Collection TYPES = Arrays.asList( UNSUPPORTED, NULL, @@ -55,6 +57,7 @@ public final class DataTypes { SHORT, INTEGER, LONG, + UNSIGNED_LONG, DOUBLE, FLOAT, HALF_FLOAT, @@ -69,14 +72,14 @@ public final class DataTypes { .stream() .sorted(Comparator.comparing(DataType::typeName)) .collect(toUnmodifiableList()); - + private static final Map NAME_TO_TYPE = TYPES.stream() .collect(toUnmodifiableMap(DataType::typeName, t -> t)); - + private static final Map ES_TO_TYPE = TYPES.stream() .filter(e -> e.esType() != null) .collect(toUnmodifiableMap(DataType::esType, t -> t)); - + private DataTypes() {} public static Collection types() { @@ -102,6 +105,10 @@ public static DataType fromJava(Object value) { if (value instanceof Long) { return LONG; } + if (value instanceof BigInteger) { + // TODO: range check needed at all? (((BigInteger) value).signum() < 0 + return UNSIGNED_LONG; + } if (value instanceof Boolean) { return BOOLEAN; } @@ -148,7 +155,7 @@ public static boolean isNullOrNumeric(DataType t) { } public static boolean isSigned(DataType t) { - return t.isNumeric(); + return t.isNumeric() && t.equals(UNSIGNED_LONG) == false; } public static boolean areCompatible(DataType left, DataType right) { diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/Check.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/Check.java index 604ab8002dfc3..af23408b14d39 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/Check.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/Check.java @@ -7,6 +7,8 @@ import org.elasticsearch.xpack.ql.QlIllegalArgumentException; +import java.math.BigInteger; + /** * Utility class used for checking various conditions at runtime, with minimum amount of code. */ @@ -47,4 +49,13 @@ public static void isBoolean(Object obj) { throw new QlIllegalArgumentException("A boolean is required; received [{}]", obj); } } + + // 18446744073709551615 + public static final BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); + + public static void isUnsignedLong(BigInteger bi) { + if (bi.signum() < 0 || bi.compareTo(UNSIGNED_LONG_MAX) > 0) { + throw new ArithmeticException("unsigned_long overflow"); + } + } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java index 5fd369c1e3639..e51024af53ca9 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java @@ -22,6 +22,7 @@ import java.util.Locale; import static java.util.stream.Collectors.toList; +import static org.elasticsearch.xpack.ql.util.Check.isUnsignedLong; public final class StringUtils { @@ -326,6 +327,23 @@ public static long parseLong(String string) throws QlIllegalArgumentException { } } + public static Number parseInteger(String string) throws QlIllegalArgumentException { + try { + BigInteger bi = new BigInteger(string); + try { + if (bi.signum() < 0 || bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) <= 0) { + return bi.longValueExact(); + } + isUnsignedLong(bi); + return bi; + } catch (ArithmeticException ae) { + throw new QlIllegalArgumentException("Number [{}] is too large", string); + } + } catch (NumberFormatException ex) { + throw new QlIllegalArgumentException("Cannot parse number [{}]", string); + } + } + public static String ordinal(int i) { switch (i % 100) { case 11: diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/VersionUtil.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/VersionUtil.java new file mode 100644 index 0000000000000..ca47ccb8e249e --- /dev/null +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/VersionUtil.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ql.util; + +import org.elasticsearch.Version; + +import static org.elasticsearch.Version.V_7_11_0; + +/** + * Utility class to enable gradual feature enabling, version-dependent. + */ +public final class VersionUtil { + + private VersionUtil() {} + + /** + * Does the provided {@code version} support the unsigned_long type (PR#60050)? + */ + public static boolean isUnsignedLongSupported(Version version) { + return V_7_11_0.compareTo(version) <= 0; + } +} diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java index dc5dbdbca9d77..a7446ccf1ef2e 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java @@ -14,6 +14,9 @@ import org.elasticsearch.xpack.ql.expression.gen.processor.ConstantProcessor; import org.elasticsearch.xpack.ql.expression.gen.processor.Processor; import org.elasticsearch.xpack.ql.expression.processor.Processors; +import org.elasticsearch.xpack.ql.util.Check; + +import java.math.BigInteger; import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; @@ -45,16 +48,58 @@ public void testAdd() { assertEquals(10, ba.process(null)); } + public void testUnsignedLongAdd() { + Processor ba = new Add(EMPTY, l(BigInteger.valueOf(7)), l(3)).makePipe().asProcessor(); + assertEquals(BigInteger.valueOf(10), ba.process(null)); + + ba = new Add(EMPTY, l(BigInteger.valueOf(7)), l((short) -3)).makePipe().asProcessor(); + assertEquals(BigInteger.valueOf(4), ba.process(null)); + + ba = new Add(EMPTY, l(BigInteger.valueOf(7)), l(-3f)).makePipe().asProcessor(); + assertEquals(4f, ba.process(null)); + + Processor pn = new Add(EMPTY, l(BigInteger.valueOf(7)), l(-8)).makePipe().asProcessor(); + expectThrows(ArithmeticException.class, () -> pn.process(null)); + + Processor pm = new Add(EMPTY, l(Check.UNSIGNED_LONG_MAX), l(1)).makePipe().asProcessor(); + expectThrows(ArithmeticException.class, () -> pm.process(null)); + } + public void testSub() { Processor ba = new Sub(EMPTY, l(7), l(3)).makePipe().asProcessor(); assertEquals(4, ba.process(null)); } + public void testUnsignedLongSub() { + Processor bs = new Sub(EMPTY, l(BigInteger.valueOf(7)), l(3)).makePipe().asProcessor(); + assertEquals(BigInteger.valueOf(4), bs.process(null)); + + bs = new Sub(EMPTY, l(BigInteger.valueOf(7)), l((short) -3)).makePipe().asProcessor(); + assertEquals(BigInteger.valueOf(10), bs.process(null)); + + bs = new Sub(EMPTY, l(BigInteger.valueOf(7)), l(3f)).makePipe().asProcessor(); + assertEquals(4f, bs.process(null)); + + Processor proc = new Sub(EMPTY, l(BigInteger.valueOf(7)), l(8)).makePipe().asProcessor(); + expectThrows(ArithmeticException.class, () -> proc.process(null)); + } + public void testMul() { Processor ba = new Mul(EMPTY, l(7), l(3)).makePipe().asProcessor(); assertEquals(21, ba.process(null)); } + public void testUnsignedLongMul() { + Processor bm = new Mul(EMPTY, l(BigInteger.valueOf(7)), l(3)).makePipe().asProcessor(); + assertEquals(BigInteger.valueOf(21), bm.process(null)); + + bm = new Mul(EMPTY, l(BigInteger.valueOf(7)), l(3f)).makePipe().asProcessor(); + assertEquals(21f, bm.process(null)); + + Processor proc = new Mul(EMPTY, l(BigInteger.valueOf(7)), l(-8)).makePipe().asProcessor(); + expectThrows(ArithmeticException.class, () -> proc.process(null)); + } + public void testDiv() { Processor ba = new Div(EMPTY, l(7), l(3)).makePipe().asProcessor(); assertEquals(2, ((Number) ba.process(null)).longValue()); @@ -62,16 +107,46 @@ public void testDiv() { assertEquals(2.33, ((Number) ba.process(null)).doubleValue(), 0.01d); } + public void testUnsignedLongDiv() { + Processor bd = new Div(EMPTY, l(BigInteger.valueOf(7)), l(3)).makePipe().asProcessor(); + assertEquals(BigInteger.TWO, bd.process(null)); + + bd = new Div(EMPTY, l(7), l(BigInteger.valueOf(8))).makePipe().asProcessor(); + assertEquals(BigInteger.ZERO, bd.process(null)); + + bd = new Div(EMPTY, l(BigInteger.valueOf(7)), l(3f)).makePipe().asProcessor(); + assertEquals(7/3f, bd.process(null)); + + Processor proc = new Div(EMPTY, l(BigInteger.valueOf(7)), l(-2)).makePipe().asProcessor(); + expectThrows(ArithmeticException.class, () -> proc.process(null)); + } + public void testMod() { Processor ba = new Mod(EMPTY, l(7), l(3)).makePipe().asProcessor(); assertEquals(1, ba.process(null)); } + public void testUnsignedLongMod() { + Processor bm = new Mod(EMPTY, l(BigInteger.valueOf(7)), l(3)).makePipe().asProcessor(); + assertEquals(BigInteger.valueOf(1), bm.process(null)); + + Processor proc = new Mod(EMPTY, l(-7), l(BigInteger.valueOf(3))).makePipe().asProcessor(); + expectThrows(ArithmeticException.class, () -> proc.process(null)); + } + public void testNegate() { Processor ba = new Neg(EMPTY, l(7)).asPipe().asProcessor(); assertEquals(-7, ba.process(null)); } - + + public void testUnsignedLongNegate() { + Processor nm = new Neg(EMPTY, l(BigInteger.valueOf(0))).makePipe().asProcessor(); + assertEquals(BigInteger.ZERO, nm.process(null)); + + Processor proc = new Neg(EMPTY, l(BigInteger.valueOf(3))).makePipe().asProcessor(); + expectThrows(ArithmeticException.class, () -> proc.process(null)); + } + // ((3*2+4)/2-2)%2 public void testTree() { Expression mul = new Mul(EMPTY, l(3), l(2)); @@ -79,11 +154,23 @@ public void testTree() { Expression div = new Div(EMPTY, add, l(2)); Expression sub = new Sub(EMPTY, div, l(2)); Mod mod = new Mod(EMPTY, sub, l(2)); - + Processor proc = mod.makePipe().asProcessor(); assertEquals(1, proc.process(null)); } + // ((3*2+4)/2-2)%2 + public void testUnsignedLongTree() { + Expression mul = new Mul(EMPTY, l(3), l(BigInteger.TWO)); + Expression add = new Add(EMPTY, mul, l(4)); + Expression div = new Div(EMPTY, add, l(2)); + Expression sub = new Sub(EMPTY, div, l(2)); + Mod mod = new Mod(EMPTY, sub, l(2)); + + Processor proc = mod.makePipe().asProcessor(); + assertEquals(BigInteger.ONE, proc.process(null)); + } + public void testHandleNull() { assertNull(new Add(EMPTY, l(null), l(3)).makePipe().asProcessor().process(null)); assertNull(new Sub(EMPTY, l(null), l(3)).makePipe().asProcessor().process(null)); @@ -92,8 +179,8 @@ public void testHandleNull() { assertNull(new Mod(EMPTY, l(null), l(3)).makePipe().asProcessor().process(null)); assertNull(new Neg(EMPTY, l(null)).makePipe().asProcessor().process(null)); } - + private static Literal l(Object value) { return TestUtils.of(EMPTY, value); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java index 39193ce901863..4f1e32037c99c 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java @@ -11,6 +11,8 @@ import org.elasticsearch.xpack.ql.tree.Location; import org.elasticsearch.xpack.ql.tree.Source; +import java.math.BigDecimal; +import java.math.BigInteger; import java.time.ZonedDateTime; import static org.elasticsearch.xpack.ql.type.DataTypeConverter.commonType; @@ -27,8 +29,10 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.NULL; import static org.elasticsearch.xpack.ql.type.DataTypes.SHORT; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; import static org.elasticsearch.xpack.ql.type.DateUtils.asDateTime; +import static org.elasticsearch.xpack.ql.util.Check.UNSIGNED_LONG_MAX; public class DataTypeConversionTests extends ESTestCase { @@ -39,6 +43,12 @@ public void testConversionToString() { assertNull(conversion.convert(null)); assertEquals("10.0", conversion.convert(10.0)); } + { + Converter conversion = converterFor(UNSIGNED_LONG, to); + assertNull(conversion.convert(null)); + BigInteger bi = randomBigInteger(); + assertEquals(bi.toString(), conversion.convert(bi)); + } { Converter conversion = converterFor(DATETIME, to); assertNull(conversion.convert(null)); @@ -61,6 +71,16 @@ public void testConversionToLong() { Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(Double.MAX_VALUE)); assertEquals("[" + Double.MAX_VALUE + "] out of [long] range", e.getMessage()); } + { + Converter conversion = converterFor(UNSIGNED_LONG, to); + assertNull(conversion.convert(null)); + BigInteger bi = randomBigInteger(); + assertEquals(bi.longValue(), conversion.convert(bi)); + + BigInteger longPlus = bi.add(BigInteger.valueOf(Long.MAX_VALUE)); + Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(longPlus)); + assertEquals("[" + longPlus + "] out of [long] range", e.getMessage()); + } { Converter conversion = converterFor(INTEGER, to); assertNull(conversion.convert(null)); @@ -100,6 +120,16 @@ public void testConversionToDateTime() { Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(Double.MAX_VALUE)); assertEquals("[" + Double.MAX_VALUE + "] out of [long] range", e.getMessage()); } + { + Converter conversion = converterFor(UNSIGNED_LONG, to); + assertNull(conversion.convert(null)); + BigInteger bi = randomBigInteger(); + assertEquals(asDateTime(bi.longValue()), conversion.convert(bi)); + + BigInteger longPlus = bi.add(BigInteger.valueOf(Long.MAX_VALUE)); + Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(longPlus)); + assertEquals("[" + longPlus + "] out of [long] range", e.getMessage()); + } { Converter conversion = converterFor(INTEGER, to); assertNull(conversion.convert(null)); @@ -154,6 +184,13 @@ public void testConversionToFloat() { assertEquals(10.1f, (float) conversion.convert(10.1d), 0.00001); assertEquals(10.6f, (float) conversion.convert(10.6d), 0.00001); } + { + Converter conversion = converterFor(UNSIGNED_LONG, to); + assertNull(conversion.convert(null)); + + BigInteger bi = randomBigInteger(); + assertEquals(bi.floatValue(), (float) conversion.convert(bi), 0); + } { Converter conversion = converterFor(INTEGER, to); assertNull(conversion.convert(null)); @@ -192,6 +229,13 @@ public void testConversionToDouble() { assertEquals(10.1, (double) conversion.convert(10.1f), 0.00001); assertEquals(10.6, (double) conversion.convert(10.6f), 0.00001); } + { + Converter conversion = converterFor(UNSIGNED_LONG, to); + assertNull(conversion.convert(null)); + + BigInteger bi = randomBigInteger(); + assertEquals(bi.doubleValue(), (double) conversion.convert(bi), 0); + } { Converter conversion = converterFor(INTEGER, to); assertNull(conversion.convert(null)); @@ -230,6 +274,12 @@ public void testConversionToBoolean() { assertEquals(true, conversion.convert(-10.0f)); assertEquals(false, conversion.convert(0.0f)); } + { + Converter conversion = converterFor(UNSIGNED_LONG, to); + assertNull(conversion.convert(null)); + assertEquals(true, conversion.convert(BigInteger.valueOf(randomNonNegativeLong()))); + assertEquals(false, conversion.convert(BigInteger.ZERO)); + } { Converter conversion = converterFor(INTEGER, to); assertNull(conversion.convert(null)); @@ -282,6 +332,68 @@ public void testConversionToBoolean() { } } + public void testConversiontoUnsignedLong() { + DataType to = UNSIGNED_LONG; + { + Converter conversion = converterFor(DOUBLE, to); + assertNull(conversion.convert(null)); + double d = Math.abs(randomDouble()); + assertEquals(BigDecimal.valueOf(d).toBigInteger(), conversion.convert(d)); + + Double ulmAsDouble = UNSIGNED_LONG_MAX.doubleValue(); + Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(ulmAsDouble)); + assertEquals("[" + ulmAsDouble + "] out of [unsigned_long] range", e.getMessage()); + + Double nd = -Math.abs(randomDouble()); + e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(nd)); + assertEquals("[" + nd + "] out of [unsigned_long] range", e.getMessage()); + } + { + Converter conversion = converterFor(LONG, to); + assertNull(conversion.convert(null)); + + BigInteger bi = randomBigInteger(); + assertEquals(bi, conversion.convert(bi)); + + Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(bi.negate())); + assertEquals("[" + bi.negate() + "] out of [unsigned_long] range", e.getMessage()); + } + { + Converter conversion = converterFor(DATETIME, to); + assertNull(conversion.convert(null)); + + long l = randomNonNegativeLong(); + assertEquals(BigInteger.valueOf(l), conversion.convert(asDateTime(l))); + + Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(asDateTime(-l))); + assertEquals("[" + -l + "] out of [unsigned_long] range", e.getMessage()); + } + { + Converter conversion = converterFor(BOOLEAN, to); + assertNull(conversion.convert(null)); + + assertEquals(BigInteger.ONE, conversion.convert(true)); + assertEquals(BigInteger.ZERO, conversion.convert(false)); + } + { + Converter conversion = converterFor(KEYWORD, to); + assertNull(conversion.convert(null)); + BigInteger bi = randomBigInteger(); + assertEquals(bi, conversion.convert(bi.toString())); + + assertEquals(UNSIGNED_LONG_MAX, conversion.convert(UNSIGNED_LONG_MAX.toString())); + assertEquals(UNSIGNED_LONG_MAX, conversion.convert(UNSIGNED_LONG_MAX.toString() + ".0")); + + assertEquals(bi, conversion.convert(bi.toString() + "." + randomNonNegativeLong())); + + Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(BigInteger.ONE.negate().toString())); + assertEquals("[-1] out of [unsigned_long] range", e.getMessage()); + e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(UNSIGNED_LONG_MAX.add(BigInteger.ONE).toString())); + assertEquals("[" + UNSIGNED_LONG_MAX.add(BigInteger.ONE).toString() + "] out of [unsigned_long] range", + e.getMessage()); + } + } + public void testConversionToInt() { DataType to = INTEGER; { @@ -293,6 +405,16 @@ public void testConversionToInt() { Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(Long.MAX_VALUE)); assertEquals("[" + Long.MAX_VALUE + "] out of [integer] range", e.getMessage()); } + { + Converter conversion = converterFor(UNSIGNED_LONG, to); + assertNull(conversion.convert(null)); + BigInteger bi = BigInteger.valueOf(randomIntBetween(0, Integer.MAX_VALUE)); + assertEquals(bi.intValueExact(), conversion.convert(bi)); + + BigInteger bip = BigInteger.valueOf(randomLongBetween(Integer.MAX_VALUE, Long.MAX_VALUE)); + Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(bip)); + assertEquals("[" + bip + "] out of [integer] range", e.getMessage()); + } { Converter conversion = converterFor(DATETIME, to); assertNull(conversion.convert(null)); @@ -315,6 +437,16 @@ public void testConversionToShort() { Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(Integer.MAX_VALUE)); assertEquals("[" + Integer.MAX_VALUE + "] out of [short] range", e.getMessage()); } + { + Converter conversion = converterFor(UNSIGNED_LONG, to); + assertNull(conversion.convert(null)); + BigInteger bi = BigInteger.valueOf(randomIntBetween(0, Short.MAX_VALUE)); + assertEquals(bi.shortValueExact(), conversion.convert(bi)); + + BigInteger bip = BigInteger.valueOf(randomLongBetween(Short.MAX_VALUE, Long.MAX_VALUE)); + Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(bip)); + assertEquals("[" + bip + "] out of [short] range", e.getMessage()); + } { Converter conversion = converterFor(DATETIME, to); assertNull(conversion.convert(null)); @@ -337,6 +469,16 @@ public void testConversionToByte() { Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(Short.MAX_VALUE)); assertEquals("[" + Short.MAX_VALUE + "] out of [byte] range", e.getMessage()); } + { + Converter conversion = converterFor(UNSIGNED_LONG, to); + assertNull(conversion.convert(null)); + BigInteger bi = BigInteger.valueOf(randomIntBetween(0, Byte.MAX_VALUE)); + assertEquals(bi.byteValueExact(), conversion.convert(bi)); + + BigInteger bip = BigInteger.valueOf(randomLongBetween(Byte.MAX_VALUE, Long.MAX_VALUE)); + Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(bip)); + assertEquals("[" + bip + "] out of [byte] range", e.getMessage()); + } { Converter conversion = converterFor(DATETIME, to); assertNull(conversion.convert(null)); @@ -376,7 +518,9 @@ public void testCommonType() { assertEquals(SHORT, commonType(SHORT, BYTE)); assertEquals(FLOAT, commonType(BYTE, FLOAT)); assertEquals(FLOAT, commonType(FLOAT, INTEGER)); + assertEquals(UNSIGNED_LONG, commonType(UNSIGNED_LONG, LONG)); assertEquals(DOUBLE, commonType(DOUBLE, FLOAT)); + assertEquals(FLOAT, commonType(FLOAT, UNSIGNED_LONG)); // strings assertEquals(TEXT, commonType(TEXT, KEYWORD)); diff --git a/x-pack/plugin/ql/src/test/resources/mapping-multi-field-variation.json b/x-pack/plugin/ql/src/test/resources/mapping-multi-field-variation.json index c75ecfdc845c0..9a38243fde035 100644 --- a/x-pack/plugin/ql/src/test/resources/mapping-multi-field-variation.json +++ b/x-pack/plugin/ql/src/test/resources/mapping-multi-field-variation.json @@ -2,6 +2,7 @@ "properties" : { "bool" : { "type" : "boolean" }, "int" : { "type" : "integer" }, + "unsigned_long" : { "type" : "unsigned_long" }, "text" : { "type" : "text" }, "keyword" : { "type" : "keyword" }, "date" : { "type" : "date" }, diff --git a/x-pack/plugin/ql/src/test/resources/mapping-multi-field-with-nested.json b/x-pack/plugin/ql/src/test/resources/mapping-multi-field-with-nested.json index d66dd7efd5fca..a3ebf56ec814c 100644 --- a/x-pack/plugin/ql/src/test/resources/mapping-multi-field-with-nested.json +++ b/x-pack/plugin/ql/src/test/resources/mapping-multi-field-with-nested.json @@ -2,6 +2,7 @@ "properties" : { "bool" : { "type" : "boolean" }, "int" : { "type" : "integer" }, + "unsigned_long" : { "type" : "unsigned_long" }, "text" : { "type" : "text" }, "keyword" : { "type" : "keyword" }, "unsupported" : { "type" : "ip_range" }, diff --git a/x-pack/plugin/ql/src/test/resources/mapping-numeric.json b/x-pack/plugin/ql/src/test/resources/mapping-numeric.json index 15b02ab5f311e..119be02a4f098 100644 --- a/x-pack/plugin/ql/src/test/resources/mapping-numeric.json +++ b/x-pack/plugin/ql/src/test/resources/mapping-numeric.json @@ -12,6 +12,9 @@ "long" : { "type" : "long" }, + "unsigned_long" : { + "type" : "unsigned_long" + }, "meta_subfield" : { "type" : "text", "fields" : { diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java index 0b96204fd5d7e..af11cf8c9c7b6 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java @@ -19,6 +19,7 @@ public enum EsType implements SQLType { SHORT( Types.SMALLINT), INTEGER( Types.INTEGER), LONG( Types.BIGINT), + UNSIGNED_LONG( Types.BIGINT), DOUBLE( Types.DOUBLE), FLOAT( Types.REAL), HALF_FLOAT( Types.FLOAT), diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatement.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatement.java index 1814d61c39711..4b1f7677149c2 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatement.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatement.java @@ -8,6 +8,7 @@ import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; +import java.math.BigInteger; import java.net.URL; import java.sql.Array; import java.sql.Blob; @@ -39,7 +40,9 @@ import java.util.List; import java.util.Locale; +import static java.sql.Types.BIGINT; import static java.time.ZoneOffset.UTC; +import static org.elasticsearch.xpack.sql.jdbc.TypeUtils.scaleOrLenght; class JdbcPreparedStatement extends JdbcStatement implements PreparedStatement { @@ -111,7 +114,7 @@ public void setInt(int parameterIndex, int x) throws SQLException { @Override public void setLong(int parameterIndex, long x) throws SQLException { - setObject(parameterIndex, x, Types.BIGINT); + setObject(parameterIndex, x, BIGINT); } @Override @@ -204,7 +207,7 @@ public void setObject(int parameterIndex, Object x) throws SQLException { // {@code java.sql.Array} etc) will generate the correct exception message. Otherwise, the method call // {@code TypeConverter.fromJavaToJDBC(x.getClass())} will report the implementing class as not being supported. checkKnownUnsupportedTypes(x); - setObject(parameterIndex, x, TypeUtils.of(x.getClass()).getVendorTypeNumber(), 0); + setObject(parameterIndex, x, TypeUtils.of(x.getClass()).getVendorTypeNumber(), scaleOrLenght(x)); } @@ -347,7 +350,7 @@ public void setObject(int parameterIndex, Object x, int targetSqlType, int scale @Override public void setObject(int parameterIndex, Object x, SQLType targetSqlType, int scaleOrLength) throws SQLException { - setObject(parameterIndex, x, TypeUtils.of(targetSqlType), targetSqlType.getName()); + setObject(parameterIndex, x, TypeUtils.of(targetSqlType, scaleOrLength), targetSqlType.getName()); } private void setObject(int parameterIndex, Object x, EsType dataType, String typeString) throws SQLException { @@ -409,6 +412,7 @@ private void setObject(int parameterIndex, Object x, EsType dataType, String typ || x instanceof Short || x instanceof Integer || x instanceof Long + || x instanceof BigInteger || x instanceof Float || x instanceof Double || x instanceof String) { diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java index 4589a32955dc9..4b5dd65ca62ea 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.math.BigDecimal; +import java.math.BigInteger; import java.sql.Date; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; @@ -161,6 +162,9 @@ static T convert(Object val, EsType columnType, Class type, String typeSt if (type == Long.class) { return (T) asLong(val, columnType, typeString); } + if (type == BigInteger.class) { + return (T) asBigInteger(val, columnType, typeString); + } if (type == Float.class) { return (T) asFloat(val, columnType, typeString); } @@ -223,6 +227,8 @@ static Object convert(Object v, EsType columnType, String typeString) throws SQL return ((Number) v).intValue(); case LONG: return ((Number) v).longValue(); + case UNSIGNED_LONG: + return new BigInteger(v.toString()); case HALF_FLOAT: case SCALED_FLOAT: case DOUBLE: @@ -320,11 +326,12 @@ private static Boolean asBoolean(Object val, EsType columnType, String typeStrin case SHORT: case INTEGER: case LONG: + case UNSIGNED_LONG: case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: case DOUBLE: - return Boolean.valueOf(Integer.signum(((Number) val).intValue()) != 0); + return Boolean.valueOf(((Number) val).doubleValue() != 0); case KEYWORD: case TEXT: return Boolean.valueOf((String) val); @@ -342,6 +349,12 @@ private static Byte asByte(Object val, EsType columnType, String typeString) thr case INTEGER: case LONG: return safeToByte(((Number) val).longValue()); + case UNSIGNED_LONG: + try { + return ((BigInteger) val).byteValueExact(); + } catch (ArithmeticException ae) { + return failConversion(val, columnType, typeString, Byte.class, ae); + } case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -369,6 +382,12 @@ private static Short asShort(Object val, EsType columnType, String typeString) t case INTEGER: case LONG: return safeToShort(((Number) val).longValue()); + case UNSIGNED_LONG: + try { + return ((BigInteger) val).shortValueExact(); + } catch (ArithmeticException ae) { + return failConversion(val, columnType, typeString, Short.class, ae); + } case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -395,6 +414,12 @@ private static Integer asInteger(Object val, EsType columnType, String typeStrin case INTEGER: case LONG: return safeToInt(((Number) val).longValue()); + case UNSIGNED_LONG: + try { + return ((BigInteger) val).intValueExact(); + } catch (ArithmeticException ae) { + return failConversion(val, columnType, typeString, Integer.class, ae); + } case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -421,6 +446,12 @@ private static Long asLong(Object val, EsType columnType, String typeString) thr case INTEGER: case LONG: return Long.valueOf(((Number) val).longValue()); + case UNSIGNED_LONG: + try { + return ((BigInteger) val).longValueExact(); + } catch (ArithmeticException ae) { + return failConversion(val, columnType, typeString, Long.class, ae); + } case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -453,6 +484,8 @@ private static Float asFloat(Object val, EsType columnType, String typeString) t case INTEGER: case LONG: return Float.valueOf(((Number) val).longValue()); + case UNSIGNED_LONG: + return ((BigInteger) val).floatValue(); case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -479,6 +512,8 @@ private static Double asDouble(Object val, EsType columnType, String typeString) case INTEGER: case LONG: return Double.valueOf(((Number) val).longValue()); + case UNSIGNED_LONG: + return ((BigInteger) val).doubleValue(); case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -533,6 +568,34 @@ private static byte[] asByteArray(Object val, EsType columnType, String typeStri throw new SQLFeatureNotSupportedException(); } + private static BigInteger asBigInteger(Object val, EsType columnType, String typeString) throws SQLException { + switch (columnType) { + case BOOLEAN: + return ((Boolean) val).booleanValue() ? BigInteger.ONE : BigInteger.ZERO; + case BYTE: + case SHORT: + case INTEGER: + case LONG: + return BigInteger.valueOf(((Number)val).longValue()); + case UNSIGNED_LONG: + return new BigInteger(val.toString()); + case FLOAT: + case HALF_FLOAT: + case SCALED_FLOAT: + case DOUBLE: + return BigDecimal.valueOf(((Number) val).doubleValue()).toBigInteger(); // no safeguarding limits checking needed + case KEYWORD: + case TEXT: + try { + return new BigDecimal(val.toString()).toBigInteger(); + } catch (NumberFormatException e) { + return failConversion(val, columnType, typeString, BigInteger.class, e); + } + default: + } + return failConversion(val, columnType, typeString, BigInteger.class); + } + private static BigDecimal asBigDecimal(Object val, EsType columnType, String typeString) throws SQLException { switch (columnType) { case BOOLEAN: @@ -542,6 +605,8 @@ private static BigDecimal asBigDecimal(Object val, EsType columnType, String typ case INTEGER: case LONG: return BigDecimal.valueOf(((Number) val).longValue()); + case UNSIGNED_LONG: + return new BigDecimal((BigInteger) val); case FLOAT: case HALF_FLOAT: // floats are passed in as doubles here, so we need to dip into string to keep original float's (reduced) precision. diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java index 57cfb55cab34a..7d7c9ac00655e 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.jdbc; +import java.math.BigInteger; import java.sql.JDBCType; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; @@ -24,6 +25,7 @@ import java.util.Map.Entry; import java.util.Set; +import static java.sql.Types.BIGINT; import static java.util.Collections.unmodifiableMap; final class TypeUtils { @@ -39,6 +41,7 @@ private TypeUtils() {} EsType.SHORT, EsType.INTEGER, EsType.LONG, EsType.FLOAT, EsType.HALF_FLOAT, EsType.SCALED_FLOAT, EsType.DOUBLE, EsType.DATETIME); + public static final int LONG_MAX_LENGTH = String.valueOf(Long.MAX_VALUE).length(); /* type length value as defined in ES */ static { Map, EsType> aMap = new LinkedHashMap<>(); @@ -47,11 +50,11 @@ private TypeUtils() {} aMap.put(Short.class, EsType.SHORT); aMap.put(Integer.class, EsType.INTEGER); aMap.put(Long.class, EsType.LONG); + aMap.put(BigInteger.class, EsType.UNSIGNED_LONG); aMap.put(Float.class, EsType.FLOAT); aMap.put(Double.class, EsType.DOUBLE); aMap.put(String.class, EsType.KEYWORD); aMap.put(byte[].class, EsType.BINARY); - aMap.put(String.class, EsType.KEYWORD); aMap.put(Timestamp.class, EsType.DATETIME); // apart from the mappings in {@code DataType} three more Java classes can be mapped to a {@code JDBCType.TIMESTAMP} @@ -70,6 +73,7 @@ private TypeUtils() {} types.put(EsType.SHORT, Short.class); types.put(EsType.INTEGER, Integer.class); types.put(EsType.LONG, Long.class); + types.put(EsType.UNSIGNED_LONG, BigInteger.class); types.put(EsType.DOUBLE, Double.class); types.put(EsType.FLOAT, Float.class); types.put(EsType.HALF_FLOAT, Double.class); @@ -148,6 +152,16 @@ static EsType of(int sqlType) throws SQLException { return dataType; } + static EsType of(SQLType sqlType, int scaleOrLength) throws SQLException { + EsType esType; + if (sqlType.getVendorTypeNumber() == BIGINT) { + esType = scaleOrLength > LONG_MAX_LENGTH ? EsType.UNSIGNED_LONG : EsType.LONG; + } else { + esType = TypeUtils.of(sqlType); + } + return esType; + } + static EsType of(String name) throws SQLException { EsType dataType = ENUM_NAME_TO_TYPE.get(name); if (dataType == null) { @@ -175,4 +189,8 @@ static EsType of(Class clazz) throws SQLException { } return dataType; } + + static int scaleOrLenght(Object val) { + return val instanceof BigInteger ? LONG_MAX_LENGTH + 1 : 0; + } } diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java index 878fb9422e755..af555fcc0354d 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java @@ -8,8 +8,10 @@ import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.test.ESTestCase; +import java.math.BigInteger; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.sql.JDBCType; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.sql.Struct; @@ -38,6 +40,7 @@ import static org.elasticsearch.xpack.sql.jdbc.EsType.LONG; import static org.elasticsearch.xpack.sql.jdbc.EsType.SHORT; import static org.elasticsearch.xpack.sql.jdbc.EsType.TIME; +import static org.elasticsearch.xpack.sql.jdbc.EsType.UNSIGNED_LONG; public class JdbcPreparedStatementTests extends ESTestCase { @@ -121,6 +124,14 @@ public void testSettingByteTypeValues() throws SQLException { assertEquals(123, value(jps)); assertEquals(INTEGER, jdbcType(jps)); + jps.setObject(1, (byte) 123, Types.BIGINT); + assertEquals(123L, value(jps)); + assertEquals(LONG, jdbcType(jps)); + + jps.setObject(1, (byte) 123, Types.BIGINT, 20); + assertEquals(BigInteger.valueOf(123), value(jps)); + assertEquals(UNSIGNED_LONG, jdbcType(jps)); + jps.setObject(1, (byte) -128, Types.DOUBLE); assertEquals(-128.0, value(jps)); assertEquals(DOUBLE, jdbcType(jps)); @@ -234,6 +245,51 @@ public void testSettingLongValues() throws SQLException { assertEquals(HALF_FLOAT, jdbcType(jps)); } + public void testSettingBigIntegerValues() throws SQLException { + JdbcPreparedStatement jps = createJdbcPreparedStatement(); + + BigInteger bi = BigInteger.valueOf(randomLong()).abs(); + + jps.setObject(1, bi); + assertEquals(bi, value(jps)); + assertEquals(UNSIGNED_LONG, jdbcType(jps)); + assertTrue(value(jps) instanceof BigInteger); + + jps.setObject(1, bi, Types.VARCHAR); + assertEquals(String.valueOf(bi), value(jps)); + assertEquals(KEYWORD, jdbcType(jps)); + + jps.setObject(1, bi, Types.BIGINT); + assertEquals(bi.longValueExact(), value(jps)); + assertEquals(LONG, jdbcType(jps)); + + jps.setObject(1, bi, Types.DOUBLE); + assertEquals(bi.doubleValue(), value(jps)); + assertEquals(DOUBLE, jdbcType(jps)); + + jps.setObject(1, bi, Types.FLOAT); + assertEquals(bi.doubleValue(), value(jps)); + assertEquals(HALF_FLOAT, jdbcType(jps)); + + jps.setObject(1, BigInteger.ZERO, Types.BOOLEAN); + assertEquals(false, value(jps)); + assertEquals(BOOLEAN, jdbcType(jps)); + + jps.setObject(1, BigInteger.TEN, Types.BOOLEAN); + assertEquals(true, value(jps)); + assertEquals(BOOLEAN, jdbcType(jps)); + + jps.setObject(1, bi.longValueExact(), JDBCType.BIGINT, 19); + assertTrue(value(jps) instanceof Long); + assertEquals(bi.longValueExact(), value(jps)); + assertEquals(LONG, jdbcType(jps)); + + jps.setObject(1, bi.longValueExact(), JDBCType.BIGINT, 20); + assertTrue(value(jps) instanceof BigInteger); + assertEquals(bi, value(jps)); + assertEquals(UNSIGNED_LONG, jdbcType(jps)); + } + public void testThrownExceptionsWhenSettingLongValues() throws SQLException { JdbcPreparedStatement jps = createJdbcPreparedStatement(); long someLong = randomLong(); diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java index 206d12bafd80c..49cec0e8043d3 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java @@ -11,6 +11,8 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.ESTestCase; +import java.math.BigDecimal; +import java.math.BigInteger; import java.sql.Date; import java.sql.Timestamp; import java.time.ZoneId; @@ -24,6 +26,32 @@ public class TypeConverterTests extends ESTestCase { private static final ZoneId UTC = ZoneId.of("Z"); + public void testConvertToBigInteger() throws Exception { + { + assertEquals(BigInteger.ZERO, convertAsNative(false, BigInteger.class)); + assertEquals(BigInteger.ONE, convertAsNative(true, BigInteger.class)); + } + { + BigInteger bi = randomBigInteger(); + assertEquals(bi, convertAsNative(bi, BigInteger.class)); + } + // 18446744073709551615 + BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); + BigDecimal bd = BigDecimal.valueOf(randomDouble()).abs().remainder(new BigDecimal(UNSIGNED_LONG_MAX)); + { + float f = bd.floatValue(); + assertEquals(BigDecimal.valueOf(f).toBigInteger(), convertAsNative(f, BigInteger.class)); + } + { + double d = bd.doubleValue(); + assertEquals(BigDecimal.valueOf(d).toBigInteger(), convertAsNative(d, BigInteger.class)); + } + { + String s = bd.toString(); + assertEquals(new BigDecimal(s).toBigInteger(), convertAsNative(s, BigInteger.class)); + } + } + public void testFloatAsNative() throws Exception { assertThat(convertAsNative(42.0f, EsType.FLOAT), instanceOf(Float.class)); assertThat(convertAsNative(42.0, EsType.FLOAT), instanceOf(Float.class)); @@ -61,15 +89,25 @@ public void testDateAsNative() throws Exception { assertEquals(now.toLocalDate().atStartOfDay(ZoneId.of("Etc/GMT-10")).toInstant().toEpochMilli(), ((Date) nativeObject).getTime()); } - private Object convertAsNative(Object value, EsType type) throws Exception { - // Simulate sending over XContent + // Simulate sending over XContent + private static Object throughXContent(Object value) throws Exception { XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); builder.field("value"); builder.value(value); builder.endObject(); builder.close(); - Object copy = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2().get("value"); + return XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2().get("value"); + + } + private Object convertAsNative(Object value, EsType type) throws Exception { + Object copy = throughXContent(value); return TypeConverter.convert(copy, type, type.toString()); } + + private Object convertAsNative(Object value, Class nativeType) throws Exception { + Object copy = throughXContent(value); + EsType esType = TypeUtils.of(value.getClass()); + return TypeConverter.convert(value, esType, nativeType, esType.toString()); + } } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java index b437b86aa53aa..8b9ee6cb12da1 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java @@ -5,25 +5,64 @@ */ package org.elasticsearch.xpack.sql.qa.jdbc; +import org.elasticsearch.xpack.sql.jdbc.EsType; import org.elasticsearch.xpack.sql.proto.StringUtils; +import java.math.BigInteger; import java.sql.Date; import java.sql.Time; +import java.sql.Timestamp; import java.time.Instant; import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.Calendar; +import java.util.Collections; +import java.util.GregorianCalendar; +import java.util.LinkedHashMap; +import java.util.Map; final class JdbcTestUtils { private JdbcTestUtils() {} + private static final Map, EsType> CLASS_TO_ES_TYPE; + static final ZoneId UTC = ZoneId.of("Z"); static final String JDBC_TIMEZONE = "timezone"; static final LocalDate EPOCH = LocalDate.of(1970, 1, 1); + static { + Map, EsType> aMap = new LinkedHashMap<>(); + aMap.put(Boolean.class, EsType.BOOLEAN); + aMap.put(Byte.class, EsType.BYTE); + aMap.put(Short.class, EsType.SHORT); + aMap.put(Integer.class, EsType.INTEGER); + aMap.put(Long.class, EsType.LONG); + aMap.put(BigInteger.class, EsType.UNSIGNED_LONG); + aMap.put(Float.class, EsType.FLOAT); + aMap.put(Double.class, EsType.DOUBLE); + aMap.put(String.class, EsType.KEYWORD); + aMap.put(byte[].class, EsType.BINARY); + aMap.put(Timestamp.class, EsType.DATETIME); + + // apart from the mappings in {@code DataType} three more Java classes can be mapped to a {@code JDBCType.TIMESTAMP} + // according to B-4 table from the jdbc4.2 spec + aMap.put(Calendar.class, EsType.DATETIME); + aMap.put(GregorianCalendar.class, EsType.DATETIME); + aMap.put(java.util.Date.class, EsType.DATETIME); + aMap.put(java.sql.Date.class, EsType.DATETIME); + aMap.put(java.sql.Time.class, EsType.TIME); + aMap.put(LocalDateTime.class, EsType.DATETIME); + CLASS_TO_ES_TYPE = Collections.unmodifiableMap(aMap); + } + + static EsType of(Class clazz) { + return CLASS_TO_ES_TYPE.get(clazz); + } + static String of(long millis, String zoneId) { return StringUtils.toString(ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneId.of(zoneId))); } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java index 10b23994dd81d..d70e8a6913f90 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.math.BigDecimal; +import java.math.BigInteger; import java.sql.Connection; import java.sql.Date; import java.sql.JDBCType; @@ -27,11 +28,22 @@ import java.util.Locale; import java.util.StringJoiner; +import static org.elasticsearch.xpack.sql.jdbc.EsType.BOOLEAN; +import static org.elasticsearch.xpack.sql.jdbc.EsType.BYTE; +import static org.elasticsearch.xpack.sql.jdbc.EsType.DOUBLE; +import static org.elasticsearch.xpack.sql.jdbc.EsType.FLOAT; +import static org.elasticsearch.xpack.sql.jdbc.EsType.INTEGER; +import static org.elasticsearch.xpack.sql.jdbc.EsType.KEYWORD; +import static org.elasticsearch.xpack.sql.jdbc.EsType.LONG; +import static org.elasticsearch.xpack.sql.jdbc.EsType.SHORT; +import static org.elasticsearch.xpack.sql.jdbc.EsType.UNSIGNED_LONG; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.startsWith; public abstract class PreparedStatementTestCase extends JdbcIntegrationTestCase { + static final BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); + public void testSupportedTypes() throws SQLException { String stringVal = randomAlphaOfLength(randomIntBetween(0, 1000)); int intVal = randomInt(); @@ -42,6 +54,7 @@ public void testSupportedTypes() throws SQLException { byte byteVal = randomByte(); short shortVal = randomShort(); BigDecimal bigDecimalVal = BigDecimal.valueOf(randomDouble()); + BigInteger bigIntegerVal = bigDecimalVal.abs().remainder(new BigDecimal(UNSIGNED_LONG_MAX)).toBigInteger(); long millis = randomNonNegativeLong(); Calendar calendarVal = Calendar.getInstance(randomTimeZone(), Locale.ROOT); Timestamp timestampVal = new Timestamp(millis); @@ -61,29 +74,31 @@ public void testSupportedTypes() throws SQLException { try (Connection connection = esJdbc()) { StringJoiner sql = new StringJoiner(",", "SELECT ", ""); - for (int i = 0; i < 19; i++) { + for (int i = 0; i < 20; i++) { sql.add("?"); } try (PreparedStatement statement = connection.prepareStatement(sql.toString())) { - statement.setString(1, stringVal); - statement.setInt(2, intVal); - statement.setLong(3, longVal); - statement.setFloat(4, floatVal); - statement.setDouble(5, doubleVal); - statement.setNull(6, JDBCType.DOUBLE.getVendorTypeNumber()); - statement.setBoolean(7, booleanVal); - statement.setByte(8, byteVal); - statement.setShort(9, shortVal); - statement.setBigDecimal(10, bigDecimalVal); - statement.setTimestamp(11, timestampVal); - statement.setTimestamp(12, timestampVal, calendarVal); - statement.setDate(13, dateVal); - statement.setDate(14, dateVal, calendarVal); - statement.setTime(15, timeVal); - statement.setTime(16, timeVal, calendarVal); - statement.setObject(17, calendarVal); - statement.setObject(18, utilDateVal); - statement.setObject(19, localDateTimeVal); + int index = 1; + statement.setString(index ++, stringVal); + statement.setInt(index ++, intVal); + statement.setLong(index ++, longVal); + statement.setFloat(index ++, floatVal); + statement.setDouble(index ++, doubleVal); + statement.setNull(index ++, JDBCType.DOUBLE.getVendorTypeNumber()); + statement.setBoolean(index ++, booleanVal); + statement.setByte(index ++, byteVal); + statement.setShort(index ++, shortVal); + statement.setBigDecimal(index ++, bigDecimalVal); + statement.setObject(index ++, bigIntegerVal); + statement.setTimestamp(index ++, timestampVal); + statement.setTimestamp(index ++, timestampVal, calendarVal); + statement.setDate(index ++, dateVal); + statement.setDate(index ++, dateVal, calendarVal); + statement.setTime(index ++, timeVal); + statement.setTime(index ++, timeVal, calendarVal); + statement.setObject(index ++, calendarVal); + statement.setObject(index ++, utilDateVal); + statement.setObject(index ++, localDateTimeVal); try (ResultSet results = statement.executeQuery()) { ResultSetMetaData resultSetMetaData = results.getMetaData(); @@ -94,25 +109,27 @@ public void testSupportedTypes() throws SQLException { assertEquals(parameterMetaData.getParameterType(i), resultSetMetaData.getColumnType(i)); } assertTrue(results.next()); - assertEquals(stringVal, results.getString(1)); - assertEquals(intVal, results.getInt(2)); - assertEquals(longVal, results.getLong(3)); - assertEquals(floatVal, results.getFloat(4), 0.00001f); - assertEquals(doubleVal, results.getDouble(5), 0.00001f); - assertNull(results.getObject(6)); - assertEquals(booleanVal, results.getBoolean(7)); - assertEquals(byteVal, results.getByte(8)); - assertEquals(shortVal, results.getShort(9)); - assertEquals(bigDecimalVal, results.getBigDecimal(10)); - assertEquals(timestampVal, results.getTimestamp(11)); - assertEquals(timestampValWithCal, results.getTimestamp(12)); - assertEquals(dateVal, results.getDate(13)); - assertEquals(dateValWithCal, results.getDate(14)); - assertEquals(timeVal, results.getTime(15)); - assertEquals(timeValWithCal, results.getTime(16)); - assertEquals(new Timestamp(calendarVal.getTimeInMillis()), results.getObject(17)); - assertEquals(timestampVal, results.getObject(18)); - assertEquals(timestampVal, results.getObject(19)); + index = 1; + assertEquals(stringVal, results.getString(index ++)); + assertEquals(intVal, results.getInt(index ++)); + assertEquals(longVal, results.getLong(index ++)); + assertEquals(floatVal, results.getFloat(index ++), 0.00001f); + assertEquals(doubleVal, results.getDouble(index ++), 0.00001f); + assertNull(results.getObject(index ++)); + assertEquals(booleanVal, results.getBoolean(index ++)); + assertEquals(byteVal, results.getByte(index ++)); + assertEquals(shortVal, results.getShort(index ++)); + assertEquals(bigDecimalVal, results.getBigDecimal(index ++)); + assertEquals(bigIntegerVal, results.getObject(index ++)); + assertEquals(timestampVal, results.getTimestamp(index ++)); + assertEquals(timestampValWithCal, results.getTimestamp(index ++)); + assertEquals(dateVal, results.getDate(index ++)); + assertEquals(dateValWithCal, results.getDate(index ++)); + assertEquals(timeVal, results.getTime(index ++)); + assertEquals(timeValWithCal, results.getTime(index ++)); + assertEquals(new Timestamp(calendarVal.getTimeInMillis()), results.getObject(index ++)); + assertEquals(timestampVal, results.getObject(index ++)); + assertEquals(timestampVal, results.getObject(index ++)); assertFalse(results.next()); } } @@ -331,6 +348,7 @@ public void testSingleParameterMultipleTypes() throws SQLException { String stringVal = randomAlphaOfLength(randomIntBetween(0, 1000)); int intVal = randomInt(); long longVal = randomLong(); + BigInteger bigIntegerVal = randomBigInteger(); double doubleVal = randomDouble(); float floatVal = randomFloat(); boolean booleanVal = randomBoolean(); @@ -341,32 +359,39 @@ public void testSingleParameterMultipleTypes() throws SQLException { try (PreparedStatement statement = connection.prepareStatement("SELECT ?")) { statement.setString(1, stringVal); - assertEquals(new Tuple<>(JDBCType.VARCHAR.getVendorTypeNumber(), stringVal), execute(statement)); + assertEquals(new Tuple<>(KEYWORD.getName(), stringVal), execute(statement)); statement.setInt(1, intVal); - assertEquals(new Tuple<>(JDBCType.INTEGER.getVendorTypeNumber(), intVal), execute(statement)); + assertEquals(new Tuple<>(INTEGER.getName(), intVal), execute(statement)); statement.setLong(1, longVal); - assertEquals(new Tuple<>(JDBCType.BIGINT.getVendorTypeNumber(), longVal), execute(statement)); + assertEquals(new Tuple<>(LONG.getName(), longVal), execute(statement)); + statement.setObject(1, bigIntegerVal); + assertEquals(new Tuple<>(UNSIGNED_LONG.getName(), bigIntegerVal), execute(statement)); statement.setFloat(1, floatVal); - assertEquals(new Tuple<>(JDBCType.REAL.getVendorTypeNumber(), floatVal), execute(statement)); + assertEquals(new Tuple<>(FLOAT.getName(), floatVal), execute(statement)); statement.setDouble(1, doubleVal); - assertEquals(new Tuple<>(JDBCType.DOUBLE.getVendorTypeNumber(), doubleVal), execute(statement)); + assertEquals(new Tuple<>(DOUBLE.getName(), doubleVal), execute(statement)); statement.setNull(1, JDBCType.DOUBLE.getVendorTypeNumber()); - assertEquals(new Tuple<>(JDBCType.DOUBLE.getVendorTypeNumber(), null), execute(statement)); + assertEquals(new Tuple<>(DOUBLE.getName(), null), execute(statement)); statement.setBoolean(1, booleanVal); - assertEquals(new Tuple<>(JDBCType.BOOLEAN.getVendorTypeNumber(), booleanVal), execute(statement)); + assertEquals(new Tuple<>(BOOLEAN.getName(), booleanVal), execute(statement)); statement.setByte(1, byteVal); - assertEquals(new Tuple<>(JDBCType.TINYINT.getVendorTypeNumber(), byteVal), execute(statement)); + assertEquals(new Tuple<>(BYTE.getName(), byteVal), execute(statement)); statement.setShort(1, shortVal); - assertEquals(new Tuple<>(JDBCType.SMALLINT.getVendorTypeNumber(), shortVal), execute(statement)); + assertEquals(new Tuple<>(SHORT.getName(), shortVal), execute(statement)); + + statement.setObject(1, longVal, JDBCType.BIGINT, 19); + assertEquals(new Tuple<>(LONG.getName(), longVal), execute(statement)); + statement.setObject(1, Math.abs(longVal), JDBCType.BIGINT, 20); + assertEquals(new Tuple<>(UNSIGNED_LONG.getName(), BigInteger.valueOf(Math.abs(longVal))), execute(statement)); } } } - private Tuple execute(PreparedStatement statement) throws SQLException { + private Tuple execute(PreparedStatement statement) throws SQLException { try (ResultSet results = statement.executeQuery()) { ResultSetMetaData resultSetMetaData = results.getMetaData(); assertTrue(results.next()); - Tuple result = new Tuple<>(resultSetMetaData.getColumnType(1), results.getObject(1)); + Tuple result = new Tuple<>(resultSetMetaData.getColumnTypeName(1), results.getObject(1)); assertFalse(results.next()); return result; } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java index 2bad7ea0e3560..2d1161a83c482 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java @@ -21,6 +21,7 @@ public abstract class ResultSetMetaDataTestCase extends JdbcIntegrationTestCase "test_byte", "test_integer", "test_long", + "test_unsigned_long", "test_short", "test_double", "test_float", @@ -36,13 +37,14 @@ public void testValidGetObjectCalls() throws IOException, SQLException { } }); - String q = "SELECT test_byte, test_integer, test_long, test_short, test_double, test_float, test_keyword, " + String q = "SELECT test_byte, test_integer, test_long, test_unsigned_long, test_short, test_double, test_float, test_keyword, " + "test_boolean, test_date FROM test"; doWithQuery(q, r -> assertColumnNamesAndLabels(r.getMetaData(), fieldsNames)); - q = "SELECT test_byte AS b, test_integer AS i, test_long AS l, test_short AS s, test_double AS d, test_float AS f, " - + "test_keyword AS k, test_boolean AS bool, test_date AS dt FROM test"; - doWithQuery(q, r -> assertColumnNamesAndLabels(r.getMetaData(), new String[] { "b", "i", "l", "s", "d", "f", "k", "bool", "dt" })); + q = "SELECT test_byte AS b, test_integer AS i, test_long AS l, test_unsigned_long AS ul, test_short AS s, test_double AS d, " + + "test_float AS f, test_keyword AS k, test_boolean AS bool, test_date AS dt FROM test"; + doWithQuery(q, r -> assertColumnNamesAndLabels(r.getMetaData(), + new String[] { "b", "i", "l", "ul", "s", "d", "f", "k", "bool", "dt" })); } private void doWithQuery(String query, CheckedConsumer consumer) throws SQLException { diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java index aa7dfb63991cb..031d849864c62 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java @@ -23,6 +23,7 @@ import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; +import java.math.BigInteger; import java.sql.Blob; import java.sql.Clob; import java.sql.Connection; @@ -64,6 +65,8 @@ import static java.util.Calendar.MONTH; import static java.util.Calendar.SECOND; import static java.util.Calendar.YEAR; +import static org.elasticsearch.xpack.sql.jdbc.EsType.UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.of; public abstract class ResultSetTestCase extends JdbcIntegrationTestCase { @@ -71,6 +74,7 @@ public abstract class ResultSetTestCase extends JdbcIntegrationTestCase { "test_byte", "test_integer", "test_long", + "test_unsigned_long", "test_short", "test_double", "test_float", @@ -78,13 +82,14 @@ public abstract class ResultSetTestCase extends JdbcIntegrationTestCase { ).collect(Collectors.toCollection(HashSet::new)); static final Map, SQLType> dateTimeTestingFields = new HashMap<>(); static final String SELECT_ALL_FIELDS = "SELECT test_boolean, test_byte, test_integer," - + "test_long, test_short, test_double, test_float, test_keyword, test_date FROM test"; + + "test_long, test_unsigned_long, test_short, test_double, test_float, test_keyword, test_date FROM test"; static final String SELECT_WILDCARD = "SELECT * FROM test"; static { dateTimeTestingFields.put(new Tuple<>("test_boolean", true), EsType.BOOLEAN); dateTimeTestingFields.put(new Tuple<>("test_byte", 1), EsType.BYTE); dateTimeTestingFields.put(new Tuple<>("test_integer", 1), EsType.INTEGER); dateTimeTestingFields.put(new Tuple<>("test_long", 1L), EsType.LONG); + dateTimeTestingFields.put(new Tuple<>("test_unsigned_long", BigInteger.ONE), UNSIGNED_LONG); dateTimeTestingFields.put(new Tuple<>("test_short", 1), EsType.SHORT); dateTimeTestingFields.put(new Tuple<>("test_double", 1d), EsType.DOUBLE); dateTimeTestingFields.put(new Tuple<>("test_float", 1f), EsType.FLOAT); @@ -231,6 +236,7 @@ public void testGettingInvalidByte() throws IOException, SQLException { short shortNotByte = (short) randomIntBetween(Byte.MAX_VALUE + 1, Short.MAX_VALUE); double doubleNotByte = randomDoubleBetween(Byte.MAX_VALUE + 1, Double.MAX_VALUE, true); float floatNotByte = randomFloatBetween(Byte.MAX_VALUE + 1, Float.MAX_VALUE); + BigInteger bigIntegerNotByte = BigInteger.valueOf(longNotByte); String randomString = randomUnicodeOfCodepointLengthBetween(128, 256); long randomDate = randomNonNegativeLong(); @@ -241,6 +247,7 @@ public void testGettingInvalidByte() throws IOException, SQLException { index("test", "1", builder -> { builder.field("test_integer", intNotByte); builder.field("test_long", longNotByte); + builder.field("test_unsigned_long", bigIntegerNotByte); builder.field("test_short", shortNotByte); builder.field("test_double", doubleNotByte); builder.field("test_float", floatNotByte); @@ -261,6 +268,13 @@ public void testGettingInvalidByte() throws IOException, SQLException { sqle = expectThrows(SQLException.class, () -> results.getObject("test_short", Byte.class)); assertEquals(format(Locale.ROOT, "Numeric %s out of range", shortNotByte), sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getByte("test_unsigned_long")); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Byte]", bigIntegerNotByte), + sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Byte.class)); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Byte]", bigIntegerNotByte), + sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getByte("test_long")); assertEquals(format(Locale.ROOT, "Numeric %s out of range", Long.toString(longNotByte)), sqle.getMessage()); sqle = expectThrows(SQLException.class, () -> results.getObject("test_long", Byte.class)); @@ -367,6 +381,7 @@ public void testGettingInvalidShort() throws IOException, SQLException { long longNotShort = randomLongBetween(Short.MAX_VALUE + 1, Long.MAX_VALUE); double doubleNotShort = randomDoubleBetween(Short.MAX_VALUE + 1, Double.MAX_VALUE, true); float floatNotShort = randomFloatBetween(Short.MAX_VALUE + 1, Float.MAX_VALUE); + BigInteger bigIntegerNotShort = BigInteger.valueOf(longNotShort); String randomString = randomUnicodeOfCodepointLengthBetween(128, 256); long randomDate = randomNonNegativeLong(); @@ -377,6 +392,7 @@ public void testGettingInvalidShort() throws IOException, SQLException { index("test", "1", builder -> { builder.field("test_integer", intNotShort); builder.field("test_long", longNotShort); + builder.field("test_unsigned_long", bigIntegerNotShort); builder.field("test_double", doubleNotShort); builder.field("test_float", floatNotShort); builder.field("test_keyword", randomString); @@ -396,6 +412,13 @@ public void testGettingInvalidShort() throws IOException, SQLException { sqle = expectThrows(SQLException.class, () -> results.getObject("test_long", Short.class)); assertEquals(format(Locale.ROOT, "Numeric %s out of range", longNotShort), sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getShort("test_unsigned_long")); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Short]", bigIntegerNotShort), + sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Short.class)); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Short]", bigIntegerNotShort), + sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getShort("test_double")); assertEquals(format(Locale.ROOT, "Numeric %s out of range", doubleErrorMessage), sqle.getMessage()); sqle = expectThrows(SQLException.class, () -> results.getObject("test_double", Short.class)); @@ -496,6 +519,7 @@ public void testGettingInvalidInteger() throws IOException, SQLException { long longNotInt = randomLongBetween(getMaxIntPlusOne(), Long.MAX_VALUE); double doubleNotInt = randomDoubleBetween(getMaxIntPlusOne().doubleValue(), Double.MAX_VALUE, true); float floatNotInt = randomFloatBetween(getMaxIntPlusOne().floatValue(), Float.MAX_VALUE); + BigInteger bigIntegerNotInt = BigInteger.valueOf(longNotInt); String randomString = randomUnicodeOfCodepointLengthBetween(128, 256); long randomDate = randomNonNegativeLong(); @@ -505,6 +529,7 @@ public void testGettingInvalidInteger() throws IOException, SQLException { index("test", "1", builder -> { builder.field("test_long", longNotInt); + builder.field("test_unsigned_long", bigIntegerNotInt); builder.field("test_double", doubleNotInt); builder.field("test_float", floatNotInt); builder.field("test_keyword", randomString); @@ -519,6 +544,13 @@ public void testGettingInvalidInteger() throws IOException, SQLException { sqle = expectThrows(SQLException.class, () -> results.getObject("test_long", Integer.class)); assertEquals(format(Locale.ROOT, "Numeric %s out of range", Long.toString(longNotInt)), sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getInt("test_unsigned_long")); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Integer]", bigIntegerNotInt), + sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Integer.class)); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Integer]", bigIntegerNotInt), + sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getInt("test_double")); assertEquals(format(Locale.ROOT, "Numeric %s out of range", doubleErrorMessage), sqle.getMessage()); sqle = expectThrows(SQLException.class, () -> results.getObject("test_double", Integer.class)); @@ -615,10 +647,12 @@ public void testGettingInvalidLong() throws IOException, SQLException { double doubleNotLong = randomDoubleBetween(getMaxLongPlusOne(), Double.MAX_VALUE, true); float floatNotLong = randomFloatBetween(getMaxLongPlusOne().floatValue(), Float.MAX_VALUE); + BigInteger bigIntegerNotLong = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.valueOf(randomNonNegativeLong())); String randomString = randomUnicodeOfCodepointLengthBetween(128, 256); long randomDate = randomNonNegativeLong(); index("test", "1", builder -> { + builder.field("test_unsigned_long", bigIntegerNotLong); builder.field("test_double", doubleNotLong); builder.field("test_float", floatNotLong); builder.field("test_keyword", randomString); @@ -628,7 +662,14 @@ public void testGettingInvalidLong() throws IOException, SQLException { doWithQuery(SELECT_WILDCARD, results -> { results.next(); - SQLException sqle = expectThrows(SQLException.class, () -> results.getLong("test_double")); + SQLException sqle = expectThrows(SQLException.class, () -> results.getLong("test_unsigned_long")); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Long]", bigIntegerNotLong), + sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Long.class)); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Long]", bigIntegerNotLong), + sqle.getMessage()); + + sqle = expectThrows(SQLException.class, () -> results.getLong("test_double")); assertEquals(format(Locale.ROOT, "Numeric %s out of range", Double.toString(doubleNotLong)), sqle.getMessage()); sqle = expectThrows(SQLException.class, () -> results.getObject("test_double", Long.class)); assertEquals(format(Locale.ROOT, "Numeric %s out of range", Double.toString(doubleNotLong)), sqle.getMessage()); @@ -662,6 +703,92 @@ public void testGettingInvalidLong() throws IOException, SQLException { }); } + // BigInteger/unsigned_long values testing + public void testGettingValidBigIntegerWithoutCasting() throws IOException, SQLException { + List bigIntegerTestValues = createTestDataForNumericValueTests(ESTestCase::randomBigInteger); + BigInteger random1 = bigIntegerTestValues.get(0); + BigInteger random2 = bigIntegerTestValues.get(1); + BigInteger random3 = bigIntegerTestValues.get(2); + + doWithQuery("SELECT test_unsigned_long, test_null_unsigned_long, test_keyword FROM test", results -> { + ResultSetMetaData resultSetMetaData = results.getMetaData(); + + results.next(); + assertEquals(3, resultSetMetaData.getColumnCount()); + assertEquals(UNSIGNED_LONG.getName(), resultSetMetaData.getColumnTypeName(1)); + assertEquals(random1, results.getObject(1)); + assertEquals(random1, results.getObject("test_unsigned_long")); + assertEquals(random1, results.getObject("test_unsigned_long", BigInteger.class)); + assertTrue(results.getObject(1) instanceof BigInteger); + + assertEquals(UNSIGNED_LONG.getName(), resultSetMetaData.getColumnTypeName(2)); + assertNull(results.getObject(2)); + assertTrue(results.wasNull()); + assertNull(results.getObject("test_null_unsigned_long")); + assertTrue(results.wasNull()); + + assertTrue(results.next()); + assertEquals(random2, results.getObject(1)); + assertEquals(random2, results.getObject("test_unsigned_long")); + assertTrue(results.getObject(1) instanceof BigInteger); + assertEquals(random3.toString(), results.getObject("test_keyword")); + + assertFalse(results.next()); + }); + } + + public void testGettingValidBigIntegerWithCasting() throws IOException, SQLException { + Map map = createTestDataForNumericValueTypes(ESTestCase::randomBigInteger); + + doWithQuery(SELECT_WILDCARD, results -> { + results.next(); + for (Entry e : map.entrySet()) { + BigInteger actual = results.getObject(e.getKey(), BigInteger.class); + if (e.getValue() instanceof Float) { + assertEquals("For field " + e.getKey(), e.getValue(), actual.floatValue()); + } else if (e.getValue() instanceof Double) { + assertEquals("For field " + e.getKey(), e.getValue(), actual.doubleValue()); + } else { + assertEquals("For field " + e.getKey(), e.getValue().longValue(), results.getLong(e.getKey())); + assertEquals("For field " + e.getKey(), e.getValue().longValue(), actual.longValue()); + } + } + }); + } + + public void testGettingInvalidBigInteger() throws IOException, SQLException { + createIndex("test"); + updateMappingForNumericValuesTests("test"); + updateMapping("test", builder -> { + builder.startObject("test_keyword").field("type", "keyword").endObject(); + builder.startObject("test_date").field("type", "date").endObject(); + }); + + String randomString = randomUnicodeOfCodepointLengthBetween(128, 256); + long randomDate = randomNonNegativeLong(); + + index("test", "1", builder -> { + builder.field("test_keyword", randomString); + builder.field("test_date", randomDate); + }); + + doWithQuery(SELECT_WILDCARD, results -> { + results.next(); + + SQLException sqle = expectThrows(SQLException.class, () -> results.getObject("test_keyword", BigInteger.class)); + assertEquals( + format(Locale.ROOT, "Unable to convert value [%.128s] of type [KEYWORD] to [BigInteger]", randomString), + sqle.getMessage() + ); + + sqle = expectThrows(SQLException.class, () -> results.getObject("test_date", BigInteger.class)); + assertEquals( + format(Locale.ROOT, "Unable to convert value [%.128s] of type [DATETIME] to [BigInteger]", asDateString(randomDate)), + sqle.getMessage() + ); + }); + } + // Double values testing public void testGettingValidDoubleWithoutCasting() throws IOException, SQLException { List doubleTestValues = createTestDataForNumericValueTests(ESTestCase::randomDouble); @@ -1082,6 +1209,7 @@ public void testGettingBooleanValues() throws IOException, SQLException { builder.field("test_byte", 0); builder.field("test_integer", 0); builder.field("test_long", 0L); + builder.field("test_unsigned_long", 0L); builder.field("test_short", 0); builder.field("test_double", 0d); builder.field("test_float", 0f); @@ -1094,6 +1222,7 @@ public void testGettingBooleanValues() throws IOException, SQLException { builder.field("test_byte", randomValueOtherThan((byte) 0, ESTestCase::randomByte)); builder.field("test_integer", randomValueOtherThan(0, ESTestCase::randomInt)); builder.field("test_long", randomValueOtherThan(0L, ESTestCase::randomLong)); + builder.field("test_unsigned_long", randomValueOtherThan(BigInteger.ZERO, ESTestCase::randomBigInteger)); builder.field("test_short", randomValueOtherThan((short) 0, ESTestCase::randomShort)); builder.field( "test_double", @@ -1174,9 +1303,9 @@ public void testGettingDateWithoutCalendar() throws IOException, SQLException { java.sql.Date expectedDate = JdbcTestUtils.asDate(randomLongDate, getZoneFromOffset(randomLongDate)); assertEquals(expectedDate, results.getDate("test_date")); - assertEquals(expectedDate, results.getDate(9)); + assertEquals(expectedDate, results.getDate(10)); assertEquals(expectedDate, results.getObject("test_date", java.sql.Date.class)); - assertEquals(expectedDate, results.getObject(9, java.sql.Date.class)); + assertEquals(expectedDate, results.getObject(10, java.sql.Date.class)); // bulk validation for all fields which are not of type date validateErrorsForDateTestsWithoutCalendar(results::getDate); @@ -1206,7 +1335,7 @@ public void testGettingDateWithCalendar() throws IOException, SQLException { c.set(MILLISECOND, 0); assertEquals(results.getDate("test_date", c), new java.sql.Date(c.getTimeInMillis())); - assertEquals(results.getDate(9, c), new java.sql.Date(c.getTimeInMillis())); + assertEquals(results.getDate(10, c), new java.sql.Date(c.getTimeInMillis())); // bulk validation for all fields which are not of type date validateErrorsForDateTimeTestsWithCalendar(c, results::getDate); @@ -1232,9 +1361,9 @@ public void testGettingTimeWithoutCalendar() throws IOException, SQLException { java.sql.Time expectedTime = JdbcTestUtils.asTime(randomLongDate, getZoneFromOffset(randomLongDate)); assertEquals(expectedTime, results.getTime("test_date")); - assertEquals(expectedTime, results.getTime(9)); + assertEquals(expectedTime, results.getTime(10)); assertEquals(expectedTime, results.getObject("test_date", java.sql.Time.class)); - assertEquals(expectedTime, results.getObject(9, java.sql.Time.class)); + assertEquals(expectedTime, results.getObject(10, java.sql.Time.class)); validateErrorsForTimeTestsWithoutCalendar(results::getTime); }); @@ -1263,7 +1392,7 @@ public void testGettingTimeWithCalendar() throws IOException, SQLException { c.set(DAY_OF_MONTH, 1); assertEquals(results.getTime("test_date", c), new java.sql.Time(c.getTimeInMillis())); - assertEquals(results.getTime(9, c), new java.sql.Time(c.getTimeInMillis())); + assertEquals(results.getTime(10, c), new java.sql.Time(c.getTimeInMillis())); validateErrorsForDateTimeTestsWithCalendar(c, results::getTime); @@ -1333,7 +1462,7 @@ public void testGettingTimestampWithCalendar() throws IOException, SQLException c.setTimeInMillis(randomLongDate); assertEquals(results.getTimestamp("test_date", c), new java.sql.Timestamp(c.getTimeInMillis())); - assertEquals(results.getTimestamp(9, c), new java.sql.Timestamp(c.getTimeInMillis())); + assertEquals(results.getTimestamp(10, c), new java.sql.Timestamp(c.getTimeInMillis())); validateErrorsForDateTimeTestsWithCalendar(c, results::getTimestamp); @@ -1943,22 +2072,23 @@ private List createTestDataForNumericValueTests(Supplier clazz = random1.getClass(); - String primitiveName = clazz.getSimpleName().toLowerCase(Locale.ROOT); + EsType esType = of(random1.getClass()); + assertNotNull(esType); + String esTypeName = esType.getName().toLowerCase(Locale.ROOT); createIndex("test"); updateMapping("test", builder -> { - builder.startObject("test_" + primitiveName).field("type", primitiveName).endObject(); - builder.startObject("test_null_" + primitiveName).field("type", primitiveName).endObject(); + builder.startObject("test_" + esTypeName).field("type", esTypeName).endObject(); + builder.startObject("test_null_" + esTypeName).field("type", esTypeName).endObject(); builder.startObject("test_keyword").field("type", "keyword").endObject(); }); index("test", "1", builder -> { - builder.field("test_" + primitiveName, random1); - builder.field("test_null_" + primitiveName, (Byte) null); + builder.field("test_" + esTypeName, random1); + builder.field("test_null_" + esTypeName, (Byte) null); }); index("test", "2", builder -> { - builder.field("test_" + primitiveName, random2); + builder.field("test_" + esTypeName, random2); builder.field("test_keyword", random3); }); @@ -1991,6 +2121,7 @@ private void indexSimpleDocumentWithTrueValues(Long randomLongDate) throws IOExc builder.field("test_byte", 1); builder.field("test_integer", 1); builder.field("test_long", 1L); + builder.field("test_unsigned_long", 1L); builder.field("test_short", 1); builder.field("test_double", 1d); builder.field("test_float", 1f); @@ -2029,6 +2160,12 @@ private Map createTestDataForNumericValueTypes(Supplier builder.field("test_long", test_long); map.put("test_long", test_long); + // random BigInteger/unsigned_long + BigInteger test_unsigned_long = BigInteger.valueOf(randomValueOtherThanMany(map::containsValue, randomGenerator).longValue()) + .abs(); + builder.field("test_unsigned_long", test_unsigned_long); + map.put("test_unsigned_long", test_unsigned_long); + // random Double double test_double = randomValueOtherThanMany(map::containsValue, randomGenerator).doubleValue(); builder.field("test_double", test_double); @@ -2127,7 +2264,7 @@ private Connection esWithLeniency(boolean multiValueLeniency) throws SQLExceptio } private String asDateString(long millis) { - return JdbcTestUtils.of(millis, timeZoneId); + return of(millis, timeZoneId); } private ZoneId getZoneFromOffset(Long randomLongDate) { diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java index c17c0c8086f67..74f8e32927e24 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java @@ -28,6 +28,7 @@ import java.util.Locale; import java.util.Map; +import static org.elasticsearch.xpack.ql.util.Check.UNSIGNED_LONG_MAX; import static org.elasticsearch.xpack.sql.proto.Mode.CLI; import static org.elasticsearch.xpack.sql.proto.Protocol.SQL_QUERY_REST_ENDPOINT; import static org.elasticsearch.xpack.sql.proto.RequestInfo.CLIENT_IDS; @@ -53,6 +54,7 @@ public void testNumericTypes() throws IOException { assertQuery("SELECT -2123", "-2123", "integer", -2123, 11); assertQuery("SELECT 1234567890123", "1234567890123", "long", 1234567890123L, 20); assertQuery("SELECT -1234567890123", "-1234567890123", "long", -1234567890123L, 20); + assertQuery("SELECT 18446744073709551615", "18446744073709551615", "unsigned_long", UNSIGNED_LONG_MAX, 20); assertQuery("SELECT 1234567890123.34", "1234567890123.34", "double", 1234567890123.34, 25); assertQuery("SELECT -1234567890123.34", "-1234567890123.34", "double", -1234567890123.34, 25); assertQuery("SELECT CAST(1234.34 AS REAL)", "CAST(1234.34 AS REAL)", "float", 1234.34f, 15); diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java index ecbd693011e37..1da6160316e74 100644 --- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java @@ -65,6 +65,13 @@ public static SqlVersion fromString(String version) { return new SqlVersion(version, from(version)); } + public static SqlVersion fromId(int id) { + byte revision = (byte) ((id / REVISION_MULTIPLIER) % REVISION_MULTIPLIER); + byte minor = (byte) ((id / MINOR_MULTIPLIER) % REVISION_MULTIPLIER); + byte major = (byte) ((id / MAJOR_MULTIPLIER) % REVISION_MULTIPLIER); + return new SqlVersion(major, minor, revision); + } + protected static byte[] from(String ver) { String[] parts = ver.split("[.-]"); // Allow for optional snapshot and qualifier (Major.Minor.Revision-Qualifier-SNAPSHOT) diff --git a/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java b/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java index de8d8bb6ec0ea..ef8737a001156 100644 --- a/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java +++ b/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java @@ -36,6 +36,11 @@ public void test123AlphaSnapshotFromString() { assertEquals("1.2.3-Alpha-SNAPSHOT", ver.version); } + public void testFromId() { + SqlVersion ver = new SqlVersion((byte)randomIntBetween(0, 99), (byte)randomIntBetween(0, 99), (byte)randomIntBetween(0, 99)); + assertEquals(ver, SqlVersion.fromId(ver.id)); + } + public void testVersionsEqual() { SqlVersion ver1 = SqlVersion.fromString("1.2.3"); SqlVersion ver2 = SqlVersion.fromString("1.2.3"); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathProcessor.java index c5ac5bd4406a9..8799ae91120c2 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathProcessor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathProcessor.java @@ -14,13 +14,14 @@ import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import java.io.IOException; +import java.math.BigInteger; import java.util.Random; import java.util.function.DoubleFunction; import java.util.function.Function; import java.util.function.Supplier; public class MathProcessor implements Processor { - + public enum MathOperation { ABS((Object l) -> { if (l instanceof Double) { @@ -29,6 +30,9 @@ public enum MathOperation { if (l instanceof Float) { return Math.abs(((Float) l).floatValue()); } + if (l instanceof BigInteger) { + return ((BigInteger) l).abs(); + } // fallback to integer long lo = ((Number) l).longValue(); @@ -82,6 +86,9 @@ public enum MathOperation { if (l instanceof Float) { return (int) Math.signum((Float) l); } + if (l instanceof BigInteger) { + return ((BigInteger) l).signum(); + } return Long.signum(((Number) l).longValue()); }), @@ -121,7 +128,7 @@ public final Number apply(Object l) { return apply.apply(l); } } - + public static final String NAME = "m"; private final MathOperation processor; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java index 37a0296a7089c..ce249a1760ccc 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java @@ -117,6 +117,7 @@ import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; import org.elasticsearch.xpack.sql.type.SqlDataTypes; +import java.math.BigInteger; import java.time.Duration; import java.time.Period; import java.time.ZoneId; @@ -681,21 +682,27 @@ public Literal visitDecimalLiteral(DecimalLiteralContext ctx) { public Literal visitIntegerLiteral(IntegerLiteralContext ctx) { Tuple tuple = withMinus(ctx); - long value; + Number value; try { - value = Long.valueOf(StringUtils.parseLong(tuple.v2())); + value = StringUtils.parseInteger(tuple.v2()); } catch (QlIllegalArgumentException siae) { throw new ParsingException(tuple.v1(), siae.getMessage()); } - Object val = Long.valueOf(value); - DataType type = DataTypes.LONG; - // try to downsize to int if possible (since that's the most common type) - if ((int) value == value) { - type = DataTypes.INTEGER; - val = Integer.valueOf((int) value); + DataType type; + if (value instanceof BigInteger) { + type = DataTypes.UNSIGNED_LONG; + } else { + assert value instanceof Long : "Expected value [" + value + "] of type Long but got: " + value.getClass(); + // try to downsize to int if possible (since that's the most common type) + if (value.longValue() == value.intValue()) { + type = DataTypes.INTEGER; + value = Integer.valueOf(value.intValue()); + } else { + type = DataTypes.LONG; + } } - return new Literal(tuple.v1(), val, type); + return new Literal(tuple.v1(), value, type); } @Override diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java index d510e2c4926f8..c3631b0a75603 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.sql.plan.logical.command; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.FieldAttribute; @@ -26,6 +27,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyList; +import static org.elasticsearch.xpack.sql.proto.Mode.isDriver; public class ShowColumns extends Command { @@ -66,7 +68,8 @@ public void execute(SqlSession session, ActionListener listener) { String regex = pattern != null ? pattern.asJavaRegex() : null; boolean withFrozen = includeFrozen || session.configuration().includeFrozen(); - session.indexResolver().resolveAsMergedMapping(idx, regex, withFrozen, ActionListener.wrap( + Version version = isDriver(session.configuration().mode()) ? Version.fromId(session.configuration().version().id) : null; + session.indexResolver().resolveAsMergedMapping(idx, regex, withFrozen, version, ActionListener.wrap( indexResult -> { List> rows = emptyList(); if (indexResult.isValid()) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java index 69d7257ae84a0..17019b00485b8 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.plan.logical.command.sys; import org.apache.lucene.util.Counter; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.Strings; import org.elasticsearch.xpack.ql.expression.Attribute; @@ -37,6 +38,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.SHORT; import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; import static org.elasticsearch.xpack.ql.type.DataTypes.isString; +import static org.elasticsearch.xpack.sql.proto.Mode.isDriver; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.displaySize; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDataType; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDateTimeSub; @@ -126,9 +128,10 @@ public void execute(SqlSession session, ActionListener listener) { Pattern columnMatcher = columnPattern != null ? Pattern.compile(columnPattern.asJavaRegex()) : null; boolean includeFrozen = session.configuration().includeFrozen(); + Version version = isDriver(session.configuration().mode()) ? Version.fromId(session.configuration().version().id) : null; // special case for '%' (translated to *) if ("*".equals(idx)) { - session.indexResolver().resolveAsSeparateMappings(idx, regex, includeFrozen, + session.indexResolver().resolveAsSeparateMappings(idx, regex, includeFrozen, version, ActionListener.wrap(esIndices -> { List> rows = new ArrayList<>(); for (EsIndex esIndex : esIndices) { @@ -139,7 +142,7 @@ public void execute(SqlSession session, ActionListener listener) { } // otherwise use a merged mapping else { - session.indexResolver().resolveAsMergedMapping(idx, regex, includeFrozen, + session.indexResolver().resolveAsMergedMapping(idx, regex, includeFrozen, version, ActionListener.wrap(r -> { List> rows = new ArrayList<>(); // populate the data only when a target is found diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java index 189c5e1a10757..2fcb948ff6893 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java @@ -100,7 +100,7 @@ public final void execute(SqlSession session, ActionListener listener) { isString(t), // everything is searchable, DatabaseMetaData.typeSearchable, - // only numerics are signed + // only numerics (sans UNSIGNED_LONG) are signed isSigned(t) == false, //no fixed precision scale SQL_FALSE Boolean.FALSE, diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlClearCursorAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlClearCursorAction.java index 1a93408af1ce2..b1a3f3c13c3dd 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlClearCursorAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlClearCursorAction.java @@ -46,8 +46,8 @@ public static void operation(PlanExecutor planExecutor, SqlClearCursorRequest re Cursor cursor = Cursors.decodeFromStringWithZone(request.getCursor()).v1(); planExecutor.cleanCursor( new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, null, - request.mode(), StringUtils.EMPTY, StringUtils.EMPTY, StringUtils.EMPTY, Protocol.FIELD_MULTI_VALUE_LENIENCY, - Protocol.INDEX_INCLUDE_FROZEN), + request.mode(), StringUtils.EMPTY, request.version(), StringUtils.EMPTY, StringUtils.EMPTY, + Protocol.FIELD_MULTI_VALUE_LENIENCY, Protocol.INDEX_INCLUDE_FROZEN), cursor, ActionListener.wrap( success -> listener.onResponse(new SqlClearCursorResponse(success)), listener::onFailure)); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java index 153f405950488..f2435af2d6c6a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java @@ -76,8 +76,8 @@ static void operation(PlanExecutor planExecutor, SqlQueryRequest request, Action // The configuration is always created however when dealing with the next page, only the timeouts are relevant // the rest having default values (since the query is already created) SqlConfiguration cfg = new SqlConfiguration(request.zoneId(), request.fetchSize(), request.requestTimeout(), request.pageTimeout(), - request.filter(), request.mode(), request.clientId(), username, clusterName, request.fieldMultiValueLeniency(), - request.indexIncludeFrozen()); + request.filter(), request.mode(), request.clientId(), request.version(), username, clusterName, + request.fieldMultiValueLeniency(), request.indexIncludeFrozen()); if (Strings.hasText(request.cursor()) == false) { planExecutor.sql(cfg, request.query(), request.params(), diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlTranslateAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlTranslateAction.java index 9fecaf43e89d4..7587612663c20 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlTranslateAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlTranslateAction.java @@ -54,7 +54,7 @@ protected void doExecute(Task task, SqlTranslateRequest request, ActionListener< SqlConfiguration cfg = new SqlConfiguration(request.zoneId(), request.fetchSize(), request.requestTimeout(), request.pageTimeout(), request.filter(), - request.mode(), request.clientId(), + request.mode(), request.clientId(), request.version(), username(securityContext), clusterName(clusterService), Protocol.FIELD_MULTI_VALUE_LENIENCY, Protocol.INDEX_INCLUDE_FROZEN); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java index 1988b216f64bd..eaa6feb6c91f6 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java @@ -9,17 +9,19 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.sql.proto.Mode; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import java.time.ZoneId; // Typed object holding properties for a given query public class SqlConfiguration extends org.elasticsearch.xpack.ql.session.Configuration { - + private final int pageSize; private final TimeValue requestTimeout; private final TimeValue pageTimeout; private final Mode mode; private final String clientId; + private final SqlVersion version; private final boolean multiValueFieldLeniency; private final boolean includeFrozenIndices; @@ -27,7 +29,7 @@ public class SqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu private QueryBuilder filter; public SqlConfiguration(ZoneId zi, int pageSize, TimeValue requestTimeout, TimeValue pageTimeout, QueryBuilder filter, - Mode mode, String clientId, + Mode mode, String clientId, SqlVersion version, String username, String clusterName, boolean multiValueFieldLeniency, boolean includeFrozen) { @@ -40,6 +42,7 @@ public SqlConfiguration(ZoneId zi, int pageSize, TimeValue requestTimeout, TimeV this.filter = filter; this.mode = mode == null ? Mode.PLAIN : mode; this.clientId = clientId; + this.version = version; this.multiValueFieldLeniency = multiValueFieldLeniency; this.includeFrozenIndices = includeFrozen; } @@ -67,6 +70,10 @@ public String clientId() { return clientId; } + public SqlVersion version() { + return version; + } + public boolean multiValueFieldLeniency() { return multiValueFieldLeniency; } @@ -74,4 +81,4 @@ public boolean multiValueFieldLeniency() { public boolean includeFrozen() { return includeFrozenIndices; } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java index c93000308cd2c..a1c9e96d4a358 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.sql.session; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; @@ -32,6 +33,7 @@ import java.util.function.Function; import static org.elasticsearch.action.ActionListener.wrap; +import static org.elasticsearch.xpack.sql.proto.Mode.isDriver; public class SqlSession implements Session { @@ -44,7 +46,7 @@ public class SqlSession implements Session { private final Optimizer optimizer; private final Planner planner; private final PlanExecutor planExecutor; - + private final SqlConfiguration configuration; public SqlSession(SqlConfiguration configuration, Client client, FunctionRegistry functionRegistry, @@ -86,7 +88,7 @@ public IndexResolver indexResolver() { public Optimizer optimizer() { return optimizer; } - + public Verifier verifier() { return verifier; } @@ -141,6 +143,7 @@ private void preAnalyze(LogicalPlan parsed, Function act boolean includeFrozen = configuration.includeFrozen() || tableInfo.isFrozen(); indexResolver.resolveAsMergedMapping(table.index(), null, includeFrozen, + isDriver(configuration.mode()) ? Version.fromId(configuration.version().id) : null, wrap(indexResult -> listener.onResponse(action.apply(indexResult)), listener::onFailure)); } else { try { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java index e958ce1aa830d..3f720a0e9f67e 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverter.java @@ -8,7 +8,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.type.Converter; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypeConverter; @@ -19,7 +18,6 @@ import java.io.IOException; import java.time.OffsetTime; import java.time.ZonedDateTime; -import java.time.format.DateTimeParseException; import java.util.function.Function; import static org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter.BOOL_TO_INT; @@ -30,10 +28,12 @@ import static org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter.DATETIME_TO_INT; import static org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter.DATETIME_TO_LONG; import static org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter.DATETIME_TO_SHORT; +import static org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter.DATETIME_TO_UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter.IDENTITY; import static org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter.INTEGER_TO_LONG; import static org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter.RATIONAL_TO_LONG; import static org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter.TO_NULL; +import static org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter.fromString; import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; import static org.elasticsearch.xpack.ql.type.DataTypes.BYTE; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; @@ -45,6 +45,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.NULL; import static org.elasticsearch.xpack.ql.type.DataTypes.SHORT; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; import static org.elasticsearch.xpack.ql.type.DataTypes.isString; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.DATE; @@ -80,23 +81,17 @@ public static DataType commonType(DataType left, DataType right) { return DATETIME; } if (left == TIME) { - if (right == DATE) { - return DATETIME; - } if (isInterval(right)) { return left; } } if (right == TIME) { - if (left == DATE) { - return DATETIME; - } if (isInterval(left)) { return right; } } if (left == DATETIME) { - if (right == DATE || right == TIME) { + if (right == TIME) { return left; } if (isInterval(right)) { @@ -104,7 +99,7 @@ public static DataType commonType(DataType left, DataType right) { } } if (right == DATETIME) { - if (left == DATE || left == TIME) { + if (left == TIME) { return right; } if (isInterval(left)) { @@ -162,6 +157,9 @@ public static Converter converterFor(DataType from, DataType to) { if (to == KEYWORD || to == TEXT) { return conversionToString(from); } + if (to == UNSIGNED_LONG) { + return conversionToUnsignedLong(from); + } if (to == LONG) { return conversionToLong(from); } @@ -190,7 +188,7 @@ public static Converter converterFor(DataType from, DataType to) { // fallback to default return DataTypeConverter.converterFor(from, to); } - + private static Converter conversionToString(DataType from) { if (from == DATE) { return SqlConverter.DATE_TO_STRING; @@ -200,7 +198,17 @@ private static Converter conversionToString(DataType from) { } return null; } - + + private static Converter conversionToUnsignedLong(DataType from) { + if (from == DATE) { + return SqlConverter.DATE_TO_UNSIGNED_LONG; + } + if (from == TIME) { + return SqlConverter.TIME_TO_UNSIGNED_LONG; + } + return null; + } + private static Converter conversionToLong(DataType from) { if (from == DATE) { return SqlConverter.DATE_TO_LONG; @@ -347,7 +355,10 @@ public static Object convert(Object value, DataType dataType) { public enum SqlConverter implements Converter { DATE_TO_STRING(o -> DateUtils.toDateString((ZonedDateTime) o)), TIME_TO_STRING(o -> DateUtils.toTimeString((OffsetTime) o)), - + + DATE_TO_UNSIGNED_LONG(delegate(DATETIME_TO_UNSIGNED_LONG)), + TIME_TO_UNSIGNED_LONG(fromTime(DataTypeConverter::safeToUnsignedLong)), + DATE_TO_LONG(delegate(DATETIME_TO_LONG)), TIME_TO_LONG(fromTime(value -> value)), @@ -385,7 +396,7 @@ public enum SqlConverter implements Converter { TIME_TO_BOOLEAN(fromTime(value -> value != 0)); public static final String NAME = "dtc-sql"; - + private final Function converter; SqlConverter(Function converter) { @@ -416,18 +427,6 @@ private static Function delegate(Converter converter) { return converter::convert; } - private static Function fromString(Function converter, String to) { - return (Object value) -> { - try { - return converter.apply(value.toString()); - } catch (NumberFormatException e) { - throw new QlIllegalArgumentException(e, "cannot cast [{}] to [{}]", value, to); - } catch (DateTimeParseException | IllegalArgumentException e) { - throw new QlIllegalArgumentException(e, "cannot cast [{}] to [{}]: {}", value, to, e.getMessage()); - } - }; - } - @Override public Object convert(Object l) { if (l == null) { @@ -450,4 +449,4 @@ public static Converter read(StreamInput in) throws IOException { return in.readEnum(SqlConverter.class); } } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeRegistry.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeRegistry.java index 12554a547bb9b..fecf0380ff00f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeRegistry.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeRegistry.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.type; +import org.elasticsearch.Version; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypeRegistry; import org.elasticsearch.xpack.ql.type.DataTypes; @@ -28,6 +29,11 @@ public DataType fromEs(String typeName) { return SqlDataTypes.fromEs(typeName); } + @Override + public DataType fromEs(String typeName, Version version) { + return SqlDataTypes.fromEs(typeName, version); + } + @Override public DataType fromJava(Object value) { return SqlDataTypes.fromJava(value); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java index 9b0ed868174b9..e7a065cac88fd 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.sql.type; +import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.sql.expression.literal.geo.GeoShape; @@ -43,8 +45,10 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.SCALED_FLOAT; import static org.elasticsearch.xpack.ql.type.DataTypes.SHORT; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; import static org.elasticsearch.xpack.ql.util.CollectionUtils.mapSize; +import static org.elasticsearch.xpack.ql.util.VersionUtil.isUnsignedLongSupported; public class SqlDataTypes { @@ -94,6 +98,7 @@ public class SqlDataTypes { ODBC_TO_ES.put("SQL_SMALLINT", SHORT); ODBC_TO_ES.put("SQL_INTEGER", INTEGER); ODBC_TO_ES.put("SQL_BIGINT", LONG); + ODBC_TO_ES.put("SQL_UBIGINT", UNSIGNED_LONG); ODBC_TO_ES.put("SQL_REAL", FLOAT); ODBC_TO_ES.put("SQL_FLOAT", DOUBLE); ODBC_TO_ES.put("SQL_DOUBLE", DOUBLE); @@ -204,6 +209,16 @@ public static DataType fromEs(String name) { return type != null ? type : UNSUPPORTED; } + public static DataType fromEs(String name, @Nullable Version version) { + DataType type = fromEs(name); + if (version != null) { + if (type == UNSIGNED_LONG && isUnsignedLongSupported(version) == false) { + return UNSUPPORTED; + } + } + return type; + } + public static DataType fromJava(Object value) { DataType type = DataTypes.fromJava(value); @@ -314,7 +329,7 @@ public static SQLType sqlType(DataType dataType) { if (dataType == INTEGER) { return JDBCType.INTEGER; } - if (dataType == LONG) { + if (dataType == LONG || dataType == UNSIGNED_LONG) { return JDBCType.BIGINT; } if (dataType == DOUBLE) { @@ -440,6 +455,9 @@ public static int defaultPrecision(DataType dataType) { if (dataType == LONG) { return 19; } + if (dataType == UNSIGNED_LONG) { + return 20; + } if (dataType == DOUBLE) { return 15; } @@ -557,7 +575,7 @@ public static int displaySize(DataType dataType) { if (dataType == INTEGER) { return 11; } - if (dataType == LONG) { + if (dataType == LONG || dataType == UNSIGNED_LONG) { return 20; } if (dataType == DOUBLE) { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java index c03a473d38cc1..489309134c844 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java @@ -38,7 +38,7 @@ private SqlTestUtils() {} public static final SqlConfiguration TEST_CFG = new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, null, Mode.PLAIN, - null, null, null, false, false); + null, null, null, null, false, false); public static SqlConfiguration randomConfiguration() { return new SqlConfiguration(randomZone(), @@ -48,6 +48,7 @@ public static SqlConfiguration randomConfiguration() { null, randomFrom(Mode.values()), randomAlphaOfLength(10), + null, randomAlphaOfLength(10), randomAlphaOfLength(10), false, @@ -62,6 +63,7 @@ public static SqlConfiguration randomConfiguration(ZoneId providedZoneId) { null, randomFrom(Mode.values()), randomAlphaOfLength(10), + null, randomAlphaOfLength(10), randomAlphaOfLength(10), false, diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java index 7b8582dcbc3a0..f93417c6ff95b 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java @@ -165,7 +165,7 @@ public void testDottedFieldPathTypo() { public void testStarExpansionExcludesObjectAndUnsupportedTypes() { LogicalPlan plan = plan("SELECT * FROM test"); List list = ((Project) plan).projections(); - assertThat(list, hasSize(10)); + assertThat(list, hasSize(11)); List names = Expressions.names(list); assertThat(names, not(hasItem("some"))); assertThat(names, not(hasItem("some.dotted"))); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 7adbb6278b5c3..164748186c85f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -5,7 +5,9 @@ */ package org.elasticsearch.xpack.sql.analysis.index; +import org.elasticsearch.Version; import org.elasticsearch.action.fieldcaps.FieldCapabilities; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; @@ -17,6 +19,7 @@ import org.elasticsearch.xpack.sql.type.SqlDataTypeRegistry; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -25,12 +28,14 @@ import java.util.stream.Stream; import static java.util.Collections.singletonMap; +import static org.elasticsearch.common.collect.Tuple.tuple; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; @@ -227,6 +232,38 @@ public void testRandomMappingFieldTypeMappedAsUnsupported() throws Exception { assertEquals(UNSUPPORTED, esIndex.mapping().get("another_field").getProperties().get("_foo").getDataType()); } + public void testMergeCompatibleMappingVersionAware() throws Exception { + Map basicMapping = loadMapping("mapping-basic.json", true); + Map numericMapping = loadMapping("mapping-numeric.json", true); + + assertNotEquals(basicMapping, numericMapping); + + for (Tuple t : Arrays.asList(tuple(Version.V_7_10_0, UNSUPPORTED), tuple(Version.V_7_11_0, UNSIGNED_LONG))) { + IndexResolution resolution = merge(t.v1(), new EsIndex("basic", basicMapping), new EsIndex("numeric", numericMapping)); + + assertTrue(resolution.isValid()); + assertEquals(basicMapping.size() + numericMapping.size(), resolution.get().mapping().size()); + assertEquals(t.v2(), resolution.get().mapping().get("unsigned_long").getDataType()); + } + } + + public void testSeparateMappingVersionAware() throws Exception { + Map numericMapping = loadMapping("mapping-numeric.json", true); + Map sameMapping = loadMapping("mapping-numeric.json", true); + + assertEquals(sameMapping, numericMapping); + + for (Tuple t : Arrays.asList(tuple(Version.V_7_10_0, UNSUPPORTED), tuple(Version.V_7_11_0, UNSIGNED_LONG))) { + List indices = separate(t.v1(), new EsIndex("numeric", numericMapping), new EsIndex("same", sameMapping)); + + assertEquals(2, indices.size()); + assertEquals(numericMapping.size(), indices.get(0).mapping().size()); + assertEquals(t.v2(), indices.get(0).mapping().get("unsigned_long").getDataType()); + assertEquals(sameMapping.size(), indices.get(1).mapping().size()); + assertEquals(t.v2(), indices.get(1).mapping().get("unsigned_long").getDataType()); + } + } + public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception { Map> fieldCaps = new HashMap<>(); @@ -324,12 +361,20 @@ public void testIndexWithNoMapping() { } public static IndexResolution merge(EsIndex... indices) { - return mergedMappings("*", Stream.of(indices).map(EsIndex::name).toArray(String[]::new), fromMappings(indices)); + return merge(null, indices); + } + + public static IndexResolution merge(Version version, EsIndex... indices) { + return mergedMappings("*", Stream.of(indices).map(EsIndex::name).toArray(String[]::new), fromMappings(indices), version); } public static List separate(EsIndex... indices) { + return separate(null, indices); + } + + public static List separate(Version version, EsIndex... indices) { return separateMappings(null, Stream.of(indices).map(EsIndex::name).toArray(String[]::new), - fromMappings(indices)); + fromMappings(indices), version); } public static Map> fromMappings(EsIndex... indices) { @@ -433,12 +478,17 @@ private void addFieldCaps(Map> fieldCaps, } private static IndexResolution mergedMappings(String indexPattern, String[] indexNames, - Map> fieldCaps) { - return IndexResolver.mergedMappings(SqlDataTypeRegistry.INSTANCE, indexPattern, indexNames, fieldCaps); + Map> fieldCaps) { + return mergedMappings(indexPattern, indexNames, fieldCaps, null); + } + + private static IndexResolution mergedMappings(String indexPattern, String[] indexNames, + Map> fieldCaps, Version version) { + return IndexResolver.mergedMappings(SqlDataTypeRegistry.INSTANCE, indexPattern, indexNames, fieldCaps, version); } private static List separateMappings(String javaRegex, String[] indexNames, - Map> fieldCaps) { - return IndexResolver.separateMappings(SqlDataTypeRegistry.INSTANCE, javaRegex, indexNames, fieldCaps, null); + Map> fieldCaps, Version version) { + return IndexResolver.separateMappings(SqlDataTypeRegistry.INSTANCE, javaRegex, indexNames, fieldCaps, null, version); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java index 225e93b417a52..0d5b2d87ef9d5 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java @@ -33,12 +33,12 @@ public void testDatabaseFunctionOutput() { new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, null, randomFrom(Mode.values()), randomAlphaOfLength(10), - null, clusterName, randomBoolean(), randomBoolean()), + null, null, clusterName, randomBoolean(), randomBoolean()), new SqlFunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics()) ); - + Project result = (Project) analyzer.analyze(parser.createStatement("SELECT DATABASE()"), true); NamedExpression ne = result.projections().get(0); assertTrue(ne instanceof Alias); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java index f3a645477cd64..7380f0057744e 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java @@ -32,13 +32,13 @@ public void testNoUsernameFunctionOutput() { new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, null, randomFrom(Mode.values()), randomAlphaOfLength(10), - null, randomAlphaOfLengthBetween(1, 15), + null, null, randomAlphaOfLengthBetween(1, 15), randomBoolean(), randomBoolean()), new SqlFunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics()) ); - + Project result = (Project) analyzer.analyze(parser.createStatement("SELECT USER()"), true); NamedExpression ne = result.projections().get(0); assertTrue(ne instanceof Alias); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunctionProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunctionProcessorTests.java index 579ca5f056d63..c9c7dab483cb6 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunctionProcessorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunctionProcessorTests.java @@ -11,6 +11,8 @@ import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import java.io.IOException; +import java.math.BigInteger; +import java.util.Arrays; public class MathFunctionProcessorTests extends AbstractWireSerializingTestCase { public static MathProcessor randomMathFunctionProcessor() { @@ -55,7 +57,7 @@ public void testRandom() { assertNotNull(proc.process(null)); assertNotNull(proc.process(randomLong())); } - + public void testFloor() { MathProcessor proc = new MathProcessor(MathOperation.FLOOR); assertNull(proc.process(null)); @@ -64,7 +66,7 @@ public void testFloor() { assertEquals(3.0, proc.process(3.9)); assertEquals(-13.0, proc.process(-12.1)); } - + public void testCeil() { MathProcessor proc = new MathProcessor(MathOperation.CEIL); assertNull(proc.process(null)); @@ -73,4 +75,19 @@ public void testCeil() { assertEquals(4.0, proc.process(3.9)); assertEquals(-12.0, proc.process(-12.1)); } + + public void testUnsignedLongAbs() { + MathProcessor proc = new MathProcessor(MathOperation.ABS); + BigInteger bi = randomBigInteger(); + assertEquals(bi, proc.process(bi)); + } + + public void testUnsignedLongSign() { + MathProcessor proc = new MathProcessor(MathOperation.SIGN); + for (BigInteger bi : Arrays.asList(BigInteger.valueOf(randomNonNegativeLong()), BigInteger.ZERO)) { + Object val = proc.process(bi); + assertEquals(bi.intValue() == 0 ? 0 : 1, val); + assertTrue(val instanceof Integer); + } + } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index 61743900c8266..d707864c231e3 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.sql.plan.logical.command.sys; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.test.ESTestCase; @@ -20,6 +21,7 @@ import org.elasticsearch.xpack.sql.proto.Mode; import org.elasticsearch.xpack.sql.proto.Protocol; import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.session.Cursor; import org.elasticsearch.xpack.sql.session.SchemaRowSet; import org.elasticsearch.xpack.sql.session.SqlConfiguration; @@ -52,89 +54,98 @@ public class SysColumnsTests extends ESTestCase { private final SqlParser parser = new SqlParser(); private final Map mapping = loadMapping("mapping-multi-field-with-nested.json", true); + private static final int FIELD_COUNT = 16; public void testSysColumns() { List> rows = new ArrayList<>(); SysColumns.fillInRows("test", "index", loadMapping("mapping-multi-field-variation.json", true), null, rows, null, randomValueOtherThanMany(Mode::isDriver, () -> randomFrom(Mode.values()))); // nested fields are ignored - assertEquals(15, rows.size()); + assertEquals(FIELD_COUNT, rows.size()); assertEquals(24, rows.get(0).size()); - List row = rows.get(0); + int index = 0; + + List row = rows.get(index ++); assertEquals("bool", name(row)); assertEquals(Types.BOOLEAN, sqlType(row)); assertEquals(null, radix(row)); assertEquals(1, bufferLength(row)); - row = rows.get(1); + row = rows.get(index ++); assertEquals("int", name(row)); assertEquals(Types.INTEGER, sqlType(row)); assertEquals(10, radix(row)); assertEquals(4, bufferLength(row)); - row = rows.get(2); + row = rows.get(index ++); + assertEquals("unsigned_long", name(row)); + assertEquals(Types.BIGINT, sqlType(row)); + assertEquals(10, radix(row)); + assertEquals(Long.BYTES, bufferLength(row)); + + row = rows.get(index ++); assertEquals("text", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - - row = rows.get(3); + + row = rows.get(index ++); assertEquals("keyword", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(4); + row = rows.get(index ++); assertEquals("date", name(row)); assertEquals(Types.TIMESTAMP, sqlType(row)); assertEquals(null, radix(row)); assertEquals(29, precision(row)); assertEquals(8, bufferLength(row)); - row = rows.get(5); + row = rows.get(index ++); assertEquals("some.dotted.field", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(6); + row = rows.get(index ++); assertEquals("some.string", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - - row = rows.get(7); + + row = rows.get(index ++); assertEquals("some.string.normalized", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - - row = rows.get(8); + + row = rows.get(index ++); assertEquals("some.string.typical", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - - row = rows.get(9); + + row = rows.get(index ++); assertEquals("some.ambiguous", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - - row = rows.get(10); + + row = rows.get(index ++); assertEquals("some.ambiguous.one", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - - row = rows.get(11); + + row = rows.get(index ++); assertEquals("some.ambiguous.two", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(12); + row = rows.get(index ++); assertEquals("some.ambiguous.normalized", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -145,16 +156,18 @@ public void testSysColumnsInOdbcMode() { List> rows = new ArrayList<>(); SysColumns.fillInRows("test", "index", loadMapping("mapping-multi-field-variation.json", true), null, rows, null, Mode.ODBC); - assertEquals(15, rows.size()); + assertEquals(FIELD_COUNT, rows.size()); assertEquals(24, rows.get(0).size()); - List row = rows.get(0); + int index = 0; + + List row = rows.get(index ++); assertEquals("bool", name(row)); assertEquals((short) Types.BOOLEAN, sqlType(row)); assertEquals(null, radix(row)); assertEquals(1, bufferLength(row)); - row = rows.get(1); + row = rows.get(index ++); assertEquals("int", name(row)); assertEquals((short) Types.INTEGER, sqlType(row)); assertEquals(Short.class, radix(row).getClass()); @@ -164,7 +177,17 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - row = rows.get(2); + row = rows.get(index ++); + assertEquals("unsigned_long", name(row)); + assertEquals((short) Types.BIGINT, sqlType(row)); + assertEquals(Short.class, radix(row).getClass()); + assertEquals(Long.BYTES, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(Short.class, nullable(row).getClass()); + assertEquals(Short.class, sqlDataType(row).getClass()); + assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + + row = rows.get(index ++); assertEquals("text", name(row)); assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -174,7 +197,7 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - row = rows.get(3); + row = rows.get(index ++); assertEquals("keyword", name(row)); assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -184,7 +207,7 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - row = rows.get(4); + row = rows.get(index ++); assertEquals("date", name(row)); assertEquals((short) Types.TIMESTAMP, sqlType(row)); assertEquals(null, radix(row)); @@ -195,7 +218,7 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - row = rows.get(5); + row = rows.get(index ++); assertEquals("some.dotted.field", name(row)); assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -205,7 +228,7 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - row = rows.get(6); + row = rows.get(index ++); assertEquals("some.string", name(row)); assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -215,7 +238,7 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - row = rows.get(7); + row = rows.get(index ++); assertEquals("some.string.normalized", name(row)); assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -225,7 +248,7 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - row = rows.get(8); + row = rows.get(index ++); assertEquals("some.string.typical", name(row)); assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -235,7 +258,7 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - row = rows.get(9); + row = rows.get(index ++); assertEquals("some.ambiguous", name(row)); assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -244,8 +267,8 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, nullable(row).getClass()); assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(10); + + row = rows.get(index ++); assertEquals("some.ambiguous.one", name(row)); assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -254,8 +277,8 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, nullable(row).getClass()); assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(11); + + row = rows.get(index ++); assertEquals("some.ambiguous.two", name(row)); assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -265,7 +288,7 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - row = rows.get(12); + row = rows.get(index ++); assertEquals("some.ambiguous.normalized", name(row)); assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -275,21 +298,23 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); } - + public void testSysColumnsInJdbcMode() { List> rows = new ArrayList<>(); SysColumns.fillInRows("test", "index", loadMapping("mapping-multi-field-variation.json", true), null, rows, null, Mode.JDBC); - assertEquals(15, rows.size()); + assertEquals(FIELD_COUNT, rows.size()); assertEquals(24, rows.get(0).size()); - List row = rows.get(0); + int index = 0; + + List row = rows.get(index ++); assertEquals("bool", name(row)); assertEquals(Types.BOOLEAN, sqlType(row)); assertEquals(null, radix(row)); assertEquals(1, bufferLength(row)); - row = rows.get(1); + row = rows.get(index ++); assertEquals("int", name(row)); assertEquals(Types.INTEGER, sqlType(row)); assertEquals(Integer.class, radix(row).getClass()); @@ -299,7 +324,17 @@ public void testSysColumnsInJdbcMode() { assertEquals(Integer.class, sqlDataType(row).getClass()); assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - row = rows.get(2); + row = rows.get(index ++); + assertEquals("unsigned_long", name(row)); + assertEquals(Types.BIGINT, sqlType(row)); + assertEquals(Integer.class, radix(row).getClass()); + assertEquals(Long.BYTES, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(Integer.class, nullable(row).getClass()); + assertEquals(Integer.class, sqlDataType(row).getClass()); + assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); + + row = rows.get(index ++); assertEquals("text", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -309,7 +344,7 @@ public void testSysColumnsInJdbcMode() { assertEquals(Integer.class, sqlDataType(row).getClass()); assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - row = rows.get(3); + row = rows.get(index ++); assertEquals("keyword", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -319,7 +354,7 @@ public void testSysColumnsInJdbcMode() { assertEquals(Integer.class, sqlDataType(row).getClass()); assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - row = rows.get(4); + row = rows.get(index ++); assertEquals("date", name(row)); assertEquals(Types.TIMESTAMP, sqlType(row)); assertEquals(null, radix(row)); @@ -330,7 +365,7 @@ public void testSysColumnsInJdbcMode() { assertEquals(Integer.class, sqlDataType(row).getClass()); assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - row = rows.get(5); + row = rows.get(index ++); assertEquals("some.dotted.field", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -340,7 +375,7 @@ public void testSysColumnsInJdbcMode() { assertEquals(Integer.class, sqlDataType(row).getClass()); assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - row = rows.get(6); + row = rows.get(index ++); assertEquals("some.string", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -350,7 +385,7 @@ public void testSysColumnsInJdbcMode() { assertEquals(Integer.class, sqlDataType(row).getClass()); assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - row = rows.get(7); + row = rows.get(index ++); assertEquals("some.string.normalized", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -360,7 +395,7 @@ public void testSysColumnsInJdbcMode() { assertEquals(Integer.class, sqlDataType(row).getClass()); assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - row = rows.get(8); + row = rows.get(index ++); assertEquals("some.string.typical", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -369,8 +404,8 @@ public void testSysColumnsInJdbcMode() { assertEquals(Integer.class, nullable(row).getClass()); assertEquals(Integer.class, sqlDataType(row).getClass()); assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(9); + + row = rows.get(index ++); assertEquals("some.ambiguous", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -379,8 +414,8 @@ public void testSysColumnsInJdbcMode() { assertEquals(Integer.class, nullable(row).getClass()); assertEquals(Integer.class, sqlDataType(row).getClass()); assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(10); + + row = rows.get(index ++); assertEquals("some.ambiguous.one", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -389,8 +424,8 @@ public void testSysColumnsInJdbcMode() { assertEquals(Integer.class, nullable(row).getClass()); assertEquals(Integer.class, sqlDataType(row).getClass()); assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(11); + + row = rows.get(index ++); assertEquals("some.ambiguous.two", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -400,7 +435,7 @@ public void testSysColumnsInJdbcMode() { assertEquals(Integer.class, sqlDataType(row).getClass()); assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - row = rows.get(12); + row = rows.get(index ++); assertEquals("some.ambiguous.normalized", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); @@ -449,7 +484,7 @@ private static Object sqlDataTypeSub(List list) { public void testSysColumnsNoArg() throws Exception { executeCommand("SYS COLUMNS", emptyList(), r -> { - assertEquals(15, r.size()); + assertEquals(FIELD_COUNT, r.size()); assertEquals(CLUSTER_NAME, r.column(0)); // no index specified assertEquals("test", r.column(2)); @@ -464,7 +499,7 @@ public void testSysColumnsNoArg() throws Exception { public void testSysColumnsWithCatalogWildcard() throws Exception { executeCommand("SYS COLUMNS CATALOG 'cluster' TABLE LIKE 'test' LIKE '%'", emptyList(), r -> { - assertEquals(15, r.size()); + assertEquals(FIELD_COUNT, r.size()); assertEquals(CLUSTER_NAME, r.column(0)); assertEquals("test", r.column(2)); assertEquals("bool", r.column(3)); @@ -477,7 +512,7 @@ public void testSysColumnsWithCatalogWildcard() throws Exception { public void testSysColumnsWithMissingCatalog() throws Exception { executeCommand("SYS COLUMNS TABLE LIKE 'test' LIKE '%'", emptyList(), r -> { - assertEquals(15, r.size()); + assertEquals(FIELD_COUNT, r.size()); assertEquals(CLUSTER_NAME, r.column(0)); assertEquals("test", r.column(2)); assertEquals("bool", r.column(3)); @@ -490,7 +525,7 @@ public void testSysColumnsWithMissingCatalog() throws Exception { public void testSysColumnsWithNullCatalog() throws Exception { executeCommand("SYS COLUMNS CATALOG ? TABLE LIKE 'test' LIKE '%'", singletonList(new SqlTypedParamValue("keyword", null)), r -> { - assertEquals(15, r.size()); + assertEquals(FIELD_COUNT, r.size()); assertEquals(CLUSTER_NAME, r.column(0)); assertEquals("test", r.column(2)); assertEquals("bool", r.column(3)); @@ -507,13 +542,13 @@ public void testSysColumnsTypesInOdbcMode() { } public void testSysColumnsPaginationInOdbcMode() { - assertEquals(15, executeCommandInOdbcModeAndCountRows("SYS COLUMNS")); - assertEquals(15, executeCommandInOdbcModeAndCountRows("SYS COLUMNS TABLE LIKE 'test'")); + assertEquals(FIELD_COUNT, executeCommandInOdbcModeAndCountRows("SYS COLUMNS")); + assertEquals(FIELD_COUNT, executeCommandInOdbcModeAndCountRows("SYS COLUMNS TABLE LIKE 'test'")); } private int executeCommandInOdbcModeAndCountRows(String sql) { final SqlConfiguration config = new SqlConfiguration(DateUtils.UTC, randomIntBetween(1, 15), Protocol.REQUEST_TIMEOUT, - Protocol.PAGE_TIMEOUT, null, Mode.ODBC, null, null, null, false, false); + Protocol.PAGE_TIMEOUT, null, Mode.ODBC, null, SqlVersion.fromId(Version.CURRENT.id), null, null, false, false); Tuple tuple = sql(sql, emptyList(), config, mapping); int[] rowCount = {0}; @@ -538,7 +573,7 @@ public void onFailure(Exception e) { private void executeCommand(String sql, List params, Mode mode, Consumer consumer, Map mapping) { final SqlConfiguration config = new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, - Protocol.PAGE_TIMEOUT, null, mode, null, null, null, false, false); + Protocol.PAGE_TIMEOUT, null, mode, null, SqlVersion.fromId(Version.CURRENT.id), null, null, false, false); Tuple tuple = sql(sql, params, config, mapping); tuple.v1().execute(tuple.v2(), wrap(p -> consumer.accept((SchemaRowSet) p.rowSet()), ex -> fail(ex.getMessage()))); @@ -559,20 +594,20 @@ private Tuple sql(String sql, List para IndexResolver resolver = mock(IndexResolver.class); when(resolver.clusterName()).thenReturn(CLUSTER_NAME); doAnswer(invocation -> { - ((ActionListener) invocation.getArguments()[3]).onResponse(IndexResolution.valid(test)); + ((ActionListener) invocation.getArguments()[4]).onResponse(IndexResolution.valid(test)); return Void.TYPE; - }).when(resolver).resolveAsMergedMapping(any(), any(), anyBoolean(), any()); + }).when(resolver).resolveAsMergedMapping(any(), any(), anyBoolean(), any(), any()); doAnswer(invocation -> { - ((ActionListener>) invocation.getArguments()[3]).onResponse(singletonList(test)); + ((ActionListener>) invocation.getArguments()[4]).onResponse(singletonList(test)); return Void.TYPE; - }).when(resolver).resolveAsSeparateMappings(any(), any(), anyBoolean(), any()); + }).when(resolver).resolveAsSeparateMappings(any(), any(), anyBoolean(), any(), any()); SqlSession session = new SqlSession(config, null, null, resolver, null, null, null, null, null); return new Tuple<>(cmd, session); } private static void checkOdbcShortTypes(SchemaRowSet r) { - assertEquals(15, r.size()); + assertEquals(FIELD_COUNT, r.size()); // https://github.com/elastic/elasticsearch/issues/35376 // cols that need to be of short type: DATA_TYPE, DECIMAL_DIGITS, NUM_PREC_RADIX, NULLABLE, SQL_DATA_TYPE, SQL_DATETIME_SUB List cols = Arrays.asList(4, 8, 9, 10, 13, 14); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java index 90dbda045d1d3..17c58134ba0dd 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java @@ -59,7 +59,7 @@ public class SysTablesTests extends ESTestCase { private final IndexInfo frozen = new IndexInfo("frozen", IndexType.FROZEN_INDEX); private final SqlConfiguration FROZEN_CFG = new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, - Protocol.PAGE_TIMEOUT, null, Mode.PLAIN, null, null, null, false, true); + Protocol.PAGE_TIMEOUT, null, Mode.PLAIN, null, null, null, null, false, true); // // catalog enumeration diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java index 17bc17bbec4b8..5d3b70f5164e7 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java @@ -45,8 +45,8 @@ private Tuple sql(String sql) { public void testSysTypes() { Command cmd = sql("SYS TYPES").v1(); - List names = asList("BYTE", "LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "FLOAT", "DOUBLE", "SCALED_FLOAT", - "IP", "KEYWORD", "TEXT", "BOOLEAN", "DATE", "TIME", "DATETIME", + List names = asList("BYTE", "LONG", "UNSIGNED_LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "FLOAT", "DOUBLE", + "SCALED_FLOAT", "IP", "KEYWORD", "TEXT", "BOOLEAN", "DATE", "TIME", "DATETIME", "INTERVAL_YEAR", "INTERVAL_MONTH", "INTERVAL_DAY", "INTERVAL_HOUR", "INTERVAL_MINUTE", "INTERVAL_SECOND", "INTERVAL_YEAR_TO_MONTH", "INTERVAL_DAY_TO_HOUR", "INTERVAL_DAY_TO_MINUTE", "INTERVAL_DAY_TO_SECOND", "INTERVAL_HOUR_TO_MINUTE", "INTERVAL_HOUR_TO_SECOND", "INTERVAL_MINUTE_TO_SECOND", @@ -57,7 +57,7 @@ public void testSysTypes() { assertEquals(19, r.columnCount()); assertEquals(SqlDataTypes.types().size(), r.size()); assertFalse(r.schema().types().contains(DataTypes.NULL)); - // test numeric as signed + // test first numeric (BYTE) as signed assertFalse(r.column(9, Boolean.class)); // make sure precision is returned as boolean (not int) assertFalse(r.column(10, Boolean.class)); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java index ddbcacc2c1666..1c1deb909d127 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.sql.util.DateUtils; +import java.math.BigInteger; import java.time.OffsetTime; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -32,6 +33,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.NULL; import static org.elasticsearch.xpack.ql.type.DataTypes.SHORT; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; import static org.elasticsearch.xpack.sql.type.SqlDataTypeConverter.commonType; import static org.elasticsearch.xpack.sql.type.SqlDataTypeConverter.converterFor; @@ -147,6 +149,16 @@ public void testConversionToDate() { Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(Double.MAX_VALUE)); assertEquals("[" + Double.MAX_VALUE + "] out of [long] range", e.getMessage()); } + { + Converter conversion = converterFor(UNSIGNED_LONG, to); + assertNull(conversion.convert(null)); + BigInteger bi = BigInteger.valueOf(randomNonNegativeLong()); + assertEquals(date(bi.longValue()), conversion.convert(bi)); + + BigInteger tooLarge = bi.add(BigInteger.valueOf(Long.MAX_VALUE)); + Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(tooLarge)); + assertEquals("[" + tooLarge + "] out of [long] range", e.getMessage()); + } { Converter conversion = converterFor(INTEGER, to); assertNull(conversion.convert(null)); @@ -219,6 +231,16 @@ public void testConversionToTime() { Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(Double.MAX_VALUE)); assertEquals("[" + Double.MAX_VALUE + "] out of [long] range", e.getMessage()); } + { + Converter conversion = converterFor(UNSIGNED_LONG, to); + assertNull(conversion.convert(null)); + BigInteger bi = BigInteger.valueOf(randomNonNegativeLong()); + assertEquals(time(bi.longValue()), conversion.convert(bi)); + + BigInteger tooLarge = bi.add(BigInteger.valueOf(Long.MAX_VALUE)); + Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(tooLarge)); + assertEquals("[" + tooLarge + "] out of [long] range", e.getMessage()); + } { Converter conversion = converterFor(INTEGER, to); assertNull(conversion.convert(null)); @@ -506,6 +528,30 @@ public void testConversionToBoolean() { } } + public void testConversiontoUnsignedLong() { + DataType to = UNSIGNED_LONG; + { + Converter conversion = converterFor(DATE, to); + assertNull(conversion.convert(null)); + + long l = randomNonNegativeLong(); + ZonedDateTime zdt = asDateOnly(l); + assertEquals(BigInteger.valueOf(zdt.toEpochSecond() * 1000), conversion.convert(zdt)); + + ZonedDateTime zdtn = asDateOnly(-l); + Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(zdtn)); + assertEquals("[" + zdtn.toEpochSecond() * 1000 + "] out of [unsigned_long] range", e.getMessage()); + } + { + Converter conversion = converterFor(TIME, to); + assertNull(conversion.convert(null)); + + long l = randomLong(); + OffsetTime ot = asTimeOnly(l); + assertEquals(BigInteger.valueOf(ot.atDate(DateUtils.EPOCH).toInstant().toEpochMilli()), conversion.convert(ot)); + } + } + public void testConversionToInt() { DataType to = INTEGER; { @@ -711,7 +757,7 @@ private DataType randomInterval() { .filter(SqlDataTypes::isInterval) .collect(toList())); } - + static ZonedDateTime dateTime(long millisSinceEpoch) { return DateUtils.asDateTime(millisSinceEpoch); } From 56b6e3df13001488c03b9b5a701d1075a955a005 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Tue, 17 Nov 2020 17:13:19 +0100 Subject: [PATCH 02/28] style fixes Remove unused imports. --- .../org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java | 1 - .../execution/search/extractor/AbstractFieldHitExtractor.java | 2 -- 2 files changed, 3 deletions(-) diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java index 50e34ef167225..ae5b9f5514002 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java @@ -9,7 +9,6 @@ import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.TerminalNode; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.xpack.eql.expression.predicate.operator.comparison.InsensitiveEquals; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.ArithmeticUnaryContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.ComparisonContext; diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java index 96e07b28cfd93..373e3e133c541 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java @@ -28,8 +28,6 @@ import java.util.Map; import java.util.Objects; import java.util.StringJoiner; -import java.util.function.BiFunction; -import java.util.function.Function; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; From 9eda68342c12e21bf87e913138b5a7f0d1b57c00 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Thu, 19 Nov 2020 12:00:02 +0100 Subject: [PATCH 03/28] Make JDBC QA tests version aware Disable UNSIGNED_LONG testing for those bwc tests with drivers older than 7.11 - first target release to support them. --- .../xpack/sql/qa/jdbc/JdbcTestUtils.java | 18 +- .../qa/jdbc/PreparedStatementTestCase.java | 27 ++- .../qa/jdbc/ResultSetMetaDataTestCase.java | 55 +++-- .../xpack/sql/qa/jdbc/ResultSetTestCase.java | 201 ++++++++++-------- 4 files changed, 181 insertions(+), 120 deletions(-) diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java index 8b9ee6cb12da1..d7ba41c28fd7d 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.sql.qa.jdbc; +import org.elasticsearch.Version; import org.elasticsearch.xpack.sql.jdbc.EsType; import org.elasticsearch.xpack.sql.proto.StringUtils; @@ -24,6 +25,8 @@ import java.util.LinkedHashMap; import java.util.Map; +import static org.elasticsearch.Version.V_7_11_0; + final class JdbcTestUtils { private JdbcTestUtils() {} @@ -32,16 +35,25 @@ private JdbcTestUtils() {} static final ZoneId UTC = ZoneId.of("Z"); static final String JDBC_TIMEZONE = "timezone"; + private static final String DRIVER_VERSION_PROPERTY_NAME = "jdbc.driver.version"; static final LocalDate EPOCH = LocalDate.of(1970, 1, 1); + static final Version JDBC_DRIVER_VERSION; + static { + // master's version is x.0.0-SNAPSHOT, tho Version#fromString() won't accept that back for recent versions + String jdbcDriverVersion = System.getProperty(DRIVER_VERSION_PROPERTY_NAME, "").replace("-SNAPSHOT", ""); + JDBC_DRIVER_VERSION = Version.fromString(jdbcDriverVersion); // takes empty and null strings, resolves them to CURRENT + Map, EsType> aMap = new LinkedHashMap<>(); aMap.put(Boolean.class, EsType.BOOLEAN); aMap.put(Byte.class, EsType.BYTE); aMap.put(Short.class, EsType.SHORT); aMap.put(Integer.class, EsType.INTEGER); aMap.put(Long.class, EsType.LONG); - aMap.put(BigInteger.class, EsType.UNSIGNED_LONG); + if (isUnsignedLongSupported()) { + aMap.put(BigInteger.class, EsType.UNSIGNED_LONG); + } aMap.put(Float.class, EsType.FLOAT); aMap.put(Double.class, EsType.DOUBLE); aMap.put(String.class, EsType.KEYWORD); @@ -96,4 +108,8 @@ static long convertFromCalendarToUTC(long value, Calendar cal) { return convertedDateTime.toInstant().toEpochMilli(); } + + public static boolean isUnsignedLongSupported() { + return V_7_11_0.compareTo(JDBC_DRIVER_VERSION) <= 0; + } } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java index d70e8a6913f90..b545e7a6bd5d2 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java @@ -7,6 +7,7 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.sql.jdbc.EsType; import java.io.IOException; import java.math.BigDecimal; @@ -36,13 +37,14 @@ import static org.elasticsearch.xpack.sql.jdbc.EsType.KEYWORD; import static org.elasticsearch.xpack.sql.jdbc.EsType.LONG; import static org.elasticsearch.xpack.sql.jdbc.EsType.SHORT; -import static org.elasticsearch.xpack.sql.jdbc.EsType.UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.isUnsignedLongSupported; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.startsWith; public abstract class PreparedStatementTestCase extends JdbcIntegrationTestCase { static final BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); + EsType UNSIGNED_LONG = isUnsignedLongSupported() ? EsType.valueOf("UNSIGNED_LONG") : null; public void testSupportedTypes() throws SQLException { String stringVal = randomAlphaOfLength(randomIntBetween(0, 1000)); @@ -89,7 +91,6 @@ public void testSupportedTypes() throws SQLException { statement.setByte(index ++, byteVal); statement.setShort(index ++, shortVal); statement.setBigDecimal(index ++, bigDecimalVal); - statement.setObject(index ++, bigIntegerVal); statement.setTimestamp(index ++, timestampVal); statement.setTimestamp(index ++, timestampVal, calendarVal); statement.setDate(index ++, dateVal); @@ -99,6 +100,9 @@ public void testSupportedTypes() throws SQLException { statement.setObject(index ++, calendarVal); statement.setObject(index ++, utilDateVal); statement.setObject(index ++, localDateTimeVal); + if (isUnsignedLongSupported()) { + statement.setObject(index++, bigIntegerVal); + } try (ResultSet results = statement.executeQuery()) { ResultSetMetaData resultSetMetaData = results.getMetaData(); @@ -120,7 +124,6 @@ public void testSupportedTypes() throws SQLException { assertEquals(byteVal, results.getByte(index ++)); assertEquals(shortVal, results.getShort(index ++)); assertEquals(bigDecimalVal, results.getBigDecimal(index ++)); - assertEquals(bigIntegerVal, results.getObject(index ++)); assertEquals(timestampVal, results.getTimestamp(index ++)); assertEquals(timestampValWithCal, results.getTimestamp(index ++)); assertEquals(dateVal, results.getDate(index ++)); @@ -130,6 +133,9 @@ public void testSupportedTypes() throws SQLException { assertEquals(new Timestamp(calendarVal.getTimeInMillis()), results.getObject(index ++)); assertEquals(timestampVal, results.getObject(index ++)); assertEquals(timestampVal, results.getObject(index ++)); + if (isUnsignedLongSupported()) { + assertEquals(bigIntegerVal, results.getObject(index++)); + } assertFalse(results.next()); } } @@ -364,8 +370,6 @@ public void testSingleParameterMultipleTypes() throws SQLException { assertEquals(new Tuple<>(INTEGER.getName(), intVal), execute(statement)); statement.setLong(1, longVal); assertEquals(new Tuple<>(LONG.getName(), longVal), execute(statement)); - statement.setObject(1, bigIntegerVal); - assertEquals(new Tuple<>(UNSIGNED_LONG.getName(), bigIntegerVal), execute(statement)); statement.setFloat(1, floatVal); assertEquals(new Tuple<>(FLOAT.getName(), floatVal), execute(statement)); statement.setDouble(1, doubleVal); @@ -379,10 +383,15 @@ public void testSingleParameterMultipleTypes() throws SQLException { statement.setShort(1, shortVal); assertEquals(new Tuple<>(SHORT.getName(), shortVal), execute(statement)); - statement.setObject(1, longVal, JDBCType.BIGINT, 19); - assertEquals(new Tuple<>(LONG.getName(), longVal), execute(statement)); - statement.setObject(1, Math.abs(longVal), JDBCType.BIGINT, 20); - assertEquals(new Tuple<>(UNSIGNED_LONG.getName(), BigInteger.valueOf(Math.abs(longVal))), execute(statement)); + if (isUnsignedLongSupported()) { + statement.setObject(1, bigIntegerVal); + assertEquals(new Tuple<>(UNSIGNED_LONG.getName(), bigIntegerVal), execute(statement)); + + statement.setObject(1, longVal, JDBCType.BIGINT, 19); + assertEquals(new Tuple<>(LONG.getName(), longVal), execute(statement)); + statement.setObject(1, Math.abs(longVal), JDBCType.BIGINT, 20); + assertEquals(new Tuple<>(UNSIGNED_LONG.getName(), BigInteger.valueOf(Math.abs(longVal))), execute(statement)); + } } } } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java index 2d1161a83c482..9bd5e28b9d158 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java @@ -14,20 +14,32 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.isUnsignedLongSupported; public abstract class ResultSetMetaDataTestCase extends JdbcIntegrationTestCase { - private final String[] fieldsNames = new String[] { - "test_byte", - "test_integer", - "test_long", - "test_unsigned_long", - "test_short", - "test_double", - "test_float", - "test_keyword", - "test_boolean", - "test_date" }; + private static final List fieldsNames = new ArrayList<>(); + + static { + fieldsNames.addAll(List.of( + "test_byte", + "test_integer", + "test_long", + "test_short", + "test_double", + "test_float", + "test_keyword", + "test_boolean", + "test_date" + )); + if (isUnsignedLongSupported()) { + fieldsNames.add("test_unsigned_long"); + } + } public void testValidGetObjectCalls() throws IOException, SQLException { ResultSetTestCase.createIndex("test"); @@ -37,31 +49,30 @@ public void testValidGetObjectCalls() throws IOException, SQLException { } }); - String q = "SELECT test_byte, test_integer, test_long, test_unsigned_long, test_short, test_double, test_float, test_keyword, " - + "test_boolean, test_date FROM test"; + String q = "SELECT " + String.join(", ", fieldsNames) + " FROM test"; doWithQuery(q, r -> assertColumnNamesAndLabels(r.getMetaData(), fieldsNames)); - q = "SELECT test_byte AS b, test_integer AS i, test_long AS l, test_unsigned_long AS ul, test_short AS s, test_double AS d, " - + "test_float AS f, test_keyword AS k, test_boolean AS bool, test_date AS dt FROM test"; - doWithQuery(q, r -> assertColumnNamesAndLabels(r.getMetaData(), - new String[] { "b", "i", "l", "ul", "s", "d", "f", "k", "bool", "dt" })); + + q = "SELECT " + fieldsNames.stream().map(x -> x + " AS " + x.replace("_", "")).collect(Collectors.joining(", ")) + " FROM test"; + doWithQuery(q, r -> assertColumnNamesAndLabels(r.getMetaData(), fieldsNames.stream() + .map(x -> x.replace("_", "")).collect(Collectors.toList()))); } private void doWithQuery(String query, CheckedConsumer consumer) throws SQLException { try (Connection connection = esJdbc()) { try (PreparedStatement statement = connection.prepareStatement(query)) { try (ResultSet results = statement.executeQuery()) { - assertEquals(fieldsNames.length, results.getMetaData().getColumnCount()); + assertEquals(fieldsNames.size(), results.getMetaData().getColumnCount()); consumer.accept(results); } } } } - private void assertColumnNamesAndLabels(ResultSetMetaData metaData, String[] names) throws SQLException { - for (int i = 0; i < fieldsNames.length; i++) { - assertEquals(names[i], metaData.getColumnName(i + 1)); - assertEquals(names[i], metaData.getColumnLabel(i + 1)); + private void assertColumnNamesAndLabels(ResultSetMetaData metaData, List names) throws SQLException { + for (int i = 0; i < fieldsNames.size(); i++) { + assertEquals(names.get(i), metaData.getColumnName(i + 1)); + assertEquals(names.get(i), metaData.getColumnLabel(i + 1)); } } } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java index 031d849864c62..fe91d44550962 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java @@ -39,6 +39,7 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Date; @@ -50,11 +51,9 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Properties; -import java.util.Set; import java.util.TimeZone; import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.stream.Stream; import static java.lang.String.format; import static java.util.Calendar.DAY_OF_MONTH; @@ -65,35 +64,47 @@ import static java.util.Calendar.MONTH; import static java.util.Calendar.SECOND; import static java.util.Calendar.YEAR; -import static org.elasticsearch.xpack.sql.jdbc.EsType.UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.JDBC_DRIVER_VERSION; +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.isUnsignedLongSupported; import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.of; public abstract class ResultSetTestCase extends JdbcIntegrationTestCase { - static final Set fieldsNames = Stream.of( - "test_byte", - "test_integer", - "test_long", - "test_unsigned_long", - "test_short", - "test_double", - "test_float", - "test_keyword" - ).collect(Collectors.toCollection(HashSet::new)); + static final List fieldsNames = new ArrayList<>(); static final Map, SQLType> dateTimeTestingFields = new HashMap<>(); - static final String SELECT_ALL_FIELDS = "SELECT test_boolean, test_byte, test_integer," - + "test_long, test_unsigned_long, test_short, test_double, test_float, test_keyword, test_date FROM test"; + static final String SELECT_ALL_FIELDS; static final String SELECT_WILDCARD = "SELECT * FROM test"; + static final EsType UNSIGNED_LONG; + static { + fieldsNames.addAll(List.of( + "test_byte", + "test_integer", + "test_long", + "test_short", + "test_double", + "test_float", + "test_keyword" + )); + if (isUnsignedLongSupported()) { + fieldsNames.add("test_unsigned_long"); + } + + SELECT_ALL_FIELDS = "SELECT test_boolean, " + String.join(", ", fieldsNames) + ", test_date FROM test"; + + UNSIGNED_LONG = isUnsignedLongSupported() ? EsType.valueOf("UNSIGNED_LONG") : null; + dateTimeTestingFields.put(new Tuple<>("test_boolean", true), EsType.BOOLEAN); dateTimeTestingFields.put(new Tuple<>("test_byte", 1), EsType.BYTE); dateTimeTestingFields.put(new Tuple<>("test_integer", 1), EsType.INTEGER); dateTimeTestingFields.put(new Tuple<>("test_long", 1L), EsType.LONG); - dateTimeTestingFields.put(new Tuple<>("test_unsigned_long", BigInteger.ONE), UNSIGNED_LONG); dateTimeTestingFields.put(new Tuple<>("test_short", 1), EsType.SHORT); dateTimeTestingFields.put(new Tuple<>("test_double", 1d), EsType.DOUBLE); dateTimeTestingFields.put(new Tuple<>("test_float", 1f), EsType.FLOAT); dateTimeTestingFields.put(new Tuple<>("test_keyword", "true"), EsType.KEYWORD); + if (isUnsignedLongSupported()) { + dateTimeTestingFields.put(new Tuple<>("test_unsigned_long", BigInteger.ONE), UNSIGNED_LONG); + } } private String timeZoneId; @@ -223,6 +234,22 @@ public void testGettingValidByteWithCasting() throws IOException, SQLException { }); } + static void indexTestFieldsDoc(String index, String docId, int i, long l, short s, double d, float f, String k, long date, + BigInteger bi) throws IOException { + index(index, "1", builder -> { + builder.field("test_integer", i); + builder.field("test_long", l); + builder.field("test_short", s); + builder.field("test_double", d); + builder.field("test_float", f); + builder.field("test_keyword", k); + builder.field("test_date", date); + if (isUnsignedLongSupported()) { + builder.field("test_unsigned_long", bi); + } + }); + } + public void testGettingInvalidByte() throws IOException, SQLException { createIndex("test"); updateMappingForNumericValuesTests("test"); @@ -244,16 +271,8 @@ public void testGettingInvalidByte() throws IOException, SQLException { ? Double.toString(doubleNotByte) : Long.toString(Math.round(doubleNotByte)); - index("test", "1", builder -> { - builder.field("test_integer", intNotByte); - builder.field("test_long", longNotByte); - builder.field("test_unsigned_long", bigIntegerNotByte); - builder.field("test_short", shortNotByte); - builder.field("test_double", doubleNotByte); - builder.field("test_float", floatNotByte); - builder.field("test_keyword", randomString); - builder.field("test_date", randomDate); - }); + indexTestFieldsDoc("test", "1", intNotByte, longNotByte, shortNotByte, doubleNotByte, floatNotByte, randomString, randomDate, + bigIntegerNotByte); doWithQuery(SELECT_WILDCARD, results -> { results.next(); @@ -268,13 +287,6 @@ public void testGettingInvalidByte() throws IOException, SQLException { sqle = expectThrows(SQLException.class, () -> results.getObject("test_short", Byte.class)); assertEquals(format(Locale.ROOT, "Numeric %s out of range", shortNotByte), sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getByte("test_unsigned_long")); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Byte]", bigIntegerNotByte), - sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Byte.class)); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Byte]", bigIntegerNotByte), - sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getByte("test_long")); assertEquals(format(Locale.ROOT, "Numeric %s out of range", Long.toString(longNotByte)), sqle.getMessage()); sqle = expectThrows(SQLException.class, () -> results.getObject("test_long", Byte.class)); @@ -311,6 +323,15 @@ public void testGettingInvalidByte() throws IOException, SQLException { format(Locale.ROOT, "Unable to convert value [%.128s] of type [DATETIME] to [Byte]", asDateString(randomDate)), sqle.getMessage() ); + + if (isUnsignedLongSupported()) { + sqle = expectThrows(SQLException.class, () -> results.getByte("test_unsigned_long")); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Byte]", bigIntegerNotByte), + sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Byte.class)); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Byte]", bigIntegerNotByte), + sqle.getMessage()); + } }); } @@ -389,15 +410,8 @@ public void testGettingInvalidShort() throws IOException, SQLException { ? Double.toString(doubleNotShort) : Long.toString(Math.round(doubleNotShort)); - index("test", "1", builder -> { - builder.field("test_integer", intNotShort); - builder.field("test_long", longNotShort); - builder.field("test_unsigned_long", bigIntegerNotShort); - builder.field("test_double", doubleNotShort); - builder.field("test_float", floatNotShort); - builder.field("test_keyword", randomString); - builder.field("test_date", randomDate); - }); + indexTestFieldsDoc("test", "1", intNotShort, longNotShort, (short) 0, doubleNotShort, floatNotShort, randomString, randomDate, + bigIntegerNotShort); doWithQuery(SELECT_WILDCARD, results -> { results.next(); @@ -412,13 +426,6 @@ public void testGettingInvalidShort() throws IOException, SQLException { sqle = expectThrows(SQLException.class, () -> results.getObject("test_long", Short.class)); assertEquals(format(Locale.ROOT, "Numeric %s out of range", longNotShort), sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getShort("test_unsigned_long")); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Short]", bigIntegerNotShort), - sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Short.class)); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Short]", bigIntegerNotShort), - sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getShort("test_double")); assertEquals(format(Locale.ROOT, "Numeric %s out of range", doubleErrorMessage), sqle.getMessage()); sqle = expectThrows(SQLException.class, () -> results.getObject("test_double", Short.class)); @@ -450,6 +457,15 @@ public void testGettingInvalidShort() throws IOException, SQLException { format(Locale.ROOT, "Unable to convert value [%.128s] of type [DATETIME] to [Short]", asDateString(randomDate)), sqle.getMessage() ); + + if (isUnsignedLongSupported()) { + sqle = expectThrows(SQLException.class, () -> results.getShort("test_unsigned_long")); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Short]", bigIntegerNotShort), + sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Short.class)); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Short]", bigIntegerNotShort), + sqle.getMessage()); + } }); } @@ -527,14 +543,7 @@ public void testGettingInvalidInteger() throws IOException, SQLException { ? Double.toString(doubleNotInt) : Long.toString(Math.round(doubleNotInt)); - index("test", "1", builder -> { - builder.field("test_long", longNotInt); - builder.field("test_unsigned_long", bigIntegerNotInt); - builder.field("test_double", doubleNotInt); - builder.field("test_float", floatNotInt); - builder.field("test_keyword", randomString); - builder.field("test_date", randomDate); - }); + indexTestFieldsDoc("test", "1", 0, longNotInt, (short) 0, doubleNotInt, floatNotInt, randomString, randomDate, bigIntegerNotInt); doWithQuery(SELECT_WILDCARD, results -> { results.next(); @@ -544,13 +553,6 @@ public void testGettingInvalidInteger() throws IOException, SQLException { sqle = expectThrows(SQLException.class, () -> results.getObject("test_long", Integer.class)); assertEquals(format(Locale.ROOT, "Numeric %s out of range", Long.toString(longNotInt)), sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getInt("test_unsigned_long")); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Integer]", bigIntegerNotInt), - sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Integer.class)); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Integer]", bigIntegerNotInt), - sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getInt("test_double")); assertEquals(format(Locale.ROOT, "Numeric %s out of range", doubleErrorMessage), sqle.getMessage()); sqle = expectThrows(SQLException.class, () -> results.getObject("test_double", Integer.class)); @@ -582,6 +584,15 @@ public void testGettingInvalidInteger() throws IOException, SQLException { format(Locale.ROOT, "Unable to convert value [%.128s] of type [DATETIME] to [Integer]", asDateString(randomDate)), sqle.getMessage() ); + + if (isUnsignedLongSupported()) { + sqle = expectThrows(SQLException.class, () -> results.getInt("test_unsigned_long")); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Integer]", bigIntegerNotInt), + sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Integer.class)); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Integer]", bigIntegerNotInt), + sqle.getMessage()); + } }); } @@ -651,25 +662,12 @@ public void testGettingInvalidLong() throws IOException, SQLException { String randomString = randomUnicodeOfCodepointLengthBetween(128, 256); long randomDate = randomNonNegativeLong(); - index("test", "1", builder -> { - builder.field("test_unsigned_long", bigIntegerNotLong); - builder.field("test_double", doubleNotLong); - builder.field("test_float", floatNotLong); - builder.field("test_keyword", randomString); - builder.field("test_date", randomDate); - }); + indexTestFieldsDoc("test", "1", 0, 0L, (short) 0, doubleNotLong, floatNotLong, randomString, randomDate, bigIntegerNotLong); doWithQuery(SELECT_WILDCARD, results -> { results.next(); - SQLException sqle = expectThrows(SQLException.class, () -> results.getLong("test_unsigned_long")); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Long]", bigIntegerNotLong), - sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Long.class)); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Long]", bigIntegerNotLong), - sqle.getMessage()); - - sqle = expectThrows(SQLException.class, () -> results.getLong("test_double")); + SQLException sqle = expectThrows(SQLException.class, () -> results.getLong("test_double")); assertEquals(format(Locale.ROOT, "Numeric %s out of range", Double.toString(doubleNotLong)), sqle.getMessage()); sqle = expectThrows(SQLException.class, () -> results.getObject("test_double", Long.class)); assertEquals(format(Locale.ROOT, "Numeric %s out of range", Double.toString(doubleNotLong)), sqle.getMessage()); @@ -700,11 +698,22 @@ public void testGettingInvalidLong() throws IOException, SQLException { format(Locale.ROOT, "Unable to convert value [%.128s] of type [DATETIME] to [Long]", asDateString(randomDate)), sqle.getMessage() ); + + if (isUnsignedLongSupported()) { + sqle = expectThrows(SQLException.class, () -> results.getLong("test_unsigned_long")); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Long]", bigIntegerNotLong), + sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Long.class)); + assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Long]", bigIntegerNotLong), + sqle.getMessage()); + } }); } // BigInteger/unsigned_long values testing public void testGettingValidBigIntegerWithoutCasting() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); + List bigIntegerTestValues = createTestDataForNumericValueTests(ESTestCase::randomBigInteger); BigInteger random1 = bigIntegerTestValues.get(0); BigInteger random2 = bigIntegerTestValues.get(1); @@ -738,6 +747,8 @@ public void testGettingValidBigIntegerWithoutCasting() throws IOException, SQLEx } public void testGettingValidBigIntegerWithCasting() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); + Map map = createTestDataForNumericValueTypes(ESTestCase::randomBigInteger); doWithQuery(SELECT_WILDCARD, results -> { @@ -757,6 +768,8 @@ public void testGettingValidBigIntegerWithCasting() throws IOException, SQLExcep } public void testGettingInvalidBigInteger() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); + createIndex("test"); updateMappingForNumericValuesTests("test"); updateMapping("test", builder -> { @@ -1209,12 +1222,14 @@ public void testGettingBooleanValues() throws IOException, SQLException { builder.field("test_byte", 0); builder.field("test_integer", 0); builder.field("test_long", 0L); - builder.field("test_unsigned_long", 0L); builder.field("test_short", 0); builder.field("test_double", 0d); builder.field("test_float", 0f); builder.field("test_keyword", "false"); builder.field("test_date", randomDate2); + if (isUnsignedLongSupported()) { + builder.field("test_unsigned_long", 0L); + } }); // other (non 0 = true) values @@ -1222,7 +1237,6 @@ public void testGettingBooleanValues() throws IOException, SQLException { builder.field("test_byte", randomValueOtherThan((byte) 0, ESTestCase::randomByte)); builder.field("test_integer", randomValueOtherThan(0, ESTestCase::randomInt)); builder.field("test_long", randomValueOtherThan(0L, ESTestCase::randomLong)); - builder.field("test_unsigned_long", randomValueOtherThan(BigInteger.ZERO, ESTestCase::randomBigInteger)); builder.field("test_short", randomValueOtherThan((short) 0, ESTestCase::randomShort)); builder.field( "test_double", @@ -1239,6 +1253,9 @@ public void testGettingBooleanValues() throws IOException, SQLException { ) ); builder.field("test_keyword", "1"); + if (isUnsignedLongSupported()) { + builder.field("test_unsigned_long", randomValueOtherThan(BigInteger.ZERO, ESTestCase::randomBigInteger)); + } }); // other false values @@ -1303,9 +1320,9 @@ public void testGettingDateWithoutCalendar() throws IOException, SQLException { java.sql.Date expectedDate = JdbcTestUtils.asDate(randomLongDate, getZoneFromOffset(randomLongDate)); assertEquals(expectedDate, results.getDate("test_date")); - assertEquals(expectedDate, results.getDate(10)); + assertEquals(expectedDate, results.getDate(fieldsNames.size() + 2)); assertEquals(expectedDate, results.getObject("test_date", java.sql.Date.class)); - assertEquals(expectedDate, results.getObject(10, java.sql.Date.class)); + assertEquals(expectedDate, results.getObject(fieldsNames.size() + 2, java.sql.Date.class)); // bulk validation for all fields which are not of type date validateErrorsForDateTestsWithoutCalendar(results::getDate); @@ -1335,7 +1352,7 @@ public void testGettingDateWithCalendar() throws IOException, SQLException { c.set(MILLISECOND, 0); assertEquals(results.getDate("test_date", c), new java.sql.Date(c.getTimeInMillis())); - assertEquals(results.getDate(10, c), new java.sql.Date(c.getTimeInMillis())); + assertEquals(results.getDate(fieldsNames.size() + 2, c), new java.sql.Date(c.getTimeInMillis())); // bulk validation for all fields which are not of type date validateErrorsForDateTimeTestsWithCalendar(c, results::getDate); @@ -1361,9 +1378,9 @@ public void testGettingTimeWithoutCalendar() throws IOException, SQLException { java.sql.Time expectedTime = JdbcTestUtils.asTime(randomLongDate, getZoneFromOffset(randomLongDate)); assertEquals(expectedTime, results.getTime("test_date")); - assertEquals(expectedTime, results.getTime(10)); + assertEquals(expectedTime, results.getTime(fieldsNames.size() + 2)); assertEquals(expectedTime, results.getObject("test_date", java.sql.Time.class)); - assertEquals(expectedTime, results.getObject(10, java.sql.Time.class)); + assertEquals(expectedTime, results.getObject(fieldsNames.size() + 2, java.sql.Time.class)); validateErrorsForTimeTestsWithoutCalendar(results::getTime); }); @@ -1392,7 +1409,7 @@ public void testGettingTimeWithCalendar() throws IOException, SQLException { c.set(DAY_OF_MONTH, 1); assertEquals(results.getTime("test_date", c), new java.sql.Time(c.getTimeInMillis())); - assertEquals(results.getTime(10, c), new java.sql.Time(c.getTimeInMillis())); + assertEquals(results.getTime(fieldsNames.size() + 2, c), new java.sql.Time(c.getTimeInMillis())); validateErrorsForDateTimeTestsWithCalendar(c, results::getTime); @@ -1462,7 +1479,7 @@ public void testGettingTimestampWithCalendar() throws IOException, SQLException c.setTimeInMillis(randomLongDate); assertEquals(results.getTimestamp("test_date", c), new java.sql.Timestamp(c.getTimeInMillis())); - assertEquals(results.getTimestamp(10, c), new java.sql.Timestamp(c.getTimeInMillis())); + assertEquals(results.getTimestamp(fieldsNames.size() + 2, c), new java.sql.Timestamp(c.getTimeInMillis())); validateErrorsForDateTimeTestsWithCalendar(c, results::getTimestamp); @@ -1686,6 +1703,9 @@ public void testGettingNullValues() throws SQLException { String query = "SELECT CAST(NULL AS BOOLEAN) b, CAST(NULL AS TINYINT) t, CAST(NULL AS SMALLINT) s, CAST(NULL AS INTEGER) i," + "CAST(NULL AS BIGINT) bi, CAST(NULL AS DOUBLE) d, CAST(NULL AS REAL) r, CAST(NULL AS FLOAT) f, CAST(NULL AS VARCHAR) v," + "CAST(NULL AS DATE) dt, CAST(NULL AS TIME) tm, CAST(NULL AS TIMESTAMP) ts"; + if (isUnsignedLongSupported()) { + query += ", CAST(NULL AS UNSIGNED_LONG) as ul"; + } doWithQuery(query, results -> { results.next(); @@ -1727,6 +1747,11 @@ public void testGettingNullValues() throws SQLException { assertNull(results.getObject("ts")); assertNull(results.getTimestamp("ts")); assertNull(results.getTimestamp("ts", randomCalendar())); + + if (isUnsignedLongSupported()) { + assertNull(results.getObject("ul")); + assertEquals(0.0f, results.getFloat("f"), 0f); + } }); } From dced577701607db0f03ebbeecd617d8b7ed09ec5 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Thu, 19 Nov 2020 13:40:37 +0100 Subject: [PATCH 04/28] Disable UL for 7.11 till after merging it in Also, undo change of BigInteger.valueOf(randomNonNegativeLong()) to randomBigInteger() for long-requiring tests. --- .../xpack/ql/type/DataTypeConversionTests.java | 8 ++++---- .../elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java index 4f1e32037c99c..3d3f9ead14650 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java @@ -74,7 +74,7 @@ public void testConversionToLong() { { Converter conversion = converterFor(UNSIGNED_LONG, to); assertNull(conversion.convert(null)); - BigInteger bi = randomBigInteger(); + BigInteger bi = BigInteger.valueOf(randomNonNegativeLong()); assertEquals(bi.longValue(), conversion.convert(bi)); BigInteger longPlus = bi.add(BigInteger.valueOf(Long.MAX_VALUE)); @@ -123,7 +123,7 @@ public void testConversionToDateTime() { { Converter conversion = converterFor(UNSIGNED_LONG, to); assertNull(conversion.convert(null)); - BigInteger bi = randomBigInteger(); + BigInteger bi = BigInteger.valueOf(randomNonNegativeLong()); assertEquals(asDateTime(bi.longValue()), conversion.convert(bi)); BigInteger longPlus = bi.add(BigInteger.valueOf(Long.MAX_VALUE)); @@ -352,8 +352,8 @@ public void testConversiontoUnsignedLong() { Converter conversion = converterFor(LONG, to); assertNull(conversion.convert(null)); - BigInteger bi = randomBigInteger(); - assertEquals(bi, conversion.convert(bi)); + BigInteger bi = BigInteger.valueOf(randomNonNegativeLong()); + assertEquals(bi, conversion.convert(bi.longValue())); Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert(bi.negate())); assertEquals("[" + bi.negate() + "] out of [unsigned_long] range", e.getMessage()); diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java index d7ba41c28fd7d..33e856dc62c25 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java @@ -110,6 +110,7 @@ static long convertFromCalendarToUTC(long value, Calendar cal) { } public static boolean isUnsignedLongSupported() { - return V_7_11_0.compareTo(JDBC_DRIVER_VERSION) <= 0; + // TODO: add equality only once actually ported to 7.11 + return V_7_11_0.compareTo(JDBC_DRIVER_VERSION) < 0; } } From e2a23637bf5936a7c7cc940ba9046596e2c14117 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Mon, 23 Nov 2020 00:06:17 +0100 Subject: [PATCH 05/28] Refactor UL gating based on version Move the condition checking wheatehr unsigned_long fields should be exposed to the clients from the IndexResolver to the Verifier, SysColumns and SysTypes commands. --- .../xpack/ql/index/IndexResolver.java | 45 ++- .../xpack/ql/type/DataTypeRegistry.java | 7 - .../xpack/ql/util/VersionUtil.java | 26 -- .../xpack/sql/qa/jdbc/ResultSetTestCase.java | 5 +- .../xpack/sql/proto/SqlVersion.java | 23 +- .../xpack/sql/proto/SqlVersionTests.java | 13 +- .../xpack/sql/analysis/analyzer/Analyzer.java | 48 ++- .../xpack/sql/analysis/analyzer/Verifier.java | 18 +- .../xpack/sql/execution/PlanExecutor.java | 14 +- .../sql/plan/logical/command/ShowColumns.java | 5 +- .../plan/logical/command/sys/SysColumns.java | 24 +- .../plan/logical/command/sys/SysTypes.java | 6 +- .../xpack/sql/session/Compatibility.java | 39 +++ .../xpack/sql/session/SqlConfiguration.java | 9 +- .../xpack/sql/session/SqlSession.java | 3 - .../xpack/sql/type/SqlDataTypeRegistry.java | 6 - .../xpack/sql/type/SqlDataTypes.java | 13 - .../elasticsearch/xpack/sql/SqlTestUtils.java | 18 +- .../analyzer/FieldAttributeTests.java | 70 +++- .../analyzer/VerifierErrorMessagesTests.java | 6 +- .../analysis/index/IndexResolverTests.java | 60 +--- .../scalar/DatabaseFunctionTests.java | 11 +- .../function/scalar/UserFunctionTests.java | 13 +- .../scalar/datetime/CurrentDateTimeTests.java | 7 +- .../scalar/datetime/CurrentTimeTests.java | 3 +- .../sql/optimizer/OptimizerRunTests.java | 5 +- .../logical/command/sys/SysColumnsTests.java | 305 ++++++------------ .../logical/command/sys/SysTablesTests.java | 2 +- .../logical/command/sys/SysTypesTests.java | 91 ++++-- .../planner/PostOptimizerVerifierTests.java | 3 +- .../xpack/sql/planner/QueryFolderTests.java | 3 +- .../sql/planner/QueryTranslatorTests.java | 3 +- .../xpack/sql/stats/VerifierMetricsTests.java | 62 ++-- 33 files changed, 490 insertions(+), 476 deletions(-) delete mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/VersionUtil.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/Compatibility.java diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index 8086eb56a30ed..ed85985585763 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -285,26 +285,24 @@ public void resolveAsMergedMapping(String indexWildcard, String javaRegex, Indic FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, indicesOptions); client.fieldCaps(fieldRequest, ActionListener.wrap( - response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response.getIndices(), response.get(), - version)), + response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response.getIndices(), response.get())), listener::onFailure)); } /** * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. */ - public void resolveAsMergedMapping(String indexWildcard, String javaRegex, boolean includeFrozen, @Nullable Version version, + public void resolveAsMergedMapping(String indexWildcard, String javaRegex, boolean includeFrozen, ActionListener listener) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen); client.fieldCaps(fieldRequest, ActionListener.wrap( - response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response.getIndices(), response.get(), - version)), + response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response.getIndices(), response.get())), listener::onFailure)); } public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, String indexPattern, String[] indexNames, - Map> fieldCaps, @Nullable Version version) { + Map> fieldCaps) { if (indexNames.length == 0) { return IndexResolution.notFound(indexPattern); @@ -363,8 +361,7 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri // everything checks return null; - }, - version); + }); if (indices.size() > 1) { throw new QlIllegalArgumentException( @@ -379,8 +376,7 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa Map> globalCaps, Map hierarchicalMapping, Map flattedMapping, - Function field, - Version version) { + Function field) { Map parentProps = hierarchicalMapping; @@ -399,7 +395,7 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa // lack of parent implies the field is an alias if (map == null) { // as such, create the field manually, marking the field to also be an alias - fieldFunction = s -> createField(typeRegistry, s, OBJECT.esType(), new TreeMap<>(), false, true, version); + fieldFunction = s -> createField(typeRegistry, s, OBJECT.esType(), new TreeMap<>(), false, true); } else { Iterator iterator = map.values().iterator(); FieldCapabilities parentCap = iterator.next(); @@ -407,11 +403,10 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa parentCap = iterator.next(); } final FieldCapabilities parentC = parentCap; - fieldFunction = s -> createField(typeRegistry, s, parentC.getType(), new TreeMap<>(), parentC.isAggregatable(), false, - version); + fieldFunction = s -> createField(typeRegistry, s, parentC.getType(), new TreeMap<>(), parentC.isAggregatable(), false); } - parent = createField(typeRegistry, parentName, globalCaps, hierarchicalMapping, flattedMapping, fieldFunction, version); + parent = createField(typeRegistry, parentName, globalCaps, hierarchicalMapping, flattedMapping, fieldFunction); } parentProps = parent.getProperties(); } @@ -440,8 +435,8 @@ private static EsField createField(DataTypeRegistry typeRegistry, String fieldNa } private static EsField createField(DataTypeRegistry typeRegistry, String fieldName, String typeName, Map props, - boolean isAggregateable, boolean isAlias, Version version) { - DataType esType = typeRegistry.fromEs(typeName, version); + boolean isAggregateable, boolean isAlias) { + DataType esType = typeRegistry.fromEs(typeName); if (esType == TEXT) { return new TextEsField(fieldName, props, false, isAlias); @@ -480,17 +475,15 @@ private static FieldCapabilitiesRequest createFieldCapsRequest(String index, boo /** * Resolves a pattern to multiple, separate indices. Doesn't perform validation. */ - public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, boolean includeFrozen, @Nullable Version version, + public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, boolean includeFrozen, ActionListener> listener) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen); client.fieldCaps(fieldRequest, wrap(response -> { client.admin().indices().getAliases(createGetAliasesRequest(response, includeFrozen), wrap(aliases -> - listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), aliases.getAliases(), - version)), + listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), aliases.getAliases())), ex -> { if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) { - listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), null, - version)); + listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), null)); } else { listener.onFailure(ex); } @@ -509,9 +502,8 @@ private GetAliasesRequest createGetAliasesRequest(FieldCapabilitiesResponse resp } public static List separateMappings(DataTypeRegistry typeRegistry, String javaRegex, String[] indexNames, - Map> fieldCaps, ImmutableOpenMap> aliases, - @Nullable Version version) { - return buildIndices(typeRegistry, indexNames, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null, version); + Map> fieldCaps, ImmutableOpenMap> aliases) { + return buildIndices(typeRegistry, indexNames, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null); } private static class Fields { @@ -526,7 +518,7 @@ private static class Fields { private static List buildIndices(DataTypeRegistry typeRegistry, String[] indexNames, String javaRegex, Map> fieldCaps, ImmutableOpenMap> aliases, Function indexNameProcessor, - BiFunction, InvalidMappedField> validityVerifier, @Nullable Version version) { + BiFunction, InvalidMappedField> validityVerifier) { if ((indexNames == null || indexNames.length == 0) && (aliases == null || aliases.isEmpty())) { return emptyList(); @@ -655,8 +647,7 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ createField(typeRegistry, fieldName, fieldCaps, indexFields.hierarchicalMapping, indexFields.flattedMapping, s -> invalidField != null ? invalidField : createField(typeRegistry, s, typeCap.getType(), emptyMap(), typeCap.isAggregatable(), - isAliasFieldType.get(), version), - version); + isAliasFieldType.get())); } } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeRegistry.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeRegistry.java index 5b90c0e35c3af..714eef3586362 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeRegistry.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeRegistry.java @@ -6,8 +6,6 @@ package org.elasticsearch.xpack.ql.type; -import org.elasticsearch.Version; - import java.util.Collection; /** @@ -22,11 +20,6 @@ public interface DataTypeRegistry { DataType fromEs(String typeName); - // version-dependent type resolution - default DataType fromEs(String typeName, Version version) { - return fromEs(typeName); - } - DataType fromJava(Object value); boolean isUnsupported(DataType type); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/VersionUtil.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/VersionUtil.java deleted file mode 100644 index ca47ccb8e249e..0000000000000 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/VersionUtil.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.ql.util; - -import org.elasticsearch.Version; - -import static org.elasticsearch.Version.V_7_11_0; - -/** - * Utility class to enable gradual feature enabling, version-dependent. - */ -public final class VersionUtil { - - private VersionUtil() {} - - /** - * Does the provided {@code version} support the unsigned_long type (PR#60050)? - */ - public static boolean isUnsignedLongSupported(Version version) { - return V_7_11_0.compareTo(version) <= 0; - } -} diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java index fe91d44550962..5de0b571233ad 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java @@ -2186,8 +2186,9 @@ private Map createTestDataForNumericValueTypes(Supplier map.put("test_long", test_long); // random BigInteger/unsigned_long - BigInteger test_unsigned_long = BigInteger.valueOf(randomValueOtherThanMany(map::containsValue, randomGenerator).longValue()) - .abs(); + BigInteger test_unsigned_long = BigInteger.valueOf(randomValueOtherThanMany(map::containsValue, randomGenerator).longValue()); + test_unsigned_long = test_unsigned_long.abs().subtract(test_unsigned_long.equals(BigInteger.ZERO) ? + BigInteger.ZERO : BigInteger.ONE); builder.field("test_unsigned_long", test_unsigned_long); map.put("test_unsigned_long", test_unsigned_long); diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java index 1da6160316e74..ffdafc1234299 100644 --- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java @@ -23,6 +23,7 @@ public class SqlVersion implements Comparable{ public final byte major; public final byte minor; public final byte revision; + public final byte build; // the build is required for id-compatibility with ES's version public static final int REVISION_MULTIPLIER = 100; public static final int MINOR_MULTIPLIER = REVISION_MULTIPLIER * REVISION_MULTIPLIER; @@ -45,17 +46,21 @@ protected SqlVersion(String version, byte... parts) { this.version = version; assert parts.length >= 3 : "Version must be initialized with all Major.Minor.Revision components"; - this.major = parts[0]; - this.minor = parts[1]; - this.revision = parts[2]; - - if ((major | minor | revision) < 0 || minor >= REVISION_MULTIPLIER || revision >= REVISION_MULTIPLIER) { - throw new InvalidParameterException("Invalid version initialisers [" + major + ", " + minor + ", " + revision + "]"); + major = parts[0]; + minor = parts[1]; + revision = parts[2]; + build = (parts.length >= 4) ? parts[3] : REVISION_MULTIPLIER - 1; + + if ((major | minor | revision | build) < 0 || + minor >= REVISION_MULTIPLIER || revision >= REVISION_MULTIPLIER || build >= REVISION_MULTIPLIER) { + throw new InvalidParameterException("Invalid version initialisers [" + major + ", " + minor + ", " + revision + ", " + + build + "]"); } id = Integer.valueOf(major) * MAJOR_MULTIPLIER + Integer.valueOf(minor) * MINOR_MULTIPLIER - + Integer.valueOf(revision) * REVISION_MULTIPLIER; + + Integer.valueOf(revision) * REVISION_MULTIPLIER + + Integer.valueOf(build); } public static SqlVersion fromString(String version) { @@ -66,10 +71,12 @@ public static SqlVersion fromString(String version) { } public static SqlVersion fromId(int id) { + byte build = (byte) (id % REVISION_MULTIPLIER); byte revision = (byte) ((id / REVISION_MULTIPLIER) % REVISION_MULTIPLIER); byte minor = (byte) ((id / MINOR_MULTIPLIER) % REVISION_MULTIPLIER); byte major = (byte) ((id / MAJOR_MULTIPLIER) % REVISION_MULTIPLIER); - return new SqlVersion(major, minor, revision); + /* the build is not reflected in the string format */ + return new SqlVersion(toString(major, minor, revision), major, minor, revision, build); } protected static byte[] from(String ver) { diff --git a/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java b/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java index ef8737a001156..382955b9ab075 100644 --- a/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java +++ b/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/proto/SqlVersionTests.java @@ -8,13 +8,18 @@ import org.elasticsearch.test.ESTestCase; +import static org.elasticsearch.xpack.sql.proto.SqlVersion.MAJOR_MULTIPLIER; +import static org.elasticsearch.xpack.sql.proto.SqlVersion.MINOR_MULTIPLIER; +import static org.elasticsearch.xpack.sql.proto.SqlVersion.REVISION_MULTIPLIER; + public class SqlVersionTests extends ESTestCase { public void test123FromString() { SqlVersion ver = SqlVersion.fromString("1.2.3"); assertEquals(1, ver.major); assertEquals(2, ver.minor); assertEquals(3, ver.revision); - assertEquals(1 * SqlVersion.MAJOR_MULTIPLIER + 2 * SqlVersion.MINOR_MULTIPLIER + 3 * SqlVersion.REVISION_MULTIPLIER, ver.id); + assertEquals(REVISION_MULTIPLIER - 1, ver.build); + assertEquals(1 * MAJOR_MULTIPLIER + 2 * MINOR_MULTIPLIER + 3 * REVISION_MULTIPLIER + REVISION_MULTIPLIER - 1, ver.id); assertEquals("1.2.3", ver.version); } @@ -23,7 +28,8 @@ public void test123AlphaFromString() { assertEquals(1, ver.major); assertEquals(2, ver.minor); assertEquals(3, ver.revision); - assertEquals(1 * SqlVersion.MAJOR_MULTIPLIER + 2 * SqlVersion.MINOR_MULTIPLIER + 3 * SqlVersion.REVISION_MULTIPLIER, ver.id); + assertEquals(REVISION_MULTIPLIER - 1, ver.build); + assertEquals(1 * MAJOR_MULTIPLIER + 2 * MINOR_MULTIPLIER + 3 * REVISION_MULTIPLIER + REVISION_MULTIPLIER - 1, ver.id); assertEquals("1.2.3-Alpha", ver.version); } @@ -32,7 +38,8 @@ public void test123AlphaSnapshotFromString() { assertEquals(1, ver.major); assertEquals(2, ver.minor); assertEquals(3, ver.revision); - assertEquals(1 * SqlVersion.MAJOR_MULTIPLIER + 2 * SqlVersion.MINOR_MULTIPLIER + 3 * SqlVersion.REVISION_MULTIPLIER, ver.id); + assertEquals(REVISION_MULTIPLIER - 1, ver.build); + assertEquals(1 * MAJOR_MULTIPLIER + 2 * MINOR_MULTIPLIER + 3 * REVISION_MULTIPLIER + REVISION_MULTIPLIER - 1, ver.id); assertEquals("1.2.3-Alpha-SNAPSHOT", ver.version); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index 2cf91b6687f44..38e1cb5812e4a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -115,6 +115,7 @@ protected Iterable.Batch> batches() { new ResolveAggsInHaving(), new ResolveAggsInOrderBy() //new ImplicitCasting() + //new VersionCompatibility() ); Batch finish = new Batch("Finish Analysis", new PruneSubqueryAliases(), @@ -496,6 +497,37 @@ private LogicalPlan dedupRight(LogicalPlan left, LogicalPlan right) { } } + /* + private class VersionCompatibility extends BaseAnalyzeRule { + @Override + protected LogicalPlan doRule(LogicalPlan plan) { + if (configuration.version() == null) { + return plan; + } + if (plan instanceof Project) { + Project project = (Project) plan; + return new Project(project.source(), project.child(), + project.projections().stream().map(this::typeSupported).collect(toList())); + } + return plan; + } + + private NamedExpression typeSupported(NamedExpression ne) { + if (ne.resolved() && isTypeVersionGated(ne.dataType(), configuration.version())) { + return new UnresolvedAttribute(ne.source(), null, ne.toAttribute().qualifier(), + "Cannot use field [" + ne.name() + "] with type [" + ne.dataType() + "] unsupported in version [" + + configuration.version() + "]"); + } + return ne; + } + + @Override + protected boolean skipResolved() { + return false; + } + } + */ + // Allow ordinal positioning in order/sort by (quite useful when dealing with aggs) // Note that ordering starts at 1 private static class ResolveOrdinalInOrderByAndGroupBy extends BaseAnalyzeRule { @@ -1220,7 +1252,6 @@ private Expression implicitCast(Expression e) { } } - public static class PruneSubqueryAliases extends AnalyzeRule { @Override @@ -1300,6 +1331,21 @@ protected boolean skipResolved() { } } + private class VersionCompatibility2 extends Rule { + + @Override + public LogicalPlan apply(LogicalPlan logicalPlan) { + List projects = logicalPlan.collectFirstChildren(x -> x instanceof Project); + assert projects.size() > 0; + projects.get(0); + return logicalPlan.transformDown(this::rule, typeToken()); + } + + @Override + protected LogicalPlan rule(Project project) { + return null; + } + } abstract static class AnalyzeRule extends Rule { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index 4d58f0db1f4b3..b6937a943413e 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -53,6 +53,7 @@ import org.elasticsearch.xpack.sql.plan.logical.LocalRelation; import org.elasticsearch.xpack.sql.plan.logical.Pivot; import org.elasticsearch.xpack.sql.plan.logical.command.Command; +import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.stats.FeatureMetric; import org.elasticsearch.xpack.sql.stats.Metrics; import org.elasticsearch.xpack.sql.type.SqlDataTypes; @@ -71,6 +72,7 @@ import static java.util.stream.Collectors.toMap; import static org.elasticsearch.xpack.ql.common.Failure.fail; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; +import static org.elasticsearch.xpack.sql.session.Compatibility.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.COMMAND; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.GROUPBY; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.HAVING; @@ -87,9 +89,11 @@ */ public final class Verifier { private final Metrics metrics; + private final SqlConfiguration config; - public Verifier(Metrics metrics) { + public Verifier(Metrics metrics, SqlConfiguration config) { this.metrics = metrics; + this.config = config; } public Map, String> verifyFailures(LogicalPlan plan) { @@ -231,6 +235,18 @@ Collection verify(LogicalPlan plan) { failures.addAll(localFailures); }); + + if (config.version() != null) { + List projects = plan.collectFirstChildren(x -> x instanceof Project); + if (projects.size() > 0) { // not selecting just literals + ((Project) projects.get(0)).projections().forEach(e -> { + if (e.resolved() && isTypeSupportedInVersion(e.dataType(), config.version()) == false) { + failures.add(fail(e, "Cannot use field [" + e.name() + "] with type [" + e.dataType() + "] unsupported " + + "in version [" + config.version() + "]")); + } + }); + } + } } // gather metrics diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java index 8aeea3132e8f1..fb34c53e64711 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java @@ -41,10 +41,9 @@ public class PlanExecutor { private final IndexResolver indexResolver; private final PreAnalyzer preAnalyzer; - private final Verifier verifier; private final Optimizer optimizer; private final Planner planner; - + private final Metrics metrics; public PlanExecutor(Client client, IndexResolver indexResolver, NamedWriteableRegistry writeableRegistry) { @@ -53,17 +52,20 @@ public PlanExecutor(Client client, IndexResolver indexResolver, NamedWriteableRe this.indexResolver = indexResolver; this.functionRegistry = new SqlFunctionRegistry(); - + this.metrics = new Metrics(); this.preAnalyzer = new PreAnalyzer(); - this.verifier = new Verifier(metrics); this.optimizer = new Optimizer(); this.planner = new Planner(); } private SqlSession newSession(SqlConfiguration cfg) { - return new SqlSession(cfg, client, functionRegistry, indexResolver, preAnalyzer, verifier, optimizer, planner, this); + // TODO: SqlSession exposes no functionality that doesn't involve the Analyzer; would it make sense to: + // 1) instantiate an Analyzer in its c'tor? and + // 2) instantiate the Verifier in Analyzer's c'tor and add a getter for it? + return new SqlSession(cfg, client, functionRegistry, indexResolver, preAnalyzer, new Verifier(metrics, cfg), optimizer, planner, + this); } public void searchSource(SqlConfiguration cfg, String sql, List params, @@ -116,7 +118,7 @@ public void nextPage(SqlConfiguration cfg, Cursor cursor, ActionListener l public void cleanCursor(SqlConfiguration cfg, Cursor cursor, ActionListener listener) { cursor.clear(cfg, client, listener); } - + public Metrics metrics() { return this.metrics; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java index c3631b0a75603..d510e2c4926f8 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.sql.plan.logical.command; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.FieldAttribute; @@ -27,7 +26,6 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyList; -import static org.elasticsearch.xpack.sql.proto.Mode.isDriver; public class ShowColumns extends Command { @@ -68,8 +66,7 @@ public void execute(SqlSession session, ActionListener listener) { String regex = pattern != null ? pattern.asJavaRegex() : null; boolean withFrozen = includeFrozen || session.configuration().includeFrozen(); - Version version = isDriver(session.configuration().mode()) ? Version.fromId(session.configuration().version().id) : null; - session.indexResolver().resolveAsMergedMapping(idx, regex, withFrozen, version, ActionListener.wrap( + session.indexResolver().resolveAsMergedMapping(idx, regex, withFrozen, ActionListener.wrap( indexResult -> { List> rows = emptyList(); if (indexResult.isValid()) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java index 17019b00485b8..599991cbd13b0 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.sql.plan.logical.command.sys; import org.apache.lucene.util.Counter; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.Strings; import org.elasticsearch.xpack.ql.expression.Attribute; @@ -19,6 +18,7 @@ import org.elasticsearch.xpack.ql.util.StringUtils; import org.elasticsearch.xpack.sql.plan.logical.command.Command; import org.elasticsearch.xpack.sql.proto.Mode; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.session.Cursor.Page; import org.elasticsearch.xpack.sql.session.ListCursor; import org.elasticsearch.xpack.sql.session.Rows; @@ -39,6 +39,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; import static org.elasticsearch.xpack.ql.type.DataTypes.isString; import static org.elasticsearch.xpack.sql.proto.Mode.isDriver; +import static org.elasticsearch.xpack.sql.session.Compatibility.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.displaySize; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDataType; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDateTimeSub; @@ -128,27 +129,28 @@ public void execute(SqlSession session, ActionListener listener) { Pattern columnMatcher = columnPattern != null ? Pattern.compile(columnPattern.asJavaRegex()) : null; boolean includeFrozen = session.configuration().includeFrozen(); - Version version = isDriver(session.configuration().mode()) ? Version.fromId(session.configuration().version().id) : null; // special case for '%' (translated to *) if ("*".equals(idx)) { - session.indexResolver().resolveAsSeparateMappings(idx, regex, includeFrozen, version, + session.indexResolver().resolveAsSeparateMappings(idx, regex, includeFrozen, ActionListener.wrap(esIndices -> { List> rows = new ArrayList<>(); for (EsIndex esIndex : esIndices) { - fillInRows(cluster, esIndex.name(), esIndex.mapping(), null, rows, columnMatcher, mode); + fillInRows(cluster, esIndex.name(), esIndex.mapping(), null, rows, columnMatcher, mode, + session.configuration().version()); } listener.onResponse(ListCursor.of(Rows.schema(output), rows, session.configuration().pageSize())); }, listener::onFailure)); } // otherwise use a merged mapping else { - session.indexResolver().resolveAsMergedMapping(idx, regex, includeFrozen, version, + session.indexResolver().resolveAsMergedMapping(idx, regex, includeFrozen, ActionListener.wrap(r -> { List> rows = new ArrayList<>(); // populate the data only when a target is found if (r.isValid()) { EsIndex esIndex = r.get(); - fillInRows(cluster, indexName, esIndex.mapping(), null, rows, columnMatcher, mode); + fillInRows(cluster, indexName, esIndex.mapping(), null, rows, columnMatcher, mode, + session.configuration().version()); } listener.onResponse(ListCursor.of(Rows.schema(output), rows, session.configuration().pageSize())); }, listener::onFailure)); @@ -156,12 +158,12 @@ public void execute(SqlSession session, ActionListener listener) { } static void fillInRows(String clusterName, String indexName, Map mapping, String prefix, List> rows, - Pattern columnMatcher, Mode mode) { - fillInRows(clusterName, indexName, mapping, prefix, rows, columnMatcher, Counter.newCounter(), mode); + Pattern columnMatcher, Mode mode, SqlVersion version) { + fillInRows(clusterName, indexName, mapping, prefix, rows, columnMatcher, Counter.newCounter(), mode, version); } private static void fillInRows(String clusterName, String indexName, Map mapping, String prefix, List> rows, - Pattern columnMatcher, Counter position, Mode mode) { + Pattern columnMatcher, Counter position, Mode mode, SqlVersion version) { boolean isOdbcClient = mode == Mode.ODBC; for (Map.Entry entry : mapping.entrySet()) { position.addAndGet(1); // JDBC is 1-based so we start with 1 here @@ -172,7 +174,7 @@ private static void fillInRows(String clusterName, String indexName, Map output() { @Override public final void execute(SqlSession session, ActionListener listener) { - Stream values = SqlDataTypes.types().stream(); + Stream values = SqlDataTypes.types().stream() + .filter(t -> Mode.isDriver(session.configuration().mode()) == false || + isTypeSupportedInVersion(t, session.configuration().version())); if (type.intValue() != 0) { values = values.filter(t -> type.equals(sqlType(t).getVendorTypeNumber())); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/Compatibility.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/Compatibility.java new file mode 100644 index 0000000000000..219a3a221945b --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/Compatibility.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.session; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.sql.proto.SqlVersion; + +import static org.elasticsearch.Version.V_7_11_0; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; + +public final class Compatibility { + + public static final SqlVersion INTRODUCING_UNSIGNED_LONG = SqlVersion.fromId(V_7_11_0.id); + + private Compatibility() {} + + /** + * Is the provided {@code dataType} being supported in the provided {@code version}? + */ + public static boolean isTypeSupportedInVersion(DataType dataType, @Nullable SqlVersion version) { + if (version != null) { + if (dataType == UNSIGNED_LONG) { + return supportsUnsignedLong(version); + } + } + return true; + } + /** + * Does the provided {@code version} support the unsigned_long type (PR#60050)? + */ + public static boolean supportsUnsignedLong(SqlVersion version) { + return INTRODUCING_UNSIGNED_LONG.compareTo(version) <= 0; + } +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java index eaa6feb6c91f6..54ff4d00bcfaa 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java @@ -21,6 +21,7 @@ public class SqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu private final TimeValue pageTimeout; private final Mode mode; private final String clientId; + @Nullable private final SqlVersion version; private final boolean multiValueFieldLeniency; private final boolean includeFrozenIndices; @@ -70,10 +71,6 @@ public String clientId() { return clientId; } - public SqlVersion version() { - return version; - } - public boolean multiValueFieldLeniency() { return multiValueFieldLeniency; } @@ -81,4 +78,8 @@ public boolean multiValueFieldLeniency() { public boolean includeFrozen() { return includeFrozenIndices; } + + public SqlVersion version() { + return version; + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java index a1c9e96d4a358..bca0c9ec95592 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.sql.session; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; @@ -33,7 +32,6 @@ import java.util.function.Function; import static org.elasticsearch.action.ActionListener.wrap; -import static org.elasticsearch.xpack.sql.proto.Mode.isDriver; public class SqlSession implements Session { @@ -143,7 +141,6 @@ private void preAnalyze(LogicalPlan parsed, Function act boolean includeFrozen = configuration.includeFrozen() || tableInfo.isFrozen(); indexResolver.resolveAsMergedMapping(table.index(), null, includeFrozen, - isDriver(configuration.mode()) ? Version.fromId(configuration.version().id) : null, wrap(indexResult -> listener.onResponse(action.apply(indexResult)), listener::onFailure)); } else { try { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeRegistry.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeRegistry.java index fecf0380ff00f..12554a547bb9b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeRegistry.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypeRegistry.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.sql.type; -import org.elasticsearch.Version; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypeRegistry; import org.elasticsearch.xpack.ql.type.DataTypes; @@ -29,11 +28,6 @@ public DataType fromEs(String typeName) { return SqlDataTypes.fromEs(typeName); } - @Override - public DataType fromEs(String typeName, Version version) { - return SqlDataTypes.fromEs(typeName, version); - } - @Override public DataType fromJava(Object value) { return SqlDataTypes.fromJava(value); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java index e7a065cac88fd..cc7eda2020a00 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java @@ -6,8 +6,6 @@ package org.elasticsearch.xpack.sql.type; -import org.elasticsearch.Version; -import org.elasticsearch.common.Nullable; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.sql.expression.literal.geo.GeoShape; @@ -48,7 +46,6 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; import static org.elasticsearch.xpack.ql.util.CollectionUtils.mapSize; -import static org.elasticsearch.xpack.ql.util.VersionUtil.isUnsignedLongSupported; public class SqlDataTypes { @@ -209,16 +206,6 @@ public static DataType fromEs(String name) { return type != null ? type : UNSUPPORTED; } - public static DataType fromEs(String name, @Nullable Version version) { - DataType type = fromEs(name); - if (version != null) { - if (type == UNSIGNED_LONG && isUnsignedLongSupported(version) == false) { - return UNSUPPORTED; - } - } - return type; - } - public static DataType fromJava(Object value) { DataType type = DataTypes.fromJava(value); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java index 489309134c844..5554666ee2211 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.sql.proto.Mode; import org.elasticsearch.xpack.sql.proto.Protocol; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.type.SqlDataTypes; import org.elasticsearch.xpack.sql.util.DateUtils; @@ -57,13 +58,28 @@ public static SqlConfiguration randomConfiguration() { public static SqlConfiguration randomConfiguration(ZoneId providedZoneId) { return new SqlConfiguration(providedZoneId, + randomIntBetween(0, 1000), + new TimeValue(randomNonNegativeLong()), + new TimeValue(randomNonNegativeLong()), + null, + randomFrom(Mode.values()), + randomAlphaOfLength(10), + null, + randomAlphaOfLength(10), + randomAlphaOfLength(10), + false, + randomBoolean()); + } + + public static SqlConfiguration randomConfiguration(SqlVersion version) { + return new SqlConfiguration(randomZone(), randomIntBetween(0, 1000), new TimeValue(randomNonNegativeLong()), new TimeValue(randomNonNegativeLong()), null, randomFrom(Mode.values()), randomAlphaOfLength(10), - null, + version, randomAlphaOfLength(10), randomAlphaOfLength(10), false, diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java index f93417c6ff95b..58c708eca6deb 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java @@ -24,6 +24,8 @@ import org.elasticsearch.xpack.sql.SqlTestUtils; import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry; import org.elasticsearch.xpack.sql.parser.SqlParser; +import org.elasticsearch.xpack.sql.proto.SqlVersion; +import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.stats.Metrics; import java.util.List; @@ -31,8 +33,11 @@ import java.util.stream.Collectors; import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; +import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.session.Compatibility.INTRODUCING_UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.contains; @@ -53,7 +58,7 @@ public class FieldAttributeTests extends ESTestCase { public FieldAttributeTests() { parser = new SqlParser(); functionRegistry = new SqlFunctionRegistry(); - verifier = new Verifier(new Metrics()); + verifier = new Verifier(new Metrics(), SqlTestUtils.TEST_CFG); Map mapping = loadMapping("mapping-multi-field-variation.json"); @@ -278,6 +283,69 @@ public void testGroupByAmbiguity() { ex.getMessage()); } + public void testUnsignedLongVersionCompatibility() { + Map mapping = TypesTests.loadMapping("mapping-numeric.json"); + EsIndex index = new EsIndex("test", mapping); + getIndexResult = IndexResolution.valid(index); + + String query = "SELECT unsigned_long FROM test"; + String queryWithLiteral = "SELECT 18446744073709551615 as unsigned_long"; + String queryWithAlias = "SELECT unsigned_long AS unsigned_long FROM test"; + String queryWithArithmetic = "SELECT unsigned_long + 1 AS unsigned_long FROM test"; + String queryWithCast = "SELECT long + 1::unsigned_long AS unsigned_long FROM test"; + + SqlVersion introducingUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id); + SqlVersion preIntroducingVersion = SqlVersion.fromId(introducingUnsignedLong.id - SqlVersion.MINOR_MULTIPLIER); + SqlVersion postIntroducingVersion = SqlVersion.fromId(introducingUnsignedLong.id + SqlVersion.MINOR_MULTIPLIER); + + + for (String sql : List.of(query, queryWithLiteral, queryWithAlias, queryWithArithmetic, queryWithCast)) { + SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preIntroducingVersion); + analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics(), sqlConfig)); + VerificationException ex = expectThrows(VerificationException.class, () -> plan(sql)); + assertEquals( + "Found 1 problem\nline 1:8: Cannot use field [unsigned_long] with type [UNSIGNED_LONG] unsupported in version [" + + preIntroducingVersion + "]", + ex.getMessage()); + + for (SqlVersion v : List.of(introducingUnsignedLong, postIntroducingVersion)) { + analyzer = new Analyzer(SqlTestUtils.randomConfiguration(v), functionRegistry, getIndexResult, + verifier); + LogicalPlan plan = plan(sql); + assertThat(plan, instanceOf(Project.class)); + Project p = (Project) plan; + List projections = p.projections(); + assertThat(projections, hasSize(1)); + Attribute attribute = projections.get(0).toAttribute(); + assertThat(attribute.dataType(), is(UNSIGNED_LONG)); + assertThat(attribute.name(), is("unsigned_long")); + } + } + } + + public void testNonProjectedUnsignedLongVersionCompatibility() { + Map mapping = TypesTests.loadMapping("mapping-numeric.json"); + EsIndex index = new EsIndex("test", mapping); + getIndexResult = IndexResolution.valid(index); + SqlVersion preIntroducingVersion = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); + SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preIntroducingVersion); + analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics(), sqlConfig)); + + String query = "SELECT unsigned_long = 1, unsigned_long::double FROM test"; + String queryWithSubquery = "SELECT l = 1, SQRT(ul) FROM " + + "(SELECT unsigned_long AS ul, long AS l FROM test WHERE ul > 10) WHERE l < 100 "; + + for (String sql : List.of(query, queryWithSubquery)) { + LogicalPlan plan = plan(sql); + assertThat(plan, instanceOf(Project.class)); + Project p = (Project) plan; + List projections = p.projections(); + assertThat(projections, hasSize(2)); + assertEquals(projections.get(0).dataType(), BOOLEAN); + assertEquals(projections.get(1).dataType(), DOUBLE); + } + } + public void testFunctionOverNonExistingFieldAsArgumentAndSameAlias() throws Exception { Map mapping = TypesTests.loadMapping("mapping-basic.json"); EsIndex index = new EsIndex("test", mapping); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index def83dc19cca8..62ed673989a39 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -48,7 +48,8 @@ private String error(String sql) { } private String error(IndexResolution getIndexResult, String sql) { - Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics())); + Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics(), + SqlTestUtils.TEST_CFG)); VerificationException e = expectThrows(VerificationException.class, () -> analyzer.analyze(parser.createStatement(sql), true)); String message = e.getMessage(); assertTrue(message.startsWith("Found ")); @@ -68,7 +69,8 @@ private EsIndex getTestEsIndex() { } private LogicalPlan accept(IndexResolution resolution, String sql) { - Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), resolution, new Verifier(new Metrics())); + Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), resolution, + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); return analyzer.analyze(parser.createStatement(sql), true); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 164748186c85f..4da5d0154ffe7 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -5,9 +5,7 @@ */ package org.elasticsearch.xpack.sql.analysis.index; -import org.elasticsearch.Version; import org.elasticsearch.action.fieldcaps.FieldCapabilities; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; @@ -19,7 +17,6 @@ import org.elasticsearch.xpack.sql.type.SqlDataTypeRegistry; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -28,14 +25,12 @@ import java.util.stream.Stream; import static java.util.Collections.singletonMap; -import static org.elasticsearch.common.collect.Tuple.tuple; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; -import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; @@ -232,38 +227,6 @@ public void testRandomMappingFieldTypeMappedAsUnsupported() throws Exception { assertEquals(UNSUPPORTED, esIndex.mapping().get("another_field").getProperties().get("_foo").getDataType()); } - public void testMergeCompatibleMappingVersionAware() throws Exception { - Map basicMapping = loadMapping("mapping-basic.json", true); - Map numericMapping = loadMapping("mapping-numeric.json", true); - - assertNotEquals(basicMapping, numericMapping); - - for (Tuple t : Arrays.asList(tuple(Version.V_7_10_0, UNSUPPORTED), tuple(Version.V_7_11_0, UNSIGNED_LONG))) { - IndexResolution resolution = merge(t.v1(), new EsIndex("basic", basicMapping), new EsIndex("numeric", numericMapping)); - - assertTrue(resolution.isValid()); - assertEquals(basicMapping.size() + numericMapping.size(), resolution.get().mapping().size()); - assertEquals(t.v2(), resolution.get().mapping().get("unsigned_long").getDataType()); - } - } - - public void testSeparateMappingVersionAware() throws Exception { - Map numericMapping = loadMapping("mapping-numeric.json", true); - Map sameMapping = loadMapping("mapping-numeric.json", true); - - assertEquals(sameMapping, numericMapping); - - for (Tuple t : Arrays.asList(tuple(Version.V_7_10_0, UNSUPPORTED), tuple(Version.V_7_11_0, UNSIGNED_LONG))) { - List indices = separate(t.v1(), new EsIndex("numeric", numericMapping), new EsIndex("same", sameMapping)); - - assertEquals(2, indices.size()); - assertEquals(numericMapping.size(), indices.get(0).mapping().size()); - assertEquals(t.v2(), indices.get(0).mapping().get("unsigned_long").getDataType()); - assertEquals(sameMapping.size(), indices.get(1).mapping().size()); - assertEquals(t.v2(), indices.get(1).mapping().get("unsigned_long").getDataType()); - } - } - public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception { Map> fieldCaps = new HashMap<>(); @@ -361,20 +324,12 @@ public void testIndexWithNoMapping() { } public static IndexResolution merge(EsIndex... indices) { - return merge(null, indices); - } - - public static IndexResolution merge(Version version, EsIndex... indices) { - return mergedMappings("*", Stream.of(indices).map(EsIndex::name).toArray(String[]::new), fromMappings(indices), version); + return mergedMappings("*", Stream.of(indices).map(EsIndex::name).toArray(String[]::new), fromMappings(indices)); } public static List separate(EsIndex... indices) { - return separate(null, indices); - } - - public static List separate(Version version, EsIndex... indices) { return separateMappings(null, Stream.of(indices).map(EsIndex::name).toArray(String[]::new), - fromMappings(indices), version); + fromMappings(indices)); } public static Map> fromMappings(EsIndex... indices) { @@ -479,16 +434,11 @@ private void addFieldCaps(Map> fieldCaps, private static IndexResolution mergedMappings(String indexPattern, String[] indexNames, Map> fieldCaps) { - return mergedMappings(indexPattern, indexNames, fieldCaps, null); - } - - private static IndexResolution mergedMappings(String indexPattern, String[] indexNames, - Map> fieldCaps, Version version) { - return IndexResolver.mergedMappings(SqlDataTypeRegistry.INSTANCE, indexPattern, indexNames, fieldCaps, version); + return IndexResolver.mergedMappings(SqlDataTypeRegistry.INSTANCE, indexPattern, indexNames, fieldCaps); } private static List separateMappings(String javaRegex, String[] indexNames, - Map> fieldCaps, Version version) { - return IndexResolver.separateMappings(SqlDataTypeRegistry.INSTANCE, javaRegex, indexNames, fieldCaps, null, version); + Map> fieldCaps) { + return IndexResolver.separateMappings(SqlDataTypeRegistry.INSTANCE, javaRegex, indexNames, fieldCaps, null); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java index 0d5b2d87ef9d5..49f10064bb5cb 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java @@ -29,14 +29,15 @@ public void testDatabaseFunctionOutput() { String clusterName = randomAlphaOfLengthBetween(1, 15); SqlParser parser = new SqlParser(); EsIndex test = new EsIndex("test", SqlTypesTests.loadMapping("mapping-basic.json", true)); + SqlConfiguration sqlConfig = new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, + Protocol.PAGE_TIMEOUT, null, + randomFrom(Mode.values()), randomAlphaOfLength(10), + null, null, clusterName, randomBoolean(), randomBoolean()); Analyzer analyzer = new Analyzer( - new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, - Protocol.PAGE_TIMEOUT, null, - randomFrom(Mode.values()), randomAlphaOfLength(10), - null, null, clusterName, randomBoolean(), randomBoolean()), + sqlConfig, new SqlFunctionRegistry(), IndexResolution.valid(test), - new Verifier(new Metrics()) + new Verifier(new Metrics(), sqlConfig) ); Project result = (Project) analyzer.analyze(parser.createStatement("SELECT DATABASE()"), true); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java index 7380f0057744e..6f810de688d2b 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java @@ -28,15 +28,16 @@ public class UserFunctionTests extends ESTestCase { public void testNoUsernameFunctionOutput() { SqlParser parser = new SqlParser(); EsIndex test = new EsIndex("test", SqlTypesTests.loadMapping("mapping-basic.json", true)); + SqlConfiguration sqlConfig = new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, + Protocol.PAGE_TIMEOUT, null, + randomFrom(Mode.values()), randomAlphaOfLength(10), + null, null, randomAlphaOfLengthBetween(1, 15), + randomBoolean(), randomBoolean()); Analyzer analyzer = new Analyzer( - new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, - Protocol.PAGE_TIMEOUT, null, - randomFrom(Mode.values()), randomAlphaOfLength(10), - null, null, randomAlphaOfLengthBetween(1, 15), - randomBoolean(), randomBoolean()), + sqlConfig, new SqlFunctionRegistry(), IndexResolution.valid(test), - new Verifier(new Metrics()) + new Verifier(new Metrics(), sqlConfig) ); Project result = (Project) analyzer.analyze(parser.createStatement("SELECT USER()"), true); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java index ab6a182be36d9..d420d93d80daf 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java @@ -74,14 +74,14 @@ public void testNanoPrecision() { assertEquals(123_456_780, CurrentDateTime.nanoPrecision(zdt, literal(8)).getNano()); assertEquals(123_456_789, CurrentDateTime.nanoPrecision(zdt, literal(9)).getNano()); } - + public void testDefaultPrecision() { Configuration configuration = SqlTestUtils.randomConfiguration(); // null precision means default precision CurrentDateTime cdt = new CurrentDateTime(EMPTY, null, configuration); ZonedDateTime now = configuration.now(); assertEquals(now.get(ChronoField.MILLI_OF_SECOND), ((ZonedDateTime) cdt.fold()).get(ChronoField.MILLI_OF_SECOND)); - + ZonedDateTime zdt = ZonedDateTime.parse("2019-02-26T12:34:56.123456789Z"); assertEquals(123_000_000, CurrentDateTime.nanoPrecision(zdt, null).getNano()); } @@ -91,7 +91,8 @@ public void testInvalidPrecision() { IndexResolution indexResolution = IndexResolution.valid(new EsIndex("test", SqlTypesTests.loadMapping("mapping-multi-field-with-nested.json"))); - Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, new Verifier(new Metrics())); + Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); ParsingException e = expectThrows(ParsingException.class, () -> analyzer.analyze(parser.createStatement("SELECT CURRENT_TIMESTAMP(100000000000000)"), true)); assertEquals("line 1:27: invalid precision; [100000000000000] out of [integer] range", e.getMessage()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java index b1fc816974712..61e638ba85ee1 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java @@ -92,7 +92,8 @@ public void testInvalidPrecision() { IndexResolution indexResolution = IndexResolution.valid(new EsIndex("test", SqlTypesTests.loadMapping("mapping-multi-field-with-nested.json"))); - Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, new Verifier(new Metrics())); + Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); ParsingException e = expectThrows(ParsingException.class, () -> analyzer.analyze(parser.createStatement("SELECT CURRENT_TIME(100000000000000)"), true)); assertEquals("line 1:22: invalid precision; [100000000000000] out of [integer] range", e.getMessage()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java index 4a54a1e48b78b..6e7b5c59b930b 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java @@ -36,7 +36,8 @@ public OptimizerRunTests() { EsIndex test = new EsIndex("test", mapping); getIndexResult = IndexResolution.valid(test); - analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, new Verifier(new Metrics())); + analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); optimizer = new Optimizer(); } @@ -48,4 +49,4 @@ public void testWhereClause() { LogicalPlan p = plan("SELECT some.string l FROM test WHERE int IS NOT NULL AND int < 10005 ORDER BY int"); assertNotNull(p); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index d707864c231e3..2c4d3b1115d55 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -32,15 +32,19 @@ import java.sql.Types; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.elasticsearch.action.ActionListener.wrap; import static org.elasticsearch.xpack.ql.TestUtils.UTC; +import static org.elasticsearch.xpack.sql.session.Compatibility.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.session.Compatibility.supportsUnsignedLong; import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; @@ -59,7 +63,7 @@ public class SysColumnsTests extends ESTestCase { public void testSysColumns() { List> rows = new ArrayList<>(); SysColumns.fillInRows("test", "index", loadMapping("mapping-multi-field-variation.json", true), null, rows, null, - randomValueOtherThanMany(Mode::isDriver, () -> randomFrom(Mode.values()))); + randomValueOtherThanMany(Mode::isDriver, () -> randomFrom(Mode.values())), SqlVersion.fromId(Version.CURRENT.id)); // nested fields are ignored assertEquals(FIELD_COUNT, rows.size()); assertEquals(24, rows.get(0).size()); @@ -152,298 +156,169 @@ public void testSysColumns() { assertEquals(Integer.MAX_VALUE, bufferLength(row)); } - public void testSysColumnsInOdbcMode() { + public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { + boolean unsignedLongSupported = version == null || supportsUnsignedLong(version); + Class typeClass = mode == Mode.ODBC ? Short.class : Integer.class; List> rows = new ArrayList<>(); - SysColumns.fillInRows("test", "index", loadMapping("mapping-multi-field-variation.json", true), null, rows, null, - Mode.ODBC); - assertEquals(FIELD_COUNT, rows.size()); + SysColumns.fillInRows("test", "index", loadMapping("mapping-multi-field-variation.json", true), null, rows, null, mode, version); + assertEquals(FIELD_COUNT - (unsignedLongSupported ? 0 : 1), rows.size()); assertEquals(24, rows.get(0).size()); int index = 0; List row = rows.get(index ++); assertEquals("bool", name(row)); - assertEquals((short) Types.BOOLEAN, sqlType(row)); + assertEquals(Types.BOOLEAN, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); assertEquals(1, bufferLength(row)); row = rows.get(index ++); assertEquals("int", name(row)); - assertEquals((short) Types.INTEGER, sqlType(row)); - assertEquals(Short.class, radix(row).getClass()); + assertEquals(Types.INTEGER, typeClass.cast(sqlType(row)).intValue()); + assertEquals(typeClass, radix(row).getClass()); assertEquals(4, bufferLength(row)); assertNull(decimalPrecision(row)); - assertEquals(Short.class, nullable(row).getClass()); - assertEquals(Short.class, sqlDataType(row).getClass()); - assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(index ++); - assertEquals("unsigned_long", name(row)); - assertEquals((short) Types.BIGINT, sqlType(row)); - assertEquals(Short.class, radix(row).getClass()); - assertEquals(Long.BYTES, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Short.class, nullable(row).getClass()); - assertEquals(Short.class, sqlDataType(row).getClass()); - assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); + + if (unsignedLongSupported) { + row = rows.get(index++); + assertEquals("unsigned_long", name(row)); + assertEquals(Types.BIGINT, typeClass.cast(sqlType(row)).intValue()); + assertEquals(typeClass, radix(row).getClass()); + assertEquals(Long.BYTES, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); + } row = rows.get(index ++); assertEquals("text", name(row)); - assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); assertNull(decimalPrecision(row)); - assertEquals(Short.class, nullable(row).getClass()); - assertEquals(Short.class, sqlDataType(row).getClass()); - assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); row = rows.get(index ++); assertEquals("keyword", name(row)); - assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); assertNull(decimalPrecision(row)); - assertEquals(Short.class, nullable(row).getClass()); - assertEquals(Short.class, sqlDataType(row).getClass()); - assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); row = rows.get(index ++); assertEquals("date", name(row)); - assertEquals((short) Types.TIMESTAMP, sqlType(row)); + assertEquals(Types.TIMESTAMP, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); assertEquals(29, precision(row)); assertEquals(8, bufferLength(row)); assertNull(decimalPrecision(row)); - assertEquals(Short.class, nullable(row).getClass()); - assertEquals(Short.class, sqlDataType(row).getClass()); - assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); row = rows.get(index ++); assertEquals("some.dotted.field", name(row)); - assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); assertNull(decimalPrecision(row)); - assertEquals(Short.class, nullable(row).getClass()); - assertEquals(Short.class, sqlDataType(row).getClass()); - assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); row = rows.get(index ++); assertEquals("some.string", name(row)); - assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); assertNull(decimalPrecision(row)); - assertEquals(Short.class, nullable(row).getClass()); - assertEquals(Short.class, sqlDataType(row).getClass()); - assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); row = rows.get(index ++); assertEquals("some.string.normalized", name(row)); - assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); assertNull(decimalPrecision(row)); - assertEquals(Short.class, nullable(row).getClass()); - assertEquals(Short.class, sqlDataType(row).getClass()); - assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); row = rows.get(index ++); assertEquals("some.string.typical", name(row)); - assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); assertNull(decimalPrecision(row)); - assertEquals(Short.class, nullable(row).getClass()); - assertEquals(Short.class, sqlDataType(row).getClass()); - assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); row = rows.get(index ++); assertEquals("some.ambiguous", name(row)); - assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); assertNull(decimalPrecision(row)); - assertEquals(Short.class, nullable(row).getClass()); - assertEquals(Short.class, sqlDataType(row).getClass()); - assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); row = rows.get(index ++); assertEquals("some.ambiguous.one", name(row)); - assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); assertNull(decimalPrecision(row)); - assertEquals(Short.class, nullable(row).getClass()); - assertEquals(Short.class, sqlDataType(row).getClass()); - assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); row = rows.get(index ++); assertEquals("some.ambiguous.two", name(row)); - assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); assertNull(decimalPrecision(row)); - assertEquals(Short.class, nullable(row).getClass()); - assertEquals(Short.class, sqlDataType(row).getClass()); - assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); row = rows.get(index ++); assertEquals("some.ambiguous.normalized", name(row)); - assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); assertNull(decimalPrecision(row)); - assertEquals(Short.class, nullable(row).getClass()); - assertEquals(Short.class, sqlDataType(row).getClass()); - assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); } - public void testSysColumnsInJdbcMode() { - List> rows = new ArrayList<>(); - SysColumns.fillInRows("test", "index", loadMapping("mapping-multi-field-variation.json", true), null, rows, null, - Mode.JDBC); - assertEquals(FIELD_COUNT, rows.size()); - assertEquals(24, rows.get(0).size()); - - int index = 0; - - List row = rows.get(index ++); - assertEquals("bool", name(row)); - assertEquals(Types.BOOLEAN, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(1, bufferLength(row)); - - row = rows.get(index ++); - assertEquals("int", name(row)); - assertEquals(Types.INTEGER, sqlType(row)); - assertEquals(Integer.class, radix(row).getClass()); - assertEquals(4, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Integer.class, nullable(row).getClass()); - assertEquals(Integer.class, sqlDataType(row).getClass()); - assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(index ++); - assertEquals("unsigned_long", name(row)); - assertEquals(Types.BIGINT, sqlType(row)); - assertEquals(Integer.class, radix(row).getClass()); - assertEquals(Long.BYTES, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Integer.class, nullable(row).getClass()); - assertEquals(Integer.class, sqlDataType(row).getClass()); - assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(index ++); - assertEquals("text", name(row)); - assertEquals(Types.VARCHAR, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(Integer.MAX_VALUE, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Integer.class, nullable(row).getClass()); - assertEquals(Integer.class, sqlDataType(row).getClass()); - assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(index ++); - assertEquals("keyword", name(row)); - assertEquals(Types.VARCHAR, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(Integer.MAX_VALUE, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Integer.class, nullable(row).getClass()); - assertEquals(Integer.class, sqlDataType(row).getClass()); - assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(index ++); - assertEquals("date", name(row)); - assertEquals(Types.TIMESTAMP, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(29, precision(row)); - assertEquals(8, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Integer.class, nullable(row).getClass()); - assertEquals(Integer.class, sqlDataType(row).getClass()); - assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(index ++); - assertEquals("some.dotted.field", name(row)); - assertEquals(Types.VARCHAR, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(Integer.MAX_VALUE, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Integer.class, nullable(row).getClass()); - assertEquals(Integer.class, sqlDataType(row).getClass()); - assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(index ++); - assertEquals("some.string", name(row)); - assertEquals(Types.VARCHAR, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(Integer.MAX_VALUE, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Integer.class, nullable(row).getClass()); - assertEquals(Integer.class, sqlDataType(row).getClass()); - assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(index ++); - assertEquals("some.string.normalized", name(row)); - assertEquals(Types.VARCHAR, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(Integer.MAX_VALUE, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Integer.class, nullable(row).getClass()); - assertEquals(Integer.class, sqlDataType(row).getClass()); - assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(index ++); - assertEquals("some.string.typical", name(row)); - assertEquals(Types.VARCHAR, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(Integer.MAX_VALUE, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Integer.class, nullable(row).getClass()); - assertEquals(Integer.class, sqlDataType(row).getClass()); - assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(index ++); - assertEquals("some.ambiguous", name(row)); - assertEquals(Types.VARCHAR, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(Integer.MAX_VALUE, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Integer.class, nullable(row).getClass()); - assertEquals(Integer.class, sqlDataType(row).getClass()); - assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(index ++); - assertEquals("some.ambiguous.one", name(row)); - assertEquals(Types.VARCHAR, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(Integer.MAX_VALUE, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Integer.class, nullable(row).getClass()); - assertEquals(Integer.class, sqlDataType(row).getClass()); - assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(index ++); - assertEquals("some.ambiguous.two", name(row)); - assertEquals(Types.VARCHAR, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(Integer.MAX_VALUE, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Integer.class, nullable(row).getClass()); - assertEquals(Integer.class, sqlDataType(row).getClass()); - assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); - - row = rows.get(index ++); - assertEquals("some.ambiguous.normalized", name(row)); - assertEquals(Types.VARCHAR, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(Integer.MAX_VALUE, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(Integer.class, nullable(row).getClass()); - assertEquals(Integer.class, sqlDataType(row).getClass()); - assertEquals(Integer.class, sqlDataTypeSub(row).getClass()); + public void testSysColumnsInDriverMode() { + for (Mode mode : List.of(Mode.JDBC, Mode.ODBC)) { + Set versions = new HashSet<>(List.of( + SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER), + INTRODUCING_UNSIGNED_LONG, + SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER), + SqlVersion.fromId(Version.CURRENT.id) + )); + versions.add(null); + for (SqlVersion version : versions) { + sysColumnsInDriverModeAtVersion(mode, version); + } + } } private static Object name(List list) { @@ -588,19 +463,19 @@ private void executeCommand(String sql, List params, Consume private Tuple sql(String sql, List params, SqlConfiguration config, Map mapping) { EsIndex test = new EsIndex("test", mapping); - Analyzer analyzer = new Analyzer(config, new FunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics())); + Analyzer analyzer = new Analyzer(config, new FunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics(), config)); Command cmd = (Command) analyzer.analyze(parser.createStatement(sql, params, UTC), true); IndexResolver resolver = mock(IndexResolver.class); when(resolver.clusterName()).thenReturn(CLUSTER_NAME); doAnswer(invocation -> { - ((ActionListener) invocation.getArguments()[4]).onResponse(IndexResolution.valid(test)); + ((ActionListener) invocation.getArguments()[3]).onResponse(IndexResolution.valid(test)); return Void.TYPE; - }).when(resolver).resolveAsMergedMapping(any(), any(), anyBoolean(), any(), any()); + }).when(resolver).resolveAsMergedMapping(any(), any(), anyBoolean(), any()); doAnswer(invocation -> { - ((ActionListener>) invocation.getArguments()[4]).onResponse(singletonList(test)); + ((ActionListener>) invocation.getArguments()[3]).onResponse(singletonList(test)); return Void.TYPE; - }).when(resolver).resolveAsSeparateMappings(any(), any(), anyBoolean(), any(), any()); + }).when(resolver).resolveAsSeparateMappings(any(), any(), anyBoolean(), any()); SqlSession session = new SqlSession(config, null, null, resolver, null, null, null, null, null); return new Tuple<>(cmd, session); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java index 17c58134ba0dd..8e43934c3c411 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java @@ -334,7 +334,7 @@ private SqlTypedParamValue param(Object value) { private Tuple sql(String sql, List params, SqlConfiguration cfg) { EsIndex test = new EsIndex("test", mapping); Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new FunctionRegistry(), IndexResolution.valid(test), - new Verifier(new Metrics())); + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); Command cmd = (Command) analyzer.analyze(parser.createStatement(sql, params, cfg.zoneId()), true); IndexResolver resolver = mock(IndexResolver.class); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java index 5d3b70f5164e7..53f342556d186 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.sql.plan.logical.command.sys; +import org.elasticsearch.Version; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; @@ -12,52 +13,67 @@ import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.index.IndexResolver; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.elasticsearch.xpack.sql.SqlTestUtils; import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer; import org.elasticsearch.xpack.sql.parser.SqlParser; import org.elasticsearch.xpack.sql.plan.logical.command.Command; +import org.elasticsearch.xpack.sql.proto.Mode; +import org.elasticsearch.xpack.sql.proto.Protocol; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.session.SchemaRowSet; +import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.session.SqlSession; import org.elasticsearch.xpack.sql.type.SqlDataTypes; import org.elasticsearch.xpack.sql.types.SqlTypesTests; +import org.elasticsearch.xpack.sql.util.DateUtils; import java.sql.JDBCType; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import static java.util.Arrays.asList; import static org.elasticsearch.action.ActionListener.wrap; +import static org.elasticsearch.xpack.sql.session.Compatibility.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.session.Compatibility.isTypeSupportedInVersion; import static org.mockito.Mockito.mock; public class SysTypesTests extends ESTestCase { private final SqlParser parser = new SqlParser(); - private Tuple sql(String sql) { + private Tuple sql(String sql, Mode mode, SqlVersion version) { + SqlConfiguration configuration = new SqlConfiguration(DateUtils.UTC, Protocol.FETCH_SIZE, + Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, null, mode, null, version, null, null, false, false); EsIndex test = new EsIndex("test", SqlTypesTests.loadMapping("mapping-multi-field-with-nested.json", true)); - Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new FunctionRegistry(), IndexResolution.valid(test), null); + Analyzer analyzer = new Analyzer(configuration, new FunctionRegistry(), IndexResolution.valid(test), null); Command cmd = (Command) analyzer.analyze(parser.createStatement(sql), false); IndexResolver resolver = mock(IndexResolver.class); - SqlSession session = new SqlSession(SqlTestUtils.TEST_CFG, null, null, resolver, null, null, null, null, null); + SqlSession session = new SqlSession(configuration, null, null, resolver, null, null, null, null, null); return new Tuple<>(cmd, session); } + private Tuple sql(String sql) { + return sql(sql, randomFrom(Mode.values()), randomBoolean() ? null : SqlVersion.fromId(Version.CURRENT.id)); + } - public void testSysTypes() { - Command cmd = sql("SYS TYPES").v1(); + static final List ALL_TYPES = asList("BYTE", "LONG", "UNSIGNED_LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", + "FLOAT", "DOUBLE", "SCALED_FLOAT", "IP", "KEYWORD", "TEXT", "BOOLEAN", "DATE", "TIME", "DATETIME", + "INTERVAL_YEAR", "INTERVAL_MONTH", "INTERVAL_DAY", "INTERVAL_HOUR", "INTERVAL_MINUTE", "INTERVAL_SECOND", + "INTERVAL_YEAR_TO_MONTH", "INTERVAL_DAY_TO_HOUR", "INTERVAL_DAY_TO_MINUTE", "INTERVAL_DAY_TO_SECOND", + "INTERVAL_HOUR_TO_MINUTE", "INTERVAL_HOUR_TO_SECOND", "INTERVAL_MINUTE_TO_SECOND", + "GEO_POINT", "GEO_SHAPE", "SHAPE", "UNSUPPORTED", "NESTED", "OBJECT"); - List names = asList("BYTE", "LONG", "UNSIGNED_LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "FLOAT", "DOUBLE", - "SCALED_FLOAT", "IP", "KEYWORD", "TEXT", "BOOLEAN", "DATE", "TIME", "DATETIME", - "INTERVAL_YEAR", "INTERVAL_MONTH", "INTERVAL_DAY", "INTERVAL_HOUR", "INTERVAL_MINUTE", "INTERVAL_SECOND", - "INTERVAL_YEAR_TO_MONTH", "INTERVAL_DAY_TO_HOUR", "INTERVAL_DAY_TO_MINUTE", "INTERVAL_DAY_TO_SECOND", - "INTERVAL_HOUR_TO_MINUTE", "INTERVAL_HOUR_TO_SECOND", "INTERVAL_MINUTE_TO_SECOND", - "GEO_POINT", "GEO_SHAPE", "SHAPE", "UNSUPPORTED", "NESTED", "OBJECT"); + public void testSysTypes() { + Tuple cmd = sql("SYS TYPES"); - cmd.execute(session(), wrap(p -> { + cmd.v1().execute(cmd.v2(), wrap(p -> { SchemaRowSet r = (SchemaRowSet) p.rowSet(); assertEquals(19, r.columnCount()); assertEquals(SqlDataTypes.types().size(), r.size()); assertFalse(r.schema().types().contains(DataTypes.NULL)); - // test first numeric (BYTE) as signed + // test first numeric (i.e. BYTE) as signed assertFalse(r.column(9, Boolean.class)); // make sure precision is returned as boolean (not int) assertFalse(r.column(10, Boolean.class)); @@ -65,17 +81,42 @@ public void testSysTypes() { assertFalse(r.column(11, Boolean.class)); for (int i = 0; i < r.size(); i++) { - assertEquals(names.get(i), r.column(0)); + assertEquals(ALL_TYPES.get(i), r.column(0)); r.advanceRow(); } }, ex -> fail(ex.getMessage()))); } + public void testUnsignedLongFiltering() { + Set versions = new HashSet<>(List.of( + SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER), + INTRODUCING_UNSIGNED_LONG, + SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER), + SqlVersion.fromId(Version.CURRENT.id) + )); + versions.add(null); + for (SqlVersion version : versions) { + for (Mode mode : Mode.values()) { + Tuple cmd = sql("SYS TYPES", mode, version); + + cmd.v1().execute(cmd.v2(), wrap(p -> { + SchemaRowSet r = (SchemaRowSet) p.rowSet(); + List types = new ArrayList<>(); + r.forEachRow(rv -> types.add((String) rv.column(0))); + List expectedTypes = ALL_TYPES.stream() + .filter(t -> Mode.isDriver(mode) == false || isTypeSupportedInVersion(DataTypes.fromTypeName(t), version)) + .collect(Collectors.toList()); + assertEquals(expectedTypes, types); + }, ex -> fail(ex.getMessage()))); + } + } + } + public void testSysTypesDefaultFiltering() { - Command cmd = sql("SYS TYPES 0").v1(); + Tuple cmd = sql("SYS TYPES 0"); - cmd.execute(session(), wrap(p -> { + cmd.v1().execute(cmd.v2(), wrap(p -> { SchemaRowSet r = (SchemaRowSet) p.rowSet(); assertEquals(SqlDataTypes.types().size(), r.size()); }, ex -> fail(ex.getMessage()))); @@ -83,9 +124,9 @@ public void testSysTypesDefaultFiltering() { public void testSysTypesPositiveFiltering() { // boolean = 16 - Command cmd = sql("SYS TYPES " + JDBCType.BOOLEAN.getVendorTypeNumber()).v1(); + Tuple cmd = sql("SYS TYPES " + JDBCType.BOOLEAN.getVendorTypeNumber()); - cmd.execute(session(), wrap(p -> { + cmd.v1().execute(cmd.v2(), wrap(p -> { SchemaRowSet r = (SchemaRowSet) p.rowSet(); assertEquals(1, r.size()); assertEquals("BOOLEAN", r.column(0)); @@ -93,9 +134,9 @@ public void testSysTypesPositiveFiltering() { } public void testSysTypesNegativeFiltering() { - Command cmd = sql("SYS TYPES " + JDBCType.TINYINT.getVendorTypeNumber()).v1(); + Tuple cmd = sql("SYS TYPES " + JDBCType.TINYINT.getVendorTypeNumber()); - cmd.execute(session(), wrap(p -> { + cmd.v1().execute(cmd.v2(), wrap(p -> { SchemaRowSet r = (SchemaRowSet) p.rowSet(); assertEquals(1, r.size()); assertEquals("BYTE", r.column(0)); @@ -103,9 +144,9 @@ public void testSysTypesNegativeFiltering() { } public void testSysTypesMultipleMatches() { - Command cmd = sql("SYS TYPES " + JDBCType.VARCHAR.getVendorTypeNumber()).v1(); + Tuple cmd = sql("SYS TYPES " + JDBCType.VARCHAR.getVendorTypeNumber()); - cmd.execute(session(), wrap(p -> { + cmd.v1().execute(cmd.v2(), wrap(p -> { SchemaRowSet r = (SchemaRowSet) p.rowSet(); assertEquals(3, r.size()); assertEquals("IP", r.column(0)); @@ -115,8 +156,4 @@ public void testSysTypesMultipleMatches() { assertEquals("TEXT", r.column(0)); }, ex -> fail(ex.getMessage()))); } - - private static SqlSession session() { - return new SqlSession(SqlTestUtils.TEST_CFG, null, null, null, null, null, null, null, null); - } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/PostOptimizerVerifierTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/PostOptimizerVerifierTests.java index 0f1072f462fd2..c5bbe0292e11b 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/PostOptimizerVerifierTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/PostOptimizerVerifierTests.java @@ -39,7 +39,8 @@ public void init() { Map mapping = SqlTypesTests.loadMapping("mapping-multi-field-variation.json"); EsIndex test = new EsIndex("test", mapping); indexResolution = IndexResolution.valid(test); - analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, new Verifier(new Metrics())); + analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); optimizer = new Optimizer(); planner = new Planner(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java index 77df4913b552a..be252f68131e0 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java @@ -48,7 +48,8 @@ public static void init() { Map mapping = SqlTypesTests.loadMapping("mapping-multi-field-variation.json"); EsIndex test = new EsIndex("test", mapping); IndexResolution getIndexResult = IndexResolution.valid(test); - analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics())); + analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), getIndexResult, + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); optimizer = new Optimizer(); planner = new Planner(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java index 521e9a1d4792b..8dee1858d605c 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java @@ -122,7 +122,8 @@ private static class TestContext { Map mapping = SqlTypesTests.loadMapping(mappingFile); EsIndex test = new EsIndex("test", mapping); IndexResolution getIndexResult = IndexResolution.valid(test); - analyzer = new Analyzer(SqlTestUtils.TEST_CFG, sqlFunctionRegistry, getIndexResult, new Verifier(new Metrics())); + analyzer = new Analyzer(SqlTestUtils.TEST_CFG, sqlFunctionRegistry, getIndexResult, + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); optimizer = new Optimizer(); planner = new Planner(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java index 55eb0c234f533..b8ea511c652a8 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java @@ -30,11 +30,11 @@ import static org.elasticsearch.xpack.sql.stats.Metrics.FPREFIX; public class VerifierMetricsTests extends ESTestCase { - + private SqlParser parser = new SqlParser(); private String[] commands = {"SHOW FUNCTIONS", "SHOW COLUMNS FROM library", "SHOW SCHEMAS", "SHOW TABLES", "SYS COLUMNS LIKE '%name'", "SYS TABLES", "SYS TYPES"}; - + public void testWhereQuery() { Counters c = sql("SELECT emp_no FROM test WHERE languages > 2"); assertEquals(1L, where(c)); @@ -45,7 +45,7 @@ public void testWhereQuery() { assertEquals(0, command(c)); assertEquals(0, local(c)); } - + public void testLimitQuery() { Counters c = sql("SELECT emp_no FROM test LIMIT 4"); assertEquals(0, where(c)); @@ -56,7 +56,7 @@ public void testLimitQuery() { assertEquals(0, command(c)); assertEquals(0, local(c)); } - + public void testGroupByQuery() { Counters c = sql("SELECT languages, MAX(languages) FROM test GROUP BY languages"); assertEquals(0, where(c)); @@ -67,7 +67,7 @@ public void testGroupByQuery() { assertEquals(0, command(c)); assertEquals(0, local(c)); } - + public void testHavingQuery() { Counters c = sql("SELECT UCASE(gender), MAX(languages) FROM test GROUP BY gender HAVING MAX(languages) > 3"); assertEquals(0, where(c)); @@ -78,7 +78,7 @@ public void testHavingQuery() { assertEquals(0, command(c)); assertEquals(0, local(c)); } - + public void testOrderByQuery() { Counters c = sql("SELECT UCASE(gender) FROM test ORDER BY emp_no"); assertEquals(0, where(c)); @@ -89,7 +89,7 @@ public void testOrderByQuery() { assertEquals(0, command(c)); assertEquals(0, local(c)); } - + public void testCommand() { Counters c = sql(randomFrom("SHOW FUNCTIONS", "SHOW COLUMNS FROM library", "SHOW SCHEMAS", "SHOW TABLES", "SYS COLUMNS LIKE '%name'", "SYS TABLES", "SYS TYPES")); @@ -101,7 +101,7 @@ public void testCommand() { assertEquals(1L, command(c)); assertEquals(0, local(c)); } - + public void testLocalQuery() { Counters c = sql("SELECT CONCAT('Elastic','search')"); assertEquals(0, where(c)); @@ -112,7 +112,7 @@ public void testLocalQuery() { assertEquals(0, command(c)); assertEquals(1L, local(c)); } - + public void testWhereAndLimitQuery() { Counters c = sql("SELECT emp_no FROM test WHERE languages > 2 LIMIT 5"); assertEquals(1L, where(c)); @@ -123,7 +123,7 @@ public void testWhereAndLimitQuery() { assertEquals(0, command(c)); assertEquals(0, local(c)); } - + public void testWhereLimitGroupByQuery() { Counters c = sql("SELECT languages FROM test WHERE languages > 2 GROUP BY languages LIMIT 5"); assertEquals(1L, where(c)); @@ -134,7 +134,7 @@ public void testWhereLimitGroupByQuery() { assertEquals(0, command(c)); assertEquals(0, local(c)); } - + public void testWhereLimitGroupByHavingQuery() { Counters c = sql("SELECT languages FROM test WHERE languages > 2 GROUP BY languages HAVING MAX(languages) > 3 LIMIT 5"); assertEquals(1L, where(c)); @@ -145,7 +145,7 @@ public void testWhereLimitGroupByHavingQuery() { assertEquals(0, command(c)); assertEquals(0, local(c)); } - + public void testWhereLimitGroupByHavingOrderByQuery() { Counters c = sql("SELECT languages FROM test WHERE languages > 2 GROUP BY languages HAVING MAX(languages) > 3" + " ORDER BY languages LIMIT 5"); @@ -157,15 +157,15 @@ public void testWhereLimitGroupByHavingOrderByQuery() { assertEquals(0, command(c)); assertEquals(0, local(c)); } - + public void testTwoQueriesExecuted() { Metrics metrics = new Metrics(); - Verifier verifier = new Verifier(metrics); + Verifier verifier = new Verifier(metrics, SqlTestUtils.TEST_CFG); sqlWithVerifier("SELECT languages FROM test WHERE languages > 2 GROUP BY languages LIMIT 5", verifier); sqlWithVerifier("SELECT languages FROM test WHERE languages > 2 GROUP BY languages HAVING MAX(languages) > 3 " + "ORDER BY languages LIMIT 5", verifier); Counters c = metrics.stats(); - + assertEquals(2L, where(c)); assertEquals(2L, limit(c)); assertEquals(2L, groupby(c)); @@ -174,15 +174,15 @@ public void testTwoQueriesExecuted() { assertEquals(0, command(c)); assertEquals(0, local(c)); } - + public void testTwoCommandsExecuted() { String command1 = randomFrom(commands); Metrics metrics = new Metrics(); - Verifier verifier = new Verifier(metrics); + Verifier verifier = new Verifier(metrics, SqlTestUtils.TEST_CFG); sqlWithVerifier(command1, verifier); sqlWithVerifier(randomValueOtherThan(command1, () -> randomFrom(commands)), verifier); Counters c = metrics.stats(); - + assertEquals(0, where(c)); assertEquals(0, limit(c)); assertEquals(0, groupby(c)); @@ -191,39 +191,39 @@ public void testTwoCommandsExecuted() { assertEquals(2, command(c)); assertEquals(0, local(c)); } - + private long where(Counters c) { return c.get(FPREFIX + WHERE); } - + private long groupby(Counters c) { return c.get(FPREFIX + GROUPBY); } - + private long limit(Counters c) { return c.get(FPREFIX + LIMIT); } - + private long local(Counters c) { return c.get(FPREFIX + LOCAL); } - + private long having(Counters c) { return c.get(FPREFIX + HAVING); } - + private long orderby(Counters c) { return c.get(FPREFIX + ORDERBY); } - + private long command(Counters c) { return c.get(FPREFIX + COMMAND); } - + private Counters sql(String sql) { return sql(sql, null); } - + private void sqlWithVerifier(String sql, Verifier verifier) { sql(sql, verifier); } @@ -231,17 +231,17 @@ private void sqlWithVerifier(String sql, Verifier verifier) { private Counters sql(String sql, Verifier v) { Map mapping = SqlTypesTests.loadMapping("mapping-basic.json"); EsIndex test = new EsIndex("test", mapping); - + Verifier verifier = v; Metrics metrics = null; if (v == null) { metrics = new Metrics(); - verifier = new Verifier(metrics); + verifier = new Verifier(metrics, SqlTestUtils.TEST_CFG); } Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), IndexResolution.valid(test), verifier); analyzer.analyze(parser.createStatement(sql), true); - + return metrics == null ? null : metrics.stats(); } -} \ No newline at end of file +} From 672df3286e6f1c29952e41fe024be22e22309d02 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Mon, 23 Nov 2020 09:42:14 +0100 Subject: [PATCH 06/28] Minor clean-ups Vars renaming, dead code removal. --- .../xpack/eql/session/EqlSession.java | 2 +- .../xpack/ql/index/IndexResolver.java | 4 +- .../xpack/sql/analysis/analyzer/Analyzer.java | 47 ------------------- .../analyzer/FieldAttributeTests.java | 12 ++--- 4 files changed, 8 insertions(+), 57 deletions(-) diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java index 05e053e647f19..4ba9fdff687e0 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java @@ -100,7 +100,7 @@ private void preAnalyze(LogicalPlan parsed, ActionListener list listener.onFailure(new TaskCancelledException("cancelled")); return; } - indexResolver.resolveAsMergedMapping(indexWildcard, null, configuration.indicesOptions(), null, + indexResolver.resolveAsMergedMapping(indexWildcard, null, configuration.indicesOptions(), map(listener, r -> preAnalyzer.preAnalyze(parsed, r)) ); } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index ed85985585763..92dd8a7035d4b 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -8,7 +8,6 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.ElasticsearchSecurityException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; @@ -23,7 +22,6 @@ import org.elasticsearch.action.support.IndicesOptions.WildcardStates; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.AliasMetadata; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.util.set.Sets; @@ -280,7 +278,7 @@ private void filterResults(String javaRegex, GetAliasesResponse aliases, GetInde /** * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. */ - public void resolveAsMergedMapping(String indexWildcard, String javaRegex, IndicesOptions indicesOptions, @Nullable Version version, + public void resolveAsMergedMapping(String indexWildcard, String javaRegex, IndicesOptions indicesOptions, ActionListener listener) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, indicesOptions); client.fieldCaps(fieldRequest, diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index 38e1cb5812e4a..221cb98955efc 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -497,37 +497,6 @@ private LogicalPlan dedupRight(LogicalPlan left, LogicalPlan right) { } } - /* - private class VersionCompatibility extends BaseAnalyzeRule { - @Override - protected LogicalPlan doRule(LogicalPlan plan) { - if (configuration.version() == null) { - return plan; - } - if (plan instanceof Project) { - Project project = (Project) plan; - return new Project(project.source(), project.child(), - project.projections().stream().map(this::typeSupported).collect(toList())); - } - return plan; - } - - private NamedExpression typeSupported(NamedExpression ne) { - if (ne.resolved() && isTypeVersionGated(ne.dataType(), configuration.version())) { - return new UnresolvedAttribute(ne.source(), null, ne.toAttribute().qualifier(), - "Cannot use field [" + ne.name() + "] with type [" + ne.dataType() + "] unsupported in version [" + - configuration.version() + "]"); - } - return ne; - } - - @Override - protected boolean skipResolved() { - return false; - } - } - */ - // Allow ordinal positioning in order/sort by (quite useful when dealing with aggs) // Note that ordering starts at 1 private static class ResolveOrdinalInOrderByAndGroupBy extends BaseAnalyzeRule { @@ -1331,22 +1300,6 @@ protected boolean skipResolved() { } } - private class VersionCompatibility2 extends Rule { - - @Override - public LogicalPlan apply(LogicalPlan logicalPlan) { - List projects = logicalPlan.collectFirstChildren(x -> x instanceof Project); - assert projects.size() > 0; - projects.get(0); - return logicalPlan.transformDown(this::rule, typeToken()); - } - - @Override - protected LogicalPlan rule(Project project) { - return null; - } - } - abstract static class AnalyzeRule extends Rule { // transformUp (post-order) - that is first children and then the node diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java index 58c708eca6deb..d83359f3ca4c4 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java @@ -294,21 +294,21 @@ public void testUnsignedLongVersionCompatibility() { String queryWithArithmetic = "SELECT unsigned_long + 1 AS unsigned_long FROM test"; String queryWithCast = "SELECT long + 1::unsigned_long AS unsigned_long FROM test"; - SqlVersion introducingUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id); - SqlVersion preIntroducingVersion = SqlVersion.fromId(introducingUnsignedLong.id - SqlVersion.MINOR_MULTIPLIER); - SqlVersion postIntroducingVersion = SqlVersion.fromId(introducingUnsignedLong.id + SqlVersion.MINOR_MULTIPLIER); + SqlVersion introducingUnsignedLong = INTRODUCING_UNSIGNED_LONG; + SqlVersion preUnsignedLong = SqlVersion.fromId(introducingUnsignedLong.id - SqlVersion.MINOR_MULTIPLIER); + SqlVersion postUnsignedLong = SqlVersion.fromId(introducingUnsignedLong.id + SqlVersion.MINOR_MULTIPLIER); for (String sql : List.of(query, queryWithLiteral, queryWithAlias, queryWithArithmetic, queryWithCast)) { - SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preIntroducingVersion); + SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preUnsignedLong); analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics(), sqlConfig)); VerificationException ex = expectThrows(VerificationException.class, () -> plan(sql)); assertEquals( "Found 1 problem\nline 1:8: Cannot use field [unsigned_long] with type [UNSIGNED_LONG] unsupported in version [" + - preIntroducingVersion + "]", + preUnsignedLong + "]", ex.getMessage()); - for (SqlVersion v : List.of(introducingUnsignedLong, postIntroducingVersion)) { + for (SqlVersion v : List.of(introducingUnsignedLong, postUnsignedLong)) { analyzer = new Analyzer(SqlTestUtils.randomConfiguration(v), functionRegistry, getIndexResult, verifier); LogicalPlan plan = plan(sql); From 85447c1711d9733da77c65501e706853f9794afc Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Fri, 27 Nov 2020 22:15:35 +0100 Subject: [PATCH 07/28] Address reivew comments - refactoring of the QA JDBC tests; - introduce csv/sql-spec files; - list of punctual changes as suggested. --- .../xpack/eql/parser/ExpressionBuilder.java | 5 +- .../xpack/eql/analysis/VerifierTests.java | 2 +- .../src/test/resources/mapping-numeric.json | 6 +- .../extractor/AbstractFieldHitExtractor.java | 6 +- .../operator/arithmetic/Arithmetics.java | 63 +-- .../xpack/ql/type/DataTypeConverter.java | 19 +- .../xpack/ql/type/DataTypes.java | 1 - .../elasticsearch/xpack/ql/util/Check.java | 11 - .../xpack/ql/util/StringUtils.java | 23 +- .../xpack/ql/util/UnsignedLongUtils.java | 36 ++ .../BinaryArithmeticProcessorTests.java | 21 +- .../ql/type/DataTypeConversionTests.java | 2 +- .../xpack/sql/jdbc/JdbcPreparedStatement.java | 7 +- .../xpack/sql/jdbc/TypeConverter.java | 70 ++- .../xpack/sql/jdbc/TypeUtils.java | 5 +- .../xpack/sql/jdbc/TypeConverterTests.java | 1 - .../xpack/sql/qa/jdbc/JdbcTestUtils.java | 13 + .../qa/jdbc/PreparedStatementTestCase.java | 116 ++--- .../qa/jdbc/ResultSetMetaDataTestCase.java | 54 +- .../xpack/sql/qa/jdbc/ResultSetTestCase.java | 460 +++++++++++------- .../xpack/sql/qa/FieldExtractorTestCase.java | 16 +- .../xpack/sql/qa/SqlProtocolTestCase.java | 2 +- .../xpack/sql/qa/jdbc/CsvTestUtils.java | 2 + .../xpack/sql/qa/jdbc/DataLoader.java | 11 +- .../xpack/sql/qa/jdbc/JdbcAssert.java | 15 + .../server/src/main/resources/alias.csv-spec | 6 +- .../src/main/resources/command.csv-spec | 160 +++--- .../single-node-only/command-sys.csv-spec | 236 ++++----- .../src/main/resources/unsigned-long.csv-spec | 99 ++++ .../src/main/resources/unsigned-long.sql-spec | 73 +++ .../xpack/sql/analysis/analyzer/Analyzer.java | 1 - .../xpack/sql/analysis/analyzer/Verifier.java | 37 +- .../xpack/sql/execution/PlanExecutor.java | 7 +- .../xpack/sql/parser/ExpressionBuilder.java | 2 +- .../plan/logical/command/sys/SysColumns.java | 5 +- .../plan/logical/command/sys/SysTypes.java | 6 +- .../xpack/sql/session/SqlConfiguration.java | 4 +- ...y.java => VersionCompatibilityChecks.java} | 18 +- .../elasticsearch/xpack/sql/SqlTestUtils.java | 6 +- .../analyzer/FieldAttributeTests.java | 10 +- .../analyzer/VerifierErrorMessagesTests.java | 4 +- .../analysis/index/IndexResolverTests.java | 2 +- .../scalar/DatabaseFunctionTests.java | 2 +- .../function/scalar/UserFunctionTests.java | 2 +- .../scalar/datetime/CurrentDateTimeTests.java | 2 +- .../scalar/datetime/CurrentTimeTests.java | 2 +- .../sql/optimizer/OptimizerRunTests.java | 2 +- .../logical/command/sys/SysColumnsTests.java | 124 ++--- .../logical/command/sys/SysTablesTests.java | 2 +- .../logical/command/sys/SysTypesTests.java | 12 +- .../planner/PostOptimizerVerifierTests.java | 2 +- .../xpack/sql/planner/QueryFolderTests.java | 2 +- .../sql/planner/QueryTranslatorTests.java | 2 +- .../xpack/sql/stats/VerifierMetricsTests.java | 6 +- 54 files changed, 1088 insertions(+), 715 deletions(-) create mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/UnsignedLongUtils.java create mode 100644 x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec create mode 100644 x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.sql-spec rename x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/{Compatibility.java => VersionCompatibilityChecks.java} (67%) diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java index 9c4c52b175efc..3622eb11f3121 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java @@ -210,7 +210,7 @@ public Literal visitIntegerLiteral(EqlBaseParser.IntegerLiteralContext ctx) { Number value; try { - value = StringUtils.parseInteger(text); + value = StringUtils.parseIntegral(text); } catch (QlIllegalArgumentException siae) { // if it's too large, then quietly try to parse as a float instead try { @@ -225,11 +225,10 @@ public Literal visitIntegerLiteral(EqlBaseParser.IntegerLiteralContext ctx) { if (value instanceof BigInteger) { type = DataTypes.UNSIGNED_LONG; } else { - assert value instanceof Long : "Expected value [" + value + "] of type Long but got: " + value.getClass(); // try to downsize to int if possible (since that's the most common type) if (value.longValue() == value.intValue()) { type = DataTypes.INTEGER; - value = Integer.valueOf(value.intValue()); + value = value.intValue(); } else { type = DataTypes.LONG; } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java index fcf2f44238af6..ffbb13da3b629 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -176,7 +176,6 @@ public void testAliasErrors() { // Test all elasticsearch numeric field types public void testNumeric() { final IndexResolution idxr = loadIndexResolution("mapping-numeric.json"); - accept(idxr, "foo where unsigned_long_field == 0"); accept(idxr, "foo where long_field == 0"); accept(idxr, "foo where integer_field == 0"); accept(idxr, "foo where short_field == 0"); @@ -185,6 +184,7 @@ public void testNumeric() { accept(idxr, "foo where float_field == 0"); accept(idxr, "foo where half_float_field == 0"); accept(idxr, "foo where scaled_float_field == 0"); + accept(idxr, "foo where unsigned_long_field == 0"); // Test query against unsupported field type int assertEquals("1:11: Cannot use field [wrong_int_type_field] with unsupported type [int]", diff --git a/x-pack/plugin/eql/src/test/resources/mapping-numeric.json b/x-pack/plugin/eql/src/test/resources/mapping-numeric.json index e750446d9f149..4f0587f35c4e0 100644 --- a/x-pack/plugin/eql/src/test/resources/mapping-numeric.json +++ b/x-pack/plugin/eql/src/test/resources/mapping-numeric.json @@ -10,9 +10,6 @@ "@timestamp" : { "type" : "date" }, - "unsigned_long_field" : { - "type": "unsigned_long" - }, "long_field" : { "type" : "long" }, @@ -37,6 +34,9 @@ "scaled_float_field": { "type" : "scaled_float" }, + "unsigned_long_field" : { + "type": "unsigned_long" + }, "wrong_int_type_field": { "type" : "int" } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java index 373e3e133c541..aeb4e930fce24 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java @@ -188,7 +188,7 @@ protected Object unwrapMultiValue(Object values) { Number result = null; try { // TODO: don't mapper modules expose _source parsing methods? should the _source be (re)validated? - result = dataType == UNSIGNED_LONG ? new BigInteger(values.toString()) : numberType(dataType).parse(values, true); + result = createNumeric(dataType, values); } catch(IllegalArgumentException iae) { return null; } @@ -214,6 +214,10 @@ private static NumberType numberType(DataType dataType) { return NumberType.valueOf(dataType.esType().toUpperCase(Locale.ROOT)); } + private static Number createNumeric(DataType dataType, Object value) { + return dataType == UNSIGNED_LONG ? new BigInteger(value.toString()) : numberType(dataType).parse(value, true); + } + protected abstract Object unwrapCustomValue(Object values); protected abstract boolean isPrimitive(List list); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java index 6d2061c75ff6f..75b0746aefeea 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java @@ -10,7 +10,7 @@ import java.math.BigInteger; import java.util.function.BiFunction; -import static org.elasticsearch.xpack.ql.util.Check.isUnsignedLong; +import static org.elasticsearch.xpack.ql.util.UnsignedLongUtils.asUnsignedLong; /** * Arithmetic operation using the type widening rules of the JLS 5.6.2 namely @@ -34,22 +34,6 @@ default Object wrap(Object l, Object r) { } } - private static BigInteger unsignedLongOperation(Number l, Number r, BiFunction op) { - BigInteger biLeft, biRight; - if (l instanceof BigInteger) { - biLeft = (BigInteger) l; - biRight = BigInteger.valueOf(r.longValue()); - } else if (r instanceof BigInteger) { - biLeft = BigInteger.valueOf(l.longValue()); - biRight = (BigInteger) r; - } else { - return null; - } - BigInteger bi = op.apply(biLeft, biRight); - isUnsignedLong(bi); - return bi; - } - public static Number add(Number l, Number r) { if (l == null || r == null) { return null; @@ -61,9 +45,9 @@ public static Number add(Number l, Number r) { if (l instanceof Float || r instanceof Float) { return Float.valueOf(l.floatValue() + r.floatValue()); } - BigInteger bi = unsignedLongOperation(l, r, BigInteger::add); - if (bi != null) { - return bi; + if (l instanceof BigInteger || r instanceof BigInteger) { + BigInteger bi = asBigInteger(l).add(asBigInteger(r)); + return asUnsignedLong(bi); } if (l instanceof Long || r instanceof Long) { return Long.valueOf(Math.addExact(l.longValue(), r.longValue())); @@ -83,9 +67,9 @@ public static Number sub(Number l, Number r) { if (l instanceof Float || r instanceof Float) { return Float.valueOf(l.floatValue() - r.floatValue()); } - BigInteger bi = unsignedLongOperation(l, r, BigInteger::subtract); - if (bi != null) { - return bi; + if (l instanceof BigInteger || r instanceof BigInteger) { + BigInteger bi = asBigInteger(l).subtract(asBigInteger(r)); + return asUnsignedLong(bi); } if (l instanceof Long || r instanceof Long) { return Long.valueOf(Math.subtractExact(l.longValue(), r.longValue())); @@ -105,15 +89,12 @@ public static Number mul(Number l, Number r) { if (l instanceof Float || r instanceof Float) { return Float.valueOf(l.floatValue() * r.floatValue()); } - // Note: in case of unsigned_long overflow (or underflow, with negative fixed point numbers), the exception is thrown. - // This is unlike the way some other traditional RDBMS that support unsigned types work, which simply promote the result to a - // floating point type, but in line with how our implementation treats other fixed point type operations (i.e. Math#xxExact()). - // The reason for our behavior is (prolly) the need to establish a schema based on an index mapping. Noteworthy however is also - // that we're not strictly consistent with the returned type: a `SUM(field)` aggregation can return a floating point, for example. - // TODO: document this last point or address it? - BigInteger bi = unsignedLongOperation(l, r, BigInteger::multiply); - if (bi != null) { - return bi; + if (l instanceof BigInteger || r instanceof BigInteger) { + BigInteger bi = asBigInteger(l).multiply(asBigInteger(r)); + // Note: in case of unsigned_long overflow (or underflow, with negative fixed point numbers), the exception is thrown. + // This is unlike the way some other traditional RDBMS that support unsigned types work, which simply promote the result to a + // floating point type, but in line with how our implementation treats other fixed point type operations (i.e. Math#xxExact()). + return asUnsignedLong(bi); } if (l instanceof Long || r instanceof Long) { return Long.valueOf(Math.multiplyExact(l.longValue(), r.longValue())); @@ -133,9 +114,9 @@ public static Number div(Number l, Number r) { if (l instanceof Float || r instanceof Float) { return l.floatValue() / r.floatValue(); } - BigInteger bi = unsignedLongOperation(l, r, BigInteger::divide); - if (bi != null) { - return bi; + if (l instanceof BigInteger || r instanceof BigInteger) { + BigInteger bi = asBigInteger(l).divide(asBigInteger(r)); + return asUnsignedLong(bi); } if (l instanceof Long || r instanceof Long) { return l.longValue() / r.longValue(); @@ -155,9 +136,9 @@ public static Number mod(Number l, Number r) { if (l instanceof Float || r instanceof Float) { return Float.valueOf(l.floatValue() % r.floatValue()); } - BigInteger bi = unsignedLongOperation(l, r, BigInteger::remainder); - if (bi != null) { - return bi; + if (l instanceof BigInteger || r instanceof BigInteger) { + BigInteger bi = asBigInteger(l).remainder(asBigInteger(r)); + return asUnsignedLong(bi); } if (l instanceof Long || r instanceof Long) { return Long.valueOf(l.longValue() % r.longValue()); @@ -187,7 +168,7 @@ static Number negate(Number n) { } if (n instanceof BigInteger) { if (((BigInteger) n).signum() != 0) { - throw new ArithmeticException("unsigned_long overflow"); + throw new ArithmeticException("unsigned_long overflow"); // in the scope of the unsigned_long type } return n; } @@ -197,4 +178,8 @@ static Number negate(Number n) { return Integer.valueOf(Math.negateExact(n.intValue())); } + + private static BigInteger asBigInteger(Number n) { + return n instanceof BigInteger ? (BigInteger) n : BigInteger.valueOf(n.longValue()); + } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java index 7c9e253098741..eb53e6cccadef 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java @@ -35,8 +35,8 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; import static org.elasticsearch.xpack.ql.type.DataTypes.isString; -import static org.elasticsearch.xpack.ql.util.Check.UNSIGNED_LONG_MAX; -import static org.elasticsearch.xpack.ql.util.Check.isUnsignedLong; +import static org.elasticsearch.xpack.ql.util.UnsignedLongUtils.inUnsignedLongRange; +import static org.elasticsearch.xpack.ql.util.UnsignedLongUtils.isUnsignedLong; /** * Conversion utility from one Elasticsearch data type to another Elasticsearch data types. @@ -385,15 +385,8 @@ public static Long safeToLong(Number x) { } } - // 18446744073709551615.0 - private static final double UNSIGNED_LONG_MAX_AS_DOUBLE = UNSIGNED_LONG_MAX.doubleValue(); - public static BigInteger safeToUnsignedLong(Double x) { - // UNSIGNED_LONG_MAX can't be represented precisely enough on a double, being converted as a rounded up value. - // Converting it to a double and back will yield a larger unsigned long, so the double comparison is still preferred, but - // it'll require the equality check. (BigDecimal comparisons only make sense for string-recovered floating point numbers.) - // This also means that 18446744073709551615.0 is actually a double too high to be converted as an unsigned long. - if (x < 0 || x >= UNSIGNED_LONG_MAX_AS_DOUBLE) { + if (inUnsignedLongRange(x) == false) { throw new QlIllegalArgumentException("[" + x + "] out of [unsigned_long] range"); } return BigDecimal.valueOf(x).toBigInteger(); @@ -408,10 +401,8 @@ public static BigInteger safeToUnsignedLong(Long x) { public static BigInteger safeToUnsignedLong(String x) { BigInteger bi = new BigDecimal(x).toBigInteger(); - try { - isUnsignedLong(bi); - } catch (ArithmeticException ae) { - throw new QlIllegalArgumentException("[" + x + "] out of [unsigned_long] range", ae); + if (isUnsignedLong(bi) == false) { + throw new QlIllegalArgumentException("[" + x + "] out of [unsigned_long] range"); } return bi; } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java index bcc17c5f86c25..3c154f13a4be7 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java @@ -106,7 +106,6 @@ public static DataType fromJava(Object value) { return LONG; } if (value instanceof BigInteger) { - // TODO: range check needed at all? (((BigInteger) value).signum() < 0 return UNSIGNED_LONG; } if (value instanceof Boolean) { diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/Check.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/Check.java index af23408b14d39..604ab8002dfc3 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/Check.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/Check.java @@ -7,8 +7,6 @@ import org.elasticsearch.xpack.ql.QlIllegalArgumentException; -import java.math.BigInteger; - /** * Utility class used for checking various conditions at runtime, with minimum amount of code. */ @@ -49,13 +47,4 @@ public static void isBoolean(Object obj) { throw new QlIllegalArgumentException("A boolean is required; received [{}]", obj); } } - - // 18446744073709551615 - public static final BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); - - public static void isUnsignedLong(BigInteger bi) { - if (bi.signum() < 0 || bi.compareTo(UNSIGNED_LONG_MAX) > 0) { - throw new ArithmeticException("unsigned_long overflow"); - } - } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java index 86d2e294a11da..d66a33a4a2f29 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java @@ -22,7 +22,7 @@ import java.util.Locale; import static java.util.stream.Collectors.toList; -import static org.elasticsearch.xpack.ql.util.Check.isUnsignedLong; +import static org.elasticsearch.xpack.ql.util.UnsignedLongUtils.isUnsignedLong; public final class StringUtils { @@ -327,21 +327,20 @@ public static long parseLong(String string) throws QlIllegalArgumentException { } } - public static Number parseInteger(String string) throws QlIllegalArgumentException { + public static Number parseIntegral(String string) throws QlIllegalArgumentException { + BigInteger bi; try { - BigInteger bi = new BigInteger(string); - try { - if (bi.signum() < 0 || bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) <= 0) { - return bi.longValueExact(); - } - isUnsignedLong(bi); - return bi; - } catch (ArithmeticException ae) { - throw new QlIllegalArgumentException("Number [{}] is too large", string); - } + bi = new BigInteger(string); } catch (NumberFormatException ex) { throw new QlIllegalArgumentException("Cannot parse number [{}]", string); } + if (bi.signum() < 0 || bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) <= 0) { + return bi.longValueExact(); + } + if (isUnsignedLong(bi) == false) { + throw new QlIllegalArgumentException("Number [{}] is too large", string); + } + return bi; } public static String ordinal(int i) { diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/UnsignedLongUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/UnsignedLongUtils.java new file mode 100644 index 0000000000000..07d33e71ae74f --- /dev/null +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/UnsignedLongUtils.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ql.util; + +import java.math.BigInteger; + +public abstract class UnsignedLongUtils { + // 18446744073709551615 + public static final BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); + + // 18446744073709551615.0 + public static final double UNSIGNED_LONG_MAX_AS_DOUBLE = UNSIGNED_LONG_MAX.doubleValue(); + + public static boolean isUnsignedLong(BigInteger bi) { + return bi.signum() >= 0 && bi.compareTo(UNSIGNED_LONG_MAX) <= 0; + } + + public static boolean inUnsignedLongRange(double d) { + // UNSIGNED_LONG_MAX can't be represented precisely enough on a double, being converted as a rounded up value. + // Converting it to a double and back will yield a larger unsigned long, so the double comparison is still preferred, but + // it'll require the equality check. (BigDecimal comparisons only make sense for string-recovered floating point numbers.) + // This also means that 18446744073709551615.0 is actually a double too high to be converted as an unsigned long. + return d >= 0 && d < UNSIGNED_LONG_MAX_AS_DOUBLE; + } + + public static BigInteger asUnsignedLong(BigInteger bi) { + if (isUnsignedLong(bi) == false) { + throw new ArithmeticException("unsigned_long overflow"); + } + return bi; + } +} diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java index a7446ccf1ef2e..711f58ebc8635 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java @@ -14,7 +14,7 @@ import org.elasticsearch.xpack.ql.expression.gen.processor.ConstantProcessor; import org.elasticsearch.xpack.ql.expression.gen.processor.Processor; import org.elasticsearch.xpack.ql.expression.processor.Processors; -import org.elasticsearch.xpack.ql.util.Check; +import org.elasticsearch.xpack.ql.util.UnsignedLongUtils; import java.math.BigInteger; @@ -48,10 +48,13 @@ public void testAdd() { assertEquals(10, ba.process(null)); } - public void testUnsignedLongAdd() { + public void testAddUnsignedLong() { Processor ba = new Add(EMPTY, l(BigInteger.valueOf(7)), l(3)).makePipe().asProcessor(); assertEquals(BigInteger.valueOf(10), ba.process(null)); + ba = new Add(EMPTY, l(BigInteger.ONE), l(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE))).makePipe().asProcessor(); + assertEquals(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.TWO), ba.process(null)); + ba = new Add(EMPTY, l(BigInteger.valueOf(7)), l((short) -3)).makePipe().asProcessor(); assertEquals(BigInteger.valueOf(4), ba.process(null)); @@ -61,7 +64,7 @@ public void testUnsignedLongAdd() { Processor pn = new Add(EMPTY, l(BigInteger.valueOf(7)), l(-8)).makePipe().asProcessor(); expectThrows(ArithmeticException.class, () -> pn.process(null)); - Processor pm = new Add(EMPTY, l(Check.UNSIGNED_LONG_MAX), l(1)).makePipe().asProcessor(); + Processor pm = new Add(EMPTY, l(UnsignedLongUtils.UNSIGNED_LONG_MAX), l(1)).makePipe().asProcessor(); expectThrows(ArithmeticException.class, () -> pm.process(null)); } @@ -70,7 +73,7 @@ public void testSub() { assertEquals(4, ba.process(null)); } - public void testUnsignedLongSub() { + public void testSubUnsignedLong() { Processor bs = new Sub(EMPTY, l(BigInteger.valueOf(7)), l(3)).makePipe().asProcessor(); assertEquals(BigInteger.valueOf(4), bs.process(null)); @@ -89,7 +92,7 @@ public void testMul() { assertEquals(21, ba.process(null)); } - public void testUnsignedLongMul() { + public void testMulUnsignedLong() { Processor bm = new Mul(EMPTY, l(BigInteger.valueOf(7)), l(3)).makePipe().asProcessor(); assertEquals(BigInteger.valueOf(21), bm.process(null)); @@ -107,7 +110,7 @@ public void testDiv() { assertEquals(2.33, ((Number) ba.process(null)).doubleValue(), 0.01d); } - public void testUnsignedLongDiv() { + public void testDivUnsignedLong() { Processor bd = new Div(EMPTY, l(BigInteger.valueOf(7)), l(3)).makePipe().asProcessor(); assertEquals(BigInteger.TWO, bd.process(null)); @@ -126,7 +129,7 @@ public void testMod() { assertEquals(1, ba.process(null)); } - public void testUnsignedLongMod() { + public void testModUnsignedLong() { Processor bm = new Mod(EMPTY, l(BigInteger.valueOf(7)), l(3)).makePipe().asProcessor(); assertEquals(BigInteger.valueOf(1), bm.process(null)); @@ -139,7 +142,7 @@ public void testNegate() { assertEquals(-7, ba.process(null)); } - public void testUnsignedLongNegate() { + public void testNegateUnsignedLong() { Processor nm = new Neg(EMPTY, l(BigInteger.valueOf(0))).makePipe().asProcessor(); assertEquals(BigInteger.ZERO, nm.process(null)); @@ -160,7 +163,7 @@ public void testTree() { } // ((3*2+4)/2-2)%2 - public void testUnsignedLongTree() { + public void testTreeUnsignedLong() { Expression mul = new Mul(EMPTY, l(3), l(BigInteger.TWO)); Expression add = new Add(EMPTY, mul, l(4)); Expression div = new Div(EMPTY, add, l(2)); diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java index 3d3f9ead14650..6f7e9f571c07e 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java @@ -32,7 +32,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; import static org.elasticsearch.xpack.ql.type.DateUtils.asDateTime; -import static org.elasticsearch.xpack.ql.util.Check.UNSIGNED_LONG_MAX; +import static org.elasticsearch.xpack.ql.util.UnsignedLongUtils.UNSIGNED_LONG_MAX; public class DataTypeConversionTests extends ESTestCase { diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatement.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatement.java index 4b1f7677149c2..cd7cf6f8ebda0 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatement.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatement.java @@ -40,9 +40,8 @@ import java.util.List; import java.util.Locale; -import static java.sql.Types.BIGINT; import static java.time.ZoneOffset.UTC; -import static org.elasticsearch.xpack.sql.jdbc.TypeUtils.scaleOrLenght; +import static org.elasticsearch.xpack.sql.jdbc.TypeUtils.scaleOrLength; class JdbcPreparedStatement extends JdbcStatement implements PreparedStatement { @@ -114,7 +113,7 @@ public void setInt(int parameterIndex, int x) throws SQLException { @Override public void setLong(int parameterIndex, long x) throws SQLException { - setObject(parameterIndex, x, BIGINT); + setObject(parameterIndex, x, Types.BIGINT); } @Override @@ -207,7 +206,7 @@ public void setObject(int parameterIndex, Object x) throws SQLException { // {@code java.sql.Array} etc) will generate the correct exception message. Otherwise, the method call // {@code TypeConverter.fromJavaToJDBC(x.getClass())} will report the implementing class as not being supported. checkKnownUnsupportedTypes(x); - setObject(parameterIndex, x, TypeUtils.of(x.getClass()).getVendorTypeNumber(), scaleOrLenght(x)); + setObject(parameterIndex, x, TypeUtils.of(x.getClass()).getVendorTypeNumber(), scaleOrLength(x)); } diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java index 4b5dd65ca62ea..6c7f56d296b00 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java @@ -350,11 +350,7 @@ private static Byte asByte(Object val, EsType columnType, String typeString) thr case LONG: return safeToByte(((Number) val).longValue()); case UNSIGNED_LONG: - try { - return ((BigInteger) val).byteValueExact(); - } catch (ArithmeticException ae) { - return failConversion(val, columnType, typeString, Byte.class, ae); - } + return safeToByte(((BigInteger) val)); case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -383,11 +379,7 @@ private static Short asShort(Object val, EsType columnType, String typeString) t case LONG: return safeToShort(((Number) val).longValue()); case UNSIGNED_LONG: - try { - return ((BigInteger) val).shortValueExact(); - } catch (ArithmeticException ae) { - return failConversion(val, columnType, typeString, Short.class, ae); - } + return safeToShort((BigInteger) val); case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -415,11 +407,7 @@ private static Integer asInteger(Object val, EsType columnType, String typeStrin case LONG: return safeToInt(((Number) val).longValue()); case UNSIGNED_LONG: - try { - return ((BigInteger) val).intValueExact(); - } catch (ArithmeticException ae) { - return failConversion(val, columnType, typeString, Integer.class, ae); - } + return safeToInt((BigInteger) val); case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -447,11 +435,7 @@ private static Long asLong(Object val, EsType columnType, String typeString) thr case LONG: return Long.valueOf(((Number) val).longValue()); case UNSIGNED_LONG: - try { - return ((BigInteger) val).longValueExact(); - } catch (ArithmeticException ae) { - return failConversion(val, columnType, typeString, Long.class, ae); - } + return safeToLong((BigInteger) val); case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -647,28 +631,60 @@ private static OffsetDateTime asOffsetDateTime(Object val, EsType columnType, St throw new SQLFeatureNotSupportedException(); } - private static byte safeToByte(long x) throws SQLException { + private static byte safeToByte(Number n) throws SQLException { + if (n instanceof BigInteger) { + try { + return ((BigInteger) n).byteValueExact(); + } catch (ArithmeticException ae) { + throw new SQLException(format(Locale.ROOT, "Numeric %s out of range", n)); + } + } + long x = n.longValue(); if (x > Byte.MAX_VALUE || x < Byte.MIN_VALUE) { - throw new SQLException(format(Locale.ROOT, "Numeric %s out of range", Long.toString(x))); + throw new SQLException(format(Locale.ROOT, "Numeric %s out of range", n)); } return (byte) x; } - private static short safeToShort(long x) throws SQLException { + private static short safeToShort(Number n) throws SQLException { + if (n instanceof BigInteger) { + try { + return ((BigInteger) n).shortValueExact(); + } catch (ArithmeticException ae) { + throw new SQLException(format(Locale.ROOT, "Numeric %s out of range", n)); + } + } + long x = n.longValue(); if (x > Short.MAX_VALUE || x < Short.MIN_VALUE) { - throw new SQLException(format(Locale.ROOT, "Numeric %s out of range", Long.toString(x))); + throw new SQLException(format(Locale.ROOT, "Numeric %s out of range", n)); } return (short) x; } - private static int safeToInt(long x) throws SQLException { + private static int safeToInt(Number n) throws SQLException { + if (n instanceof BigInteger) { + try { + return ((BigInteger) n).intValueExact(); + } catch (ArithmeticException ae) { + throw new SQLException(format(Locale.ROOT, "Numeric %s out of range", n)); + } + } + long x = n.longValue(); if (x > Integer.MAX_VALUE || x < Integer.MIN_VALUE) { - throw new SQLException(format(Locale.ROOT, "Numeric %s out of range", Long.toString(x))); + throw new SQLException(format(Locale.ROOT, "Numeric %s out of range", n)); } return (int) x; } - private static long safeToLong(double x) throws SQLException { + private static long safeToLong(Number n) throws SQLException { + if (n instanceof BigInteger) { + try { + return ((BigInteger) n).longValueExact(); + } catch (ArithmeticException ae) { + throw new SQLException(format(Locale.ROOT, "Numeric %s out of range", n)); + } + } + double x = n.doubleValue(); if (x > Long.MAX_VALUE || x < Long.MIN_VALUE) { throw new SQLException(format(Locale.ROOT, "Numeric %s out of range", Double.toString(x))); } diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java index 7d7c9ac00655e..d9195053ceafc 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java @@ -41,9 +41,10 @@ private TypeUtils() {} EsType.SHORT, EsType.INTEGER, EsType.LONG, EsType.FLOAT, EsType.HALF_FLOAT, EsType.SCALED_FLOAT, EsType.DOUBLE, EsType.DATETIME); - public static final int LONG_MAX_LENGTH = String.valueOf(Long.MAX_VALUE).length(); /* type length value as defined in ES */ + public static final int LONG_MAX_LENGTH = String.valueOf(Long.MAX_VALUE).length(); // type length value as defined in ES static { + // Note: keep in sync with org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils#CLASS_TO_ES_TYPE Map, EsType> aMap = new LinkedHashMap<>(); aMap.put(Boolean.class, EsType.BOOLEAN); aMap.put(Byte.class, EsType.BYTE); @@ -190,7 +191,7 @@ static EsType of(Class clazz) throws SQLException { return dataType; } - static int scaleOrLenght(Object val) { + static int scaleOrLength(Object val) { return val instanceof BigInteger ? LONG_MAX_LENGTH + 1 : 0; } } diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java index 49cec0e8043d3..352bf3020ae84 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java @@ -106,7 +106,6 @@ private Object convertAsNative(Object value, EsType type) throws Exception { } private Object convertAsNative(Object value, Class nativeType) throws Exception { - Object copy = throughXContent(value); EsType esType = TypeUtils.of(value.getClass()); return TypeConverter.convert(value, esType, nativeType, esType.toString()); } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java index 33e856dc62c25..a23a9883cc729 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java @@ -38,6 +38,18 @@ private JdbcTestUtils() {} private static final String DRIVER_VERSION_PROPERTY_NAME = "jdbc.driver.version"; static final LocalDate EPOCH = LocalDate.of(1970, 1, 1); + static final String UNSIGNED_LONG_TYPE_NAME = "UNSIGNED_LONG"; + public static final BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); + + /* + * The version of the driver that the QA (bwc-)tests run against. + * Note: when adding a version-gated feature (i.e. new feature that would not be supported by old drivers) and add code in these QA + * tests to check the feature, you'll want to compare the target release version of the feature against this variable, to selectively + * run the new tests only for drivers that will support the feature (i.e. of the target release version and newer). The check would + * look like if (TARGET_VERSION.compareTo(JDBC_DRIVER_VERSION) <= 0) {run_tests();} (see {@code isUnsignedLongSupported}. + * However, until the feature + QA tests are actually ported to the target branch, the comparison will hold true only for the master + * branch. So you'll need to remove the equality, port the feature and subsequently add the equality; i.e. a two-step commit. + */ static final Version JDBC_DRIVER_VERSION; static { @@ -45,6 +57,7 @@ private JdbcTestUtils() {} String jdbcDriverVersion = System.getProperty(DRIVER_VERSION_PROPERTY_NAME, "").replace("-SNAPSHOT", ""); JDBC_DRIVER_VERSION = Version.fromString(jdbcDriverVersion); // takes empty and null strings, resolves them to CURRENT + // Note: keep in sync with org.elasticsearch.xpack.sql.jdbc.TypeUtils#CLASS_TO_TYPE Map, EsType> aMap = new LinkedHashMap<>(); aMap.put(Boolean.class, EsType.BOOLEAN); aMap.put(Byte.class, EsType.BYTE); diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java index b545e7a6bd5d2..a372446758752 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java @@ -37,15 +37,14 @@ import static org.elasticsearch.xpack.sql.jdbc.EsType.KEYWORD; import static org.elasticsearch.xpack.sql.jdbc.EsType.LONG; import static org.elasticsearch.xpack.sql.jdbc.EsType.SHORT; +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.JDBC_DRIVER_VERSION; +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.UNSIGNED_LONG_TYPE_NAME; import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.isUnsignedLongSupported; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.startsWith; public abstract class PreparedStatementTestCase extends JdbcIntegrationTestCase { - static final BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); - EsType UNSIGNED_LONG = isUnsignedLongSupported() ? EsType.valueOf("UNSIGNED_LONG") : null; - public void testSupportedTypes() throws SQLException { String stringVal = randomAlphaOfLength(randomIntBetween(0, 1000)); int intVal = randomInt(); @@ -56,7 +55,6 @@ public void testSupportedTypes() throws SQLException { byte byteVal = randomByte(); short shortVal = randomShort(); BigDecimal bigDecimalVal = BigDecimal.valueOf(randomDouble()); - BigInteger bigIntegerVal = bigDecimalVal.abs().remainder(new BigDecimal(UNSIGNED_LONG_MAX)).toBigInteger(); long millis = randomNonNegativeLong(); Calendar calendarVal = Calendar.getInstance(randomTimeZone(), Locale.ROOT); Timestamp timestampVal = new Timestamp(millis); @@ -76,33 +74,30 @@ public void testSupportedTypes() throws SQLException { try (Connection connection = esJdbc()) { StringJoiner sql = new StringJoiner(",", "SELECT ", ""); - for (int i = 0; i < 20; i++) { + for (int i = 0; i < 19; i++) { sql.add("?"); } try (PreparedStatement statement = connection.prepareStatement(sql.toString())) { int index = 1; - statement.setString(index ++, stringVal); - statement.setInt(index ++, intVal); - statement.setLong(index ++, longVal); - statement.setFloat(index ++, floatVal); - statement.setDouble(index ++, doubleVal); - statement.setNull(index ++, JDBCType.DOUBLE.getVendorTypeNumber()); - statement.setBoolean(index ++, booleanVal); - statement.setByte(index ++, byteVal); - statement.setShort(index ++, shortVal); - statement.setBigDecimal(index ++, bigDecimalVal); - statement.setTimestamp(index ++, timestampVal); - statement.setTimestamp(index ++, timestampVal, calendarVal); - statement.setDate(index ++, dateVal); - statement.setDate(index ++, dateVal, calendarVal); - statement.setTime(index ++, timeVal); - statement.setTime(index ++, timeVal, calendarVal); - statement.setObject(index ++, calendarVal); - statement.setObject(index ++, utilDateVal); - statement.setObject(index ++, localDateTimeVal); - if (isUnsignedLongSupported()) { - statement.setObject(index++, bigIntegerVal); - } + statement.setString(index++, stringVal); + statement.setInt(index++, intVal); + statement.setLong(index++, longVal); + statement.setFloat(index++, floatVal); + statement.setDouble(index++, doubleVal); + statement.setNull(index++, JDBCType.DOUBLE.getVendorTypeNumber()); + statement.setBoolean(index++, booleanVal); + statement.setByte(index++, byteVal); + statement.setShort(index++, shortVal); + statement.setBigDecimal(index++, bigDecimalVal); + statement.setTimestamp(index++, timestampVal); + statement.setTimestamp(index++, timestampVal, calendarVal); + statement.setDate(index++, dateVal); + statement.setDate(index++, dateVal, calendarVal); + statement.setTime(index++, timeVal); + statement.setTime(index++, timeVal, calendarVal); + statement.setObject(index++, calendarVal); + statement.setObject(index++, utilDateVal); + statement.setObject(index++, localDateTimeVal); try (ResultSet results = statement.executeQuery()) { ResultSetMetaData resultSetMetaData = results.getMetaData(); @@ -114,28 +109,25 @@ public void testSupportedTypes() throws SQLException { } assertTrue(results.next()); index = 1; - assertEquals(stringVal, results.getString(index ++)); - assertEquals(intVal, results.getInt(index ++)); - assertEquals(longVal, results.getLong(index ++)); - assertEquals(floatVal, results.getFloat(index ++), 0.00001f); - assertEquals(doubleVal, results.getDouble(index ++), 0.00001f); - assertNull(results.getObject(index ++)); - assertEquals(booleanVal, results.getBoolean(index ++)); - assertEquals(byteVal, results.getByte(index ++)); - assertEquals(shortVal, results.getShort(index ++)); - assertEquals(bigDecimalVal, results.getBigDecimal(index ++)); - assertEquals(timestampVal, results.getTimestamp(index ++)); - assertEquals(timestampValWithCal, results.getTimestamp(index ++)); - assertEquals(dateVal, results.getDate(index ++)); - assertEquals(dateValWithCal, results.getDate(index ++)); - assertEquals(timeVal, results.getTime(index ++)); - assertEquals(timeValWithCal, results.getTime(index ++)); - assertEquals(new Timestamp(calendarVal.getTimeInMillis()), results.getObject(index ++)); - assertEquals(timestampVal, results.getObject(index ++)); - assertEquals(timestampVal, results.getObject(index ++)); - if (isUnsignedLongSupported()) { - assertEquals(bigIntegerVal, results.getObject(index++)); - } + assertEquals(stringVal, results.getString(index++)); + assertEquals(intVal, results.getInt(index++)); + assertEquals(longVal, results.getLong(index++)); + assertEquals(floatVal, results.getFloat(index++), 0.00001f); + assertEquals(doubleVal, results.getDouble(index++), 0.00001f); + assertNull(results.getObject(index++)); + assertEquals(booleanVal, results.getBoolean(index++)); + assertEquals(byteVal, results.getByte(index++)); + assertEquals(shortVal, results.getShort(index++)); + assertEquals(bigDecimalVal, results.getBigDecimal(index++)); + assertEquals(timestampVal, results.getTimestamp(index++)); + assertEquals(timestampValWithCal, results.getTimestamp(index++)); + assertEquals(dateVal, results.getDate(index++)); + assertEquals(dateValWithCal, results.getDate(index++)); + assertEquals(timeVal, results.getTime(index++)); + assertEquals(timeValWithCal, results.getTime(index++)); + assertEquals(new Timestamp(calendarVal.getTimeInMillis()), results.getObject(index++)); + assertEquals(timestampVal, results.getObject(index++)); + assertEquals(timestampVal, results.getObject(index++)); assertFalse(results.next()); } } @@ -382,16 +374,26 @@ public void testSingleParameterMultipleTypes() throws SQLException { assertEquals(new Tuple<>(BYTE.getName(), byteVal), execute(statement)); statement.setShort(1, shortVal); assertEquals(new Tuple<>(SHORT.getName(), shortVal), execute(statement)); + } + } + } - if (isUnsignedLongSupported()) { - statement.setObject(1, bigIntegerVal); - assertEquals(new Tuple<>(UNSIGNED_LONG.getName(), bigIntegerVal), execute(statement)); + public void testSingleParameterUnsignedLong() throws SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); - statement.setObject(1, longVal, JDBCType.BIGINT, 19); - assertEquals(new Tuple<>(LONG.getName(), longVal), execute(statement)); - statement.setObject(1, Math.abs(longVal), JDBCType.BIGINT, 20); - assertEquals(new Tuple<>(UNSIGNED_LONG.getName(), BigInteger.valueOf(Math.abs(longVal))), execute(statement)); - } + BigInteger bigIntegerVal = randomBigInteger(); + long longVal = randomLong(); + + try (Connection connection = esJdbc()) { + try (PreparedStatement statement = connection.prepareStatement("SELECT ?")) { + + statement.setObject(1, bigIntegerVal); + assertEquals(new Tuple<>(UNSIGNED_LONG_TYPE_NAME, bigIntegerVal), execute(statement)); + + statement.setObject(1, longVal, JDBCType.BIGINT, 19); + assertEquals(new Tuple<>(LONG.getName(), longVal), execute(statement)); + statement.setObject(1, Math.abs(longVal), JDBCType.BIGINT, 20); + assertEquals(new Tuple<>(UNSIGNED_LONG_TYPE_NAME, BigInteger.valueOf(Math.abs(longVal))), execute(statement)); } } } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java index 9bd5e28b9d158..90ba593b0ef91 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java @@ -14,34 +14,31 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.stream.Collectors; +import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.JDBC_DRIVER_VERSION; +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.UNSIGNED_LONG_TYPE_NAME; import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.isUnsignedLongSupported; public abstract class ResultSetMetaDataTestCase extends JdbcIntegrationTestCase { - private static final List fieldsNames = new ArrayList<>(); + private static final List FIELDS_NAMES = List.of( + "test_byte", + "test_integer", + "test_long", + "test_short", + "test_double", + "test_float", + "test_keyword", + "test_boolean", + "test_date" + ); + private static final String UNSIGNED_LONG_FIELD = "test_" + UNSIGNED_LONG_TYPE_NAME.toLowerCase(Locale.ROOT); - static { - fieldsNames.addAll(List.of( - "test_byte", - "test_integer", - "test_long", - "test_short", - "test_double", - "test_float", - "test_keyword", - "test_boolean", - "test_date" - )); - if (isUnsignedLongSupported()) { - fieldsNames.add("test_unsigned_long"); - } - } - - public void testValidGetObjectCalls() throws IOException, SQLException { + public void doTestValidGetObjectCalls(List fieldsNames) throws IOException, SQLException { ResultSetTestCase.createIndex("test"); ResultSetTestCase.updateMapping("test", builder -> { for (String field : fieldsNames) { @@ -53,16 +50,26 @@ public void testValidGetObjectCalls() throws IOException, SQLException { doWithQuery(q, r -> assertColumnNamesAndLabels(r.getMetaData(), fieldsNames)); - q = "SELECT " + fieldsNames.stream().map(x -> x + " AS " + x.replace("_", "")).collect(Collectors.joining(", ")) + " FROM test"; + String selectedFields = fieldsNames.stream().map(x -> x + " AS " + x.replace("_", "")).collect(Collectors.joining(", ")); + q = "SELECT " + selectedFields + " FROM test"; doWithQuery(q, r -> assertColumnNamesAndLabels(r.getMetaData(), fieldsNames.stream() .map(x -> x.replace("_", "")).collect(Collectors.toList()))); } + public void testValidGetObjectCalls() throws IOException, SQLException { + doTestValidGetObjectCalls(FIELDS_NAMES); + } + + public void testValidGetObjectCallsWithUnsignedLong() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); + + doTestValidGetObjectCalls(singletonList(UNSIGNED_LONG_FIELD)); + } + private void doWithQuery(String query, CheckedConsumer consumer) throws SQLException { try (Connection connection = esJdbc()) { try (PreparedStatement statement = connection.prepareStatement(query)) { try (ResultSet results = statement.executeQuery()) { - assertEquals(fieldsNames.size(), results.getMetaData().getColumnCount()); consumer.accept(results); } } @@ -70,7 +77,8 @@ private void doWithQuery(String query, CheckedConsumer } private void assertColumnNamesAndLabels(ResultSetMetaData metaData, List names) throws SQLException { - for (int i = 0; i < fieldsNames.size(); i++) { + assertEquals(names.size(), metaData.getColumnCount()); + for (int i = 0; i < names.size(); i++) { assertEquals(names.get(i), metaData.getColumnName(i + 1)); assertEquals(names.get(i), metaData.getColumnLabel(i + 1)); } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java index 5de0b571233ad..32005ffd1eebe 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java @@ -39,7 +39,6 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; -import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Date; @@ -64,35 +63,32 @@ import static java.util.Calendar.MONTH; import static java.util.Calendar.SECOND; import static java.util.Calendar.YEAR; +import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.JDBC_DRIVER_VERSION; +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.UNSIGNED_LONG_MAX; +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.UNSIGNED_LONG_TYPE_NAME; import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.isUnsignedLongSupported; import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.of; public abstract class ResultSetTestCase extends JdbcIntegrationTestCase { - static final List fieldsNames = new ArrayList<>(); + static final List FIELDS_NAMES = List.of( + "test_byte", + "test_integer", + "test_long", + "test_short", + "test_double", + "test_float", + "test_keyword" + ); + static final String UNSIGNED_LONG_FIELD = "test_" + UNSIGNED_LONG_TYPE_NAME.toLowerCase(Locale.ROOT); + static final Map, SQLType> dateTimeTestingFields = new HashMap<>(); static final String SELECT_ALL_FIELDS; static final String SELECT_WILDCARD = "SELECT * FROM test"; - static final EsType UNSIGNED_LONG; static { - fieldsNames.addAll(List.of( - "test_byte", - "test_integer", - "test_long", - "test_short", - "test_double", - "test_float", - "test_keyword" - )); - if (isUnsignedLongSupported()) { - fieldsNames.add("test_unsigned_long"); - } - - SELECT_ALL_FIELDS = "SELECT test_boolean, " + String.join(", ", fieldsNames) + ", test_date FROM test"; - - UNSIGNED_LONG = isUnsignedLongSupported() ? EsType.valueOf("UNSIGNED_LONG") : null; + SELECT_ALL_FIELDS = "SELECT test_boolean, " + String.join(", ", FIELDS_NAMES) + ", test_date FROM test"; dateTimeTestingFields.put(new Tuple<>("test_boolean", true), EsType.BOOLEAN); dateTimeTestingFields.put(new Tuple<>("test_byte", 1), EsType.BYTE); @@ -102,9 +98,6 @@ public abstract class ResultSetTestCase extends JdbcIntegrationTestCase { dateTimeTestingFields.put(new Tuple<>("test_double", 1d), EsType.DOUBLE); dateTimeTestingFields.put(new Tuple<>("test_float", 1f), EsType.FLOAT); dateTimeTestingFields.put(new Tuple<>("test_keyword", "true"), EsType.KEYWORD); - if (isUnsignedLongSupported()) { - dateTimeTestingFields.put(new Tuple<>("test_unsigned_long", BigInteger.ONE), UNSIGNED_LONG); - } } private String timeZoneId; @@ -234,22 +227,6 @@ public void testGettingValidByteWithCasting() throws IOException, SQLException { }); } - static void indexTestFieldsDoc(String index, String docId, int i, long l, short s, double d, float f, String k, long date, - BigInteger bi) throws IOException { - index(index, "1", builder -> { - builder.field("test_integer", i); - builder.field("test_long", l); - builder.field("test_short", s); - builder.field("test_double", d); - builder.field("test_float", f); - builder.field("test_keyword", k); - builder.field("test_date", date); - if (isUnsignedLongSupported()) { - builder.field("test_unsigned_long", bi); - } - }); - } - public void testGettingInvalidByte() throws IOException, SQLException { createIndex("test"); updateMappingForNumericValuesTests("test"); @@ -258,12 +235,11 @@ public void testGettingInvalidByte() throws IOException, SQLException { builder.startObject("test_date").field("type", "date").endObject(); }); - int intNotByte = randomIntBetween(Byte.MAX_VALUE + 1, Integer.MAX_VALUE); + Integer intNotByte = randomIntBetween(Byte.MAX_VALUE + 1, Integer.MAX_VALUE); long longNotByte = randomLongBetween(Byte.MAX_VALUE + 1, Long.MAX_VALUE); - short shortNotByte = (short) randomIntBetween(Byte.MAX_VALUE + 1, Short.MAX_VALUE); + Short shortNotByte = (short) randomIntBetween(Byte.MAX_VALUE + 1, Short.MAX_VALUE); double doubleNotByte = randomDoubleBetween(Byte.MAX_VALUE + 1, Double.MAX_VALUE, true); float floatNotByte = randomFloatBetween(Byte.MAX_VALUE + 1, Float.MAX_VALUE); - BigInteger bigIntegerNotByte = BigInteger.valueOf(longNotByte); String randomString = randomUnicodeOfCodepointLengthBetween(128, 256); long randomDate = randomNonNegativeLong(); @@ -271,8 +247,8 @@ public void testGettingInvalidByte() throws IOException, SQLException { ? Double.toString(doubleNotByte) : Long.toString(Math.round(doubleNotByte)); - indexTestFieldsDoc("test", "1", intNotByte, longNotByte, shortNotByte, doubleNotByte, floatNotByte, randomString, randomDate, - bigIntegerNotByte); + indexTestFieldsDoc("1", intNotByte, longNotByte, shortNotByte, doubleNotByte, floatNotByte, randomString, + new Date(randomDate)); doWithQuery(SELECT_WILDCARD, results -> { results.next(); @@ -323,15 +299,25 @@ public void testGettingInvalidByte() throws IOException, SQLException { format(Locale.ROOT, "Unable to convert value [%.128s] of type [DATETIME] to [Byte]", asDateString(randomDate)), sqle.getMessage() ); + }); + } - if (isUnsignedLongSupported()) { - sqle = expectThrows(SQLException.class, () -> results.getByte("test_unsigned_long")); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Byte]", bigIntegerNotByte), - sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Byte.class)); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Byte]", bigIntegerNotByte), - sqle.getMessage()); - } + public void testGettingInvalidByteFromUnsignedLong() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); + + createIndex("test"); + updateMappingForNumericValuesTests("test", singletonList(UNSIGNED_LONG_FIELD)); + + BigInteger bigIntegerNotByte = BigInteger.valueOf(randomLongBetween(Byte.MAX_VALUE + 1, Long.MAX_VALUE)); + indexTestFieldsDoc("1", bigIntegerNotByte); + + doWithQuery(SELECT_WILDCARD, results -> { + results.next(); + + SQLException sqle = expectThrows(SQLException.class, () -> results.getByte(UNSIGNED_LONG_FIELD)); + assertEquals(format(Locale.ROOT, "Numeric %s out of range", bigIntegerNotByte), sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getObject(UNSIGNED_LONG_FIELD, Byte.class)); + assertEquals(format(Locale.ROOT, "Numeric %s out of range", bigIntegerNotByte), sqle.getMessage()); }); } @@ -394,7 +380,6 @@ public void testGettingInvalidShort() throws IOException, SQLException { createIndex("test"); updateMappingForNumericValuesTests("test"); updateMapping("test", builder -> { - builder.startObject("test_keyword").field("type", "keyword").endObject(); builder.startObject("test_date").field("type", "date").endObject(); }); @@ -402,7 +387,6 @@ public void testGettingInvalidShort() throws IOException, SQLException { long longNotShort = randomLongBetween(Short.MAX_VALUE + 1, Long.MAX_VALUE); double doubleNotShort = randomDoubleBetween(Short.MAX_VALUE + 1, Double.MAX_VALUE, true); float floatNotShort = randomFloatBetween(Short.MAX_VALUE + 1, Float.MAX_VALUE); - BigInteger bigIntegerNotShort = BigInteger.valueOf(longNotShort); String randomString = randomUnicodeOfCodepointLengthBetween(128, 256); long randomDate = randomNonNegativeLong(); @@ -410,8 +394,7 @@ public void testGettingInvalidShort() throws IOException, SQLException { ? Double.toString(doubleNotShort) : Long.toString(Math.round(doubleNotShort)); - indexTestFieldsDoc("test", "1", intNotShort, longNotShort, (short) 0, doubleNotShort, floatNotShort, randomString, randomDate, - bigIntegerNotShort); + indexTestFieldsDoc("1", intNotShort, longNotShort, doubleNotShort, floatNotShort, randomString, new Date(randomDate)); doWithQuery(SELECT_WILDCARD, results -> { results.next(); @@ -457,15 +440,25 @@ public void testGettingInvalidShort() throws IOException, SQLException { format(Locale.ROOT, "Unable to convert value [%.128s] of type [DATETIME] to [Short]", asDateString(randomDate)), sqle.getMessage() ); + }); + } - if (isUnsignedLongSupported()) { - sqle = expectThrows(SQLException.class, () -> results.getShort("test_unsigned_long")); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Short]", bigIntegerNotShort), - sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Short.class)); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Short]", bigIntegerNotShort), - sqle.getMessage()); - } + public void testGettingInvalidShortFromUnsignedLong() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); + + createIndex("test"); + updateMappingForNumericValuesTests("test", singletonList(UNSIGNED_LONG_FIELD)); + + BigInteger bigIntegerNotShort = BigInteger.valueOf(randomLongBetween(Short.MAX_VALUE + 1, Long.MAX_VALUE)); + indexTestFieldsDoc("1", bigIntegerNotShort); + + doWithQuery(SELECT_WILDCARD, results -> { + results.next(); + + SQLException sqle = expectThrows(SQLException.class, () -> results.getShort(UNSIGNED_LONG_FIELD)); + assertEquals(format(Locale.ROOT, "Numeric %s out of range", bigIntegerNotShort), sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getObject(UNSIGNED_LONG_FIELD, Short.class)); + assertEquals(format(Locale.ROOT, "Numeric %s out of range", bigIntegerNotShort), sqle.getMessage()); }); } @@ -528,14 +521,12 @@ public void testGettingInvalidInteger() throws IOException, SQLException { createIndex("test"); updateMappingForNumericValuesTests("test"); updateMapping("test", builder -> { - builder.startObject("test_keyword").field("type", "keyword").endObject(); builder.startObject("test_date").field("type", "date").endObject(); }); long longNotInt = randomLongBetween(getMaxIntPlusOne(), Long.MAX_VALUE); double doubleNotInt = randomDoubleBetween(getMaxIntPlusOne().doubleValue(), Double.MAX_VALUE, true); float floatNotInt = randomFloatBetween(getMaxIntPlusOne().floatValue(), Float.MAX_VALUE); - BigInteger bigIntegerNotInt = BigInteger.valueOf(longNotInt); String randomString = randomUnicodeOfCodepointLengthBetween(128, 256); long randomDate = randomNonNegativeLong(); @@ -543,7 +534,7 @@ public void testGettingInvalidInteger() throws IOException, SQLException { ? Double.toString(doubleNotInt) : Long.toString(Math.round(doubleNotInt)); - indexTestFieldsDoc("test", "1", 0, longNotInt, (short) 0, doubleNotInt, floatNotInt, randomString, randomDate, bigIntegerNotInt); + indexTestFieldsDoc("1", longNotInt, doubleNotInt, floatNotInt, randomString, new Date(randomDate)); doWithQuery(SELECT_WILDCARD, results -> { results.next(); @@ -584,15 +575,25 @@ public void testGettingInvalidInteger() throws IOException, SQLException { format(Locale.ROOT, "Unable to convert value [%.128s] of type [DATETIME] to [Integer]", asDateString(randomDate)), sqle.getMessage() ); + }); + } - if (isUnsignedLongSupported()) { - sqle = expectThrows(SQLException.class, () -> results.getInt("test_unsigned_long")); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Integer]", bigIntegerNotInt), - sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Integer.class)); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Integer]", bigIntegerNotInt), - sqle.getMessage()); - } + public void testGettingInvalidIntegerFromUnsignedLong() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); + + createIndex("test"); + updateMappingForNumericValuesTests("test", singletonList(UNSIGNED_LONG_FIELD)); + + BigInteger bigIntegerNotInt = BigInteger.valueOf(randomLongBetween(getMaxIntPlusOne(), Long.MAX_VALUE)); + indexTestFieldsDoc("1", bigIntegerNotInt); + + doWithQuery(SELECT_WILDCARD, results -> { + results.next(); + + SQLException sqle = expectThrows(SQLException.class, () -> results.getInt(UNSIGNED_LONG_FIELD)); + assertEquals(format(Locale.ROOT, "Numeric %s out of range", bigIntegerNotInt), sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getObject(UNSIGNED_LONG_FIELD, Integer.class)); + assertEquals(format(Locale.ROOT, "Numeric %s out of range", bigIntegerNotInt), sqle.getMessage()); }); } @@ -652,17 +653,15 @@ public void testGettingInvalidLong() throws IOException, SQLException { createIndex("test"); updateMappingForNumericValuesTests("test"); updateMapping("test", builder -> { - builder.startObject("test_keyword").field("type", "keyword").endObject(); builder.startObject("test_date").field("type", "date").endObject(); }); double doubleNotLong = randomDoubleBetween(getMaxLongPlusOne(), Double.MAX_VALUE, true); float floatNotLong = randomFloatBetween(getMaxLongPlusOne().floatValue(), Float.MAX_VALUE); - BigInteger bigIntegerNotLong = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.valueOf(randomNonNegativeLong())); String randomString = randomUnicodeOfCodepointLengthBetween(128, 256); long randomDate = randomNonNegativeLong(); - indexTestFieldsDoc("test", "1", 0, 0L, (short) 0, doubleNotLong, floatNotLong, randomString, randomDate, bigIntegerNotLong); + indexTestFieldsDoc("1", doubleNotLong, floatNotLong, randomString, new Date(randomDate)); doWithQuery(SELECT_WILDCARD, results -> { results.next(); @@ -698,15 +697,25 @@ public void testGettingInvalidLong() throws IOException, SQLException { format(Locale.ROOT, "Unable to convert value [%.128s] of type [DATETIME] to [Long]", asDateString(randomDate)), sqle.getMessage() ); + }); + } - if (isUnsignedLongSupported()) { - sqle = expectThrows(SQLException.class, () -> results.getLong("test_unsigned_long")); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Long]", bigIntegerNotLong), - sqle.getMessage()); - sqle = expectThrows(SQLException.class, () -> results.getObject("test_unsigned_long", Long.class)); - assertEquals(format(Locale.ROOT, "Unable to convert value [%s] of type [UNSIGNED_LONG] to [Long]", bigIntegerNotLong), - sqle.getMessage()); - } + public void testGettingInvalidLongFromUnsignedLong() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); + + createIndex("test"); + updateMappingForNumericValuesTests("test", singletonList(UNSIGNED_LONG_FIELD)); + + BigInteger bigIntegerNotLong = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.valueOf(randomNonNegativeLong())); + indexTestFieldsDoc("1", bigIntegerNotLong); + + doWithQuery(SELECT_WILDCARD, results -> { + results.next(); + + SQLException sqle = expectThrows(SQLException.class, () -> results.getLong(UNSIGNED_LONG_FIELD)); + assertEquals(format(Locale.ROOT, "Numeric %s out of range", bigIntegerNotLong), sqle.getMessage()); + sqle = expectThrows(SQLException.class, () -> results.getObject(UNSIGNED_LONG_FIELD, Long.class)); + assertEquals(format(Locale.ROOT, "Numeric %s out of range", bigIntegerNotLong), sqle.getMessage()); }); } @@ -724,13 +733,13 @@ public void testGettingValidBigIntegerWithoutCasting() throws IOException, SQLEx results.next(); assertEquals(3, resultSetMetaData.getColumnCount()); - assertEquals(UNSIGNED_LONG.getName(), resultSetMetaData.getColumnTypeName(1)); + assertEquals(UNSIGNED_LONG_TYPE_NAME, resultSetMetaData.getColumnTypeName(1)); assertEquals(random1, results.getObject(1)); - assertEquals(random1, results.getObject("test_unsigned_long")); - assertEquals(random1, results.getObject("test_unsigned_long", BigInteger.class)); + assertEquals(random1, results.getObject(UNSIGNED_LONG_FIELD)); + assertEquals(random1, results.getObject(UNSIGNED_LONG_FIELD, BigInteger.class)); assertTrue(results.getObject(1) instanceof BigInteger); - assertEquals(UNSIGNED_LONG.getName(), resultSetMetaData.getColumnTypeName(2)); + assertEquals(UNSIGNED_LONG_TYPE_NAME, resultSetMetaData.getColumnTypeName(2)); assertNull(results.getObject(2)); assertTrue(results.wasNull()); assertNull(results.getObject("test_null_unsigned_long")); @@ -738,7 +747,7 @@ public void testGettingValidBigIntegerWithoutCasting() throws IOException, SQLEx assertTrue(results.next()); assertEquals(random2, results.getObject(1)); - assertEquals(random2, results.getObject("test_unsigned_long")); + assertEquals(random2, results.getObject(UNSIGNED_LONG_FIELD)); assertTrue(results.getObject(1) instanceof BigInteger); assertEquals(random3.toString(), results.getObject("test_keyword")); @@ -750,6 +759,10 @@ public void testGettingValidBigIntegerWithCasting() throws IOException, SQLExcep assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); Map map = createTestDataForNumericValueTypes(ESTestCase::randomBigInteger); + updateMappingForNumericValuesTests("test", singletonList(UNSIGNED_LONG_FIELD)); + + BigInteger randomBigInteger = randomBigInteger(); + indexTestFieldsDoc("2", randomBigInteger); doWithQuery(SELECT_WILDCARD, results -> { results.next(); @@ -764,6 +777,10 @@ public void testGettingValidBigIntegerWithCasting() throws IOException, SQLExcep assertEquals("For field " + e.getKey(), e.getValue().longValue(), actual.longValue()); } } + + results.next(); + BigInteger actual = results.getObject(UNSIGNED_LONG_FIELD, BigInteger.class); + assertEquals("For field " + UNSIGNED_LONG_FIELD, randomBigInteger, actual); }); } @@ -802,6 +819,45 @@ public void testGettingInvalidBigInteger() throws IOException, SQLException { }); } + public void testGettingValidNumbersWithCastingFromUnsignedLong() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); + + createIndex("test"); + updateMappingForNumericValuesTests("test", singletonList(UNSIGNED_LONG_FIELD)); + + byte randomNonNegativeByte = randomNonNegativeByte(); + short randomNonNegativeShort = (short) (Math.abs(randomShort()) - 1); + int randomNonNegativeInt = Math.abs(randomInt()) - 1; + long randomNonNegativeLong = randomNonNegativeLong(); + double randomNonNegativeFloat = (float) randomDoubleBetween(0, UNSIGNED_LONG_MAX.doubleValue(), true); + double randomNonNegativeDouble = randomDoubleBetween(0, UNSIGNED_LONG_MAX.doubleValue(), true); + + int docId = 1; + indexTestFieldsDoc(String.valueOf(docId++), BigInteger.valueOf(randomNonNegativeByte)); + indexTestFieldsDoc(String.valueOf(docId++), BigInteger.valueOf(randomNonNegativeShort)); + indexTestFieldsDoc(String.valueOf(docId++), BigInteger.valueOf(randomNonNegativeInt)); + indexTestFieldsDoc(String.valueOf(docId++), BigInteger.valueOf(randomNonNegativeLong)); + indexTestFieldsDoc(String.valueOf(docId++), BigDecimal.valueOf(randomNonNegativeFloat).toBigInteger()); + indexTestFieldsDoc(String.valueOf(docId++), BigDecimal.valueOf(randomNonNegativeDouble).toBigInteger()); + + doWithQuery(SELECT_WILDCARD, results -> { + results.next(); + assertEquals(randomNonNegativeByte, results.getByte(1)); + results.next(); + assertEquals(randomNonNegativeShort, results.getShort(1)); + results.next(); + assertEquals(randomNonNegativeInt, results.getInt(1)); + results.next(); + assertEquals(randomNonNegativeLong, results.getLong(1)); + results.next(); + assertEquals(randomNonNegativeFloat, results.getFloat(1), 0f); + results.next(); + assertEquals(randomNonNegativeDouble, results.getDouble(1), 0d); + }); + + } + + // Double values testing public void testGettingValidDoubleWithoutCasting() throws IOException, SQLException { List doubleTestValues = createTestDataForNumericValueTests(ESTestCase::randomDouble); @@ -1214,57 +1270,34 @@ public void testGettingBooleanValues() throws IOException, SQLException { long randomDate2 = randomNonNegativeLong(); // true values - indexSimpleDocumentWithTrueValues(randomDate1); - + indexSimpleDocumentWithBooleanValues("1", true, randomDate1); // false values - index("test", "2", builder -> { - builder.field("test_boolean", false); - builder.field("test_byte", 0); - builder.field("test_integer", 0); - builder.field("test_long", 0L); - builder.field("test_short", 0); - builder.field("test_double", 0d); - builder.field("test_float", 0f); - builder.field("test_keyword", "false"); - builder.field("test_date", randomDate2); - if (isUnsignedLongSupported()) { - builder.field("test_unsigned_long", 0L); - } - }); + indexSimpleDocumentWithBooleanValues("2", false, randomDate2); // other (non 0 = true) values - index("test", "3", builder -> { - builder.field("test_byte", randomValueOtherThan((byte) 0, ESTestCase::randomByte)); - builder.field("test_integer", randomValueOtherThan(0, ESTestCase::randomInt)); - builder.field("test_long", randomValueOtherThan(0L, ESTestCase::randomLong)); - builder.field("test_short", randomValueOtherThan((short) 0, ESTestCase::randomShort)); - builder.field( - "test_double", - randomValueOtherThanMany( - i -> i < 1.0d && i > -1.0d && i < Double.MAX_VALUE && i > Double.MIN_VALUE, - () -> randomDouble() * randomInt() - ) - ); - builder.field( - "test_float", - randomValueOtherThanMany( - i -> i < 1.0f && i > -1.0f && i < Float.MAX_VALUE && i > Float.MIN_VALUE, - () -> randomFloat() * randomInt() - ) - ); - builder.field("test_keyword", "1"); - if (isUnsignedLongSupported()) { - builder.field("test_unsigned_long", randomValueOtherThan(BigInteger.ZERO, ESTestCase::randomBigInteger)); - } - }); + indexTestFieldsDoc("3", + randomValueOtherThan((byte) 0, ESTestCase::randomByte), + randomValueOtherThan(0, ESTestCase::randomInt), + randomValueOtherThan(0L, ESTestCase::randomLong), + randomValueOtherThan((short) 0, ESTestCase::randomShort), + randomValueOtherThanMany( + i -> i < 1.0d && i > -1.0d && i < Double.MAX_VALUE && i > Double.MIN_VALUE, + () -> randomDouble() * randomInt() + ), + randomValueOtherThanMany( + i -> i < 1.0f && i > -1.0f && i < Float.MAX_VALUE && i > Float.MIN_VALUE, + () -> randomFloat() * randomInt() + ), + "1" + ); // other false values index("test", "4", builder -> builder.field("test_keyword", "0")); doWithQuery(SELECT_WILDCARD, results -> { - results.next(); + results.next(); // docId: 1 assertTrue(results.getBoolean("test_boolean")); - for (String fld : fieldsNames) { + for (String fld : FIELDS_NAMES) { assertTrue("Expected: but was: for field " + fld, results.getBoolean(fld)); assertEquals("Expected: but was: for field " + fld, true, results.getObject(fld, Boolean.class)); } @@ -1274,9 +1307,9 @@ public void testGettingBooleanValues() throws IOException, SQLException { sqle.getMessage() ); - results.next(); + results.next(); // docId: 2 assertFalse(results.getBoolean("test_boolean")); - for (String fld : fieldsNames) { + for (String fld : FIELDS_NAMES) { assertFalse("Expected: but was: for field " + fld, results.getBoolean(fld)); assertEquals("Expected: but was: for field " + fld, false, results.getObject(fld, Boolean.class)); } @@ -1292,18 +1325,43 @@ public void testGettingBooleanValues() throws IOException, SQLException { sqle.getMessage() ); - results.next(); - for (String fld : fieldsNames.stream().filter(f -> !f.equals("test_keyword")).collect(Collectors.toCollection(HashSet::new))) { + results.next(); // docId: 3 + for (String fld : FIELDS_NAMES.stream().filter(f -> !f.equals("test_keyword")).collect(Collectors.toCollection(HashSet::new))) { assertTrue("Expected: but was: for field " + fld, results.getBoolean(fld)); assertEquals("Expected: but was: for field " + fld, true, results.getObject(fld, Boolean.class)); } - results.next(); + results.next(); // docId: 4 assertFalse(results.getBoolean("test_keyword")); assertEquals(false, results.getObject("test_keyword", Boolean.class)); }); } + public void testGettingBooleanValuesFromUnsignedLong() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); + + createIndex("test"); + updateMappingForNumericValuesTests("test", singletonList(UNSIGNED_LONG_FIELD)); + + indexTestFieldsDoc("1", BigInteger.ONE); + indexTestFieldsDoc("2", BigInteger.ZERO); + indexTestFieldsDoc("3", randomValueOtherThan(BigInteger.ZERO, ESTestCase::randomBigInteger)); + + doWithQuery(SELECT_WILDCARD, results -> { + results.next(); // docId: 1 + assertTrue(results.getBoolean(UNSIGNED_LONG_FIELD)); + assertTrue(results.getObject(UNSIGNED_LONG_FIELD, Boolean.class)); + + results.next(); // docId: 2 + assertFalse(results.getBoolean(UNSIGNED_LONG_FIELD)); + assertFalse(results.getObject(UNSIGNED_LONG_FIELD, Boolean.class)); + + results.next(); // docId: 3 + assertTrue(results.getBoolean(UNSIGNED_LONG_FIELD)); + assertTrue(results.getObject(UNSIGNED_LONG_FIELD, Boolean.class)); + }); + } + public void testGettingDateWithoutCalendar() throws IOException, SQLException { createIndex("test"); updateMappingForNumericValuesTests("test"); @@ -1312,7 +1370,7 @@ public void testGettingDateWithoutCalendar() throws IOException, SQLException { builder.startObject("test_date").field("type", "date").endObject(); }); long randomLongDate = randomNonNegativeLong(); - indexSimpleDocumentWithTrueValues(randomLongDate); + indexSimpleDocumentWithBooleanValues("1", true, randomLongDate); doWithQuery(SELECT_ALL_FIELDS, results -> { results.next(); @@ -1320,9 +1378,9 @@ public void testGettingDateWithoutCalendar() throws IOException, SQLException { java.sql.Date expectedDate = JdbcTestUtils.asDate(randomLongDate, getZoneFromOffset(randomLongDate)); assertEquals(expectedDate, results.getDate("test_date")); - assertEquals(expectedDate, results.getDate(fieldsNames.size() + 2)); + assertEquals(expectedDate, results.getDate(FIELDS_NAMES.size() + 2)); assertEquals(expectedDate, results.getObject("test_date", java.sql.Date.class)); - assertEquals(expectedDate, results.getObject(fieldsNames.size() + 2, java.sql.Date.class)); + assertEquals(expectedDate, results.getObject(FIELDS_NAMES.size() + 2, java.sql.Date.class)); // bulk validation for all fields which are not of type date validateErrorsForDateTestsWithoutCalendar(results::getDate); @@ -1337,7 +1395,7 @@ public void testGettingDateWithCalendar() throws IOException, SQLException { builder.startObject("test_date").field("type", "date").endObject(); }); long randomLongDate = randomNonNegativeLong(); - indexSimpleDocumentWithTrueValues(randomLongDate); + indexSimpleDocumentWithBooleanValues("1", true, randomLongDate); index("test", "2", builder -> builder.timeField("test_date", null)); String anotherTZId = randomValueOtherThan(timeZoneId, JdbcIntegrationTestCase::randomKnownTimeZone); @@ -1352,7 +1410,7 @@ public void testGettingDateWithCalendar() throws IOException, SQLException { c.set(MILLISECOND, 0); assertEquals(results.getDate("test_date", c), new java.sql.Date(c.getTimeInMillis())); - assertEquals(results.getDate(fieldsNames.size() + 2, c), new java.sql.Date(c.getTimeInMillis())); + assertEquals(results.getDate(FIELDS_NAMES.size() + 2, c), new java.sql.Date(c.getTimeInMillis())); // bulk validation for all fields which are not of type date validateErrorsForDateTimeTestsWithCalendar(c, results::getDate); @@ -1370,7 +1428,7 @@ public void testGettingTimeWithoutCalendar() throws IOException, SQLException { builder.startObject("test_date").field("type", "date").endObject(); }); Long randomLongDate = randomNonNegativeLong(); - indexSimpleDocumentWithTrueValues(randomLongDate); + indexSimpleDocumentWithBooleanValues("1", true, randomLongDate); doWithQuery(SELECT_ALL_FIELDS, results -> { results.next(); @@ -1378,9 +1436,9 @@ public void testGettingTimeWithoutCalendar() throws IOException, SQLException { java.sql.Time expectedTime = JdbcTestUtils.asTime(randomLongDate, getZoneFromOffset(randomLongDate)); assertEquals(expectedTime, results.getTime("test_date")); - assertEquals(expectedTime, results.getTime(fieldsNames.size() + 2)); + assertEquals(expectedTime, results.getTime(FIELDS_NAMES.size() + 2)); assertEquals(expectedTime, results.getObject("test_date", java.sql.Time.class)); - assertEquals(expectedTime, results.getObject(fieldsNames.size() + 2, java.sql.Time.class)); + assertEquals(expectedTime, results.getObject(FIELDS_NAMES.size() + 2, java.sql.Time.class)); validateErrorsForTimeTestsWithoutCalendar(results::getTime); }); @@ -1394,7 +1452,7 @@ public void testGettingTimeWithCalendar() throws IOException, SQLException { builder.startObject("test_date").field("type", "date").endObject(); }); long randomLongDate = randomNonNegativeLong(); - indexSimpleDocumentWithTrueValues(randomLongDate); + indexSimpleDocumentWithBooleanValues("1", true, randomLongDate); index("test", "2", builder -> builder.timeField("test_date", null)); String anotherTZId = randomValueOtherThan(timeZoneId, JdbcIntegrationTestCase::randomKnownTimeZone); @@ -1409,7 +1467,7 @@ public void testGettingTimeWithCalendar() throws IOException, SQLException { c.set(DAY_OF_MONTH, 1); assertEquals(results.getTime("test_date", c), new java.sql.Time(c.getTimeInMillis())); - assertEquals(results.getTime(fieldsNames.size() + 2, c), new java.sql.Time(c.getTimeInMillis())); + assertEquals(results.getTime(FIELDS_NAMES.size() + 2, c), new java.sql.Time(c.getTimeInMillis())); validateErrorsForDateTimeTestsWithCalendar(c, results::getTime); @@ -1468,7 +1526,7 @@ public void testGettingTimestampWithCalendar() throws IOException, SQLException builder.startObject("test_date").field("type", "date").endObject(); }); long randomLongDate = randomNonNegativeLong(); - indexSimpleDocumentWithTrueValues(randomLongDate); + indexSimpleDocumentWithBooleanValues("1", true, randomLongDate); index("test", "2", builder -> builder.timeField("test_date", null)); String anotherTZId = randomValueOtherThan(timeZoneId, JdbcIntegrationTestCase::randomKnownTimeZone); @@ -1479,7 +1537,7 @@ public void testGettingTimestampWithCalendar() throws IOException, SQLException c.setTimeInMillis(randomLongDate); assertEquals(results.getTimestamp("test_date", c), new java.sql.Timestamp(c.getTimeInMillis())); - assertEquals(results.getTimestamp(fieldsNames.size() + 2, c), new java.sql.Timestamp(c.getTimeInMillis())); + assertEquals(results.getTimestamp(FIELDS_NAMES.size() + 2, c), new java.sql.Timestamp(c.getTimeInMillis())); validateErrorsForDateTimeTestsWithCalendar(c, results::getTimestamp); @@ -1488,6 +1546,32 @@ public void testGettingTimestampWithCalendar() throws IOException, SQLException }); } + public void testErrorsValidationForDateTimeTypesConvertingToUnsignedLong() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); + + createIndex("test"); + updateMappingForNumericValuesTests("test", singletonList(UNSIGNED_LONG_FIELD)); + + indexTestFieldsDoc("1", BigInteger.ONE); + + doWithQuery(SELECT_WILDCARD, results -> { + results.next(); + + SQLException sqle = expectThrows(SQLException.class, () -> results.getDate(UNSIGNED_LONG_FIELD)); + assertEquals( + format(Locale.ROOT, "Unable to convert value [%.127s] of type [%s] to a Date", BigInteger.ONE, UNSIGNED_LONG_TYPE_NAME), + sqle.getMessage() + ); + + sqle = expectThrows(SQLException.class, () -> results.getTime(UNSIGNED_LONG_FIELD)); + assertEquals( + format(Locale.ROOT, "Unable to convert value [%.127s] of type [%s] to a Time", BigInteger.ONE, UNSIGNED_LONG_TYPE_NAME), + sqle.getMessage() + ); + }); + + } + public void testScalarOnDates() throws IOException, SQLException { createIndex("test"); updateMapping("test", builder -> builder.startObject("test_date").field("type", "date").endObject()); @@ -1703,9 +1787,6 @@ public void testGettingNullValues() throws SQLException { String query = "SELECT CAST(NULL AS BOOLEAN) b, CAST(NULL AS TINYINT) t, CAST(NULL AS SMALLINT) s, CAST(NULL AS INTEGER) i," + "CAST(NULL AS BIGINT) bi, CAST(NULL AS DOUBLE) d, CAST(NULL AS REAL) r, CAST(NULL AS FLOAT) f, CAST(NULL AS VARCHAR) v," + "CAST(NULL AS DATE) dt, CAST(NULL AS TIME) tm, CAST(NULL AS TIMESTAMP) ts"; - if (isUnsignedLongSupported()) { - query += ", CAST(NULL AS UNSIGNED_LONG) as ul"; - } doWithQuery(query, results -> { results.next(); @@ -1747,11 +1828,17 @@ public void testGettingNullValues() throws SQLException { assertNull(results.getObject("ts")); assertNull(results.getTimestamp("ts")); assertNull(results.getTimestamp("ts", randomCalendar())); + }); + } + public void testGettingNullUnsignedLong() throws SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); - if (isUnsignedLongSupported()) { - assertNull(results.getObject("ul")); - assertEquals(0.0f, results.getFloat("f"), 0f); - } + String query = "SELECT CAST(NULL AS UNSIGNED_LONG) as ul"; + doWithQuery(query, results -> { + results.next(); + + assertNull(results.getObject("ul")); + assertEquals(0.0f, results.getFloat("ul"), 0f); }); } @@ -1785,7 +1872,7 @@ public void testUnsupportedGetMethods() throws IOException, SQLException { index("test", "1", builder -> builder.field("test", "test")); try ( Connection conn = esJdbc(); - PreparedStatement statement = conn.prepareStatement("SELECT * FROM test"); + PreparedStatement statement = conn.prepareStatement(SELECT_WILDCARD); ResultSet r = statement.executeQuery() ) { @@ -1823,7 +1910,7 @@ public void testUnsupportedUpdateMethods() throws IOException, SQLException { index("test", "1", builder -> builder.field("test", "test")); try ( Connection conn = esJdbc(); - PreparedStatement statement = conn.prepareStatement("SELECT * FROM test"); + PreparedStatement statement = conn.prepareStatement(SELECT_WILDCARD); ResultSet r = statement.executeQuery() ) { r.next(); @@ -2140,21 +2227,41 @@ private void createTestDataForBooleanValueTests() throws IOException { }); } - private void indexSimpleDocumentWithTrueValues(Long randomLongDate) throws IOException { - index("test", "1", builder -> { - builder.field("test_boolean", true); - builder.field("test_byte", 1); - builder.field("test_integer", 1); - builder.field("test_long", 1L); - builder.field("test_unsigned_long", 1L); - builder.field("test_short", 1); - builder.field("test_double", 1d); - builder.field("test_float", 1f); - builder.field("test_keyword", "true"); + private void indexSimpleDocumentWithBooleanValues(String docId, boolean bool, Long randomLongDate) throws IOException { + index("test", docId, builder -> { + builder.field("test_boolean", bool); + builder.field("test_byte", bool ? 1 : 0); + builder.field("test_integer", bool ? 1 : 0); + builder.field("test_long", bool ? 1L : 0L); + builder.field("test_short", bool ? 1 : 0); + builder.field("test_double", bool ? 1d : 0d); + builder.field("test_float", bool ? 1f : 0f); + builder.field("test_keyword", bool ? "true" : "false"); builder.field("test_date", randomLongDate); }); } + protected static void indexTestFieldsDoc(String docId, Object... values) throws IOException { + index("test", docId, builder -> { + for (Object value : values) { + String classSimpleName = value.getClass().getSimpleName().toLowerCase(Locale.ROOT); + switch (classSimpleName) { + case "biginteger": + builder.field(UNSIGNED_LONG_FIELD, value); + break; + case "date": + builder.field("test_" + classSimpleName, ((Date) value).getTime()); + break; + case "string": + builder.field("test_keyword", value); + break; + default: + builder.field("test_" + classSimpleName, value); + } + } + }); + } + /** * Creates test data for all numeric get* methods. All values random and different from the other numeric fields already generated. * It returns a map containing the field name and its randomly generated value to be later used in checking the returned values. @@ -2185,13 +2292,6 @@ private Map createTestDataForNumericValueTypes(Supplier builder.field("test_long", test_long); map.put("test_long", test_long); - // random BigInteger/unsigned_long - BigInteger test_unsigned_long = BigInteger.valueOf(randomValueOtherThanMany(map::containsValue, randomGenerator).longValue()); - test_unsigned_long = test_unsigned_long.abs().subtract(test_unsigned_long.equals(BigInteger.ZERO) ? - BigInteger.ZERO : BigInteger.ONE); - builder.field("test_unsigned_long", test_unsigned_long); - map.put("test_unsigned_long", test_unsigned_long); - // random Double double test_double = randomValueOtherThanMany(map::containsValue, randomGenerator).doubleValue(); builder.field("test_double", test_double); @@ -2205,14 +2305,18 @@ private Map createTestDataForNumericValueTypes(Supplier return map; } - private static void updateMappingForNumericValuesTests(String indexName) throws IOException { + private static void updateMappingForNumericValuesTests(String indexName, List fieldsNames) throws IOException { updateMapping(indexName, builder -> { for (String field : fieldsNames) { - builder.startObject(field).field("type", field.substring(5)).endObject(); + builder.startObject(field).field("type", field.substring("test_".length())).endObject(); } }); } + private static void updateMappingForNumericValuesTests(String indexName) throws IOException { + updateMappingForNumericValuesTests(indexName, FIELDS_NAMES); + } + private void assertThrowsUnsupportedAndExpectErrorMessage(ThrowingRunnable runnable, String message) { SQLException sqle = expectThrows(SQLFeatureNotSupportedException.class, runnable); assertEquals(message, sqle.getMessage()); diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java index 25dde1cef9ee4..e4b9745f15613 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java @@ -168,13 +168,13 @@ public void testWildcardField() throws IOException { } /* - * "long/integer/short/byte_field": { + * "unsigned_long/long/integer/short/byte_field": { * "type": "long/integer/short/byte" * } */ public void testFractionsForNonFloatingPointTypes() throws IOException { String floatingPointNumber = "123.456"; - String fieldType = randomFrom("long", "integer", "short", "byte"); + String fieldType = randomFrom("unsigned_long", "long", "integer", "short", "byte"); createIndexWithFieldTypeAndProperties(fieldType, null, null); index("{\"" + fieldType + "_field\":\"" + floatingPointNumber + "\"}"); @@ -276,6 +276,16 @@ public void testByteFieldType() throws IOException { testField("byte", ((Number) randomByte()).intValue()); } + /* + * "unsigned_long_field": { + * "type": "unsigned_long", + * "ignore_malformed": true/false + * } + */ + public void testUnsignedLongFieldType() throws IOException { + testField("unsigned_long", randomBigInteger()); + } + private void testField(String fieldType, Object value) throws IOException { String fieldName = fieldType + "_field"; String query = "SELECT " + fieldName + " FROM test"; @@ -946,6 +956,8 @@ private JDBCType jdbcTypeFor(String esType) { return JDBCType.FLOAT; case "scaled_float": return JDBCType.DOUBLE; + case "unsigned_long": + return JDBCType.BIGINT; default: throw new AssertionError("Illegal value [" + esType + "] for data type"); } diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java index 74f8e32927e24..7004ecd340e07 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java @@ -28,7 +28,7 @@ import java.util.Locale; import java.util.Map; -import static org.elasticsearch.xpack.ql.util.Check.UNSIGNED_LONG_MAX; +import static org.elasticsearch.xpack.ql.util.UnsignedLongUtils.UNSIGNED_LONG_MAX; import static org.elasticsearch.xpack.sql.proto.Mode.CLI; import static org.elasticsearch.xpack.sql.proto.Protocol.SQL_QUERY_REST_ENDPOINT; import static org.elasticsearch.xpack.sql.proto.RequestInfo.CLIENT_IDS; diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java index b14d79452045b..30f47b527b153 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java @@ -161,6 +161,8 @@ private static String resolveColumnType(String type) { return "byte"; case "sh": return "short"; + case "ul": + return "bigdecimal"; // CSV default: return type; } diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java index 759e5a9f33d72..31716a60d46bc 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java @@ -136,6 +136,9 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String createIndex.startObject("birth_date").field("type", "date").endObject(); createIndex.startObject("hire_date").field("type", "date").endObject(); createIndex.startObject("salary").field("type", "integer").endObject(); + if (extraFields) { + createIndex.startObject("salary_ul").field("type", "unsigned_long").endObject(); + } createIndex.startObject("languages").field("type", "byte").endObject(); { createIndex.startObject("dep").field("type", "nested"); @@ -197,8 +200,12 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String } hadLastItem = true; bulk.append('"').append(titles.get(f)).append("\":\"").append(fields.get(f)).append('"'); - if (titles.get(f).equals("gender") && extraFields) { - bulk.append(",\"extra_gender\":\"Female\""); + if (extraFields) { + if (titles.get(f).equals("gender")) { + bulk.append(",\"extra_gender\":\"Female\""); + } else if (titles.get(f).equals("salary")) { + bulk.append(",\"salary_ul\":" + fields.get(f)); + } } } if ((titles.get(f).equals("first_name") || titles.get(f).equals("last_name")) && extraFields && setWildcardName) { diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java index db41cd4a997e2..780ba791ea157 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java @@ -18,6 +18,8 @@ import org.relique.jdbc.csv.CsvResultSet; import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; import java.sql.Date; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -163,6 +165,7 @@ public static void assertResultSetMetaData(ResultSet expected, ResultSet actual, // use the type not the name (timestamp with timezone returns spaces for example) int expectedType = typeOf(expectedMeta.getColumnType(column), lenientDataType); int actualType = typeOf(actualMeta.getColumnType(column), lenientDataType); + String actualTypeName = actualMeta.getColumnTypeName(column); // since H2 cannot use a fixed timezone, the data is stored in UTC (and thus with timezone) if (expectedType == Types.TIMESTAMP_WITH_TIMEZONE) { @@ -188,6 +191,11 @@ public static void assertResultSetMetaData(ResultSet expected, ResultSet actual, expectedType = Types.NULL; } + // csv and h2 both map values larger than Long.MAX_VALUE to Decimal types + if (expectedType == Types.DECIMAL && actualTypeName.compareTo(EsType.UNSIGNED_LONG.getName()) == 0) { + expectedType = EsType.UNSIGNED_LONG.getVendorTypeNumber(); + } + // when lenient is used, an int is equivalent to a short, etc... assertEquals( "Different column type for column [" + expectedName + "] (" + nameOf(expectedType) + " != " + nameOf(actualType) + ")", @@ -263,6 +271,9 @@ private static void doAssertResultSetData( case "Int": columnClassName = "java.lang.Integer"; break; + case "BigDecimal": + columnClassName = "java.math.BigDecimal"; + break; default: columnClassName = "java.lang." + columnClassName; break; @@ -333,6 +344,10 @@ else if (type == Types.DOUBLE) { else if (type == Types.VARCHAR && actualObject instanceof TemporalAmount) { assertEquals(msg, expectedObject, StringUtils.toString(actualObject)); } + // unsigned_long + else if (expectedObject instanceof BigDecimal && actualObject instanceof BigInteger) { + assertEquals(expectedObject, new BigDecimal((BigInteger) actualObject)); + } // finally the actual comparison else { assertEquals(msg, expectedObject, actualObject); diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec index 3c72cb630b0c0..67c31aef7993f 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec @@ -28,7 +28,7 @@ emp_no:i | first_name:s describeAlias DESCRIBE test_alias; - column | type | mapping + column | type | mapping --------------------+---------------+--------------- birth_date |TIMESTAMP |datetime dep |STRUCT |nested @@ -52,13 +52,14 @@ last_name |VARCHAR |text last_name.keyword |VARCHAR |keyword null_constant |VARCHAR |keyword salary |INTEGER |integer +salary_ul |BIGINT |unsigned_long wildcard_name |VARCHAR |keyword ; describePattern DESCRIBE "test_*"; - column | type | mapping + column | type | mapping --------------------+---------------+--------------- birth_date |TIMESTAMP |datetime dep |STRUCT |nested @@ -82,6 +83,7 @@ last_name |VARCHAR |text last_name.keyword |VARCHAR |keyword null_constant |VARCHAR |keyword salary |INTEGER |integer +salary_ul |BIGINT |unsigned_long wildcard_name |VARCHAR |keyword ; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec index ec403ab44e14e..03ec1ed604316 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec @@ -9,15 +9,15 @@ showFunctions SHOW FUNCTIONS; name:s | type:s -AVG |AGGREGATE +AVG |AGGREGATE COUNT |AGGREGATE FIRST |AGGREGATE FIRST_VALUE |AGGREGATE LAST |AGGREGATE LAST_VALUE |AGGREGATE MAX |AGGREGATE -MIN |AGGREGATE -SUM |AGGREGATE +MIN |AGGREGATE +SUM |AGGREGATE KURTOSIS |AGGREGATE MAD |AGGREGATE PERCENTILE |AGGREGATE @@ -64,11 +64,11 @@ DAY_OF_MONTH |SCALAR DAY_OF_WEEK |SCALAR DAY_OF_YEAR |SCALAR DOM |SCALAR -DOW |SCALAR +DOW |SCALAR DOY |SCALAR FORMAT |SCALAR HOUR |SCALAR -HOUR_OF_DAY |SCALAR +HOUR_OF_DAY |SCALAR IDOW |SCALAR ISODAYOFWEEK |SCALAR ISODOW |SCALAR @@ -78,16 +78,16 @@ ISO_DAY_OF_WEEK |SCALAR ISO_WEEK_OF_YEAR |SCALAR IW |SCALAR IWOY |SCALAR -MINUTE |SCALAR -MINUTE_OF_DAY |SCALAR -MINUTE_OF_HOUR |SCALAR -MONTH |SCALAR -MONTHNAME |SCALAR -MONTH_NAME |SCALAR -MONTH_OF_YEAR |SCALAR +MINUTE |SCALAR +MINUTE_OF_DAY |SCALAR +MINUTE_OF_HOUR |SCALAR +MONTH |SCALAR +MONTHNAME |SCALAR +MONTH_NAME |SCALAR +MONTH_OF_YEAR |SCALAR NOW |SCALAR -QUARTER |SCALAR -SECOND |SCALAR +QUARTER |SCALAR +SECOND |SCALAR SECOND_OF_MINUTE |SCALAR TIMESTAMPADD |SCALAR TIMESTAMPDIFF |SCALAR @@ -96,60 +96,60 @@ TIMESTAMP_DIFF |SCALAR TIME_PARSE |SCALAR TODAY |SCALAR WEEK |SCALAR -WEEK_OF_YEAR |SCALAR -YEAR |SCALAR -ABS |SCALAR -ACOS |SCALAR -ASIN |SCALAR -ATAN |SCALAR -ATAN2 |SCALAR -CBRT |SCALAR -CEIL |SCALAR -CEILING |SCALAR -COS |SCALAR -COSH |SCALAR -COT |SCALAR -DEGREES |SCALAR -E |SCALAR -EXP |SCALAR -EXPM1 |SCALAR -FLOOR |SCALAR -LOG |SCALAR -LOG10 |SCALAR -MOD |SCALAR -PI |SCALAR -POWER |SCALAR -RADIANS |SCALAR -RAND |SCALAR -RANDOM |SCALAR -ROUND |SCALAR -SIGN |SCALAR -SIGNUM |SCALAR -SIN |SCALAR -SINH |SCALAR -SQRT |SCALAR -TAN |SCALAR +WEEK_OF_YEAR |SCALAR +YEAR |SCALAR +ABS |SCALAR +ACOS |SCALAR +ASIN |SCALAR +ATAN |SCALAR +ATAN2 |SCALAR +CBRT |SCALAR +CEIL |SCALAR +CEILING |SCALAR +COS |SCALAR +COSH |SCALAR +COT |SCALAR +DEGREES |SCALAR +E |SCALAR +EXP |SCALAR +EXPM1 |SCALAR +FLOOR |SCALAR +LOG |SCALAR +LOG10 |SCALAR +MOD |SCALAR +PI |SCALAR +POWER |SCALAR +RADIANS |SCALAR +RAND |SCALAR +RANDOM |SCALAR +ROUND |SCALAR +SIGN |SCALAR +SIGNUM |SCALAR +SIN |SCALAR +SINH |SCALAR +SQRT |SCALAR +TAN |SCALAR TRUNC |SCALAR TRUNCATE |SCALAR ASCII |SCALAR -BIT_LENGTH |SCALAR -CHAR |SCALAR -CHARACTER_LENGTH |SCALAR -CHAR_LENGTH |SCALAR -CONCAT |SCALAR -INSERT |SCALAR -LCASE |SCALAR -LEFT |SCALAR -LENGTH |SCALAR -LOCATE |SCALAR -LTRIM |SCALAR -OCTET_LENGTH |SCALAR -POSITION |SCALAR -REPEAT |SCALAR -REPLACE |SCALAR -RIGHT |SCALAR -RTRIM |SCALAR -SPACE |SCALAR +BIT_LENGTH |SCALAR +CHAR |SCALAR +CHARACTER_LENGTH |SCALAR +CHAR_LENGTH |SCALAR +CONCAT |SCALAR +INSERT |SCALAR +LCASE |SCALAR +LEFT |SCALAR +LENGTH |SCALAR +LOCATE |SCALAR +LTRIM |SCALAR +OCTET_LENGTH |SCALAR +POSITION |SCALAR +REPEAT |SCALAR +REPLACE |SCALAR +RIGHT |SCALAR +RTRIM |SCALAR +SPACE |SCALAR STARTS_WITH |SCALAR SUBSTRING |SCALAR TRIM |SCALAR @@ -212,22 +212,22 @@ DAY_NAME |SCALAR DAY_OF_MONTH |SCALAR DAY_OF_WEEK |SCALAR DAY_OF_YEAR |SCALAR -HOUR_OF_DAY |SCALAR +HOUR_OF_DAY |SCALAR ISODAYOFWEEK |SCALAR ISO_DAY_OF_WEEK|SCALAR -MINUTE_OF_DAY |SCALAR +MINUTE_OF_DAY |SCALAR TODAY |SCALAR ; showTables SHOW TABLES; - name | type | kind + name | type | kind logs |TABLE |INDEX test_alias |VIEW |ALIAS -test_alias_emp |VIEW |ALIAS -test_emp |TABLE |INDEX -test_emp_copy |TABLE |INDEX +test_alias_emp |VIEW |ALIAS +test_emp |TABLE |INDEX +test_emp_copy |TABLE |INDEX ; showTablesSimpleLike @@ -241,8 +241,8 @@ showTablesMultiLike SHOW TABLES LIKE 'test_emp%'; name:s |type:s |kind:s -test_emp |TABLE |INDEX -test_emp_copy |TABLE |INDEX +test_emp |TABLE |INDEX +test_emp_copy |TABLE |INDEX ; showTablesIdentifier @@ -263,8 +263,8 @@ showTablesIdentifierPatternOnAliases SHOW TABLES "test*,-test_emp*"; name:s | type:s | kind:s -test_alias |VIEW |ALIAS -test_alias_emp |VIEW |ALIAS +test_alias |VIEW |ALIAS +test_alias_emp |VIEW |ALIAS ; // DESCRIBE @@ -272,7 +272,7 @@ test_alias_emp |VIEW |ALIAS describeSimpleLike DESCRIBE LIKE 'test_emp'; - column | type | mapping + column | type | mapping --------------------+---------------+--------------- birth_date |TIMESTAMP |datetime dep |STRUCT |nested @@ -296,13 +296,14 @@ last_name |VARCHAR |text last_name.keyword |VARCHAR |keyword null_constant |VARCHAR |keyword salary |INTEGER |integer +salary_ul |BIGINT |unsigned_long wildcard_name |VARCHAR |keyword ; describeMultiLike DESCRIBE LIKE 'test_emp%'; - column | type | mapping + column | type | mapping --------------------+---------------+--------------- birth_date |TIMESTAMP |datetime dep |STRUCT |nested @@ -326,13 +327,14 @@ last_name |VARCHAR |text last_name.keyword |VARCHAR |keyword null_constant |VARCHAR |keyword salary |INTEGER |integer +salary_ul |BIGINT |unsigned_long wildcard_name |VARCHAR |keyword ; describeSimpleIdentifier DESCRIBE "test_emp"; - column | type | mapping + column | type | mapping --------------------+---------------+--------------- birth_date |TIMESTAMP |datetime dep |STRUCT |nested @@ -358,7 +360,7 @@ salary |INTEGER |integer describeIncludeExcludeIdentifier-Ignore DESCRIBE "test_*,-test_alias*"; - column | type | mapping + column | type | mapping --------------------+---------------+--------------- birth_date |TIMESTAMP |datetime dep |STRUCT |nested diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec index 7b3331f81aa3e..a8a3ccaaff75a 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec @@ -9,16 +9,16 @@ SYS COLUMNS TABLE LIKE 'test\_emp' ESCAPE '\'; TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO ; sysColumnsWithTableLikeNoEscape @@ -29,21 +29,22 @@ SYS COLUMNS TABLE LIKE 'test_emp'; TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |salary_ul |-5 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |-5 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO ; sysColumnsWithCatalogAndLike @@ -51,21 +52,22 @@ SYS COLUMNS CATALOG 'integTest' TABLE LIKE 'test\_emp\_copy' ESCAPE '\'; TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s ---------------+---------------+---------------+-------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -integTest |null |test_emp_copy|birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO integTest |null |test_emp_copy|last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|salary_ul |-5 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |-5 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO ; sysColumnsOnAliasWithTableLike @@ -73,21 +75,22 @@ SYS COLUMNS TABLE LIKE 'test\_alias' ESCAPE '\'; TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -integTest |null |test_alias |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |salary_ul |-5 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |-5 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO ; sysColumnsAllTables @@ -95,67 +98,70 @@ SYS COLUMNS TABLE LIKE '%'; TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i|BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s -----------------+---------------+---------------+------------------+---------------+----------------+---------------+---------------+----------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -integTest |null |logs |@timestamp |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO -integTest |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |logs |client_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO -integTest |null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO -integTest |null |logs |dest_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO -integTest |null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |18 |YES |null |null |null |null |NO |NO +integTest |null |logs |@timestamp |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO +integTest |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |logs |client_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO +integTest |null |logs |dest_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO +integTest |null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO +integTest |null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |salary_ul |-5 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |-5 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |salary_ul |-5 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |-5 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |salary_ul |-5 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |-5 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO ; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec new file mode 100644 index 0000000000000..bba6d6cd655dd --- /dev/null +++ b/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec @@ -0,0 +1,99 @@ +// To mute tests follow example in file: example.csv-spec + +// +// Unsigned long tests +// + +arithmeticUnsignedLongCast +SELECT 1::unsigned_long + 1 AS x; + + x:ul +--------------- +2 +; + +nullArithmetics +SELECT null + 18446744073709551614 AS x; + + x:ul +--------------- +null +; + +arithmeticPlus +SELECT 18446744073709551614 + 1 AS x; + + x:ul +--------------- +18446744073709551615 +; + +arithmeticMultiply +SELECT 9223372036854775807::unsigned_long * 2 AS x; + + x:ul +--------------- +18446744073709551614 +; + +arithmeticDivide +SELECT 18446744073709551614 / 2 AS x; + + x:ul +--------------- +9223372036854775807 +; + +arithmeticModulo +SELECT 18446744073709551614 % 10 AS x; + + x:ul +--------------- +4 +; + +selectWithOrderByAndComparison +SELECT salary_ul AS x FROM test_emp_copy WHERE salary_ul > 70000 ORDER by salary_ul DESC LIMIT 1; + + x:ul +--------------- +74999 +; + +selectWithImplicitAgg +SELECT MAX(salary_ul)::STRING AS x FROM test_emp_copy; + + x:s +--------------- +74999.0 +; + +selectWithGroupByCountAndOrderBy +SELECT salary_ul AS s, count(1) AS c FROM test_emp_copy GROUP BY 1 ORDER BY 1 LIMIT 3; + + s:ul | c:l +---------------+--------------- +25324 |1 +25945 |1 +25976 |1 +; + +selectWithMathFunction +SELECT ROUND(SIN(salary_ul), 3) AS sin, salary_ul AS sal FROM test_emp_copy ORDER BY sal DESC LIMIT 3; + + sin:d | sal:ul +---------------+--------------- +0.239 |74999 +-0.823 |74970 +-0.015 |74572 +; + +selectWithIn +SELECT salary_ul, salary_ul + languages AS s, languages FROM test_emp_copy WHERE salary_ul IN ('74999', 74970, 74572::unsigned_long); + + salary_ul:ul | s:ul | languages:bt +------------------+---------------+--------------- +74572 |74576 |4 +74999 |null |null +74970 |74973 |3 +; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.sql-spec b/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.sql-spec new file mode 100644 index 0000000000000..6e893af92e445 --- /dev/null +++ b/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.sql-spec @@ -0,0 +1,73 @@ +// To mute tests follow example in file: example.sql-spec + +// +// Unsigned long tests +// + +// H2 doesn't support our unsigned_long type, casting to it isn't possible. +// Values larger than Long.MAX_VALUE are returned as BigDecimal. +// EXPM1 and CBRT functions not available. +// RAND takes an int. +// ES's CEIL & FLOOR don't work with unsigned_longs (TODO) + they can return longs, while H2's always returns doubles + +plus +SELECT 18446744073709551614 + 1 AS x; +plusLongMax +SELECT 9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000) AS x; +minus +SELECT 18446744073709551615 - ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000) AS x; +divide +SELECT 18446744073709551615 / ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000) AS x; +multiply +SELECT 18446744073709551615 * 1 AS x; +mod +SELECT 18446744073709551615 % ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000) AS x; + +abs +SELECT ABS(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +// ceil +// SELECT CEIL(9223372036854775808 + RAND(SECOND(CURRENT_TIMESTAMP())) * 10000) AS x; +exp +SELECT EXP(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +// floor +// SELECT FLOOR(9223372036854775808 + RAND(SECOND(CURRENT_TIMESTAMP())) * 10000) AS x; +log +SELECT LOG(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +log10 +SELECT LOG10(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +power +SELECT POWER(4294967295, 2) AS x; +round +SELECT ROUND(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +sign +SELECT SIGN(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +sqrt +SELECT SQRT(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +truncate +SELECT TRUNCATE(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +acos + +// trigonometric +SELECT ACOS(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +asin +SELECT ASIN(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +atan +SELECT ATAN(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +atan2 +SELECT ATAN2(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000), 45) AS x; +cos +SELECT COS(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +cosh +SELECT COSH(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +cot +SELECT COT(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +degrees +SELECT DEGREES(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +radians +SELECT RADIANS(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +sin +SELECT SIN(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +sinh +SELECT SINH(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; +tan +SELECT TAN(9223372036854775808 + ROUND(RAND(SECOND(CURRENT_TIMESTAMP())) * 10000)) AS x; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index c16d7b4b77f15..f5976930ac417 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -116,7 +116,6 @@ protected Iterable.Batch> batches() { new ResolveAggsInHaving(), new ResolveAggsInOrderBy() //new ImplicitCasting() - //new VersionCompatibility() ); Batch finish = new Batch("Finish Analysis", new PruneSubqueryAliases(), diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index b6937a943413e..9e0df6a7a9930 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -53,7 +53,7 @@ import org.elasticsearch.xpack.sql.plan.logical.LocalRelation; import org.elasticsearch.xpack.sql.plan.logical.Pivot; import org.elasticsearch.xpack.sql.plan.logical.command.Command; -import org.elasticsearch.xpack.sql.session.SqlConfiguration; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.stats.FeatureMetric; import org.elasticsearch.xpack.sql.stats.Metrics; import org.elasticsearch.xpack.sql.type.SqlDataTypes; @@ -72,7 +72,8 @@ import static java.util.stream.Collectors.toMap; import static org.elasticsearch.xpack.ql.common.Failure.fail; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; -import static org.elasticsearch.xpack.sql.session.Compatibility.isTypeSupportedInVersion; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.COMMAND; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.GROUPBY; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.HAVING; @@ -89,11 +90,11 @@ */ public final class Verifier { private final Metrics metrics; - private final SqlConfiguration config; + private final SqlVersion version; - public Verifier(Metrics metrics, SqlConfiguration config) { + public Verifier(Metrics metrics, SqlVersion version) { this.metrics = metrics; - this.config = config; + this.version = version; } public Map, String> verifyFailures(LogicalPlan plan) { @@ -236,17 +237,7 @@ Collection verify(LogicalPlan plan) { failures.addAll(localFailures); }); - if (config.version() != null) { - List projects = plan.collectFirstChildren(x -> x instanceof Project); - if (projects.size() > 0) { // not selecting just literals - ((Project) projects.get(0)).projections().forEach(e -> { - if (e.resolved() && isTypeSupportedInVersion(e.dataType(), config.version()) == false) { - failures.add(fail(e, "Cannot use field [" + e.name() + "] with type [" + e.dataType() + "] unsupported " + - "in version [" + config.version() + "]")); - } - }); - } - } + checkClientSupportsDataTypes(plan, failures, version); } // gather metrics @@ -898,4 +889,18 @@ private static void checkCastOnInexact(LogicalPlan p, Set localFailures } }, Cast.class)), Filter.class); } + + private static void checkClientSupportsDataTypes(LogicalPlan p, Set localFailures, SqlVersion version) { + List projects = p.collectFirstChildren(x -> x instanceof Project); + if (projects.size() > 0) { + ((Project) projects.get(0)).projections().forEach(e -> { + if (e.resolved() && isTypeSupportedInVersion(e.dataType(), version) == false) { + localFailures.add(fail(e, "Cannot use field [" + e.name() + "] with type [" + e.dataType() + "] unsupported " + + "in version [" + version + "], upgrade required (to version [" + INTRODUCING_UNSIGNED_LONG + + "] or higher)")); + } + }); + } + + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java index fb34c53e64711..67ffa1cc3ec6d 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java @@ -61,11 +61,8 @@ public PlanExecutor(Client client, IndexResolver indexResolver, NamedWriteableRe } private SqlSession newSession(SqlConfiguration cfg) { - // TODO: SqlSession exposes no functionality that doesn't involve the Analyzer; would it make sense to: - // 1) instantiate an Analyzer in its c'tor? and - // 2) instantiate the Verifier in Analyzer's c'tor and add a getter for it? - return new SqlSession(cfg, client, functionRegistry, indexResolver, preAnalyzer, new Verifier(metrics, cfg), optimizer, planner, - this); + return new SqlSession(cfg, client, functionRegistry, indexResolver, preAnalyzer, new Verifier(metrics, cfg.version()), optimizer, + planner, this); } public void searchSource(SqlConfiguration cfg, String sql, List params, diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java index ce249a1760ccc..5c9ac5e91edf4 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java @@ -684,7 +684,7 @@ public Literal visitIntegerLiteral(IntegerLiteralContext ctx) { Number value; try { - value = StringUtils.parseInteger(tuple.v2()); + value = StringUtils.parseIntegral(tuple.v2()); } catch (QlIllegalArgumentException siae) { throw new ParsingException(tuple.v1(), siae.getMessage()); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java index 599991cbd13b0..9b81b88fb5cf2 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java @@ -38,8 +38,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.SHORT; import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; import static org.elasticsearch.xpack.ql.type.DataTypes.isString; -import static org.elasticsearch.xpack.sql.proto.Mode.isDriver; -import static org.elasticsearch.xpack.sql.session.Compatibility.isTypeSupportedInVersion; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedByClient; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.displaySize; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDataType; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDateTimeSub; @@ -174,7 +173,7 @@ private static void fillInRows(String clusterName, String indexName, Map output() { @Override public final void execute(SqlSession session, ActionListener listener) { Stream values = SqlDataTypes.types().stream() - .filter(t -> Mode.isDriver(session.configuration().mode()) == false || - isTypeSupportedInVersion(t, session.configuration().version())); + .filter(t -> isTypeSupportedByClient(session.configuration().mode(), t, session.configuration().version())); if (type.intValue() != 0) { values = values.filter(t -> type.equals(sqlType(t).getVendorTypeNumber())); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java index 54ff4d00bcfaa..0cd57b9ea35ad 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlConfiguration.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.sql.session; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilder; @@ -21,7 +22,6 @@ public class SqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu private final TimeValue pageTimeout; private final Mode mode; private final String clientId; - @Nullable private final SqlVersion version; private final boolean multiValueFieldLeniency; private final boolean includeFrozenIndices; @@ -43,7 +43,7 @@ public SqlConfiguration(ZoneId zi, int pageSize, TimeValue requestTimeout, TimeV this.filter = filter; this.mode = mode == null ? Mode.PLAIN : mode; this.clientId = clientId; - this.version = version; + this.version = version != null ? version : SqlVersion.fromId(Version.CURRENT.id); this.multiValueFieldLeniency = multiValueFieldLeniency; this.includeFrozenIndices = includeFrozen; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/Compatibility.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java similarity index 67% rename from x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/Compatibility.java rename to x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java index 219a3a221945b..e17b00c6940fd 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/Compatibility.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java @@ -6,27 +6,29 @@ package org.elasticsearch.xpack.sql.session; -import org.elasticsearch.common.Nullable; import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.sql.proto.Mode; import org.elasticsearch.xpack.sql.proto.SqlVersion; import static org.elasticsearch.Version.V_7_11_0; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.proto.Mode.isDriver; -public final class Compatibility { +public final class VersionCompatibilityChecks { public static final SqlVersion INTRODUCING_UNSIGNED_LONG = SqlVersion.fromId(V_7_11_0.id); - private Compatibility() {} + private VersionCompatibilityChecks() {} + public static boolean isTypeSupportedByClient(Mode mode, DataType dataType, SqlVersion version) { + return isDriver(mode) == false || isTypeSupportedInVersion(dataType, version); + } /** * Is the provided {@code dataType} being supported in the provided {@code version}? */ - public static boolean isTypeSupportedInVersion(DataType dataType, @Nullable SqlVersion version) { - if (version != null) { - if (dataType == UNSIGNED_LONG) { - return supportsUnsignedLong(version); - } + public static boolean isTypeSupportedInVersion(DataType dataType, SqlVersion version) { + if (dataType == UNSIGNED_LONG) { + return supportsUnsignedLong(version); } return true; } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java index 5554666ee2211..720da3c9e494d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlTestUtils.java @@ -43,7 +43,7 @@ private SqlTestUtils() {} public static SqlConfiguration randomConfiguration() { return new SqlConfiguration(randomZone(), - randomIntBetween(0, 1000), + randomIntBetween(0, 1000), new TimeValue(randomNonNegativeLong()), new TimeValue(randomNonNegativeLong()), null, @@ -58,7 +58,7 @@ public static SqlConfiguration randomConfiguration() { public static SqlConfiguration randomConfiguration(ZoneId providedZoneId) { return new SqlConfiguration(providedZoneId, - randomIntBetween(0, 1000), + randomIntBetween(0, 1000), new TimeValue(randomNonNegativeLong()), new TimeValue(randomNonNegativeLong()), null, @@ -73,7 +73,7 @@ public static SqlConfiguration randomConfiguration(ZoneId providedZoneId) { public static SqlConfiguration randomConfiguration(SqlVersion version) { return new SqlConfiguration(randomZone(), - randomIntBetween(0, 1000), + randomIntBetween(0, 1000), new TimeValue(randomNonNegativeLong()), new TimeValue(randomNonNegativeLong()), null, diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java index d83359f3ca4c4..62ceabda3ca85 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java @@ -37,7 +37,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; -import static org.elasticsearch.xpack.sql.session.Compatibility.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.contains; @@ -58,7 +58,7 @@ public class FieldAttributeTests extends ESTestCase { public FieldAttributeTests() { parser = new SqlParser(); functionRegistry = new SqlFunctionRegistry(); - verifier = new Verifier(new Metrics(), SqlTestUtils.TEST_CFG); + verifier = new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version()); Map mapping = loadMapping("mapping-multi-field-variation.json"); @@ -301,11 +301,11 @@ public void testUnsignedLongVersionCompatibility() { for (String sql : List.of(query, queryWithLiteral, queryWithAlias, queryWithArithmetic, queryWithCast)) { SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preUnsignedLong); - analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics(), sqlConfig)); + analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics(), sqlConfig.version())); VerificationException ex = expectThrows(VerificationException.class, () -> plan(sql)); assertEquals( "Found 1 problem\nline 1:8: Cannot use field [unsigned_long] with type [UNSIGNED_LONG] unsupported in version [" + - preUnsignedLong + "]", + preUnsignedLong + "], upgrade required (to version [" + INTRODUCING_UNSIGNED_LONG + "] or higher)", ex.getMessage()); for (SqlVersion v : List.of(introducingUnsignedLong, postUnsignedLong)) { @@ -329,7 +329,7 @@ public void testNonProjectedUnsignedLongVersionCompatibility() { getIndexResult = IndexResolution.valid(index); SqlVersion preIntroducingVersion = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preIntroducingVersion); - analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics(), sqlConfig)); + analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics(), sqlConfig.version())); String query = "SELECT unsigned_long = 1, unsigned_long::double FROM test"; String queryWithSubquery = "SELECT l = 1, SQRT(ul) FROM " + diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index b3aaa427da35c..5886fc6dbbdd5 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -49,7 +49,7 @@ private String error(String sql) { private String error(IndexResolution getIndexResult, String sql) { Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics(), - SqlTestUtils.TEST_CFG)); + SqlTestUtils.TEST_CFG.version())); VerificationException e = expectThrows(VerificationException.class, () -> analyzer.analyze(parser.createStatement(sql), true)); String message = e.getMessage(); assertTrue(message.startsWith("Found ")); @@ -70,7 +70,7 @@ private EsIndex getTestEsIndex() { private LogicalPlan accept(IndexResolution resolution, String sql) { Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), resolution, - new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version())); return analyzer.analyze(parser.createStatement(sql), true); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 4da5d0154ffe7..7adbb6278b5c3 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -433,7 +433,7 @@ private void addFieldCaps(Map> fieldCaps, } private static IndexResolution mergedMappings(String indexPattern, String[] indexNames, - Map> fieldCaps) { + Map> fieldCaps) { return IndexResolver.mergedMappings(SqlDataTypeRegistry.INSTANCE, indexPattern, indexNames, fieldCaps); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java index 49f10064bb5cb..47546118e8beb 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java @@ -37,7 +37,7 @@ public void testDatabaseFunctionOutput() { sqlConfig, new SqlFunctionRegistry(), IndexResolution.valid(test), - new Verifier(new Metrics(), sqlConfig) + new Verifier(new Metrics(), sqlConfig.version()) ); Project result = (Project) analyzer.analyze(parser.createStatement("SELECT DATABASE()"), true); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java index 6f810de688d2b..d47d93191ced3 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java @@ -37,7 +37,7 @@ null, null, randomAlphaOfLengthBetween(1, 15), sqlConfig, new SqlFunctionRegistry(), IndexResolution.valid(test), - new Verifier(new Metrics(), sqlConfig) + new Verifier(new Metrics(), sqlConfig.version()) ); Project result = (Project) analyzer.analyze(parser.createStatement("SELECT USER()"), true); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java index d420d93d80daf..fe66ad7001dbc 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java @@ -92,7 +92,7 @@ public void testInvalidPrecision() { SqlTypesTests.loadMapping("mapping-multi-field-with-nested.json"))); Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, - new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version())); ParsingException e = expectThrows(ParsingException.class, () -> analyzer.analyze(parser.createStatement("SELECT CURRENT_TIMESTAMP(100000000000000)"), true)); assertEquals("line 1:27: invalid precision; [100000000000000] out of [integer] range", e.getMessage()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java index 61e638ba85ee1..bbcb369ddd4ff 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java @@ -93,7 +93,7 @@ public void testInvalidPrecision() { SqlTypesTests.loadMapping("mapping-multi-field-with-nested.json"))); Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, - new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version())); ParsingException e = expectThrows(ParsingException.class, () -> analyzer.analyze(parser.createStatement("SELECT CURRENT_TIME(100000000000000)"), true)); assertEquals("line 1:22: invalid precision; [100000000000000] out of [integer] range", e.getMessage()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java index 6e7b5c59b930b..0905fe4d0df60 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java @@ -37,7 +37,7 @@ public OptimizerRunTests() { EsIndex test = new EsIndex("test", mapping); getIndexResult = IndexResolution.valid(test); analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, - new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version())); optimizer = new Optimizer(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index 2c4d3b1115d55..483eb27b8a3ab 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -32,19 +32,19 @@ import java.sql.Types; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; import java.util.function.Consumer; +import java.util.stream.Collectors; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.elasticsearch.action.ActionListener.wrap; import static org.elasticsearch.xpack.ql.TestUtils.UTC; -import static org.elasticsearch.xpack.sql.session.Compatibility.INTRODUCING_UNSIGNED_LONG; -import static org.elasticsearch.xpack.sql.session.Compatibility.supportsUnsignedLong; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedByClient; import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; @@ -70,109 +70,109 @@ public void testSysColumns() { int index = 0; - List row = rows.get(index ++); + List row = rows.get(index++); assertEquals("bool", name(row)); assertEquals(Types.BOOLEAN, sqlType(row)); assertEquals(null, radix(row)); assertEquals(1, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("int", name(row)); assertEquals(Types.INTEGER, sqlType(row)); assertEquals(10, radix(row)); assertEquals(4, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("unsigned_long", name(row)); assertEquals(Types.BIGINT, sqlType(row)); assertEquals(10, radix(row)); assertEquals(Long.BYTES, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("text", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("keyword", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("date", name(row)); assertEquals(Types.TIMESTAMP, sqlType(row)); assertEquals(null, radix(row)); assertEquals(29, precision(row)); assertEquals(8, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.dotted.field", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.string", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.string.normalized", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.string.typical", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.ambiguous", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.ambiguous.one", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.ambiguous.two", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.ambiguous.normalized", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); } - public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { - boolean unsignedLongSupported = version == null || supportsUnsignedLong(version); + public void sysColumnsInDriverMode(Mode mode) { Class typeClass = mode == Mode.ODBC ? Short.class : Integer.class; List> rows = new ArrayList<>(); - SysColumns.fillInRows("test", "index", loadMapping("mapping-multi-field-variation.json", true), null, rows, null, mode, version); - assertEquals(FIELD_COUNT - (unsignedLongSupported ? 0 : 1), rows.size()); + SysColumns.fillInRows("test", "index", loadMapping("mapping-multi-field-variation.json", true), null, rows, null, mode, + SqlVersion.fromId(Version.CURRENT.id)); + assertEquals(FIELD_COUNT, rows.size()); assertEquals(24, rows.get(0).size()); int index = 0; - List row = rows.get(index ++); + List row = rows.get(index++); assertEquals("bool", name(row)); assertEquals(Types.BOOLEAN, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); assertEquals(1, bufferLength(row)); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("int", name(row)); assertEquals(Types.INTEGER, typeClass.cast(sqlType(row)).intValue()); assertEquals(typeClass, radix(row).getClass()); @@ -182,19 +182,17 @@ public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { assertEquals(typeClass, sqlDataType(row).getClass()); assertEquals(typeClass, sqlDataTypeSub(row).getClass()); - if (unsignedLongSupported) { - row = rows.get(index++); - assertEquals("unsigned_long", name(row)); - assertEquals(Types.BIGINT, typeClass.cast(sqlType(row)).intValue()); - assertEquals(typeClass, radix(row).getClass()); - assertEquals(Long.BYTES, bufferLength(row)); - assertNull(decimalPrecision(row)); - assertEquals(typeClass, nullable(row).getClass()); - assertEquals(typeClass, sqlDataType(row).getClass()); - assertEquals(typeClass, sqlDataTypeSub(row).getClass()); - } + row = rows.get(index++); + assertEquals("unsigned_long", name(row)); + assertEquals(Types.BIGINT, typeClass.cast(sqlType(row)).intValue()); + assertEquals(typeClass, radix(row).getClass()); + assertEquals(Long.BYTES, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(typeClass, nullable(row).getClass()); + assertEquals(typeClass, sqlDataType(row).getClass()); + assertEquals(typeClass, sqlDataTypeSub(row).getClass()); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("text", name(row)); assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); @@ -204,7 +202,7 @@ public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { assertEquals(typeClass, sqlDataType(row).getClass()); assertEquals(typeClass, sqlDataTypeSub(row).getClass()); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("keyword", name(row)); assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); @@ -214,7 +212,7 @@ public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { assertEquals(typeClass, sqlDataType(row).getClass()); assertEquals(typeClass, sqlDataTypeSub(row).getClass()); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("date", name(row)); assertEquals(Types.TIMESTAMP, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); @@ -225,7 +223,7 @@ public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { assertEquals(typeClass, sqlDataType(row).getClass()); assertEquals(typeClass, sqlDataTypeSub(row).getClass()); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.dotted.field", name(row)); assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); @@ -235,7 +233,7 @@ public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { assertEquals(typeClass, sqlDataType(row).getClass()); assertEquals(typeClass, sqlDataTypeSub(row).getClass()); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.string", name(row)); assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); @@ -245,7 +243,7 @@ public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { assertEquals(typeClass, sqlDataType(row).getClass()); assertEquals(typeClass, sqlDataTypeSub(row).getClass()); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.string.normalized", name(row)); assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); @@ -255,7 +253,7 @@ public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { assertEquals(typeClass, sqlDataType(row).getClass()); assertEquals(typeClass, sqlDataTypeSub(row).getClass()); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.string.typical", name(row)); assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); @@ -265,7 +263,7 @@ public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { assertEquals(typeClass, sqlDataType(row).getClass()); assertEquals(typeClass, sqlDataTypeSub(row).getClass()); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.ambiguous", name(row)); assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); @@ -275,7 +273,7 @@ public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { assertEquals(typeClass, sqlDataType(row).getClass()); assertEquals(typeClass, sqlDataTypeSub(row).getClass()); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.ambiguous.one", name(row)); assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); @@ -285,7 +283,7 @@ public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { assertEquals(typeClass, sqlDataType(row).getClass()); assertEquals(typeClass, sqlDataTypeSub(row).getClass()); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.ambiguous.two", name(row)); assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); @@ -295,7 +293,7 @@ public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { assertEquals(typeClass, sqlDataType(row).getClass()); assertEquals(typeClass, sqlDataTypeSub(row).getClass()); - row = rows.get(index ++); + row = rows.get(index++); assertEquals("some.ambiguous.normalized", name(row)); assertEquals(Types.VARCHAR, typeClass.cast(sqlType(row)).intValue()); assertEquals(null, radix(row)); @@ -306,21 +304,32 @@ public void sysColumnsInDriverModeAtVersion(Mode mode, SqlVersion version) { assertEquals(typeClass, sqlDataTypeSub(row).getClass()); } - public void testSysColumnsInDriverMode() { + public void testSysColumnsInJdbcMode() { + sysColumnsInDriverMode(Mode.JDBC); + } + + public void testSysColumnsInOdbcMode() { + sysColumnsInDriverMode(Mode.ODBC); + } + + public void testUnsignedLongFiltering() { + List versions = List.of( + SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER), + INTRODUCING_UNSIGNED_LONG, + SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER), + SqlVersion.fromId(Version.CURRENT.id) + ); for (Mode mode : List.of(Mode.JDBC, Mode.ODBC)) { - Set versions = new HashSet<>(List.of( - SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER), - INTRODUCING_UNSIGNED_LONG, - SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER), - SqlVersion.fromId(Version.CURRENT.id) - )); - versions.add(null); for (SqlVersion version : versions) { - sysColumnsInDriverModeAtVersion(mode, version); + List> rows = new ArrayList<>(); + SysColumns.fillInRows("test", "index", loadMapping("mapping-multi-field-variation.json", true), null, rows, null, mode, + version); + List types = rows.stream().map(row -> name(row).toString()).collect(Collectors.toList()); + assertEquals(isTypeSupportedByClient(mode, UNSIGNED_LONG, version), + types.contains(UNSIGNED_LONG.toString().toLowerCase(Locale.ROOT))); } } } - private static Object name(List list) { return list.get(3); } @@ -463,7 +472,8 @@ private void executeCommand(String sql, List params, Consume private Tuple sql(String sql, List params, SqlConfiguration config, Map mapping) { EsIndex test = new EsIndex("test", mapping); - Analyzer analyzer = new Analyzer(config, new FunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics(), config)); + Analyzer analyzer = new Analyzer(config, new FunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics(), + config.version())); Command cmd = (Command) analyzer.analyze(parser.createStatement(sql, params, UTC), true); IndexResolver resolver = mock(IndexResolver.class); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java index 8e43934c3c411..43fec3c0d0111 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java @@ -334,7 +334,7 @@ private SqlTypedParamValue param(Object value) { private Tuple sql(String sql, List params, SqlConfiguration cfg) { EsIndex test = new EsIndex("test", mapping); Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new FunctionRegistry(), IndexResolution.valid(test), - new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version())); Command cmd = (Command) analyzer.analyze(parser.createStatement(sql, params, cfg.zoneId()), true); IndexResolver resolver = mock(IndexResolver.class); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java index 53f342556d186..efc303059e537 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java @@ -31,12 +31,12 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; import static java.util.Arrays.asList; import static org.elasticsearch.action.ActionListener.wrap; -import static org.elasticsearch.xpack.sql.session.Compatibility.INTRODUCING_UNSIGNED_LONG; -import static org.elasticsearch.xpack.sql.session.Compatibility.isTypeSupportedInVersion; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedByClient; import static org.mockito.Mockito.mock; public class SysTypesTests extends ESTestCase { @@ -104,10 +104,8 @@ public void testUnsignedLongFiltering() { SchemaRowSet r = (SchemaRowSet) p.rowSet(); List types = new ArrayList<>(); r.forEachRow(rv -> types.add((String) rv.column(0))); - List expectedTypes = ALL_TYPES.stream() - .filter(t -> Mode.isDriver(mode) == false || isTypeSupportedInVersion(DataTypes.fromTypeName(t), version)) - .collect(Collectors.toList()); - assertEquals(expectedTypes, types); + assertEquals(isTypeSupportedByClient(mode, UNSIGNED_LONG, cmd.v2().configuration().version()), + types.contains(UNSIGNED_LONG.toString())); }, ex -> fail(ex.getMessage()))); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/PostOptimizerVerifierTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/PostOptimizerVerifierTests.java index c5bbe0292e11b..2b92ad1d17920 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/PostOptimizerVerifierTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/PostOptimizerVerifierTests.java @@ -40,7 +40,7 @@ public void init() { EsIndex test = new EsIndex("test", mapping); indexResolution = IndexResolution.valid(test); analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, - new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version())); optimizer = new Optimizer(); planner = new Planner(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java index be252f68131e0..ac41f0d524d58 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java @@ -49,7 +49,7 @@ public static void init() { EsIndex test = new EsIndex("test", mapping); IndexResolution getIndexResult = IndexResolution.valid(test); analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), getIndexResult, - new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version())); optimizer = new Optimizer(); planner = new Planner(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java index e235bed219a32..4c812564b38c7 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java @@ -132,7 +132,7 @@ private static class TestContext { EsIndex test = new EsIndex("test", mapping); IndexResolution getIndexResult = IndexResolution.valid(test); analyzer = new Analyzer(SqlTestUtils.TEST_CFG, sqlFunctionRegistry, getIndexResult, - new Verifier(new Metrics(), SqlTestUtils.TEST_CFG)); + new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version())); optimizer = new Optimizer(); planner = new Planner(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java index b8ea511c652a8..9e7f5d693fbb7 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java @@ -160,7 +160,7 @@ public void testWhereLimitGroupByHavingOrderByQuery() { public void testTwoQueriesExecuted() { Metrics metrics = new Metrics(); - Verifier verifier = new Verifier(metrics, SqlTestUtils.TEST_CFG); + Verifier verifier = new Verifier(metrics, SqlTestUtils.TEST_CFG.version()); sqlWithVerifier("SELECT languages FROM test WHERE languages > 2 GROUP BY languages LIMIT 5", verifier); sqlWithVerifier("SELECT languages FROM test WHERE languages > 2 GROUP BY languages HAVING MAX(languages) > 3 " + "ORDER BY languages LIMIT 5", verifier); @@ -178,7 +178,7 @@ public void testTwoQueriesExecuted() { public void testTwoCommandsExecuted() { String command1 = randomFrom(commands); Metrics metrics = new Metrics(); - Verifier verifier = new Verifier(metrics, SqlTestUtils.TEST_CFG); + Verifier verifier = new Verifier(metrics, SqlTestUtils.TEST_CFG.version()); sqlWithVerifier(command1, verifier); sqlWithVerifier(randomValueOtherThan(command1, () -> randomFrom(commands)), verifier); Counters c = metrics.stats(); @@ -236,7 +236,7 @@ private Counters sql(String sql, Verifier v) { Metrics metrics = null; if (v == null) { metrics = new Metrics(); - verifier = new Verifier(metrics, SqlTestUtils.TEST_CFG); + verifier = new Verifier(metrics, SqlTestUtils.TEST_CFG.version()); } Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), IndexResolution.valid(test), verifier); From 6801bd587aafca356a519d326a6c26986d736fc1 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Mon, 30 Nov 2020 12:14:18 +0100 Subject: [PATCH 08/28] Remove stale comment - remoce comment - undo primitive -> object change in one test. --- .../execution/search/extractor/AbstractFieldHitExtractor.java | 1 - .../elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java index aeb4e930fce24..e78752a2149e6 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java @@ -187,7 +187,6 @@ protected Object unwrapMultiValue(Object values) { if (dataType.isNumeric() && isFromDocValuesOnly(dataType) == false) { Number result = null; try { - // TODO: don't mapper modules expose _source parsing methods? should the _source be (re)validated? result = createNumeric(dataType, values); } catch(IllegalArgumentException iae) { return null; diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java index 32005ffd1eebe..415c150708006 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java @@ -235,9 +235,9 @@ public void testGettingInvalidByte() throws IOException, SQLException { builder.startObject("test_date").field("type", "date").endObject(); }); - Integer intNotByte = randomIntBetween(Byte.MAX_VALUE + 1, Integer.MAX_VALUE); + int intNotByte = randomIntBetween(Byte.MAX_VALUE + 1, Integer.MAX_VALUE); long longNotByte = randomLongBetween(Byte.MAX_VALUE + 1, Long.MAX_VALUE); - Short shortNotByte = (short) randomIntBetween(Byte.MAX_VALUE + 1, Short.MAX_VALUE); + short shortNotByte = (short) randomIntBetween(Byte.MAX_VALUE + 1, Short.MAX_VALUE); double doubleNotByte = randomDoubleBetween(Byte.MAX_VALUE + 1, Double.MAX_VALUE, true); float floatNotByte = randomFloatBetween(Byte.MAX_VALUE + 1, Float.MAX_VALUE); String randomString = randomUnicodeOfCodepointLengthBetween(128, 256); From c9bfffdd1279c155c75fa030930eb0c274caa75d Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Mon, 30 Nov 2020 18:15:19 +0100 Subject: [PATCH 09/28] Filter out UL from * expansion. Drop client check - Filters out UL from star expansion (SELECT * ...). This is required since the verifier will otherwise later fail an otherwise "supportable" query for old clients. - Drop checking for the client type when enforcing version compatibility. If the client inserts a version, we should answer according to that version's compability no matter the client. --- .../qa/jdbc/ResultSetMetaDataTestCase.java | 23 ++++++++++++++- .../xpack/sql/analysis/analyzer/Analyzer.java | 19 ++++++++---- .../plan/logical/command/sys/SysColumns.java | 4 +-- .../plan/logical/command/sys/SysTypes.java | 4 +-- .../session/VersionCompatibilityChecks.java | 8 ++--- .../analyzer/FieldAttributeTests.java | 29 +++++++++++++++++-- .../logical/command/sys/SysColumnsTests.java | 4 +-- .../logical/command/sys/SysTypesTests.java | 4 +-- 8 files changed, 72 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java index 90ba593b0ef91..e349d5db13f93 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java @@ -14,6 +14,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.stream.Collectors; @@ -38,13 +39,17 @@ public abstract class ResultSetMetaDataTestCase extends JdbcIntegrationTestCase ); private static final String UNSIGNED_LONG_FIELD = "test_" + UNSIGNED_LONG_TYPE_NAME.toLowerCase(Locale.ROOT); - public void doTestValidGetObjectCalls(List fieldsNames) throws IOException, SQLException { + private static void createMappedIndex(List fieldsNames) throws IOException { ResultSetTestCase.createIndex("test"); ResultSetTestCase.updateMapping("test", builder -> { for (String field : fieldsNames) { builder.startObject(field).field("type", field.substring(5)).endObject(); } }); + } + + public void doTestValidGetObjectCalls(List fieldsNames) throws IOException, SQLException { + createMappedIndex(fieldsNames); String q = "SELECT " + String.join(", ", fieldsNames) + " FROM test"; doWithQuery(q, r -> assertColumnNamesAndLabels(r.getMetaData(), fieldsNames)); @@ -66,6 +71,22 @@ public void testValidGetObjectCallsWithUnsignedLong() throws IOException, SQLExc doTestValidGetObjectCalls(singletonList(UNSIGNED_LONG_FIELD)); } + public void testUnsignedLongConditionallyReturnedOnStarExpansion() throws IOException, SQLException { + List fieldsNames = new ArrayList<>(FIELDS_NAMES); + fieldsNames.add(UNSIGNED_LONG_FIELD); + createMappedIndex(fieldsNames); + + String query = "SELECT * FROM test"; + doWithQuery(query, r -> { + List columnTypeNames = new ArrayList<>(fieldsNames.size()); + for (int i = 0; i < r.getMetaData().getColumnCount(); i++) { + columnTypeNames.add(r.getMetaData().getColumnTypeName(i + 1).toLowerCase(Locale.ROOT)); + } + // the assert executes only if UL is supported; the failing case would be triggered earlier in the driver already + assertEquals(isUnsignedLongSupported(), columnTypeNames.contains(UNSIGNED_LONG_TYPE_NAME.toLowerCase(Locale.ROOT))); + }); + } + private void doWithQuery(String query, CheckedConsumer consumer) throws SQLException { try (Connection connection = esJdbc()) { try (PreparedStatement statement = connection.prepareStatement(query)) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index f5976930ac417..3daf0d73baaf9 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -39,7 +39,6 @@ import org.elasticsearch.xpack.ql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.ql.rule.RuleExecutor; -import org.elasticsearch.xpack.ql.session.Configuration; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.type.InvalidMappedField; @@ -53,6 +52,7 @@ import org.elasticsearch.xpack.sql.plan.logical.Pivot; import org.elasticsearch.xpack.sql.plan.logical.SubQueryAlias; import org.elasticsearch.xpack.sql.plan.logical.With; +import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.type.SqlDataTypeConverter; import java.util.ArrayList; @@ -72,6 +72,7 @@ import static org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.AnalyzerRule; import static org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.BaseAnalyzerRule; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; public class Analyzer extends RuleExecutor { /** @@ -86,13 +87,13 @@ public class Analyzer extends RuleExecutor { * Per-request specific settings needed in some of the functions (timezone, username and clustername), * to which they are attached. */ - private final Configuration configuration; + private final SqlConfiguration configuration; /** * The verifier has the role of checking the analyzed tree for failures and build a list of failures. */ private final Verifier verifier; - public Analyzer(Configuration configuration, FunctionRegistry functionRegistry, IndexResolution results, Verifier verifier) { + public Analyzer(SqlConfiguration configuration, FunctionRegistry functionRegistry, IndexResolution results, Verifier verifier) { this.configuration = configuration; this.functionRegistry = functionRegistry; this.indexResolution = results; @@ -317,7 +318,7 @@ protected LogicalPlan rule(UnresolvedRelation plan) { } } - private static class ResolveRefs extends BaseAnalyzerRule { + private class ResolveRefs extends BaseAnalyzerRule { @Override protected LogicalPlan doRule(LogicalPlan plan) { @@ -434,10 +435,16 @@ private List expandProjections(List } } - return result; + return filterUnsupportedProjections(result); } - static List expandStar(UnresolvedStar us, List output) { + private List filterUnsupportedProjections(List projections) { + return projections.stream() + .filter(p -> p.resolved() == false || isTypeSupportedInVersion(p.dataType(), configuration.version())) + .collect(toList()); + } + + List expandStar(UnresolvedStar us, List output) { List expanded = new ArrayList<>(); // a qualifier is specified - since this is a star, it should be a CompoundDataType diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java index 9b81b88fb5cf2..8658dabb6263d 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java @@ -38,7 +38,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.SHORT; import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; import static org.elasticsearch.xpack.ql.type.DataTypes.isString; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedByClient; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.displaySize; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDataType; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDateTimeSub; @@ -173,7 +173,7 @@ private static void fillInRows(String clusterName, String indexName, Map output() { @Override public final void execute(SqlSession session, ActionListener listener) { Stream values = SqlDataTypes.types().stream() - .filter(t -> isTypeSupportedByClient(session.configuration().mode(), t, session.configuration().version())); + .filter(t -> isTypeSupportedInVersion(t, session.configuration().version())); if (type.intValue() != 0) { values = values.filter(t -> type.equals(sqlType(t).getVendorTypeNumber())); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java index e17b00c6940fd..de30328b21428 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java @@ -7,12 +7,10 @@ package org.elasticsearch.xpack.sql.session; import org.elasticsearch.xpack.ql.type.DataType; -import org.elasticsearch.xpack.sql.proto.Mode; import org.elasticsearch.xpack.sql.proto.SqlVersion; import static org.elasticsearch.Version.V_7_11_0; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; -import static org.elasticsearch.xpack.sql.proto.Mode.isDriver; public final class VersionCompatibilityChecks { @@ -20,9 +18,6 @@ public final class VersionCompatibilityChecks { private VersionCompatibilityChecks() {} - public static boolean isTypeSupportedByClient(Mode mode, DataType dataType, SqlVersion version) { - return isDriver(mode) == false || isTypeSupportedInVersion(dataType, version); - } /** * Is the provided {@code dataType} being supported in the provided {@code version}? */ @@ -36,6 +31,7 @@ public static boolean isTypeSupportedInVersion(DataType dataType, SqlVersion ver * Does the provided {@code version} support the unsigned_long type (PR#60050)? */ public static boolean supportsUnsignedLong(SqlVersion version) { - return INTRODUCING_UNSIGNED_LONG.compareTo(version) <= 0; + // TODO: add equality only once actually ported to 7.11 + return INTRODUCING_UNSIGNED_LONG.compareTo(version) < 0; } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java index 62ceabda3ca85..d44571e4db75d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.ql.plan.logical.Aggregate; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.plan.logical.Project; +import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.ql.type.TypesTests; import org.elasticsearch.xpack.sql.SqlTestUtils; @@ -38,6 +39,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.contains; @@ -327,8 +329,8 @@ public void testNonProjectedUnsignedLongVersionCompatibility() { Map mapping = TypesTests.loadMapping("mapping-numeric.json"); EsIndex index = new EsIndex("test", mapping); getIndexResult = IndexResolution.valid(index); - SqlVersion preIntroducingVersion = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); - SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preIntroducingVersion); + SqlVersion preUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); + SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preUnsignedLong); analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics(), sqlConfig.version())); String query = "SELECT unsigned_long = 1, unsigned_long::double FROM test"; @@ -346,6 +348,29 @@ public void testNonProjectedUnsignedLongVersionCompatibility() { } } + public void testUnsignedLongStarExpandedVersionControlled() { + Map mapping = TypesTests.loadMapping("mapping-numeric.json"); + EsIndex index = new EsIndex("test", mapping); + getIndexResult = IndexResolution.valid(index); + + SqlVersion preUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); + SqlVersion postUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER); + String query = "SELECT * FROM test"; + + for (SqlVersion version : List.of(preUnsignedLong, INTRODUCING_UNSIGNED_LONG, postUnsignedLong)) { + SqlConfiguration config = SqlTestUtils.randomConfiguration(version); + analyzer = new Analyzer(config, functionRegistry, getIndexResult, new Verifier(new Metrics(), config.version())); + + LogicalPlan plan = plan(query); + assertThat(plan, instanceOf(Project.class)); + Project p = (Project) plan; + + List projectedDataTypes = p.projections().stream().map(Expression::dataType).collect(Collectors.toList()); + assertEquals(isTypeSupportedInVersion(UNSIGNED_LONG, config.version()), projectedDataTypes.contains(UNSIGNED_LONG)); + } + + } + public void testFunctionOverNonExistingFieldAsArgumentAndSameAlias() throws Exception { Map mapping = TypesTests.loadMapping("mapping-basic.json"); EsIndex index = new EsIndex("test", mapping); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index 483eb27b8a3ab..f25833b44f455 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -44,7 +44,7 @@ import static org.elasticsearch.xpack.ql.TestUtils.UTC; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedByClient; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; @@ -325,7 +325,7 @@ public void testUnsignedLongFiltering() { SysColumns.fillInRows("test", "index", loadMapping("mapping-multi-field-variation.json", true), null, rows, null, mode, version); List types = rows.stream().map(row -> name(row).toString()).collect(Collectors.toList()); - assertEquals(isTypeSupportedByClient(mode, UNSIGNED_LONG, version), + assertEquals(isTypeSupportedInVersion(UNSIGNED_LONG, version), types.contains(UNSIGNED_LONG.toString().toLowerCase(Locale.ROOT))); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java index efc303059e537..c067b2d7a7c02 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java @@ -36,7 +36,7 @@ import static org.elasticsearch.action.ActionListener.wrap; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedByClient; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.mockito.Mockito.mock; public class SysTypesTests extends ESTestCase { @@ -104,7 +104,7 @@ public void testUnsignedLongFiltering() { SchemaRowSet r = (SchemaRowSet) p.rowSet(); List types = new ArrayList<>(); r.forEachRow(rv -> types.add((String) rv.column(0))); - assertEquals(isTypeSupportedByClient(mode, UNSIGNED_LONG, cmd.v2().configuration().version()), + assertEquals(isTypeSupportedInVersion(UNSIGNED_LONG, cmd.v2().configuration().version()), types.contains(UNSIGNED_LONG.toString())); }, ex -> fail(ex.getMessage()))); } From 2d64608fdb187bc83e8fa338e83acbb4dc25fcf0 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Mon, 30 Nov 2020 18:45:00 +0100 Subject: [PATCH 10/28] Revert FieldExtractorTestCase: no float to UL The UL mapper won't accept any floating point representation. --- .../elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java index e4b9745f15613..524bec8d71009 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java @@ -168,13 +168,14 @@ public void testWildcardField() throws IOException { } /* - * "unsigned_long/long/integer/short/byte_field": { + * "long/integer/short/byte_field": { * "type": "long/integer/short/byte" * } + * Note: no unsigned_long tested -- the mapper for it won't accept float formats. */ public void testFractionsForNonFloatingPointTypes() throws IOException { String floatingPointNumber = "123.456"; - String fieldType = randomFrom("unsigned_long", "long", "integer", "short", "byte"); + String fieldType = randomFrom("long", "integer", "short", "byte"); createIndexWithFieldTypeAndProperties(fieldType, null, null); index("{\"" + fieldType + "_field\":\"" + floatingPointNumber + "\"}"); From 754d0266a7349435d7b9266bb75686e28c94f100 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Tue, 8 Dec 2020 14:05:51 +0100 Subject: [PATCH 11/28] Address review comments - rename class; - make one function private; - undo one line break. --- .../operator/arithmetic/Arithmetics.java | 2 +- .../xpack/ql/type/DataTypeConverter.java | 4 ++-- ...signedLongUtils.java => NumericUtils.java} | 2 +- .../xpack/ql/util/StringUtils.java | 2 +- .../BinaryArithmeticProcessorTests.java | 4 ++-- .../ql/type/DataTypeConversionTests.java | 2 +- .../qa/jdbc/ResultSetMetaDataTestCase.java | 22 +++++++++---------- .../xpack/sql/qa/SqlProtocolTestCase.java | 2 +- .../xpack/sql/analysis/analyzer/Verifier.java | 18 ++++++--------- .../analyzer/FieldAttributeTests.java | 10 ++++----- 10 files changed, 31 insertions(+), 37 deletions(-) rename x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/{UnsignedLongUtils.java => NumericUtils.java} (97%) diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java index 75b0746aefeea..6bf4adb6557f2 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java @@ -10,7 +10,7 @@ import java.math.BigInteger; import java.util.function.BiFunction; -import static org.elasticsearch.xpack.ql.util.UnsignedLongUtils.asUnsignedLong; +import static org.elasticsearch.xpack.ql.util.NumericUtils.asUnsignedLong; /** * Arithmetic operation using the type widening rules of the JLS 5.6.2 namely diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java index eb53e6cccadef..6a0406c1985de 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java @@ -35,8 +35,8 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; import static org.elasticsearch.xpack.ql.type.DataTypes.isString; -import static org.elasticsearch.xpack.ql.util.UnsignedLongUtils.inUnsignedLongRange; -import static org.elasticsearch.xpack.ql.util.UnsignedLongUtils.isUnsignedLong; +import static org.elasticsearch.xpack.ql.util.NumericUtils.inUnsignedLongRange; +import static org.elasticsearch.xpack.ql.util.NumericUtils.isUnsignedLong; /** * Conversion utility from one Elasticsearch data type to another Elasticsearch data types. diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/UnsignedLongUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/NumericUtils.java similarity index 97% rename from x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/UnsignedLongUtils.java rename to x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/NumericUtils.java index 07d33e71ae74f..793cd1bc22c28 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/UnsignedLongUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/NumericUtils.java @@ -8,7 +8,7 @@ import java.math.BigInteger; -public abstract class UnsignedLongUtils { +public abstract class NumericUtils { // 18446744073709551615 public static final BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java index d66a33a4a2f29..544458cad8162 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java @@ -22,7 +22,7 @@ import java.util.Locale; import static java.util.stream.Collectors.toList; -import static org.elasticsearch.xpack.ql.util.UnsignedLongUtils.isUnsignedLong; +import static org.elasticsearch.xpack.ql.util.NumericUtils.isUnsignedLong; public final class StringUtils { diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java index 711f58ebc8635..252d159440c61 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java @@ -14,7 +14,7 @@ import org.elasticsearch.xpack.ql.expression.gen.processor.ConstantProcessor; import org.elasticsearch.xpack.ql.expression.gen.processor.Processor; import org.elasticsearch.xpack.ql.expression.processor.Processors; -import org.elasticsearch.xpack.ql.util.UnsignedLongUtils; +import org.elasticsearch.xpack.ql.util.NumericUtils; import java.math.BigInteger; @@ -64,7 +64,7 @@ public void testAddUnsignedLong() { Processor pn = new Add(EMPTY, l(BigInteger.valueOf(7)), l(-8)).makePipe().asProcessor(); expectThrows(ArithmeticException.class, () -> pn.process(null)); - Processor pm = new Add(EMPTY, l(UnsignedLongUtils.UNSIGNED_LONG_MAX), l(1)).makePipe().asProcessor(); + Processor pm = new Add(EMPTY, l(NumericUtils.UNSIGNED_LONG_MAX), l(1)).makePipe().asProcessor(); expectThrows(ArithmeticException.class, () -> pm.process(null)); } diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java index 6f7e9f571c07e..3e9c77aa2c631 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java @@ -32,7 +32,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; import static org.elasticsearch.xpack.ql.type.DateUtils.asDateTime; -import static org.elasticsearch.xpack.ql.util.UnsignedLongUtils.UNSIGNED_LONG_MAX; +import static org.elasticsearch.xpack.ql.util.NumericUtils.UNSIGNED_LONG_MAX; public class DataTypeConversionTests extends ESTestCase { diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java index e349d5db13f93..6d1ee2191e775 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java @@ -48,7 +48,17 @@ private static void createMappedIndex(List fieldsNames) throws IOExcepti }); } - public void doTestValidGetObjectCalls(List fieldsNames) throws IOException, SQLException { + public void testValidGetObjectCalls() throws IOException, SQLException { + doTestValidGetObjectCalls(FIELDS_NAMES); + } + + public void testValidGetObjectCallsWithUnsignedLong() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); + + doTestValidGetObjectCalls(singletonList(UNSIGNED_LONG_FIELD)); + } + + private void doTestValidGetObjectCalls(List fieldsNames) throws IOException, SQLException { createMappedIndex(fieldsNames); String q = "SELECT " + String.join(", ", fieldsNames) + " FROM test"; @@ -61,16 +71,6 @@ public void doTestValidGetObjectCalls(List fieldsNames) throws IOExcepti .map(x -> x.replace("_", "")).collect(Collectors.toList()))); } - public void testValidGetObjectCalls() throws IOException, SQLException { - doTestValidGetObjectCalls(FIELDS_NAMES); - } - - public void testValidGetObjectCallsWithUnsignedLong() throws IOException, SQLException { - assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support UNSIGNED_LONGs", isUnsignedLongSupported()); - - doTestValidGetObjectCalls(singletonList(UNSIGNED_LONG_FIELD)); - } - public void testUnsignedLongConditionallyReturnedOnStarExpansion() throws IOException, SQLException { List fieldsNames = new ArrayList<>(FIELDS_NAMES); fieldsNames.add(UNSIGNED_LONG_FIELD); diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java index 7004ecd340e07..463aacf2f30ea 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java @@ -28,7 +28,7 @@ import java.util.Locale; import java.util.Map; -import static org.elasticsearch.xpack.ql.util.UnsignedLongUtils.UNSIGNED_LONG_MAX; +import static org.elasticsearch.xpack.ql.util.NumericUtils.UNSIGNED_LONG_MAX; import static org.elasticsearch.xpack.sql.proto.Mode.CLI; import static org.elasticsearch.xpack.sql.proto.Protocol.SQL_QUERY_REST_ENDPOINT; import static org.elasticsearch.xpack.sql.proto.RequestInfo.CLIENT_IDS; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index 9e0df6a7a9930..aba81ae544bc5 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -891,16 +891,12 @@ private static void checkCastOnInexact(LogicalPlan p, Set localFailures } private static void checkClientSupportsDataTypes(LogicalPlan p, Set localFailures, SqlVersion version) { - List projects = p.collectFirstChildren(x -> x instanceof Project); - if (projects.size() > 0) { - ((Project) projects.get(0)).projections().forEach(e -> { - if (e.resolved() && isTypeSupportedInVersion(e.dataType(), version) == false) { - localFailures.add(fail(e, "Cannot use field [" + e.name() + "] with type [" + e.dataType() + "] unsupported " + - "in version [" + version + "], upgrade required (to version [" + INTRODUCING_UNSIGNED_LONG + - "] or higher)")); - } - }); - } - + p.output().forEach(e -> { + if (e.resolved() && isTypeSupportedInVersion(e.dataType(), version) == false) { + localFailures.add(fail(e, "Cannot use field [" + e.name() + "] with type [" + e.dataType() + "] unsupported " + + "in version [" + version + "], upgrade required (to version [" + INTRODUCING_UNSIGNED_LONG + + "] or higher)")); + } + }); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java index d44571e4db75d..6a2f3537ae4d9 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java @@ -296,9 +296,8 @@ public void testUnsignedLongVersionCompatibility() { String queryWithArithmetic = "SELECT unsigned_long + 1 AS unsigned_long FROM test"; String queryWithCast = "SELECT long + 1::unsigned_long AS unsigned_long FROM test"; - SqlVersion introducingUnsignedLong = INTRODUCING_UNSIGNED_LONG; - SqlVersion preUnsignedLong = SqlVersion.fromId(introducingUnsignedLong.id - SqlVersion.MINOR_MULTIPLIER); - SqlVersion postUnsignedLong = SqlVersion.fromId(introducingUnsignedLong.id + SqlVersion.MINOR_MULTIPLIER); + SqlVersion preUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); + SqlVersion postUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER); for (String sql : List.of(query, queryWithLiteral, queryWithAlias, queryWithArithmetic, queryWithCast)) { @@ -310,9 +309,8 @@ public void testUnsignedLongVersionCompatibility() { preUnsignedLong + "], upgrade required (to version [" + INTRODUCING_UNSIGNED_LONG + "] or higher)", ex.getMessage()); - for (SqlVersion v : List.of(introducingUnsignedLong, postUnsignedLong)) { - analyzer = new Analyzer(SqlTestUtils.randomConfiguration(v), functionRegistry, getIndexResult, - verifier); + for (SqlVersion v : List.of(INTRODUCING_UNSIGNED_LONG, postUnsignedLong)) { + analyzer = new Analyzer(SqlTestUtils.randomConfiguration(v), functionRegistry, getIndexResult, verifier); LogicalPlan plan = plan(sql); assertThat(plan, instanceOf(Project.class)); Project p = (Project) plan; From 46a072e56cf5903517eb21be06af6bf890a1e160 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Wed, 16 Dec 2020 17:28:15 +0100 Subject: [PATCH 12/28] Add 'SHOW COLUMNS' tests Add tests for `SHOW COLUMNS`. --- .../sql/plan/logical/command/ShowColumns.java | 10 ++- .../logical/command/ShowFunctionsTests.java | 83 +++++++++++++++++++ 2 files changed, 89 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java index d510e2c4926f8..c45821fbafeff 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.ql.type.KeywordEsField; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.session.Cursor.Page; import org.elasticsearch.xpack.sql.session.SqlSession; import org.elasticsearch.xpack.sql.type.SqlDataTypes; @@ -26,6 +27,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyList; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; public class ShowColumns extends Command { @@ -71,24 +73,24 @@ public void execute(SqlSession session, ActionListener listener) { List> rows = emptyList(); if (indexResult.isValid()) { rows = new ArrayList<>(); - fillInRows(indexResult.get().mapping(), null, rows); + fillInRows(indexResult.get().mapping(), null, session.configuration().version(), rows); } listener.onResponse(of(session, rows)); }, listener::onFailure)); } - private void fillInRows(Map mapping, String prefix, List> rows) { + static void fillInRows(Map mapping, String prefix, SqlVersion version, List> rows) { for (Entry e : mapping.entrySet()) { EsField field = e.getValue(); DataType dt = field.getDataType(); String name = e.getKey(); - if (dt != null) { + if (dt != null && isTypeSupportedInVersion(dt, version)) { // show only fields that exist in ES rows.add(asList(prefix != null ? prefix + "." + name : name, SqlDataTypes.sqlType(dt).getName(), dt.typeName())); if (field.getProperties().isEmpty() == false) { String newPrefix = prefix != null ? prefix + "." + name : name; - fillInRows(field.getProperties(), newPrefix, rows); + fillInRows(field.getProperties(), newPrefix, version, rows); } } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java index c7b06571a01e7..5c9e2477e908f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java @@ -6,17 +6,43 @@ package org.elasticsearch.xpack.sql.plan.logical.command; +import org.elasticsearch.Version; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.sql.SqlTestUtils; import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.session.SchemaRowSet; import org.elasticsearch.xpack.sql.session.SqlSession; +import java.sql.JDBCType; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static java.util.Arrays.asList; +import static java.util.stream.Collectors.toList; +import static org.elasticsearch.Version.CURRENT; import static org.elasticsearch.action.ActionListener.wrap; +import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; +import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; +import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; +import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; +import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT; +import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; +import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_POINT; +import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_SHAPE; +import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; public class ShowFunctionsTests extends ESTestCase { + public static final String JDBC_TYPE_GEOMETRY = "GEOMETRY"; + public void testShowFunctions() throws Exception { ShowFunctions showFunctions = new ShowFunctions(Source.EMPTY, null); SqlSession session = new SqlSession(SqlTestUtils.TEST_CFG, null, new SqlFunctionRegistry(), null, null, null, null, null, null); @@ -29,4 +55,61 @@ public void testShowFunctions() throws Exception { assertEquals("AGGREGATE", r.column(1)); }, ex -> fail(ex.getMessage()))); } + + public void testShowColumns() { + String prefix = "myIndex"; + List> rows = new ArrayList<>(); + ShowColumns.fillInRows(loadMapping("mapping-multi-field-variation.json", true), prefix, SqlVersion.fromId(CURRENT.id), rows); + + List> expect = asList( + asList("bool", JDBCType.BOOLEAN.getName(), BOOLEAN.typeName()), + asList("int", JDBCType.INTEGER.getName(), INTEGER.typeName()), + asList("unsigned_long", JDBCType.BIGINT.getName(), UNSIGNED_LONG.typeName()), + asList("text", JDBCType.VARCHAR.getName(), TEXT.typeName()), + asList("keyword", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("date", JDBCType.TIMESTAMP.getName(), DATETIME.typeName()), + asList("unsupported", JDBCType.OTHER.getName(), UNSUPPORTED.typeName()), + asList("some", JDBCType.STRUCT.getName(), OBJECT.typeName()), + asList("some.dotted", JDBCType.STRUCT.getName(), OBJECT.typeName()), + asList("some.dotted.field", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("some.string", JDBCType.VARCHAR.getName(), TEXT.typeName()), + asList("some.string.normalized", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("some.string.typical", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("some.ambiguous", JDBCType.VARCHAR.getName(), TEXT.typeName()), + asList("some.ambiguous.one", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("some.ambiguous.two", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("some.ambiguous.normalized", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("foo_type", JDBCType.OTHER.getName(), UNSUPPORTED.typeName()), + asList("point", JDBC_TYPE_GEOMETRY, GEO_POINT.typeName()), + asList("shape", JDBC_TYPE_GEOMETRY, GEO_SHAPE.typeName()) + ); + + + assertEquals(expect.size(), rows.size()); + assertEquals(expect.get(0).size(), rows.get(0).size()); + + for (int i = 0; i < expect.size(); i ++) { + List expectedRow = expect.get(i); + List receivedRow = rows.get(i); + assertEquals("Name mismatch in row " + i, prefix + "." + expectedRow.get(0), receivedRow.get(0)); + assertEquals("Type mismatch in row " + i, expectedRow.get(1), receivedRow.get(1)); + assertEquals("Mapping mismatch in row " + i, expectedRow.get(2), receivedRow.get(2)); + } + } + + public void testUnsignedLongFiltering() { + Set versions = new HashSet<>(List.of( + SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER), + INTRODUCING_UNSIGNED_LONG, + SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER), + SqlVersion.fromId(Version.CURRENT.id) + )); + + for (SqlVersion version : versions) { + List> rows = new ArrayList<>(); + ShowColumns.fillInRows(loadMapping("mapping-multi-field-variation.json", true), null, version, rows); + List typeNames = rows.stream().map(row -> (String) row.get(2)).collect(toList()); + assertEquals(isTypeSupportedInVersion(UNSIGNED_LONG, version), typeNames.contains(UNSIGNED_LONG.typeName())); + } + } } From 0897d8e762bd2b7a26f7583962faa2e5331cbc07 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Wed, 16 Dec 2020 17:56:28 +0100 Subject: [PATCH 13/28] Switch UNSIGNED_LONG to NUMERIC xDBC type Most client applications don't support/check unsigned types. Furthermore, seeing a BIGINT type, they assume it's signed (although xDBC APIs allow checking). Consequently the NUMERIC type offers more conversion safty. --- .../elasticsearch/xpack/sql/jdbc/EsType.java | 2 +- .../xpack/sql/jdbc/JdbcResultSetMetaData.java | 3 +-- .../xpack/sql/jdbc/TypeConverter.java | 24 +++++++++---------- .../server/src/main/resources/alias.csv-spec | 4 ++-- .../src/main/resources/command.csv-spec | 4 ++-- .../single-node-only/command-sys.csv-spec | 12 +++++----- .../xpack/sql/type/SqlDataTypes.java | 5 +++- .../logical/command/sys/SysColumnsTests.java | 4 ++-- .../logical/command/sys/SysTypesTests.java | 2 +- 9 files changed, 31 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java index af11cf8c9c7b6..c10c579a9c8de 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java @@ -19,7 +19,7 @@ public enum EsType implements SQLType { SHORT( Types.SMALLINT), INTEGER( Types.INTEGER), LONG( Types.BIGINT), - UNSIGNED_LONG( Types.BIGINT), + UNSIGNED_LONG( Types.NUMERIC), DOUBLE( Types.DOUBLE), FLOAT( Types.REAL), HALF_FLOAT( Types.FLOAT), diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSetMetaData.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSetMetaData.java index 75e03a240bc96..ce3f55cbc133d 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSetMetaData.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSetMetaData.java @@ -93,8 +93,7 @@ public int getPrecision(int column) throws SQLException { @Override public int getScale(int column) throws SQLException { - column(column); - return 0; + return column(column).displaySize(); } @Override diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java index 6c7f56d296b00..ca57def43a15b 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java @@ -228,7 +228,7 @@ static Object convert(Object v, EsType columnType, String typeString) throws SQL case LONG: return ((Number) v).longValue(); case UNSIGNED_LONG: - return new BigInteger(v.toString()); + return asBigInteger(v, columnType, typeString); case HALF_FLOAT: case SCALED_FLOAT: case DOUBLE: @@ -350,7 +350,7 @@ private static Byte asByte(Object val, EsType columnType, String typeString) thr case LONG: return safeToByte(((Number) val).longValue()); case UNSIGNED_LONG: - return safeToByte(((BigInteger) val)); + return safeToByte(asBigInteger(val, columnType, typeString)); case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -379,7 +379,7 @@ private static Short asShort(Object val, EsType columnType, String typeString) t case LONG: return safeToShort(((Number) val).longValue()); case UNSIGNED_LONG: - return safeToShort((BigInteger) val); + return safeToShort(asBigInteger(val, columnType, typeString)); case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -407,7 +407,7 @@ private static Integer asInteger(Object val, EsType columnType, String typeStrin case LONG: return safeToInt(((Number) val).longValue()); case UNSIGNED_LONG: - return safeToInt((BigInteger) val); + return safeToInt(asBigInteger(val, columnType, typeString)); case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -435,7 +435,7 @@ private static Long asLong(Object val, EsType columnType, String typeString) thr case LONG: return Long.valueOf(((Number) val).longValue()); case UNSIGNED_LONG: - return safeToLong((BigInteger) val); + return safeToLong(asBigInteger(val, columnType, typeString)); case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -469,7 +469,7 @@ private static Float asFloat(Object val, EsType columnType, String typeString) t case LONG: return Float.valueOf(((Number) val).longValue()); case UNSIGNED_LONG: - return ((BigInteger) val).floatValue(); + return asBigInteger(val, columnType, typeString).floatValue(); case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -497,7 +497,7 @@ private static Double asDouble(Object val, EsType columnType, String typeString) case LONG: return Double.valueOf(((Number) val).longValue()); case UNSIGNED_LONG: - return ((BigInteger) val).doubleValue(); + return asBigInteger(val, columnType, typeString).doubleValue(); case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: @@ -560,14 +560,14 @@ private static BigInteger asBigInteger(Object val, EsType columnType, String typ case SHORT: case INTEGER: case LONG: - return BigInteger.valueOf(((Number)val).longValue()); - case UNSIGNED_LONG: - return new BigInteger(val.toString()); + return BigInteger.valueOf(((Number) val).longValue()); case FLOAT: case HALF_FLOAT: case SCALED_FLOAT: case DOUBLE: - return BigDecimal.valueOf(((Number) val).doubleValue()).toBigInteger(); // no safeguarding limits checking needed + return BigDecimal.valueOf(((Number) val).doubleValue()).toBigInteger(); + // Aggs can return floats dressed as UL types (bugUrl="https://github.com/elastic/elasticsearch/issues/65413") + case UNSIGNED_LONG: case KEYWORD: case TEXT: try { @@ -590,7 +590,7 @@ private static BigDecimal asBigDecimal(Object val, EsType columnType, String typ case LONG: return BigDecimal.valueOf(((Number) val).longValue()); case UNSIGNED_LONG: - return new BigDecimal((BigInteger) val); + return new BigDecimal(asBigInteger(val, columnType, typeString)); case FLOAT: case HALF_FLOAT: // floats are passed in as doubles here, so we need to dip into string to keep original float's (reduced) precision. diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec index 67c31aef7993f..d07606d97b60d 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/alias.csv-spec @@ -52,7 +52,7 @@ last_name |VARCHAR |text last_name.keyword |VARCHAR |keyword null_constant |VARCHAR |keyword salary |INTEGER |integer -salary_ul |BIGINT |unsigned_long +salary_ul |NUMERIC |unsigned_long wildcard_name |VARCHAR |keyword ; @@ -83,7 +83,7 @@ last_name |VARCHAR |text last_name.keyword |VARCHAR |keyword null_constant |VARCHAR |keyword salary |INTEGER |integer -salary_ul |BIGINT |unsigned_long +salary_ul |NUMERIC |unsigned_long wildcard_name |VARCHAR |keyword ; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec index 03ec1ed604316..78b6b8bc4fdb0 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec @@ -296,7 +296,7 @@ last_name |VARCHAR |text last_name.keyword |VARCHAR |keyword null_constant |VARCHAR |keyword salary |INTEGER |integer -salary_ul |BIGINT |unsigned_long +salary_ul |NUMERIC |unsigned_long wildcard_name |VARCHAR |keyword ; @@ -327,7 +327,7 @@ last_name |VARCHAR |text last_name.keyword |VARCHAR |keyword null_constant |VARCHAR |keyword salary |INTEGER |integer -salary_ul |BIGINT |unsigned_long +salary_ul |NUMERIC |unsigned_long wildcard_name |VARCHAR |keyword ; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec index a8a3ccaaff75a..8a8346e89e783 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec @@ -43,7 +43,7 @@ integTest |null |test_emp |last_name |12 integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO integTest |null |test_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |salary_ul |-5 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |-5 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |18 |YES |null |null |null |null |NO |NO integTest |null |test_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO ; @@ -66,7 +66,7 @@ integTest |null |test_emp_copy|last_name |12 |TEX integTest |null |test_emp_copy|last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO integTest |null |test_emp_copy|null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO integTest |null |test_emp_copy|salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy|salary_ul |-5 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |-5 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy|salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |18 |YES |null |null |null |null |NO |NO integTest |null |test_emp_copy|wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO ; @@ -89,7 +89,7 @@ integTest |null |test_alias |last_name |12 |T integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |salary_ul |-5 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |-5 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |18 |YES |null |null |null |null |NO |NO integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO ; @@ -120,7 +120,7 @@ integTest |null |test_alias |last_name |12 integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |salary_ul |-5 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |-5 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |18 |YES |null |null |null |null |NO |NO integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO integTest |null |test_alias_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO integTest |null |test_alias_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO @@ -136,7 +136,7 @@ integTest |null |test_alias_emp |last_name |12 integTest |null |test_alias_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO integTest |null |test_alias_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO integTest |null |test_alias_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |salary_ul |-5 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |-5 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |18 |YES |null |null |null |null |NO |NO integTest |null |test_alias_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO @@ -162,6 +162,6 @@ integTest |null |test_emp_copy |last_name |12 integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO integTest |null |test_emp_copy |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |salary_ul |-5 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |-5 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |18 |YES |null |null |null |null |NO |NO integTest |null |test_emp_copy |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |19 |YES |null |null |null |null |NO |NO ; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java index cc7eda2020a00..8a5fb6621c705 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java @@ -316,9 +316,12 @@ public static SQLType sqlType(DataType dataType) { if (dataType == INTEGER) { return JDBCType.INTEGER; } - if (dataType == LONG || dataType == UNSIGNED_LONG) { + if (dataType == LONG) { return JDBCType.BIGINT; } + if (dataType == UNSIGNED_LONG) { + return JDBCType.NUMERIC; + } if (dataType == DOUBLE) { return JDBCType.DOUBLE; } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index f25833b44f455..4ad36870b92f7 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -84,7 +84,7 @@ public void testSysColumns() { row = rows.get(index++); assertEquals("unsigned_long", name(row)); - assertEquals(Types.BIGINT, sqlType(row)); + assertEquals(Types.NUMERIC, sqlType(row)); assertEquals(10, radix(row)); assertEquals(Long.BYTES, bufferLength(row)); @@ -184,7 +184,7 @@ public void sysColumnsInDriverMode(Mode mode) { row = rows.get(index++); assertEquals("unsigned_long", name(row)); - assertEquals(Types.BIGINT, typeClass.cast(sqlType(row)).intValue()); + assertEquals(Types.NUMERIC, typeClass.cast(sqlType(row)).intValue()); assertEquals(typeClass, radix(row).getClass()); assertEquals(Long.BYTES, bufferLength(row)); assertNull(decimalPrecision(row)); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java index c067b2d7a7c02..6839fa34c4f5c 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java @@ -58,7 +58,7 @@ private Tuple sql(String sql) { return sql(sql, randomFrom(Mode.values()), randomBoolean() ? null : SqlVersion.fromId(Version.CURRENT.id)); } - static final List ALL_TYPES = asList("BYTE", "LONG", "UNSIGNED_LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", + static final List ALL_TYPES = asList("BYTE", "LONG", "BINARY", "NULL", "UNSIGNED_LONG", "INTEGER", "SHORT", "HALF_FLOAT", "FLOAT", "DOUBLE", "SCALED_FLOAT", "IP", "KEYWORD", "TEXT", "BOOLEAN", "DATE", "TIME", "DATETIME", "INTERVAL_YEAR", "INTERVAL_MONTH", "INTERVAL_DAY", "INTERVAL_HOUR", "INTERVAL_MINUTE", "INTERVAL_SECOND", "INTERVAL_YEAR_TO_MONTH", "INTERVAL_DAY_TO_HOUR", "INTERVAL_DAY_TO_MINUTE", "INTERVAL_DAY_TO_SECOND", From 1b906ec5248815e5ba988fbd3b9c5f11ce98597f Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Wed, 16 Dec 2020 18:35:18 +0100 Subject: [PATCH 14/28] Update test - update SHOW commands to newly changed NUMERIC type; - pass a BigInteger value that won't fit into a Long for field extr. test. --- .../elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java | 4 +++- .../xpack/sql/plan/logical/command/ShowFunctionsTests.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java index 524bec8d71009..24f907ab74fa8 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.InputStream; +import java.math.BigInteger; import java.sql.JDBCType; import java.util.Arrays; import java.util.Collections; @@ -284,7 +285,8 @@ public void testByteFieldType() throws IOException { * } */ public void testUnsignedLongFieldType() throws IOException { - testField("unsigned_long", randomBigInteger()); + //randomBigInteger() can produce a value that fits into a Long, which is what testField() will then recover + testField("unsigned_long", BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.valueOf(randomNonNegativeLong()))); } private void testField(String fieldType, Object value) throws IOException { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java index 5c9e2477e908f..d096cff6a3f13 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java @@ -64,7 +64,7 @@ public void testShowColumns() { List> expect = asList( asList("bool", JDBCType.BOOLEAN.getName(), BOOLEAN.typeName()), asList("int", JDBCType.INTEGER.getName(), INTEGER.typeName()), - asList("unsigned_long", JDBCType.BIGINT.getName(), UNSIGNED_LONG.typeName()), + asList("unsigned_long", JDBCType.NUMERIC.getName(), UNSIGNED_LONG.typeName()), asList("text", JDBCType.VARCHAR.getName(), TEXT.typeName()), asList("keyword", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), asList("date", JDBCType.TIMESTAMP.getName(), DATETIME.typeName()), From f99eeec8e1541bea053f7ef9e894ee85a6f23cd3 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Wed, 16 Dec 2020 18:45:44 +0100 Subject: [PATCH 15/28] Style fix Update one comment line. --- .../org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java index 24f907ab74fa8..3026928ae0346 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java @@ -285,7 +285,7 @@ public void testByteFieldType() throws IOException { * } */ public void testUnsignedLongFieldType() throws IOException { - //randomBigInteger() can produce a value that fits into a Long, which is what testField() will then recover + // randomBigInteger() can produce a value that fits into a Long, which is what testField() will then recover testField("unsigned_long", BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.valueOf(randomNonNegativeLong()))); } From 9664e23c79c234b3a8e3beac35976a224cc34d15 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Thu, 27 May 2021 15:43:35 +0200 Subject: [PATCH 16/28] License header fix Fix headers of new files added to this PR before EL2. --- .../java/org/elasticsearch/xpack/ql/util/NumericUtils.java | 5 +++-- .../xpack/sql/session/VersionCompatibilityChecks.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/NumericUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/NumericUtils.java index 793cd1bc22c28..5ec05397816c9 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/NumericUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/NumericUtils.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.xpack.ql.util; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java index 9864d909b786e..2928426baa112 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.xpack.sql.session; From f8c20db0b8d778c323e525f2d2348d0d57a85b57 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Wed, 21 Jul 2021 18:06:58 +0200 Subject: [PATCH 17/28] Update target release.Update test to 4-dig. years - Update target release to 7.15 - Update test to the newly restricted 4-digits year. --- .../org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java | 6 +++--- .../elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java | 2 +- .../xpack/sql/session/VersionCompatibilityChecks.java | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java index 66dbebdc9929e..e6a056f4ee4f9 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java @@ -26,7 +26,7 @@ import java.util.LinkedHashMap; import java.util.Map; -import static org.elasticsearch.Version.V_7_14_0; +import static org.elasticsearch.Version.V_7_15_0; import static org.elasticsearch.common.time.DateUtils.toMilliSeconds; import static org.elasticsearch.test.ESTestCase.randomLongBetween; @@ -154,7 +154,7 @@ static boolean versionSupportsDateNanos() { } public static boolean isUnsignedLongSupported() { - // TODO: add equality only once actually ported to 7.14 - return V_7_14_0.compareTo(JDBC_DRIVER_VERSION) < 0; + // TODO: add equality only once actually ported to 7.15 + return V_7_15_0.compareTo(JDBC_DRIVER_VERSION) < 0; } } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java index 0b54edcec8d8b..8cb5f2c5ef5f8 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java @@ -786,7 +786,7 @@ public void testGettingInvalidBigInteger() throws IOException, SQLException { }); String randomString = randomUnicodeOfCodepointLengthBetween(128, 256); - long randomDate = randomNonNegativeLong(); + long randomDate = randomMillisUpToYear9999(); index("test", "1", builder -> { builder.field("test_keyword", randomString); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java index 2928426baa112..becf5274bb153 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java @@ -10,12 +10,12 @@ import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.sql.proto.SqlVersion; -import static org.elasticsearch.Version.V_7_14_0; +import static org.elasticsearch.Version.V_7_15_0; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; public final class VersionCompatibilityChecks { - public static final SqlVersion INTRODUCING_UNSIGNED_LONG = SqlVersion.fromId(V_7_14_0.id); + public static final SqlVersion INTRODUCING_UNSIGNED_LONG = SqlVersion.fromId(V_7_15_0.id); private VersionCompatibilityChecks() {} @@ -32,7 +32,7 @@ public static boolean isTypeSupportedInVersion(DataType dataType, SqlVersion ver * Does the provided {@code version} support the unsigned_long type (PR#60050)? */ public static boolean supportsUnsignedLong(SqlVersion version) { - // TODO: add equality only once actually ported to 7.14 + // TODO: add equality only once actually ported to 7.15 return INTRODUCING_UNSIGNED_LONG.compareTo(version) < 0; } } From 51a4f84b4fa80559a9df208c688e3e493cfec6bf Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Thu, 2 Sep 2021 21:06:15 +0200 Subject: [PATCH 18/28] Add Painless-driven testing Add queries to test unsigned long going through Painless scripts. --- .../operator/arithmetic/Arithmetics.java | 2 +- .../operator/comparison/Comparisons.java | 6 + .../BinaryComparisonProcessorTests.java | 13 + .../xpack/sql/qa/jdbc/JdbcTestUtils.java | 7 +- .../src/main/resources/unsigned-long.csv-spec | 357 ++++++++++++++++++ .../expression/function/aggregate/Sum.java | 4 +- .../session/VersionCompatibilityChecks.java | 6 +- 7 files changed, 386 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java index 2a2e63a980418..2543b62652795 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java @@ -180,7 +180,7 @@ static Number negate(Number n) { return Integer.valueOf(Math.negateExact(n.intValue())); } - private static BigInteger asBigInteger(Number n) { + public static BigInteger asBigInteger(Number n) { return n instanceof BigInteger ? (BigInteger) n : BigInteger.valueOf(n.longValue()); } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/Comparisons.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/Comparisons.java index cb4d94615cddb..cbe032c253ad8 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/Comparisons.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/Comparisons.java @@ -6,8 +6,11 @@ */ package org.elasticsearch.xpack.ql.expression.predicate.operator.comparison; +import java.math.BigInteger; import java.util.Set; +import static org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Arithmetics.asBigInteger; + /** * Comparison utilities. */ @@ -92,6 +95,9 @@ private static Integer compare(Number l, Number r) { if (l instanceof Float || r instanceof Float) { return Float.compare(l.floatValue(), r.floatValue()); } + if (l instanceof BigInteger || r instanceof BigInteger) { + return asBigInteger(l).compareTo(asBigInteger(r)); + } if (l instanceof Long || r instanceof Long) { return Long.compare(l.longValue(), r.longValue()); } diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/BinaryComparisonProcessorTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/BinaryComparisonProcessorTests.java index d95ff3a816225..944652b5de967 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/BinaryComparisonProcessorTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/BinaryComparisonProcessorTests.java @@ -14,6 +14,8 @@ import org.elasticsearch.xpack.ql.expression.gen.processor.ConstantProcessor; import org.elasticsearch.xpack.ql.expression.processor.Processors; +import java.math.BigInteger; + import static org.elasticsearch.xpack.ql.TestUtils.equalsOf; import static org.elasticsearch.xpack.ql.TestUtils.greaterThanOf; import static org.elasticsearch.xpack.ql.TestUtils.greaterThanOrEqualOf; @@ -50,6 +52,8 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { public void testEq() { assertEquals(true, equalsOf(l(4), l(4)).makePipe().asProcessor().process(null)); assertEquals(false, equalsOf(l(3), l(4)).makePipe().asProcessor().process(null)); + assertEquals(true, equalsOf(l(BigInteger.valueOf(4)), l(4L)).makePipe().asProcessor().process(null)); + assertEquals(false, equalsOf(l(BigInteger.valueOf(3)), l(4L)).makePipe().asProcessor().process(null)); } public void testNullEq() { @@ -63,30 +67,39 @@ public void testNullEq() { public void testNEq() { assertEquals(false, notEqualsOf(l(4), l(4)).makePipe().asProcessor().process(null)); assertEquals(true, notEqualsOf(l(3), l(4)).makePipe().asProcessor().process(null)); + assertEquals(true, notEqualsOf(l(BigInteger.valueOf(3)), l(4)).makePipe().asProcessor().process(null)); } public void testGt() { assertEquals(true, greaterThanOf(l(4), l(3)).makePipe().asProcessor().process(null)); assertEquals(false, greaterThanOf(l(3), l(4)).makePipe().asProcessor().process(null)); assertEquals(false, greaterThanOf(l(3), l(3)).makePipe().asProcessor().process(null)); + assertEquals(true, greaterThanOf(l(4), l(BigInteger.valueOf(3))).makePipe().asProcessor().process(null)); } public void testGte() { assertEquals(true, greaterThanOrEqualOf(l(4), l(3)).makePipe().asProcessor().process(null)); assertEquals(false, greaterThanOrEqualOf(l(3), l(4)).makePipe().asProcessor().process(null)); assertEquals(true, greaterThanOrEqualOf(l(3), l(3)).makePipe().asProcessor().process(null)); + assertEquals(true, greaterThanOrEqualOf(l(BigInteger.valueOf(3)), l(3L)).makePipe().asProcessor().process(null)); + assertEquals(true, greaterThanOrEqualOf(l(BigInteger.valueOf(4)), l(3L)).makePipe().asProcessor().process(null)); + assertEquals(false, greaterThanOrEqualOf(l(BigInteger.valueOf(3)), l(4L)).makePipe().asProcessor().process(null)); } public void testLt() { assertEquals(false, lessThanOf(l(4), l(3)).makePipe().asProcessor().process(null)); assertEquals(true, lessThanOf(l(3), l(4)).makePipe().asProcessor().process(null)); assertEquals(false, lessThanOf(l(3), l(3)).makePipe().asProcessor().process(null)); + assertEquals(false, lessThanOf(l(3), l(BigInteger.valueOf(3))).makePipe().asProcessor().process(null)); } public void testLte() { assertEquals(false, lessThanOrEqualOf(l(4), l(3)).makePipe().asProcessor().process(null)); assertEquals(true, lessThanOrEqualOf(l(3), l(4)).makePipe().asProcessor().process(null)); assertEquals(true, lessThanOrEqualOf(l(3), l(3)).makePipe().asProcessor().process(null)); + assertEquals(false, lessThanOrEqualOf(l(4), l(BigInteger.valueOf(3))).makePipe().asProcessor().process(null)); + assertEquals(true, lessThanOrEqualOf(l(3), l(BigInteger.valueOf(4))).makePipe().asProcessor().process(null)); + assertEquals(true, lessThanOrEqualOf(l(3), l(BigInteger.valueOf(3))).makePipe().asProcessor().process(null)); } public void testHandleNull() { diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java index e6a056f4ee4f9..013a7bdf00bf8 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java @@ -26,8 +26,7 @@ import java.util.LinkedHashMap; import java.util.Map; -import static org.elasticsearch.Version.V_7_15_0; - +import static org.elasticsearch.Version.V_7_16_0; import static org.elasticsearch.common.time.DateUtils.toMilliSeconds; import static org.elasticsearch.test.ESTestCase.randomLongBetween; @@ -154,7 +153,7 @@ static boolean versionSupportsDateNanos() { } public static boolean isUnsignedLongSupported() { - // TODO: add equality only once actually ported to 7.15 - return V_7_15_0.compareTo(JDBC_DRIVER_VERSION) < 0; + // TODO: add equality only once actually ported to 7.16 + return V_7_16_0.compareTo(JDBC_DRIVER_VERSION) < 0; } } diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec index 0213c40c3d1f3..d39aaf3230d27 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec @@ -97,3 +97,360 @@ SELECT salary_ul, salary_ul + languages AS s, languages FROM test_emp_copy WHERE 74999 |null |null 74970 |74973 |3 ; + +countWithImplicitGroupBy +SELECT MAX(salary_ul) AS m FROM test_emp_copy ORDER BY COUNT(*); + + m:ul +--------------- +74999 +; + +countWithImplicitGroupByWithHaving +SELECT MAX(salary_ul) AS m FROM test_emp_copy HAVING MIN(salary_ul) > 1 ORDER BY COUNT(*); + + m:ul +--------------- +74999 +; + +kurtosisAndSkewnessGroup +SELECT gender, KURTOSIS(salary_ul) k, SKEWNESS(salary_ul) s FROM test_emp_copy GROUP BY gender; + +gender:s | k:d | s:d + +null |2.2215791166941923 |-0.03373126000214023 +F |1.7873117044424276 |0.05504995122217512 +M |2.280646181070106 |0.44302407229580243 +; + +sumFieldWithSumLiteralAsCondition +SELECT first_name, last_name, SUM(salary_ul) AS s, birth_date AS y, COUNT(1) FROM test_emp_copy GROUP BY 1, 2, 4 HAVING ((SUM(1) >= 1) AND (SUM(1) <= 577)) AND ((SUM(salary_ul) >= 35000) AND (SUM(salary_ul) <= 45000)); + + first_name:s | last_name:s | s:ul | y:ts | COUNT(1):l +---------------+---------------+---------------+------------------------+--------------- +null |Brender |36051 |1959-10-01T00:00:00.000Z|1 +null |Joslin |37716 |1959-01-27T00:00:00.000Z|1 +null |Lortz |35222 |1960-07-20T00:00:00.000Z|1 +null |Makrucki |37691 |1963-07-22T00:00:00.000Z|1 +null |Swan |39878 |1962-12-29T00:00:00.000Z|1 +Alejandro |McAlpine |44307 |1953-09-19T00:00:00.000Z|1 +Amabile |Gomatam |38645 |1955-10-04T00:00:00.000Z|1 +Basil |Tramer |37853 |null |1 +Berhard |McFarlin |38376 |1954-10-01T00:00:00.000Z|1 +Berni |Genin |37137 |1956-02-12T00:00:00.000Z|1 +Chirstian |Koblick |36174 |1954-05-01T00:00:00.000Z|1 +Domenick |Tempesti |39356 |1963-11-26T00:00:00.000Z|1 +Hilari |Morton |37702 |1965-01-03T00:00:00.000Z|1 +Hisao |Lipner |40612 |1958-01-21T00:00:00.000Z|1 +Jayson |Mandell |43889 |1954-09-16T00:00:00.000Z|1 +Jungsoon |Syrzycki |39638 |1954-02-25T00:00:00.000Z|1 +Kendra |Hofting |44956 |1961-05-30T00:00:00.000Z|1 +Kenroku |Malabarba |35742 |1962-11-07T00:00:00.000Z|1 +Margareta |Bierman |41933 |1960-09-06T00:00:00.000Z|1 +Mayuko |Warwick |40031 |1952-12-24T00:00:00.000Z|1 +Mingsen |Casley |39728 |null |1 +Mokhtar |Bernatsky |38992 |1955-08-28T00:00:00.000Z|1 +Saniya |Kalloufi |43906 |1958-02-19T00:00:00.000Z|1 +Sreekrishna |Servieres |44817 |1961-09-23T00:00:00.000Z|1 +Sudharsan |Flasterstein |43602 |1963-03-21T00:00:00.000Z|1 +Vishv |Zockler |39110 |1959-07-23T00:00:00.000Z|1 +Weiyi |Meriste |37112 |null |1 +Yinghua |Dredge |43026 |1958-05-21T00:00:00.000Z|1 +Zvonko |Nyanchama |42716 |null |1 +; + +histogramNumeric +SELECT HISTOGRAM(salary_ul, 5000) AS h FROM test_emp_copy GROUP BY h; + + h:ul +--------------- +25000 +30000 +35000 +40000 +45000 +50000 +55000 +60000 +65000 +70000 +; + +medianAbsoluteDeviation +schema::gender:s|mad:d +SELECT gender, MAD(salary_ul) AS mad FROM test_emp_copy GROUP BY gender ORDER BY gender; + + gender | mad +---------------+--------------- +null |10789.0 +F |12719.0 +M |8905.0 +; + +groupAndAggNotSpecifiedInTheAggregateWithHaving +SELECT gender, MIN(salary_ul) AS min, COUNT(*) AS c FROM test_emp_copy GROUP BY gender HAVING c > 1 ORDER BY gender NULLS FIRST, MAX(salary_ul); + + gender:s | min:ul | c:l +---------------+---------------+--------------- +null |25324 |10 +F |25976 |33 +M |25945 |57 +; + +multipleAggsThatGetRewrittenWithAliasOnAMediumGroupByWithHaving +SELECT languages, MAX(salary_ul) AS max, MIN(salary_ul) AS min FROM test_emp_copy GROUP BY languages HAVING min BETWEEN 1000 AND 99999 ORDER BY max; + + languages:bt| max:ul | min:ul +---------------+---------------+--------------- +5 |66817 |25324 +2 |73578 |29175 +1 |73717 |25976 +4 |74572 |27215 +3 |74970 |26436 +null |74999 |28336 +; + +aggNotSpecifiedWithHavingOnLargeGroupBy +SELECT MAX(salary_ul) AS max FROM test_emp_copy GROUP BY emp_no HAVING AVG(salary_ul) > 1000 ORDER BY MIN(salary_ul) LIMIT 5; + + max:ul +--------------- +25324 +25945 +25976 +26436 +27215 +; + +multipleGroupingsAndOrderingByGroupsAndAggs +SELECT gender, MIN(salary_ul + 1) AS min, COUNT(*) AS c, MAX(salary_ul) AS max FROM test_emp_copy GROUP BY gender HAVING c > 1 ORDER BY gender DESC NULLS LAST, MAX(salary_ul) ASC; + + gender:s | min:ul | c:l | max:ul +---------------+---------------+---------------+--------------- +M |25946 |57 |74999 +F |25977 |33 |74572 +null |25325 |10 |73717 +; + +aggSumWithAliasWithColumnRepeatedWithOrderDesc +SELECT gender AS g, gender, SUM(salary_ul) AS s3, SUM(salary_ul), SUM(salary_ul) AS s5 FROM test_emp_copy GROUP BY g ORDER BY s5 DESC; + +g:s | gender:s | s3:ul | SUM(salary_ul):ul | s5:ul +-----+-----------+-------+-------------------+------ +M |M |2671054|2671054 |2671054 +F |F |1666196|1666196 |1666196 +null |null |487605 |487605 |487605 +; + +aggregateFunctionsWithScalars +SELECT MAX(CASE WHEN (salary_ul - 10) > 70000 THEN (salary_ul + 12345) * 1.2 ELSE (salary_ul - 12345) * 2.7 END) AS "max", +MIN(CASE WHEN (salary_ul - 20) > 50000 THEN (salary_ul * 1.2) - 1234 ELSE (salary_ul - 20) * 0.93 END) AS "min", +AVG(COS(salary_ul * 1.2) + 100 * (salary_ul / 5)) AS "avg", +SUM(-salary_ul / 0.765 + sin((salary_ul + 12345) / 12)) AS "sum", +MAD(ABS(salary_ul / -0.813) / 2 + (12345 * (salary_ul % 10))) AS "mad" +FROM test_emp_copy; + + max | min | avg | sum | mad +------------------+---------------+-----------------+------------------+----------------- +155409.30000000002|23532.72 |964937.9295477575|-6307004.517507723|30811.76199261993 +; + +aggregatesWithScalarsAndGroupByOrderByAggWithoutProjection +schema::gender:s +SELECT gender FROM test_emp_copy GROUP BY gender ORDER BY MAX(salary_ul % 100) DESC; + + gender +--------------- +M +null +F +; + +percentileAggregateFunctionsWithScalars +schema::percentile:d|percentile_rank:d|gender:s +SELECT PERCENTILE(CASE WHEN (salary_ul / 2) > 10000 THEN (salary_ul + 12345) * 1.2 ELSE (salary_ul - 12345) * 2.7 END, 80) AS "percentile", +PERCENTILE_RANK(CASE WHEN (salary_ul - 20) > 50000 THEN (salary_ul * 1.2) - 1234 ELSE (salary_ul - 20) * 0.93 END, 40000) AS "percentile_rank", +gender FROM test_emp_copy +GROUP BY gender ORDER BY gender; + + percentile | percentile_rank | gender +-----------------+------------------+--------------- +86857.79999999999|32.69659025378865 |null +94042.92000000001|37.03569653103581 |F +87348.36 |44.337514210592246|M +; + +extendedStatsAggregateFunctionsWithScalars +schema::stddev_pop:d|stddev_samp:d|sum_of_squares:d|var_pop:d|var_samp:d|gender:s +SELECT STDDEV_POP(CASE WHEN (salary_ul / 2) > 10000 THEN (salary_ul + 12345) * 1.2 ELSE (salary_ul - 12345) * 2.7 END) AS "stddev_pop", +STDDEV_SAMP(CASE WHEN (salary_ul / 2) > 10000 THEN (salary_ul + 12345) * 1.2 ELSE (salary_ul - 12345) * 2.7 END) AS "stddev_samp", +SUM_OF_SQUARES(CASE WHEN (salary_ul - 20) > 50000 THEN (salary_ul * 1.2) - 1234 ELSE (salary_ul - 20) * 0.93 END) AS "sum_of_squares", +VAR_POP(CASE WHEN (salary_ul - 20) % 1000 > 200 THEN (salary_ul * 1.2) - 1234 ELSE (salary_ul - 20) * 0.93 END) AS "var_pop", +VAR_SAMP(CASE WHEN (salary_ul - 20) % 1000 > 200 THEN (salary_ul * 1.2) - 1234 ELSE (salary_ul - 20) * 0.93 END) AS "var_samp", +gender FROM test_emp_copy +GROUP BY gender ORDER BY gender; + + stddev_pop | stddev_samp | sum_of_squares | var_pop | var_samp | gender +------------------+------------------+---------------------+--------------------+--------------------+--------------- +16752.73244172422 |17658.930515747525|3.06310583829007E10 |3.460331137445282E8 |3.844812374939202E8 |null +17427.462400181845|17697.67172930331 |1.148127725047658E11 |3.1723426960671306E8|3.271478405319228E8 |F +15702.798665784752|15842.381843421828|1.5882243113919238E11|2.529402043805585E8 |2.5745699374449703E8|M +; + +groupByRoundAndTruncateWithTwoParams +SELECT ROUND(SIN(TRUNCATE("salary_ul", 2)), 2) FROM "test_emp_copy" GROUP BY ROUND(SIN(TRUNCATE("salary_ul", 2)), 2) ORDER BY ROUND(SIN(TRUNCATE("salary_ul", 2)), 2) LIMIT 5; + +ROUND(SIN(TRUNCATE("salary_ul", 2)), 2) +--------------------------------------- +-1.0 +-0.99 +-0.98 +-0.97 +-0.96 +; + +sumLiteralAndSumFieldWithComplexHaving +SELECT gender, CAST(SUM("salary_ul") AS BIGINT), CAST(SUM(1) AS BIGINT), CAST(SUM(10) AS BIGINT), COUNT(*) FROM test_emp_copy GROUP BY gender HAVING ((SUM(1) >= 0) AND (SUM(1) <= 50) AND (SUM(salary_ul) >= 250000) AND (SUM(salary_ul) <= 5000000)) ORDER BY gender; + + gender:s |CAST(SUM("salary_ul") AS BIGINT):l|CAST(SUM(1) AS BIGINT):l|CAST(SUM(10) AS BIGINT):l| COUNT(*):l +---------------+----------------------------------+------------------------+-------------------------+--------------- +null |487605 |10 |100 |10 +F |1666196 |33 |330 |33 +; + +aggAvg +SELECT MAX(salary_ul)/100 AS max FROM test_emp_copy HAVING max = 749; + + max:ul +--------------- +749 +; + +aggGroupByOnScalarWithHaving +SELECT salary_ul + 1 AS e FROM test_emp_copy GROUP BY e HAVING AVG(salary_ul) BETWEEN 50000 AND 60000 ORDER BY e LIMIT 2; + + e:ul +--------------- +50065 +50129 +; + +aggMultiGroupByHavingWithMultipleScalarFunctionsBasedOnAliasFromGroupByAndAggNotInGroupBy +SELECT MIN(salary_ul) mi, MAX(salary_ul) ma, MAX(salary_ul) - MIN(salary_ul) AS d FROM test_emp_copy GROUP BY gender, languages HAVING ROUND((d - ABS(ma % mi))) + AVG(salary_ul) > 0 AND AVG(salary_ul) > 30000 ORDER BY gender, languages; + + mi:ul | ma:ul | d:ul +---------------+---------------+--------------- +48735 |73717 |24982 +56760 |61358 |4598 +45797 |45797 |0 +25324 |48942 |23618 +47896 |73851 |25955 +25976 |66174 |40198 +32263 |73578 |41315 +30404 |62405 |32001 +27215 |74572 |47357 +32272 |66817 |34545 +28336 |74999 |46663 +28035 |70011 |41976 +29175 |57305 |28130 +26436 |74970 |48534 +32568 |65367 |32799 +25945 |52833 |26888 +; + +aggMultiGroupByMultiWithHavingUsingInAndNullHandling +SELECT MIN(salary_ul) min, MAX(salary_ul) max, gender g, languages l, COUNT(*) c FROM "test_emp_copy" WHERE languages > 0 GROUP BY g, languages HAVING max IN (74572, null, 74970) ORDER BY gender, languages; + + min:ul | max:ul | g:s | l:bt | c:l +---------------+---------------+---------------+---------------+--------------- +27215 |74572 |F |4 |6 +26436 |74970 |M |3 |11 +; + +caseGroupByProtectedDivisionByZero +schema::x:ul +SELECT CASE WHEN languages = 1 THEN NULL ELSE ( salary_ul / (languages - 1) ) END AS x FROM test_emp_copy GROUP BY 1 ORDER BY 1 LIMIT 10; + + x +--------------- +null +6331 +6486 +7780 +7974 +8068 +8489 +8935 +9043 +9071 +; + +iifWithCompatibleIntervals +schema::hire_date + IIF(salary_ul > 70000, INTERVAL 2 HOURS, INTERVAL 2 DAYS):ts|salary_ul:ul +SELECT hire_date + IIF(salary_ul > 70000, INTERVAL 2 HOURS, INTERVAL 2 DAYS), salary_ul FROM test_emp_copy ORDER BY salary DESC LIMIT 10; + +hire_date + IIF(salary_ul > 70000, INTERVAL 2 HOURS, INTERVAL 2 DAYS)| salary_ul +---------------------------------------------------------------------+--------------- +1985-11-20T02:00:00.000Z |74999 +1989-09-02T02:00:00.000Z |74970 +1989-02-10T02:00:00.000Z |74572 +1989-07-07T02:00:00.000Z |73851 +1999-04-30T02:00:00.000Z |73717 +1988-10-18T02:00:00.000Z |73578 +1990-09-15T02:00:00.000Z |71165 +1987-03-18T02:00:00.000Z |70011 +1987-05-28T00:00:00.000Z |69904 +1990-02-18T00:00:00.000Z |68547 +; + +aggPercentile +SELECT languages, PERCENTILE(salary_ul, 95) AS "95th" FROM test_emp_copy GROUP BY languages; + + languages:bt| 95th:d +---------------+----------------- +null |74999.0 +1 |72790.5 +2 |71924.70000000001 +3 |73638.25 +4 |72115.59999999999 +5 |61071.7 +; + +mathPowerNegative +SELECT POWER(salary_ul, -1) m, first_name FROM "test_emp_copy" WHERE emp_no < 10010 ORDER BY emp_no; + + m | first_name +---------------------+--------------- +1.7450484250937964E-5|Georgi +1.773961788863068E-5 |Bezalel +1.617992071838848E-5 |Parto +2.76441643169127E-5 |Chirstian +1.57410905427528E-5 |Kyoichi +1.6574127786525235E-5|Anneke +1.3409858928284075E-5|Tzvetan +2.2775930396756706E-5|Saniya +1.5111675280321576E-5|Sumant +; + +averageWithOneValueAndOrder +schema::languages:bt|'F':d +SELECT * FROM (SELECT languages, gender, salary_ul FROM test_emp_copy) PIVOT (AVG(salary_ul + 1) FOR gender IN ('F')) ORDER BY languages DESC LIMIT 4; + languages | 'F' +---------------+------------------ +5 |46706.555555555555 +4 |49292.5 +3 |53661.0 +2 |50685.4 +; + +sumWithInnerAggregateSumOfSquaresRound +schema::birth_date:ts|emp_no:i|extra.info.gender:s|extra_gender:s|extra_no:i|first_name:s|gender:s|hire_date:ts|last_name:s|name:s|null_constant:s|salary:i|wildcard_name:s|1:d|2:d +SELECT * FROM test_emp_copy PIVOT (ROUND(SUM_OF_SQUARES(salary_ul + 1)/1E6, 2) FOR languages IN (1, 2)) LIMIT 3; + + birth_date | emp_no |extra.info.gender| extra_gender | extra_no | first_name | gender | hire_date | last_name | name | null_constant | salary | wildcard_name | 1 | 2 +---------------+---------------+-----------------+---------------+---------------+---------------+---------------+------------------------+---------------+---------------+---------------+---------------+---------------+---------------+--------------- +null |10041 |F |Female |10041 |Uri |F |1989-11-12T00:00:00.000Z|Lenart |Uri Lenart |null |56415 |Uri Lenart |3182.77 |null +null |10043 |M |Female |10043 |Yishay |M |1990-10-20T00:00:00.000Z|Tzvieli |Yishay Tzvieli |null |34341 |Yishay Tzvieli |1179.37 |null +null |10044 |F |Female |10044 |Mingsen |F |1994-05-21T00:00:00.000Z|Casley |Mingsen Casley |null |39728 |Mingsen Casley |1578.39 |null +; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Sum.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Sum.java index 19e5c00dc79b5..b2810b5bcff57 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Sum.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Sum.java @@ -16,6 +16,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE; import static org.elasticsearch.xpack.ql.type.DataTypes.LONG; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; /** * Sum all values of a field in matching documents. @@ -38,7 +39,8 @@ public Sum replaceChildren(List newChildren) { @Override public DataType dataType() { - return field().dataType().isInteger() ? LONG : DOUBLE; + DataType dt = field().dataType(); + return dt.isInteger() ? (dt == UNSIGNED_LONG ? UNSIGNED_LONG : LONG) : DOUBLE; } @Override diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java index becf5274bb153..ee2e2461cdc00 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java @@ -10,12 +10,12 @@ import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.sql.proto.SqlVersion; -import static org.elasticsearch.Version.V_7_15_0; +import static org.elasticsearch.Version.V_7_16_0; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; public final class VersionCompatibilityChecks { - public static final SqlVersion INTRODUCING_UNSIGNED_LONG = SqlVersion.fromId(V_7_15_0.id); + public static final SqlVersion INTRODUCING_UNSIGNED_LONG = SqlVersion.fromId(V_7_16_0.id); private VersionCompatibilityChecks() {} @@ -32,7 +32,7 @@ public static boolean isTypeSupportedInVersion(DataType dataType, SqlVersion ver * Does the provided {@code version} support the unsigned_long type (PR#60050)? */ public static boolean supportsUnsignedLong(SqlVersion version) { - // TODO: add equality only once actually ported to 7.15 + // TODO: add equality only once actually ported to 7.16 return INTRODUCING_UNSIGNED_LONG.compareTo(version) < 0; } } From 509892c27b242e77c6ee4c91cb19de5ed0c04206 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Thu, 23 Dec 2021 11:20:01 +0100 Subject: [PATCH 19/28] Extend value extraction as unsigned long The values returned for unsigned long fields needed further value and type conversion, like in the case of scripts returns, or type promotion, like in the case of the extractors. Testing has also been extended to include unsigned long values exceeding the Long range all the way up unsigned long upper limit. This revealed the need to further change some aggs implementation, like MAX/MIN delegating to LAST/FIRST. --- .../xpack/eql/plugin/eql_whitelist.txt | 1 + .../function/scalar/ScalarFunction.java | 13 +- .../whitelist/InternalQlScriptUtils.java | 5 + .../xpack/ql/type/DataTypeConverter.java | 8 + .../xpack/sql/qa/jdbc/DataLoader.java | 109 +++-- .../src/main/resources/command.csv-spec | 34 +- .../src/main/resources/logs_unsigned_long.csv | 102 +++++ .../multi-cluster-command-sys.csv-spec | 119 +++--- .../multi-cluster-command.csv-spec | 62 +-- .../single-node-only/command-sys.csv-spec | 155 ++++---- .../src/main/resources/slow/frozen.csv-spec | 1 + .../src/main/resources/unsigned-long.csv-spec | 371 ++++++++++++++---- .../xpack/sql/analysis/analyzer/Verifier.java | 6 +- .../xpack/sql/execution/search/Querier.java | 2 +- .../extractor/CompositeKeyExtractor.java | 59 ++- .../search/extractor/FieldHitExtractor.java | 7 + .../search/extractor/TopHitsAggExtractor.java | 12 + .../expression/function/aggregate/Sum.java | 2 +- .../xpack/sql/optimizer/Optimizer.java | 9 +- .../xpack/sql/planner/QueryFolder.java | 35 +- .../sql/querydsl/container/GroupByRef.java | 11 +- .../analyzer/VerifierErrorMessagesTests.java | 14 + .../extractor/CompositeKeyExtractorTests.java | 37 +- .../extractor/FieldHitExtractorTests.java | 14 + .../extractor/TopHitsAggExtractorTests.java | 11 + .../xpack/sql/optimizer/OptimizerTests.java | 104 ++--- 26 files changed, 886 insertions(+), 417 deletions(-) create mode 100644 x-pack/plugin/sql/qa/server/src/main/resources/logs_unsigned_long.csv diff --git a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt index c3722d377f39d..cbdd21d451ea5 100644 --- a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt +++ b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt @@ -19,6 +19,7 @@ class org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQl double nullSafeSortNumeric(Number) String nullSafeSortString(Object) Number nullSafeCastNumeric(Number, String) + Number nullSafeCastToUnsignedLong(Number) # # ASCII Functions diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java index 58c15c769ddb5..391ccb4a1ac92 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.ql.expression.function.Function; import org.elasticsearch.xpack.ql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.ql.expression.function.grouping.GroupingFunction; +import org.elasticsearch.xpack.ql.expression.gen.script.Params; import org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder; import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; import org.elasticsearch.xpack.ql.expression.gen.script.Scripts; @@ -24,10 +25,12 @@ import java.util.List; import static java.util.Collections.emptyList; +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder; import static org.elasticsearch.xpack.ql.expression.gen.script.Scripts.PARAM; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.LONG; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; /** * A {@code ScalarFunction} is a {@code Function} that takes values from some @@ -150,7 +153,15 @@ protected ScriptTemplate scriptWithGrouping(GroupingFunction grouping) { } protected ScriptTemplate scriptWithField(FieldAttribute field) { - return new ScriptTemplate(processScript(Scripts.DOC_VALUE), paramsBuilder().variable(field.name()).build(), dataType()); + Params params = paramsBuilder().variable(field.name()).build(); + // unsigned_long fields get returned in scripts as plain longs, so a conversion is required + return field.dataType() != UNSIGNED_LONG + ? new ScriptTemplate(processScript(Scripts.DOC_VALUE), params, dataType()) + : new ScriptTemplate( + processScript(formatTemplate(format("{ql}.", "nullSafeCastToUnsignedLong({})", Scripts.DOC_VALUE))), + params, + UNSIGNED_LONG + ); } protected String processScript(String script) { diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java index c975c876c9b12..25a6575dc7411 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java @@ -23,6 +23,7 @@ import java.util.Map; import static org.elasticsearch.xpack.ql.type.DataTypeConverter.convert; +import static org.elasticsearch.xpack.ql.type.DataTypeConverter.toUnsignedLong; import static org.elasticsearch.xpack.ql.type.DataTypes.fromTypeName; public class InternalQlScriptUtils { @@ -58,6 +59,10 @@ public static Number nullSafeCastNumeric(Number number, String typeName) { return number == null || Double.isNaN(number.doubleValue()) ? null : (Number) convert(number, fromTypeName(typeName)); } + public static Number nullSafeCastToUnsignedLong(Number number) { + return number == null || Double.isNaN(number.doubleValue()) ? null : toUnsignedLong(number); + } + // // Operators // diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java index 3f6891b757d8b..c2319ab8bde9d 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypeConverter.java @@ -37,6 +37,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.isDateTime; import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; import static org.elasticsearch.xpack.ql.type.DataTypes.isString; +import static org.elasticsearch.xpack.ql.util.NumericUtils.UNSIGNED_LONG_MAX; import static org.elasticsearch.xpack.ql.util.NumericUtils.inUnsignedLongRange; import static org.elasticsearch.xpack.ql.util.NumericUtils.isUnsignedLong; @@ -413,6 +414,13 @@ public static BigInteger safeToUnsignedLong(String x) { return bi; } + // "unsafe" value conversion to unsigned long (vs. "safe", type-only conversion of safeToUnsignedLong()); + // -1L -> 18446744073709551615 (=UNSIGNED_LONG_MAX) + public static BigInteger toUnsignedLong(Number number) { + BigInteger bi = BigInteger.valueOf(number.longValue()); + return bi.signum() < 0 ? bi.and(UNSIGNED_LONG_MAX) : bi; + } + public static Number toInteger(double x, DataType dataType) { long l = safeDoubleToLong(x); diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java index a5bdb40b96be0..bf2f77d7f0c96 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java @@ -67,6 +67,7 @@ protected static void loadEmpDatasetIntoEs(RestClient client) throws Exception { loadEmpDatasetWithExtraIntoEs(client, "test_emp_copy", "employees"); loadLogsDatasetIntoEs(client, "logs", "logs"); loadLogNanosDatasetIntoEs(client, "logs_nanos", "logs_nanos"); + loadLogUnsignedLongIntoEs(client, "logs_unsigned_long", "logs_unsigned_long"); makeAlias(client, "test_alias", "test_emp", "test_emp_copy"); makeAlias(client, "test_alias_emp", "test_emp", "test_emp_copy"); // frozen index @@ -269,14 +270,7 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String } protected static void loadLogsDatasetIntoEs(RestClient client, String index, String filename) throws Exception { - Request request = new Request("PUT", "/" + index); XContentBuilder createIndex = JsonXContent.contentBuilder().startObject(); - createIndex.startObject("settings"); - { - createIndex.field("number_of_shards", 1); - createIndex.field("number_of_replicas", 1); - } - createIndex.endObject(); createIndex.startObject("mappings"); { createIndex.startObject("properties"); @@ -292,82 +286,49 @@ protected static void loadLogsDatasetIntoEs(RestClient client, String index, Str } createIndex.endObject(); } - createIndex.endObject().endObject(); - request.setJsonEntity(Strings.toString(createIndex)); - client.performRequest(request); + createIndex.endObject(); - request = new Request("POST", "/" + index + "/_bulk?refresh=wait_for"); - request.addParameter("refresh", "true"); - StringBuilder bulk = new StringBuilder(); - csvToLines(filename, (titles, fields) -> { - bulk.append("{\"index\":{\"_id\":\"" + fields.get(0) + "\"}}\n"); - bulk.append("{"); - for (int f = 0; f < titles.size(); f++) { - if (Strings.hasText(fields.get(f))) { - if (f > 0) { - bulk.append(","); - } - bulk.append('"').append(titles.get(f)).append("\":\"").append(fields.get(f)).append('"'); - } - } - bulk.append("}\n"); - }); - request.setJsonEntity(bulk.toString()); - client.performRequest(request); + loadDatasetIntoEs(client, index, filename, createIndex); } protected static void loadLogNanosDatasetIntoEs(RestClient client, String index, String filename) throws Exception { - Request request = new Request("PUT", "/" + index); XContentBuilder createIndex = JsonXContent.contentBuilder().startObject(); - createIndex.startObject("settings"); + createIndex.startObject("mappings"); { - createIndex.field("number_of_shards", 1); - createIndex.field("number_of_replicas", 1); + createIndex.startObject("properties"); + { + createIndex.startObject("id").field("type", "integer").endObject(); + createIndex.startObject("@timestamp").field("type", "date_nanos").endObject(); + createIndex.startObject("status").field("type", "keyword").endObject(); + } + createIndex.endObject(); } createIndex.endObject(); + + loadDatasetIntoEs(client, index, filename, createIndex); + } + + protected static void loadLogUnsignedLongIntoEs(RestClient client, String index, String filename) throws Exception { + XContentBuilder createIndex = JsonXContent.contentBuilder().startObject(); createIndex.startObject("mappings"); { createIndex.startObject("properties"); { createIndex.startObject("id").field("type", "integer").endObject(); createIndex.startObject("@timestamp").field("type", "date_nanos").endObject(); + createIndex.startObject("bytes_in").field("type", "unsigned_long").endObject(); + createIndex.startObject("bytes_out").field("type", "unsigned_long").endObject(); createIndex.startObject("status").field("type", "keyword").endObject(); } createIndex.endObject(); } - createIndex.endObject().endObject(); - request.setJsonEntity(Strings.toString(createIndex)); - client.performRequest(request); + createIndex.endObject(); - request = new Request("POST", "/" + index + "/_bulk?refresh=wait_for"); - request.addParameter("refresh", "true"); - StringBuilder bulk = new StringBuilder(); - csvToLines(filename, (titles, fields) -> { - bulk.append("{\"index\":{\"_id\":\"" + fields.get(0) + "\"}}\n"); - bulk.append("{"); - for (int f = 0; f < titles.size(); f++) { - if (Strings.hasText(fields.get(f))) { - if (f > 0) { - bulk.append(","); - } - bulk.append('"').append(titles.get(f)).append("\":\"").append(fields.get(f)).append('"'); - } - } - bulk.append("}\n"); - }); - request.setJsonEntity(bulk.toString()); - client.performRequest(request); + loadDatasetIntoEs(client, index, filename, createIndex); } protected static void loadLibDatasetIntoEs(RestClient client, String index) throws Exception { - Request request = new Request("PUT", "/" + index); XContentBuilder createIndex = JsonXContent.contentBuilder().startObject(); - createIndex.startObject("settings"); - { - createIndex.field("number_of_shards", 1); - createIndex.field("number_of_replicas", 1); - } - createIndex.endObject(); createIndex.startObject("mappings"); { createIndex.startObject("properties"); @@ -379,21 +340,39 @@ protected static void loadLibDatasetIntoEs(RestClient client, String index) thro } createIndex.endObject(); } - createIndex.endObject().endObject(); + createIndex.endObject(); + + loadDatasetIntoEs(client, index, "library", createIndex); + } + + // createIndex must be startObject()'d, but not endObject()'d: the index settings are added at the end + protected static void loadDatasetIntoEs(RestClient client, String index, String filename, XContentBuilder createIndex) + throws Exception { + createIndex.startObject("settings"); + { + createIndex.field("number_of_shards", 1); + createIndex.field("number_of_replicas", 1); + } + createIndex.endObject(); + createIndex.endObject(); + + Request request = new Request("PUT", "/" + index); request.setJsonEntity(Strings.toString(createIndex)); client.performRequest(request); request = new Request("POST", "/" + index + "/_bulk?refresh=wait_for"); request.addParameter("refresh", "true"); StringBuilder bulk = new StringBuilder(); - csvToLines("library", (titles, fields) -> { + csvToLines(filename, (titles, fields) -> { bulk.append("{\"index\":{\"_id\":\"" + fields.get(0) + "\"}}\n"); bulk.append("{"); for (int f = 0; f < titles.size(); f++) { - if (f > 0) { - bulk.append(","); + if (fields.size() > f && Strings.hasText(fields.get(f))) { + if (f > 0) { + bulk.append(","); + } + bulk.append('"').append(titles.get(f)).append("\":\"").append(fields.get(f)).append('"'); } - bulk.append('"').append(titles.get(f)).append("\":\"").append(fields.get(f)).append('"'); } bulk.append("}\n"); }); diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec index 5c71a1043dbd8..c3f3af296adb3 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec @@ -231,14 +231,15 @@ integTest |local showTables SHOW TABLES; -catalog | name | type | kind -integTest |empty_mapping |TABLE |INDEX -integTest |logs |TABLE |INDEX -integTest |logs_nanos |TABLE |INDEX -integTest |test_alias |VIEW |ALIAS -integTest |test_alias_emp |VIEW |ALIAS -integTest |test_emp |TABLE |INDEX -integTest |test_emp_copy |TABLE |INDEX +catalog | name | type | kind +integTest |empty_mapping |TABLE |INDEX +integTest |logs |TABLE |INDEX +integTest |logs_nanos |TABLE |INDEX +integTest |logs_unsigned_long |TABLE |INDEX +integTest |test_alias |VIEW |ALIAS +integTest |test_alias_emp |VIEW |ALIAS +integTest |test_emp |TABLE |INDEX +integTest |test_emp_copy |TABLE |INDEX ; showTablesSimpleLike @@ -281,14 +282,15 @@ integTest |test_alias_emp |VIEW |ALIAS showTablesLocalCatalog SHOW TABLES CATALOG 'integTest'; -catalog | name | type | kind -integTest |empty_mapping |TABLE |INDEX -integTest |logs |TABLE |INDEX -integTest |logs_nanos |TABLE |INDEX -integTest |test_alias |VIEW |ALIAS -integTest |test_alias_emp |VIEW |ALIAS -integTest |test_emp |TABLE |INDEX -integTest |test_emp_copy |TABLE |INDEX +catalog | name | type | kind +integTest |empty_mapping |TABLE |INDEX +integTest |logs |TABLE |INDEX +integTest |logs_nanos |TABLE |INDEX +integTest |logs_unsigned_long |TABLE |INDEX +integTest |test_alias |VIEW |ALIAS +integTest |test_alias_emp |VIEW |ALIAS +integTest |test_emp |TABLE |INDEX +integTest |test_emp_copy |TABLE |INDEX ; // DESCRIBE diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/logs_unsigned_long.csv b/x-pack/plugin/sql/qa/server/src/main/resources/logs_unsigned_long.csv new file mode 100644 index 0000000000000..cd4c3d136caa5 --- /dev/null +++ b/x-pack/plugin/sql/qa/server/src/main/resources/logs_unsigned_long.csv @@ -0,0 +1,102 @@ +id,@timestamp,bytes_in,bytes_out,status +1,2017-11-10T21:15:54Z,4348801185987554667,12749081495402663265,OK +2,2017-11-10T21:15:39Z,11054572642507476486,2215793005711196537,OK +3,2017-11-10T21:15:39Z,7239423344688551324,4747671420480199905,OK +4,2017-11-10T21:15:39Z,12880875341157998416,10347160802894727455,OK +5,2017-11-10T21:15:40Z,6480569113728286781,4628689249901172357,OK +6,2017-11-10T21:15:40Z,8847365258155648277,18107197698386620672,OK +7,2017-11-10T21:15:40Z,18081123477485622121,6254036056888007861,OK +8,2017-11-10T21:15:41Z,17159009460398071592,6041947699951197416,OK +9,2017-11-10T21:15:41Z,18317075104972913640,3738987414350619907,OK +10,2017-11-10T20:36:07Z,9223372036854775807,13014552081688587417,OK +11,2017-11-10T20:36:08Z,10618481193158417699,7645257133789254601,OK +12,2017-11-10T20:36:07Z,14243423136348863449,1851693232606252132,OK +13,2017-11-10T20:36:07Z,8014838889043461601,12855878692699288887,OK +14,2017-11-10T20:36:15Z,9704166250476073712,9243820354371174974,OK +15,2017-11-10T20:36:15Z,16673466483681919036,17281501450843634251,OK +16,2017-11-10T20:35:54Z,11414099303186823563,4552407785188434877,OK +17,2017-11-10T20:35:54Z,9614024902524991937,583785103161450865,OK +18,2017-11-10T20:35:55Z,2703254959364209157,15688732125935676003,OK +19,2017-11-10T17:54:43Z,16907772202142018796,1978055896356244912,OK +20,2017-11-10T23:23:24Z,18446744073709551614,9891957732954625161,OK +21,2017-11-10T17:54:59Z,18098466156271475039,10560599221675458546,OK +22,2017-11-10T21:13:27Z,12113814789427553914,17695317925249333633,OK +23,2017-11-10T22:37:41Z,369412756671598363,4454824974559554214,OK +24,2017-11-10T20:34:43Z,17764691215469285192,751496841062464739,OK +25,2017-11-10T23:30:46Z,316080452389500167,13471731928228498458,OK +26,2017-11-10T21:13:16Z,3987249898147090269,857017108209908030,OK +27,2017-11-10T23:36:32Z,9343007301895818617,13652755194722568502,OK +28,2017-11-10T23:36:33Z,12951716972543168268,9336652471323200906,OK +29,2017-11-10T20:35:26Z,16002960716282089759,6754707638562449159,OK +30,2017-11-10T23:36:41Z,18446744073709550591,14393839423240122480,OK +31,2017-11-10T23:56:36Z,5495907774457032585,8384790841458113028,OK +32,2017-11-10T20:29:25Z,905851433235877972,11682551086136399874,Error +33,2017-11-10T21:35:01Z,4368413537705409055,10386906319745215430,OK +34,2017-11-10T21:12:17Z,7953313938735885073,14008282674840239286,OK +35,2017-11-10T23:17:14Z,9188929021194043442,991636820083925493,OK +36,2017-11-10T23:28:11Z,13875498092704386047,17953153966527637143,OK +37,2017-11-10T22:36:27Z,8156660980420095219,901610289258538340,OK +38,2017-11-10T20:35:55Z,2408213296071189837,419872666232023984,OK +39,2017-11-10T20:35:55Z,17460378829280278708,10724795375261191248,OK +40,2017-11-10T20:35:55Z,18446744073709551614,14524142879756567901,OK +41,2017-11-10T20:35:55Z,,,Error +42,2017-11-10T21:34:49Z,154551962150890561,4317649615355527138,Error +43,2017-11-10T20:35:55Z,6713823401157015713,768392740554438381,OK +44,2017-11-10T20:14:04Z,13007085541148329579,1262767764958640849,OK +45,2017-11-10T19:38:06Z,4008445367955620676,2444837981761911481,OK +46,2017-11-10T21:14:18Z,9056948257586320738,3660006000364826492,OK +47,2017-11-10T20:35:56Z,10640542847470647209,3071012467454913482,OK +48,2017-11-10T20:53:05Z,14463699407888333801,16193000254773667372,OK +49,2017-11-10T21:25:42Z,4691003749418709874,16735032755695343779,OK +50,2017-11-10T21:14:44Z,18446744073709551615,8359170160363687272,OK +51,2017-11-10T21:28:34Z,10414368669933920698,17857609920324506371,OK +52,2017-11-10T20:35:55Z,14591698995327831783,837800054257171070,OK +53,2017-11-10T20:15:24Z,9149768745019330607,9934783425401329847,OK +54,2017-11-10T20:35:57Z,5826090293715995525,13263580863583654980,OK +55,2017-11-10T17:14:10Z,15352019942832250739,1498178946494790227,OK +56,2017-11-10T20:35:57Z,9732690250707058359,2520919358333960813,OK +57,2017-11-10T23:22:13Z,8914368988247035466,16187631537609304549,OK +58,2017-11-10T20:32:57Z,8420006392678593250,14938622925960605968,OK +59,2017-11-10T21:24:00Z,17056885385468285787,9973198429366930442,OK +60,2017-11-10T20:35:56Z,9223372036854775808,6620615504579533702,OK +61,2017-11-10T23:43:10Z,2390976293435536689,16020561580624977312,OK +62,2017-11-10T20:35:57Z,10993546521190430203,18184253384683076090,OK +63,2017-11-10T20:21:58Z,5246566629176459718,9382204513185396493,OK +64,2017-11-10T20:35:57Z,9983398877364735609,10626289664367265415,OK +65,2017-11-10T20:33:06Z,5480608687137202404,6895880056122579688,Error +66,2017-11-10T20:35:57Z,7538807943450220608,11745980216826561015,OK +67,2017-11-10T20:26:21Z,17067060651018256448,1722789377000665830,OK +68,2017-11-10T21:23:25Z,16873365461162643186,10056378788277261033,OK +69,2017-11-10T21:23:54Z,9991932520184465636,16110121334900810541,OK +70,2017-11-10T20:35:57Z,0,2507200025082562692,OK +71,2017-11-10T00:27:03Z,0,18223615477147360166,OK +72,2017-11-10T00:27:46Z,0,11206857258468587792,OK +73,2017-11-10T20:35:58Z,13986802678251316321,1330575423003442317,OK +74,2017-11-10T20:35:57Z,13922094693483143156,14343149449348005776,OK +75,2017-11-10T22:27:09Z,13999070515664268533,8422074124513216267,OK +76,2017-11-10T20:35:58Z,15968566213936682639,3784845108080773823,OK +77,2017-11-10T22:26:44Z,1729864283282545225,11105009496753939058,OK +78,2017-11-10T22:27:31Z,14241624006161076477,11563896463355414928,OK +79,2017-11-10T20:35:52Z,2294690022638798960,14564159158999105001,OK +80,2017-11-10T00:00:22Z,0,11060623717086222747,OK +81,2017-11-10T20:35:52Z,7470203340634956368,7490193999241578548,OK +82,2017-11-10T00:01:20Z,74330435873664882,4875216609683497742,OK +83,2017-11-10T00:01:04Z,9636626466125797351,14208813483941526550,OK +84,2017-11-10T00:32:48Z,11949176856304796477,8190769023162854115,OK +85,2017-11-10T00:01:45Z,754822992931077409,12647826153259487490,OK +86,2017-11-10T20:36:08Z,16424089095262982944,12394320926003300611,OK +87,2017-11-10T21:17:37Z,10580536762493152413,13605535835272740587,OK +88,2017-11-10T20:06:49Z,195161570976258241,15395084776572180858,Error +89,2017-11-10T21:17:37Z,15084788733189711518,6353233118260828721,OK +90,2017-11-10T19:51:38Z,,,Error +91,2017-11-10T19:51:38Z,11628588779507401305,8500236459902170712,Error +92,2017-11-10T20:06:50Z,2706408999083639864,594246218266628121,OK +93,2017-11-10T21:17:46Z,9007528787465012783,15931740851225178582,OK +94,2017-11-10T19:51:38Z,18345360876889252152,16119381686035586648,Error +95,2017-11-10T21:17:46Z,2788944430410706777,11087293691148056886,OK +96,2017-11-10T00:04:50Z,9932469097722733505,14925592145374204307,OK +97,2017-11-10T21:17:48Z,11620953158540412267,3809712277266935082,OK +98,2017-11-10T21:12:24Z,3448205404634246112,5409549730889481641,OK +99,2017-11-10T21:17:37Z,1957665857956635540,352442273299370793,OK +100,2017-11-10T03:21:36Z,16462768484251021236,15616395223975497926,OK +101,2017-11-10T23:22:36Z,,,Error diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/multi-cluster-with-security/multi-cluster-command-sys.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/multi-cluster-with-security/multi-cluster-command-sys.csv-spec index 2af437c274828..843b7ea3aa056 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/multi-cluster-with-security/multi-cluster-command-sys.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/multi-cluster-with-security/multi-cluster-command-sys.csv-spec @@ -119,59 +119,65 @@ my_remote_cluster|null |test_alias |wildcard_name |12 sysColumnsAllTables SYS COLUMNS CATALOG 'my_remote_cluster' TABLE LIKE '%'; - TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s ------------------+---------------+---------------+------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -my_remote_cluster|null |logs |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |logs |client_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |logs |dest_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |logs_nanos |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |logs_nanos |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |logs_nanos |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |3 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp |name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |12 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |19 |YES |null |null |null |null |NO |NO -my_remote_cluster|null |test_emp_copy |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |20 |YES |null |null |null |null |NO |NO + TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s +-----------------+---------------+-------------------+------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ +my_remote_cluster|null |logs |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs |client_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs |dest_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs_nanos |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs_nanos |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs_nanos |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |3 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs_unsigned_long |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs_unsigned_long |bytes_in |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |2 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs_unsigned_long |bytes_out |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |3 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs_unsigned_long |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |4 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |logs_unsigned_long |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp |name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |12 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |19 |YES |null |null |null |null |NO |NO +my_remote_cluster|null |test_emp_copy |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |20 |YES |null |null |null |null |NO |NO ; sysTablesSimple SYS TABLES CATALOG LIKE 'my_remote_cluster'; - TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | TABLE_TYPE:s | REMARKS:s | TYPE_CAT:s | TYPE_SCHEM:S | TYPE_NAME:s |SELF_REFERENCING_COL_NAME:s|REF_GENERATION:s ------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+-------------------------+--------------- -my_remote_cluster|null |empty_mapping |TABLE | |null |null |null |null |null -my_remote_cluster|null |logs |TABLE | |null |null |null |null |null -my_remote_cluster|null |logs_nanos |TABLE | |null |null |null |null |null -my_remote_cluster|null |test_emp |TABLE | |null |null |null |null |null -my_remote_cluster|null |test_emp_copy |TABLE | |null |null |null |null |null + TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | TABLE_TYPE:s | REMARKS:s | TYPE_CAT:s | TYPE_SCHEM:S | TYPE_NAME:s |SELF_REFERENCING_COL_NAME:s|REF_GENERATION:s +-----------------+---------------+-------------------+---------------+---------------+---------------+---------------+---------------+-------------------------+--------------- +my_remote_cluster|null |empty_mapping |TABLE | |null |null |null |null |null +my_remote_cluster|null |logs |TABLE | |null |null |null |null |null +my_remote_cluster|null |logs_nanos |TABLE | |null |null |null |null |null +my_remote_cluster|null |logs_unsigned_long |TABLE | |null |null |null |null |null +my_remote_cluster|null |test_emp |TABLE | |null |null |null |null |null +my_remote_cluster|null |test_emp_copy |TABLE | |null |null |null |null |null ; sysTablesIndexLikeFilter @@ -190,16 +196,17 @@ SYS TABLES CATALOG LIKE 'my_remote_cluster' "empty*"; my_remote_cluster|null |empty_mapping |TABLE | |null |null |null |null |null ; -sysTablesEmptyFilters +sysTablesEmptyFiltersExceptCatalog SYS TABLES CATALOG LIKE 'my_remote_cluster' "" TYPE ''; - TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | TABLE_TYPE:s | REMARKS:s | TYPE_CAT:s | TYPE_SCHEM:S | TYPE_NAME:s |SELF_REFERENCING_COL_NAME:s|REF_GENERATION:s ------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+-------------------------+--------------- -my_remote_cluster|null |empty_mapping |TABLE | |null |null |null |null |null -my_remote_cluster|null |logs |TABLE | |null |null |null |null |null -my_remote_cluster|null |logs_nanos |TABLE | |null |null |null |null |null -my_remote_cluster|null |test_emp |TABLE | |null |null |null |null |null -my_remote_cluster|null |test_emp_copy |TABLE | |null |null |null |null |null + TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | TABLE_TYPE:s | REMARKS:s | TYPE_CAT:s | TYPE_SCHEM:S | TYPE_NAME:s |SELF_REFERENCING_COL_NAME:s|REF_GENERATION:s +-----------------+---------------+-------------------+---------------+---------------+---------------+---------------+---------------+-------------------------+--------------- +my_remote_cluster|null |empty_mapping |TABLE | |null |null |null |null |null +my_remote_cluster|null |logs |TABLE | |null |null |null |null |null +my_remote_cluster|null |logs_nanos |TABLE | |null |null |null |null |null +my_remote_cluster|null |logs_unsigned_long |TABLE | |null |null |null |null |null +my_remote_cluster|null |test_emp |TABLE | |null |null |null |null |null +my_remote_cluster|null |test_emp_copy |TABLE | |null |null |null |null |null ; sysTablesCatalogEnumeration diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/multi-cluster-with-security/multi-cluster-command.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/multi-cluster-with-security/multi-cluster-command.csv-spec index 19ff982c25e78..8b32f9bdc7d17 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/multi-cluster-with-security/multi-cluster-command.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/multi-cluster-with-security/multi-cluster-command.csv-spec @@ -18,38 +18,41 @@ my_remote_cluster |remote showTables SHOW TABLES CATALOG 'my_remote_cluster'; - catalog | name | type | kind ------------------+---------------+---------------+--------------- -my_remote_cluster|empty_mapping |TABLE |INDEX -my_remote_cluster|logs |TABLE |INDEX -my_remote_cluster|logs_nanos |TABLE |INDEX -my_remote_cluster|test_emp |TABLE |INDEX -my_remote_cluster|test_emp_copy |TABLE |INDEX + catalog | name | type | kind +-----------------+-------------------+---------------+--------------- +my_remote_cluster|empty_mapping |TABLE |INDEX +my_remote_cluster|logs |TABLE |INDEX +my_remote_cluster|logs_nanos |TABLE |INDEX +my_remote_cluster|logs_unsigned_long |TABLE |INDEX +my_remote_cluster|test_emp |TABLE |INDEX +my_remote_cluster|test_emp_copy |TABLE |INDEX ; showTablesWithLike SHOW TABLES CATALOG LIKE 'my_remote_%'; - catalog | name | type | kind ------------------+---------------+---------------+--------------- -my_remote_cluster|empty_mapping |TABLE |INDEX -my_remote_cluster|logs |TABLE |INDEX -my_remote_cluster|logs_nanos |TABLE |INDEX -my_remote_cluster|test_emp |TABLE |INDEX -my_remote_cluster|test_emp_copy |TABLE |INDEX + catalog | name | type | kind +-----------------+-------------------+---------------+--------------- +my_remote_cluster|empty_mapping |TABLE |INDEX +my_remote_cluster|logs |TABLE |INDEX +my_remote_cluster|logs_nanos |TABLE |INDEX +my_remote_cluster|logs_unsigned_long |TABLE |INDEX +my_remote_cluster|test_emp |TABLE |INDEX +my_remote_cluster|test_emp_copy |TABLE |INDEX ; showTablesWithFrozen SHOW TABLES CATALOG 'my_remote_cluster' INCLUDE FROZEN; - catalog | name | type | kind ------------------+---------------+---------------+--------------- -my_remote_cluster|empty_mapping |TABLE |INDEX -my_remote_cluster|frozen_emp |TABLE |INDEX -my_remote_cluster|logs |TABLE |INDEX -my_remote_cluster|logs_nanos |TABLE |INDEX -my_remote_cluster|test_emp |TABLE |INDEX -my_remote_cluster|test_emp_copy |TABLE |INDEX + catalog | name | type | kind +-----------------+-------------------+---------------+--------------- +my_remote_cluster|empty_mapping |TABLE |INDEX +my_remote_cluster|frozen_emp |TABLE |INDEX +my_remote_cluster|logs |TABLE |INDEX +my_remote_cluster|logs_nanos |TABLE |INDEX +my_remote_cluster|logs_unsigned_long |TABLE |INDEX +my_remote_cluster|test_emp |TABLE |INDEX +my_remote_cluster|test_emp_copy |TABLE |INDEX ; showTablesSimpleLike @@ -109,13 +112,14 @@ test_alias_emp |VIEW |ALIAS showTablesAllCatalogs SHOW TABLES CATALOG '*'; - catalog | name | type | kind ------------------+---------------+---------------+--------------- -my_remote_cluster|empty_mapping |TABLE |INDEX -my_remote_cluster|logs |TABLE |INDEX -my_remote_cluster|logs_nanos |TABLE |INDEX -my_remote_cluster|test_emp |TABLE |INDEX -my_remote_cluster|test_emp_copy |TABLE |INDEX + catalog | name | type | kind +-----------------+-------------------+---------------+--------------- +my_remote_cluster|empty_mapping |TABLE |INDEX +my_remote_cluster|logs |TABLE |INDEX +my_remote_cluster|logs_nanos |TABLE |INDEX +my_remote_cluster|logs_unsigned_long |TABLE |INDEX +my_remote_cluster|test_emp |TABLE |INDEX +my_remote_cluster|test_emp_copy |TABLE |INDEX ; // DESCRIBE diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec index 9a4bf744b9164..006b1ffa3e37f 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec @@ -100,79 +100,84 @@ integTest |null |test_alias |wildcard_name |12 sysColumnsAllTables SYS COLUMNS TABLE LIKE '%'; - TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i|BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s ------------------+---------------+---------------+------------------+---------------+----------------+---------------+---------------+----------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -integTest |null |logs |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO -integTest |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |logs |client_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO -integTest |null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO -integTest |null |logs |dest_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO -integTest |null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO -integTest |null |logs_nanos |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |logs_nanos |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO -integTest |null |logs_nanos |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |3 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |19 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |20 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |19 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |20 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |19 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |20 |YES |null |null |null |null |NO |NO + TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i|BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s +-----------------+---------------+-------------------+------------------+---------------+----------------+---------------+---------------+----------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ +integTest |null |logs |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO +integTest |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |logs |client_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO +integTest |null |logs |dest_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO +integTest |null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO +integTest |null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +integTest |null |logs_nanos |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |logs_nanos |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO +integTest |null |logs_nanos |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |3 |YES |null |null |null |null |NO |NO +integTest |null |logs_unsigned_long |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |logs_unsigned_long |bytes_in |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |2 |YES |null |null |null |null |NO |NO +integTest |null |logs_unsigned_long |bytes_out |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |logs_unsigned_long |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |logs_unsigned_long |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |19 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |20 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |19 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |20 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |birth_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |hire_date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |null_constant |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |17 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |18 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |salary_ul |2 |UNSIGNED_LONG |20 |8 |null |10 |1 |null |null |2 |0 |null |19 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |wildcard_name |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |20 |YES |null |null |null |null |NO |NO ; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/slow/frozen.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/slow/frozen.csv-spec index 2b5da3da451f5..b68d6e49443f1 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/slow/frozen.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/slow/frozen.csv-spec @@ -12,6 +12,7 @@ integTest |empty_mapping |TABLE |INDEX integTest |frozen_emp |TABLE |FROZEN INDEX integTest |logs |TABLE |INDEX integTest |logs_nanos |TABLE |INDEX +integTest |logs_unsigned_long |TABLE |INDEX integTest |test_alias |VIEW |ALIAS integTest |test_alias_emp |VIEW |ALIAS integTest |test_emp |TABLE |INDEX diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec index d39aaf3230d27..2b4df08dc7618 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/unsigned-long.csv-spec @@ -106,14 +106,6 @@ SELECT MAX(salary_ul) AS m FROM test_emp_copy ORDER BY COUNT(*); 74999 ; -countWithImplicitGroupByWithHaving -SELECT MAX(salary_ul) AS m FROM test_emp_copy HAVING MIN(salary_ul) > 1 ORDER BY COUNT(*); - - m:ul ---------------- -74999 -; - kurtosisAndSkewnessGroup SELECT gender, KURTOSIS(salary_ul) k, SKEWNESS(salary_ul) s FROM test_emp_copy GROUP BY gender; @@ -127,7 +119,7 @@ M |2.280646181070106 |0.44302407229580243 sumFieldWithSumLiteralAsCondition SELECT first_name, last_name, SUM(salary_ul) AS s, birth_date AS y, COUNT(1) FROM test_emp_copy GROUP BY 1, 2, 4 HAVING ((SUM(1) >= 1) AND (SUM(1) <= 577)) AND ((SUM(salary_ul) >= 35000) AND (SUM(salary_ul) <= 45000)); - first_name:s | last_name:s | s:ul | y:ts | COUNT(1):l + first_name:s | last_name:s | s:d | y:ts | COUNT(1):l ---------------+---------------+---------------+------------------------+--------------- null |Brender |36051 |1959-10-01T00:00:00.000Z|1 null |Joslin |37716 |1959-01-27T00:00:00.000Z|1 @@ -198,19 +190,6 @@ F |25976 |33 M |25945 |57 ; -multipleAggsThatGetRewrittenWithAliasOnAMediumGroupByWithHaving -SELECT languages, MAX(salary_ul) AS max, MIN(salary_ul) AS min FROM test_emp_copy GROUP BY languages HAVING min BETWEEN 1000 AND 99999 ORDER BY max; - - languages:bt| max:ul | min:ul ----------------+---------------+--------------- -5 |66817 |25324 -2 |73578 |29175 -1 |73717 |25976 -4 |74572 |27215 -3 |74970 |26436 -null |74999 |28336 -; - aggNotSpecifiedWithHavingOnLargeGroupBy SELECT MAX(salary_ul) AS max FROM test_emp_copy GROUP BY emp_no HAVING AVG(salary_ul) > 1000 ORDER BY MIN(salary_ul) LIMIT 5; @@ -236,7 +215,7 @@ null |25325 |10 |73717 aggSumWithAliasWithColumnRepeatedWithOrderDesc SELECT gender AS g, gender, SUM(salary_ul) AS s3, SUM(salary_ul), SUM(salary_ul) AS s5 FROM test_emp_copy GROUP BY g ORDER BY s5 DESC; -g:s | gender:s | s3:ul | SUM(salary_ul):ul | s5:ul +g:s | gender:s | s3:d | SUM(salary_ul):d | s5:d -----+-----------+-------+-------------------+------ M |M |2671054|2671054 |2671054 F |F |1666196|1666196 |1666196 @@ -247,13 +226,13 @@ aggregateFunctionsWithScalars SELECT MAX(CASE WHEN (salary_ul - 10) > 70000 THEN (salary_ul + 12345) * 1.2 ELSE (salary_ul - 12345) * 2.7 END) AS "max", MIN(CASE WHEN (salary_ul - 20) > 50000 THEN (salary_ul * 1.2) - 1234 ELSE (salary_ul - 20) * 0.93 END) AS "min", AVG(COS(salary_ul * 1.2) + 100 * (salary_ul / 5)) AS "avg", -SUM(-salary_ul / 0.765 + sin((salary_ul + 12345) / 12)) AS "sum", +SUM(salary_ul / 0.765 + sin((salary_ul + 12345) / 12)) AS "sum", MAD(ABS(salary_ul / -0.813) / 2 + (12345 * (salary_ul % 10))) AS "mad" FROM test_emp_copy; - max | min | avg | sum | mad -------------------+---------------+-----------------+------------------+----------------- -155409.30000000002|23532.72 |964937.9295477575|-6307004.517507723|30811.76199261993 + max | min | avg | sum | mad +------------------+---------------+-----------------+-----------------+----------------- +155409.30000000002|23532.72 |964937.9295477575|6306995.482492277|30811.76199261993 ; aggregatesWithScalarsAndGroupByOrderByAggWithoutProjection @@ -319,14 +298,6 @@ null |487605 |10 |100 F |1666196 |33 |330 |33 ; -aggAvg -SELECT MAX(salary_ul)/100 AS max FROM test_emp_copy HAVING max = 749; - - max:ul ---------------- -749 -; - aggGroupByOnScalarWithHaving SELECT salary_ul + 1 AS e FROM test_emp_copy GROUP BY e HAVING AVG(salary_ul) BETWEEN 50000 AND 60000 ORDER BY e LIMIT 2; @@ -336,38 +307,6 @@ SELECT salary_ul + 1 AS e FROM test_emp_copy GROUP BY e HAVING AVG(salary_ul) BE 50129 ; -aggMultiGroupByHavingWithMultipleScalarFunctionsBasedOnAliasFromGroupByAndAggNotInGroupBy -SELECT MIN(salary_ul) mi, MAX(salary_ul) ma, MAX(salary_ul) - MIN(salary_ul) AS d FROM test_emp_copy GROUP BY gender, languages HAVING ROUND((d - ABS(ma % mi))) + AVG(salary_ul) > 0 AND AVG(salary_ul) > 30000 ORDER BY gender, languages; - - mi:ul | ma:ul | d:ul ----------------+---------------+--------------- -48735 |73717 |24982 -56760 |61358 |4598 -45797 |45797 |0 -25324 |48942 |23618 -47896 |73851 |25955 -25976 |66174 |40198 -32263 |73578 |41315 -30404 |62405 |32001 -27215 |74572 |47357 -32272 |66817 |34545 -28336 |74999 |46663 -28035 |70011 |41976 -29175 |57305 |28130 -26436 |74970 |48534 -32568 |65367 |32799 -25945 |52833 |26888 -; - -aggMultiGroupByMultiWithHavingUsingInAndNullHandling -SELECT MIN(salary_ul) min, MAX(salary_ul) max, gender g, languages l, COUNT(*) c FROM "test_emp_copy" WHERE languages > 0 GROUP BY g, languages HAVING max IN (74572, null, 74970) ORDER BY gender, languages; - - min:ul | max:ul | g:s | l:bt | c:l ----------------+---------------+---------------+---------------+--------------- -27215 |74572 |F |4 |6 -26436 |74970 |M |3 |11 -; - caseGroupByProtectedDivisionByZero schema::x:ul SELECT CASE WHEN languages = 1 THEN NULL ELSE ( salary_ul / (languages - 1) ) END AS x FROM test_emp_copy GROUP BY 1 ORDER BY 1 LIMIT 10; @@ -454,3 +393,301 @@ null |10041 |F |Female |10041 null |10043 |M |Female |10043 |Yishay |M |1990-10-20T00:00:00.000Z|Tzvieli |Yishay Tzvieli |null |34341 |Yishay Tzvieli |1179.37 |null null |10044 |F |Female |10044 |Mingsen |F |1994-05-21T00:00:00.000Z|Casley |Mingsen Casley |null |39728 |Mingsen Casley |1578.39 |null ; + + +implicitAggAndCastMaxLargeInput +SELECT MAX(bytes_out)::DOUBLE xd, MAX(bytes_in)::STRING xs FROM logs_unsigned_long; + + xd:d | xs:s +--------------------+-------------------- +1.822361547714736E19|18446744073709551615 +; + +implicitAggFirstLast +SELECT FIRST(bytes_in)::BYTE xd, FIRST(bytes_out)::LONG xs FROM logs_unsigned_long; + + xd:bt | xs:l +---------------+------------------ +0 |352442273299370793 +; + +withMathFunctionLargeInput +SELECT ROUND(SIN(bytes_out), 3) AS sin, bytes_out AS out FROM logs_unsigned_long ORDER BY sin DESC NULLS LAST LIMIT 3; + + sin:d | out:ul +---------------+-------------------- +0.999 |8422074124513216267 +0.999 |11563896463355414928 +0.997 |16110121334900810541 +; + +withMathIntoInLargeInput +SELECT id, bytes_in, bytes_out, + ((bytes_out::DOUBLE + CASE WHEN status = 'OK' THEN bytes_in ELSE 0 END)/POWER(1024, 3))::UNSIGNED_LONG AS traffic_gb +FROM logs_unsigned_long +WHERE bytes_in IN (9223372036854775807::UNSIGNED_LONG * 2, 18446744073709551614, '18446744073709551615'::UNSIGNED_LONG); + + id:i | bytes_in:ul | bytes_out:ul | traffic_gb:ul +---------------+--------------------+--------------------+--------------- +20 |18446744073709551614|9891957732954625161 |26392472727 +40 |18446744073709551614|14524142879756567901|30706531324 +50 |18446744073709551615|8359170160363687272 |24964953059 +; + +countWithImplicitGroupByAndArithmeticLargeInput +SELECT MAX(bytes_out) max, MIN(bytes_out) min, MAX(bytes_out) - MIN(bytes_out) diff FROM logs_unsigned_long ORDER BY COUNT(*); + + max:ul | min:ul | diff:ul +--------------------+------------------+-------------------- +18223615477147360166|352442273299370793|17871173203847989373 + +; + +kurtosisAndSkewnessGroupLargeInput +SELECT status, KURTOSIS(bytes_in) k, SKEWNESS(bytes_in) s FROM logs_unsigned_long GROUP BY status; + + status | k | s +---------------+------------------+-------------------- +Error |2.056501455580364 |0.7571349191844446 +OK |1.9651462247673082|-0.15628563062367107 +; + + +fieldWithSumLiteralAsConditionLargeInput +SELECT status, SUM(bytes_in) AS s, "@timestamp" AS y, COUNT(1) FROM logs_unsigned_long GROUP BY 1, 3 HAVING SUM(1) >= 3; + + status:s | s:d | y:ts | COUNT(1):l +---------------+---------------------+------------------------+--------------- +Error |2.9973949656396653E19|2017-11-10T19:51:38.000Z|3 +OK |6.232411355491008E19 |2017-11-10T20:35:55.000Z|6 +OK |5.799662857991158E19 |2017-11-10T20:35:57.000Z|7 +OK |3.14816340622471E19 |2017-11-10T20:36:07.000Z|3 +OK |3.1174871328354025E19|2017-11-10T21:15:39.000Z|3 +OK |3.3409057849369555E19|2017-11-10T21:15:40.000Z|3 +OK |2.76229913536395E19 |2017-11-10T21:17:37.000Z|3 +; + +histogramNumericLargeInput +SELECT HISTOGRAM(bytes_in, POWER(10, 9)) AS gb FROM logs_unsigned_long GROUP BY 1 LIMIT 10; + + gb:ul +------------------- +null +0 +74330435000000000 +154551962000000000 +195161570000000000 +316080452000000000 +369412756000000000 +754822992000000000 +905851433000000000 +1729864283000000000 +; + +medianAbsoluteDeviationLargeInput +SELECT status, MAD(bytes_in) AS mad FROM logs_unsigned_long GROUP BY status ORDER BY status; + + status | mad +---------------+--------------------- +Error |3.0183732936229658E18 +OK |4.5240953206634045E18 +; + + +groupAndAggNotSpecifiedInTheAggregateWithHavingLargeInput +SELECT status, MIN(bytes_out) AS min, COUNT(*) AS c FROM logs_unsigned_long GROUP BY 1 HAVING c > 1 ORDER BY 1 NULLS FIRST, MAX(bytes_out); + + status:s | min:ul | c:l +---------------+-------------------+--------------- +Error |4317649615355527138|9 +OK |352442273299370793 |92 +; + +aggNotSpecifiedWithHavingOnLargeGroupByLargeInput +SELECT MAX(bytes_in) AS max FROM logs_unsigned_long GROUP BY id HAVING AVG(bytes_in) > 1000 ORDER BY MIN(bytes_out) LIMIT 5; + + max:ul +-------------------- +1957665857956635540 +2408213296071189837 +9614024902524991937 +2706408999083639864 +17764691215469285192 +; + +multipleGroupingsAndOrderingByGroupsAndAggsLargeInput +SELECT status, MIN(bytes_out + 1) AS min, COUNT(*) AS c, MAX(bytes_in) AS max FROM logs_unsigned_long GROUP BY status HAVING c > 1 ORDER BY status DESC NULLS LAST, MAX(bytes_in) ASC; + + status:s | min:ul | c:l | max:ul +---------------+------------------+---------------+-------------------- +OK |352442273299370794|92 |18446744073709551615 +Error |null |9 |18345360876889252152 +; + +aggSumWithAliasWithColumnRepeatedWithOrderDescLargeInput +SELECT status AS s, status, SUM(bytes_in) AS s3, SUM(bytes_in), SUM(bytes_in) AS s5 FROM logs_unsigned_long GROUP BY s ORDER BY s5 DESC; + + s:s | status:s | s3:d | SUM(bytes_in):d | s5:d +---------------+---------------+--------------------+--------------------+-------------------- +OK |OK |9.045581107970589E20|9.045581107970589E20|9.045581107970589E20 +Error |Error |3.671012330989688E19|3.671012330989688E19|3.671012330989688E19 +; + +aggregateFunctionsWithScalarsLargeInput + +SELECT MAX(CASE WHEN (bytes_out - 10) > 9223372036854775807 THEN (bytes_out + 12345) * 1.2 ELSE (bytes_out - 12345) * 2.7 END) AS max, +MIN(CASE WHEN (bytes_out - 20) > 9223372036854775807 THEN (bytes_out * 1.2) - 1234 ELSE (bytes_out - 20) * 0.93 END) AS min, +AVG((COS(bytes_out * 1.2) + 100) * (bytes_out / 5)) AS avg, +SUM(bytes_out / 0.765 + sin((bytes_out + 12345) / 12)) AS sum, +MAD(ABS(bytes_out / -0.813) / 2 + (12345 * (bytes_out % 10))) AS mad +FROM logs_unsigned_long; + + max:d | min:d | avg:d | sum:d | mad:d +--------------------+---------------------+--------------------+---------------------+--------------------- +2.295063844173583E19|3.2777131416841485E17|1.820279192886511E20|1.1661879109640491E21|3.0865540937819786E18 +; + + +aggregatesWithScalarsAndGroupByOrderByAggWithoutProjectionLargeInput +SELECT status, MAX(bytes_in % 100) max FROM logs_unsigned_long GROUP BY status ORDER BY 2 DESC; + + status:s | max:ul +---------------+--------------- +OK |99 +Error |72 +; + +percentileAggregateFunctionsWithScalarsLargeInput +SELECT PERCENTILE(CASE WHEN (bytes_out / 2) > 4E18 THEN (bytes_out + 12345) * 1.2 ELSE (bytes_out - 12345) * 2.7 END, 80) AS percentile, +PERCENTILE_RANK(CASE WHEN (bytes_out - 20) > 4E18 THEN (bytes_out * 1.2) - 1234 ELSE (bytes_out - 20) * 0.93 END, 4E18) AS percentile_rank, +status +FROM logs_unsigned_long +GROUP BY status +ORDER BY status; + + percentile | percentile_rank | status +---------------------+------------------+--------------- +1.8836190713044468E19|1.970336796004502 |Error +1.7957483822449326E19|26.644793296251386|OK +; + +extendedStatsAggregateFunctionsWithScalarsLargeInput +SELECT STDDEV_POP(CASE WHEN (bytes_out / 2) > 10000 THEN (bytes_out + 12345) * 1.2 ELSE (bytes_out - 12345) * 2.7 END) AS stddev_pop, +STDDEV_SAMP(CASE WHEN (bytes_out / 2) > 10000 THEN (bytes_out + 12345) * 1.2 ELSE (bytes_out - 12345) * 2.7 END) AS stddev_samp, +SUM_OF_SQUARES(CASE WHEN (bytes_out - 20) > 50000 THEN (bytes_out * 1.2) - 1234 ELSE (bytes_out - 20) * 0.93 END) AS sum_of_squares, +VAR_POP(CASE WHEN (bytes_out - 20) % 1000 > 200 THEN (bytes_out * 1.2) - 1234 ELSE (bytes_out - 20) * 0.93 END) AS var_pop, +VAR_SAMP(CASE WHEN (bytes_out - 20) % 1000 > 200 THEN (bytes_out * 1.2) - 1234 ELSE (bytes_out - 20) * 0.93 END) AS var_samp, +status +FROM logs_unsigned_long +GROUP BY status +ORDER BY status; + + stddev_pop | stddev_samp | sum_of_squares | var_pop | var_samp | status +---------------------+---------------------+---------------------+---------------------+--------------------+--------------- +5.1879845114777528E18|5.6831522898475694E18|1.1113551085273773E39|2.9979868907159907E37|3.597584268859189E37|Error +6.7116742512203459E18|6.7484508237837148E18|1.4906887301541836E40|4.304729535850427E37 |4.352034256024607E37|OK +; + +groupByRoundAndTruncateWithTwoParamsLargeInput +SELECT ROUND(SIN(TRUNCATE(bytes_in, 2)), 2) rst FROM logs_unsigned_long GROUP BY 1 ORDER BY 1 LIMIT 5; + + rst:d +--------------- +null +-1.0 +-0.99 +-0.97 +-0.96 +; + +sumLiteralAndSumFieldWithComplexHavingLargeInput +SELECT status, CAST(SUM(bytes_out) AS STRING), CAST(SUM(1) AS BIGINT), CAST(SUM(10) AS BIGINT), COUNT(*) FROM logs_unsigned_long GROUP BY status HAVING ((SUM(1) >= 0) AND (SUM(1) <= 50) AND (SUM(bytes_out) >= 250000) AND (SUM(bytes_out) <= 7E19)) ORDER BY status; + + status:s |CAST(SUM(bytes_out) AS STRING):s|CAST(SUM(1) AS BIGINT):l|CAST(SUM(10) AS BIGINT):l| COUNT(*):l +---------------+--------------------------------+------------------------+-------------------------+--------------- +Error |6.2910783680124445E19 |9 |90 |9 +; + +aggGroupByOnScalarWithHavingLargeInput +SELECT bytes_out + 1 AS e FROM logs_unsigned_long GROUP BY e HAVING AVG(bytes_out) BETWEEN 9E18 AND 1E19 ORDER BY e LIMIT 2; + + e:ul +------------------- +9243820354371174975 +9336652471323200907 +; + +iifWithCompatibleIntervalsLargeInput +SELECT "@timestamp" ts, "@timestamp" + IIF(bytes_out > 9E18, INTERVAL 2 HOURS, INTERVAL 2 DAYS) ts_plus, bytes_out FROM logs_unsigned_long ORDER BY bytes_out DESC LIMIT 10; + + ts:ts | ts_plus:ts | bytes_out:ul +------------------------+------------------------+-------------------- +2017-11-10T20:35:55.000Z|2017-11-12T20:35:55.000Z|null +2017-11-10T19:51:38.000Z|2017-11-12T19:51:38.000Z|null +2017-11-10T23:22:36.000Z|2017-11-12T23:22:36.000Z|null +2017-11-10T00:27:03.000Z|2017-11-10T02:27:03.000Z|18223615477147360166 +2017-11-10T20:35:57.000Z|2017-11-10T22:35:57.000Z|18184253384683076090 +2017-11-10T21:15:40.000Z|2017-11-10T23:15:40.000Z|18107197698386620672 +2017-11-10T23:28:11.000Z|2017-11-11T01:28:11.000Z|17953153966527637143 +2017-11-10T21:28:34.000Z|2017-11-10T23:28:34.000Z|17857609920324506371 +2017-11-10T21:13:27.000Z|2017-11-10T23:13:27.000Z|17695317925249333633 +2017-11-10T20:36:15.000Z|2017-11-10T22:36:15.000Z|17281501450843634251 +; + +aggPercentileLargeInput +SELECT status, PERCENTILE(bytes_in, 95) AS "95th" FROM logs_unsigned_long GROUP BY status; + + status:s | 95th:d +---------------+--------------------- +Error |1.8345360876889252E19 +OK |1.8295214210102768E19 +; + +mathPowerNegativeLargeInput +SELECT POWER(bytes_in, -1) m, id FROM logs_unsigned_long WHERE id < 10 ORDER BY id; + + m:d | id:i +----------------------+--------------- +2.2994842882726847E-19|1 +9.046030383433917E-20 |2 +1.3813254901492728E-19|3 +7.763447541524763E-20 |4 +1.5430743541977252E-19|5 +1.1302799995492255E-19|6 +5.530629782187966E-20 |7 +5.827842232431528E-20 |8 +5.459386906856704E-20 |9 +; + +averageWithOneValueAndOrderLargeInput +SELECT * FROM (SELECT id, status, bytes_out FROM logs_unsigned_long) PIVOT (AVG(bytes_out + 1) FOR status IN ('OK', 'Error')) ORDER BY id DESC LIMIT 4; + + id:i | 'OK':d | 'Error':d +---------------+---------------------+--------------- +101 |null |null +100 |1.5616395223975498E19|null +99 |3.5244227329937082E17|null +98 |5.4095497308894812E18|null +; + +sumWithInnerAggregateSumOfSquaresRoundLargeInput +SELECT * FROM logs_unsigned_long PIVOT (ROUND(SUM_OF_SQUARES(bytes_out + 1)/1E6, 2) FOR status IN ('Error', 'OK')) LIMIT 3; + + @timestamp:ts | bytes_in:ul | id:i | 'Error':d | 'OK':d +------------------------+-------------------+---------------+---------------+-------------------- +2017-11-10T00:00:22.000Z|0 |80 |null |9.223372036854776E16 +2017-11-10T00:01:04.000Z|9636626466125797351|83 |null |9.223372036854776E16 +2017-11-10T00:01:20.000Z|74330435873664882 |82 |null |9.223372036854776E16 +; + +castWithGroupByLargeInput +SELECT (bytes_in/1E12)::LONG::STRING in_tib FROM logs_unsigned_long WHERE bytes_in - 18E18 > 0 GROUP BY 1; + + in_tib:s +-------------------- +18081123 +18098466 +18317075 +18345361 +18446744 +; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index 0cb17d51bf223..db3fd87eb2d66 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -77,6 +77,7 @@ import static org.elasticsearch.xpack.ql.analyzer.VerifierChecks.checkFilterConditionType; import static org.elasticsearch.xpack.ql.common.Failure.fail; import static org.elasticsearch.xpack.ql.type.DataTypes.BINARY; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; @@ -470,8 +471,9 @@ private static boolean checkGroupByHavingHasOnlyAggs( unsupported.add(e); return true; } else if (e instanceof Min || e instanceof Max) { - if (DataTypes.isString(((AggregateFunction) e).field().dataType())) { - // Min & Max on a Keyword field will be translated to First & Last respectively + DataType aggType = ((AggregateFunction) e).field().dataType(); + if (DataTypes.isString(aggType) || aggType == UNSIGNED_LONG) { + // Min & Max on a Keyword or unsigned_long field will be translated to First & Last respectively unsupported.add(e); return true; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java index e89dbea225276..1ce3fd81e303c 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java @@ -468,7 +468,7 @@ protected List initBucketExtractors(SearchResponse response) { private BucketExtractor createExtractor(FieldExtraction ref, BucketExtractor totalCount) { if (ref instanceof GroupByRef r) { - return new CompositeKeyExtractor(r.key(), r.property(), cfg.zoneId(), r.isDateTimeBased()); + return new CompositeKeyExtractor(r.key(), r.property(), cfg.zoneId(), r.dataType()); } if (ref instanceof MetricAggRef r) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java index 41f6345ccf34b..a79a15d30acaf 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java @@ -6,13 +6,16 @@ */ package org.elasticsearch.xpack.sql.execution.search.extractor; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket; import org.elasticsearch.xpack.ql.execution.search.extractor.BucketExtractor; +import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.common.io.SqlStreamInput; import org.elasticsearch.xpack.sql.querydsl.container.GroupByRef.Property; +import org.elasticsearch.xpack.sql.type.SqlDataTypes; import org.elasticsearch.xpack.sql.util.DateUtils; import java.io.IOException; @@ -20,6 +23,13 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.xpack.ql.type.DataTypeConverter.toUnsignedLong; +import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; +import static org.elasticsearch.xpack.ql.type.DataTypes.NULL; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.type.SqlDataTypes.isDateBased; + public class CompositeKeyExtractor implements BucketExtractor { /** @@ -30,22 +40,27 @@ public class CompositeKeyExtractor implements BucketExtractor { private final String key; private final Property property; private final ZoneId zoneId; - private final boolean isDateTimeBased; + private final DataType dataType; /** * Constructs a new CompositeKeyExtractor instance. */ - public CompositeKeyExtractor(String key, Property property, ZoneId zoneId, boolean isDateTimeBased) { + public CompositeKeyExtractor(String key, Property property, ZoneId zoneId, DataType dataType) { this.key = key; this.property = property; this.zoneId = zoneId; - this.isDateTimeBased = isDateTimeBased; + this.dataType = dataType; } CompositeKeyExtractor(StreamInput in) throws IOException { key = in.readString(); property = in.readEnum(Property.class); - isDateTimeBased = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.fromId(INTRODUCING_UNSIGNED_LONG.id))) { + dataType = SqlDataTypes.fromTypeName(in.readString()); + } else { + // for pre-UNSIGNED_LONG versions, the only relevant fact about the dataType was if this isDateBased() or not. + dataType = in.readBoolean() ? DATETIME : NULL; + } zoneId = SqlStreamInput.asSqlStream(in).zoneId(); } @@ -54,7 +69,11 @@ public CompositeKeyExtractor(String key, Property property, ZoneId zoneId, boole public void writeTo(StreamOutput out) throws IOException { out.writeString(key); out.writeEnum(property); - out.writeBoolean(isDateTimeBased); + if (out.getVersion().onOrAfter(Version.fromId(INTRODUCING_UNSIGNED_LONG.id))) { + out.writeString(dataType.typeName()); + } else { + out.writeBoolean(isDateBased(dataType)); + } } String key() { @@ -69,8 +88,8 @@ ZoneId zoneId() { return zoneId; } - public boolean isDateTimeBased() { - return isDateTimeBased; + public DataType dataType() { + return dataType; } @Override @@ -92,13 +111,21 @@ public Object extract(Bucket bucket) { Object object = ((Map) m).get(key); - if (isDateTimeBased) { - if (object == null) { - return object; - } else if (object instanceof Long) { - object = DateUtils.asDateTimeWithMillis(((Long) object).longValue(), zoneId); - } else { - throw new SqlIllegalArgumentException("Invalid date key returned: {}", object); + if (object != null) { + if (isDateBased(dataType)) { + if (object instanceof Long l) { + object = DateUtils.asDateTimeWithMillis(l, zoneId); + } else { + throw new SqlIllegalArgumentException("Invalid date key returned: {}", object); + } + } else if (dataType == UNSIGNED_LONG) { + // For integral types we coerce the bucket type to long in composite aggs (unsigned_long is not an available choice). So + // when getting back a long value, this needs to be type- and value-converted to an UNSIGNED_LONG + if (object instanceof Number number) { + object = toUnsignedLong(number); + } else { + throw new SqlIllegalArgumentException("Invalid unsigned_long key returned: {}", object); + } } } @@ -107,7 +134,7 @@ public Object extract(Bucket bucket) { @Override public int hashCode() { - return Objects.hash(key, property, zoneId, isDateTimeBased); + return Objects.hash(key, property, zoneId, dataType); } @Override @@ -124,7 +151,7 @@ public boolean equals(Object obj) { return Objects.equals(key, other.key) && Objects.equals(property, other.property) && Objects.equals(zoneId, other.zoneId) - && Objects.equals(isDateTimeBased, other.isDateTimeBased); + && Objects.equals(dataType, other.dataType); } @Override diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java index 989aa2c9ded85..54187aa0ffde2 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java @@ -26,6 +26,8 @@ import java.util.Map; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.type.SqlDataTypeConverter.convert; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_POINT; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_SHAPE; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.SHAPE; @@ -126,6 +128,11 @@ protected Object unwrapCustomValue(Object values) { return DateUtils.asDateTimeWithNanos(values.toString()).withZoneSameInstant(zoneId()); } } + if (dataType == UNSIGNED_LONG) { + // Unsigned longs can be returned either as such (for values exceeding long range) or as longs. Value conversion is needed + // since its later processing will be type dependent. (ex.: negation of UL is only "safe" for 0 values) + return convert(values, UNSIGNED_LONG); + } return null; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractor.java index fe9125fb2f581..78976ea7e83c0 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractor.java @@ -21,7 +21,9 @@ import java.time.ZoneId; import java.util.Objects; +import static org.elasticsearch.xpack.ql.type.DataTypeConverter.toUnsignedLong; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.DATE; public class TopHitsAggExtractor implements BucketExtractor { @@ -83,6 +85,16 @@ public Object extract(Bucket bucket) { return DateUtils.asDateTimeWithNanos(value.toString()).withZoneSameInstant(zoneId()); } else if (SqlDataTypes.isTimeBased(fieldDataType)) { return DateUtils.asTimeOnly(Long.parseLong(value.toString()), zoneId); + } else if (fieldDataType == UNSIGNED_LONG) { + if (value == null) { + return null; + } else if (value instanceof Number number) { + // values can be returned either as unsigned_longs or longs (if range allows), which can lead to cast exceptions + // when sorting rows locally -> upcast to BigInteger + return toUnsignedLong(number); + } else { + throw new SqlIllegalArgumentException("Invalid unsigned_long key returned: {}", value); + } } else { return value; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Sum.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Sum.java index b2810b5bcff57..4b6d004e7558f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Sum.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Sum.java @@ -40,7 +40,7 @@ public Sum replaceChildren(List newChildren) { @Override public DataType dataType() { DataType dt = field().dataType(); - return dt.isInteger() ? (dt == UNSIGNED_LONG ? UNSIGNED_LONG : LONG) : DOUBLE; + return dt.isInteger() == false || dt == UNSIGNED_LONG ? DOUBLE : LONG; } @Override diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java index 6e32f22c505e4..7e11e194958a0 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java @@ -53,6 +53,7 @@ import org.elasticsearch.xpack.ql.rule.Rule; import org.elasticsearch.xpack.ql.rule.RuleExecutor; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.util.Holder; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; @@ -107,6 +108,7 @@ import static org.elasticsearch.xpack.ql.expression.Expressions.equalsAsAttribute; import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BinaryComparisonSimplification; import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PushDownAndCombineFilters; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; public class Optimizer extends RuleExecutor { @@ -1149,12 +1151,15 @@ protected LogicalPlan rule(LogicalPlan plan) { Map maxs = new HashMap<>(); return plan.transformExpressionsDown(NumericAggregate.class, e -> { if (e instanceof Min min) { - if (DataTypes.isString(min.field().dataType())) { + DataType minType = min.field().dataType(); + // upper range unsigned longs can't be represented exactly on doubles (returned by stats agg) + if (DataTypes.isString(minType) || minType == UNSIGNED_LONG) { return mins.computeIfAbsent(min.field(), k -> new First(min.source(), k, null)); } } if (e instanceof Max max) { - if (DataTypes.isString(max.field().dataType())) { + DataType maxType = max.field().dataType(); + if (DataTypes.isString(maxType) || maxType == UNSIGNED_LONG) { return maxs.computeIfAbsent(max.field(), k -> new Last(max.source(), k, null)); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java index 3b7e44722f6c5..a7477fc9781e2 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java @@ -40,6 +40,7 @@ import org.elasticsearch.xpack.ql.querydsl.query.Query; import org.elasticsearch.xpack.ql.rule.Rule; import org.elasticsearch.xpack.ql.rule.RuleExecutor; +import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.expression.function.Score; @@ -95,6 +96,7 @@ import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicReference; +import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; import static org.elasticsearch.xpack.sql.expression.function.grouping.Histogram.DAY_INTERVAL; import static org.elasticsearch.xpack.sql.expression.function.grouping.Histogram.MONTH_INTERVAL; @@ -554,7 +556,7 @@ else if (target instanceof Function) { if (matchingGroup != null) { if (exp instanceof Attribute || exp instanceof ScalarFunction || exp instanceof GroupingFunction) { Processor action = null; - boolean isDateBased = isDateBased(exp.dataType()); + DataType dataType = exp.dataType(); /* * special handling of dates since aggs return the typed Date object which needs * extraction instead of handling this in the scroller, the folder handles this @@ -562,14 +564,9 @@ else if (target instanceof Function) { */ if (exp instanceof DateTimeHistogramFunction) { action = ((UnaryPipe) p).action(); - isDateBased = true; + dataType = DATETIME; } - return new AggPathInput( - exp.source(), - exp, - new GroupByRef(matchingGroup.id(), null, isDateBased), - action - ); + return new AggPathInput(exp.source(), exp, new GroupByRef(matchingGroup.id(), null, dataType), action); } } // or found an aggregate expression (which has to work on an attribute used for grouping) @@ -607,19 +604,11 @@ else if (target instanceof Function) { // attributes can only refer to declared groups if (target instanceof Attribute) { Check.notNull(matchingGroup, "Cannot find group [{}]", Expressions.name(target)); - queryC = queryC.addColumn( - new GroupByRef(matchingGroup.id(), null, isDateBased(target.dataType())), - id, - ne.toAttribute() - ); + queryC = queryC.addColumn(new GroupByRef(matchingGroup.id(), null, target.dataType()), id, ne.toAttribute()); } // handle histogram else if (target instanceof GroupingFunction) { - queryC = queryC.addColumn( - new GroupByRef(matchingGroup.id(), null, isDateBased(target.dataType())), - id, - ne.toAttribute() - ); + queryC = queryC.addColumn(new GroupByRef(matchingGroup.id(), null, target.dataType()), id, ne.toAttribute()); } // handle literal else if (target.foldable()) { @@ -650,11 +639,7 @@ else if (target.foldable()) { matchingGroup = groupingContext.groupFor(target); Check.notNull(matchingGroup, "Cannot find group [{}]", Expressions.name(ne)); - queryC = queryC.addColumn( - new GroupByRef(matchingGroup.id(), null, isDateBased(ne.dataType())), - id, - ne.toAttribute() - ); + queryC = queryC.addColumn(new GroupByRef(matchingGroup.id(), null, ne.dataType()), id, ne.toAttribute()); } // fallback else { @@ -667,7 +652,7 @@ else if (target.foldable()) { if (a.aggregates().stream().allMatch(e -> e.anyMatch(Expression::foldable))) { for (Expression grouping : a.groupings()) { GroupByKey matchingGroup = groupingContext.groupFor(grouping); - queryC = queryC.addColumn(new GroupByRef(matchingGroup.id(), null, false), id, null); + queryC = queryC.addColumn(new GroupByRef(matchingGroup.id(), null, grouping.dataType()), id, null); } } return new EsQueryExec(exec.source(), exec.index(), a.output(), queryC); @@ -692,7 +677,7 @@ private static Tuple addAggFunction( // if the count points to the total track hits, enable accurate count retrieval queryC = queryC.withTrackHits(); } else { - ref = new GroupByRef(groupingAgg.id(), Property.COUNT, false); + ref = new GroupByRef(groupingAgg.id(), Property.COUNT, c.dataType()); } Map pseudoFunctions = new LinkedHashMap<>(queryC.pseudoFunctions()); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/GroupByRef.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/GroupByRef.java index 3ce1d6c5cc694..c22dd6a13e1ea 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/GroupByRef.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/GroupByRef.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.sql.querydsl.container; import org.elasticsearch.xpack.ql.execution.search.AggRef; +import org.elasticsearch.xpack.ql.type.DataType; /** * Reference to a GROUP BY agg (typically this gets translated to a composite key). @@ -20,12 +21,12 @@ public enum Property { private final String key; private final Property property; - private final boolean isDateTimeBased; + private final DataType dataType; - public GroupByRef(String key, Property property, boolean isDateTimeBased) { + public GroupByRef(String key, Property property, DataType dataType) { this.key = key; this.property = property == null ? Property.VALUE : property; - this.isDateTimeBased = isDateTimeBased; + this.dataType = dataType; } public String key() { @@ -36,8 +37,8 @@ public Property property() { return property; } - public boolean isDateTimeBased() { - return isDateTimeBased; + public DataType dataType() { + return dataType; } @Override diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index 8f4bd716e9ccc..61b56aaadb309 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -1468,6 +1468,20 @@ public void testMaxOnKeywordGroupByHavingUnsupported() { ); } + public void testMinOnUnsignedLongGroupByHavingUnsupported() { + assertEquals( + "1:62: HAVING filter is unsupported for function [MIN(unsigned_long)]", + error("SELECT MIN(unsigned_long) min FROM test GROUP BY text HAVING min > 10") + ); + } + + public void testMaxOnUnsignedLongGroupByHavingUnsupported() { + assertEquals( + "1:62: HAVING filter is unsupported for function [MAX(unsigned_long)]", + error("SELECT MAX(unsigned_long) max FROM test GROUP BY text HAVING max > 10") + ); + } + public void testProjectAliasInFilter() { accept("SELECT int AS i FROM test WHERE i > 10"); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractorTests.java index 88773071f465a..ef4b3d66b5fc1 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractorTests.java @@ -14,24 +14,40 @@ import org.elasticsearch.xpack.sql.AbstractSqlWireSerializingTestCase; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.querydsl.container.GroupByRef.Property; +import org.elasticsearch.xpack.sql.type.SqlDataTypes; import org.elasticsearch.xpack.sql.util.DateUtils; +import java.math.BigInteger; import java.time.ZoneId; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; +import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; +import static org.elasticsearch.xpack.ql.type.DataTypes.NULL; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.ql.util.NumericUtils.UNSIGNED_LONG_MAX; import static org.elasticsearch.xpack.sql.util.DateUtils.UTC; public class CompositeKeyExtractorTests extends AbstractSqlWireSerializingTestCase { public static CompositeKeyExtractor randomCompositeKeyExtractor() { - return new CompositeKeyExtractor(randomAlphaOfLength(16), randomFrom(asList(Property.values())), randomZone(), randomBoolean()); + return new CompositeKeyExtractor( + randomAlphaOfLength(16), + randomFrom(asList(Property.values())), + randomZone(), + randomFrom(SqlDataTypes.types()) + ); } public static CompositeKeyExtractor randomCompositeKeyExtractor(ZoneId zoneId) { - return new CompositeKeyExtractor(randomAlphaOfLength(16), randomFrom(asList(Property.values())), zoneId, randomBoolean()); + return new CompositeKeyExtractor( + randomAlphaOfLength(16), + randomFrom(asList(Property.values())), + zoneId, + randomFrom(SqlDataTypes.types()) + ); } @Override @@ -55,18 +71,18 @@ protected CompositeKeyExtractor mutateInstance(CompositeKeyExtractor instance) { instance.key() + "mutated", randomValueOtherThan(instance.property(), () -> randomFrom(Property.values())), randomValueOtherThan(instance.zoneId(), ESTestCase::randomZone), - instance.isDateTimeBased() == false + randomValueOtherThan(instance.dataType(), () -> randomFrom(SqlDataTypes.types())) ); } public void testExtractBucketCount() { Bucket bucket = new TestBucket(emptyMap(), randomLong(), new Aggregations(emptyList())); - CompositeKeyExtractor extractor = new CompositeKeyExtractor(randomAlphaOfLength(16), Property.COUNT, randomZone(), false); + CompositeKeyExtractor extractor = new CompositeKeyExtractor(randomAlphaOfLength(16), Property.COUNT, randomZone(), NULL); assertEquals(bucket.getDocCount(), extractor.extract(bucket)); } public void testExtractKey() { - CompositeKeyExtractor extractor = new CompositeKeyExtractor(randomAlphaOfLength(16), Property.VALUE, UTC, false); + CompositeKeyExtractor extractor = new CompositeKeyExtractor(randomAlphaOfLength(16), Property.VALUE, UTC, NULL); Object value = new Object(); Bucket bucket = new TestBucket(singletonMap(extractor.key(), value), randomLong(), new Aggregations(emptyList())); @@ -74,7 +90,7 @@ public void testExtractKey() { } public void testExtractDate() { - CompositeKeyExtractor extractor = new CompositeKeyExtractor(randomAlphaOfLength(16), Property.VALUE, randomZone(), true); + CompositeKeyExtractor extractor = new CompositeKeyExtractor(randomAlphaOfLength(16), Property.VALUE, randomZone(), DATETIME); long millis = System.currentTimeMillis(); Bucket bucket = new TestBucket(singletonMap(extractor.key(), millis), randomLong(), new Aggregations(emptyList())); @@ -82,7 +98,7 @@ public void testExtractDate() { } public void testExtractIncorrectDateKey() { - CompositeKeyExtractor extractor = new CompositeKeyExtractor(randomAlphaOfLength(16), Property.VALUE, randomZone(), true); + CompositeKeyExtractor extractor = new CompositeKeyExtractor(randomAlphaOfLength(16), Property.VALUE, randomZone(), DATETIME); Object value = new Object(); Bucket bucket = new TestBucket(singletonMap(extractor.key(), value), randomLong(), new Aggregations(emptyList())); @@ -90,6 +106,13 @@ public void testExtractIncorrectDateKey() { assertEquals("Invalid date key returned: " + value, exception.getMessage()); } + public void testExtractUnsignedLong() { + CompositeKeyExtractor extractor = new CompositeKeyExtractor(randomAlphaOfLength(16), Property.VALUE, randomZone(), UNSIGNED_LONG); + Long value = randomLong(); + Bucket bucket = new TestBucket(singletonMap(extractor.key(), value), randomLong(), new Aggregations(emptyList())); + assertEquals(BigInteger.valueOf(value).and(UNSIGNED_LONG_MAX), extractor.extract(bucket)); + } + public static ZoneId extractZoneId(BucketExtractor extractor) { return extractor instanceof CompositeKeyExtractor ? ((CompositeKeyExtractor) extractor).zoneId() : null; } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java index 7b4b91e25add3..2f5edb30e07b3 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java @@ -34,6 +34,7 @@ import static java.util.Collections.singletonMap; import static org.elasticsearch.common.time.DateUtils.toMilliSeconds; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_SHAPE; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.SHAPE; import static org.elasticsearch.xpack.sql.util.DateUtils.UTC; @@ -206,6 +207,19 @@ public void testMultipleGeoShapeExtraction() { ); } + public void testUnsignedLongExtraction() { + BigInteger bi = randomBigInteger(); + Number number = bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) <= 0 ? bi.longValue() : bi; + Object value = randomBoolean() ? number.toString() : number; + + String fieldName = randomAlphaOfLength(10); + DocumentField field = new DocumentField(fieldName, singletonList(value)); + SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + FieldHitExtractor fe = new FieldHitExtractor(fieldName, UNSIGNED_LONG, randomZone(), randomBoolean()); + + assertEquals(bi, fe.extract(hit)); + } + private FieldHitExtractor getFieldHitExtractor(String fieldName) { return new FieldHitExtractor(fieldName, null, UTC); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractorTests.java index d8d70cf9b4520..c816544a92e2b 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractorTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.sql.type.SqlDataTypes; import org.elasticsearch.xpack.sql.util.DateUtils; +import java.math.BigInteger; import java.time.ZoneId; import java.util.Collections; @@ -103,6 +104,16 @@ public void testExtractDateValue() { assertEquals(DateUtils.asDateTimeWithMillis(value, zoneId), extractor.extract(bucket)); } + public void testExtractUnsignedLong() { + BigInteger bi = randomBigInteger(); + Object value = bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) <= 0 ? bi.longValue() : bi; + + TopHitsAggExtractor extractor = new TopHitsAggExtractor(randomAlphaOfLength(10), DataTypes.UNSIGNED_LONG, randomZone()); + Aggregation agg = new InternalTopHits(extractor.name(), 0, 1, null, searchHitsOf(value), null); + Bucket bucket = new TestBucket(emptyMap(), 0, new Aggregations(singletonList(agg))); + assertEquals(bi, extractor.extract(bucket)); + } + private SearchHits searchHitsOf(Object value) { TotalHits totalHits = new TotalHits(10, TotalHits.Relation.EQUAL_TO); return new SearchHits( diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java index 74759812bbd7c..fe307baa24f0e 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java @@ -51,6 +51,7 @@ import org.elasticsearch.xpack.ql.plan.logical.OrderBy; import org.elasticsearch.xpack.ql.plan.logical.Project; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.ql.util.CollectionUtils; @@ -143,6 +144,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.SqlTestUtils.literal; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.DATE; import static org.elasticsearch.xpack.sql.util.DateUtils.UTC; @@ -873,60 +875,64 @@ public void testCombineUnbalancedComparisonsMixedWithEqualsIntoRange() { } public void testTranslateMinToFirst() { - Min min1 = new Min(EMPTY, new FieldAttribute(EMPTY, "str", new EsField("str", KEYWORD, emptyMap(), true))); - Min min2 = new Min(EMPTY, getFieldAttribute()); + for (DataType dataType : List.of(KEYWORD, UNSIGNED_LONG)) { + Min min1 = new Min(EMPTY, new FieldAttribute(EMPTY, "field", new EsField("field", dataType, emptyMap(), true))); + Min min2 = new Min(EMPTY, getFieldAttribute()); - OrderBy plan = new OrderBy( - EMPTY, - new Aggregate(EMPTY, FROM(), emptyList(), asList(a("min1", min1), a("min2", min2))), - asList( - new Order(EMPTY, min1, OrderDirection.ASC, Order.NullsPosition.LAST), - new Order(EMPTY, min2, OrderDirection.ASC, Order.NullsPosition.LAST) - ) - ); - LogicalPlan result = new ReplaceMinMaxWithTopHits().apply(plan); - assertTrue(result instanceof OrderBy); - List order = ((OrderBy) result).order(); - assertEquals(2, order.size()); - assertEquals(First.class, order.get(0).child().getClass()); - assertEquals(min2, order.get(1).child()); - First first = (First) order.get(0).child(); - - assertTrue(((OrderBy) result).child() instanceof Aggregate); - List aggregates = ((Aggregate) ((OrderBy) result).child()).aggregates(); - assertEquals(2, aggregates.size()); - assertEquals(Alias.class, aggregates.get(0).getClass()); - assertEquals(Alias.class, aggregates.get(1).getClass()); - assertSame(first, ((Alias) aggregates.get(0)).child()); - assertEquals(min2, ((Alias) aggregates.get(1)).child()); + OrderBy plan = new OrderBy( + EMPTY, + new Aggregate(EMPTY, FROM(), emptyList(), asList(a("min1", min1), a("min2", min2))), + asList( + new Order(EMPTY, min1, OrderDirection.ASC, Order.NullsPosition.LAST), + new Order(EMPTY, min2, OrderDirection.ASC, Order.NullsPosition.LAST) + ) + ); + LogicalPlan result = new ReplaceMinMaxWithTopHits().apply(plan); + assertTrue(result instanceof OrderBy); + List order = ((OrderBy) result).order(); + assertEquals(2, order.size()); + assertEquals(First.class, order.get(0).child().getClass()); + assertEquals(min2, order.get(1).child()); + First first = (First) order.get(0).child(); + + assertTrue(((OrderBy) result).child() instanceof Aggregate); + List aggregates = ((Aggregate) ((OrderBy) result).child()).aggregates(); + assertEquals(2, aggregates.size()); + assertEquals(Alias.class, aggregates.get(0).getClass()); + assertEquals(Alias.class, aggregates.get(1).getClass()); + assertSame(first, ((Alias) aggregates.get(0)).child()); + assertEquals(min2, ((Alias) aggregates.get(1)).child()); + } } public void testTranslateMaxToLast() { - Max max1 = new Max(EMPTY, new FieldAttribute(EMPTY, "str", new EsField("str", KEYWORD, emptyMap(), true))); - Max max2 = new Max(EMPTY, getFieldAttribute()); + for (DataType dataType : List.of(KEYWORD, UNSIGNED_LONG)) { + Max max1 = new Max(EMPTY, new FieldAttribute(EMPTY, "field", new EsField("field", dataType, emptyMap(), true))); + Max max2 = new Max(EMPTY, getFieldAttribute()); - OrderBy plan = new OrderBy( - EMPTY, - new Aggregate(EMPTY, FROM(), emptyList(), asList(a("max1", max1), a("max2", max2))), - asList( - new Order(EMPTY, max1, OrderDirection.ASC, Order.NullsPosition.LAST), - new Order(EMPTY, max2, OrderDirection.ASC, Order.NullsPosition.LAST) - ) - ); - LogicalPlan result = new ReplaceMinMaxWithTopHits().apply(plan); - assertTrue(result instanceof OrderBy); - List order = ((OrderBy) result).order(); - assertEquals(Last.class, order.get(0).child().getClass()); - assertEquals(max2, order.get(1).child()); - Last last = (Last) order.get(0).child(); - - assertTrue(((OrderBy) result).child() instanceof Aggregate); - List aggregates = ((Aggregate) ((OrderBy) result).child()).aggregates(); - assertEquals(2, aggregates.size()); - assertEquals(Alias.class, aggregates.get(0).getClass()); - assertEquals(Alias.class, aggregates.get(1).getClass()); - assertSame(last, ((Alias) aggregates.get(0)).child()); - assertEquals(max2, ((Alias) aggregates.get(1)).child()); + OrderBy plan = new OrderBy( + EMPTY, + new Aggregate(EMPTY, FROM(), emptyList(), asList(a("max1", max1), a("max2", max2))), + asList( + new Order(EMPTY, max1, OrderDirection.ASC, Order.NullsPosition.LAST), + new Order(EMPTY, max2, OrderDirection.ASC, Order.NullsPosition.LAST) + ) + ); + LogicalPlan result = new ReplaceMinMaxWithTopHits().apply(plan); + assertTrue(result instanceof OrderBy); + List order = ((OrderBy) result).order(); + assertEquals(Last.class, order.get(0).child().getClass()); + assertEquals(max2, order.get(1).child()); + Last last = (Last) order.get(0).child(); + + assertTrue(((OrderBy) result).child() instanceof Aggregate); + List aggregates = ((Aggregate) ((OrderBy) result).child()).aggregates(); + assertEquals(2, aggregates.size()); + assertEquals(Alias.class, aggregates.get(0).getClass()); + assertEquals(Alias.class, aggregates.get(1).getClass()); + assertSame(last, ((Alias) aggregates.get(0)).child()); + assertEquals(max2, ((Alias) aggregates.get(1)).child()); + } } public void testSortAggregateOnOrderByWithTwoFields() { From d03cb1d3eee58784a981377723418465ec3c1677 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Thu, 23 Dec 2021 11:26:44 +0100 Subject: [PATCH 20/28] Address review comments - extend comments; - move client's version argument from Verifier's constructor to methods; - extract unsigned long testing version to reusable var; - StringUtils#parseIntegral() now returns the "smallest" fitting Number; - adjust some member and methods' visibility. --- .../xpack/eql/parser/ExpressionBuilder.java | 20 ++--------------- .../xpack/ql/util/StringUtils.java | 15 ++++++++----- .../ql/type/DataTypeConversionTests.java | 2 +- .../sql/jdbc/JdbcPreparedStatementTests.java | 4 ++++ .../xpack/sql/qa/jdbc/JdbcTestUtils.java | 2 +- .../xpack/sql/qa/jdbc/ResultSetTestCase.java | 2 +- .../xpack/sql/qa/jdbc/CsvTestUtils.java | 2 +- .../xpack/sql/proto/SqlVersion.java | 3 ++- .../xpack/sql/analysis/analyzer/Analyzer.java | 2 +- .../xpack/sql/analysis/analyzer/Verifier.java | 14 +++++------- .../xpack/sql/execution/PlanExecutor.java | 14 +++--------- .../xpack/sql/parser/ExpressionBuilder.java | 21 ++---------------- .../sql/plan/logical/command/Explain.java | 2 +- .../session/VersionCompatibilityChecks.java | 9 ++++++++ .../analyzer/FieldAttributeTests.java | 8 +++---- .../analyzer/VerifierErrorMessagesTests.java | 9 ++------ .../scalar/DatabaseFunctionTests.java | 7 +----- .../function/scalar/UserFunctionTests.java | 7 +----- .../scalar/datetime/CurrentDateTimeTests.java | 7 +----- .../scalar/datetime/CurrentTimeTests.java | 7 +----- .../sql/optimizer/OptimizerRunTests.java | 7 +----- .../logical/command/ShowFunctionsTests.java | 16 ++------------ .../logical/command/sys/SysColumnsTests.java | 22 ++++++++----------- .../logical/command/sys/SysTablesTests.java | 2 +- .../logical/command/sys/SysTypesTests.java | 11 ++-------- .../xpack/sql/planner/QueryFolderTests.java | 7 +----- .../sql/planner/QueryTranslatorSpecTests.java | 2 +- .../sql/planner/QueryTranslatorTests.java | 2 +- .../xpack/sql/planner/VerifierTests.java | 7 +----- .../xpack/sql/stats/VerifierMetricsTests.java | 6 ++--- .../sql/type/SqlDataTypeConverterTests.java | 2 +- 31 files changed, 77 insertions(+), 164 deletions(-) diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java index 674896ffc4d91..5e873a67fc1e8 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java @@ -49,11 +49,9 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.ql.expression.predicate.regex.Like; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.util.StringUtils; -import java.math.BigInteger; import java.time.ZoneId; import java.util.List; @@ -254,9 +252,9 @@ public Literal visitIntegerLiteral(EqlBaseParser.IntegerLiteralContext ctx) { Source source = source(ctx); String text = ctx.getText(); - Number value; try { - value = StringUtils.parseIntegral(text); + Number value = StringUtils.parseIntegral(text); + return new Literal(source, value, DataTypes.fromJava(value)); } catch (QlIllegalArgumentException siae) { // if it's too large, then quietly try to parse as a float instead try { @@ -265,20 +263,6 @@ public Literal visitIntegerLiteral(EqlBaseParser.IntegerLiteralContext ctx) { throw new ParsingException(source, siae.getMessage()); } - - DataType type; - if (value instanceof BigInteger) { - type = DataTypes.UNSIGNED_LONG; - } else { - // try to downsize to int if possible (since that's the most common type) - if (value.longValue() == value.intValue()) { - type = DataTypes.INTEGER; - value = value.intValue(); - } else { - type = DataTypes.LONG; - } - } - return new Literal(source, value, type); } @Override diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java index fec78df32c9c9..81d53d1fb713c 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java @@ -334,13 +334,18 @@ public static Number parseIntegral(String string) throws QlIllegalArgumentExcept } catch (NumberFormatException ex) { throw new QlIllegalArgumentException("Cannot parse number [{}]", string); } - if (bi.signum() < 0 || bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) <= 0) { - return bi.longValueExact(); + if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + if (isUnsignedLong(bi) == false) { + throw new QlIllegalArgumentException("Number [{}] is too large", string); + } + return bi; } - if (isUnsignedLong(bi) == false) { - throw new QlIllegalArgumentException("Number [{}] is too large", string); + // try to downsize to int if possible (since that's the most common type) + if (bi.intValue() == bi.longValue()) { // ternary operator would always promote to Long + return bi.intValueExact(); + } else { + return bi.longValueExact(); } - return bi; } public static String ordinal(int i) { diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java index 66c44cf7d2e68..13a1182c1f7f6 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/DataTypeConversionTests.java @@ -339,7 +339,7 @@ public void testConversionToBoolean() { } } - public void testConversiontoUnsignedLong() { + public void testConversionToUnsignedLong() { DataType to = UNSIGNED_LONG; { Converter conversion = converterFor(DOUBLE, to); diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java index e4f787b401a6f..43a47611c79c9 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java @@ -271,6 +271,10 @@ public void testSettingBigIntegerValues() throws SQLException { assertEquals(bi.doubleValue(), value(jps)); assertEquals(HALF_FLOAT, jdbcType(jps)); + jps.setObject(1, bi, Types.REAL); + assertEquals(bi.floatValue(), value(jps)); + assertEquals(FLOAT, jdbcType(jps)); + jps.setObject(1, BigInteger.ZERO, Types.BOOLEAN); assertEquals(false, value(jps)); assertEquals(BOOLEAN, jdbcType(jps)); diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java index 3102a1c344fea..6ca1cb9acd55c 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java @@ -42,7 +42,7 @@ private JdbcTestUtils() {} static final LocalDate EPOCH = LocalDate.of(1970, 1, 1); static final String UNSIGNED_LONG_TYPE_NAME = "UNSIGNED_LONG"; - public static final BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); + static final BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); /* * The version of the driver that the QA (bwc-)tests run against. diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java index b4edd80d5d5d6..7aafcce432a52 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java @@ -2449,7 +2449,7 @@ private void indexSimpleDocumentWithBooleanValues(String docId, boolean bool, Lo }); } - protected static void indexTestFieldsDoc(String docId, Object... values) throws IOException { + private static void indexTestFieldsDoc(String docId, Object... values) throws IOException { index("test", docId, builder -> { for (Object value : values) { String classSimpleName = value.getClass().getSimpleName().toLowerCase(Locale.ROOT); diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java index 5c660aed31565..bc4507e1688e3 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java @@ -163,7 +163,7 @@ private static String resolveColumnType(String type) { case "sh": return "short"; case "ul": - return "bigdecimal"; // CSV + return "bigdecimal"; // CSV JDBC driver lacks biginteger support default: return type; } diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java index ff74c14797a50..09a931dc7204a 100644 --- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlVersion.java @@ -32,7 +32,7 @@ public class SqlVersion implements Comparable { public static final SqlVersion V_7_7_0 = new SqlVersion(7, 7, 0); public static final SqlVersion V_7_12_0 = new SqlVersion(7, 12, 0); - public static final SqlVersion DATE_NANOS_SUPPORT_VERSION = V_7_12_0; + public static final SqlVersion DATE_NANOS_SUPPORT_VERSION = V_7_12_0; // TODO: move to VersionCompatibilityChecks public SqlVersion(byte major, byte minor, byte revision) { this(toString(major, minor, revision), major, minor, revision); @@ -169,6 +169,7 @@ public static boolean isClientCompatible(SqlVersion server, SqlVersion client) { return hasVersionCompatibility(client) && server.compareTo(client) >= 0 && server.major - client.major <= 1; } + // TODO: move to VersionCompatibilityChecks public static boolean supportsDateNanos(SqlVersion version) { return DATE_NANOS_SUPPORT_VERSION.compareTo(version) <= 0; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index 997b8935d4398..ae8a5636f8e6f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -150,7 +150,7 @@ public ExecutionInfo debugAnalyze(LogicalPlan plan) { } public LogicalPlan verify(LogicalPlan plan) { - Collection failures = verifier.verify(plan); + Collection failures = verifier.verify(plan, configuration.version()); if (failures.isEmpty() == false) { throw new VerificationException(failures); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index db3fd87eb2d66..52f0e93928a37 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -79,8 +79,8 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.BINARY; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.versionIntroducingType; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.COMMAND; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.GROUPBY; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.HAVING; @@ -97,19 +97,17 @@ */ public final class Verifier { private final Metrics metrics; - private final SqlVersion version; - public Verifier(Metrics metrics, SqlVersion version) { + public Verifier(Metrics metrics) { this.metrics = metrics; - this.version = version; } - public Map, String> verifyFailures(LogicalPlan plan) { - Collection failures = verify(plan); + public Map, String> verifyFailures(LogicalPlan plan, SqlVersion version) { + Collection failures = verify(plan, version); return failures.stream().collect(toMap(Failure::node, Failure::message)); } - Collection verify(LogicalPlan plan) { + Collection verify(LogicalPlan plan, SqlVersion version) { Set failures = new LinkedHashSet<>(); // start bottom-up @@ -1017,7 +1015,7 @@ private static void checkClientSupportsDataTypes(LogicalPlan p, Set loc + "] unsupported in version [" + version + "], upgrade required (to version [" - + INTRODUCING_UNSIGNED_LONG + + versionIntroducingType(e.dataType()) + "] or higher)" ) ); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java index a47acdd4170b2..a9bcd745d62f7 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java @@ -42,6 +42,7 @@ public class PlanExecutor { private final IndexResolver indexResolver; private final PreAnalyzer preAnalyzer; + private final Verifier verifier; private final Optimizer optimizer; private final Planner planner; @@ -57,22 +58,13 @@ public PlanExecutor(Client client, IndexResolver indexResolver, NamedWriteableRe this.metrics = new Metrics(); this.preAnalyzer = new PreAnalyzer(); + this.verifier = new Verifier(metrics); this.optimizer = new Optimizer(); this.planner = new Planner(); } private SqlSession newSession(SqlConfiguration cfg) { - return new SqlSession( - cfg, - client, - functionRegistry, - indexResolver, - preAnalyzer, - new Verifier(metrics, cfg.version()), - optimizer, - planner, - this - ); + return new SqlSession(cfg, client, functionRegistry, indexResolver, preAnalyzer, verifier, optimizer, planner, this); } public void searchSource( diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java index d393c30f03411..34f36e8eacf19 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java @@ -120,7 +120,6 @@ import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; import org.elasticsearch.xpack.sql.type.SqlDataTypes; -import java.math.BigInteger; import java.time.Duration; import java.time.Period; import java.time.ZoneId; @@ -728,28 +727,12 @@ public Literal visitDecimalLiteral(DecimalLiteralContext ctx) { @Override public Literal visitIntegerLiteral(IntegerLiteralContext ctx) { Tuple tuple = withMinus(ctx); - - Number value; try { - value = StringUtils.parseIntegral(tuple.v2()); + Number value = StringUtils.parseIntegral(tuple.v2()); + return new Literal(tuple.v1(), value, DataTypes.fromJava(value)); } catch (QlIllegalArgumentException siae) { throw new ParsingException(tuple.v1(), siae.getMessage()); } - - DataType type; - if (value instanceof BigInteger) { - type = DataTypes.UNSIGNED_LONG; - } else { - assert value instanceof Long : "Expected value [" + value + "] of type Long but got: " + value.getClass(); - // try to downsize to int if possible (since that's the most common type) - if (value.longValue() == value.intValue()) { - type = DataTypes.INTEGER; - value = Integer.valueOf(value.intValue()); - } else { - type = DataTypes.LONG; - } - } - return new Literal(tuple.v1(), value, type); } @Override diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/Explain.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/Explain.java index 1fe9e121bcc17..e82e9070f1cdb 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/Explain.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/Explain.java @@ -140,7 +140,7 @@ public void execute(SqlSession session, ActionListener listener) { // check errors manually to see how far the plans work out else { // no analysis failure, can move on - if (session.verifier().verifyFailures(analyzedPlan).isEmpty()) { + if (session.verifier().verifyFailures(analyzedPlan, session.configuration().version()).isEmpty()) { session.optimizedPlan(analyzedPlan, wrap(optimizedPlan -> { if (type == Type.OPTIMIZED) { listener.onResponse(Page.last(Rows.singleton(output(), formatPlan(format, optimizedPlan)))); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java index ec66b26737656..b7587e6dac2cd 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.sql.session; +import org.elasticsearch.core.Nullable; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.sql.proto.SqlVersion; @@ -35,4 +36,12 @@ public static boolean isTypeSupportedInVersion(DataType dataType, SqlVersion ver public static boolean supportsUnsignedLong(SqlVersion version) { return INTRODUCING_UNSIGNED_LONG.compareTo(version) <= 0; } + + public static @Nullable SqlVersion versionIntroducingType(DataType dataType) { + if (dataType == UNSIGNED_LONG) { + return INTRODUCING_UNSIGNED_LONG; + } + + return null; + } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java index ee8a5d5a77006..653437eb7c5d0 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java @@ -62,7 +62,7 @@ public class FieldAttributeTests extends ESTestCase { public FieldAttributeTests() { parser = new SqlParser(); functionRegistry = new SqlFunctionRegistry(); - verifier = new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version()); + verifier = new Verifier(new Metrics()); Map mapping = loadMapping("mapping-multi-field-variation.json"); @@ -319,7 +319,7 @@ public void testUnsignedLongVersionCompatibility() { for (String sql : List.of(query, queryWithLiteral, queryWithAlias, queryWithArithmetic, queryWithCast)) { SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preUnsignedLong); - analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics(), sqlConfig.version())); + analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics())); VerificationException ex = expectThrows(VerificationException.class, () -> plan(sql)); assertEquals( "Found 1 problem\nline 1:8: Cannot use field [unsigned_long] with type [UNSIGNED_LONG] unsupported in version [" @@ -350,7 +350,7 @@ public void testNonProjectedUnsignedLongVersionCompatibility() { getIndexResult = IndexResolution.valid(index); SqlVersion preUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preUnsignedLong); - analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics(), sqlConfig.version())); + analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics())); String query = "SELECT unsigned_long = 1, unsigned_long::double FROM test"; String queryWithSubquery = "SELECT l = 1, SQRT(ul) FROM " @@ -378,7 +378,7 @@ public void testUnsignedLongStarExpandedVersionControlled() { for (SqlVersion version : List.of(preUnsignedLong, INTRODUCING_UNSIGNED_LONG, postUnsignedLong)) { SqlConfiguration config = SqlTestUtils.randomConfiguration(version); - analyzer = new Analyzer(config, functionRegistry, getIndexResult, new Verifier(new Metrics(), config.version())); + analyzer = new Analyzer(config, functionRegistry, getIndexResult, new Verifier(new Metrics())); LogicalPlan plan = plan(query); assertThat(plan, instanceOf(Project.class)); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index 61b56aaadb309..0d4e7994214b1 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -54,12 +54,7 @@ private String error(String sql) { } private String error(IndexResolution getIndexResult, String sql) { - Analyzer analyzer = new Analyzer( - TEST_CFG, - new SqlFunctionRegistry(), - getIndexResult, - new Verifier(new Metrics(), TEST_CFG.version()) - ); + Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics())); VerificationException e = expectThrows(VerificationException.class, () -> analyzer.analyze(parser.createStatement(sql), true)); String message = e.getMessage(); assertTrue(message.startsWith("Found ")); @@ -79,7 +74,7 @@ private EsIndex getTestEsIndex() { } private LogicalPlan accept(IndexResolution resolution, String sql) { - Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), resolution, new Verifier(new Metrics(), TEST_CFG.version())); + Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), resolution, new Verifier(new Metrics())); return analyzer.analyze(parser.createStatement(sql), true); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java index f03db58c0b320..84db6a231c02a 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java @@ -48,12 +48,7 @@ public void testDatabaseFunctionOutput() { null, null ); - Analyzer analyzer = new Analyzer( - sqlConfig, - new SqlFunctionRegistry(), - IndexResolution.valid(test), - new Verifier(new Metrics(), sqlConfig.version()) - ); + Analyzer analyzer = new Analyzer(sqlConfig, new SqlFunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics())); Project result = (Project) analyzer.analyze(parser.createStatement("SELECT DATABASE()"), true); NamedExpression ne = result.projections().get(0); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java index 2b3de1b42bb3f..67090f21a209f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java @@ -47,12 +47,7 @@ public void testNoUsernameFunctionOutput() { null, null ); - Analyzer analyzer = new Analyzer( - sqlConfig, - new SqlFunctionRegistry(), - IndexResolution.valid(test), - new Verifier(new Metrics(), sqlConfig.version()) - ); + Analyzer analyzer = new Analyzer(sqlConfig, new SqlFunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics())); Project result = (Project) analyzer.analyze(parser.createStatement("SELECT USER()"), true); NamedExpression ne = result.projections().get(0); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java index 3b7eabfde464c..090794b869416 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java @@ -93,12 +93,7 @@ public void testInvalidPrecision() { new EsIndex("test", SqlTypesTests.loadMapping("mapping-multi-field-with-nested.json")) ); - Analyzer analyzer = new Analyzer( - SqlTestUtils.TEST_CFG, - new SqlFunctionRegistry(), - indexResolution, - new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version()) - ); + Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, new Verifier(new Metrics())); ParsingException e = expectThrows( ParsingException.class, () -> analyzer.analyze(parser.createStatement("SELECT CURRENT_TIMESTAMP(100000000000000)"), true) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java index 499d6075370a4..4c845728f1c2b 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java @@ -94,12 +94,7 @@ public void testInvalidPrecision() { new EsIndex("test", SqlTypesTests.loadMapping("mapping-multi-field-with-nested.json")) ); - Analyzer analyzer = new Analyzer( - SqlTestUtils.TEST_CFG, - new SqlFunctionRegistry(), - indexResolution, - new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version()) - ); + Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, new Verifier(new Metrics())); ParsingException e = expectThrows( ParsingException.class, () -> analyzer.analyze(parser.createStatement("SELECT CURRENT_TIME(100000000000000)"), true) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java index e878e99260a45..c56a2eaeae34d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java @@ -77,12 +77,7 @@ public OptimizerRunTests() { EsIndex test = new EsIndex("test", mapping); getIndexResult = IndexResolution.valid(test); - analyzer = new Analyzer( - SqlTestUtils.TEST_CFG, - functionRegistry, - getIndexResult, - new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version()) - ); + analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, new Verifier(new Metrics())); optimizer = new Optimizer(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java index da373b200b05f..c3a6452e6d5cf 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.sql.plan.logical.command; -import org.elasticsearch.Version; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.sql.SqlTestUtils; @@ -18,9 +17,7 @@ import java.sql.JDBCType; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import static java.util.Arrays.asList; import static java.util.stream.Collectors.toList; @@ -36,7 +33,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.plan.logical.command.sys.SysColumnsTests.UNSIGNED_LONG_TEST_VERSIONS; import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_POINT; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_SHAPE; @@ -104,16 +101,7 @@ public void testShowColumns() { } public void testUnsignedLongFiltering() { - Set versions = new HashSet<>( - List.of( - SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER), - INTRODUCING_UNSIGNED_LONG, - SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER), - SqlVersion.fromId(Version.CURRENT.id) - ) - ); - - for (SqlVersion version : versions) { + for (SqlVersion version : UNSIGNED_LONG_TEST_VERSIONS) { List> rows = new ArrayList<>(); ShowColumns.fillInRows(loadMapping("mapping-multi-field-variation.json", true), null, version, rows); List typeNames = rows.stream().map(row -> (String) row.get(2)).collect(toList()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index 954de4b93d313..f8b481c845e8c 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -57,6 +57,13 @@ public class SysColumnsTests extends ESTestCase { + public static List UNSIGNED_LONG_TEST_VERSIONS = List.of( + SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER), + INTRODUCING_UNSIGNED_LONG, + SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER), + SqlVersion.fromId(Version.CURRENT.id) + ); + private static final String CLUSTER_NAME = "cluster"; private static final Map MAPPING1 = loadMapping("mapping-multi-field-with-nested.json", true); private static final Map MAPPING2 = loadMapping("mapping-multi-field-variation.json", true); @@ -139,14 +146,8 @@ public void testInNonDriverMode() { } public void testUnsignedLongFiltering() { - List versions = List.of( - SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER), - INTRODUCING_UNSIGNED_LONG, - SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER), - SqlVersion.fromId(Version.CURRENT.id) - ); for (Mode mode : List.of(Mode.JDBC, Mode.ODBC)) { - for (SqlVersion version : versions) { + for (SqlVersion version : UNSIGNED_LONG_TEST_VERSIONS) { List> rows = new ArrayList<>(); SysColumns.fillInRows( "test", @@ -354,12 +355,7 @@ private Tuple sql( Map mapping ) { EsIndex test = new EsIndex("test", mapping); - Analyzer analyzer = new Analyzer( - config, - new FunctionRegistry(), - IndexResolution.valid(test), - new Verifier(new Metrics(), config.version()) - ); + Analyzer analyzer = new Analyzer(config, new FunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics())); Command cmd = (Command) analyzer.analyze(parser.createStatement(sql, params, UTC), true); IndexResolver resolver = mock(IndexResolver.class); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java index 5f7a6268a6eee..1ad7371405bf3 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java @@ -371,7 +371,7 @@ private Tuple sql(String sql, List para SqlTestUtils.TEST_CFG, new FunctionRegistry(), IndexResolution.valid(test), - new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version()) + new Verifier(new Metrics()) ); Command cmd = (Command) analyzer.analyze(parser.createStatement(sql, params, cfg.zoneId()), true); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java index 13279d6942100..62a9a4765352a 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java @@ -36,7 +36,7 @@ import static java.util.Arrays.asList; import static org.elasticsearch.action.ActionListener.wrap; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.plan.logical.command.sys.SysColumnsTests.UNSIGNED_LONG_TEST_VERSIONS; import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.mockito.Mockito.mock; @@ -140,14 +140,7 @@ public void testSysTypes() { } public void testUnsignedLongFiltering() { - Set versions = new HashSet<>( - List.of( - SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER), - INTRODUCING_UNSIGNED_LONG, - SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER), - SqlVersion.fromId(Version.CURRENT.id) - ) - ); + Set versions = new HashSet<>(UNSIGNED_LONG_TEST_VERSIONS); versions.add(null); for (SqlVersion version : versions) { for (Mode mode : Mode.values()) { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java index a3c4b40cac8b8..c3413662a152c 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java @@ -60,12 +60,7 @@ public static void init() { Map mapping = SqlTypesTests.loadMapping("mapping-multi-field-variation.json"); EsIndex test = new EsIndex("test", mapping); IndexResolution getIndexResult = IndexResolution.valid(test); - analyzer = new Analyzer( - SqlTestUtils.TEST_CFG, - new SqlFunctionRegistry(), - getIndexResult, - new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version()) - ); + analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics())); optimizer = new Optimizer(); planner = new Planner(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorSpecTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorSpecTests.java index d7525d4ede3b5..3cc1023e37d1f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorSpecTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorSpecTests.java @@ -49,7 +49,7 @@ private static class TestContext { Map mapping = SqlTypesTests.loadMapping(mappingFile); EsIndex test = new EsIndex("test", mapping); IndexResolution getIndexResult = IndexResolution.valid(test); - analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics(), TEST_CFG.version())); + analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics())); optimizer = new Optimizer(); planner = new Planner(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java index 1a1a2ab036770..093096e152b31 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java @@ -130,7 +130,7 @@ private static class TestContext { Map mapping = SqlTypesTests.loadMapping(mappingFile); EsIndex test = new EsIndex("test", mapping); IndexResolution getIndexResult = IndexResolution.valid(test); - analyzer = new Analyzer(TEST_CFG, sqlFunctionRegistry, getIndexResult, new Verifier(new Metrics(), TEST_CFG.version())); + analyzer = new Analyzer(TEST_CFG, sqlFunctionRegistry, getIndexResult, new Verifier(new Metrics())); optimizer = new Optimizer(); planner = new Planner(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierTests.java index eb5f9197400d5..fdb87f9ab1c92 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierTests.java @@ -27,12 +27,7 @@ public class VerifierTests extends ESTestCase { private final IndexResolution indexResolution = IndexResolution.valid( new EsIndex("test", loadMapping("mapping-multi-field-with-nested.json")) ); - private final Analyzer analyzer = new Analyzer( - TEST_CFG, - new SqlFunctionRegistry(), - indexResolution, - new Verifier(new Metrics(), TEST_CFG.version()) - ); + private final Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), indexResolution, new Verifier(new Metrics())); private final Planner planner = new Planner(); private PhysicalPlan verify(String sql) { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java index 90aa3c9053c24..da6c96a84684e 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java @@ -177,7 +177,7 @@ public void testWhereLimitGroupByHavingOrderByQuery() { public void testTwoQueriesExecuted() { Metrics metrics = new Metrics(); - Verifier verifier = new Verifier(metrics, SqlTestUtils.TEST_CFG.version()); + Verifier verifier = new Verifier(metrics); sqlWithVerifier("SELECT languages FROM test WHERE languages > 2 GROUP BY languages LIMIT 5", verifier); sqlWithVerifier( "SELECT languages FROM test WHERE languages > 2 GROUP BY languages HAVING MAX(languages) > 3 " + "ORDER BY languages LIMIT 5", @@ -197,7 +197,7 @@ public void testTwoQueriesExecuted() { public void testTwoCommandsExecuted() { String command1 = randomFrom(commands); Metrics metrics = new Metrics(); - Verifier verifier = new Verifier(metrics, SqlTestUtils.TEST_CFG.version()); + Verifier verifier = new Verifier(metrics); sqlWithVerifier(command1, verifier); sqlWithVerifier(randomValueOtherThan(command1, () -> randomFrom(commands)), verifier); Counters c = metrics.stats(); @@ -255,7 +255,7 @@ private Counters sql(String sql, Verifier v) { Metrics metrics = null; if (v == null) { metrics = new Metrics(); - verifier = new Verifier(metrics, SqlTestUtils.TEST_CFG.version()); + verifier = new Verifier(metrics); } Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), IndexResolution.valid(test), verifier); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java index d95a479a85bcf..490fb935308d2 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java @@ -535,7 +535,7 @@ public void testConversionToBoolean() { } } - public void testConversiontoUnsignedLong() { + public void testConversionToUnsignedLong() { DataType to = UNSIGNED_LONG; { Converter conversion = converterFor(DATE, to); From 0943d4b1daffa0b1176398465c85a9cc4ba409b4 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Thu, 23 Dec 2021 14:06:58 +0100 Subject: [PATCH 21/28] Remove ShowColumns UL version-dep filtering Unsigned long will now always show in 'SHOW COLUMNS', irrespective of client's version (unlike 'SYS COLUMNS'). --- .../sql/plan/logical/command/ShowColumns.java | 3 +- .../logical/command/ShowColumnsTests.java | 90 +++++++++++++++++++ .../logical/command/ShowFunctionsTests.java | 78 ---------------- 3 files changed, 91 insertions(+), 80 deletions(-) create mode 100644 x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java index 8edd4c7833c48..5d1b792ed1ac3 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java @@ -31,7 +31,6 @@ import static java.util.Collections.emptyMap; import static org.elasticsearch.common.Strings.hasText; import static org.elasticsearch.transport.RemoteClusterAware.buildRemoteIndexName; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; public class ShowColumns extends Command { @@ -95,7 +94,7 @@ static void fillInRows(Map mapping, String prefix, SqlVersion v EsField field = e.getValue(); DataType dt = field.getDataType(); String name = e.getKey(); - if (dt != null && isTypeSupportedInVersion(dt, version)) { + if (dt != null) { // show only fields that exist in ES rows.add(asList(prefix != null ? prefix + "." + name : name, SqlDataTypes.sqlType(dt).getName(), dt.typeName())); if (field.getProperties().isEmpty() == false) { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java new file mode 100644 index 0000000000000..2aacdc1f96ae7 --- /dev/null +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.sql.plan.logical.command; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.sql.proto.SqlVersion; + +import java.sql.JDBCType; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.elasticsearch.Version.CURRENT; +import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; +import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; +import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT; +import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; +import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; +import static org.elasticsearch.xpack.ql.type.DataTypes.NESTED; +import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT; +import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; +import static org.elasticsearch.xpack.sql.plan.logical.command.sys.SysColumnsTests.UNSIGNED_LONG_TEST_VERSIONS; +import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_POINT; +import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_SHAPE; +import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; + +public class ShowColumnsTests extends ESTestCase { + + public static final String JDBC_TYPE_GEOMETRY = "GEOMETRY"; + + public void testShowColumns() { + String prefix = "myIndex"; + List> rows = new ArrayList<>(); + ShowColumns.fillInRows(loadMapping("mapping-multi-field-variation.json", true), prefix, SqlVersion.fromId(CURRENT.id), rows); + + List> expect = asList( + asList("bool", JDBCType.BOOLEAN.getName(), BOOLEAN.typeName()), + asList("int", JDBCType.INTEGER.getName(), INTEGER.typeName()), + asList("unsigned_long", JDBCType.NUMERIC.getName(), UNSIGNED_LONG.typeName()), + asList("float", JDBCType.REAL.getName(), FLOAT.typeName()), + asList("text", JDBCType.VARCHAR.getName(), TEXT.typeName()), + asList("keyword", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("date", JDBCType.TIMESTAMP.getName(), DATETIME.typeName()), + asList("date_nanos", JDBCType.TIMESTAMP.getName(), DATETIME.typeName()), + asList("unsupported", JDBCType.OTHER.getName(), UNSUPPORTED.typeName()), + asList("some", JDBCType.STRUCT.getName(), OBJECT.typeName()), + asList("some.dotted", JDBCType.STRUCT.getName(), OBJECT.typeName()), + asList("some.dotted.field", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("some.string", JDBCType.VARCHAR.getName(), TEXT.typeName()), + asList("some.string.normalized", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("some.string.typical", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("some.ambiguous", JDBCType.VARCHAR.getName(), TEXT.typeName()), + asList("some.ambiguous.one", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("some.ambiguous.two", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("some.ambiguous.normalized", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), + asList("foo_type", JDBCType.OTHER.getName(), UNSUPPORTED.typeName()), + asList("point", JDBC_TYPE_GEOMETRY, GEO_POINT.typeName()), + asList("shape", JDBC_TYPE_GEOMETRY, GEO_SHAPE.typeName()), + asList("nested", JDBCType.STRUCT.getName(), NESTED.typeName()), + asList("nested.point", JDBC_TYPE_GEOMETRY, GEO_POINT.typeName()) + ); + + assertEquals(expect.size(), rows.size()); + assertEquals(expect.get(0).size(), rows.get(0).size()); + + for (int i = 0; i < expect.size(); i++) { + List expectedRow = expect.get(i); + List receivedRow = rows.get(i); + assertEquals("Name mismatch in row " + i, prefix + "." + expectedRow.get(0), receivedRow.get(0)); + assertEquals("Type mismatch in row " + i, expectedRow.get(1), receivedRow.get(1)); + assertEquals("Mapping mismatch in row " + i, expectedRow.get(2), receivedRow.get(2)); + } + } + + public void testUnsignedLongFiltering() { + for (SqlVersion version : UNSIGNED_LONG_TEST_VERSIONS) { + List> rows = new ArrayList<>(); + ShowColumns.fillInRows(loadMapping("mapping-multi-field-variation.json", true), null, version, rows); + List typeNames = rows.stream().map(row -> (String) row.get(2)).toList(); + assertTrue(typeNames.contains(UNSIGNED_LONG.typeName())); + } + } +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java index c3a6452e6d5cf..b8b2ee07f085f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowFunctionsTests.java @@ -11,38 +11,13 @@ import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.sql.SqlTestUtils; import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry; -import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.session.SchemaRowSet; import org.elasticsearch.xpack.sql.session.SqlSession; -import java.sql.JDBCType; -import java.util.ArrayList; -import java.util.List; - -import static java.util.Arrays.asList; -import static java.util.stream.Collectors.toList; -import static org.elasticsearch.Version.CURRENT; import static org.elasticsearch.action.ActionListener.wrap; -import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; -import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; -import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT; -import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; -import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; -import static org.elasticsearch.xpack.ql.type.DataTypes.NESTED; -import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT; -import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; -import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; -import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; -import static org.elasticsearch.xpack.sql.plan.logical.command.sys.SysColumnsTests.UNSIGNED_LONG_TEST_VERSIONS; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; -import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_POINT; -import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_SHAPE; -import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; public class ShowFunctionsTests extends ESTestCase { - public static final String JDBC_TYPE_GEOMETRY = "GEOMETRY"; - public void testShowFunctions() throws Exception { ShowFunctions showFunctions = new ShowFunctions(Source.EMPTY, null); SqlSession session = new SqlSession(SqlTestUtils.TEST_CFG, null, new SqlFunctionRegistry(), null, null, null, null, null, null); @@ -55,57 +30,4 @@ public void testShowFunctions() throws Exception { assertEquals("AGGREGATE", r.column(1)); }, ex -> fail(ex.getMessage()))); } - - public void testShowColumns() { - String prefix = "myIndex"; - List> rows = new ArrayList<>(); - ShowColumns.fillInRows(loadMapping("mapping-multi-field-variation.json", true), prefix, SqlVersion.fromId(CURRENT.id), rows); - - List> expect = asList( - asList("bool", JDBCType.BOOLEAN.getName(), BOOLEAN.typeName()), - asList("int", JDBCType.INTEGER.getName(), INTEGER.typeName()), - asList("unsigned_long", JDBCType.NUMERIC.getName(), UNSIGNED_LONG.typeName()), - asList("float", JDBCType.REAL.getName(), FLOAT.typeName()), - asList("text", JDBCType.VARCHAR.getName(), TEXT.typeName()), - asList("keyword", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), - asList("date", JDBCType.TIMESTAMP.getName(), DATETIME.typeName()), - asList("date_nanos", JDBCType.TIMESTAMP.getName(), DATETIME.typeName()), - asList("unsupported", JDBCType.OTHER.getName(), UNSUPPORTED.typeName()), - asList("some", JDBCType.STRUCT.getName(), OBJECT.typeName()), - asList("some.dotted", JDBCType.STRUCT.getName(), OBJECT.typeName()), - asList("some.dotted.field", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), - asList("some.string", JDBCType.VARCHAR.getName(), TEXT.typeName()), - asList("some.string.normalized", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), - asList("some.string.typical", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), - asList("some.ambiguous", JDBCType.VARCHAR.getName(), TEXT.typeName()), - asList("some.ambiguous.one", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), - asList("some.ambiguous.two", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), - asList("some.ambiguous.normalized", JDBCType.VARCHAR.getName(), KEYWORD.typeName()), - asList("foo_type", JDBCType.OTHER.getName(), UNSUPPORTED.typeName()), - asList("point", JDBC_TYPE_GEOMETRY, GEO_POINT.typeName()), - asList("shape", JDBC_TYPE_GEOMETRY, GEO_SHAPE.typeName()), - asList("nested", JDBCType.STRUCT.getName(), NESTED.typeName()), - asList("nested.point", JDBC_TYPE_GEOMETRY, GEO_POINT.typeName()) - ); - - assertEquals(expect.size(), rows.size()); - assertEquals(expect.get(0).size(), rows.get(0).size()); - - for (int i = 0; i < expect.size(); i++) { - List expectedRow = expect.get(i); - List receivedRow = rows.get(i); - assertEquals("Name mismatch in row " + i, prefix + "." + expectedRow.get(0), receivedRow.get(0)); - assertEquals("Type mismatch in row " + i, expectedRow.get(1), receivedRow.get(1)); - assertEquals("Mapping mismatch in row " + i, expectedRow.get(2), receivedRow.get(2)); - } - } - - public void testUnsignedLongFiltering() { - for (SqlVersion version : UNSIGNED_LONG_TEST_VERSIONS) { - List> rows = new ArrayList<>(); - ShowColumns.fillInRows(loadMapping("mapping-multi-field-variation.json", true), null, version, rows); - List typeNames = rows.stream().map(row -> (String) row.get(2)).collect(toList()); - assertEquals(isTypeSupportedInVersion(UNSIGNED_LONG, version), typeNames.contains(UNSIGNED_LONG.typeName())); - } - } } From 8ebfad1b13a38e443ddc327638cb274b7c13067b Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Thu, 23 Dec 2021 17:16:42 +0100 Subject: [PATCH 22/28] Extract mapping reading out of loops Extract the reading of an index mapping out of the loops in a couple of tests. --- .../sql/plan/logical/command/ShowColumnsTests.java | 5 ++++- .../plan/logical/command/sys/SysColumnsTests.java | 12 ++---------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java index 2aacdc1f96ae7..e01d6db956406 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java @@ -8,11 +8,13 @@ package org.elasticsearch.xpack.sql.plan.logical.command; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.sql.proto.SqlVersion; import java.sql.JDBCType; import java.util.ArrayList; import java.util.List; +import java.util.Map; import static java.util.Arrays.asList; import static org.elasticsearch.Version.CURRENT; @@ -80,9 +82,10 @@ public void testShowColumns() { } public void testUnsignedLongFiltering() { + Map mapping = loadMapping("mapping-multi-field-variation.json", true); for (SqlVersion version : UNSIGNED_LONG_TEST_VERSIONS) { List> rows = new ArrayList<>(); - ShowColumns.fillInRows(loadMapping("mapping-multi-field-variation.json", true), null, version, rows); + ShowColumns.fillInRows(mapping, null, version, rows); List typeNames = rows.stream().map(row -> (String) row.get(2)).toList(); assertTrue(typeNames.contains(UNSIGNED_LONG.typeName())); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index f8b481c845e8c..4171a64851b75 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -146,19 +146,11 @@ public void testInNonDriverMode() { } public void testUnsignedLongFiltering() { + Map mapping = loadMapping("mapping-multi-field-variation.json", true); for (Mode mode : List.of(Mode.JDBC, Mode.ODBC)) { for (SqlVersion version : UNSIGNED_LONG_TEST_VERSIONS) { List> rows = new ArrayList<>(); - SysColumns.fillInRows( - "test", - "index", - loadMapping("mapping-multi-field-variation.json", true), - null, - rows, - null, - mode, - version - ); + SysColumns.fillInRows("test", "index", mapping, null, rows, null, mode, version); List types = rows.stream().map(row -> name(row).toString()).collect(Collectors.toList()); assertEquals( isTypeSupportedInVersion(UNSIGNED_LONG, version), From 64f02f75437ad581ef8eb0c58dc2e85d8ed3a3a8 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Tue, 4 Jan 2022 14:55:31 +0100 Subject: [PATCH 23/28] Resolve UL fields to unsupported in resolver Resolve the unsigned long fiels to UNSUPPORTED directly in the index resolver to remove the need for downstream type filtering. Also rearange enum to prevent any possible serialization issues. --- .../xpack/eql/session/EqlSession.java | 2 + .../xpack/ql/index/IndexResolver.java | 61 ++++++++++---- .../ql/index}/VersionCompatibilityChecks.java | 15 ++-- .../elasticsearch/xpack/ql/type/Types.java | 27 ++++--- .../xpack/ql/type/TypesTests.java | 19 ++--- .../elasticsearch/xpack/sql/jdbc/EsType.java | 4 +- .../xpack/sql/qa/rest/RestSqlTestCase.java | 29 +++++++ .../xpack/sql/analysis/analyzer/Analyzer.java | 5 +- .../xpack/sql/analysis/analyzer/Verifier.java | 7 +- .../extractor/CompositeKeyExtractor.java | 2 +- .../sql/plan/logical/command/ShowColumns.java | 29 ++++--- .../plan/logical/command/sys/SysColumns.java | 80 +++++++++---------- .../plan/logical/command/sys/SysTypes.java | 7 +- .../xpack/sql/session/SqlSession.java | 2 + .../analyzer/FieldAttributeTests.java | 19 +++-- .../analysis/index/IndexResolverTests.java | 7 +- .../logical/command/ShowColumnsTests.java | 14 ++-- .../logical/command/sys/SysColumnsTests.java | 22 ++--- .../logical/command/sys/SysTypesTests.java | 4 +- .../xpack/sql/types/SqlTypesTests.java | 7 +- 20 files changed, 225 insertions(+), 137 deletions(-) rename x-pack/plugin/{sql/src/main/java/org/elasticsearch/xpack/sql/session => ql/src/main/java/org/elasticsearch/xpack/ql/index}/VersionCompatibilityChecks.java (68%) diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java index 9ec908c696a0c..c8e2bd2c50fed 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.eql.session; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.ParentTaskAssigningClient; @@ -120,6 +121,7 @@ private void preAnalyze(LogicalPlan parsed, ActionListener list indexWildcard, configuration.indicesOptions(), configuration.runtimeMappings(), + Version.CURRENT, map(listener, r -> preAnalyzer.preAnalyze(parsed, r)) ); } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index ec88674f6b382..3a285daf7d6a5 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -9,6 +9,7 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; @@ -68,10 +69,12 @@ import static org.elasticsearch.common.Strings.hasText; import static org.elasticsearch.common.regex.Regex.simpleMatch; import static org.elasticsearch.transport.RemoteClusterAware.buildRemoteIndexName; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.supportsUnsignedLong; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; import static org.elasticsearch.xpack.ql.util.StringUtils.qualifyAndJoinIndices; import static org.elasticsearch.xpack.ql.util.StringUtils.splitQualifiedIndex; @@ -366,12 +369,16 @@ public void resolveAsMergedMapping( String indexWildcard, IndicesOptions indicesOptions, Map runtimeMappings, + Version version, ActionListener listener ) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, indicesOptions, runtimeMappings); client.fieldCaps( fieldRequest, - ActionListener.wrap(response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response)), listener::onFailure) + ActionListener.wrap( + response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response, version)), + listener::onFailure + ) ); } @@ -382,19 +389,24 @@ public void resolveAsMergedMapping( String indexWildcard, boolean includeFrozen, Map runtimeMappings, + Version version, ActionListener listener ) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen, runtimeMappings); client.fieldCaps( fieldRequest, - ActionListener.wrap(response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response)), listener::onFailure) + ActionListener.wrap( + response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response, version)), + listener::onFailure + ) ); } public static IndexResolution mergedMappings( DataTypeRegistry typeRegistry, String indexPattern, - FieldCapabilitiesResponse fieldCapsResponse + FieldCapabilitiesResponse fieldCapsResponse, + Version version ) { if (fieldCapsResponse.getIndices().length == 0) { @@ -402,7 +414,7 @@ public static IndexResolution mergedMappings( } // merge all indices onto the same one - List indices = buildIndices(typeRegistry, null, fieldCapsResponse, null, i -> indexPattern, (n, types) -> { + List indices = buildIndices(typeRegistry, null, fieldCapsResponse, null, i -> indexPattern, version, (n, types) -> { StringBuilder errorMessage = new StringBuilder(); boolean hasUnmapped = types.containsKey(UNMAPPED); @@ -473,7 +485,8 @@ private static EsField createField( Map> globalCaps, Map hierarchicalMapping, Map flattedMapping, - Function field + Function field, + Version version ) { Map parentProps = hierarchicalMapping; @@ -493,7 +506,7 @@ private static EsField createField( // lack of parent implies the field is an alias if (map == null) { // as such, create the field manually, marking the field to also be an alias - fieldFunction = s -> createField(typeRegistry, s, OBJECT.esType(), new TreeMap<>(), false, true); + fieldFunction = s -> createField(typeRegistry, s, OBJECT.esType(), new TreeMap<>(), false, true, version); } else { Iterator iterator = map.values().iterator(); FieldCapabilities parentCap = iterator.next(); @@ -501,10 +514,18 @@ private static EsField createField( parentCap = iterator.next(); } final FieldCapabilities parentC = parentCap; - fieldFunction = s -> createField(typeRegistry, s, parentC.getType(), new TreeMap<>(), parentC.isAggregatable(), false); + fieldFunction = s -> createField( + typeRegistry, + s, + parentC.getType(), + new TreeMap<>(), + parentC.isAggregatable(), + false, + version + ); } - parent = createField(typeRegistry, parentName, globalCaps, hierarchicalMapping, flattedMapping, fieldFunction); + parent = createField(typeRegistry, parentName, globalCaps, hierarchicalMapping, flattedMapping, fieldFunction, version); } parentProps = parent.getProperties(); } @@ -537,7 +558,8 @@ private static EsField createField( String typeName, Map props, boolean isAggregateable, - boolean isAlias + boolean isAlias, + Version version ) { DataType esType = typeRegistry.fromEs(typeName); @@ -553,7 +575,7 @@ private static EsField createField( if (esType == DATETIME) { return DateEsField.dateEsField(fieldName, props, isAggregateable); } - if (esType == UNSUPPORTED) { + if (esType == UNSUPPORTED || (esType == UNSIGNED_LONG && supportsUnsignedLong(version) == false)) { return new UnsupportedEsField(fieldName, typeName, null, props); } @@ -591,6 +613,7 @@ public void resolveAsSeparateMappings( String javaRegex, boolean includeFrozen, Map runtimeMappings, + Version version, ActionListener> listener ) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen, runtimeMappings); @@ -600,10 +623,12 @@ public void resolveAsSeparateMappings( .getAliases( createGetAliasesRequest(response, includeFrozen), wrap( - aliases -> { listener.onResponse(separateMappings(typeRegistry, javaRegex, response, aliases.getAliases())); }, + aliases -> { + listener.onResponse(separateMappings(typeRegistry, javaRegex, response, aliases.getAliases(), version)); + }, ex -> { if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) { - listener.onResponse(separateMappings(typeRegistry, javaRegex, response, null)); + listener.onResponse(separateMappings(typeRegistry, javaRegex, response, null, version)); } else { listener.onFailure(ex); } @@ -625,9 +650,10 @@ public static List separateMappings( DataTypeRegistry typeRegistry, String javaRegex, FieldCapabilitiesResponse fieldCaps, - ImmutableOpenMap> aliases + ImmutableOpenMap> aliases, + Version version ) { - return buildIndices(typeRegistry, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null); + return buildIndices(typeRegistry, javaRegex, fieldCaps, aliases, Function.identity(), version, (s, cap) -> null); } private static class Fields { @@ -645,6 +671,7 @@ private static List buildIndices( FieldCapabilitiesResponse fieldCapsResponse, ImmutableOpenMap> aliases, Function indexNameProcessor, + Version version, BiFunction, InvalidMappedField> validityVerifier ) { @@ -780,8 +807,10 @@ private static List buildIndices( typeCap.getType(), emptyMap(), typeCap.isAggregatable(), - isAliasFieldType.get() - ) + isAliasFieldType.get(), + version + ), + version ); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java similarity index 68% rename from x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java rename to x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java index b7587e6dac2cd..e66f010ea89c3 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java @@ -5,25 +5,22 @@ * 2.0. */ -package org.elasticsearch.xpack.sql.session; +package org.elasticsearch.xpack.ql.index; +import org.elasticsearch.Version; import org.elasticsearch.core.Nullable; import org.elasticsearch.xpack.ql.type.DataType; -import org.elasticsearch.xpack.sql.proto.SqlVersion; import static org.elasticsearch.Version.V_8_1_0; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; public final class VersionCompatibilityChecks { - public static final SqlVersion INTRODUCING_UNSIGNED_LONG = SqlVersion.fromId(V_8_1_0.id); + public static final Version INTRODUCING_UNSIGNED_LONG = V_8_1_0; private VersionCompatibilityChecks() {} - /** - * Is the provided {@code dataType} being supported in the provided {@code version}? - */ - public static boolean isTypeSupportedInVersion(DataType dataType, SqlVersion version) { + public static boolean isTypeSupportedInVersion(DataType dataType, Version version) { if (dataType == UNSIGNED_LONG) { return supportsUnsignedLong(version); } @@ -33,11 +30,11 @@ public static boolean isTypeSupportedInVersion(DataType dataType, SqlVersion ver /** * Does the provided {@code version} support the unsigned_long type (PR#60050)? */ - public static boolean supportsUnsignedLong(SqlVersion version) { + public static boolean supportsUnsignedLong(Version version) { return INTRODUCING_UNSIGNED_LONG.compareTo(version) <= 0; } - public static @Nullable SqlVersion versionIntroducingType(DataType dataType) { + public static @Nullable Version versionIntroducingType(DataType dataType) { if (dataType == UNSIGNED_LONG) { return INTRODUCING_UNSIGNED_LONG; } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java index 515d70dff05c8..818b4c9ad815d 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.ql.type; +import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.core.Booleans; @@ -15,32 +16,34 @@ import java.util.Map.Entry; import static java.util.Collections.emptyMap; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.supportsUnsignedLong; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.NESTED; import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; public abstract class Types { @SuppressWarnings("unchecked") - public static Map fromEs(DataTypeRegistry typeRegistry, Map asMap) { + public static Map fromEs(DataTypeRegistry typeRegistry, Map asMap, Version version) { Map props = null; if (asMap != null && asMap.isEmpty() == false) { props = (Map) asMap.get("properties"); } - return props == null || props.isEmpty() ? emptyMap() : startWalking(typeRegistry, props); + return props == null || props.isEmpty() ? emptyMap() : startWalking(typeRegistry, props, version); } - private static Map startWalking(DataTypeRegistry typeRegistry, Map mapping) { + private static Map startWalking(DataTypeRegistry typeRegistry, Map mapping, Version version) { Map types = new LinkedHashMap<>(); if (mapping == null) { return emptyMap(); } for (Entry entry : mapping.entrySet()) { - walkMapping(typeRegistry, entry.getKey(), entry.getValue(), types); + walkMapping(typeRegistry, entry.getKey(), entry.getValue(), types, version); } return types; @@ -65,7 +68,13 @@ private static DataType getType(DataTypeRegistry typeRegistry, Map mapping) { + private static void walkMapping( + DataTypeRegistry typeRegistry, + String name, + Object value, + Map mapping, + Version version + ) { // object type - only root or nested docs supported if (value instanceof Map) { Map content = (Map) value; @@ -74,17 +83,17 @@ private static void walkMapping(DataTypeRegistry typeRegistry, String name, Obje DataType esDataType = getType(typeRegistry, content); final Map properties; if (esDataType == OBJECT || esDataType == NESTED) { - properties = fromEs(typeRegistry, content); + properties = fromEs(typeRegistry, content, version); } else if (content.containsKey("fields")) { // Check for multifields Object fields = content.get("fields"); if (fields instanceof Map) { - properties = startWalking(typeRegistry, (Map) fields); + properties = startWalking(typeRegistry, (Map) fields, version); } else { properties = Collections.emptyMap(); } } else { - properties = fromEs(typeRegistry, content); + properties = fromEs(typeRegistry, content, version); } boolean docValues = boolSetting(content.get("doc_values"), esDataType.hasDocValues()); final EsField field; @@ -96,7 +105,7 @@ private static void walkMapping(DataTypeRegistry typeRegistry, String name, Obje field = new KeywordEsField(name, properties, docValues, length, normalized); } else if (esDataType == DATETIME) { field = DateEsField.dateEsField(name, properties, docValues); - } else if (esDataType == UNSUPPORTED) { + } else if (esDataType == UNSUPPORTED || (esDataType == UNSIGNED_LONG && supportsUnsignedLong(version) == false)) { String type = content.get("type").toString(); field = new UnsupportedEsField(name, type, null, properties); propagateUnsupportedType(name, type, properties); diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java index f565896416dce..fbf95e6c96acb 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.ql.type; +import org.elasticsearch.Version; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.json.JsonXContent; @@ -27,12 +28,12 @@ public class TypesTests extends ESTestCase { public void testNullMap() { - Map fromEs = Types.fromEs(DefaultDataTypeRegistry.INSTANCE, null); + Map fromEs = Types.fromEs(DefaultDataTypeRegistry.INSTANCE, null, Version.CURRENT); assertThat(fromEs.isEmpty(), is(true)); } public void testEmptyMap() { - Map fromEs = Types.fromEs(DefaultDataTypeRegistry.INSTANCE, emptyMap()); + Map fromEs = Types.fromEs(DefaultDataTypeRegistry.INSTANCE, emptyMap(), Version.CURRENT); assertThat(fromEs.isEmpty(), is(true)); } @@ -211,27 +212,27 @@ public void testUnsupportedTypes() { } public static Map loadMapping(String name) { - return loadMapping(DefaultDataTypeRegistry.INSTANCE, name, null); + return loadMapping(DefaultDataTypeRegistry.INSTANCE, name, null, Version.CURRENT); } public static Map loadMapping(String name, boolean ordered) { - return loadMapping(DefaultDataTypeRegistry.INSTANCE, name, ordered); + return loadMapping(DefaultDataTypeRegistry.INSTANCE, name, ordered, Version.CURRENT); } public static Map loadMapping(DataTypeRegistry registry, String name) { - return loadMapping(registry, name, null); + return loadMapping(registry, name, null, Version.CURRENT); } - public static Map loadMapping(DataTypeRegistry registry, String name, Boolean ordered) { + public static Map loadMapping(DataTypeRegistry registry, String name, Boolean ordered, Version version) { InputStream stream = TypesTests.class.getResourceAsStream("/" + name); assertNotNull("Could not find mapping resource:" + name, stream); - return loadMapping(registry, stream, ordered); + return loadMapping(registry, stream, ordered, version); } - public static Map loadMapping(DataTypeRegistry registry, InputStream stream, Boolean ordered) { + private static Map loadMapping(DataTypeRegistry registry, InputStream stream, Boolean ordered, Version version) { boolean order = ordered != null ? ordered.booleanValue() : randomBoolean(); try (InputStream in = stream) { - return Types.fromEs(registry, XContentHelper.convertToMap(JsonXContent.jsonXContent, in, order)); + return Types.fromEs(registry, XContentHelper.convertToMap(JsonXContent.jsonXContent, in, order), version); } catch (IOException ex) { throw new RuntimeException(ex); } diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java index 2541aaa8bdc28..6fd6bb7e83d6e 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java @@ -19,7 +19,6 @@ public enum EsType implements SQLType { SHORT(Types.SMALLINT), INTEGER(Types.INTEGER), LONG(Types.BIGINT), - UNSIGNED_LONG(Types.NUMERIC), DOUBLE(Types.DOUBLE), FLOAT(Types.REAL), HALF_FLOAT(Types.FLOAT), @@ -48,7 +47,8 @@ public enum EsType implements SQLType { INTERVAL_MINUTE_TO_SECOND(ExtraTypes.INTERVAL_MINUTE_SECOND), GEO_POINT(ExtraTypes.GEOMETRY), GEO_SHAPE(ExtraTypes.GEOMETRY), - SHAPE(ExtraTypes.GEOMETRY); + SHAPE(ExtraTypes.GEOMETRY), + UNSIGNED_LONG(Types.NUMERIC); private final Integer type; diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java index 9226b16591759..b5563f8651ca7 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java @@ -11,6 +11,7 @@ import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; @@ -22,6 +23,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.test.NotEqualMessageBuilder; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.sql.proto.Mode; @@ -54,6 +56,7 @@ import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.common.Strings.hasText; import static org.elasticsearch.xpack.ql.TestUtils.getNumberOfSearchContexts; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.proto.Protocol.COLUMNS_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.HEADER_NAME_ASYNC_ID; import static org.elasticsearch.xpack.sql.proto.Protocol.HEADER_NAME_ASYNC_PARTIAL; @@ -1151,6 +1154,16 @@ public void testBinaryFieldFiltering() throws IOException { deleteIndex("test_binary"); } + public void testPreventedUnsignedLongMaskedAccess() throws IOException { + loadUnsignedLongTestData(); + Version version = VersionUtils.randomVersionBetween(random(), null, VersionUtils.getPreviousVersion(INTRODUCING_UNSIGNED_LONG)); + String query = query("SELECT unsigned_long::STRING FROM " + indexPattern("test")).version(version.toString()).toString(); + expectBadRequest( + () -> runSql(new StringEntity(query, ContentType.APPLICATION_JSON), "", randomMode()), + containsString("Cannot use field [unsigned_long] with unsupported type [unsigned_long]") + ); + } + private void executeQueryWithNextPage(String format, String expectedHeader, String expectedLineFormat) throws IOException { int size = 20; String[] docs = new String[size]; @@ -1221,6 +1234,22 @@ private static void bulkLoadTestData(int count) throws IOException { provisioningClient().performRequest(request); } + private void loadUnsignedLongTestData() throws IOException { + Request request = new Request("PUT", "/test"); + request.setJsonEntity(""" + { + "mappings": { + "properties": { + "unsigned_long": { + "type": "unsigned_long" + } + } + } + }"""); + provisioningClient().performRequest(request); + index("{\"unsigned_long\": 18446744073709551615}"); + } + protected static Tuple runSqlAsText(String sql, String accept) throws IOException { return runSqlAsText(StringUtils.EMPTY, new StringEntity(query(sql).toString(), ContentType.APPLICATION_JSON), accept); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index 9670cc7bbb007..3d05acd9c3467 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.sql.analysis.analyzer; +import org.elasticsearch.Version; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.core.Tuple; import org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.AddMissingEqualsToBoolField; @@ -77,8 +78,8 @@ import static java.util.stream.Collectors.toList; import static org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.AnalyzerRule; import static org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.BaseAnalyzerRule; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; public class Analyzer extends RuleExecutor { /** @@ -451,7 +452,7 @@ private List expandProjections(List private List filterUnsupportedProjections(List projections) { return projections.stream() - .filter(p -> p.resolved() == false || isTypeSupportedInVersion(p.dataType(), configuration.version())) + .filter(p -> p.resolved() == false || isTypeSupportedInVersion(p.dataType(), Version.fromId(configuration.version().id))) .collect(toList()); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index 64b2d57db6aca..a1590f74bfdfb 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.sql.analysis.analyzer; +import org.elasticsearch.Version; import org.elasticsearch.core.Tuple; import org.elasticsearch.xpack.ql.capabilities.Unresolvable; import org.elasticsearch.xpack.ql.common.Failure; @@ -76,11 +77,11 @@ import static java.util.stream.Collectors.toMap; import static org.elasticsearch.xpack.ql.analyzer.VerifierChecks.checkFilterConditionType; import static org.elasticsearch.xpack.ql.common.Failure.fail; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.versionIntroducingType; import static org.elasticsearch.xpack.ql.type.DataTypes.BINARY; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.versionIntroducingType; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.COMMAND; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.GROUPBY; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.HAVING; @@ -1004,7 +1005,7 @@ private static void checkCastOnInexact(LogicalPlan p, Set localFailures private static void checkClientSupportsDataTypes(LogicalPlan p, Set localFailures, SqlVersion version) { p.output().forEach(e -> { - if (e.resolved() && isTypeSupportedInVersion(e.dataType(), version) == false) { + if (e.resolved() && isTypeSupportedInVersion(e.dataType(), Version.fromId(version.id)) == false) { localFailures.add( fail( e, diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java index a79a15d30acaf..010d0fafade16 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java @@ -23,11 +23,11 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypeConverter.toUnsignedLong; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.NULL; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.isDateBased; public class CompositeKeyExtractor implements BucketExtractor { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java index 5d1b792ed1ac3..e6920af0b46c8 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.sql.plan.logical.command; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.FieldAttribute; @@ -15,7 +16,6 @@ import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.ql.type.KeywordEsField; -import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.session.Cursor.Page; import org.elasticsearch.xpack.sql.session.SqlSession; import org.elasticsearch.xpack.sql.type.SqlDataTypes; @@ -79,17 +79,24 @@ public void execute(SqlSession session, ActionListener listener) { idx = hasText(cat) && cat.equals(cluster) == false ? buildRemoteIndexName(cat, idx) : idx; boolean withFrozen = includeFrozen || session.configuration().includeFrozen(); - session.indexResolver().resolveAsMergedMapping(idx, withFrozen, emptyMap(), ActionListener.wrap(indexResult -> { - List> rows = emptyList(); - if (indexResult.isValid()) { - rows = new ArrayList<>(); - fillInRows(indexResult.get().mapping(), null, session.configuration().version(), rows); - } - listener.onResponse(of(session, rows)); - }, listener::onFailure)); + session.indexResolver() + .resolveAsMergedMapping( + idx, + withFrozen, + emptyMap(), + Version.fromId(session.configuration().version().id), + ActionListener.wrap(indexResult -> { + List> rows = emptyList(); + if (indexResult.isValid()) { + rows = new ArrayList<>(); + fillInRows(indexResult.get().mapping(), null, rows); + } + listener.onResponse(of(session, rows)); + }, listener::onFailure) + ); } - static void fillInRows(Map mapping, String prefix, SqlVersion version, List> rows) { + static void fillInRows(Map mapping, String prefix, List> rows) { for (Entry e : mapping.entrySet()) { EsField field = e.getValue(); DataType dt = field.getDataType(); @@ -99,7 +106,7 @@ static void fillInRows(Map mapping, String prefix, SqlVersion v rows.add(asList(prefix != null ? prefix + "." + name : name, SqlDataTypes.sqlType(dt).getName(), dt.typeName())); if (field.getProperties().isEmpty() == false) { String newPrefix = prefix != null ? prefix + "." + name : name; - fillInRows(field.getProperties(), newPrefix, version, rows); + fillInRows(field.getProperties(), newPrefix, rows); } } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java index 40c89c0c58bc5..828b38480d01c 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.sql.plan.logical.command.sys; import org.apache.lucene.util.Counter; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.Strings; import org.elasticsearch.xpack.ql.expression.Attribute; @@ -19,7 +20,6 @@ import org.elasticsearch.xpack.ql.util.StringUtils; import org.elasticsearch.xpack.sql.plan.logical.command.Command; import org.elasticsearch.xpack.sql.proto.Mode; -import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.session.Cursor.Page; import org.elasticsearch.xpack.sql.session.ListCursor; import org.elasticsearch.xpack.sql.session.Rows; @@ -44,7 +44,6 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.isString; import static org.elasticsearch.xpack.ql.util.StringUtils.isQualified; import static org.elasticsearch.xpack.ql.util.StringUtils.splitQualifiedIndex; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.displaySize; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDataType; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDateTimeSub; @@ -158,34 +157,39 @@ public void execute(SqlSession session, ActionListener listener) { // special case for '%' (translated to *) if ("*".equals(idx)) { session.indexResolver() - .resolveAsSeparateMappings(indexPattern, regex, includeFrozen, emptyMap(), ActionListener.wrap(esIndices -> { - List> rows = new ArrayList<>(); - for (EsIndex esIndex : esIndices) { - fillInRows( - tableCat, - esIndex.name(), - esIndex.mapping(), - null, - rows, - columnMatcher, - mode, - session.configuration().version() - ); - } - listener.onResponse(ListCursor.of(Rows.schema(output), rows, session.configuration().pageSize())); - }, listener::onFailure)); + .resolveAsSeparateMappings( + indexPattern, + regex, + includeFrozen, + emptyMap(), + Version.fromId(session.configuration().version().id), + ActionListener.wrap(esIndices -> { + List> rows = new ArrayList<>(); + for (EsIndex esIndex : esIndices) { + fillInRows(tableCat, esIndex.name(), esIndex.mapping(), null, rows, columnMatcher, mode); + } + listener.onResponse(ListCursor.of(Rows.schema(output), rows, session.configuration().pageSize())); + }, listener::onFailure) + ); } // otherwise use a merged mapping else { - session.indexResolver().resolveAsMergedMapping(indexPattern, includeFrozen, emptyMap(), ActionListener.wrap(r -> { - List> rows = new ArrayList<>(); - // populate the data only when a target is found - if (r.isValid()) { - EsIndex esIndex = r.get(); - fillInRows(tableCat, indexName, esIndex.mapping(), null, rows, columnMatcher, mode, session.configuration().version()); - } - listener.onResponse(ListCursor.of(Rows.schema(output), rows, session.configuration().pageSize())); - }, listener::onFailure)); + session.indexResolver() + .resolveAsMergedMapping( + indexPattern, + includeFrozen, + emptyMap(), + Version.fromId(session.configuration().version().id), + ActionListener.wrap(r -> { + List> rows = new ArrayList<>(); + // populate the data only when a target is found + if (r.isValid()) { + EsIndex esIndex = r.get(); + fillInRows(tableCat, indexName, esIndex.mapping(), null, rows, columnMatcher, mode); + } + listener.onResponse(ListCursor.of(Rows.schema(output), rows, session.configuration().pageSize())); + }, listener::onFailure) + ); } } @@ -196,20 +200,9 @@ static void fillInRows( String prefix, List> rows, Pattern columnMatcher, - Mode mode, - SqlVersion version + Mode mode ) { - fillInRows( - clusterName, - splitQualifiedIndex(indexName).v2(), - mapping, - prefix, - rows, - columnMatcher, - Counter.newCounter(), - mode, - version - ); + fillInRows(clusterName, splitQualifiedIndex(indexName).v2(), mapping, prefix, rows, columnMatcher, Counter.newCounter(), mode); } private static void fillInRows( @@ -220,8 +213,7 @@ private static void fillInRows( List> rows, Pattern columnMatcher, Counter position, - Mode mode, - SqlVersion version + Mode mode ) { boolean isOdbcClient = mode == Mode.ODBC; for (Map.Entry entry : mapping.entrySet()) { @@ -233,7 +225,7 @@ private static void fillInRows( DataType type = field.getDataType(); // skip the nested, object and unsupported types - if (isPrimitive(type) && isTypeSupportedInVersion(type, version)) { + if (isPrimitive(type)) { if (columnMatcher == null || columnMatcher.matcher(name).matches()) { rows.add( asList( @@ -277,7 +269,7 @@ private static void fillInRows( } // skip nested fields if (field.getProperties() != null && type != NESTED) { - fillInRows(clusterName, indexName, field.getProperties(), name, rows, columnMatcher, position, mode, version); + fillInRows(clusterName, indexName, field.getProperties(), name, rows, columnMatcher, position, mode); } } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java index 2ae492de76d4e..66efd1a4ee879 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.sql.plan.logical.command.sys; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.tree.NodeInfo; @@ -23,12 +24,12 @@ import static java.util.Arrays.asList; import static java.util.stream.Collectors.toList; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; import static org.elasticsearch.xpack.ql.type.DataTypes.SHORT; import static org.elasticsearch.xpack.ql.type.DataTypes.isSigned; import static org.elasticsearch.xpack.ql.type.DataTypes.isString; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDataType; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDateTimeSub; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlMaximumScale; @@ -84,7 +85,9 @@ public List output() { @Override public final void execute(SqlSession session, ActionListener listener) { - Stream values = SqlDataTypes.types().stream().filter(t -> isTypeSupportedInVersion(t, session.configuration().version())); + Stream values = SqlDataTypes.types() + .stream() + .filter(t -> isTypeSupportedInVersion(t, Version.fromId(session.configuration().version().id))); if (type.intValue() != 0) { values = values.filter(t -> type.equals(sqlType(t).getVendorTypeNumber())); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java index 24494cf3251e9..358e309e5febe 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.sql.session; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.ParentTaskAssigningClient; @@ -158,6 +159,7 @@ private void preAnalyze(LogicalPlan parsed, Function act indexPattern, includeFrozen, configuration.runtimeMappings(), + Version.fromId(configuration.version().id), wrap(indexResult -> listener.onResponse(action.apply(indexResult)), listener::onFailure) ); } else { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java index 6ee9a6f8ecbc5..4413f4d79ed32 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.sql.analysis.analyzer; +import org.elasticsearch.Version; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.expression.Alias; @@ -35,13 +36,13 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.contains; @@ -308,7 +309,8 @@ public void testUnsignedLongVersionCompatibility() { getIndexResult = IndexResolution.valid(index); String query = "SELECT unsigned_long FROM test"; - String queryWithLiteral = "SELECT 18446744073709551615 as unsigned_long"; + String queryWithLiteral = "SELECT 18446744073709551615 AS unsigned_long"; + String queryWithCastLiteral = "SELECT '18446744073709551615'::unsigned_long AS unsigned_long"; String queryWithAlias = "SELECT unsigned_long AS unsigned_long FROM test"; String queryWithArithmetic = "SELECT unsigned_long + 1 AS unsigned_long FROM test"; String queryWithCast = "SELECT long + 1::unsigned_long AS unsigned_long FROM test"; @@ -316,7 +318,7 @@ public void testUnsignedLongVersionCompatibility() { SqlVersion preUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); SqlVersion postUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER); - for (String sql : List.of(query, queryWithLiteral, queryWithAlias, queryWithArithmetic, queryWithCast)) { + for (String sql : List.of(query, queryWithLiteral, queryWithCastLiteral, queryWithAlias, queryWithArithmetic, queryWithCast)) { SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preUnsignedLong); analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics())); VerificationException ex = expectThrows(VerificationException.class, () -> plan(sql)); @@ -329,7 +331,7 @@ public void testUnsignedLongVersionCompatibility() { ex.getMessage() ); - for (SqlVersion v : List.of(INTRODUCING_UNSIGNED_LONG, postUnsignedLong)) { + for (SqlVersion v : List.of(SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id), postUnsignedLong)) { analyzer = new Analyzer(SqlTestUtils.randomConfiguration(v), functionRegistry, getIndexResult, verifier); LogicalPlan plan = plan(sql); assertThat(plan, instanceOf(Project.class)); @@ -375,7 +377,7 @@ public void testUnsignedLongStarExpandedVersionControlled() { SqlVersion postUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER); String query = "SELECT * FROM test"; - for (SqlVersion version : List.of(preUnsignedLong, INTRODUCING_UNSIGNED_LONG, postUnsignedLong)) { + for (SqlVersion version : List.of(preUnsignedLong, SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id), postUnsignedLong)) { SqlConfiguration config = SqlTestUtils.randomConfiguration(version); analyzer = new Analyzer(config, functionRegistry, getIndexResult, new Verifier(new Metrics())); @@ -384,7 +386,10 @@ public void testUnsignedLongStarExpandedVersionControlled() { Project p = (Project) plan; List projectedDataTypes = p.projections().stream().map(Expression::dataType).collect(Collectors.toList()); - assertEquals(isTypeSupportedInVersion(UNSIGNED_LONG, config.version()), projectedDataTypes.contains(UNSIGNED_LONG)); + assertEquals( + isTypeSupportedInVersion(UNSIGNED_LONG, Version.fromId(config.version().id)), + projectedDataTypes.contains(UNSIGNED_LONG) + ); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 50edd14dd8278..f1ad6aeba4d10 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.sql.analysis.index; +import org.elasticsearch.Version; import org.elasticsearch.action.fieldcaps.FieldCapabilities; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.test.ESTestCase; @@ -495,7 +496,8 @@ private static IndexResolution mergedMappings( return IndexResolver.mergedMappings( SqlDataTypeRegistry.INSTANCE, indexPattern, - new FieldCapabilitiesResponse(indexNames, fieldCaps) + new FieldCapabilitiesResponse(indexNames, fieldCaps), + Version.CURRENT ); } @@ -508,7 +510,8 @@ private static List separateMappings( SqlDataTypeRegistry.INSTANCE, javaRegex, new FieldCapabilitiesResponse(indexNames, fieldCaps), - null + null, + Version.CURRENT ); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java index e01d6db956406..3e6c446c467a1 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.sql.plan.logical.command; +import org.elasticsearch.Version; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.sql.proto.SqlVersion; @@ -17,7 +18,7 @@ import java.util.Map; import static java.util.Arrays.asList; -import static org.elasticsearch.Version.CURRENT; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.supportsUnsignedLong; import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.FLOAT; @@ -40,7 +41,7 @@ public class ShowColumnsTests extends ESTestCase { public void testShowColumns() { String prefix = "myIndex"; List> rows = new ArrayList<>(); - ShowColumns.fillInRows(loadMapping("mapping-multi-field-variation.json", true), prefix, SqlVersion.fromId(CURRENT.id), rows); + ShowColumns.fillInRows(loadMapping("mapping-multi-field-variation.json", true), prefix, rows); List> expect = asList( asList("bool", JDBCType.BOOLEAN.getName(), BOOLEAN.typeName()), @@ -82,12 +83,13 @@ public void testShowColumns() { } public void testUnsignedLongFiltering() { - Map mapping = loadMapping("mapping-multi-field-variation.json", true); + List rowSupported = List.of("unsigned_long", "NUMERIC", "unsigned_long"); + List rowUnsupported = List.of("unsigned_long", "OTHER", "unsupported"); for (SqlVersion version : UNSIGNED_LONG_TEST_VERSIONS) { List> rows = new ArrayList<>(); - ShowColumns.fillInRows(mapping, null, version, rows); - List typeNames = rows.stream().map(row -> (String) row.get(2)).toList(); - assertTrue(typeNames.contains(UNSIGNED_LONG.typeName())); + Map mapping = loadMapping("mapping-multi-field-variation.json", true, Version.fromId(version.id)); + ShowColumns.fillInRows(mapping, null, rows); + assertTrue((supportsUnsignedLong(Version.fromId(version.id)) && rows.contains(rowSupported)) || rows.contains(rowUnsupported)); } } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index 4171a64851b75..12b1f7968d319 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -44,10 +44,10 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.action.ActionListener.wrap; import static org.elasticsearch.xpack.ql.TestUtils.UTC; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.proto.Mode.isDriver; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -59,7 +59,7 @@ public class SysColumnsTests extends ESTestCase { public static List UNSIGNED_LONG_TEST_VERSIONS = List.of( SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER), - INTRODUCING_UNSIGNED_LONG, + SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id), SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER), SqlVersion.fromId(Version.CURRENT.id) ); @@ -75,7 +75,7 @@ public class SysColumnsTests extends ESTestCase { private void sysColumnsInMode(Mode mode) { Class typeClass = mode == Mode.ODBC ? Short.class : Integer.class; List> rows = new ArrayList<>(); - SysColumns.fillInRows("test", "index", MAPPING2, null, rows, null, mode, SqlVersion.fromId(Version.CURRENT.id)); + SysColumns.fillInRows("test", "index", MAPPING2, null, rows, null, mode); assertEquals(FIELD_COUNT2, rows.size()); assertEquals(24, rows.get(0).size()); @@ -146,14 +146,14 @@ public void testInNonDriverMode() { } public void testUnsignedLongFiltering() { - Map mapping = loadMapping("mapping-multi-field-variation.json", true); for (Mode mode : List.of(Mode.JDBC, Mode.ODBC)) { for (SqlVersion version : UNSIGNED_LONG_TEST_VERSIONS) { List> rows = new ArrayList<>(); - SysColumns.fillInRows("test", "index", mapping, null, rows, null, mode, version); + Map mapping = loadMapping("mapping-multi-field-variation.json", true, Version.fromId(version.id)); + SysColumns.fillInRows("test", "index", mapping, null, rows, null, mode); List types = rows.stream().map(row -> name(row).toString()).collect(Collectors.toList()); assertEquals( - isTypeSupportedInVersion(UNSIGNED_LONG, version), + isTypeSupportedInVersion(UNSIGNED_LONG, Version.fromId(version.id)), types.contains(UNSIGNED_LONG.toString().toLowerCase(Locale.ROOT)) ); } @@ -354,13 +354,13 @@ private Tuple sql( when(resolver.clusterName()).thenReturn(CLUSTER_NAME); when(resolver.remoteClusters()).thenReturn(Set.of(CLUSTER_NAME)); doAnswer(invocation -> { - ((ActionListener) invocation.getArguments()[3]).onResponse(IndexResolution.valid(test)); + ((ActionListener) invocation.getArguments()[4]).onResponse(IndexResolution.valid(test)); return Void.TYPE; - }).when(resolver).resolveAsMergedMapping(any(), anyBoolean(), any(), any()); + }).when(resolver).resolveAsMergedMapping(any(), anyBoolean(), any(), any(), any()); doAnswer(invocation -> { - ((ActionListener>) invocation.getArguments()[4]).onResponse(singletonList(test)); + ((ActionListener>) invocation.getArguments()[5]).onResponse(singletonList(test)); return Void.TYPE; - }).when(resolver).resolveAsSeparateMappings(any(), any(), anyBoolean(), any(), any()); + }).when(resolver).resolveAsSeparateMappings(any(), any(), anyBoolean(), any(), any(), any()); SqlSession session = new SqlSession(config, null, null, resolver, null, null, null, null, null); return new Tuple<>(cmd, session); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java index 62a9a4765352a..461ba7601e054 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java @@ -35,9 +35,9 @@ import static java.util.Arrays.asList; import static org.elasticsearch.action.ActionListener.wrap; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.sql.plan.logical.command.sys.SysColumnsTests.UNSIGNED_LONG_TEST_VERSIONS; -import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.mockito.Mockito.mock; public class SysTypesTests extends ESTestCase { @@ -151,7 +151,7 @@ public void testUnsignedLongFiltering() { List types = new ArrayList<>(); r.forEachRow(rv -> types.add((String) rv.column(0))); assertEquals( - isTypeSupportedInVersion(UNSIGNED_LONG, cmd.v2().configuration().version()), + isTypeSupportedInVersion(UNSIGNED_LONG, Version.fromId(cmd.v2().configuration().version().id)), types.contains(UNSIGNED_LONG.toString()) ); }, ex -> fail(ex.getMessage()))); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/types/SqlTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/types/SqlTypesTests.java index 8e05925c54a47..c9cc33a6af67e 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/types/SqlTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/types/SqlTypesTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.sql.types; +import org.elasticsearch.Version; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.ql.type.KeywordEsField; @@ -88,6 +89,10 @@ public static Map loadMapping(String name) { } public static Map loadMapping(String name, boolean ordered) { - return TypesTests.loadMapping(SqlDataTypeRegistry.INSTANCE, name, ordered); + return loadMapping(name, ordered, Version.CURRENT); + } + + public static Map loadMapping(String name, boolean ordered, Version version) { + return TypesTests.loadMapping(SqlDataTypeRegistry.INSTANCE, name, ordered, version); } } From f6c3a8e168f75b86059dc2f208cb546c9c39b303 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Mon, 31 Jan 2022 18:53:00 +0100 Subject: [PATCH 24/28] Address review comments Take SqlVersion -> Version out of a loop. Set a minimum shard version at UL-version for searches. --- .../xpack/ql/execution/search/QlSourceBuilder.java | 1 - .../xpack/ql/expression/function/scalar/ScalarFunction.java | 2 +- .../elasticsearch/xpack/sql/analysis/analyzer/Verifier.java | 3 ++- .../org/elasticsearch/xpack/sql/execution/search/Querier.java | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java index 946fa8797ccba..fdcd87d4ef7a7 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java @@ -22,7 +22,6 @@ * the resulting ES document as a field. */ public class QlSourceBuilder { - public static final Version SWITCH_TO_FIELDS_API_VERSION = Version.V_7_10_0; public static final Version INTRODUCING_MISSING_ORDER_IN_COMPOSITE_AGGS_VERSION = Version.V_7_16_0; // The LinkedHashMaps preserve the order of the fields in the response private final Set fetchFields = new LinkedHashSet<>(); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java index 391ccb4a1ac92..5de41b3a43668 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java @@ -158,7 +158,7 @@ protected ScriptTemplate scriptWithField(FieldAttribute field) { return field.dataType() != UNSIGNED_LONG ? new ScriptTemplate(processScript(Scripts.DOC_VALUE), params, dataType()) : new ScriptTemplate( - processScript(formatTemplate(format("{ql}.", "nullSafeCastToUnsignedLong({})", Scripts.DOC_VALUE))), + processScript(format("{ql}.", "nullSafeCastToUnsignedLong({})", Scripts.DOC_VALUE)), params, UNSIGNED_LONG ); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index a1590f74bfdfb..720e17c54ecf5 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -1004,8 +1004,9 @@ private static void checkCastOnInexact(LogicalPlan p, Set localFailures } private static void checkClientSupportsDataTypes(LogicalPlan p, Set localFailures, SqlVersion version) { + Version ver = Version.fromId(version.id); p.output().forEach(e -> { - if (e.resolved() && isTypeSupportedInVersion(e.dataType(), Version.fromId(version.id)) == false) { + if (e.resolved() && isTypeSupportedInVersion(e.dataType(), ver) == false) { localFailures.add( fail( e, diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java index 715df7b722728..dbdb23b30d914 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java @@ -83,7 +83,7 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.action.ActionListener.wrap; -import static org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder.INTRODUCING_MISSING_ORDER_IN_COMPOSITE_AGGS_VERSION; +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; // TODO: add retry/back-off public class Querier { @@ -144,7 +144,7 @@ public void query(List output, QueryContainer query, String index, Ac public static SearchRequest prepareRequest(SearchSourceBuilder source, TimeValue timeout, boolean includeFrozen, String... indices) { source.timeout(timeout); - SearchRequest searchRequest = new SearchRequest(INTRODUCING_MISSING_ORDER_IN_COMPOSITE_AGGS_VERSION); + SearchRequest searchRequest = new SearchRequest(INTRODUCING_UNSIGNED_LONG); searchRequest.indices(indices); searchRequest.source(source); searchRequest.allowPartialSearchResults(false); From abda408749209982a0aad9073c58b1cd9f731cd5 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Mon, 31 Jan 2022 18:54:31 +0100 Subject: [PATCH 25/28] Update docs/changelog/65145.yaml --- docs/changelog/65145.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/65145.yaml diff --git a/docs/changelog/65145.yaml b/docs/changelog/65145.yaml new file mode 100644 index 0000000000000..b2b09cca54a6b --- /dev/null +++ b/docs/changelog/65145.yaml @@ -0,0 +1,6 @@ +pr: 65145 +summary: Add `unsigned_long` type support +area: Query Languages +type: enhancement +issues: + - 63312 From 5302100e39d29d53f5d054aa786038c479084433 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Wed, 2 Feb 2022 14:53:51 +0100 Subject: [PATCH 26/28] Extract UL to Unsupported map transf. to own class This extracts the unsigned_long replacement with UnsupportedEsField into own class, `IndexCompatibility`, in case backwards compatibility needs to apply. This leaves the responsability to ensure BWC to IndexResolver's callers. --- .../xpack/eql/session/EqlSession.java | 2 - .../xpack/ql/index/IndexCompatibility.java | 49 ++++++ .../xpack/ql/index/IndexResolver.java | 61 ++----- .../elasticsearch/xpack/ql/type/Types.java | 29 ++-- .../xpack/ql/type/TypesTests.java | 19 ++- .../xpack/sql/qa/rest/RestSqlTestCase.java | 2 +- .../xpack/sql/analysis/analyzer/Analyzer.java | 16 +- .../sql/plan/logical/command/ShowColumns.java | 25 ++- .../plan/logical/command/sys/SysColumns.java | 56 +++---- .../xpack/sql/session/SqlSession.java | 3 +- .../analyzer/FieldAttributeTests.java | 149 ++++++++++++------ .../analysis/index/IndexResolverTests.java | 7 +- .../logical/command/ShowColumnsTests.java | 6 +- .../logical/command/sys/SysColumnsTests.java | 22 ++- .../xpack/sql/types/SqlTypesTests.java | 7 +- 15 files changed, 251 insertions(+), 202 deletions(-) create mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexCompatibility.java diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java index c8e2bd2c50fed..9ec908c696a0c 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.eql.session; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.ParentTaskAssigningClient; @@ -121,7 +120,6 @@ private void preAnalyze(LogicalPlan parsed, ActionListener list indexWildcard, configuration.indicesOptions(), configuration.runtimeMappings(), - Version.CURRENT, map(listener, r -> preAnalyzer.preAnalyze(parsed, r)) ); } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexCompatibility.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexCompatibility.java new file mode 100644 index 0000000000000..2f99854679d8f --- /dev/null +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexCompatibility.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ql.index; + +import org.elasticsearch.Version; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.EsField; +import org.elasticsearch.xpack.ql.type.UnsupportedEsField; + +import java.util.Map; + +import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; +import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive; +import static org.elasticsearch.xpack.ql.type.Types.propagateUnsupportedType; + +public final class IndexCompatibility { + + public static Map compatible(Map mapping, Version version) { + for (Map.Entry entry : mapping.entrySet()) { + EsField esField = entry.getValue(); + DataType dataType = esField.getDataType(); + if (isPrimitive(dataType) == false) { + compatible(esField.getProperties(), version); + } else if (isTypeSupportedInVersion(dataType, version) == false) { + EsField field = new UnsupportedEsField(entry.getKey(), dataType.name(), null, esField.getProperties()); + entry.setValue(field); + propagateUnsupportedType(entry.getKey(), dataType.name(), esField.getProperties()); + } + } + return mapping; + } + + public static EsIndex compatible(EsIndex esIndex, Version version) { + compatible(esIndex.mapping(), version); + return esIndex; + } + + public static IndexResolution compatible(IndexResolution indexResolution, Version version) { + if (indexResolution.isValid()) { + compatible(indexResolution.get(), version); + } + return indexResolution; + } +} diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index 3a285daf7d6a5..ec88674f6b382 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -9,7 +9,6 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.ElasticsearchSecurityException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; @@ -69,12 +68,10 @@ import static org.elasticsearch.common.Strings.hasText; import static org.elasticsearch.common.regex.Regex.simpleMatch; import static org.elasticsearch.transport.RemoteClusterAware.buildRemoteIndexName; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.supportsUnsignedLong; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; -import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; import static org.elasticsearch.xpack.ql.util.StringUtils.qualifyAndJoinIndices; import static org.elasticsearch.xpack.ql.util.StringUtils.splitQualifiedIndex; @@ -369,16 +366,12 @@ public void resolveAsMergedMapping( String indexWildcard, IndicesOptions indicesOptions, Map runtimeMappings, - Version version, ActionListener listener ) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, indicesOptions, runtimeMappings); client.fieldCaps( fieldRequest, - ActionListener.wrap( - response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response, version)), - listener::onFailure - ) + ActionListener.wrap(response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response)), listener::onFailure) ); } @@ -389,24 +382,19 @@ public void resolveAsMergedMapping( String indexWildcard, boolean includeFrozen, Map runtimeMappings, - Version version, ActionListener listener ) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen, runtimeMappings); client.fieldCaps( fieldRequest, - ActionListener.wrap( - response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response, version)), - listener::onFailure - ) + ActionListener.wrap(response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response)), listener::onFailure) ); } public static IndexResolution mergedMappings( DataTypeRegistry typeRegistry, String indexPattern, - FieldCapabilitiesResponse fieldCapsResponse, - Version version + FieldCapabilitiesResponse fieldCapsResponse ) { if (fieldCapsResponse.getIndices().length == 0) { @@ -414,7 +402,7 @@ public static IndexResolution mergedMappings( } // merge all indices onto the same one - List indices = buildIndices(typeRegistry, null, fieldCapsResponse, null, i -> indexPattern, version, (n, types) -> { + List indices = buildIndices(typeRegistry, null, fieldCapsResponse, null, i -> indexPattern, (n, types) -> { StringBuilder errorMessage = new StringBuilder(); boolean hasUnmapped = types.containsKey(UNMAPPED); @@ -485,8 +473,7 @@ private static EsField createField( Map> globalCaps, Map hierarchicalMapping, Map flattedMapping, - Function field, - Version version + Function field ) { Map parentProps = hierarchicalMapping; @@ -506,7 +493,7 @@ private static EsField createField( // lack of parent implies the field is an alias if (map == null) { // as such, create the field manually, marking the field to also be an alias - fieldFunction = s -> createField(typeRegistry, s, OBJECT.esType(), new TreeMap<>(), false, true, version); + fieldFunction = s -> createField(typeRegistry, s, OBJECT.esType(), new TreeMap<>(), false, true); } else { Iterator iterator = map.values().iterator(); FieldCapabilities parentCap = iterator.next(); @@ -514,18 +501,10 @@ private static EsField createField( parentCap = iterator.next(); } final FieldCapabilities parentC = parentCap; - fieldFunction = s -> createField( - typeRegistry, - s, - parentC.getType(), - new TreeMap<>(), - parentC.isAggregatable(), - false, - version - ); + fieldFunction = s -> createField(typeRegistry, s, parentC.getType(), new TreeMap<>(), parentC.isAggregatable(), false); } - parent = createField(typeRegistry, parentName, globalCaps, hierarchicalMapping, flattedMapping, fieldFunction, version); + parent = createField(typeRegistry, parentName, globalCaps, hierarchicalMapping, flattedMapping, fieldFunction); } parentProps = parent.getProperties(); } @@ -558,8 +537,7 @@ private static EsField createField( String typeName, Map props, boolean isAggregateable, - boolean isAlias, - Version version + boolean isAlias ) { DataType esType = typeRegistry.fromEs(typeName); @@ -575,7 +553,7 @@ private static EsField createField( if (esType == DATETIME) { return DateEsField.dateEsField(fieldName, props, isAggregateable); } - if (esType == UNSUPPORTED || (esType == UNSIGNED_LONG && supportsUnsignedLong(version) == false)) { + if (esType == UNSUPPORTED) { return new UnsupportedEsField(fieldName, typeName, null, props); } @@ -613,7 +591,6 @@ public void resolveAsSeparateMappings( String javaRegex, boolean includeFrozen, Map runtimeMappings, - Version version, ActionListener> listener ) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen, runtimeMappings); @@ -623,12 +600,10 @@ public void resolveAsSeparateMappings( .getAliases( createGetAliasesRequest(response, includeFrozen), wrap( - aliases -> { - listener.onResponse(separateMappings(typeRegistry, javaRegex, response, aliases.getAliases(), version)); - }, + aliases -> { listener.onResponse(separateMappings(typeRegistry, javaRegex, response, aliases.getAliases())); }, ex -> { if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) { - listener.onResponse(separateMappings(typeRegistry, javaRegex, response, null, version)); + listener.onResponse(separateMappings(typeRegistry, javaRegex, response, null)); } else { listener.onFailure(ex); } @@ -650,10 +625,9 @@ public static List separateMappings( DataTypeRegistry typeRegistry, String javaRegex, FieldCapabilitiesResponse fieldCaps, - ImmutableOpenMap> aliases, - Version version + ImmutableOpenMap> aliases ) { - return buildIndices(typeRegistry, javaRegex, fieldCaps, aliases, Function.identity(), version, (s, cap) -> null); + return buildIndices(typeRegistry, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null); } private static class Fields { @@ -671,7 +645,6 @@ private static List buildIndices( FieldCapabilitiesResponse fieldCapsResponse, ImmutableOpenMap> aliases, Function indexNameProcessor, - Version version, BiFunction, InvalidMappedField> validityVerifier ) { @@ -807,10 +780,8 @@ private static List buildIndices( typeCap.getType(), emptyMap(), typeCap.isAggregatable(), - isAliasFieldType.get(), - version - ), - version + isAliasFieldType.get() + ) ); } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java index 818b4c9ad815d..c202e797b7566 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.ql.type; -import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.core.Booleans; @@ -16,34 +15,32 @@ import java.util.Map.Entry; import static java.util.Collections.emptyMap; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.supportsUnsignedLong; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.NESTED; import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; -import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; public abstract class Types { @SuppressWarnings("unchecked") - public static Map fromEs(DataTypeRegistry typeRegistry, Map asMap, Version version) { + public static Map fromEs(DataTypeRegistry typeRegistry, Map asMap) { Map props = null; if (asMap != null && asMap.isEmpty() == false) { props = (Map) asMap.get("properties"); } - return props == null || props.isEmpty() ? emptyMap() : startWalking(typeRegistry, props, version); + return props == null || props.isEmpty() ? emptyMap() : startWalking(typeRegistry, props); } - private static Map startWalking(DataTypeRegistry typeRegistry, Map mapping, Version version) { + private static Map startWalking(DataTypeRegistry typeRegistry, Map mapping) { Map types = new LinkedHashMap<>(); if (mapping == null) { return emptyMap(); } for (Entry entry : mapping.entrySet()) { - walkMapping(typeRegistry, entry.getKey(), entry.getValue(), types, version); + walkMapping(typeRegistry, entry.getKey(), entry.getValue(), types); } return types; @@ -68,13 +65,7 @@ private static DataType getType(DataTypeRegistry typeRegistry, Map mapping, - Version version - ) { + private static void walkMapping(DataTypeRegistry typeRegistry, String name, Object value, Map mapping) { // object type - only root or nested docs supported if (value instanceof Map) { Map content = (Map) value; @@ -83,17 +74,17 @@ private static void walkMapping( DataType esDataType = getType(typeRegistry, content); final Map properties; if (esDataType == OBJECT || esDataType == NESTED) { - properties = fromEs(typeRegistry, content, version); + properties = fromEs(typeRegistry, content); } else if (content.containsKey("fields")) { // Check for multifields Object fields = content.get("fields"); if (fields instanceof Map) { - properties = startWalking(typeRegistry, (Map) fields, version); + properties = startWalking(typeRegistry, (Map) fields); } else { properties = Collections.emptyMap(); } } else { - properties = fromEs(typeRegistry, content, version); + properties = fromEs(typeRegistry, content); } boolean docValues = boolSetting(content.get("doc_values"), esDataType.hasDocValues()); final EsField field; @@ -105,7 +96,7 @@ private static void walkMapping( field = new KeywordEsField(name, properties, docValues, length, normalized); } else if (esDataType == DATETIME) { field = DateEsField.dateEsField(name, properties, docValues); - } else if (esDataType == UNSUPPORTED || (esDataType == UNSIGNED_LONG && supportsUnsignedLong(version) == false)) { + } else if (esDataType == UNSUPPORTED) { String type = content.get("type").toString(); field = new UnsupportedEsField(name, type, null, properties); propagateUnsupportedType(name, type, properties); @@ -130,7 +121,7 @@ private static int intSetting(Object value, int defaultValue) { return value == null ? defaultValue : Integer.parseInt(value.toString()); } - private static void propagateUnsupportedType(String inherited, String originalType, Map properties) { + public static void propagateUnsupportedType(String inherited, String originalType, Map properties) { if (properties != null && properties.isEmpty() == false) { for (Entry entry : properties.entrySet()) { EsField field = entry.getValue(); diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java index fbf95e6c96acb..fa916cebd73fc 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.ql.type; -import org.elasticsearch.Version; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.json.JsonXContent; @@ -28,12 +27,12 @@ public class TypesTests extends ESTestCase { public void testNullMap() { - Map fromEs = Types.fromEs(DefaultDataTypeRegistry.INSTANCE, null, Version.CURRENT); + Map fromEs = Types.fromEs(DefaultDataTypeRegistry.INSTANCE, null); assertThat(fromEs.isEmpty(), is(true)); } public void testEmptyMap() { - Map fromEs = Types.fromEs(DefaultDataTypeRegistry.INSTANCE, emptyMap(), Version.CURRENT); + Map fromEs = Types.fromEs(DefaultDataTypeRegistry.INSTANCE, emptyMap()); assertThat(fromEs.isEmpty(), is(true)); } @@ -212,27 +211,27 @@ public void testUnsupportedTypes() { } public static Map loadMapping(String name) { - return loadMapping(DefaultDataTypeRegistry.INSTANCE, name, null, Version.CURRENT); + return loadMapping(DefaultDataTypeRegistry.INSTANCE, name, null); } public static Map loadMapping(String name, boolean ordered) { - return loadMapping(DefaultDataTypeRegistry.INSTANCE, name, ordered, Version.CURRENT); + return loadMapping(DefaultDataTypeRegistry.INSTANCE, name, ordered); } public static Map loadMapping(DataTypeRegistry registry, String name) { - return loadMapping(registry, name, null, Version.CURRENT); + return loadMapping(registry, name, null); } - public static Map loadMapping(DataTypeRegistry registry, String name, Boolean ordered, Version version) { + public static Map loadMapping(DataTypeRegistry registry, String name, Boolean ordered) { InputStream stream = TypesTests.class.getResourceAsStream("/" + name); assertNotNull("Could not find mapping resource:" + name, stream); - return loadMapping(registry, stream, ordered, version); + return loadMapping(registry, stream, ordered); } - private static Map loadMapping(DataTypeRegistry registry, InputStream stream, Boolean ordered, Version version) { + private static Map loadMapping(DataTypeRegistry registry, InputStream stream, Boolean ordered) { boolean order = ordered != null ? ordered.booleanValue() : randomBoolean(); try (InputStream in = stream) { - return Types.fromEs(registry, XContentHelper.convertToMap(JsonXContent.jsonXContent, in, order), version); + return Types.fromEs(registry, XContentHelper.convertToMap(JsonXContent.jsonXContent, in, order)); } catch (IOException ex) { throw new RuntimeException(ex); } diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java index 3a24c45995c39..c5121168116e2 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java @@ -1161,7 +1161,7 @@ public void testPreventedUnsignedLongMaskedAccess() throws IOException { String query = query("SELECT unsigned_long::STRING FROM " + indexPattern("test")).version(version.toString()).toString(); expectBadRequest( () -> runSql(new StringEntity(query, ContentType.APPLICATION_JSON), "", randomMode()), - containsString("Cannot use field [unsigned_long] with unsupported type [unsigned_long]") + containsString("Cannot use field [unsigned_long] with unsupported type [UNSIGNED_LONG]") ); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index afaed79c939b5..6ab1e210649d3 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -34,6 +34,8 @@ import org.elasticsearch.xpack.ql.expression.function.Functions; import org.elasticsearch.xpack.ql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.ArithmeticOperation; +import org.elasticsearch.xpack.ql.index.EsIndex; +import org.elasticsearch.xpack.ql.index.IndexCompatibility; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.plan.TableIdentifier; import org.elasticsearch.xpack.ql.plan.logical.Aggregate; @@ -78,7 +80,6 @@ import static java.util.stream.Collectors.toList; import static org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.AnalyzerRule; import static org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.BaseAnalyzerRule; -import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; public class Analyzer extends RuleExecutor { @@ -330,8 +331,9 @@ protected LogicalPlan rule(UnresolvedRelation plan) { "invalid [" + table + "] resolution to [" + indexResolution + "]" ); } - LogicalPlan logicalPlan = new EsRelation(plan.source(), indexResolution.get(), plan.frozen()); - SubQueryAlias sa = new SubQueryAlias(plan.source(), logicalPlan, indexResolution.get().toString()); + EsIndex esIndex = IndexCompatibility.compatible(indexResolution.get(), Version.fromId(configuration.version().id)); + LogicalPlan logicalPlan = new EsRelation(plan.source(), esIndex, plan.frozen()); + SubQueryAlias sa = new SubQueryAlias(plan.source(), logicalPlan, esIndex.toString()); if (plan.alias() != null) { sa = new SubQueryAlias(plan.source(), sa, plan.alias()); @@ -447,13 +449,7 @@ private List expandProjections(List } } - return filterUnsupportedProjections(result); - } - - private List filterUnsupportedProjections(List projections) { - return projections.stream() - .filter(p -> p.resolved() == false || isTypeSupportedInVersion(p.dataType(), Version.fromId(configuration.version().id))) - .collect(toList()); + return result; } private List expandStar(UnresolvedStar us, List output) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java index e6920af0b46c8..acee318c27077 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern; +import org.elasticsearch.xpack.ql.index.IndexCompatibility; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; @@ -79,21 +80,15 @@ public void execute(SqlSession session, ActionListener listener) { idx = hasText(cat) && cat.equals(cluster) == false ? buildRemoteIndexName(cat, idx) : idx; boolean withFrozen = includeFrozen || session.configuration().includeFrozen(); - session.indexResolver() - .resolveAsMergedMapping( - idx, - withFrozen, - emptyMap(), - Version.fromId(session.configuration().version().id), - ActionListener.wrap(indexResult -> { - List> rows = emptyList(); - if (indexResult.isValid()) { - rows = new ArrayList<>(); - fillInRows(indexResult.get().mapping(), null, rows); - } - listener.onResponse(of(session, rows)); - }, listener::onFailure) - ); + session.indexResolver().resolveAsMergedMapping(idx, withFrozen, emptyMap(), ActionListener.wrap(indexResult -> { + List> rows = emptyList(); + if (indexResult.isValid()) { + rows = new ArrayList<>(); + Version version = Version.fromId(session.configuration().version().id); + fillInRows(IndexCompatibility.compatible(indexResult, version).get().mapping(), null, rows); + } + listener.onResponse(of(session, rows)); + }, listener::onFailure)); } static void fillInRows(Map mapping, String prefix, List> rows) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java index 828b38480d01c..e18f03d503767 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern; import org.elasticsearch.xpack.ql.index.EsIndex; +import org.elasticsearch.xpack.ql.index.IndexCompatibility; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; @@ -154,42 +155,37 @@ public void execute(SqlSession session, ActionListener listener) { tableCat = cluster; } + Version version = Version.fromId(session.configuration().version().id); // special case for '%' (translated to *) if ("*".equals(idx)) { session.indexResolver() - .resolveAsSeparateMappings( - indexPattern, - regex, - includeFrozen, - emptyMap(), - Version.fromId(session.configuration().version().id), - ActionListener.wrap(esIndices -> { - List> rows = new ArrayList<>(); - for (EsIndex esIndex : esIndices) { - fillInRows(tableCat, esIndex.name(), esIndex.mapping(), null, rows, columnMatcher, mode); - } - listener.onResponse(ListCursor.of(Rows.schema(output), rows, session.configuration().pageSize())); - }, listener::onFailure) - ); + .resolveAsSeparateMappings(indexPattern, regex, includeFrozen, emptyMap(), ActionListener.wrap(esIndices -> { + List> rows = new ArrayList<>(); + for (EsIndex esIndex : esIndices) { + IndexCompatibility.compatible(esIndex, version); + fillInRows(tableCat, esIndex.name(), esIndex.mapping(), null, rows, columnMatcher, mode); + } + listener.onResponse(ListCursor.of(Rows.schema(output), rows, session.configuration().pageSize())); + }, listener::onFailure)); } // otherwise use a merged mapping else { - session.indexResolver() - .resolveAsMergedMapping( - indexPattern, - includeFrozen, - emptyMap(), - Version.fromId(session.configuration().version().id), - ActionListener.wrap(r -> { - List> rows = new ArrayList<>(); - // populate the data only when a target is found - if (r.isValid()) { - EsIndex esIndex = r.get(); - fillInRows(tableCat, indexName, esIndex.mapping(), null, rows, columnMatcher, mode); - } - listener.onResponse(ListCursor.of(Rows.schema(output), rows, session.configuration().pageSize())); - }, listener::onFailure) - ); + session.indexResolver().resolveAsMergedMapping(indexPattern, includeFrozen, emptyMap(), ActionListener.wrap(r -> { + List> rows = new ArrayList<>(); + // populate the data only when a target is found + if (r.isValid()) { + fillInRows( + tableCat, + indexName, + IndexCompatibility.compatible(r, version).get().mapping(), + null, + rows, + columnMatcher, + mode + ); + } + listener.onResponse(ListCursor.of(Rows.schema(output), rows, session.configuration().pageSize())); + }, listener::onFailure)); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java index 358e309e5febe..5916b3082e7d5 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.sql.session; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.ParentTaskAssigningClient; @@ -159,7 +158,7 @@ private void preAnalyze(LogicalPlan parsed, Function act indexPattern, includeFrozen, configuration.runtimeMappings(), - Version.fromId(configuration.version().id), + // Version.fromId(configuration.version().id), wrap(indexResult -> listener.onResponse(action.apply(indexResult)), listener::onFailure) ); } else { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java index 4413f4d79ed32..7ddaaad803e83 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java @@ -7,7 +7,9 @@ package org.elasticsearch.xpack.sql.analysis.analyzer; import org.elasticsearch.Version; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.expression.Alias; import org.elasticsearch.xpack.ql.expression.Attribute; @@ -17,12 +19,15 @@ import org.elasticsearch.xpack.ql.expression.NamedExpression; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; import org.elasticsearch.xpack.ql.index.EsIndex; +import org.elasticsearch.xpack.ql.index.IndexCompatibility; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.plan.logical.Aggregate; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.plan.logical.Project; import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DefaultDataTypeRegistry; import org.elasticsearch.xpack.ql.type.EsField; +import org.elasticsearch.xpack.ql.type.Types; import org.elasticsearch.xpack.ql.type.TypesTests; import org.elasticsearch.xpack.sql.SqlTestUtils; import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry; @@ -30,6 +35,7 @@ import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.stats.Metrics; +import org.elasticsearch.xpack.sql.types.SqlTypesTests; import java.util.Collections; import java.util.List; @@ -39,13 +45,12 @@ import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.INTRODUCING_UNSIGNED_LONG; import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; -import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; -import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.hasSize; @@ -65,7 +70,7 @@ public FieldAttributeTests() { functionRegistry = new SqlFunctionRegistry(); verifier = new Verifier(new Metrics()); - Map mapping = loadMapping("mapping-multi-field-variation.json"); + Map mapping = SqlTypesTests.loadMapping("mapping-multi-field-variation.json"); EsIndex test = new EsIndex("test", mapping); getIndexResult = IndexResolution.valid(test); @@ -304,10 +309,6 @@ public void testGroupByAmbiguity() { } public void testUnsignedLongVersionCompatibility() { - Map mapping = TypesTests.loadMapping("mapping-numeric.json"); - EsIndex index = new EsIndex("test", mapping); - getIndexResult = IndexResolution.valid(index); - String query = "SELECT unsigned_long FROM test"; String queryWithLiteral = "SELECT 18446744073709551615 AS unsigned_long"; String queryWithCastLiteral = "SELECT '18446744073709551615'::unsigned_long AS unsigned_long"; @@ -315,24 +316,27 @@ public void testUnsignedLongVersionCompatibility() { String queryWithArithmetic = "SELECT unsigned_long + 1 AS unsigned_long FROM test"; String queryWithCast = "SELECT long + 1::unsigned_long AS unsigned_long FROM test"; - SqlVersion preUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); - SqlVersion postUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER); + Version preUnsignedLong = Version.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); + Version postUnsignedLong = Version.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER); + SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(SqlVersion.fromId(preUnsignedLong.id)); for (String sql : List.of(query, queryWithLiteral, queryWithCastLiteral, queryWithAlias, queryWithArithmetic, queryWithCast)) { - SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preUnsignedLong); - analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics())); - VerificationException ex = expectThrows(VerificationException.class, () -> plan(sql)); - assertEquals( - "Found 1 problem\nline 1:8: Cannot use field [unsigned_long] with type [UNSIGNED_LONG] unsupported in version [" - + preUnsignedLong - + "], upgrade required (to version [" - + INTRODUCING_UNSIGNED_LONG - + "] or higher)", - ex.getMessage() + analyzer = new Analyzer( + sqlConfig, + functionRegistry, + loadCompatibleMapping("mapping-numeric.json", preUnsignedLong), + new Verifier(new Metrics()) ); - - for (SqlVersion v : List.of(SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id), postUnsignedLong)) { - analyzer = new Analyzer(SqlTestUtils.randomConfiguration(v), functionRegistry, getIndexResult, verifier); + VerificationException ex = expectThrows(VerificationException.class, () -> plan(sql)); + assertThat(ex.getMessage(), containsString("Found 1 problem\nline 1:8: Cannot use field [unsigned_long]")); + + for (Version v : List.of(INTRODUCING_UNSIGNED_LONG, postUnsignedLong)) { + analyzer = new Analyzer( + SqlTestUtils.randomConfiguration(SqlVersion.fromId(v.id)), + functionRegistry, + loadCompatibleMapping("mapping-numeric.json", v), + verifier + ); LogicalPlan plan = plan(sql); assertThat(plan, instanceOf(Project.class)); Project p = (Project) plan; @@ -346,59 +350,91 @@ public void testUnsignedLongVersionCompatibility() { } public void testNonProjectedUnsignedLongVersionCompatibility() { - Map mapping = TypesTests.loadMapping("mapping-numeric.json"); - EsIndex index = new EsIndex("test", mapping); - getIndexResult = IndexResolution.valid(index); - SqlVersion preUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); - SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preUnsignedLong); - analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics())); + Version preUnsignedLong = Version.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); + SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(SqlVersion.fromId(preUnsignedLong.id)); + analyzer = new Analyzer( + sqlConfig, + functionRegistry, + loadCompatibleMapping("mapping-numeric.json", preUnsignedLong), + new Verifier(new Metrics()) + ); String query = "SELECT unsigned_long = 1, unsigned_long::double FROM test"; String queryWithSubquery = "SELECT l = 1, SQRT(ul) FROM " + "(SELECT unsigned_long AS ul, long AS l FROM test WHERE ul > 10) WHERE l < 100 "; for (String sql : List.of(query, queryWithSubquery)) { + VerificationException ex = expectThrows(VerificationException.class, () -> plan(sql)); + assertThat(ex.getMessage(), containsString("Cannot use field [unsigned_long] with unsupported type [UNSIGNED_LONG]")); + } + } + + public void testNestedUnsignedLongVersionCompatibility() { + String props = """ + { + "properties": { + "container": { + "properties": { + "ul": { + "type": "unsigned_long" + } + } + } + } + } + """; + String sql = "SELECT container.ul as unsigned_long FROM test"; + + Version preUnsignedLong = Version.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); + analyzer = new Analyzer( + SqlTestUtils.randomConfiguration(SqlVersion.fromId(preUnsignedLong.id)), + functionRegistry, + compatibleMapping(props, preUnsignedLong), + new Verifier(new Metrics()) + ); + VerificationException ex = expectThrows(VerificationException.class, () -> plan(sql)); + assertThat(ex.getMessage(), containsString("Cannot use field [container.ul] with unsupported type [UNSIGNED_LONG]")); + + Version postUnsignedLong = Version.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER); + for (Version v : List.of(INTRODUCING_UNSIGNED_LONG, postUnsignedLong)) { + analyzer = new Analyzer( + SqlTestUtils.randomConfiguration(SqlVersion.fromId(v.id)), + functionRegistry, + compatibleMapping(props, v), + verifier + ); LogicalPlan plan = plan(sql); assertThat(plan, instanceOf(Project.class)); Project p = (Project) plan; List projections = p.projections(); - assertThat(projections, hasSize(2)); - assertEquals(projections.get(0).dataType(), BOOLEAN); - assertEquals(projections.get(1).dataType(), DOUBLE); + assertThat(projections, hasSize(1)); + Attribute attribute = projections.get(0).toAttribute(); + assertThat(attribute.dataType(), is(UNSIGNED_LONG)); + assertThat(attribute.name(), is("unsigned_long")); } } public void testUnsignedLongStarExpandedVersionControlled() { - Map mapping = TypesTests.loadMapping("mapping-numeric.json"); - EsIndex index = new EsIndex("test", mapping); - getIndexResult = IndexResolution.valid(index); - SqlVersion preUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER); SqlVersion postUnsignedLong = SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER); String query = "SELECT * FROM test"; for (SqlVersion version : List.of(preUnsignedLong, SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id), postUnsignedLong)) { SqlConfiguration config = SqlTestUtils.randomConfiguration(version); - analyzer = new Analyzer(config, functionRegistry, getIndexResult, new Verifier(new Metrics())); + analyzer = new Analyzer(config, functionRegistry, loadMapping("mapping-numeric.json"), new Verifier(new Metrics())); LogicalPlan plan = plan(query); assertThat(plan, instanceOf(Project.class)); Project p = (Project) plan; - List projectedDataTypes = p.projections().stream().map(Expression::dataType).collect(Collectors.toList()); - assertEquals( - isTypeSupportedInVersion(UNSIGNED_LONG, Version.fromId(config.version().id)), - projectedDataTypes.contains(UNSIGNED_LONG) - ); + List projectedDataTypes = p.projections().stream().map(Expression::dataType).toList(); + assertEquals(isTypeSupportedInVersion(UNSIGNED_LONG, Version.fromId(version.id)), projectedDataTypes.contains(UNSIGNED_LONG)); } } public void testFunctionOverNonExistingFieldAsArgumentAndSameAlias() throws Exception { - Map mapping = TypesTests.loadMapping("mapping-basic.json"); - EsIndex index = new EsIndex("test", mapping); - getIndexResult = IndexResolution.valid(index); - analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, verifier); + analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, loadMapping("mapping-basic.json"), verifier); VerificationException ex = expectThrows( VerificationException.class, @@ -408,10 +444,7 @@ public void testFunctionOverNonExistingFieldAsArgumentAndSameAlias() throws Exce } public void testFunctionWithExpressionOverNonExistingFieldAsArgumentAndSameAlias() throws Exception { - Map mapping = TypesTests.loadMapping("mapping-basic.json"); - EsIndex index = new EsIndex("test", mapping); - getIndexResult = IndexResolution.valid(index); - analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, verifier); + analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, loadMapping("mapping-basic.json"), verifier); VerificationException ex = expectThrows( VerificationException.class, @@ -431,4 +464,22 @@ public void testExpandStarOnIndexWithoutColumns() { assertTrue(((Project) plan).projections().isEmpty()); } + private static IndexResolution loadMapping(String mappingName) { + Map mapping = TypesTests.loadMapping(mappingName); + EsIndex index = new EsIndex("test", mapping); + return IndexResolution.valid(index); + } + + private static IndexResolution loadCompatibleMapping(String mappingName, Version version) { + return IndexCompatibility.compatible(loadMapping(mappingName), version); + } + + private static IndexResolution compatibleMapping(String properties, Version version) { + Map mapping = Types.fromEs( + DefaultDataTypeRegistry.INSTANCE, + XContentHelper.convertToMap(JsonXContent.jsonXContent, properties, randomBoolean()) + ); + EsIndex index = new EsIndex("test", mapping); + return IndexCompatibility.compatible(IndexResolution.valid(index), version); + } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index b191e09e456f3..f0a62f7e914d9 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.sql.analysis.index; -import org.elasticsearch.Version; import org.elasticsearch.action.fieldcaps.FieldCapabilities; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.common.util.Maps; @@ -497,8 +496,7 @@ private static IndexResolution mergedMappings( return IndexResolver.mergedMappings( SqlDataTypeRegistry.INSTANCE, indexPattern, - new FieldCapabilitiesResponse(indexNames, fieldCaps), - Version.CURRENT + new FieldCapabilitiesResponse(indexNames, fieldCaps) ); } @@ -511,8 +509,7 @@ private static List separateMappings( SqlDataTypeRegistry.INSTANCE, javaRegex, new FieldCapabilitiesResponse(indexNames, fieldCaps), - null, - Version.CURRENT + null ); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java index 3e6c446c467a1..2efd00f0e111d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumnsTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.Version; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ql.index.IndexCompatibility; import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.sql.proto.SqlVersion; @@ -87,8 +88,9 @@ public void testUnsignedLongFiltering() { List rowUnsupported = List.of("unsigned_long", "OTHER", "unsupported"); for (SqlVersion version : UNSIGNED_LONG_TEST_VERSIONS) { List> rows = new ArrayList<>(); - Map mapping = loadMapping("mapping-multi-field-variation.json", true, Version.fromId(version.id)); - ShowColumns.fillInRows(mapping, null, rows); + // mapping's mutated by IndexCompatibility.compatible, needs to stay in the loop + Map mapping = loadMapping("mapping-multi-field-variation.json", true); + ShowColumns.fillInRows(IndexCompatibility.compatible(mapping, Version.fromId(version.id)), null, rows); assertTrue((supportsUnsignedLong(Version.fromId(version.id)) && rows.contains(rowSupported)) || rows.contains(rowUnsupported)); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index 26af677644582..e96263af9f168 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; import org.elasticsearch.xpack.ql.index.EsIndex; +import org.elasticsearch.xpack.ql.index.IndexCompatibility; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.index.IndexResolver; import org.elasticsearch.xpack.ql.type.EsField; @@ -149,8 +150,17 @@ public void testUnsignedLongFiltering() { for (Mode mode : List.of(Mode.JDBC, Mode.ODBC)) { for (SqlVersion version : UNSIGNED_LONG_TEST_VERSIONS) { List> rows = new ArrayList<>(); - Map mapping = loadMapping("mapping-multi-field-variation.json", true, Version.fromId(version.id)); - SysColumns.fillInRows("test", "index", mapping, null, rows, null, mode); + // mapping's mutated by IndexCompatibility.compatible, needs to stay in the loop + Map mapping = loadMapping("mapping-multi-field-variation.json", true); + SysColumns.fillInRows( + "test", + "index", + IndexCompatibility.compatible(mapping, Version.fromId(version.id)), + null, + rows, + null, + mode + ); List types = rows.stream().map(row -> name(row).toString()).collect(Collectors.toList()); assertEquals( isTypeSupportedInVersion(UNSIGNED_LONG, Version.fromId(version.id)), @@ -354,13 +364,13 @@ private Tuple sql( when(resolver.clusterName()).thenReturn(CLUSTER_NAME); when(resolver.remoteClusters()).thenReturn(Set.of(CLUSTER_NAME)); doAnswer(invocation -> { - ((ActionListener) invocation.getArguments()[4]).onResponse(IndexResolution.valid(test)); + ((ActionListener) invocation.getArguments()[3]).onResponse(IndexResolution.valid(test)); return Void.TYPE; - }).when(resolver).resolveAsMergedMapping(any(), anyBoolean(), any(), any(), any()); + }).when(resolver).resolveAsMergedMapping(any(), anyBoolean(), any(), any()); doAnswer(invocation -> { - ((ActionListener>) invocation.getArguments()[5]).onResponse(singletonList(test)); + ((ActionListener>) invocation.getArguments()[4]).onResponse(singletonList(test)); return Void.TYPE; - }).when(resolver).resolveAsSeparateMappings(any(), any(), anyBoolean(), any(), any(), any()); + }).when(resolver).resolveAsSeparateMappings(any(), any(), anyBoolean(), any(), any()); SqlSession session = new SqlSession(config, null, null, resolver, null, null, null, null, null); return new Tuple<>(cmd, session); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/types/SqlTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/types/SqlTypesTests.java index c9cc33a6af67e..8e05925c54a47 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/types/SqlTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/types/SqlTypesTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.sql.types; -import org.elasticsearch.Version; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.ql.type.KeywordEsField; @@ -89,10 +88,6 @@ public static Map loadMapping(String name) { } public static Map loadMapping(String name, boolean ordered) { - return loadMapping(name, ordered, Version.CURRENT); - } - - public static Map loadMapping(String name, boolean ordered, Version version) { - return TypesTests.loadMapping(SqlDataTypeRegistry.INSTANCE, name, ordered, version); + return TypesTests.loadMapping(SqlDataTypeRegistry.INSTANCE, name, ordered); } } From fba1d653ec939392ed4269d6ece63a688280ea60 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Wed, 2 Feb 2022 17:05:38 +0100 Subject: [PATCH 27/28] Move compat filtering out of the Analyzer Move the compatibility filtering out of the Analyzer, which now receives the IndexResolution already filtered; this now happens in the SqlSession. --- .../xpack/sql/analysis/analyzer/Analyzer.java | 8 ++------ .../elasticsearch/xpack/sql/session/SqlSession.java | 12 +++++++++--- .../sql/analysis/analyzer/FieldAttributeTests.java | 8 +++++++- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index 6ab1e210649d3..14a25fc2afb49 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.sql.analysis.analyzer; -import org.elasticsearch.Version; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.core.Tuple; import org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.AddMissingEqualsToBoolField; @@ -34,8 +33,6 @@ import org.elasticsearch.xpack.ql.expression.function.Functions; import org.elasticsearch.xpack.ql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.ArithmeticOperation; -import org.elasticsearch.xpack.ql.index.EsIndex; -import org.elasticsearch.xpack.ql.index.IndexCompatibility; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.plan.TableIdentifier; import org.elasticsearch.xpack.ql.plan.logical.Aggregate; @@ -331,9 +328,8 @@ protected LogicalPlan rule(UnresolvedRelation plan) { "invalid [" + table + "] resolution to [" + indexResolution + "]" ); } - EsIndex esIndex = IndexCompatibility.compatible(indexResolution.get(), Version.fromId(configuration.version().id)); - LogicalPlan logicalPlan = new EsRelation(plan.source(), esIndex, plan.frozen()); - SubQueryAlias sa = new SubQueryAlias(plan.source(), logicalPlan, esIndex.toString()); + LogicalPlan logicalPlan = new EsRelation(plan.source(), indexResolution.get(), plan.frozen()); + SubQueryAlias sa = new SubQueryAlias(plan.source(), logicalPlan, indexResolution.get().toString()); if (plan.alias() != null) { sa = new SubQueryAlias(plan.source(), sa, plan.alias()); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java index 5916b3082e7d5..85b411344989d 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java @@ -6,11 +6,13 @@ */ package org.elasticsearch.xpack.sql.session; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; +import org.elasticsearch.xpack.ql.index.IndexCompatibility; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.index.IndexResolver; import org.elasticsearch.xpack.ql.index.MappingException; @@ -113,8 +115,13 @@ public void analyzedPlan(LogicalPlan parsed, boolean verify, ActionListener { - Analyzer analyzer = new Analyzer(configuration, functionRegistry, c, verifier); + preAnalyze(parsed, r -> { + Analyzer analyzer = new Analyzer( + configuration, + functionRegistry, + IndexCompatibility.compatible(r, Version.fromId(configuration.version().id)), + verifier + ); return analyzer.analyze(parsed, verify); }, listener); } @@ -158,7 +165,6 @@ private void preAnalyze(LogicalPlan parsed, Function act indexPattern, includeFrozen, configuration.runtimeMappings(), - // Version.fromId(configuration.version().id), wrap(indexResult -> listener.onResponse(action.apply(indexResult)), listener::onFailure) ); } else { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java index 7ddaaad803e83..03e2ed8db85f8 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java @@ -421,7 +421,13 @@ public void testUnsignedLongStarExpandedVersionControlled() { for (SqlVersion version : List.of(preUnsignedLong, SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id), postUnsignedLong)) { SqlConfiguration config = SqlTestUtils.randomConfiguration(version); - analyzer = new Analyzer(config, functionRegistry, loadMapping("mapping-numeric.json"), new Verifier(new Metrics())); + // the mapping is mutated when making it "compatible", so it needs to be reloaded inside the loop. + analyzer = new Analyzer( + config, + functionRegistry, + loadCompatibleMapping("mapping-numeric.json", Version.fromId(version.id)), + new Verifier(new Metrics()) + ); LogicalPlan plan = plan(query); assertThat(plan, instanceOf(Project.class)); From 6ce3a1e47972dffeb564a1ce401d63a0fc7ea707 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Thu, 3 Feb 2022 17:22:39 +0100 Subject: [PATCH 28/28] Apply review comments. Bump UL intro version Rename loadMapping to loadIndexResolution. Bump INTRODUCING_UNSIGNED_LONG version. --- .../ql/index/VersionCompatibilityChecks.java | 4 +-- .../xpack/sql/qa/jdbc/JdbcTestUtils.java | 4 +-- .../analyzer/FieldAttributeTests.java | 28 +++++++++---------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java index e66f010ea89c3..f239e6bc8b08c 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java @@ -11,12 +11,12 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.xpack.ql.type.DataType; -import static org.elasticsearch.Version.V_8_1_0; +import static org.elasticsearch.Version.V_8_2_0; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; public final class VersionCompatibilityChecks { - public static final Version INTRODUCING_UNSIGNED_LONG = V_8_1_0; + public static final Version INTRODUCING_UNSIGNED_LONG = V_8_2_0; private VersionCompatibilityChecks() {} diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java index 6ca1cb9acd55c..120081a7edf7f 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java @@ -26,7 +26,7 @@ import java.util.LinkedHashMap; import java.util.Map; -import static org.elasticsearch.Version.V_8_1_0; +import static org.elasticsearch.Version.V_8_2_0; import static org.elasticsearch.common.time.DateUtils.toMilliSeconds; import static org.elasticsearch.test.ESTestCase.randomLongBetween; @@ -153,6 +153,6 @@ static boolean versionSupportsDateNanos() { } public static boolean isUnsignedLongSupported() { - return JDBC_DRIVER_VERSION.onOrAfter(V_8_1_0); + return JDBC_DRIVER_VERSION.onOrAfter(V_8_2_0); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java index 03e2ed8db85f8..135b20c2589e1 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java @@ -35,7 +35,6 @@ import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.stats.Metrics; -import org.elasticsearch.xpack.sql.types.SqlTypesTests; import java.util.Collections; import java.util.List; @@ -48,6 +47,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; @@ -70,7 +70,7 @@ public FieldAttributeTests() { functionRegistry = new SqlFunctionRegistry(); verifier = new Verifier(new Metrics()); - Map mapping = SqlTypesTests.loadMapping("mapping-multi-field-variation.json"); + Map mapping = loadMapping("mapping-multi-field-variation.json"); EsIndex test = new EsIndex("test", mapping); getIndexResult = IndexResolution.valid(test); @@ -324,7 +324,7 @@ public void testUnsignedLongVersionCompatibility() { analyzer = new Analyzer( sqlConfig, functionRegistry, - loadCompatibleMapping("mapping-numeric.json", preUnsignedLong), + loadCompatibleIndexResolution("mapping-numeric.json", preUnsignedLong), new Verifier(new Metrics()) ); VerificationException ex = expectThrows(VerificationException.class, () -> plan(sql)); @@ -334,7 +334,7 @@ public void testUnsignedLongVersionCompatibility() { analyzer = new Analyzer( SqlTestUtils.randomConfiguration(SqlVersion.fromId(v.id)), functionRegistry, - loadCompatibleMapping("mapping-numeric.json", v), + loadCompatibleIndexResolution("mapping-numeric.json", v), verifier ); LogicalPlan plan = plan(sql); @@ -355,7 +355,7 @@ public void testNonProjectedUnsignedLongVersionCompatibility() { analyzer = new Analyzer( sqlConfig, functionRegistry, - loadCompatibleMapping("mapping-numeric.json", preUnsignedLong), + loadCompatibleIndexResolution("mapping-numeric.json", preUnsignedLong), new Verifier(new Metrics()) ); @@ -389,7 +389,7 @@ public void testNestedUnsignedLongVersionCompatibility() { analyzer = new Analyzer( SqlTestUtils.randomConfiguration(SqlVersion.fromId(preUnsignedLong.id)), functionRegistry, - compatibleMapping(props, preUnsignedLong), + compatibleIndexResolution(props, preUnsignedLong), new Verifier(new Metrics()) ); VerificationException ex = expectThrows(VerificationException.class, () -> plan(sql)); @@ -400,7 +400,7 @@ public void testNestedUnsignedLongVersionCompatibility() { analyzer = new Analyzer( SqlTestUtils.randomConfiguration(SqlVersion.fromId(v.id)), functionRegistry, - compatibleMapping(props, v), + compatibleIndexResolution(props, v), verifier ); LogicalPlan plan = plan(sql); @@ -425,7 +425,7 @@ public void testUnsignedLongStarExpandedVersionControlled() { analyzer = new Analyzer( config, functionRegistry, - loadCompatibleMapping("mapping-numeric.json", Version.fromId(version.id)), + loadCompatibleIndexResolution("mapping-numeric.json", Version.fromId(version.id)), new Verifier(new Metrics()) ); @@ -440,7 +440,7 @@ public void testUnsignedLongStarExpandedVersionControlled() { } public void testFunctionOverNonExistingFieldAsArgumentAndSameAlias() throws Exception { - analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, loadMapping("mapping-basic.json"), verifier); + analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, loadIndexResolution("mapping-basic.json"), verifier); VerificationException ex = expectThrows( VerificationException.class, @@ -450,7 +450,7 @@ public void testFunctionOverNonExistingFieldAsArgumentAndSameAlias() throws Exce } public void testFunctionWithExpressionOverNonExistingFieldAsArgumentAndSameAlias() throws Exception { - analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, loadMapping("mapping-basic.json"), verifier); + analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, loadIndexResolution("mapping-basic.json"), verifier); VerificationException ex = expectThrows( VerificationException.class, @@ -470,17 +470,17 @@ public void testExpandStarOnIndexWithoutColumns() { assertTrue(((Project) plan).projections().isEmpty()); } - private static IndexResolution loadMapping(String mappingName) { + private static IndexResolution loadIndexResolution(String mappingName) { Map mapping = TypesTests.loadMapping(mappingName); EsIndex index = new EsIndex("test", mapping); return IndexResolution.valid(index); } - private static IndexResolution loadCompatibleMapping(String mappingName, Version version) { - return IndexCompatibility.compatible(loadMapping(mappingName), version); + private static IndexResolution loadCompatibleIndexResolution(String mappingName, Version version) { + return IndexCompatibility.compatible(loadIndexResolution(mappingName), version); } - private static IndexResolution compatibleMapping(String properties, Version version) { + private static IndexResolution compatibleIndexResolution(String properties, Version version) { Map mapping = Types.fromEs( DefaultDataTypeRegistry.INSTANCE, XContentHelper.convertToMap(JsonXContent.jsonXContent, properties, randomBoolean())