QL: add unsigned_long type support (#65145)

This introduces the UNSIGNED_LONG type to QL following its availability in ES (#60050).

The type is mapped to a BigInteger whose value is checked against the UL bounds.

The SQL will now support the type as literal and in the arithmetic functions; the non-arithmetic functions however are unchanged (i.e. they still require a long / int parameter where that is the case).

The type is version-gated: for the driver SQL clients (only) the server checks their version and in case this is lower than the one introducing the UL support, it fails the request, for queries, or simply hidden in catalog functions (similar to how UNSUPPORTED is currently treated in the similar case)

The JDBC tests are adjusted to read the (bwc) version of the driver they are run against and selectively disable part of the tests accordingly.

Closes #63312

docs/changelog/65145.yaml

@@ -0,0 +1,6 @@
+pr: 65145
+summary: Add `unsigned_long` type support
+area: Query Languages
+type: enhancement
+issues:
+ - 63312

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java

@@ -49,7 +49,6 @@
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.ql.tree.Source;
-import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypes;
 import org.elasticsearch.xpack.ql.util.StringUtils;
 
@@ -240,10 +239,9 @@ public Literal visitIntegerLiteral(EqlBaseParser.IntegerLiteralContext ctx) {
         Source source = source(ctx);
         String text = ctx.getText();
 
-        long value;
-
         try {
-            value = Long.valueOf(StringUtils.parseLong(text));
+            Number value = StringUtils.parseIntegral(text);
+            return new Literal(source, value, DataTypes.fromJava(value));
         } catch (QlIllegalArgumentException siae) {
             // if it's too large, then quietly try to parse as a float instead
             try {
@@ -252,16 +250,6 @@ public Literal visitIntegerLiteral(EqlBaseParser.IntegerLiteralContext ctx) {
 
             throw new ParsingException(source, siae.getMessage());
         }
-
-        Object val = Long.valueOf(value);
-        DataType type = DataTypes.LONG;
-
-        // try to downsize to int if possible (since that's the most common type)
-        if ((int) value == value) {
-            type = DataTypes.INTEGER;
-            val = Integer.valueOf((int) value);
-        }
-        return new Literal(source, val, type);
     }
 
     @Override

x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt

@@ -19,6 +19,7 @@ class org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQl
   double nullSafeSortNumeric(Number)
   String nullSafeSortString(Object)
   Number nullSafeCastNumeric(Number, String)
+  Number nullSafeCastToUnsignedLong(Number)
 
 #
 # ASCII Functions

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java

@@ -237,6 +237,7 @@ public void testNumeric() {
         accept(idxr, "foo where float_field == 0");
         accept(idxr, "foo where half_float_field == 0");
         accept(idxr, "foo where scaled_float_field == 0");
+        accept(idxr, "foo where unsigned_long_field == 0");
 
         // Test query against unsupported field type int
         assertEquals(

x-pack/plugin/eql/src/test/resources/mapping-numeric.json

@@ -34,6 +34,9 @@
         "scaled_float_field": {
             "type" : "scaled_float"
         },
+        "unsigned_long_field" : {
+          "type": "unsigned_long"
+        },
         "wrong_int_type_field": {
             "type" : "int"
         }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java

@@ -22,7 +22,6 @@
  * the resulting ES document as a field.
  */
 public class QlSourceBuilder {
-    public static final Version SWITCH_TO_FIELDS_API_VERSION = Version.V_7_10_0;
     public static final Version INTRODUCING_MISSING_ORDER_IN_COMPOSITE_AGGS_VERSION = Version.V_7_16_0;
     // The LinkedHashMaps preserve the order of the fields in the response
     private final Set<FieldAndFormat> fetchFields = new LinkedHashSet<>();

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.xpack.ql.expression.function.Function;
 import org.elasticsearch.xpack.ql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.ql.expression.function.grouping.GroupingFunction;
+import org.elasticsearch.xpack.ql.expression.gen.script.Params;
 import org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder;
 import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
 import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
@@ -24,10 +25,12 @@
 import java.util.List;
 
 import static java.util.Collections.emptyList;
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
 import static org.elasticsearch.xpack.ql.expression.gen.script.Scripts.PARAM;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.LONG;
+import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG;
 
 /**
  * A {@code ScalarFunction} is a {@code Function} that takes values from some
@@ -150,7 +153,15 @@ protected ScriptTemplate scriptWithGrouping(GroupingFunction grouping) {
     }
 
     protected ScriptTemplate scriptWithField(FieldAttribute field) {
-        return new ScriptTemplate(processScript(Scripts.DOC_VALUE), paramsBuilder().variable(field.name()).build(), dataType());
+        Params params = paramsBuilder().variable(field.name()).build();
+        // unsigned_long fields get returned in scripts as plain longs, so a conversion is required
+        return field.dataType() != UNSIGNED_LONG
+            ? new ScriptTemplate(processScript(Scripts.DOC_VALUE), params, dataType())
+            : new ScriptTemplate(
+                processScript(format("{ql}.", "nullSafeCastToUnsignedLong({})", Scripts.DOC_VALUE)),
+                params,
+                UNSIGNED_LONG
+            );
     }
 
     protected String processScript(String script) {

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java

@@ -23,6 +23,7 @@
 import java.util.Map;
 
 import static org.elasticsearch.xpack.ql.type.DataTypeConverter.convert;
+import static org.elasticsearch.xpack.ql.type.DataTypeConverter.toUnsignedLong;
 import static org.elasticsearch.xpack.ql.type.DataTypes.fromTypeName;
 
 public class InternalQlScriptUtils {
@@ -58,6 +59,10 @@ public static Number nullSafeCastNumeric(Number number, String typeName) {
         return number == null || Double.isNaN(number.doubleValue()) ? null : (Number) convert(number, fromTypeName(typeName));
     }
 
+    public static Number nullSafeCastToUnsignedLong(Number number) {
+        return number == null || Double.isNaN(number.doubleValue()) ? null : toUnsignedLong(number);
+    }
+
     //
     // Operators
     //

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java

@@ -8,8 +8,11 @@
 
 import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
 
+import java.math.BigInteger;
 import java.util.function.BiFunction;
 
+import static org.elasticsearch.xpack.ql.util.NumericUtils.asUnsignedLong;
+
 /**
  * Arithmetic operation using the type widening rules of the JLS 5.6.2 namely
  * widen to double or float or long or int in this order.
@@ -43,6 +46,10 @@ public static Number add(Number l, Number r) {
         if (l instanceof Float || r instanceof Float) {
             return Float.valueOf(l.floatValue() + r.floatValue());
         }
+        if (l instanceof BigInteger || r instanceof BigInteger) {
+            BigInteger bi = asBigInteger(l).add(asBigInteger(r));
+            return asUnsignedLong(bi);
+        }
         if (l instanceof Long || r instanceof Long) {
             return Long.valueOf(Math.addExact(l.longValue(), r.longValue()));
         }
@@ -61,6 +68,10 @@ public static Number sub(Number l, Number r) {
         if (l instanceof Float || r instanceof Float) {
             return Float.valueOf(l.floatValue() - r.floatValue());
         }
+        if (l instanceof BigInteger || r instanceof BigInteger) {
+            BigInteger bi = asBigInteger(l).subtract(asBigInteger(r));
+            return asUnsignedLong(bi);
+        }
         if (l instanceof Long || r instanceof Long) {
             return Long.valueOf(Math.subtractExact(l.longValue(), r.longValue()));
         }
@@ -79,6 +90,13 @@ public static Number mul(Number l, Number r) {
         if (l instanceof Float || r instanceof Float) {
             return Float.valueOf(l.floatValue() * r.floatValue());
         }
+        if (l instanceof BigInteger || r instanceof BigInteger) {
+            BigInteger bi = asBigInteger(l).multiply(asBigInteger(r));
+            // Note: in case of unsigned_long overflow (or underflow, with negative fixed point numbers), the exception is thrown.
+            // This is unlike the way some other traditional RDBMS that support unsigned types work, which simply promote the result to a
+            // floating point type, but in line with how our implementation treats other fixed point type operations (i.e. Math#xxExact()).
+            return asUnsignedLong(bi);
+        }
         if (l instanceof Long || r instanceof Long) {
             return Long.valueOf(Math.multiplyExact(l.longValue(), r.longValue()));
         }
@@ -97,6 +115,10 @@ public static Number div(Number l, Number r) {
         if (l instanceof Float || r instanceof Float) {
             return l.floatValue() / r.floatValue();
         }
+        if (l instanceof BigInteger || r instanceof BigInteger) {
+            BigInteger bi = asBigInteger(l).divide(asBigInteger(r));
+            return asUnsignedLong(bi);
+        }
         if (l instanceof Long || r instanceof Long) {
             return l.longValue() / r.longValue();
         }
@@ -115,6 +137,10 @@ public static Number mod(Number l, Number r) {
         if (l instanceof Float || r instanceof Float) {
             return Float.valueOf(l.floatValue() % r.floatValue());
         }
+        if (l instanceof BigInteger || r instanceof BigInteger) {
+            BigInteger bi = asBigInteger(l).remainder(asBigInteger(r));
+            return asUnsignedLong(bi);
+        }
         if (l instanceof Long || r instanceof Long) {
             return Long.valueOf(l.longValue() % r.longValue());
         }
@@ -141,10 +167,20 @@ static Number negate(Number n) {
             }
             return Float.valueOf(-n.floatValue());
         }
+        if (n instanceof BigInteger) {
+            if (((BigInteger) n).signum() != 0) {
+                throw new ArithmeticException("unsigned_long overflow"); // in the scope of the unsigned_long type
+            }
+            return n;
+        }
         if (n instanceof Long) {
             return Long.valueOf(Math.negateExact(n.longValue()));
         }
 
         return Integer.valueOf(Math.negateExact(n.intValue()));
     }
+
+    public static BigInteger asBigInteger(Number n) {
+        return n instanceof BigInteger ? (BigInteger) n : BigInteger.valueOf(n.longValue());
+    }
 }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/Comparisons.java

@@ -6,8 +6,11 @@
  */
 package org.elasticsearch.xpack.ql.expression.predicate.operator.comparison;
 
+import java.math.BigInteger;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Arithmetics.asBigInteger;
+
 /**
  * Comparison utilities.
  */
@@ -92,6 +95,9 @@ private static Integer compare(Number l, Number r) {
         if (l instanceof Float || r instanceof Float) {
             return Float.compare(l.floatValue(), r.floatValue());
         }
+        if (l instanceof BigInteger || r instanceof BigInteger) {
+            return asBigInteger(l).compareTo(asBigInteger(r));
+        }
         if (l instanceof Long || r instanceof Long) {
             return Long.compare(l.longValue(), r.longValue());
         }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexCompatibility.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ql.index;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.EsField;
+import org.elasticsearch.xpack.ql.type.UnsupportedEsField;
+
+import java.util.Map;
+
+import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion;
+import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive;
+import static org.elasticsearch.xpack.ql.type.Types.propagateUnsupportedType;
+
+public final class IndexCompatibility {
+
+    public static Map<String, EsField> compatible(Map<String, EsField> mapping, Version version) {
+        for (Map.Entry<String, EsField> entry : mapping.entrySet()) {
+            EsField esField = entry.getValue();
+            DataType dataType = esField.getDataType();
+            if (isPrimitive(dataType) == false) {
+                compatible(esField.getProperties(), version);
+            } else if (isTypeSupportedInVersion(dataType, version) == false) {
+                EsField field = new UnsupportedEsField(entry.getKey(), dataType.name(), null, esField.getProperties());
+                entry.setValue(field);
+                propagateUnsupportedType(entry.getKey(), dataType.name(), esField.getProperties());
+            }
+        }
+        return mapping;
+    }
+
+    public static EsIndex compatible(EsIndex esIndex, Version version) {
+        compatible(esIndex.mapping(), version);
+        return esIndex;
+    }
+
+    public static IndexResolution compatible(IndexResolution indexResolution, Version version) {
+        if (indexResolution.isValid()) {
+            compatible(indexResolution.get(), version);
+        }
+        return indexResolution;
+    }
+}

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ql.index;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xpack.ql.type.DataType;
+
+import static org.elasticsearch.Version.V_8_2_0;
+import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG;
+
+public final class VersionCompatibilityChecks {
+
+    public static final Version INTRODUCING_UNSIGNED_LONG = V_8_2_0;
+
+    private VersionCompatibilityChecks() {}
+
+    public static boolean isTypeSupportedInVersion(DataType dataType, Version version) {
+        if (dataType == UNSIGNED_LONG) {
+            return supportsUnsignedLong(version);
+        }
+        return true;
+    }
+
+    /**
+     * Does the provided {@code version} support the unsigned_long type (PR#60050)?
+     */
+    public static boolean supportsUnsignedLong(Version version) {
+        return INTRODUCING_UNSIGNED_LONG.compareTo(version) <= 0;
+    }
+
+    public static @Nullable Version versionIntroducingType(DataType dataType) {
+        if (dataType == UNSIGNED_LONG) {
+            return INTRODUCING_UNSIGNED_LONG;
+        }
+
+        return null;
+    }
+}