QL: add unsigned_long type support

docs/changelog/65145.yaml

@@ -0,0 +1,6 @@
+pr: 65145
+summary: Add `unsigned_long` type support
+area: Query Languages
+type: enhancement
+issues:
+ - 63312

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java

@@ -49,7 +49,6 @@
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.ql.tree.Source;
-import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypes;
 import org.elasticsearch.xpack.ql.util.StringUtils;
 
@@ -240,10 +239,9 @@ public Literal visitIntegerLiteral(EqlBaseParser.IntegerLiteralContext ctx) {
         Source source = source(ctx);
         String text = ctx.getText();
 
-        long value;
-
         try {
-            value = Long.valueOf(StringUtils.parseLong(text));
+            Number value = StringUtils.parseIntegral(text);
+            return new Literal(source, value, DataTypes.fromJava(value));
         } catch (QlIllegalArgumentException siae) {
             // if it's too large, then quietly try to parse as a float instead
             try {
@@ -252,16 +250,6 @@ public Literal visitIntegerLiteral(EqlBaseParser.IntegerLiteralContext ctx) {
 
             throw new ParsingException(source, siae.getMessage());
         }
-
-        Object val = Long.valueOf(value);
-        DataType type = DataTypes.LONG;
-
-        // try to downsize to int if possible (since that's the most common type)
-        if ((int) value == value) {
-            type = DataTypes.INTEGER;
-            val = Integer.valueOf((int) value);
-        }
-        return new Literal(source, val, type);
     }
 
     @Override

x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt

@@ -19,6 +19,7 @@ class org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQl
   double nullSafeSortNumeric(Number)
   String nullSafeSortString(Object)
   Number nullSafeCastNumeric(Number, String)
+  Number nullSafeCastToUnsignedLong(Number)
 
 #
 # ASCII Functions

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java

@@ -237,6 +237,7 @@ public void testNumeric() {
         accept(idxr, "foo where float_field == 0");
         accept(idxr, "foo where half_float_field == 0");
         accept(idxr, "foo where scaled_float_field == 0");
+        accept(idxr, "foo where unsigned_long_field == 0");
 
         // Test query against unsupported field type int
         assertEquals(

x-pack/plugin/eql/src/test/resources/mapping-numeric.json

@@ -34,6 +34,9 @@
         "scaled_float_field": {
             "type" : "scaled_float"
         },
+        "unsigned_long_field" : {
+          "type": "unsigned_long"
+        },
         "wrong_int_type_field": {
             "type" : "int"
         }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java

@@ -22,7 +22,6 @@
  * the resulting ES document as a field.
  */
 public class QlSourceBuilder {
-    public static final Version SWITCH_TO_FIELDS_API_VERSION = Version.V_7_10_0;
     public static final Version INTRODUCING_MISSING_ORDER_IN_COMPOSITE_AGGS_VERSION = Version.V_7_16_0;
     // The LinkedHashMaps preserve the order of the fields in the response
     private final Set<FieldAndFormat> fetchFields = new LinkedHashSet<>();

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.xpack.ql.expression.function.Function;
 import org.elasticsearch.xpack.ql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.ql.expression.function.grouping.GroupingFunction;
+import org.elasticsearch.xpack.ql.expression.gen.script.Params;
 import org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder;
 import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
 import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
@@ -24,10 +25,12 @@
 import java.util.List;
 
 import static java.util.Collections.emptyList;
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
 import static org.elasticsearch.xpack.ql.expression.gen.script.Scripts.PARAM;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.LONG;
+import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG;
 
 /**
  * A {@code ScalarFunction} is a {@code Function} that takes values from some
@@ -150,7 +153,15 @@ protected ScriptTemplate scriptWithGrouping(GroupingFunction grouping) {
     }
 
     protected ScriptTemplate scriptWithField(FieldAttribute field) {
-        return new ScriptTemplate(processScript(Scripts.DOC_VALUE), paramsBuilder().variable(field.name()).build(), dataType());
+        Params params = paramsBuilder().variable(field.name()).build();
+        // unsigned_long fields get returned in scripts as plain longs, so a conversion is required
+        return field.dataType() != UNSIGNED_LONG
+            ? new ScriptTemplate(processScript(Scripts.DOC_VALUE), params, dataType())
+            : new ScriptTemplate(
+                processScript(format("{ql}.", "nullSafeCastToUnsignedLong({})", Scripts.DOC_VALUE)),
+                params,
+                UNSIGNED_LONG
+            );
     }
 
     protected String processScript(String script) {

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java

@@ -23,6 +23,7 @@
 import java.util.Map;
 
 import static org.elasticsearch.xpack.ql.type.DataTypeConverter.convert;
+import static org.elasticsearch.xpack.ql.type.DataTypeConverter.toUnsignedLong;
 import static org.elasticsearch.xpack.ql.type.DataTypes.fromTypeName;
 
 public class InternalQlScriptUtils {
@@ -58,6 +59,10 @@ public static Number nullSafeCastNumeric(Number number, String typeName) {
         return number == null || Double.isNaN(number.doubleValue()) ? null : (Number) convert(number, fromTypeName(typeName));
     }
 
+    public static Number nullSafeCastToUnsignedLong(Number number) {
+        return number == null || Double.isNaN(number.doubleValue()) ? null : toUnsignedLong(number);
+    }
+
     //
     // Operators
     //

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/Arithmetics.java

@@ -8,8 +8,11 @@
 
 import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
 
+import java.math.BigInteger;
 import java.util.function.BiFunction;
 
+import static org.elasticsearch.xpack.ql.util.NumericUtils.asUnsignedLong;
+
 /**
  * Arithmetic operation using the type widening rules of the JLS 5.6.2 namely
  * widen to double or float or long or int in this order.
@@ -43,6 +46,10 @@ public static Number add(Number l, Number r) {
         if (l instanceof Float || r instanceof Float) {
             return Float.valueOf(l.floatValue() + r.floatValue());
         }
+        if (l instanceof BigInteger || r instanceof BigInteger) {
+            BigInteger bi = asBigInteger(l).add(asBigInteger(r));
+            return asUnsignedLong(bi);
+        }
         if (l instanceof Long || r instanceof Long) {
             return Long.valueOf(Math.addExact(l.longValue(), r.longValue()));
         }
@@ -61,6 +68,10 @@ public static Number sub(Number l, Number r) {
         if (l instanceof Float || r instanceof Float) {
             return Float.valueOf(l.floatValue() - r.floatValue());
         }
+        if (l instanceof BigInteger || r instanceof BigInteger) {
+            BigInteger bi = asBigInteger(l).subtract(asBigInteger(r));
+            return asUnsignedLong(bi);
+        }
         if (l instanceof Long || r instanceof Long) {
             return Long.valueOf(Math.subtractExact(l.longValue(), r.longValue()));
         }
@@ -79,6 +90,13 @@ public static Number mul(Number l, Number r) {
         if (l instanceof Float || r instanceof Float) {
             return Float.valueOf(l.floatValue() * r.floatValue());
         }
+        if (l instanceof BigInteger || r instanceof BigInteger) {
+            BigInteger bi = asBigInteger(l).multiply(asBigInteger(r));
+            // Note: in case of unsigned_long overflow (or underflow, with negative fixed point numbers), the exception is thrown.
+            // This is unlike the way some other traditional RDBMS that support unsigned types work, which simply promote the result to a
+            // floating point type, but in line with how our implementation treats other fixed point type operations (i.e. Math#xxExact()).
+            return asUnsignedLong(bi);
+        }
         if (l instanceof Long || r instanceof Long) {
             return Long.valueOf(Math.multiplyExact(l.longValue(), r.longValue()));
         }
@@ -97,6 +115,10 @@ public static Number div(Number l, Number r) {
         if (l instanceof Float || r instanceof Float) {
             return l.floatValue() / r.floatValue();
         }
+        if (l instanceof BigInteger || r instanceof BigInteger) {
+            BigInteger bi = asBigInteger(l).divide(asBigInteger(r));
+            return asUnsignedLong(bi);
+        }
         if (l instanceof Long || r instanceof Long) {
             return l.longValue() / r.longValue();
         }
@@ -115,6 +137,10 @@ public static Number mod(Number l, Number r) {
         if (l instanceof Float || r instanceof Float) {
             return Float.valueOf(l.floatValue() % r.floatValue());
         }
+        if (l instanceof BigInteger || r instanceof BigInteger) {
+            BigInteger bi = asBigInteger(l).remainder(asBigInteger(r));
+            return asUnsignedLong(bi);
+        }
         if (l instanceof Long || r instanceof Long) {
             return Long.valueOf(l.longValue() % r.longValue());
         }
@@ -141,10 +167,20 @@ static Number negate(Number n) {
             }
             return Float.valueOf(-n.floatValue());
         }
+        if (n instanceof BigInteger) {
+            if (((BigInteger) n).signum() != 0) {
+                throw new ArithmeticException("unsigned_long overflow"); // in the scope of the unsigned_long type
+            }
+            return n;
+        }
         if (n instanceof Long) {
             return Long.valueOf(Math.negateExact(n.longValue()));
         }
 
         return Integer.valueOf(Math.negateExact(n.intValue()));
     }
+
+    public static BigInteger asBigInteger(Number n) {
+        return n instanceof BigInteger ? (BigInteger) n : BigInteger.valueOf(n.longValue());
+    }
 }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/Comparisons.java

@@ -6,8 +6,11 @@
  */
 package org.elasticsearch.xpack.ql.expression.predicate.operator.comparison;
 
+import java.math.BigInteger;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Arithmetics.asBigInteger;
+
 /**
  * Comparison utilities.
  */
@@ -92,6 +95,9 @@ private static Integer compare(Number l, Number r) {
         if (l instanceof Float || r instanceof Float) {
             return Float.compare(l.floatValue(), r.floatValue());
         }
+        if (l instanceof BigInteger || r instanceof BigInteger) {
+            return asBigInteger(l).compareTo(asBigInteger(r));
+        }
         if (l instanceof Long || r instanceof Long) {
             return Long.compare(l.longValue(), r.longValue());
         }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexCompatibility.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ql.index;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.EsField;
+import org.elasticsearch.xpack.ql.type.UnsupportedEsField;
+
+import java.util.Map;
+
+import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion;
+import static org.elasticsearch.xpack.ql.type.DataTypes.isPrimitive;
+import static org.elasticsearch.xpack.ql.type.Types.propagateUnsupportedType;
+
+public final class IndexCompatibility {
+
+    public static Map<String, EsField> compatible(Map<String, EsField> mapping, Version version) {
+        for (Map.Entry<String, EsField> entry : mapping.entrySet()) {
+            EsField esField = entry.getValue();
+            DataType dataType = esField.getDataType();
+            if (isPrimitive(dataType) == false) {
+                compatible(esField.getProperties(), version);
+            } else if (isTypeSupportedInVersion(dataType, version) == false) {
+                EsField field = new UnsupportedEsField(entry.getKey(), dataType.name(), null, esField.getProperties());
+                entry.setValue(field);
+                propagateUnsupportedType(entry.getKey(), dataType.name(), esField.getProperties());
+            }
+        }
+        return mapping;
+    }
+
+    public static EsIndex compatible(EsIndex esIndex, Version version) {
+        compatible(esIndex.mapping(), version);
+        return esIndex;
+    }
+
+    public static IndexResolution compatible(IndexResolution indexResolution, Version version) {
+        if (indexResolution.isValid()) {
+            compatible(indexResolution.get(), version);
+        }
+        return indexResolution;
+    }
+}

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ql.index;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xpack.ql.type.DataType;
+
+import static org.elasticsearch.Version.V_8_2_0;
+import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG;
+
+public final class VersionCompatibilityChecks {
+
+    public static final Version INTRODUCING_UNSIGNED_LONG = V_8_2_0;
+
+    private VersionCompatibilityChecks() {}
+
+    public static boolean isTypeSupportedInVersion(DataType dataType, Version version) {
+        if (dataType == UNSIGNED_LONG) {
+            return supportsUnsignedLong(version);
+        }
+        return true;
+    }
+
+    /**
+     * Does the provided {@code version} support the unsigned_long type (PR#60050)?
+     */
+    public static boolean supportsUnsignedLong(Version version) {
+        return INTRODUCING_UNSIGNED_LONG.compareTo(version) <= 0;
+    }
+
+    public static @Nullable Version versionIntroducingType(DataType dataType) {
+        if (dataType == UNSIGNED_LONG) {
+            return INTRODUCING_UNSIGNED_LONG;
+        }
+
+        return null;
+    }
+}