Add support for VERSION field type in SQL and EQL

docs/changelog/85502.yaml

@@ -0,0 +1,6 @@
+pr: 85502
+summary: Add support for VERSION field type in SQL and EQL
+area: Query Languages
+type: enhancement
+issues:
+ - 83375

docs/reference/sql/language/data-types.asciidoc

@@ -28,6 +28,7 @@ s|SQL precision
 | <<binary, `binary`>>                     | binary          | VARBINARY   | 2,147,483,647
 | <<date, `date`>>                         | datetime        | TIMESTAMP   | 29
 | <<ip, `ip`>>                             | ip              | VARCHAR     | 39
+| <<version, `version`>>                   | version         | VARCHAR     | 32,766
 
 4+h| Complex types
 

x-pack/plugin/eql/qa/common/src/main/resources/data/extra.data

@@ -112,6 +112,7 @@
   "transID": 2,
   "process.entity_id": 512,
   "process.pid": 123,
+  "version": "1.5.0",
   "sequence": 18
  },
  {
@@ -120,6 +121,7 @@
   "transID": 0,
   "process.entity_id": 512,
   "process.pid": 123,
+  "version": "1.5.0",
   "sequence": 19
  },
  {
@@ -128,6 +130,7 @@
   "transID": 0,
   "process.entity_id": 512,
   "process.pid": 123,
+  "version": "1.2.4",
   "sequence": 20
  },
  {
@@ -136,6 +139,7 @@
   "transID": 0,
   "process.entity_id": 512,
   "process.pid": 123,
+  "version": "1.11.3",
   "sequence": 21
  },
  {
@@ -144,6 +148,7 @@
   "transID": 0,
   "process.entity_id": 512,
   "process.pid": 123,
+  "version": "bad",
   "sequence": 22
  },
  {
@@ -152,6 +157,7 @@
   "transID": 0,
   "process.entity_id": 512,
   "process.pid": 123,
+  "version": "1.2.4-SNAPSHOT",
   "sequence": 23
  }
 ]

x-pack/plugin/eql/qa/common/src/main/resources/data/extra.mapping

@@ -27,6 +27,9 @@
     "optional_field_default_null": {
       "type": "keyword",
       "null_value": "NULL"
+    },
+    "version": {
+        "type": "version"
     }
   }
 }

x-pack/plugin/eql/qa/common/src/main/resources/test_extra.toml

@@ -76,7 +76,7 @@ expected_event_ids  = []
 [[queries]]
 name = "sequenceOptionalFieldAsKeyAndFirstFilter"
 query = '''
-sequence by ?x 
+sequence by ?x
   [OPTIONAL where ?x == null]
   [OPTIONAL where true]
 '''
@@ -137,7 +137,7 @@ expected_event_ids  = [12,13,14,
 join_keys = ["null","123",
              "null","null",
              "512","123"]
-             
+
 [[queries]]
 name = "sequenceOneKeyOptional_1"
 query = '''
@@ -159,7 +159,7 @@ expected_event_ids  = [12,13,14,
                        18,19,20]
 join_keys = ["null","123",
              "512","123"]
-             
+
 [[queries]]
 name = "sequenceNoOptionalKeys"
 query = '''
@@ -195,7 +195,7 @@ expected_event_ids  = [12,13,14,
 join_keys = ["null","123",
              "null","null",
              "512","123"]
-             
+
 [[queries]]
 name = "sequenceWithOptionalFieldsAndUntil_2"
 query = '''
@@ -209,3 +209,65 @@ expected_event_ids  = [12,13,14,
 join_keys = ["null","123",
              "null","null"]
 
+[[queries]]
+name = "sequenceWithVersionCheck"
+query = '''
+  sequence by transID
+    [ file where version == "1.2.4" ]
+    [ file where version == "1.11.3" ]
+'''
+expected_event_ids  = [20, 21]
+join_keys = ["0"]
+
+[[queries]]
+name = "sequenceWithVersionRangeCheck"
+query = '''
+  sequence by transID
+    [ file where version == "1.2.4" ]
+    [ file where version > "1.5.0" ]
+'''
+expected_event_ids  = [20, 21]
+join_keys = ["0"]
+
+[[queries]]
+name = "sequenceWithVersionRangeCheck2"
+query = '''
+  sequence by transID
+    [ file where version == "1.2.4" ]
+    [ file where version < "1.5.0" ]
+'''
+expected_event_ids  = [20, 23]
+join_keys = ["0"]
+
+
+[[queries]]
+name = "sequenceWithInvalidVersion"
+query = '''
+  sequence by transID
+    [ file where version == "1.2.4" ]
+    [ file where version == "bad" ]
+'''
+expected_event_ids  = [20, 22]
+join_keys = ["0"]
+
+
+[[queries]]
+name = "sequenceWithVersionConcat"
+query = '''
+  sequence by transID
+    [ file where CONCAT(version, "") == "1.2.4" ]
+    [ file where version == "bad" ]
+'''
+expected_event_ids  = [20, 22]
+join_keys = ["0"]
+
+
+[[queries]]
+name = "sequenceWithVersionJoinKey"
+query = '''
+  sequence by version
+    [ process where true ]
+    [ file where true ]
+'''
+expected_event_ids  = [18, 19]
+join_keys = ["1.5.0"]

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java

@@ -11,7 +11,6 @@
 import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.NestedSortBuilder;
-import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer;
 import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
@@ -133,10 +132,7 @@ private static void sorting(QueryContainer container, SearchSourceBuilder source
                     }
                 }
             } else if (sortable instanceof ScriptSort ss) {
-                sortBuilder = scriptSort(
-                    ss.script().toPainless(),
-                    ss.script().outputType().isNumeric() ? ScriptSortType.NUMBER : ScriptSortType.STRING
-                );
+                sortBuilder = scriptSort(ss.script().toPainless(), ss.script().outputType().scriptSortType());
             }
 
             if (sortBuilder != null) {

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPainlessExtension.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.painless.spi.WhitelistLoader;
 import org.elasticsearch.script.AggregationScript;
 import org.elasticsearch.script.BucketAggregationSelectorScript;
+import org.elasticsearch.script.BytesRefSortScript;
 import org.elasticsearch.script.FieldScript;
 import org.elasticsearch.script.FilterScript;
 import org.elasticsearch.script.NumberSortScript;
@@ -36,6 +37,7 @@ public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
         whitelist.put(FieldScript.CONTEXT, list);
         whitelist.put(NumberSortScript.CONTEXT, list);
         whitelist.put(StringSortScript.CONTEXT, list);
+        whitelist.put(BytesRefSortScript.CONTEXT, list);
         whitelist.put(BucketAggregationSelectorScript.CONTEXT, list);
         return whitelist;
     }

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java

@@ -93,6 +93,7 @@ public void testQueryCondition() {
         assertEquals("1:11: Condition expression needs to be boolean, found [TEXT]", error("any where hostname"));
         assertEquals("1:11: Condition expression needs to be boolean, found [KEYWORD]", error("any where constant_keyword"));
         assertEquals("1:11: Condition expression needs to be boolean, found [IP]", error("any where source_address"));
+        assertEquals("1:11: Condition expression needs to be boolean, found [VERSION]", error("any where version"));
     }
 
     public void testQueryStartsWithNumber() {
@@ -363,6 +364,11 @@ public void testIP() {
         accept(idxr, "foo where ip_addr == 0");
     }
 
+    public void testVersion() {
+        final IndexResolution idxr = loadIndexResolution("mapping-version.json");
+        accept(idxr, "foo where version_number == \"2.1.4\"");
+    }
+
     public void testJoin() {
         final IndexResolution idxr = loadIndexResolution("mapping-join.json");
         accept(idxr, "foo where serial_event_id == 0");

x-pack/plugin/eql/src/test/resources/mapping-default.json

@@ -90,6 +90,9 @@
         },
         "bool" : {
             "type" : "boolean"
+        },
+        "version" : {
+            "type" : "version"
         }
     }
 }

x-pack/plugin/eql/src/test/resources/mapping-version.json

@@ -0,0 +1,17 @@
+{
+    "properties" : {
+        "event" : {
+            "properties" : {
+                "category" : {
+                    "type" : "keyword"
+                }
+            }
+        },
+        "@timestamp" : {
+            "type" : "date"
+        },
+        "version_number" : {
+            "type" : "version"
+        }
+    }
+}

x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt

@@ -901,3 +901,17 @@ process where pid > 100 and pid < 200
 // the "not" has an undefined location of where it should be used in the Painless script
 // disjunctionInsideFunctionWithNot
 // process where string(pid > 5 and pid != 10) == \"true\"
+
+
+versionFieldAutomaticConversion
+process where version > "2"
+;
+{"range":{"version":{"gt":"2"
+;
+
+versionFieldCast
+process where CONCAT(version, constant_keyword) > "2"
+;
+{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalEqlScriptUtils.multiValueDocValues(doc,params.v1,X1->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalEqlScriptUtils.concat([X0,X1]),params.v2))))"
+"params":{"v0":"version","v1":"constant_keyword","v2":"2"}}
+;

x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionEncoder.java

@@ -35,7 +35,7 @@
  *  lexically in ASCII sort order. Numeric identifiers always have lower precedence than non-numeric identifiers.
  * </ul>
  */
-class VersionEncoder {
+public class VersionEncoder {
 
     public static final byte NUMERIC_MARKER_BYTE = (byte) 0x01;
     public static final byte PRERELEASE_SEPARATOR_BYTE = (byte) 0x02;
@@ -175,7 +175,7 @@ static boolean legalVersionString(VersionParts versionParts) {
         return legalMainVersion && legalPreRelease && legalBuildSuffix;
     }
 
-    static class EncodedVersion {
+    public static class EncodedVersion {
 
         public final boolean isLegal;
         public final boolean isPreRelease;

x-pack/plugin/ql/build.gradle

@@ -16,6 +16,7 @@ archivesBaseName = 'x-pack-ql'
 
 dependencies {
   api "org.antlr:antlr4-runtime:${antlrVersion}"
+  api project(path: xpackModule('mapper-version'))
   compileOnly project(path: xpackModule('core'))
   testApi(project(xpackModule('ql:test-fixtures'))) {
     exclude group: 'org.elasticsearch.plugin', module: 'ql'

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/Comparisons.java

@@ -6,6 +6,8 @@
  */
 package org.elasticsearch.xpack.ql.expression.predicate.operator.comparison;
 
+import org.elasticsearch.xpack.versionfield.Version;
+
 import java.math.BigInteger;
 import java.util.Set;
 
@@ -71,13 +73,21 @@ static Integer compare(Object l, Object r) {
             return null;
         }
         // typical number comparison
-        if (l instanceof Number && r instanceof Number) {
-            return compare((Number) l, (Number) r);
+        if (l instanceof Number lN && r instanceof Number rN) {
+            return compare(lN, rN);
+        }
+
+        // automatic conversion for versions
+        if (l instanceof Version lV && r instanceof String rStr) {
+            return lV.compareTo(new Version(rStr));
+        }
+        if (l instanceof String lStr && r instanceof Version rV) {
+            return new Version(lStr).compareTo(rV);
         }
 
-        if (l instanceof Comparable && r instanceof Comparable) {
+        if (l instanceof Comparable lC && r instanceof Comparable) {
             try {
-                return Integer.valueOf(((Comparable) l).compareTo(r));
+                return Integer.valueOf(lC.compareTo(r));
             } catch (ClassCastException cce) {
                 // when types are not compatible, cce is thrown
                 // fall back to null

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java

@@ -12,18 +12,24 @@
 import org.elasticsearch.xpack.ql.type.DataType;
 
 import static org.elasticsearch.Version.V_8_2_0;
+import static org.elasticsearch.Version.V_8_4_0;
 import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.ql.type.DataTypes.VERSION;
 
 public final class VersionCompatibilityChecks {
 
     public static final Version INTRODUCING_UNSIGNED_LONG = V_8_2_0;
+    public static final Version INTRODUCING_VERSION_FIELD_TYPE = V_8_4_0;
 
     private VersionCompatibilityChecks() {}
 
     public static boolean isTypeSupportedInVersion(DataType dataType, Version version) {
         if (dataType == UNSIGNED_LONG) {
             return supportsUnsignedLong(version);
         }
+        if (dataType == VERSION) {
+            return supportsVersionType(version);
+        }
         return true;
     }
 
@@ -34,10 +40,20 @@ public static boolean supportsUnsignedLong(Version version) {
         return INTRODUCING_UNSIGNED_LONG.compareTo(version) <= 0;
     }
 
+    /**
+     * Does the provided {@code version} support the version type (PR#85502)?
+     */
+    public static boolean supportsVersionType(Version version) {
+        return INTRODUCING_VERSION_FIELD_TYPE.compareTo(version) <= 0;
+    }
+
     public static @Nullable Version versionIntroducingType(DataType dataType) {
         if (dataType == UNSIGNED_LONG) {
             return INTRODUCING_UNSIGNED_LONG;
         }
+        if (dataType == VERSION) {
+            return INTRODUCING_VERSION_FIELD_TYPE;
+        }
 
         return null;
     }