diff --git a/docs/reference/esql/functions/description/mv_append.asciidoc b/docs/reference/esql/functions/description/mv_append.asciidoc new file mode 100644 index 0000000000000..26b549713e301 --- /dev/null +++ b/docs/reference/esql/functions/description/mv_append.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Concatenates values of two multi-value fields. diff --git a/docs/reference/esql/functions/kibana/definition/mv_append.json b/docs/reference/esql/functions/kibana/definition/mv_append.json new file mode 100644 index 0000000000000..8ee4e7297cc3a --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/mv_append.json @@ -0,0 +1,242 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "mv_append", + "description" : "Concatenates values of two multi-value fields.", + "signatures" : [ + { + "params" : [ + { + "name" : "field1", + "type" : "boolean", + "optional" : false, + "description" : "" + }, + { + "name" : "field2", + "type" : "boolean", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field1", + "type" : "cartesian_point", + "optional" : false, + "description" : "" + }, + { + "name" : "field2", + "type" : "cartesian_point", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "cartesian_point" + }, + { + "params" : [ + { + "name" : "field1", + "type" : "cartesian_shape", + "optional" : false, + "description" : "" + }, + { + "name" : "field2", + "type" : "cartesian_shape", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "cartesian_shape" + }, + { + "params" : [ + { + "name" : "field1", + "type" : "datetime", + "optional" : false, + "description" : "" + }, + { + "name" : "field2", + "type" : "datetime", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "field1", + "type" : "double", + "optional" : false, + "description" : "" + }, + { + "name" : "field2", + "type" : "double", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "field1", + "type" : "geo_point", + "optional" : false, + "description" : "" + }, + { + "name" : "field2", + "type" : "geo_point", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "geo_point" + }, + { + "params" : [ + { + "name" : "field1", + "type" : "geo_shape", + "optional" : false, + "description" : "" + }, + { + "name" : "field2", + "type" : "geo_shape", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "geo_shape" + }, + { + "params" : [ + { + "name" : "field1", + "type" : "integer", + "optional" : false, + "description" : "" + }, + { + "name" : "field2", + "type" : "integer", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "field1", + "type" : "ip", + "optional" : false, + "description" : "" + }, + { + "name" : "field2", + "type" : "ip", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "ip" + }, + { + "params" : [ + { + "name" : "field1", + "type" : "keyword", + "optional" : false, + "description" : "" + }, + { + "name" : "field2", + "type" : "keyword", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "field1", + "type" : "long", + "optional" : false, + "description" : "" + }, + { + "name" : "field2", + "type" : "long", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field1", + "type" : "text", + "optional" : false, + "description" : "" + }, + { + "name" : "field2", + "type" : "text", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "text" + }, + { + "params" : [ + { + "name" : "field1", + "type" : "version", + "optional" : false, + "description" : "" + }, + { + "name" : "field2", + "type" : "version", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "version" + } + ] +} diff --git a/docs/reference/esql/functions/kibana/docs/mv_append.md b/docs/reference/esql/functions/kibana/docs/mv_append.md new file mode 100644 index 0000000000000..36b285be1877c --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/mv_append.md @@ -0,0 +1,7 @@ + + +### MV_APPEND +Concatenates values of two multi-value fields. + diff --git a/docs/reference/esql/functions/layout/mv_append.asciidoc b/docs/reference/esql/functions/layout/mv_append.asciidoc new file mode 100644 index 0000000000000..4d4dbd7a24f9d --- /dev/null +++ b/docs/reference/esql/functions/layout/mv_append.asciidoc @@ -0,0 +1,14 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-mv_append]] +=== `MV_APPEND` + +*Syntax* + +[.text-center] +image::esql/functions/signature/mv_append.svg[Embedded,opts=inline] + +include::../parameters/mv_append.asciidoc[] +include::../description/mv_append.asciidoc[] +include::../types/mv_append.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/mv_append.asciidoc b/docs/reference/esql/functions/parameters/mv_append.asciidoc new file mode 100644 index 0000000000000..e08d697c25098 --- /dev/null +++ b/docs/reference/esql/functions/parameters/mv_append.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`field1`:: + + +`field2`:: + diff --git a/docs/reference/esql/functions/signature/mv_append.svg b/docs/reference/esql/functions/signature/mv_append.svg new file mode 100644 index 0000000000000..0f45435425c65 --- /dev/null +++ b/docs/reference/esql/functions/signature/mv_append.svg @@ -0,0 +1 @@ +MV_APPEND(field1,field2) \ No newline at end of file diff --git a/docs/reference/esql/functions/types/mv_append.asciidoc b/docs/reference/esql/functions/types/mv_append.asciidoc new file mode 100644 index 0000000000000..49dcef6dc8860 --- /dev/null +++ b/docs/reference/esql/functions/types/mv_append.asciidoc @@ -0,0 +1,21 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +field1 | field2 | result +boolean | boolean | boolean +cartesian_point | cartesian_point | cartesian_point +cartesian_shape | cartesian_shape | cartesian_shape +datetime | datetime | datetime +double | double | double +geo_point | geo_point | geo_point +geo_shape | geo_shape | geo_shape +integer | integer | integer +ip | ip | ip +keyword | keyword | keyword +long | long | long +text | text | text +version | version | version +|=== diff --git a/x-pack/plugin/esql-core/src/test/resources/mapping-multi-field-variation.json b/x-pack/plugin/esql-core/src/test/resources/mapping-multi-field-variation.json index b5b3d42816502..5369e50dd6bb9 100644 --- a/x-pack/plugin/esql-core/src/test/resources/mapping-multi-field-variation.json +++ b/x-pack/plugin/esql-core/src/test/resources/mapping-multi-field-variation.json @@ -8,6 +8,8 @@ "keyword" : { "type" : "keyword" }, "date" : { "type" : "date" }, "date_nanos": { "type" : "date_nanos" }, + "long" : { "type" : "long" }, + "ip" : { "type" : "ip" }, "unsupported" : { "type" : "ip_range" }, "some" : { "properties" : { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index 0c808afc9d12b..63421aec35665 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -1076,6 +1076,23 @@ required_capability: agg_values [1955-01-21T00:00:00Z, 1957-05-23T00:00:00Z, 1959-12-03T00:00:00Z] | null ; + +mvAppendDates +required_capability: fn_mv_append + +FROM employees +| WHERE emp_no == 10039 OR emp_no == 10040 +| SORT emp_no +| EVAL dates = mv_append(birth_date, hire_date) +| KEEP emp_no, birth_date, hire_date, dates +; + +emp_no:integer | birth_date:date | hire_date:date | dates:date +10039 | 1959-10-01T00:00:00Z | 1988-01-19T00:00:00Z | [1959-10-01T00:00:00Z, 1988-01-19T00:00:00Z] +10040 | null | 1993-02-14T00:00:00Z | null +; + + implicitCastingNotEqual required_capability: string_literal_auto_casting from employees | where birth_date != "1957-05-23T00:00:00Z" | keep emp_no, birth_date | sort emp_no | limit 3; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec index bbe0df9a8cda9..e88daa63ec557 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec @@ -523,6 +523,25 @@ required_capability: agg_values [1.7, 1.83, 2.05] | null ; + +mvAppend +required_capability: fn_mv_append + +FROM employees +| WHERE emp_no == 10008 OR emp_no == 10021 +| EVAL d = mv_append(salary_change, salary_change), + i = mv_append(salary_change.int, salary_change.int), + i2 = mv_append(emp_no, salary_change.int), + i3 = mv_append(emp_no, emp_no), + s = mv_append(salary_change.keyword, salary_change.keyword) +| KEEP emp_no, salary_change, d, i, i2, i3, s; + +emp_no:integer | salary_change:double | d:double | i:integer | i2:integer | i3:integer | s:keyword +10008 | [-2.92,0.75,3.54,12.68] | [-2.92,0.75,3.54,12.68,-2.92,0.75,3.54,12.68] | [-2,0,3,12,-2,0,3,12] | [10008,-2,0,3,12] | [10008, 10008] | [-2.92,0.75,12.68,3.54,-2.92,0.75,12.68,3.54] +10021 | null | null | null | null | [10021, 10021] | null +; + + signumOfPositiveDouble#[skip:-8.13.99,reason:new scalar function added in 8.14] row d = to_double(100) | eval s = signum(d); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index eff4cb05bd8c0..c81730978f284 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -42,6 +42,7 @@ double e() "double|integer|long median(number:double|integer|long)" "double|integer|long median_absolute_deviation(number:double|integer|long)" "double|integer|long min(number:double|integer|long)" +"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version mv_append(field1:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version, field2:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version)" "double mv_avg(number:double|integer|long|unsigned_long)" "keyword mv_concat(string:text|keyword, delim:text|keyword)" "integer mv_count(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)" @@ -157,6 +158,7 @@ max |number |"double|integer|long" median |number |"double|integer|long" |[""] median_absolut|number |"double|integer|long" |[""] min |number |"double|integer|long" |[""] +mv_append |[field1, field2] |["boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version", "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"] | ["", ""] mv_avg |number |"double|integer|long|unsigned_long" |Multivalue expression. mv_concat |[string, delim] |["text|keyword", "text|keyword"] |[Multivalue expression., Delimiter.] mv_count |field |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version" |Multivalue expression. @@ -273,6 +275,7 @@ max |The maximum value of a numeric field. median |The value that is greater than half of all values and less than half of all values. median_absolut|The median absolute deviation, a measure of variability. min |The minimum value of a numeric field. +mv_append |Concatenates values of two multi-value fields. mv_avg |Converts a multivalued field into a single valued field containing the average of all of the values. mv_concat |Converts a multivalued string expression into a single valued column containing the concatenation of all values separated by a delimiter. mv_count |Converts a multivalued expression into a single valued column containing a count of the number of values. @@ -390,6 +393,7 @@ max |"double|integer|long" median |"double|integer|long" |false |false |true median_absolut|"double|integer|long" |false |false |true min |"double|integer|long" |false |false |true +mv_append |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version" |[false, false] |false |false mv_avg |double |false |false |false mv_concat |keyword |[false, false] |false |false mv_count |integer |false |false |false @@ -475,5 +479,5 @@ countFunctions#[skip:-8.14.99, reason:BIN added] meta functions | stats a = count(*), b = count(*), c = count(*) | mv_expand c; a:long | b:long | c:long -107 | 107 | 107 +108 | 108 | 108 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index 13616e5146949..3cb7c6ef0f594 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -1335,6 +1335,79 @@ l1:integer | l2:integer null | 0 ; + +mvAppend +required_capability: fn_mv_append + +ROW a = "a", b = ["b", "c"], n = null +| EVAL aa = mv_append(a, a), bb = mv_append(b, b), ab = mv_append(a, b), abb = mv_append(mv_append(a, b), b), na = mv_append(n, a), an = mv_append(a, n) +; + +a:keyword | b:keyword | n:null | aa:keyword | bb:keyword | ab:keyword | abb:keyword | na:keyword | an:keyword +a | [b, c] | null |[a, a] | [b, c, b, c] | [a, b, c] | [a, b, c, b, c] | null | null +; + + +mvAppendNull +required_capability: fn_mv_append + +ROW a = "a", b = ["b", "c"], c = to_string(null) +| EVAL a_null = mv_append(a, c), + null_a = mv_append(c, a), + b_null = mv_append(b, c), + null_b = mv_append(c, b), + null_null = mv_append(c, c) +; + +a:keyword | b:keyword | c:keyword | a_null:keyword | null_a:keyword | b_null:keyword | null_b:keyword | null_null:keyword +a | [b, c] | null | null | null | null | null | null +; + + +mvAppendStrings +required_capability: fn_mv_append + +FROM employees +| WHERE emp_no == 10004 +| EVAL names = mv_sort(mv_append(first_name, last_name)), + two_jobs = mv_sort(mv_append(job_positions, job_positions)), + three_jobs = mv_sort(mv_append(job_positions, mv_append(job_positions, job_positions))) +| KEEP emp_no, names, two_jobs, three_jobs +; + +emp_no:integer | names:keyword | two_jobs:keyword | three_jobs:keyword +10004 | ["Chirstian", "Koblick"] | ["Head Human Resources","Head Human Resources","Reporting Analyst","Reporting Analyst","Support Engineer","Support Engineer","Tech Lead","Tech Lead"] | ["Head Human Resources","Head Human Resources","Head Human Resources","Reporting Analyst","Reporting Analyst","Reporting Analyst","Support Engineer","Support Engineer","Support Engineer","Tech Lead","Tech Lead","Tech Lead"] +; + + + +mvAppendStringsWhere +required_capability: fn_mv_append + +FROM employees +| EVAL two_jobs = mv_append(mv_sort(job_positions), mv_sort(job_positions)) +| WHERE emp_no == 10004 AND mv_slice(mv_append(mv_sort(job_positions), mv_sort(job_positions)), 6, 6) == "Support Engineer" +| KEEP emp_no, two_jobs +; + +emp_no:integer | two_jobs:keyword +10004 | ["Head Human Resources","Reporting Analyst","Support Engineer","Tech Lead","Head Human Resources","Reporting Analyst","Support Engineer","Tech Lead"] +; + +mvAppendNullFields +required_capability: fn_mv_append + +FROM employees +| WHERE emp_no == 10005 +| EVAL x = mv_append(first_name, job_positions), y = mv_append(job_positions, first_name), z = mv_append(job_positions, job_positions) +| keep emp_no, first_name, job_positions, x, y, z +; + +emp_no:integer | first_name:keyword | job_positions:keyword | x:keyword | y:keyword | z:keyword +10005 | Kyoichi | null | null | null | null +; + + base64Encode#[skip:-8.13.99,reason:new base64 function added in 8.14] required_capability: base64_decode_encode diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec index 3b6c41f883018..eb0d6d75a7d07 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec @@ -371,6 +371,21 @@ version:version | name:keyword 5.2.9 | mmmmm ; + +mvAppend +required_capability: fn_mv_append + +ROW a = to_version("1.2.0"), x1 = to_version("0.0.1"), x2 = to_version("1.0.0") +| EVAL b = mv_append(x1, x2) +| EVAL aa = mv_append(a, a), bb = mv_append(b, b), ab = mv_append(a, b), abb = mv_append(mv_append(a, b), b) +| KEEP a, b, aa, bb, ab, abb +; + +a:version | b:version | aa:version | bb:version | ab:version | abb:version +1.2.0 | [0.0.1, 1.0.0] | [1.2.0, 1.2.0] | [0.0.1, 1.0.0, 0.0.1, 1.0.0] | [1.2.0, 0.0.1, 1.0.0] | [1.2.0, 0.0.1, 1.0.0, 0.0.1, 1.0.0] +; + + implictCastingEqual required_capability: string_literal_auto_casting_extended from apps | where version == "1.2.3.4" | sort name | keep name, version; diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java new file mode 100644 index 0000000000000..c59d915fa2fe4 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java @@ -0,0 +1,102 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvAppend}. + * This class is generated. Do not edit it. + */ +public final class MvAppendBooleanEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator field1; + + private final EvalOperator.ExpressionEvaluator field2; + + private final DriverContext driverContext; + + public MvAppendBooleanEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + this.warnings = new Warnings(source); + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (BooleanBlock field1Block = (BooleanBlock) field1.eval(page)) { + try (BooleanBlock field2Block = (BooleanBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public BooleanBlock eval(int positionCount, BooleanBlock field1Block, BooleanBlock field2Block) { + try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; + if (!field1Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (!field2Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (allBlocksAreNulls) { + result.appendNull(); + continue position; + } + MvAppend.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvAppendBooleanEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field1; + + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvAppendBooleanEvaluator get(DriverContext context) { + return new MvAppendBooleanEvaluator(source, field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvAppendBooleanEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java new file mode 100644 index 0000000000000..c650a803a6dd4 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java @@ -0,0 +1,103 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvAppend}. + * This class is generated. Do not edit it. + */ +public final class MvAppendBytesRefEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator field1; + + private final EvalOperator.ExpressionEvaluator field2; + + private final DriverContext driverContext; + + public MvAppendBytesRefEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + this.warnings = new Warnings(source); + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock field1Block = (BytesRefBlock) field1.eval(page)) { + try (BytesRefBlock field2Block = (BytesRefBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public BytesRefBlock eval(int positionCount, BytesRefBlock field1Block, + BytesRefBlock field2Block) { + try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; + if (!field1Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (!field2Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (allBlocksAreNulls) { + result.appendNull(); + continue position; + } + MvAppend.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvAppendBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field1; + + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvAppendBytesRefEvaluator get(DriverContext context) { + return new MvAppendBytesRefEvaluator(source, field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvAppendBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java new file mode 100644 index 0000000000000..07108c0522b8d --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java @@ -0,0 +1,102 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvAppend}. + * This class is generated. Do not edit it. + */ +public final class MvAppendDoubleEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator field1; + + private final EvalOperator.ExpressionEvaluator field2; + + private final DriverContext driverContext; + + public MvAppendDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + this.warnings = new Warnings(source); + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (DoubleBlock field1Block = (DoubleBlock) field1.eval(page)) { + try (DoubleBlock field2Block = (DoubleBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public DoubleBlock eval(int positionCount, DoubleBlock field1Block, DoubleBlock field2Block) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; + if (!field1Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (!field2Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (allBlocksAreNulls) { + result.appendNull(); + continue position; + } + MvAppend.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvAppendDoubleEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field1; + + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvAppendDoubleEvaluator get(DriverContext context) { + return new MvAppendDoubleEvaluator(source, field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvAppendDoubleEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java new file mode 100644 index 0000000000000..99c5c9798a0aa --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java @@ -0,0 +1,102 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvAppend}. + * This class is generated. Do not edit it. + */ +public final class MvAppendIntEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator field1; + + private final EvalOperator.ExpressionEvaluator field2; + + private final DriverContext driverContext; + + public MvAppendIntEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + this.warnings = new Warnings(source); + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (IntBlock field1Block = (IntBlock) field1.eval(page)) { + try (IntBlock field2Block = (IntBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public IntBlock eval(int positionCount, IntBlock field1Block, IntBlock field2Block) { + try(IntBlock.Builder result = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; + if (!field1Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (!field2Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (allBlocksAreNulls) { + result.appendNull(); + continue position; + } + MvAppend.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvAppendIntEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field1; + + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvAppendIntEvaluator get(DriverContext context) { + return new MvAppendIntEvaluator(source, field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvAppendIntEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java new file mode 100644 index 0000000000000..340a9747b14aa --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java @@ -0,0 +1,102 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvAppend}. + * This class is generated. Do not edit it. + */ +public final class MvAppendLongEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator field1; + + private final EvalOperator.ExpressionEvaluator field2; + + private final DriverContext driverContext; + + public MvAppendLongEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, + EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + this.warnings = new Warnings(source); + this.field1 = field1; + this.field2 = field2; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock field1Block = (LongBlock) field1.eval(page)) { + try (LongBlock field2Block = (LongBlock) field2.eval(page)) { + return eval(page.getPositionCount(), field1Block, field2Block); + } + } + } + + public LongBlock eval(int positionCount, LongBlock field1Block, LongBlock field2Block) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; + if (!field1Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (!field2Block.isNull(p)) { + allBlocksAreNulls = false; + } + if (allBlocksAreNulls) { + result.appendNull(); + continue position; + } + MvAppend.process(result, p, field1Block, field2Block); + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvAppendLongEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(field1, field2); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field1; + + private final EvalOperator.ExpressionEvaluator.Factory field2; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, + EvalOperator.ExpressionEvaluator.Factory field2) { + this.source = source; + this.field1 = field1; + this.field2 = field2; + } + + @Override + public MvAppendLongEvaluator get(DriverContext context) { + return new MvAppendLongEvaluator(source, field1.get(context), field2.get(context), context); + } + + @Override + public String toString() { + return "MvAppendLongEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 675b99c61bfbe..61134da3ecb85 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -27,6 +27,11 @@ public class EsqlCapabilities { */ private static final String FN_CBRT = "fn_cbrt"; + /** + * Support for {@code MV_APPEND} function. #107001 + */ + private static final String FN_MV_APPEND = "fn_mv_append"; + /** * Support for function {@code IP_PREFIX}. */ @@ -61,6 +66,7 @@ private static Set capabilities() { caps.add(FN_SUBSTRING_EMPTY_NULL); caps.add(ST_CENTROID_AGG_OPTIMIZED); caps.add(METADATA_IGNORED_FIELD); + caps.add(FN_MV_APPEND); if (Build.current().isSnapshot()) { caps.add(LOOKUP); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index a8ab961fc201a..af287399a8ddb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -76,6 +76,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tan; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tanh; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tau; +import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppend; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAvg; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvConcat; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvCount; @@ -280,6 +281,7 @@ private FunctionDefinition[][] functions() { def(ToVersion.class, ToVersion::new, "to_version", "to_ver"), }, // multivalue functions new FunctionDefinition[] { + def(MvAppend.class, MvAppend::new, "mv_append"), def(MvAvg.class, MvAvg::new, "mv_avg"), def(MvConcat.class, MvConcat::new, "mv_concat"), def(MvCount.class, MvCount::new, "mv_count"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java new file mode 100644 index 0000000000000..21da054122aeb --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java @@ -0,0 +1,285 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Nullability; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.DataTypes; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.planner.PlannerUtils; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; + +/** + * Appends values to a multi-value + */ +public class MvAppend extends EsqlScalarFunction implements EvaluatorMapper { + private final Expression field1, field2; + private DataType dataType; + + @FunctionInfo( + returnType = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "version" }, + description = "Concatenates values of two multi-value fields." + ) + public MvAppend( + Source source, + @Param( + name = "field1", + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "version" } + ) Expression field1, + @Param( + name = "field2", + type = { + "boolean", + "cartesian_point", + "cartesian_shape", + "date", + "double", + "geo_point", + "geo_shape", + "integer", + "ip", + "keyword", + "long", + "text", + "version" } + ) Expression field2 + ) { + super(source, Arrays.asList(field1, field2)); + this.field1 = field1; + this.field2 = field2; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + TypeResolution resolution = isType(field1, EsqlDataTypes::isRepresentable, sourceText(), FIRST, "representable"); + if (resolution.unresolved()) { + return resolution; + } + dataType = field1.dataType(); + if (dataType == DataTypes.NULL) { + dataType = field2.dataType(); + return isType(field2, EsqlDataTypes::isRepresentable, sourceText(), SECOND, "representable"); + } + return isType(field2, t -> t == dataType, sourceText(), SECOND, dataType.typeName()); + } + + @Override + public boolean foldable() { + return field1.foldable() && field2.foldable(); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator( + Function toEvaluator + ) { + return switch (PlannerUtils.toElementType(dataType())) { + case BOOLEAN -> new MvAppendBooleanEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); + case BYTES_REF -> new MvAppendBytesRefEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); + case DOUBLE -> new MvAppendDoubleEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); + case INT -> new MvAppendIntEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); + case LONG -> new MvAppendLongEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); + case NULL -> EvalOperator.CONSTANT_NULL_FACTORY; + default -> throw EsqlIllegalArgumentException.illegalDataType(dataType); + }; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new MvAppend(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, MvAppend::new, field1, field2); + } + + @Override + public DataType dataType() { + if (dataType == null) { + resolveType(); + } + return dataType; + } + + @Override + public int hashCode() { + return Objects.hash(field1, field2); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + MvAppend other = (MvAppend) obj; + return Objects.equals(other.field1, field1) && Objects.equals(other.field2, field2); + } + + @Evaluator(extraName = "Int") + static void process(IntBlock.Builder builder, int position, IntBlock field1, IntBlock field2) { + int count1 = field1.getValueCount(position); + int count2 = field2.getValueCount(position); + if (count1 == 0 || count2 == 0) { + builder.appendNull(); + } else { + builder.beginPositionEntry(); + int first1 = field1.getFirstValueIndex(position); + int first2 = field2.getFirstValueIndex(position); + for (int i = 0; i < count1; i++) { + builder.appendInt(field1.getInt(first1 + i)); + } + for (int i = 0; i < count2; i++) { + builder.appendInt(field2.getInt(first2 + i)); + } + builder.endPositionEntry(); + } + + } + + @Evaluator(extraName = "Boolean") + static void process(BooleanBlock.Builder builder, int position, BooleanBlock field1, BooleanBlock field2) { + int count1 = field1.getValueCount(position); + int count2 = field2.getValueCount(position); + if (count1 == 0 || count2 == 0) { + builder.appendNull(); + } else { + int first1 = field1.getFirstValueIndex(position); + int first2 = field2.getFirstValueIndex(position); + builder.beginPositionEntry(); + for (int i = 0; i < count1; i++) { + builder.appendBoolean(field1.getBoolean(first1 + i)); + } + for (int i = 0; i < count2; i++) { + builder.appendBoolean(field2.getBoolean(first2 + i)); + } + builder.endPositionEntry(); + } + + } + + @Evaluator(extraName = "Long") + static void process(LongBlock.Builder builder, int position, LongBlock field1, LongBlock field2) { + int count1 = field1.getValueCount(position); + int count2 = field2.getValueCount(position); + if (count1 == 0 || count2 == 0) { + builder.appendNull(); + } else { + int first1 = field1.getFirstValueIndex(position); + int first2 = field2.getFirstValueIndex(position); + builder.beginPositionEntry(); + for (int i = 0; i < count1; i++) { + builder.appendLong(field1.getLong(first1 + i)); + } + for (int i = 0; i < count2; i++) { + builder.appendLong(field2.getLong(first2 + i)); + } + builder.endPositionEntry(); + } + } + + @Evaluator(extraName = "Double") + static void process(DoubleBlock.Builder builder, int position, DoubleBlock field1, DoubleBlock field2) { + int count1 = field1.getValueCount(position); + int count2 = field2.getValueCount(position); + if (count1 == 0 || count2 == 0) { + builder.appendNull(); + } else { + int first1 = field1.getFirstValueIndex(position); + int first2 = field2.getFirstValueIndex(position); + builder.beginPositionEntry(); + for (int i = 0; i < count1; i++) { + builder.appendDouble(field1.getDouble(first1 + i)); + } + for (int i = 0; i < count2; i++) { + builder.appendDouble(field2.getDouble(first2 + i)); + } + builder.endPositionEntry(); + } + + } + + @Evaluator(extraName = "BytesRef") + static void process(BytesRefBlock.Builder builder, int position, BytesRefBlock field1, BytesRefBlock field2) { + int count1 = field1.getValueCount(position); + int count2 = field2.getValueCount(position); + if (count1 == 0 || count2 == 0) { + builder.appendNull(); + } else { + int first1 = field1.getFirstValueIndex(position); + int first2 = field2.getFirstValueIndex(position); + builder.beginPositionEntry(); + BytesRef spare = new BytesRef(); + for (int i = 0; i < count1; i++) { + builder.appendBytesRef(field1.getBytesRef(first1 + i, spare)); + } + for (int i = 0; i < count2; i++) { + builder.appendBytesRef(field2.getBytesRef(first2 + i, spare)); + } + builder.endPositionEntry(); + } + } + + @Override + public Nullability nullable() { + return Nullability.TRUE; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index f605f898366e1..6b5e91376a335 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -118,6 +118,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tanh; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tau; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.AbstractMultivalueFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppend; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAvg; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvConcat; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvCount; @@ -425,6 +426,7 @@ public static List namedTypeEntries() { of(AggregateFunction.class, Sum.class, PlanNamedTypes::writeAggFunction, PlanNamedTypes::readAggFunction), of(AggregateFunction.class, Values.class, PlanNamedTypes::writeAggFunction, PlanNamedTypes::readAggFunction), // Multivalue functions + of(ScalarFunction.class, MvAppend.class, PlanNamedTypes::writeMvAppend, PlanNamedTypes::readMvAppend), of(ScalarFunction.class, MvAvg.class, PlanNamedTypes::writeMvFunction, PlanNamedTypes::readMvFunction), of(ScalarFunction.class, MvCount.class, PlanNamedTypes::writeMvFunction, PlanNamedTypes::readMvFunction), of(ScalarFunction.class, MvConcat.class, PlanNamedTypes::writeMvConcat, PlanNamedTypes::readMvConcat), @@ -1857,4 +1859,16 @@ static void writeMvZip(PlanStreamOutput out, MvZip fn) throws IOException { out.writeExpression(fields.get(1)); out.writeOptionalWriteable(fields.size() == 3 ? o -> out.writeExpression(fields.get(2)) : null); } + + static MvAppend readMvAppend(PlanStreamInput in) throws IOException { + return new MvAppend(Source.readFrom(in), in.readExpression(), in.readExpression()); + } + + static void writeMvAppend(PlanStreamOutput out, MvAppend fn) throws IOException { + Source.EMPTY.writeTo(out); + List fields = fn.children(); + assert fields.size() == 2; + out.writeExpression(fields.get(0)); + out.writeExpression(fields.get(1)); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 5f2b4290f48f3..2d9b52f1b433f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -62,6 +62,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -522,7 +523,9 @@ public void testDropUnsupportedFieldExplicit() { "float", "foo_type", "int", + "ip", "keyword", + "long", "point", "shape", "some.ambiguous", @@ -566,7 +569,9 @@ public void testDropUnsupportedPattern() { "float", "foo_type", "int", + "ip", "keyword", + "long", "point", "shape", "some.ambiguous", @@ -779,7 +784,9 @@ public void testDropSupportedDottedField() { "float", "foo_type", "int", + "ip", "keyword", + "long", "point", "shape", "some.ambiguous", @@ -1878,6 +1885,40 @@ public void testInOnText() { """, "mapping-multi-field-variation.json", "text"); } + public void testMvAppendValidation() { + String[][] fields = { + { "bool", "boolean" }, + { "int", "integer" }, + { "unsigned_long", "unsigned_long" }, + { "float", "double" }, + { "text", "text" }, + { "keyword", "keyword" }, + { "date", "datetime" }, + { "point", "geo_point" }, + { "shape", "geo_shape" }, + { "long", "long" }, + { "ip", "ip" }, + { "version", "version" } }; + + Supplier supplier = () -> randomInt(fields.length - 1); + int first = supplier.get(); + int second = randomValueOtherThan(first, supplier); + + String signature = "mv_append(" + fields[first][0] + ", " + fields[second][0] + ")"; + verifyUnsupported( + " from test | eval " + signature, + "second argument of [" + + signature + + "] must be [" + + fields[first][1] + + "], found value [" + + fields[second][0] + + "] type [" + + fields[second][1] + + "]" + ); + } + public void testLookup() { var e = expectThrows(ParsingException.class, () -> analyze(""" FROM test diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java new file mode 100644 index 0000000000000..6d43a1087ad88 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java @@ -0,0 +1,296 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geo.ShapeTestUtils; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataTypes; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO; +import static org.hamcrest.Matchers.equalTo; + +public class MvAppendTests extends AbstractFunctionTestCase { + public MvAppendTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List suppliers = new ArrayList<>(); + booleans(suppliers); + ints(suppliers); + longs(suppliers); + doubles(suppliers); + bytesRefs(suppliers); + nulls(suppliers); + return parameterSuppliersFromTypedData(suppliers); + } + + @Override + protected Expression build(Source source, List args) { + return new MvAppend(source, args.get(0), args.get(1)); + } + + private static void booleans(List suppliers) { + suppliers.add(new TestCaseSupplier(List.of(DataTypes.BOOLEAN, DataTypes.BOOLEAN), () -> { + List field1 = randomList(1, 10, () -> randomBoolean()); + List field2 = randomList(1, 10, () -> randomBoolean()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.BOOLEAN, "field1"), + new TestCaseSupplier.TypedData(field2, DataTypes.BOOLEAN, "field2") + ), + "MvAppendBooleanEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.BOOLEAN, + equalTo(result) + ); + })); + } + + private static void ints(List suppliers) { + suppliers.add(new TestCaseSupplier(List.of(DataTypes.INTEGER, DataTypes.INTEGER), () -> { + List field1 = randomList(1, 10, () -> randomInt()); + List field2 = randomList(1, 10, () -> randomInt()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.INTEGER, "field1"), + new TestCaseSupplier.TypedData(field2, DataTypes.INTEGER, "field2") + ), + "MvAppendIntEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.INTEGER, + equalTo(result) + ); + })); + } + + private static void longs(List suppliers) { + suppliers.add(new TestCaseSupplier(List.of(DataTypes.LONG, DataTypes.LONG), () -> { + List field1 = randomList(1, 10, () -> randomLong()); + List field2 = randomList(1, 10, () -> randomLong()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.LONG, "field1"), + new TestCaseSupplier.TypedData(field2, DataTypes.LONG, "field2") + ), + "MvAppendLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.LONG, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataTypes.DATETIME, DataTypes.DATETIME), () -> { + List field1 = randomList(1, 10, () -> randomLong()); + List field2 = randomList(1, 10, () -> randomLong()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.DATETIME, "field1"), + new TestCaseSupplier.TypedData(field2, DataTypes.DATETIME, "field2") + ), + "MvAppendLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.DATETIME, + equalTo(result) + ); + })); + } + + private static void doubles(List suppliers) { + suppliers.add(new TestCaseSupplier(List.of(DataTypes.DOUBLE, DataTypes.DOUBLE), () -> { + List field1 = randomList(1, 10, () -> randomDouble()); + List field2 = randomList(1, 10, () -> randomDouble()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.DOUBLE, "field1"), + new TestCaseSupplier.TypedData(field2, DataTypes.DOUBLE, "field2") + ), + "MvAppendDoubleEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.DOUBLE, + equalTo(result) + ); + })); + } + + private static void bytesRefs(List suppliers) { + suppliers.add(new TestCaseSupplier(List.of(DataTypes.KEYWORD, DataTypes.KEYWORD), () -> { + List field1 = randomList(1, 10, () -> randomLiteral(DataTypes.KEYWORD).value()); + List field2 = randomList(1, 10, () -> randomLiteral(DataTypes.KEYWORD).value()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.KEYWORD, "field1"), + new TestCaseSupplier.TypedData(field2, DataTypes.KEYWORD, "field2") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataTypes.TEXT, DataTypes.TEXT), () -> { + List field1 = randomList(1, 10, () -> randomLiteral(DataTypes.TEXT).value()); + List field2 = randomList(1, 10, () -> randomLiteral(DataTypes.TEXT).value()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.TEXT, "field1"), + new TestCaseSupplier.TypedData(field2, DataTypes.TEXT, "field2") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.TEXT, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataTypes.IP, DataTypes.IP), () -> { + List field1 = randomList(1, 10, () -> randomLiteral(DataTypes.IP).value()); + List field2 = randomList(1, 10, () -> randomLiteral(DataTypes.IP).value()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.IP, "field"), + new TestCaseSupplier.TypedData(field2, DataTypes.IP, "field") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.IP, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataTypes.VERSION, DataTypes.VERSION), () -> { + List field1 = randomList(1, 10, () -> randomLiteral(DataTypes.VERSION).value()); + List field2 = randomList(1, 10, () -> randomLiteral(DataTypes.VERSION).value()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.VERSION, "field"), + new TestCaseSupplier.TypedData(field2, DataTypes.VERSION, "field") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.VERSION, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataTypes.GEO_POINT, DataTypes.GEO_POINT), () -> { + List field1 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomPoint()))); + List field2 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomPoint()))); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.GEO_POINT, "field1"), + new TestCaseSupplier.TypedData(field2, DataTypes.GEO_POINT, "field2") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.GEO_POINT, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_POINT), () -> { + List field1 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomPoint()))); + List field2 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomPoint()))); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.CARTESIAN_POINT, "field1"), + new TestCaseSupplier.TypedData(field2, DataTypes.CARTESIAN_POINT, "field2") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.CARTESIAN_POINT, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataTypes.GEO_SHAPE, DataTypes.GEO_SHAPE), () -> { + List field1 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean())))); + List field2 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean())))); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.GEO_SHAPE, "field1"), + new TestCaseSupplier.TypedData(field2, DataTypes.GEO_SHAPE, "field2") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.GEO_SHAPE, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataTypes.CARTESIAN_SHAPE, DataTypes.CARTESIAN_SHAPE), () -> { + List field1 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean())))); + List field2 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean())))); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.CARTESIAN_SHAPE, "field1"), + new TestCaseSupplier.TypedData(field2, DataTypes.CARTESIAN_SHAPE, "field2") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.CARTESIAN_SHAPE, + equalTo(result) + ); + })); + } + + private static void nulls(List suppliers) { + suppliers.add(new TestCaseSupplier(List.of(DataTypes.INTEGER, DataTypes.INTEGER), () -> { + List field2 = randomList(2, 10, () -> randomInt()); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(null, DataTypes.INTEGER, "field1"), + new TestCaseSupplier.TypedData(field2, DataTypes.INTEGER, "field2") + ), + "MvAppendIntEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.INTEGER, + equalTo(null) + ); + })); + suppliers.add(new TestCaseSupplier(List.of(DataTypes.INTEGER, DataTypes.INTEGER), () -> { + List field1 = randomList(2, 10, () -> randomInt()); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataTypes.INTEGER, "field1"), + new TestCaseSupplier.TypedData(null, DataTypes.INTEGER, "field2") + ), + "MvAppendIntEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataTypes.INTEGER, + equalTo(null) + ); + })); + } +}