diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java index 265bd52eabe83..916823fd91b61 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java @@ -89,7 +89,6 @@ public static void stopHttpServers() throws IOException { } public void testBuilderUsesDefaultSSLContext() throws Exception { - assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm()); final SSLContext defaultSSLContext = SSLContext.getDefault(); try { try (RestClient client = buildRestClient()) { @@ -97,10 +96,15 @@ public void testBuilderUsesDefaultSSLContext() throws Exception { client.performRequest(new Request("GET", "/")); fail("connection should have been rejected due to SSL handshake"); } catch (Exception e) { - assertThat(e, instanceOf(SSLHandshakeException.class)); + if (inFipsJvm()) { + // Bouncy Castle throw a different exception + assertThat(e, instanceOf(IOException.class)); + assertThat(e.getCause(), instanceOf(javax.net.ssl.SSLException.class)); + } else { + assertThat(e, instanceOf(SSLHandshakeException.class)); + } } } - SSLContext.setDefault(getSslContext()); try (RestClient client = buildRestClient()) { Response response = client.performRequest(new Request("GET", "/")); @@ -112,7 +116,6 @@ public void testBuilderUsesDefaultSSLContext() throws Exception { } public void testBuilderSetsThreadName() throws Exception { - assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm()); final SSLContext defaultSSLContext = SSLContext.getDefault(); try { SSLContext.setDefault(getSslContext()); diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 285ac82a63aef..5f4beb0e7c9a0 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -70,7 +70,7 @@ static List systemJvmOptions(Settings nodeSettings, final Map s).toList(); } @@ -148,12 +148,9 @@ private static Stream maybeWorkaroundG1Bug() { return Stream.of(); } - private static Stream maybeAllowSecurityManager(boolean useEntitlements) { - if (useEntitlements == false) { - // Will become conditional on useEntitlements once entitlements can run without SM - return Stream.of("-Djava.security.manager=allow"); - } - return Stream.of(); + private static Stream maybeAllowSecurityManager() { + // Will become conditional on useEntitlements once entitlements can run without SM + return Stream.of("-Djava.security.manager=allow"); } private static Stream maybeAttachEntitlementAgent(boolean useEntitlements) { diff --git a/docs/changelog/118938.yaml b/docs/changelog/118938.yaml new file mode 100644 index 0000000000000..395da7912fd4b --- /dev/null +++ b/docs/changelog/118938.yaml @@ -0,0 +1,5 @@ +pr: 118938 +summary: Hash functions +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/119250.yaml b/docs/changelog/119250.yaml new file mode 100644 index 0000000000000..9db36957d8050 --- /dev/null +++ b/docs/changelog/119250.yaml @@ -0,0 +1,5 @@ +pr: 119250 +summary: Add rest endpoint for `create_from_source_index` +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/119542.yaml b/docs/changelog/119542.yaml new file mode 100644 index 0000000000000..aaf26c7dc4b0f --- /dev/null +++ b/docs/changelog/119542.yaml @@ -0,0 +1,5 @@ +pr: 119542 +summary: Wait while index is blocked +area: Transform +type: enhancement +issues: [] diff --git a/docs/changelog/119564.yaml b/docs/changelog/119564.yaml new file mode 100644 index 0000000000000..175eff75c8218 --- /dev/null +++ b/docs/changelog/119564.yaml @@ -0,0 +1,5 @@ +pr: 119564 +summary: Http stream activity tracker and exceptions handling +area: Network +type: enhancement +issues: [] diff --git a/docs/reference/esql/functions/description/md5.asciidoc b/docs/reference/esql/functions/description/md5.asciidoc new file mode 100644 index 0000000000000..2ad847c0ce0e3 --- /dev/null +++ b/docs/reference/esql/functions/description/md5.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Computes the MD5 hash of the input. diff --git a/docs/reference/esql/functions/description/sha1.asciidoc b/docs/reference/esql/functions/description/sha1.asciidoc new file mode 100644 index 0000000000000..5bc29f86cc591 --- /dev/null +++ b/docs/reference/esql/functions/description/sha1.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Computes the SHA1 hash of the input. diff --git a/docs/reference/esql/functions/description/sha256.asciidoc b/docs/reference/esql/functions/description/sha256.asciidoc new file mode 100644 index 0000000000000..b2a7ef01e1069 --- /dev/null +++ b/docs/reference/esql/functions/description/sha256.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Computes the SHA256 hash of the input. diff --git a/docs/reference/esql/functions/examples/hash.asciidoc b/docs/reference/esql/functions/examples/hash.asciidoc new file mode 100644 index 0000000000000..492e466eb395e --- /dev/null +++ b/docs/reference/esql/functions/examples/hash.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/hash.csv-spec[tag=hash] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/hash.csv-spec[tag=hash-result] +|=== + diff --git a/docs/reference/esql/functions/examples/md5.asciidoc b/docs/reference/esql/functions/examples/md5.asciidoc new file mode 100644 index 0000000000000..0b43bc5b791c9 --- /dev/null +++ b/docs/reference/esql/functions/examples/md5.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/hash.csv-spec[tag=md5] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/hash.csv-spec[tag=md5-result] +|=== + diff --git a/docs/reference/esql/functions/examples/sha1.asciidoc b/docs/reference/esql/functions/examples/sha1.asciidoc new file mode 100644 index 0000000000000..77786431a738a --- /dev/null +++ b/docs/reference/esql/functions/examples/sha1.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/hash.csv-spec[tag=sha1] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/hash.csv-spec[tag=sha1-result] +|=== + diff --git a/docs/reference/esql/functions/examples/sha256.asciidoc b/docs/reference/esql/functions/examples/sha256.asciidoc new file mode 100644 index 0000000000000..801c36d8effc8 --- /dev/null +++ b/docs/reference/esql/functions/examples/sha256.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/hash.csv-spec[tag=sha256] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/hash.csv-spec[tag=sha256-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/hash.json b/docs/reference/esql/functions/kibana/definition/hash.json index 17a60cf45acfe..dbf4a2542afc5 100644 --- a/docs/reference/esql/functions/kibana/definition/hash.json +++ b/docs/reference/esql/functions/kibana/definition/hash.json @@ -77,6 +77,9 @@ "returnType" : "keyword" } ], + "examples" : [ + "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL md5 = hash(\"md5\", message), sha256 = hash(\"sha256\", message) \n| KEEP message, md5, sha256;" + ], "preview" : false, "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/md5.json b/docs/reference/esql/functions/kibana/definition/md5.json new file mode 100644 index 0000000000000..4d3a88e123ff4 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/md5.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "md5", + "description" : "Computes the MD5 hash of the input.", + "signatures" : [ + { + "params" : [ + { + "name" : "input", + "type" : "keyword", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "input", + "type" : "text", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "examples" : [ + "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL md5 = md5(message)\n| KEEP message, md5;" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/sha1.json b/docs/reference/esql/functions/kibana/definition/sha1.json new file mode 100644 index 0000000000000..a6abb31368bb3 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/sha1.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "sha1", + "description" : "Computes the SHA1 hash of the input.", + "signatures" : [ + { + "params" : [ + { + "name" : "input", + "type" : "keyword", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "input", + "type" : "text", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "examples" : [ + "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL sha1 = sha1(message)\n| KEEP message, sha1;" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/sha256.json b/docs/reference/esql/functions/kibana/definition/sha256.json new file mode 100644 index 0000000000000..700425d485b61 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/sha256.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "sha256", + "description" : "Computes the SHA256 hash of the input.", + "signatures" : [ + { + "params" : [ + { + "name" : "input", + "type" : "keyword", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "input", + "type" : "text", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "examples" : [ + "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL sha256 = sha256(message)\n| KEEP message, sha256;" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/docs/hash.md b/docs/reference/esql/functions/kibana/docs/hash.md index 9826e80ec5bec..4e937778ba67a 100644 --- a/docs/reference/esql/functions/kibana/docs/hash.md +++ b/docs/reference/esql/functions/kibana/docs/hash.md @@ -5,3 +5,9 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ ### HASH Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512. +``` +FROM sample_data +| WHERE message != "Connection error" +| EVAL md5 = hash("md5", message), sha256 = hash("sha256", message) +| KEEP message, md5, sha256; +``` diff --git a/docs/reference/esql/functions/kibana/docs/md5.md b/docs/reference/esql/functions/kibana/docs/md5.md new file mode 100644 index 0000000000000..aacb8a3960165 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/md5.md @@ -0,0 +1,13 @@ + + +### MD5 +Computes the MD5 hash of the input. + +``` +FROM sample_data +| WHERE message != "Connection error" +| EVAL md5 = md5(message) +| KEEP message, md5; +``` diff --git a/docs/reference/esql/functions/kibana/docs/sha1.md b/docs/reference/esql/functions/kibana/docs/sha1.md new file mode 100644 index 0000000000000..a940aa133f06e --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/sha1.md @@ -0,0 +1,13 @@ + + +### SHA1 +Computes the SHA1 hash of the input. + +``` +FROM sample_data +| WHERE message != "Connection error" +| EVAL sha1 = sha1(message) +| KEEP message, sha1; +``` diff --git a/docs/reference/esql/functions/kibana/docs/sha256.md b/docs/reference/esql/functions/kibana/docs/sha256.md new file mode 100644 index 0000000000000..fbe576c7c20d6 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/sha256.md @@ -0,0 +1,13 @@ + + +### SHA256 +Computes the SHA256 hash of the input. + +``` +FROM sample_data +| WHERE message != "Connection error" +| EVAL sha256 = sha256(message) +| KEEP message, sha256; +``` diff --git a/docs/reference/esql/functions/layout/hash.asciidoc b/docs/reference/esql/functions/layout/hash.asciidoc index 27c55ada6319b..daf7fbf1170b2 100644 --- a/docs/reference/esql/functions/layout/hash.asciidoc +++ b/docs/reference/esql/functions/layout/hash.asciidoc @@ -12,3 +12,4 @@ image::esql/functions/signature/hash.svg[Embedded,opts=inline] include::../parameters/hash.asciidoc[] include::../description/hash.asciidoc[] include::../types/hash.asciidoc[] +include::../examples/hash.asciidoc[] diff --git a/docs/reference/esql/functions/layout/md5.asciidoc b/docs/reference/esql/functions/layout/md5.asciidoc new file mode 100644 index 0000000000000..82d3031d6bdfd --- /dev/null +++ b/docs/reference/esql/functions/layout/md5.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-md5]] +=== `MD5` + +*Syntax* + +[.text-center] +image::esql/functions/signature/md5.svg[Embedded,opts=inline] + +include::../parameters/md5.asciidoc[] +include::../description/md5.asciidoc[] +include::../types/md5.asciidoc[] +include::../examples/md5.asciidoc[] diff --git a/docs/reference/esql/functions/layout/sha1.asciidoc b/docs/reference/esql/functions/layout/sha1.asciidoc new file mode 100644 index 0000000000000..23e1e0e9ac2ab --- /dev/null +++ b/docs/reference/esql/functions/layout/sha1.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-sha1]] +=== `SHA1` + +*Syntax* + +[.text-center] +image::esql/functions/signature/sha1.svg[Embedded,opts=inline] + +include::../parameters/sha1.asciidoc[] +include::../description/sha1.asciidoc[] +include::../types/sha1.asciidoc[] +include::../examples/sha1.asciidoc[] diff --git a/docs/reference/esql/functions/layout/sha256.asciidoc b/docs/reference/esql/functions/layout/sha256.asciidoc new file mode 100644 index 0000000000000..d36a1345271f5 --- /dev/null +++ b/docs/reference/esql/functions/layout/sha256.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-sha256]] +=== `SHA256` + +*Syntax* + +[.text-center] +image::esql/functions/signature/sha256.svg[Embedded,opts=inline] + +include::../parameters/sha256.asciidoc[] +include::../description/sha256.asciidoc[] +include::../types/sha256.asciidoc[] +include::../examples/sha256.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/md5.asciidoc b/docs/reference/esql/functions/parameters/md5.asciidoc new file mode 100644 index 0000000000000..99eba4dc2cb3d --- /dev/null +++ b/docs/reference/esql/functions/parameters/md5.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`input`:: +Input to hash. diff --git a/docs/reference/esql/functions/parameters/sha1.asciidoc b/docs/reference/esql/functions/parameters/sha1.asciidoc new file mode 100644 index 0000000000000..99eba4dc2cb3d --- /dev/null +++ b/docs/reference/esql/functions/parameters/sha1.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`input`:: +Input to hash. diff --git a/docs/reference/esql/functions/parameters/sha256.asciidoc b/docs/reference/esql/functions/parameters/sha256.asciidoc new file mode 100644 index 0000000000000..99eba4dc2cb3d --- /dev/null +++ b/docs/reference/esql/functions/parameters/sha256.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`input`:: +Input to hash. diff --git a/docs/reference/esql/functions/signature/md5.svg b/docs/reference/esql/functions/signature/md5.svg new file mode 100644 index 0000000000000..419af764a212e --- /dev/null +++ b/docs/reference/esql/functions/signature/md5.svg @@ -0,0 +1 @@ +MD5(input) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/sha1.svg b/docs/reference/esql/functions/signature/sha1.svg new file mode 100644 index 0000000000000..bab03a7eb88c8 --- /dev/null +++ b/docs/reference/esql/functions/signature/sha1.svg @@ -0,0 +1 @@ +SHA1(input) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/sha256.svg b/docs/reference/esql/functions/signature/sha256.svg new file mode 100644 index 0000000000000..b77126bbefbd8 --- /dev/null +++ b/docs/reference/esql/functions/signature/sha256.svg @@ -0,0 +1 @@ +SHA256(input) \ No newline at end of file diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc index da9580a55151a..dd10e4c77581e 100644 --- a/docs/reference/esql/functions/string-functions.asciidoc +++ b/docs/reference/esql/functions/string-functions.asciidoc @@ -18,11 +18,14 @@ * <> * <> * <> +* <> * <> * <> * <> * <> * <> +* <> +* <> * <> * <> * <> @@ -43,11 +46,14 @@ include::layout/left.asciidoc[] include::layout/length.asciidoc[] include::layout/locate.asciidoc[] include::layout/ltrim.asciidoc[] +include::layout/md5.asciidoc[] include::layout/repeat.asciidoc[] include::layout/replace.asciidoc[] include::layout/reverse.asciidoc[] include::layout/right.asciidoc[] include::layout/rtrim.asciidoc[] +include::layout/sha1.asciidoc[] +include::layout/sha256.asciidoc[] include::layout/space.asciidoc[] include::layout/split.asciidoc[] include::layout/starts_with.asciidoc[] diff --git a/docs/reference/esql/functions/types/md5.asciidoc b/docs/reference/esql/functions/types/md5.asciidoc new file mode 100644 index 0000000000000..049a553397bbd --- /dev/null +++ b/docs/reference/esql/functions/types/md5.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +input | result +keyword | keyword +text | keyword +|=== diff --git a/docs/reference/esql/functions/types/sha1.asciidoc b/docs/reference/esql/functions/types/sha1.asciidoc new file mode 100644 index 0000000000000..049a553397bbd --- /dev/null +++ b/docs/reference/esql/functions/types/sha1.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +input | result +keyword | keyword +text | keyword +|=== diff --git a/docs/reference/esql/functions/types/sha256.asciidoc b/docs/reference/esql/functions/types/sha256.asciidoc new file mode 100644 index 0000000000000..049a553397bbd --- /dev/null +++ b/docs/reference/esql/functions/types/sha256.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +input | result +keyword | keyword +text | keyword +|=== diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexRestClientSslTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexRestClientSslTests.java index 766c3ff695f84..bcc6177f8363c 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexRestClientSslTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexRestClientSslTests.java @@ -113,14 +113,20 @@ private static SSLContext buildServerSslContext() throws Exception { } public void testClientFailsWithUntrustedCertificate() throws IOException { - assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm()); final List threads = new ArrayList<>(); final Settings.Builder builder = Settings.builder().put("path.home", createTempDir()); final Settings settings = builder.build(); final Environment environment = TestEnvironment.newEnvironment(settings); final ReindexSslConfig ssl = new ReindexSslConfig(settings, environment, mock(ResourceWatcherService.class)); try (RestClient client = Reindexer.buildRestClient(getRemoteInfo(), ssl, 1L, threads)) { - expectThrows(SSLHandshakeException.class, () -> client.performRequest(new Request("GET", "/"))); + if (inFipsJvm()) { + // Bouncy Castle throws a different exception + IOException exception = expectThrows(IOException.class, () -> client.performRequest(new Request("GET", "/"))); + assertThat(exception.getCause(), Matchers.instanceOf(javax.net.ssl.SSLException.class)); + } else { + expectThrows(SSLHandshakeException.class, () -> client.performRequest(new Request("GET", "/"))); + + } } } diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java index 4bb27af4bd0f5..ab2fb41d5a22b 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java @@ -594,7 +594,7 @@ record Ctx(String testName, String nodeName, Bootstrap clientBootstrap, Channel @Override public void close() throws Exception { safeGet(clientChannel.close()); - safeGet(clientBootstrap.config().group().shutdownGracefully()); + safeGet(clientBootstrap.config().group().shutdownGracefully(0, 0, TimeUnit.SECONDS)); clientRespQueue.forEach(o -> { if (o instanceof FullHttpResponse resp) resp.release(); }); for (var opaqueId : ControlServerRequestPlugin.handlers.keySet()) { if (opaqueId.startsWith(testName)) { diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java index 1a391a05add58..4809f1a1a275b 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java @@ -139,7 +139,8 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } else { var contentStream = new Netty4HttpRequestBodyStream( ctx.channel(), - serverTransport.getThreadPool().getThreadContext() + serverTransport.getThreadPool().getThreadContext(), + activityTracker ); currentRequestStream = contentStream; netty4HttpRequest = new Netty4HttpRequest(readSequence++, request, contentStream); diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java index ac3e3aecf97b9..0902e707b706e 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStream.java @@ -16,6 +16,7 @@ import io.netty.handler.codec.http.HttpContent; import io.netty.handler.codec.http.LastHttpContent; +import org.elasticsearch.common.network.ThreadWatchdog; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Releasables; import org.elasticsearch.http.HttpBody; @@ -36,6 +37,7 @@ public class Netty4HttpRequestBodyStream implements HttpBody.Stream { private final ChannelFutureListener closeListener = future -> doClose(); private final List tracingHandlers = new ArrayList<>(4); private final ThreadContext threadContext; + private final ThreadWatchdog.ActivityTracker activityTracker; private ByteBuf buf; private boolean requested = false; private boolean closing = false; @@ -46,10 +48,11 @@ public class Netty4HttpRequestBodyStream implements HttpBody.Stream { private volatile int bufSize = 0; private volatile boolean hasLast = false; - public Netty4HttpRequestBodyStream(Channel channel, ThreadContext threadContext) { + public Netty4HttpRequestBodyStream(Channel channel, ThreadContext threadContext, ThreadWatchdog.ActivityTracker activityTracker) { this.channel = channel; this.threadContext = threadContext; this.requestContext = threadContext.newStoredContext(); + this.activityTracker = activityTracker; Netty4Utils.addListener(channel.closeFuture(), closeListener); channel.config().setAutoRead(false); } @@ -76,15 +79,18 @@ public void next() { assert handler != null : "handler must be set before requesting next chunk"; requestContext = threadContext.newStoredContext(); channel.eventLoop().submit(() -> { + activityTracker.startActivity(); requested = true; - if (buf == null) { - channel.read(); - } else { - try { + try { + if (buf == null) { + channel.read(); + } else { send(); - } catch (Exception e) { - channel.pipeline().fireExceptionCaught(e); } + } catch (Throwable e) { + channel.pipeline().fireExceptionCaught(e); + } finally { + activityTracker.stopActivity(); } }); } diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java index d456bbecfbd20..7492737d4f877 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestBodyStreamTests.java @@ -11,6 +11,8 @@ import io.netty.buffer.Unpooled; import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.DefaultEventLoop; import io.netty.channel.SimpleChannelInboundHandler; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http.DefaultHttpContent; @@ -19,6 +21,7 @@ import io.netty.handler.flow.FlowControlHandler; import org.elasticsearch.common.bytes.ReleasableBytesReference; +import org.elasticsearch.common.network.ThreadWatchdog; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.http.HttpBody; @@ -27,6 +30,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -35,17 +40,18 @@ public class Netty4HttpRequestBodyStreamTests extends ESTestCase { + static HttpBody.ChunkHandler discardHandler = (chunk, isLast) -> chunk.close(); private final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); private EmbeddedChannel channel; private Netty4HttpRequestBodyStream stream; - static HttpBody.ChunkHandler discardHandler = (chunk, isLast) -> chunk.close(); + private ThreadWatchdog.ActivityTracker activityTracker; @Override public void setUp() throws Exception { super.setUp(); channel = new EmbeddedChannel(); - threadContext.putHeader("header1", "value1"); - stream = new Netty4HttpRequestBodyStream(channel, threadContext); + activityTracker = new ThreadWatchdog.ActivityTracker(); + stream = new Netty4HttpRequestBodyStream(channel, threadContext, activityTracker); stream.setHandler(discardHandler); // set default handler, each test might override one channel.pipeline().addLast(new SimpleChannelInboundHandler(false) { @Override @@ -128,57 +134,112 @@ public void testReadFromChannel() { } public void testReadFromHasCorrectThreadContext() throws InterruptedException { - var gotLast = new AtomicBoolean(false); AtomicReference> headers = new AtomicReference<>(); - stream.setHandler(new HttpBody.ChunkHandler() { - @Override - public void onNext(ReleasableBytesReference chunk, boolean isLast) { - headers.set(threadContext.getHeaders()); - gotLast.set(isLast); - chunk.close(); - } - - @Override - public void close() { - headers.set(threadContext.getHeaders()); - } - }); - channel.pipeline().addFirst(new FlowControlHandler()); // block all incoming messages, need explicit channel.read() + var eventLoop = new DefaultEventLoop(); + var gotLast = new AtomicBoolean(false); var chunkSize = 1024; + threadContext.putHeader("header1", "value1"); + try { + // activity tracker requires stream execution in the same thread, setting up stream inside event-loop + eventLoop.submit(() -> { + channel = new EmbeddedChannel(); + stream = new Netty4HttpRequestBodyStream(channel, threadContext, new ThreadWatchdog.ActivityTracker()); + channel.pipeline().addLast(new SimpleChannelInboundHandler(false) { + @Override + protected void channelRead0(ChannelHandlerContext ctx, HttpContent msg) { + stream.handleNettyContent(msg); + } + }); + stream.setHandler(new HttpBody.ChunkHandler() { + @Override + public void onNext(ReleasableBytesReference chunk, boolean isLast) { + headers.set(threadContext.getHeaders()); + gotLast.set(isLast); + chunk.close(); + } + + @Override + public void close() { + headers.set(threadContext.getHeaders()); + } + }); + channel.pipeline().addFirst(new FlowControlHandler()); // block all incoming messages, need explicit channel.read() + }).await(); - channel.writeInbound(randomContent(chunkSize)); - channel.writeInbound(randomLastContent(chunkSize)); + channel.writeInbound(randomContent(chunkSize)); + channel.writeInbound(randomLastContent(chunkSize)); - threadContext.putHeader("header2", "value2"); - stream.next(); + threadContext.putHeader("header2", "value2"); + stream.next(); - Thread thread = new Thread(() -> channel.runPendingTasks()); - thread.start(); - thread.join(); + eventLoop.submit(() -> channel.runPendingTasks()).await(); + assertThat(headers.get(), hasEntry("header1", "value1")); + assertThat(headers.get(), hasEntry("header2", "value2")); - assertThat(headers.get(), hasEntry("header1", "value1")); - assertThat(headers.get(), hasEntry("header2", "value2")); + threadContext.putHeader("header3", "value3"); + stream.next(); - threadContext.putHeader("header3", "value3"); - stream.next(); + eventLoop.submit(() -> channel.runPendingTasks()).await(); + assertThat(headers.get(), hasEntry("header1", "value1")); + assertThat(headers.get(), hasEntry("header2", "value2")); + assertThat(headers.get(), hasEntry("header3", "value3")); - thread = new Thread(() -> channel.runPendingTasks()); - thread.start(); - thread.join(); + assertTrue("should receive last content", gotLast.get()); - assertThat(headers.get(), hasEntry("header1", "value1")); - assertThat(headers.get(), hasEntry("header2", "value2")); - assertThat(headers.get(), hasEntry("header3", "value3")); + headers.set(new HashMap<>()); - assertTrue("should receive last content", gotLast.get()); + stream.close(); + + assertThat(headers.get(), hasEntry("header1", "value1")); + assertThat(headers.get(), hasEntry("header2", "value2")); + assertThat(headers.get(), hasEntry("header3", "value3")); + } finally { + eventLoop.shutdownGracefully(0, 0, TimeUnit.SECONDS); + } + } - headers.set(new HashMap<>()); + public void testStreamNextActivityTracker() { + var t0 = activityTracker.get(); + var N = between(1, 10); + for (int i = 0; i < N; i++) { + channel.writeInbound(randomContent(1024)); + stream.next(); + channel.runPendingTasks(); + } + var t1 = activityTracker.get(); + assertEquals("stream#next() must trigger activity tracker: N*step=" + N + "*2=" + N * 2L + " times", t1, t0 + N * 2L); + } - stream.close(); + // ensure that we catch all exceptions and throw them into channel pipeline + public void testCatchExceptions() { + var gotExceptions = new CountDownLatch(3); // number of tests below + + channel.pipeline().addLast(new ChannelInboundHandlerAdapter() { + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + gotExceptions.countDown(); + } + }); + + // catch exception for not buffered chunk, will be thrown on channel.fireChannelRead() + stream.setHandler((a, b) -> { throw new RuntimeException(); }); + stream.next(); + channel.runPendingTasks(); + channel.writeInbound(randomContent(1)); + + // catch exception for buffered chunk, will be thrown from eventLoop.submit() + channel.writeInbound(randomContent(1)); + stream.next(); + channel.runPendingTasks(); + + // should catch OOM exceptions too, see DieWithDignity + // swallowing exceptions can result in dangling streams, hanging channels, and delayed shutdowns + stream.setHandler((a, b) -> { throw new OutOfMemoryError(); }); + channel.writeInbound(randomContent(1)); + stream.next(); + channel.runPendingTasks(); - assertThat(headers.get(), hasEntry("header1", "value1")); - assertThat(headers.get(), hasEntry("header2", "value2")); - assertThat(headers.get(), hasEntry("header3", "value3")); + safeAwait(gotExceptions); } HttpContent randomContent(int size, boolean isLast) { diff --git a/muted-tests.yml b/muted-tests.yml index 68d7994eae678..9ff91cc5027da 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -443,8 +443,32 @@ tests: - class: org.elasticsearch.upgrades.SearchStatesIT method: testCanMatch issue: https://github.com/elastic/elasticsearch/issues/118718 -- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT - method: test {union_types.MultiIndexIndirectUseOfUnionTypesInMvExpand SYNC} - issue: https://github.com/elastic/elasticsearch/issues/119584 -- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT - issue: https://github.com/elastic/elasticsearch/issues/119622 +- class: org.elasticsearch.xpack.restart.FullClusterRestartIT + method: testTransformLegacyTemplateCleanup {cluster=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/119395 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: "test {p0=search.vectors/110_knn_query_with_filter/PRE_FILTER: knn query with internal filter as pre-filter}" + issue: https://github.com/elastic/elasticsearch/issues/119804 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: "test {p0=search.vectors/110_knn_query_with_filter/PRE_FILTER: knn query with alias filter as pre-filter}" + issue: https://github.com/elastic/elasticsearch/issues/119805 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + issue: https://github.com/elastic/elasticsearch/issues/119806 +- class: org.elasticsearch.search.profile.dfs.DfsProfilerIT + method: testProfileDfs + issue: https://github.com/elastic/elasticsearch/issues/119711 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testCreateJobInCustomSharedIndexUpdatesMapping + issue: https://github.com/elastic/elasticsearch/issues/119660 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testUsage + issue: https://github.com/elastic/elasticsearch/issues/119659 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testOpenJob_GivenTimeout_Returns408 + issue: https://github.com/elastic/elasticsearch/issues/119810 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testGetJob_GivenJobExists + issue: https://github.com/elastic/elasticsearch/issues/119811 +- class: org.elasticsearch.xpack.restart.FullClusterRestartIT + method: testSlmPolicyAndStats {cluster=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/119393 diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java index 6636b39445964..8e06ec00faaff 100644 --- a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java +++ b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java @@ -22,12 +22,19 @@ import java.util.jar.JarEntry; import java.util.jar.JarFile; +import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES; +import static org.objectweb.asm.ClassWriter.COMPUTE_MAXS; + public class HdfsClassPatcher { static final Map> patchers = Map.of( "org/apache/hadoop/util/ShutdownHookManager.class", ShutdownHookManagerPatcher::new, "org/apache/hadoop/util/Shell.class", - ShellPatcher::new + ShellPatcher::new, + "org/apache/hadoop/security/UserGroupInformation.class", + SubjectGetSubjectPatcher::new, + "org/apache/hadoop/security/authentication/client/KerberosAuthenticator.class", + SubjectGetSubjectPatcher::new ); public static void main(String[] args) throws Exception { @@ -43,7 +50,7 @@ public static void main(String[] args) throws Exception { byte[] classToPatch = jarFile.getInputStream(jarEntry).readAllBytes(); ClassReader classReader = new ClassReader(classToPatch); - ClassWriter classWriter = new ClassWriter(classReader, 0); + ClassWriter classWriter = new ClassWriter(classReader, COMPUTE_FRAMES | COMPUTE_MAXS); classReader.accept(patcher.getValue().apply(classWriter), 0); Path outputFile = outputDir.resolve(patcher.getKey()); diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java new file mode 100644 index 0000000000000..e7c0002d349ba --- /dev/null +++ b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.hdfs.patch; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Type; + +import static org.objectweb.asm.Opcodes.ASM9; +import static org.objectweb.asm.Opcodes.BIPUSH; +import static org.objectweb.asm.Opcodes.GOTO; +import static org.objectweb.asm.Opcodes.IF_ICMPLE; +import static org.objectweb.asm.Opcodes.INVOKESTATIC; +import static org.objectweb.asm.Opcodes.INVOKEVIRTUAL; +import static org.objectweb.asm.Opcodes.POP; + +class SubjectGetSubjectPatcher extends ClassVisitor { + SubjectGetSubjectPatcher(ClassWriter classWriter) { + super(ASM9, classWriter); + } + + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + return new ReplaceCallMethodVisitor(super.visitMethod(access, name, descriptor, signature, exceptions), name, access, descriptor); + } + + /** + * Replaces calls to Subject.getSubject(context); with calls to Subject.current(); + */ + private static class ReplaceCallMethodVisitor extends MethodVisitor { + private static final String SUBJECT_CLASS_INTERNAL_NAME = "javax/security/auth/Subject"; + private static final String METHOD_NAME = "getSubject"; + + ReplaceCallMethodVisitor(MethodVisitor methodVisitor, String name, int access, String descriptor) { + super(ASM9, methodVisitor); + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { + if (opcode == INVOKESTATIC && SUBJECT_CLASS_INTERNAL_NAME.equals(owner) && METHOD_NAME.equals(name)) { + Label olderJdk = new Label(); + Label end = new Label(); + mv.visitMethodInsn( + INVOKESTATIC, + Type.getInternalName(Runtime.class), + "version", + Type.getMethodDescriptor(Type.getType(Runtime.Version.class)), + false + ); + mv.visitMethodInsn( + INVOKEVIRTUAL, + Type.getInternalName(Runtime.Version.class), + "feature", + Type.getMethodDescriptor(Type.getType(int.class)), + false + ); + mv.visitIntInsn(BIPUSH, 17); + mv.visitJumpInsn(IF_ICMPLE, olderJdk); + // Get rid of the extra arg on the stack + mv.visitInsn(POP); + // Call Subject.current() + mv.visitMethodInsn( + INVOKESTATIC, + SUBJECT_CLASS_INTERNAL_NAME, + "current", + Type.getMethodDescriptor(Type.getObjectType(SUBJECT_CLASS_INTERNAL_NAME)), + false + ); + mv.visitJumpInsn(GOTO, end); + mv.visitLabel(olderJdk); + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + mv.visitLabel(end); + } else { + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + } + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.create_from.json b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.create_from.json new file mode 100644 index 0000000000000..e17a69a77b252 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.create_from.json @@ -0,0 +1,37 @@ +{ + "migrate.create_from":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-stream-reindex.html", + "description":"This API creates a destination from a source index. It copies the mappings and settings from the source index while allowing request settings and mappings to override the source values." + }, + "stability":"experimental", + "visibility":"private", + "headers":{ + "accept": [ "application/json"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_create_from/{source}/{dest}", + "methods":[ "PUT", "POST"], + "parts":{ + "source":{ + "type":"string", + "description":"The source index name" + }, + "dest":{ + "type":"string", + "description":"The destination index name" + } + } + } + ] + }, + "body":{ + "description":"The body contains the fields `mappings_override` and `settings_override`.", + "required":false + } + } +} + diff --git a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java index 76dba60689422..ba5bcd85f4807 100644 --- a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java +++ b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java @@ -268,18 +268,19 @@ private Map extractMultiFields(String field, boolean quoted) { if (allFields && this.field != null && this.field.equals(field)) { // "*" is the default field extractedFields = fieldsAndWeights; + } else { + boolean multiFields = Regex.isSimpleMatchPattern(field); + // Filters unsupported fields if a pattern is requested + // Filters metadata fields if all fields are requested + extractedFields = resolveMappingField( + context, + field, + 1.0f, + allFields == false, + multiFields == false, + quoted ? quoteFieldSuffix : null + ); } - boolean multiFields = Regex.isSimpleMatchPattern(field); - // Filters unsupported fields if a pattern is requested - // Filters metadata fields if all fields are requested - extractedFields = resolveMappingField( - context, - field, - 1.0f, - allFields == false, - multiFields == false, - quoted ? quoteFieldSuffix : null - ); } else if (quoted && quoteFieldSuffix != null) { extractedFields = resolveMappingFields(context, fieldsAndWeights, quoteFieldSuffix); } else { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java index 4d1ebf6cbaabc..68987bbd19815 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java @@ -170,7 +170,6 @@ public void testReloadingKeyStore() throws Exception { * Tests the reloading of SSLContext when a PEM key and certificate are used. */ public void testPEMKeyConfigReloading() throws Exception { - assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm()); Path tempDir = createTempDir(); Path keyPath = tempDir.resolve("testnode.pem"); Path certPath = tempDir.resolve("testnode.crt"); @@ -221,11 +220,19 @@ public void testPEMKeyConfigReloading() throws Exception { try (MockWebServer server = new MockWebServer(updatedContext, false)) { server.enqueue(new MockResponse().setResponseCode(200).setBody("body")); server.start(); - SSLHandshakeException sslException = expectThrows( - SSLHandshakeException.class, - () -> privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close()) - ); - assertThat(sslException.getCause().getMessage(), containsString("PKIX path validation failed")); + if (inFipsJvm()) { + Exception sslException = expectThrows( + IOException.class, + () -> privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close()) + ); + assertThat(sslException.getCause().getMessage(), containsString("Unable to construct a valid chain")); + } else { + SSLHandshakeException sslException = expectThrows( + SSLHandshakeException.class, + () -> privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close()) + ); + assertThat(sslException.getCause().getMessage(), containsString("PKIX path validation failed")); + } } catch (Exception e) { throw new RuntimeException("Exception starting or connecting to the mock server", e); } @@ -290,7 +297,6 @@ public void testReloadingTrustStore() throws Exception { * Test the reloading of SSLContext whose trust config is backed by PEM certificate files. */ public void testReloadingPEMTrustConfig() throws Exception { - assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm()); Path tempDir = createTempDir(); Path serverCertPath = tempDir.resolve("testnode.crt"); Path serverKeyPath = tempDir.resolve("testnode.pem"); @@ -324,11 +330,19 @@ public void testReloadingPEMTrustConfig() throws Exception { // Client doesn't trust the Server certificate anymore so SSLHandshake should fail final Consumer trustMaterialPostChecks = (updatedContext) -> { try (CloseableHttpClient client = createHttpClient(updatedContext)) { - SSLHandshakeException sslException = expectThrows( - SSLHandshakeException.class, - () -> privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close()) - ); - assertThat(sslException.getCause().getMessage(), containsString("PKIX path validation failed")); + if (inFipsJvm()) { + Exception sslException = expectThrows( + IOException.class, + () -> privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close()) + ); + assertThat(sslException.getCause().getMessage(), containsString("Unable to construct a valid chain")); + } else { + SSLHandshakeException sslException = expectThrows( + SSLHandshakeException.class, + () -> privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close()) + ); + assertThat(sslException.getCause().getMessage(), containsString("PKIX path validation failed")); + } } catch (Exception e) { throw new RuntimeException("Error closing CloseableHttpClient", e); } diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java index 5c2a05da1e8d2..5fba98b765a6b 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java @@ -10,8 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest; -import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; @@ -57,9 +55,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -144,7 +140,7 @@ public void setup(final String sourceIndex, int numOfShards, int numOfReplicas, public void testILMDownsampleRollingRestart() throws Exception { final InternalTestCluster cluster = internalCluster(); - final List masterNodes = cluster.startMasterOnlyNodes(1); + cluster.startMasterOnlyNodes(1); cluster.startDataOnlyNodes(3); ensureStableCluster(cluster.size()); ensureGreen(); @@ -169,46 +165,16 @@ public void testILMDownsampleRollingRestart() throws Exception { .endObject(); }; int indexedDocs = bulkIndex(sourceIndex, sourceSupplier, DOC_COUNT); - final CountDownLatch disruptionStart = new CountDownLatch(1); - final CountDownLatch disruptionEnd = new CountDownLatch(1); - new Thread(new Disruptor(cluster, sourceIndex, new DisruptionListener() { - @Override - public void disruptionStart() { - disruptionStart.countDown(); - } - - @Override - public void disruptionEnd() { - disruptionEnd.countDown(); - } - }, masterNodes.get(0), (ignored) -> { - try { - cluster.rollingRestart(new InternalTestCluster.RestartCallback() { - @Override - public boolean validateClusterForming() { - return true; - } - }); - } catch (Exception e) { - throw new RuntimeException(e); - } - })).start(); + cluster.rollingRestart(new InternalTestCluster.RestartCallback()); final String targetIndex = "downsample-1h-" + sourceIndex; - startDownsampleTaskViaIlm(sourceIndex, targetIndex, disruptionStart, disruptionEnd); - waitUntil(() -> getClusterPendingTasks(cluster.client()).pendingTasks().isEmpty(), 60, TimeUnit.SECONDS); - ensureStableCluster(cluster.numDataAndMasterNodes()); - assertTargetIndex(cluster, targetIndex, indexedDocs); + startDownsampleTaskViaIlm(sourceIndex, targetIndex); + assertBusy(() -> assertTargetIndex(cluster, targetIndex, indexedDocs)); + ensureGreen(targetIndex); } - private void startDownsampleTaskViaIlm( - String sourceIndex, - String targetIndex, - CountDownLatch disruptionStart, - CountDownLatch disruptionEnd - ) throws Exception { - disruptionStart.await(); + private void startDownsampleTaskViaIlm(String sourceIndex, String targetIndex) throws Exception { var request = new UpdateSettingsRequest(sourceIndex).settings( Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, POLICY_NAME) ); @@ -231,7 +197,6 @@ private void startDownsampleTaskViaIlm( var getSettingsResponse = client().admin().indices().getSettings(new GetSettingsRequest().indices(targetIndex)).actionGet(); assertThat(getSettingsResponse.getSetting(targetIndex, IndexMetadata.INDEX_DOWNSAMPLE_STATUS.getKey()), equalTo("success")); }, 60, TimeUnit.SECONDS); - disruptionEnd.await(); } private void assertTargetIndex(final InternalTestCluster cluster, final String targetIndex, int indexedDocs) { @@ -294,53 +259,4 @@ private String randomDateForRange(long start, long end) { public interface SourceSupplier { XContentBuilder get() throws IOException; } - - interface DisruptionListener { - void disruptionStart(); - - void disruptionEnd(); - } - - private class Disruptor implements Runnable { - final InternalTestCluster cluster; - private final String sourceIndex; - private final DisruptionListener listener; - private final String clientNode; - private final Consumer disruption; - - private Disruptor( - final InternalTestCluster cluster, - final String sourceIndex, - final DisruptionListener listener, - final String clientNode, - final Consumer disruption - ) { - this.cluster = cluster; - this.sourceIndex = sourceIndex; - this.listener = listener; - this.clientNode = clientNode; - this.disruption = disruption; - } - - @Override - public void run() { - listener.disruptionStart(); - try { - final String candidateNode = safeExecute( - cluster.client(clientNode), - TransportClusterSearchShardsAction.TYPE, - new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, sourceIndex) - ).getNodes()[0].getName(); - logger.info("Candidate node [" + candidateNode + "]"); - disruption.accept(candidateNode); - ensureGreen(sourceIndex); - ensureStableCluster(cluster.numDataAndMasterNodes(), clientNode); - - } catch (Exception e) { - logger.error("Ignoring Error while injecting disruption [" + e.getMessage() + "]"); - } finally { - listener.disruptionEnd(); - } - } - } } diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java new file mode 100644 index 0000000000000..708cf74bceee7 --- /dev/null +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.ccq; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.apache.http.HttpHost; +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.test.TestClustersThreadFilter; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.xpack.esql.qa.rest.RequestIndexFilteringTestCase; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; + +@ThreadLeakFilters(filters = TestClustersThreadFilter.class) +public class RequestIndexFilteringIT extends RequestIndexFilteringTestCase { + + static ElasticsearchCluster remoteCluster = Clusters.remoteCluster(); + static ElasticsearchCluster localCluster = Clusters.localCluster(remoteCluster); + + @ClassRule + public static TestRule clusterRule = RuleChain.outerRule(remoteCluster).around(localCluster); + private static RestClient remoteClient; + + @Override + protected String getTestRestCluster() { + return localCluster.getHttpAddresses(); + } + + @Before + public void setRemoteClient() throws IOException { + if (remoteClient == null) { + var clusterHosts = parseClusterHosts(remoteCluster.getHttpAddresses()); + remoteClient = buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0])); + } + } + + @BeforeClass + public static void checkVersion() { + assumeTrue("skip if version before 8.18", Clusters.localClusterVersion().onOrAfter(Version.V_8_18_0)); + } + + @AfterClass + public static void closeRemoteClients() throws IOException { + try { + IOUtils.close(remoteClient); + } finally { + remoteClient = null; + } + } + + @Override + protected void indexTimestampData(int docs, String indexName, String date, String differentiatorFieldName) throws IOException { + indexTimestampDataForClient(client(), docs, indexName, date, differentiatorFieldName); + indexTimestampDataForClient(remoteClient, docs, indexName, date, differentiatorFieldName); + } + + @Override + protected String from(String... indexName) { + if (randomBoolean()) { + return "FROM *:" + String.join(",*:", indexName); + } else { + return "FROM " + String.join(",", indexName); + } + } + + @After + public void wipeRemoteTestData() throws IOException { + try { + var response = remoteClient.performRequest(new Request("DELETE", "/test*")); + assertEquals(200, response.getStatusLine().getStatusCode()); + } catch (ResponseException re) { + assertEquals(404, re.getResponse().getStatusLine().getStatusCode()); + } + } +} diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java index ba93e9b31bb09..ac5a3d4be27f3 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java @@ -11,6 +11,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.esql.AssertWarnings; @@ -30,12 +31,13 @@ import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.entityToMap; import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.requestObjectBuilder; import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.oneOf; public abstract class RequestIndexFilteringTestCase extends ESRestTestCase { @@ -49,6 +51,10 @@ public void wipeTestData() throws IOException { } } + protected String from(String... indexName) { + return "FROM " + String.join(",", indexName); + } + public void testTimestampFilterFromQuery() throws IOException { int docsTest1 = 50; int docsTest2 = 30; @@ -56,7 +62,7 @@ public void testTimestampFilterFromQuery() throws IOException { indexTimestampData(docsTest2, "test2", "2023-11-26", "id2"); // filter includes both indices in the result (all columns, all rows) - RestEsqlTestCase.RequestObjectBuilder builder = timestampFilter("gte", "2023-01-01").query("FROM test*"); + RestEsqlTestCase.RequestObjectBuilder builder = timestampFilter("gte", "2023-01-01").query(from("test*")); Map result = runEsql(builder); assertMap( result, @@ -70,7 +76,7 @@ public void testTimestampFilterFromQuery() throws IOException { ); // filter includes only test1. Columns from test2 are filtered out, as well (not only rows)! - builder = timestampFilter("gte", "2024-01-01").query("FROM test*"); + builder = timestampFilter("gte", "2024-01-01").query(from("test*")); assertMap( runEsql(builder), matchesMap().entry( @@ -83,7 +89,7 @@ public void testTimestampFilterFromQuery() throws IOException { // filter excludes both indices (no rows); the first analysis step fails because there are no columns, a second attempt succeeds // after eliminating the index filter. All columns are returned. - builder = timestampFilter("gte", "2025-01-01").query("FROM test*"); + builder = timestampFilter("gte", "2025-01-01").query(from("test*")); assertMap( runEsql(builder), matchesMap().entry( @@ -103,7 +109,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException { indexTimestampData(docsTest2, "test2", "2023-11-26", "id2"); // filter includes only test1. Columns and rows of test2 are filtered out - RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query("FROM test*"); + RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query(from("test*")); Map result = runEsql(builder); assertMap( result, @@ -116,7 +122,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException { ); // filter includes only test1. Columns from test2 are filtered out, as well (not only rows)! - builder = existsFilter("id1").query("FROM test* METADATA _index | KEEP _index, id*"); + builder = existsFilter("id1").query(from("test*") + " METADATA _index | KEEP _index, id*"); result = runEsql(builder); assertMap( result, @@ -129,7 +135,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException { @SuppressWarnings("unchecked") var values = (List>) result.get("values"); for (List row : values) { - assertThat(row.get(0), equalTo("test1")); + assertThat(row.get(0), oneOf("test1", "remote_cluster:test1")); assertThat(row.get(1), instanceOf(Integer.class)); } } @@ -142,7 +148,7 @@ public void testFieldExistsFilter_With_ExplicitUseOfDiscardedIndexFields() throw // test2 is explicitly used in a query with "SORT id2" even if the index filter should discard test2 RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query( - "FROM test* METADATA _index | SORT id2 | KEEP _index, id*" + from("test*") + " METADATA _index | SORT id2 | KEEP _index, id*" ); Map result = runEsql(builder); assertMap( @@ -157,7 +163,7 @@ public void testFieldExistsFilter_With_ExplicitUseOfDiscardedIndexFields() throw @SuppressWarnings("unchecked") var values = (List>) result.get("values"); for (List row : values) { - assertThat(row.get(0), equalTo("test1")); + assertThat(row.get(0), oneOf("test1", "remote_cluster:test1")); assertThat(row.get(1), instanceOf(Integer.class)); assertThat(row.get(2), nullValue()); } @@ -172,59 +178,59 @@ public void testFieldNameTypo() throws IOException { // idx field name is explicitly used, though it doesn't exist in any of the indices. First test - without filter ResponseException e = expectThrows( ResponseException.class, - () -> runEsql(requestObjectBuilder().query("FROM test* | WHERE idx == 123")) + () -> runEsql(requestObjectBuilder().query(from("test*") + " | WHERE idx == 123")) ); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("verification_exception")); assertThat(e.getMessage(), containsString("Found 1 problem")); - assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]")); + assertThat(e.getMessage(), containsString("Unknown column [idx]")); - e = expectThrows(ResponseException.class, () -> runEsql(requestObjectBuilder().query("FROM test1 | WHERE idx == 123"))); + e = expectThrows(ResponseException.class, () -> runEsql(requestObjectBuilder().query(from("test1") + " | WHERE idx == 123"))); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("verification_exception")); assertThat(e.getMessage(), containsString("Found 1 problem")); - assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]")); + assertThat(e.getMessage(), containsString("Unknown column [idx]")); e = expectThrows( ResponseException.class, - () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM test* | WHERE idx == 123")) + () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test*") + " | WHERE idx == 123")) ); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("Found 1 problem")); - assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]")); + assertThat(e.getMessage(), containsString("Unknown column [idx]")); e = expectThrows( ResponseException.class, - () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM test2 | WHERE idx == 123")) + () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test2") + " | WHERE idx == 123")) ); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("Found 1 problem")); - assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]")); + assertThat(e.getMessage(), containsString("Unknown column [idx]")); } public void testIndicesDontExist() throws IOException { int docsTest1 = 0; // we are interested only in the created index, not necessarily that it has data indexTimestampData(docsTest1, "test1", "2024-11-26", "id1"); - ResponseException e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo"))); + ResponseException e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo")))); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("verification_exception")); - assertThat(e.getMessage(), containsString("Unknown index [foo]")); + assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo]"), containsString("Unknown index [remote_cluster:foo]"))); - e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo*"))); + e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo*")))); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("verification_exception")); - assertThat(e.getMessage(), containsString("Unknown index [foo*]")); + assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo*]"), containsString("Unknown index [remote_cluster:foo*]"))); - e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo,test1"))); + e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo", "test1")))); assertEquals(404, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("index_not_found_exception")); - assertThat(e.getMessage(), containsString("no such index [foo]")); + assertThat(e.getMessage(), anyOf(containsString("no such index [foo]"), containsString("no such index [remote_cluster:foo]"))); if (EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled()) { e = expectThrows( ResponseException.class, - () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM test1 | LOOKUP JOIN foo ON id1")) + () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test1") + " | LOOKUP JOIN foo ON id1")) ); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("verification_exception")); @@ -251,6 +257,11 @@ public Map runEsql(RestEsqlTestCase.RequestObjectBuilder request } protected void indexTimestampData(int docs, String indexName, String date, String differentiatorFieldName) throws IOException { + indexTimestampDataForClient(client(), docs, indexName, date, differentiatorFieldName); + } + + protected void indexTimestampDataForClient(RestClient client, int docs, String indexName, String date, String differentiatorFieldName) + throws IOException { Request createIndex = new Request("PUT", indexName); createIndex.setJsonEntity(""" { @@ -273,7 +284,7 @@ protected void indexTimestampData(int docs, String indexName, String date, Strin } } }""".replace("%differentiator_field_name%", differentiatorFieldName)); - Response response = client().performRequest(createIndex); + Response response = client.performRequest(createIndex); assertThat( entityToMap(response.getEntity(), XContentType.JSON), matchesMap().entry("shards_acknowledged", true).entry("index", indexName).entry("acknowledged", true) @@ -291,7 +302,7 @@ protected void indexTimestampData(int docs, String indexName, String date, Strin bulk.addParameter("refresh", "true"); bulk.addParameter("filter_path", "errors"); bulk.setJsonEntity(b.toString()); - response = client().performRequest(bulk); + response = client.performRequest(bulk); Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec index fcac1e1859c6d..2614ff09fed06 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec @@ -1,18 +1,22 @@ hash required_capability: hash_function +// tag::hash[] FROM sample_data | WHERE message != "Connection error" | EVAL md5 = hash("md5", message), sha256 = hash("sha256", message) | KEEP message, md5, sha256; +// end::hash[] ignoreOrder:true +// tag::hash-result[] message:keyword | md5:keyword | sha256:keyword Connected to 10.1.0.1 | abd7d1ce2bb636842a29246b3512dcae | 6d8372129ad78770f7185554dd39864749a62690216460752d6c075fa38ad85c Connected to 10.1.0.2 | 8f8f1cb60832d153f5b9ec6dc828b93f | b0db24720f15857091b3c99f4c4833586d0ea3229911b8777efb8d917cf27e9a Connected to 10.1.0.3 | 912b6dc13503165a15de43304bb77c78 | 75b0480188db8acc4d5cc666a51227eb2bc5b989cd8ca912609f33e0846eff57 Disconnected | ef70e46fd3bbc21e3e1f0b6815e750c0 | 04dfac3671b494ad53fcd152f7a14511bfb35747278aad8ce254a0d6e4ba4718 ; +// end::hash-result[] hashOfConvertedType @@ -94,12 +98,75 @@ input:integer | md5:keyword | sha256:keyword hashWithStats required_capability: hash_function +required_capability: hash_function_aliases_v1 FROM sample_data | EVAL md5="md5" -| STATS count = count(*) by hash(md5, message) +| STATS count = count(*) by hash(md5, message), md5(message), sha1(message), sha256(message) | WHERE count > 1; -count:long | hash(md5, message):keyword -3 | 2e92ae79ff32b37fee4368a594792183 +count:long | hash(md5, message):keyword | md5(message):keyword | sha1(message):keyword | sha256(message):keyword +3 | 2e92ae79ff32b37fee4368a594792183 | 2e92ae79ff32b37fee4368a594792183 | 1dbb3521876a899f82d6b0ff10eb32a01e03aba8 | 8d137af9c64fba09bbb003aba93f0b029898fe19e7927cd696f4c3e2b69f538d ; + + +md5Hash +required_capability: hash_function_aliases_v1 + +// tag::md5[] +FROM sample_data +| WHERE message != "Connection error" +| EVAL md5 = md5(message) +| KEEP message, md5; +// end::md5[] +ignoreOrder:true + +// tag::md5-result[] +message:keyword | md5:keyword +Connected to 10.1.0.1 | abd7d1ce2bb636842a29246b3512dcae +Connected to 10.1.0.2 | 8f8f1cb60832d153f5b9ec6dc828b93f +Connected to 10.1.0.3 | 912b6dc13503165a15de43304bb77c78 +Disconnected | ef70e46fd3bbc21e3e1f0b6815e750c0 +; +// end::md5-result[] + + +sha1Hash +required_capability: hash_function_aliases_v1 + +// tag::sha1[] +FROM sample_data +| WHERE message != "Connection error" +| EVAL sha1 = sha1(message) +| KEEP message, sha1; +// end::sha1[] +ignoreOrder:true + +// tag::sha1-result[] +message:keyword | sha1:keyword +Connected to 10.1.0.1 | 42b85531a79088036a17759db7d2de292b92f57f +Connected to 10.1.0.2 | d30db445da2e9237c9718d0c7e4fb7cbbe9c2cb4 +Connected to 10.1.0.3 | 2733848d943809f0b10cad3e980763e88afb9853 +Disconnected | 771e05f27b99fd59f638f41a7a4e977b1d4691fe +; +// end::sha1-result[] + +sha256Hash +required_capability: hash_function_aliases_v1 + +// tag::sha256[] +FROM sample_data +| WHERE message != "Connection error" +| EVAL sha256 = sha256(message) +| KEEP message, sha256; +// end::sha256[] +ignoreOrder:true + +// tag::sha256-result[] +message:keyword | sha256:keyword +Connected to 10.1.0.1 | 6d8372129ad78770f7185554dd39864749a62690216460752d6c075fa38ad85c +Connected to 10.1.0.2 | b0db24720f15857091b3c99f4c4833586d0ea3229911b8777efb8d917cf27e9a +Connected to 10.1.0.3 | 75b0480188db8acc4d5cc666a51227eb2bc5b989cd8ca912609f33e0846eff57 +Disconnected | 04dfac3671b494ad53fcd152f7a14511bfb35747278aad8ce254a0d6e4ba4718 +; +// end::sha256-result[] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 51080d8289353..7dcff7b2bdd98 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -931,6 +931,15 @@ null | 1950 null | 1960 ; +countAllOnly +from employees | stats c = count() +; + +c:l +100 +; + + countFieldNoGrouping from employees | where emp_no < 10050 | stats c = count(salary); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index 81164382c0541..b8277cd222348 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1375,6 +1375,7 @@ client_ip:ip | event_duration:long | message:keyword | @timestamp:keywo # Once INLINESTATS supports expressions in agg functions and groups, convert the group in the inlinestats multiIndexIndirectUseOfUnionTypesInSort +required_capability: union_types FROM sample_data, sample_data_ts_long | SORT client_ip ASC | LIMIT 1 @@ -1397,6 +1398,7 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInRename +required_capability: union_types required_capability: union_types_fix_rename_resolution FROM sample_data, sample_data_ts_long | RENAME message AS event_message @@ -1409,6 +1411,7 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInKeep +required_capability: union_types FROM sample_data, sample_data_ts_long | KEEP client_ip, event_duration, message | SORT client_ip ASC @@ -1420,6 +1423,7 @@ client_ip:ip | event_duration:long | message:keyword ; multiIndexIndirectUseOfUnionTypesInWildcardKeep +required_capability: union_types required_capability: union_types_fix_rename_resolution FROM sample_data, sample_data_ts_long | KEEP * @@ -1432,6 +1436,7 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInWildcardKeep2 +required_capability: union_types required_capability: union_types_fix_rename_resolution FROM sample_data, sample_data_ts_long | KEEP *e* @@ -1468,6 +1473,7 @@ client_ip:ip | event_duration:long | message:keyword ; multiIndexIndirectUseOfUnionTypesInWildcardDrop +required_capability: union_types required_capability: union_types_fix_rename_resolution FROM sample_data, sample_data_ts_long | DROP *time* @@ -1480,6 +1486,7 @@ client_ip:ip | event_duration:long | message:keyword ; multiIndexIndirectUseOfUnionTypesInWhere +required_capability: union_types FROM sample_data, sample_data_ts_long | WHERE message == "Disconnected" ; @@ -1490,6 +1497,7 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInDissect +required_capability: union_types FROM sample_data, sample_data_ts_long | DISSECT message "%{foo}" | SORT client_ip ASC @@ -1501,6 +1509,7 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInGrok +required_capability: union_types FROM sample_data, sample_data_ts_long | GROK message "%{WORD:foo}" | SORT client_ip ASC @@ -1512,6 +1521,7 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInEnrich +required_capability: union_types required_capability: enrich_load FROM sample_data, sample_data_ts_long | EVAL client_ip = client_ip::keyword @@ -1525,6 +1535,7 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInStats +required_capability: union_types FROM sample_data, sample_data_ts_long | STATS foo = max(event_duration) BY client_ip | SORT client_ip ASC @@ -1538,6 +1549,7 @@ foo:long | client_ip:ip ; multiIndexIndirectUseOfUnionTypesInInlineStats-Ignore +required_capability: union_types required_capability: inlinestats FROM sample_data, sample_data_ts_long | INLINESTATS foo = max(event_duration) @@ -1550,6 +1562,7 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInLookup-Ignore +required_capability: union_types required_capability: lookup_v4 FROM sample_data, sample_data_ts_long | SORT client_ip ASC @@ -1563,6 +1576,7 @@ FROM sample_data, sample_data_ts_long ; multiIndexIndirectUseOfUnionTypesInMvExpand +required_capability: union_types FROM sample_data, sample_data_ts_long | EVAL foo = MV_APPEND(message, message) | SORT client_ip ASC diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index d423453ac5ce3..ef82ca47804bc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -451,6 +451,10 @@ public enum Cap { * Hash function */ HASH_FUNCTION, + /** + * Hash function aliases such as MD5 + */ + HASH_FUNCTION_ALIASES_V1, /** * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 908c9c5f197a8..d310973d6dbe7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -134,11 +134,14 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Md5; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha1; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha256; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Space; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; @@ -333,11 +336,14 @@ private static FunctionDefinition[][] functions() { def(Left.class, Left::new, "left"), def(Length.class, Length::new, "length"), def(Locate.class, Locate::new, "locate"), + def(Md5.class, Md5::new, "md5"), def(RTrim.class, RTrim::new, "rtrim"), def(Repeat.class, Repeat::new, "repeat"), def(Replace.class, Replace::new, "replace"), def(Reverse.class, Reverse::new, "reverse"), def(Right.class, Right::new, "right"), + def(Sha1.class, Sha1::new, "sha1"), + def(Sha256.class, Sha256::new, "sha256"), def(Space.class, Space::new, "space"), def(StartsWith.class, StartsWith::new, "starts_with"), def(Substring.class, Substring::new, "substring"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java index 3cf0eef9074ad..c4b9f6885e617 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java @@ -37,10 +37,13 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Md5; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha1; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha256; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; @@ -79,6 +82,7 @@ public static List getNamedWriteables() { entries.add(Left.ENTRY); entries.add(Locate.ENTRY); entries.add(Log.ENTRY); + entries.add(Md5.ENTRY); entries.add(Now.ENTRY); entries.add(Or.ENTRY); entries.add(Pi.ENTRY); @@ -88,6 +92,8 @@ public static List getNamedWriteables() { entries.add(Replace.ENTRY); entries.add(Reverse.ENTRY); entries.add(Round.ENTRY); + entries.add(Sha1.ENTRY); + entries.add(Sha256.ENTRY); entries.add(Split.ENTRY); entries.add(Substring.ENTRY); entries.add(StartsWith.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java new file mode 100644 index 0000000000000..a39f0a4f32db2 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash.HashFunction; + +import java.io.IOException; +import java.util.function.Function; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; + +public abstract class AbstractHashFunction extends UnaryScalarFunction { + + protected AbstractHashFunction(Source source, Expression field) { + super(source, field); + } + + protected AbstractHashFunction(StreamInput in) throws IOException { + super(in); + } + + protected abstract HashFunction getHashFunction(); + + @Override + public DataType dataType() { + return DataType.KEYWORD; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + return isString(field, sourceText(), DEFAULT); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + return new HashConstantEvaluator.Factory( + source(), + context -> new BreakingBytesRefBuilder(context.breaker(), "hash"), + new Function<>() { + @Override + public HashFunction apply(DriverContext context) { + return getHashFunction().copy(); + } + + @Override + public String toString() { + return getHashFunction().toString(); + } + }, + toEvaluator.apply(field) + ); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java index 99c5908699ec2..52d33c0fc9d3d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; @@ -46,7 +47,8 @@ public class Hash extends EsqlScalarFunction { @FunctionInfo( returnType = "keyword", - description = "Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512." + description = "Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512.", + examples = { @Example(file = "hash", tag = "hash") } ) public Hash( Source source, @@ -186,10 +188,18 @@ protected NodeInfo info() { public record HashFunction(String algorithm, MessageDigest digest) { + public static HashFunction create(String algorithm) { + try { + return new HashFunction(algorithm, MessageDigest.getInstance(algorithm)); + } catch (NoSuchAlgorithmException e) { + assert false : "Expected to create a valid hashing algorithm"; + throw new IllegalStateException(e); + } + } + public static HashFunction create(BytesRef literal) throws NoSuchAlgorithmException { var algorithm = literal.utf8ToString(); - var digest = MessageDigest.getInstance(algorithm); - return new HashFunction(algorithm, digest); + return new HashFunction(algorithm, MessageDigest.getInstance(algorithm)); } public HashFunction copy() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java new file mode 100644 index 0000000000000..b42ec1036cb5b --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash.HashFunction; + +import java.io.IOException; +import java.util.List; + +public class Md5 extends AbstractHashFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MD5", Md5::new); + + private static final HashFunction MD5 = HashFunction.create("MD5"); + + @FunctionInfo( + returnType = "keyword", + description = "Computes the MD5 hash of the input.", + examples = { @Example(file = "hash", tag = "md5") } + ) + public Md5(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) { + super(source, input); + } + + private Md5(StreamInput in) throws IOException { + super(in); + } + + @Override + protected HashFunction getHashFunction() { + return MD5; + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Md5(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Md5::new, field); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java new file mode 100644 index 0000000000000..ba1f62562c03a --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.util.List; + +public class Sha1 extends AbstractHashFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "SHA1", Sha1::new); + + private static final Hash.HashFunction SHA1 = Hash.HashFunction.create("SHA1"); + + @FunctionInfo( + returnType = "keyword", + description = "Computes the SHA1 hash of the input.", + examples = { @Example(file = "hash", tag = "sha1") } + ) + public Sha1(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) { + super(source, input); + } + + private Sha1(StreamInput in) throws IOException { + super(in); + } + + @Override + protected Hash.HashFunction getHashFunction() { + return SHA1; + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Sha1(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Sha1::new, field); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java new file mode 100644 index 0000000000000..b16767a3f7948 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.util.List; + +public class Sha256 extends AbstractHashFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "SHA256", Sha256::new); + + private static final Hash.HashFunction SHA256 = Hash.HashFunction.create("SHA256"); + + @FunctionInfo( + returnType = "keyword", + description = "Computes the SHA256 hash of the input.", + examples = { @Example(file = "hash", tag = "sha256") } + ) + public Sha256(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) { + super(source, input); + } + + private Sha256(StreamInput in) throws IOException { + super(in); + } + + @Override + protected Hash.HashFunction getHashFunction() { + return SHA256; + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Sha256(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Sha256::new, field); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 1eab557c15cab..9dae485d09cf9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -191,7 +191,6 @@ protected static List anyNullIsNull( ExpectedType expectedType, ExpectedEvaluatorToString evaluatorToString ) { - typesRequired(testCaseSuppliers); List suppliers = new ArrayList<>(testCaseSuppliers.size()); suppliers.addAll(testCaseSuppliers); @@ -274,7 +273,7 @@ protected static List anyNullIsNull( } @FunctionalInterface - protected interface PositionalErrorMessageSupplier { + public interface PositionalErrorMessageSupplier { /** * This interface defines functions to supply error messages for incorrect types in specific positions. Functions which have * the same type requirements for all positions can simplify this with a lambda returning a string constant. @@ -291,7 +290,9 @@ protected interface PositionalErrorMessageSupplier { /** * Adds test cases containing unsupported parameter types that assert * that they throw type errors. + * @deprecated make a subclass of {@link ErrorsForCasesWithoutExamplesTestCase} instead */ + @Deprecated protected static List errorsForCasesWithoutExamples( List testCaseSuppliers, PositionalErrorMessageSupplier positionalErrorMessageSupplier @@ -331,11 +332,14 @@ protected interface TypeErrorMessageSupplier { String apply(boolean includeOrdinal, List> validPerPosition, List types); } + /** + * @deprecated make a subclass of {@link ErrorsForCasesWithoutExamplesTestCase} instead + */ + @Deprecated protected static List errorsForCasesWithoutExamples( List testCaseSuppliers, TypeErrorMessageSupplier typeErrorMessageSupplier ) { - typesRequired(testCaseSuppliers); List suppliers = new ArrayList<>(testCaseSuppliers.size()); suppliers.addAll(testCaseSuppliers); @@ -346,7 +350,7 @@ protected static List errorsForCasesWithoutExamples( .map(s -> s.types().size()) .collect(Collectors.toSet()) .stream() - .flatMap(count -> allPermutations(count)) + .flatMap(AbstractFunctionTestCase::allPermutations) .filter(types -> valid.contains(types) == false) /* * Skip any cases with more than one null. Our tests don't generate @@ -366,10 +370,6 @@ private static List append(List orig, DataType extra) { return longer; } - protected static Stream representable() { - return DataType.types().stream().filter(DataType::isRepresentable); - } - protected static TestCaseSupplier typeErrorSupplier( boolean includeOrdinal, List> validPerPosition, @@ -398,7 +398,7 @@ protected static TestCaseSupplier typeErrorSupplier( ); } - private static List> validPerPosition(Set> valid) { + static List> validPerPosition(Set> valid) { int max = valid.stream().mapToInt(List::size).max().getAsInt(); List> result = new ArrayList<>(max); for (int i = 0; i < max; i++) { @@ -1327,17 +1327,6 @@ public void allMemoryReleased() { } } - /** - * Validate that we know the types for all the test cases already created - * @param suppliers - list of suppliers before adding in the illegal type combinations - */ - protected static void typesRequired(List suppliers) { - String bad = suppliers.stream().filter(s -> s.types() == null).map(s -> s.name()).collect(Collectors.joining("\n")); - if (bad.equals("") == false) { - throw new IllegalArgumentException("types required but not found for these tests:\n" + bad); - } - } - /** * Returns true if the current test case is for an aggregation function. *

diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java index 958b706a3f260..65b9c447170f4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java @@ -58,7 +58,11 @@ public abstract class AbstractScalarFunctionTestCase extends AbstractFunctionTes *

* * @param entirelyNullPreservesType See {@link #anyNullIsNull(boolean, List)} + * @deprecated use {@link #parameterSuppliersFromTypedDataWithDefaultChecksNoErrors} + * and make a subclass of {@link ErrorsForCasesWithoutExamplesTestCase}. + * It's a long faster. */ + @Deprecated protected static Iterable parameterSuppliersFromTypedDataWithDefaultChecks( boolean entirelyNullPreservesType, List suppliers, @@ -72,6 +76,23 @@ protected static Iterable parameterSuppliersFromTypedDataWithDefaultCh ); } + /** + * Converts a list of test cases into a list of parameter suppliers. + * Also, adds a default set of extra test cases. + *

+ * Use if possible, as this method may get updated with new checks in the future. + *

+ * + * @param entirelyNullPreservesType See {@link #anyNullIsNull(boolean, List)} + */ + protected static Iterable parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( + // TODO remove after removing parameterSuppliersFromTypedDataWithDefaultChecks rename this to that. + boolean entirelyNullPreservesType, + List suppliers + ) { + return parameterSuppliersFromTypedData(anyNullIsNull(entirelyNullPreservesType, randomizeBytesRefsOffset(suppliers))); + } + /** * Converts a list of test cases into a list of parameter suppliers. * Also, adds a default set of extra test cases. @@ -364,43 +385,10 @@ public void testFold() { } } - public static String errorMessageStringForBinaryOperators( - boolean includeOrdinal, - List> validPerPosition, - List types, - PositionalErrorMessageSupplier positionalErrorMessageSupplier - ) { - try { - return typeErrorMessage(includeOrdinal, validPerPosition, types, positionalErrorMessageSupplier); - } catch (IllegalStateException e) { - // This means all the positional args were okay, so the expected error is from the combination - if (types.get(0).equals(DataType.UNSIGNED_LONG)) { - return "first argument of [] is [unsigned_long] and second is [" - + types.get(1).typeName() - + "]. [unsigned_long] can only be operated on together with another [unsigned_long]"; - - } - if (types.get(1).equals(DataType.UNSIGNED_LONG)) { - return "first argument of [] is [" - + types.get(0).typeName() - + "] and second is [unsigned_long]. [unsigned_long] can only be operated on together with another [unsigned_long]"; - } - return "first argument of [] is [" - + (types.get(0).isNumeric() ? "numeric" : types.get(0).typeName()) - + "] so second argument must also be [" - + (types.get(0).isNumeric() ? "numeric" : types.get(0).typeName()) - + "] but was [" - + types.get(1).typeName() - + "]"; - - } - } - /** * Adds test cases containing unsupported parameter types that immediately fail. */ protected static List failureForCasesWithoutExamples(List testCaseSuppliers) { - typesRequired(testCaseSuppliers); List suppliers = new ArrayList<>(testCaseSuppliers.size()); suppliers.addAll(testCaseSuppliers); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java new file mode 100644 index 0000000000000..e2ad9e81939d6 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.hamcrest.Matcher; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral; +import static org.hamcrest.Matchers.greaterThan; + +public abstract class ErrorsForCasesWithoutExamplesTestCase extends ESTestCase { + protected abstract List cases(); + + /** + * Build the expression being tested, for the given source and list of arguments. Test classes need to implement this + * to have something to test. + * + * @param source the source + * @param args arg list from the test case, should match the length expected + * @return an expression for evaluating the function being tested on the given arguments + */ + protected abstract Expression build(Source source, List args); + + protected abstract Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature); + + protected final List paramsToSuppliers(Iterable cases) { + List result = new ArrayList<>(); + for (Object[] c : cases) { + if (c.length != 1) { + throw new IllegalArgumentException("weird layout for test cases"); + } + TestCaseSupplier supplier = (TestCaseSupplier) c[0]; + result.add(supplier); + } + return result; + } + + public final void test() { + int checked = 0; + List cases = cases(); + Set> valid = cases.stream().map(TestCaseSupplier::types).collect(Collectors.toSet()); + List> validPerPosition = AbstractFunctionTestCase.validPerPosition(valid); + Iterable> missingSignatures = missingSignatures(cases, valid)::iterator; + for (List signature : missingSignatures) { + logger.debug("checking {}", signature); + List args = new ArrayList<>(signature.size()); + for (DataType type : signature) { + args.add(randomLiteral(type)); + } + Expression expression = build(Source.synthetic(sourceForSignature(signature)), args); + assertTrue("expected unresolved " + expression, expression.typeResolved().unresolved()); + assertThat(expression.typeResolved().message(), expectedTypeErrorMatcher(validPerPosition, signature)); + checked++; + } + logger.info("checked {} signatures", checked); + assertThat("didn't check any signatures", checked, greaterThan(0)); + } + + private Stream> missingSignatures(List cases, Set> valid) { + return cases.stream() + .map(s -> s.types().size()) + .collect(Collectors.toSet()) + .stream() + .flatMap(AbstractFunctionTestCase::allPermutations) + .filter(types -> valid.contains(types) == false) + /* + * Skip any cases with more than one null. Our tests don't generate + * the full combinatorial explosions of all nulls - just a single null. + * Hopefully , cases will function the same as , + * cases. + */ + .filter(types -> types.stream().filter(t -> t == DataType.NULL).count() <= 1); + } + + protected static String sourceForSignature(List signature) { + StringBuilder source = new StringBuilder(); + for (DataType type : signature) { + if (false == source.isEmpty()) { + source.append(", "); + } + source.append(type.typeName()); + } + return source.toString(); + } + + /** + * Build the expected error message for an invalid type signature. + */ + protected static String typeErrorMessage( + boolean includeOrdinal, + List> validPerPosition, + List signature, + AbstractFunctionTestCase.PositionalErrorMessageSupplier expectedTypeSupplier + ) { + int badArgPosition = -1; + for (int i = 0; i < signature.size(); i++) { + if (validPerPosition.get(i).contains(signature.get(i)) == false) { + badArgPosition = i; + break; + } + } + if (badArgPosition == -1) { + throw new IllegalStateException( + "Can't generate error message for these types, you probably need a custom error message function" + ); + } + String ordinal = includeOrdinal ? TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT) + " " : ""; + String source = sourceForSignature(signature); + String expectedTypeString = expectedTypeSupplier.apply(validPerPosition.get(badArgPosition), badArgPosition); + String name = signature.get(badArgPosition).typeName(); + return ordinal + "argument of [" + source + "] must be [" + expectedTypeString + "], found value [] type [" + name + "]"; + } + + protected static String errorMessageStringForBinaryOperators( + List> validPerPosition, + List signature, + AbstractFunctionTestCase.PositionalErrorMessageSupplier positionalErrorMessageSupplier + ) { + try { + return typeErrorMessage(true, validPerPosition, signature, positionalErrorMessageSupplier); + } catch (IllegalStateException e) { + String source = sourceForSignature(signature); + // This means all the positional args were okay, so the expected error is from the combination + if (signature.get(0).equals(DataType.UNSIGNED_LONG)) { + return "first argument of [" + + source + + "] is [unsigned_long] and second is [" + + signature.get(1).typeName() + + "]. [unsigned_long] can only be operated on together with another [unsigned_long]"; + + } + if (signature.get(1).equals(DataType.UNSIGNED_LONG)) { + return "first argument of [" + + source + + "] is [" + + signature.get(0).typeName() + + "] and second is [unsigned_long]. [unsigned_long] can only be operated on together with another [unsigned_long]"; + } + return "first argument of [" + + source + + "] is [" + + (signature.get(0).isNumeric() ? "numeric" : signature.get(0).typeName()) + + "] so second argument must also be [" + + (signature.get(0).isNumeric() ? "numeric" : signature.get(0).typeName()) + + "] but was [" + + signature.get(1).typeName() + + "]"; + + } + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeErrorTests.java new file mode 100644 index 0000000000000..f674f9b2c3d72 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.grouping; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class CategorizeErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(CategorizeTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Categorize(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeTests.java index 68bafd2af9960..dfdfd82d08afe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeTests.java @@ -51,7 +51,7 @@ public static Iterable parameters() { ) ); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseErrorTests.java new file mode 100644 index 0000000000000..28e1b846660c0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseErrorTests.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Locale; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class CaseErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(CaseTests.parameters()).stream() + // Take only the shorter signatures because we don't have error generation for longer cases + // TODO handle longer signatures + .filter(c -> c.types().size() < 4) + .toList(); + } + + @Override + protected Expression build(Source source, List args) { + return new Case(source, args.get(0), args.subList(1, args.size())); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + if (signature.get(0) != DataType.BOOLEAN && signature.get(0) != DataType.NULL) { + return typeErrorMessage(signature, 0, "boolean"); + } + DataType mainType = signature.get(1).noText(); + for (int i = 2; i < signature.size(); i++) { + if (i % 2 == 0 && i != signature.size() - 1) { + // condition + if (signature.get(i) != DataType.BOOLEAN && signature.get(i) != DataType.NULL) { + return typeErrorMessage(signature, i, "boolean"); + } + } else { + // value + if (signature.get(i).noText() != mainType) { + return typeErrorMessage(signature, i, mainType.typeName()); + } + } + } + throw new IllegalStateException("can't find bad arg for " + signature); + } + + private static Matcher typeErrorMessage(List signature, int badArgPosition, String expectedTypeString) { + String ordinal = TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT); + String sig = sourceForSignature(signature); + String name = signature.get(badArgPosition).typeName(); + return equalTo(ordinal + " argument of [" + sig + "] must be [" + expectedTypeString + "], found value [] type [" + name + "]"); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java index 93322ab0e35a7..05923246520fc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.NumericUtils; @@ -25,7 +24,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Locale; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -90,10 +88,6 @@ public static Iterable parameters() { ) ); } - suppliers = errorsForCasesWithoutExamples( - suppliers, - (includeOrdinal, validPerPosition, types) -> typeErrorMessage(includeOrdinal, types) - ); for (DataType type : TYPES) { fourAndFiveArgs(suppliers, true, randomSingleValuedCondition(), 0, type, List.of()); @@ -852,31 +846,4 @@ protected Matcher allNullsMatcher() { } return super.allNullsMatcher(); } - - private static String typeErrorMessage(boolean includeOrdinal, List types) { - if (types.get(0) != DataType.BOOLEAN && types.get(0) != DataType.NULL) { - return typeErrorMessage(includeOrdinal, types, 0, "boolean"); - } - DataType mainType = types.get(1).noText(); - for (int i = 2; i < types.size(); i++) { - if (i % 2 == 0 && i != types.size() - 1) { - // condition - if (types.get(i) != DataType.BOOLEAN && types.get(i) != DataType.NULL) { - return typeErrorMessage(includeOrdinal, types, i, "boolean"); - } - } else { - // value - if (types.get(i).noText() != mainType) { - return typeErrorMessage(includeOrdinal, types, i, mainType.typeName()); - } - } - } - throw new IllegalStateException("can't find bad arg for " + types); - } - - private static String typeErrorMessage(boolean includeOrdinal, List types, int badArgPosition, String expectedTypeString) { - String ordinal = includeOrdinal ? TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT) + " " : ""; - String name = types.get(badArgPosition).typeName(); - return ordinal + "argument of [] must be [" + expectedTypeString + "], found value [" + name + "] type [" + name + "]"; - } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffErrorTests.java new file mode 100644 index 0000000000000..a3a808de277d7 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffErrorTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class DateDiffErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(DateDiffTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new DateDiff(source, args.get(0), args.get(1), args.get(2)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, i) -> { + if (i == 0) { + return "string"; + } + return "datetime"; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java index 4dbdfd1e56854..da069e3c37cc4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java @@ -71,48 +71,6 @@ public static Iterable parameters() { ) ) ); - suppliers.add( - new TestCaseSupplier( - "Date Diff Error Type unit", - List.of(DataType.INTEGER, DataType.DATETIME, DataType.DATETIME), - () -> TestCaseSupplier.TestCase.typeError( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.INTEGER, "unit"), - new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"), - new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp") - ), - "first argument of [] must be [string], found value [unit] type [integer]" - ) - ) - ); - suppliers.add( - new TestCaseSupplier( - "Date Diff Error Type startTimestamp", - List.of(DataType.TEXT, DataType.INTEGER, DataType.DATETIME), - () -> TestCaseSupplier.TestCase.typeError( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataType.TEXT, "unit"), - new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.INTEGER, "startTimestamp"), - new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp") - ), - "second argument of [] must be [datetime], found value [startTimestamp] type [integer]" - ) - ) - ); - suppliers.add( - new TestCaseSupplier( - "Date Diff Error Type endTimestamp", - List.of(DataType.TEXT, DataType.DATETIME, DataType.INTEGER), - () -> TestCaseSupplier.TestCase.typeError( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataType.TEXT, "unit"), - new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"), - new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.INTEGER, "endTimestamp") - ), - "third argument of [] must be [datetime], found value [endTimestamp] type [integer]" - ) - ) - ); suppliers.add(new TestCaseSupplier("Date Diff In Year - 1", List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME), () -> { ZonedDateTime zdtStart2 = ZonedDateTime.parse("2023-12-12T00:01:01Z"); ZonedDateTime zdtEnd2 = ZonedDateTime.parse("2024-12-12T00:01:01Z"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java new file mode 100644 index 0000000000000..d5b9a06c8738e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class DateExtractErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(DateExtractTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new DateExtract(source, args.get(0), args.get(1), EsqlTestUtils.TEST_CFG); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "string"; + case 1 -> "datetime"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java index db375171b7224..be978eda06758 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java @@ -39,7 +39,7 @@ public DateExtractTests(@Name("TestCase") Supplier te @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedDataWithDefaultChecks( + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( true, List.of( new TestCaseSupplier( @@ -84,12 +84,7 @@ public static Iterable parameters() { ) .withFoldingException(InvalidArgumentException.class, "invalid date field for []: not a unit") ) - ), - (v, p) -> switch (p) { - case 0 -> "string"; - case 1 -> "datetime"; - default -> ""; - } + ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java new file mode 100644 index 0000000000000..985f1144fbcf2 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class DateFormatErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(DateFormatTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new DateFormat(source, args.get(0), args.get(1), EsqlTestUtils.TEST_CFG); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "string"; + case 1 -> "datetime"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java index d132c2ef2b365..8dfdd1ba486c7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java @@ -31,7 +31,7 @@ public DateFormatTests(@Name("TestCase") Supplier tes @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedDataWithDefaultChecks( + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( true, List.of( new TestCaseSupplier( @@ -58,12 +58,7 @@ public static Iterable parameters() { equalTo(BytesRefs.toBytesRef("2023")) ) ) - ), - (v, p) -> switch (p) { - case 0 -> "string"; - case 1 -> "datetime"; - default -> ""; - } + ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java new file mode 100644 index 0000000000000..21d9b5fb00537 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class DateParseErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(DateParseTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new DateParse(source, args.get(0), args.size() > 1 ? args.get(1) : null); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, i) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java index 04683ecb65467..b51866c0cc8a7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java @@ -36,7 +36,7 @@ public DateParseTests(@Name("TestCase") Supplier test @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedDataWithDefaultChecks( + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( true, List.of( new TestCaseSupplier( @@ -143,8 +143,7 @@ public static Iterable parameters() { + "failed to parse date field [not a date] with format [yyyy-MM-dd]" ) ) - ), - (v, p) -> "string" + ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncErrorTests.java new file mode 100644 index 0000000000000..9494888f9314f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class DateTruncErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(DateTruncTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new DateTrunc(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "dateperiod or timeduration"; + case 1 -> "date_nanos or datetime"; + default -> null; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java index 93970f64df845..50de64ff8b173 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java @@ -56,11 +56,7 @@ public static Iterable parameters() { suppliers.addAll(ofDuration(Duration.ofSeconds(30), ts, "2023-02-17T10:25:30.00Z")); suppliers.add(randomSecond()); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { - case 0 -> "dateperiod or timeduration"; - case 1 -> "date_nanos or datetime"; - default -> null; - }); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } private static List ofDatePeriod(Period period, long value, String expectedDate) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchErrorTests.java new file mode 100644 index 0000000000000..319adbc00569e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.ip; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class CIDRMatchErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(CIDRMatchTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new CIDRMatch(source, args.get(0), List.of(args.get(1))); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "ip"; + case 1 -> "string"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java index e777a0ce587e0..5047a3a4ab118 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java @@ -84,11 +84,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { - case 0 -> "ip"; - case 1 -> "string"; - default -> ""; - }); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixErrorTests.java new file mode 100644 index 0000000000000..1db58c225475e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.ip; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class IpPrefixErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(IpPrefixTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new IpPrefix(source, args.get(0), args.get(1), args.size() == 3 ? args.get(2) : null); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "ip"; + case 1, 2 -> "integer"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java index 5209d042b6408..08b517b54553d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java @@ -106,11 +106,7 @@ public static Iterable parameters() { }) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { - case 0 -> "ip"; - case 1, 2 -> "integer"; - default -> ""; - }); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2ErrorTests.java new file mode 100644 index 0000000000000..75fa802067931 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2ErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.math; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class Atan2ErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(Atan2Tests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Atan2(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, i) -> "numeric")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java index c475a75699da7..d7463be25e886 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java @@ -36,7 +36,7 @@ public static Iterable parameters() { Double.POSITIVE_INFINITY, List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinErrorTests.java new file mode 100644 index 0000000000000..58a0b0182b096 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.math; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class SinErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(SinTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Sin(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, i) -> "numeric")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java index 990356f8df6de..fba95289e3e8e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java @@ -33,7 +33,7 @@ public static Iterable parameters() { Double.POSITIVE_INFINITY, List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java index d069f7ffe2298..b1271b3a5b45e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java @@ -67,7 +67,7 @@ static Iterable parameters(String name, boolean trimLeading, boolean t })); } } - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(false, suppliers); } private static TestCaseSupplier.TestCase testCase(String name, DataType type, String data, String expected) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthErrorTests.java new file mode 100644 index 0000000000000..23411d486901c --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class BitLengthErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(BitLengthTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new BitLength(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java index bce4328a08abf..6f7c81a102996 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java @@ -40,7 +40,7 @@ public static Iterable parameters() { } } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthErrorTests.java new file mode 100644 index 0000000000000..629639485f784 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ByteLengthErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ByteLengthTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ByteLength(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthTests.java index 866b8e0cd8da3..dec96a6fb554e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthTests.java @@ -42,7 +42,7 @@ public static Iterable parameters() { cases.addAll(makeTestCases("3 bytes, 1 code point", () -> "☕", 3)); cases.addAll(makeTestCases("6 bytes, 2 code points", () -> "❗️", 6)); cases.addAll(makeTestCases("100 random alpha", () -> randomAlphaOfLength(100), 100)); - return parameterSuppliersFromTypedDataWithDefaultChecks(ENTIRELY_NULL_PRESERVES_TYPE, cases, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(ENTIRELY_NULL_PRESERVES_TYPE, cases); } private static List makeTestCases(String title, Supplier text, int expectedByteLength) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithErrorTests.java new file mode 100644 index 0000000000000..12067fa88b86b --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class EndsWithErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(EndsWithTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new EndsWith(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java index 1b2e9c41cb25c..c41b1e14257ee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java @@ -73,7 +73,7 @@ public static Iterable parameters() { ); } } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (valid, position) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } private static TestCaseSupplier.TestCase testCase( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashErrorTests.java new file mode 100644 index 0000000000000..63ef07d333eb8 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class HashErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(HashTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Hash(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java index c5cdf97eccd17..fe62b6d7b4f1b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java @@ -57,7 +57,7 @@ public static Iterable parameters() { .withWarning("Line -1:-1: java.security.NoSuchAlgorithmException: invalid MessageDigest not available") .withFoldingException(InvalidArgumentException.class, "invalid algorithm for []: invalid MessageDigest not available"); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); } private static List createTestCases(String algorithm) { @@ -87,12 +87,22 @@ private static TestCaseSupplier createTestCase(String algorithm, boolean forceLi }); } + static void addHashFunctionTestCases(List cases, String algorithm) { + TestCaseSupplier.forUnaryStrings( + cases, + "HashConstantEvaluator[algorithm=" + algorithm + ", input=Attribute[channel=0]]", + DataType.KEYWORD, + input -> new BytesRef(HashTests.hash(algorithm, BytesRefs.toString(input))), + List.of() + ); + } + private static TestCaseSupplier.TypedData createTypedData(String value, boolean forceLiteral, DataType type, String name) { var data = new TestCaseSupplier.TypedData(new BytesRef(value), type, name); return forceLiteral ? data.forceLiteral() : data; } - private static String hash(String algorithm, String input) { + static String hash(String algorithm, String input) { try { return HexFormat.of().formatHex(MessageDigest.getInstance(algorithm).digest(input.getBytes(StandardCharsets.UTF_8))); } catch (NoSuchAlgorithmException e) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimErrorTests.java new file mode 100644 index 0000000000000..8b3a20606eb93 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class LTrimErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(LTrimTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new LTrim(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftErrorTests.java new file mode 100644 index 0000000000000..44b9c13e1baa6 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class LeftErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(LeftTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Left(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "string"; + case 1 -> "integer"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java index e4e54a9e0935f..2a194a39d34e5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java @@ -181,11 +181,7 @@ public static Iterable parameters() { ); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { - case 0 -> "string"; - case 1 -> "integer"; - default -> ""; - }); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } private static String unicodeLeftSubstring(String str, int length) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthErrorTests.java new file mode 100644 index 0000000000000..ca8f2cb208209 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class LengthErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(LengthTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Length(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java index ba4c8c8ce1ea4..e4d8292fc2cb7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.hamcrest.Matcher; import java.util.ArrayList; import java.util.List; @@ -49,7 +48,7 @@ public static Iterable parameters() { cases.addAll(makeTestCases("6 bytes, 2 code points", () -> "❗️", 2)); cases.addAll(makeTestCases("100 random alpha", () -> randomAlphaOfLength(100), 100)); cases.addAll(makeTestCases("100 random code points", () -> randomUnicodeOfCodepointLength(100), 100)); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); } private static List makeTestCases(String title, Supplier text, int expectedLength) { @@ -87,10 +86,6 @@ private static List makeTestCases(String title, Supplier resultsMatcher(List typedData) { - return equalTo(UnicodeUtil.codePointCount((BytesRef) typedData.get(0).data())); - } - @Override protected Expression build(Source source, List args) { return new Length(source, args.get(0)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateErrorTests.java new file mode 100644 index 0000000000000..a9ee74a029374 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateErrorTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class LocateErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(LocateTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Locate(source, args.get(0), args.get(1), args.size() < 3 ? null : args.get(2)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> { + if (p == 0 || p == 1) { + return "string"; + } + return "integer"; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java index a10f97c45aa04..ac060b9454564 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java @@ -76,12 +76,7 @@ public static Iterable parameters() { } } - suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), (v, p) -> { - if (p == 0 || p == 1) { - return "string"; - } - return "integer"; - }); + suppliers = anyNullIsNull(true, suppliers); // Here follows some non-randomized examples that we want to cover on every run suppliers.add(supplier("a tiger", "a t", null, 1)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java new file mode 100644 index 0000000000000..701cef748c932 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class Md5ErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + + @Override + protected List cases() { + return paramsToSuppliers(Md5Tests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Md5(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java new file mode 100644 index 0000000000000..666e5a3ea5d58 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class Md5SerializationTests extends AbstractExpressionSerializationTests { + + @Override + protected Md5 createTestInstance() { + return new Md5(randomSource(), randomChild()); + } + + @Override + protected Md5 mutateInstance(Md5 instance) throws IOException { + return new Md5(instance.source(), mutateExpression(instance.field())); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java new file mode 100644 index 0000000000000..3c0a2067a81b0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class Md5Tests extends AbstractScalarFunctionTestCase { + + public Md5Tests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List cases = new ArrayList<>(); + HashTests.addHashFunctionTestCases(cases, "MD5"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); + } + + @Override + protected Expression build(Source source, List args) { + return new Md5(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimErrorTests.java new file mode 100644 index 0000000000000..38ab08345428b --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class RTrimErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(RTrimTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new RTrim(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceErrorTests.java new file mode 100644 index 0000000000000..ed1cbda5c3de0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ReplaceErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ReplaceTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Replace(source, args.get(0), args.get(1), args.get(2)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java index bf1325854f1a2..dfc4db228f8e6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java @@ -103,7 +103,7 @@ public static Iterable parameters() { "Unclosed character class near index 0\n[\n^".replaceAll("\n", System.lineSeparator()) ); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(false, suppliers); } private static TestCaseSupplier fixedCase(String name, String str, String oldStr, String newStr, String result) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightErrorTests.java new file mode 100644 index 0000000000000..3b90a26fb908f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class RightErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(RightTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Right(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "string"; + case 1 -> "integer"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java index bf93ef42ed6ad..1a65407e538c2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java @@ -179,11 +179,7 @@ public static Iterable parameters() { equalTo(new BytesRef(unicodeRightSubstring(text, length))) ); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { - case 0 -> "string"; - case 1 -> "integer"; - default -> throw new IllegalStateException("bad parameter number"); - }); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } private static String unicodeRightSubstring(String str, int length) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java new file mode 100644 index 0000000000000..613aabe658ceb --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class Sha1ErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + + @Override + protected List cases() { + return paramsToSuppliers(Sha1Tests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Sha1(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java new file mode 100644 index 0000000000000..12c9cc7580de9 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class Sha1SerializationTests extends AbstractExpressionSerializationTests { + + @Override + protected Sha1 createTestInstance() { + return new Sha1(randomSource(), randomChild()); + } + + @Override + protected Sha1 mutateInstance(Sha1 instance) throws IOException { + return new Sha1(instance.source(), mutateExpression(instance.field())); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java new file mode 100644 index 0000000000000..06c23f186d4c8 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class Sha1Tests extends AbstractScalarFunctionTestCase { + + public Sha1Tests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List cases = new ArrayList<>(); + HashTests.addHashFunctionTestCases(cases, "SHA1"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); + } + + @Override + protected Expression build(Source source, List args) { + return new Sha1(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java new file mode 100644 index 0000000000000..a9678346f8508 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class Sha256ErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + + @Override + protected List cases() { + return paramsToSuppliers(Sha256Tests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Sha256(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java new file mode 100644 index 0000000000000..2f209fef25d36 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class Sha256SerializationTests extends AbstractExpressionSerializationTests { + + @Override + protected Sha256 createTestInstance() { + return new Sha256(randomSource(), randomChild()); + } + + @Override + protected Sha256 mutateInstance(Sha256 instance) throws IOException { + return new Sha256(instance.source(), mutateExpression(instance.field())); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java new file mode 100644 index 0000000000000..0a136e3214082 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class Sha256Tests extends AbstractScalarFunctionTestCase { + + public Sha256Tests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List cases = new ArrayList<>(); + HashTests.addHashFunctionTestCases(cases, "SHA256"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); + } + + @Override + protected Expression build(Source source, List args) { + return new Sha256(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceErrorTests.java new file mode 100644 index 0000000000000..bee85a645743b --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class SpaceErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(SpaceTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Space(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "integer")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceTests.java index 308ce2c9d932f..40c9b33609a31 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceTests.java @@ -71,7 +71,7 @@ public static Iterable parameters() { .withFoldingException(IllegalArgumentException.class, "Creating strings longer than [" + max + "] bytes is not supported"); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "integer"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitErrorTests.java new file mode 100644 index 0000000000000..c113f52a1824a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class SplitErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(SplitTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Split(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java index 098be8e1fda37..c66812ccdd1df 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java @@ -63,7 +63,7 @@ public static Iterable parameters() { })); } } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithErrorTests.java new file mode 100644 index 0000000000000..9ca061997e762 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class StartsWithErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(StartsWithTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new StartsWith(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java index 60ed3b05ad642..789059fb7b6ba 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java @@ -51,7 +51,7 @@ public static Iterable parameters() { })); } } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (valid, position) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringErrorTests.java new file mode 100644 index 0000000000000..245f4e2254e50 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class SubstringErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(SubstringTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Substring(source, args.get(0), args.get(1), args.size() < 3 ? null : args.get(2)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "string"; + case 1, 2 -> "integer"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java index ae8a2a1840dfb..289cc0129c4e9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java @@ -36,7 +36,7 @@ public SubstringTests(@Name("TestCase") Supplier test @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedDataWithDefaultChecks( + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( true, List.of(new TestCaseSupplier("Substring basic test", List.of(DataType.KEYWORD, DataType.INTEGER, DataType.INTEGER), () -> { int start = between(1, 8); @@ -105,12 +105,7 @@ public static Iterable parameters() { equalTo(new BytesRef("")) ); }) - ), - (v, p) -> switch (p) { - case 0 -> "string"; - case 1, 2 -> "integer"; - default -> ""; - } + ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerErrorTests.java new file mode 100644 index 0000000000000..5744f340627c6 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerErrorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToLowerErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToLowerTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToLower(source, args.get(0), EsqlTestUtils.TEST_CFG); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java index 026d190c06e3f..b355feb6130a3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java @@ -47,9 +47,7 @@ public static Iterable parameters() { suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); suppliers.add(supplier("semantic_text ascii", DataType.SEMANTIC_TEXT, () -> randomAlphaOfLengthBetween(1, 10))); suppliers.add(supplier("semantic_text unicode", DataType.SEMANTIC_TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); - - // add null as parameter - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } public void testRandomLocale() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperErrorTests.java new file mode 100644 index 0000000000000..94c6dc6e4a29a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperErrorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToUpperErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToUpperTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToUpper(source, args.get(0), EsqlTestUtils.TEST_CFG); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java index 027ac54d15875..fdae4f953a0fa 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java @@ -47,9 +47,7 @@ public static Iterable parameters() { suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); suppliers.add(supplier("semantic_text ascii", DataType.SEMANTIC_TEXT, () -> randomAlphaOfLengthBetween(1, 10))); suppliers.add(supplier("semantic_text unicode", DataType.SEMANTIC_TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); - - // add null as parameter - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } public void testRandomLocale() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimErrorTests.java new file mode 100644 index 0000000000000..dd6de5ba9f404 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class TrimErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(TrimTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Trim(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegErrorTests.java new file mode 100644 index 0000000000000..fcb877f15e41c --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class NegErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(NegTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Neg(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "numeric, date_period or time_duration")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java index 5bfaccfbd9347..a8c7b5b5a83fd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java @@ -110,7 +110,7 @@ public static Iterable parameters() { equalTo(arg.negated()) ).withoutEvaluator(); }))); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "numeric, date_period or time_duration"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(false, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java new file mode 100644 index 0000000000000..cecb6a2987fd4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class EqualsErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(EqualsTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Equals(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(errorMessageStringForBinaryOperators(validPerPosition, signature, (l, p) -> TYPE_ERROR)); + } + + private static final String TYPE_ERROR = + "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long," + + " semantic_text, text, unsigned_long or version"; +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java index 6666eb8adab61..8e46f7d28540d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java @@ -236,18 +236,9 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples( - anyNullIsNull(true, suppliers), - (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(o, v, t, (l, p) -> typeErrorString) - ) - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } - private static String typeErrorString = - "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long," - + " semantic_text, text, unsigned_long or version"; - @Override protected Expression build(Source source, List args) { return new Equals(source, args.get(0), args.get(1)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanErrorTests.java new file mode 100644 index 0000000000000..5b56642bb6241 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanErrorTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class GreaterThanErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(GreaterThanTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new GreaterThan(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo( + errorMessageStringForBinaryOperators( + validPerPosition, + signature, + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" + ) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualErrorTests.java new file mode 100644 index 0000000000000..904affebd3b5e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualErrorTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class GreaterThanOrEqualErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(GreaterThanOrEqualTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new GreaterThanOrEqual(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo( + errorMessageStringForBinaryOperators( + validPerPosition, + signature, + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" + ) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java index 0fbd49abd885b..f00e8a19f9dac 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java @@ -158,17 +158,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples( - anyNullIsNull(true, suppliers), - (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators( - o, - v, - t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" - ) - ) - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java index ccc66df60fb3f..169f1886af47f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java @@ -172,17 +172,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples( - anyNullIsNull(true, suppliers), - (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators( - o, - v, - t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" - ) - ) - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanErrorTests.java new file mode 100644 index 0000000000000..94e3be44ba2f2 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanErrorTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class LessThanErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(LessThanTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new LessThan(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo( + errorMessageStringForBinaryOperators( + validPerPosition, + signature, + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" + ) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualErrorTests.java new file mode 100644 index 0000000000000..6074cc7c63db2 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualErrorTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class LessThanOrEqualErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(LessThanOrEqualTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new LessThanOrEqual(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo( + errorMessageStringForBinaryOperators( + validPerPosition, + signature, + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" + ) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java index 1e91a65e04c0e..7eccc2c8cc86b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java @@ -158,17 +158,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples( - anyNullIsNull(true, suppliers), - (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators( - o, - v, - t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" - ) - ) - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java index 69dc59bac6456..a17be8aa97978 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java @@ -172,17 +172,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples( - anyNullIsNull(true, suppliers), - (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators( - o, - v, - t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" - ) - ) - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java new file mode 100644 index 0000000000000..ed0a477ec613c --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class NotEqualsErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(NotEqualsTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new NotEquals(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(errorMessageStringForBinaryOperators(validPerPosition, signature, (l, p) -> TYPE_ERROR)); + } + + private static final String TYPE_ERROR = + "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long, text, " + + "unsigned_long or version"; +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java index 7b57b97dfe28e..296b4b37cfa73 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java @@ -232,18 +232,9 @@ public static Iterable parameters() { false ) ); - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples( - anyNullIsNull(true, suppliers), - (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(o, v, t, (l, p) -> typeErrorString) - ) - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } - private static String typeErrorString = - "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long, text, " - + "unsigned_long or version"; - @Override protected Expression build(Source source, List args) { return new NotEquals(source, args.get(0), args.get(1)); diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java index d9dffdefafa2c..55ec4065be8c4 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; @@ -43,6 +44,7 @@ import org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexTransportAction; import org.elasticsearch.xpack.migrate.action.ReindexDataStreamTransportAction; import org.elasticsearch.xpack.migrate.rest.RestCancelReindexDataStreamAction; +import org.elasticsearch.xpack.migrate.rest.RestCreateIndexFromSourceAction; import org.elasticsearch.xpack.migrate.rest.RestGetMigrationReindexStatusAction; import org.elasticsearch.xpack.migrate.rest.RestMigrationReindexAction; import org.elasticsearch.xpack.migrate.task.ReindexDataStreamPersistentTaskExecutor; @@ -57,6 +59,7 @@ import java.util.function.Supplier; import static org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.REINDEX_DATA_STREAM_FEATURE_FLAG; +import static org.elasticsearch.xpack.migrate.task.ReindexDataStreamPersistentTaskExecutor.MAX_CONCURRENT_INDICES_REINDEXED_PER_DATA_STREAM_SETTING; public class MigratePlugin extends Plugin implements ActionPlugin, PersistentTaskPlugin { @@ -77,6 +80,7 @@ public List getRestHandlers( handlers.add(new RestMigrationReindexAction()); handlers.add(new RestGetMigrationReindexStatusAction()); handlers.add(new RestCancelReindexDataStreamAction()); + handlers.add(new RestCreateIndexFromSourceAction()); } return handlers; } @@ -151,4 +155,11 @@ public List> getPersistentTasksExecutor( return List.of(); } } + + @Override + public List> getSettings() { + List> pluginSettings = new ArrayList<>(); + pluginSettings.add(MAX_CONCURRENT_INDICES_REINDEXED_PER_DATA_STREAM_SETTING); + return pluginSettings; + } } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java index d67eaee3d251f..b2c6a3ea4b779 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java @@ -15,6 +15,11 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xcontent.ObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.util.Map; @@ -30,18 +35,35 @@ private CreateIndexFromSourceAction() { super(NAME); } - public static class Request extends ActionRequest implements IndicesRequest { - + public static class Request extends ActionRequest implements IndicesRequest, ToXContent { private final String sourceIndex; private final String destIndex; - private final Settings settingsOverride; - private final Map mappingsOverride; + private Settings settingsOverride = Settings.EMPTY; + private Map mappingsOverride = Map.of(); + private static final ParseField SETTINGS_OVERRIDE_FIELD = new ParseField("settings_override"); + private static final ParseField MAPPINGS_OVERRIDE_FIELD = new ParseField("mappings_override"); + private static final ObjectParser PARSER = new ObjectParser<>("create_index_from_source_request"); + + static { + PARSER.declareField( + (parser, request, context) -> request.settingsOverride(Settings.fromXContent(parser)), + SETTINGS_OVERRIDE_FIELD, + ObjectParser.ValueType.OBJECT + ); + + PARSER.declareField( + (parser, request, context) -> request.mappingsOverride(Map.of("_doc", parser.map())), + MAPPINGS_OVERRIDE_FIELD, + ObjectParser.ValueType.OBJECT + ); + } public Request(String sourceIndex, String destIndex) { this(sourceIndex, destIndex, Settings.EMPTY, Map.of()); } public Request(String sourceIndex, String destIndex, Settings settingsOverride, Map mappingsOverride) { + Objects.requireNonNull(settingsOverride); Objects.requireNonNull(mappingsOverride); this.sourceIndex = sourceIndex; this.destIndex = destIndex; @@ -72,22 +94,52 @@ public ActionRequestValidationException validate() { return null; } - public String getSourceIndex() { + public String sourceIndex() { return sourceIndex; } - public String getDestIndex() { + public String destIndex() { return destIndex; } - public Settings getSettingsOverride() { + public Settings settingsOverride() { return settingsOverride; } - public Map getMappingsOverride() { + public Map mappingsOverride() { return mappingsOverride; } + public void settingsOverride(Settings settingsOverride) { + this.settingsOverride = settingsOverride; + } + + public void mappingsOverride(Map mappingsOverride) { + this.mappingsOverride = mappingsOverride; + } + + public void fromXContent(XContentParser parser) throws IOException { + PARSER.parse(parser, this, null); + } + + /* + * This only exists for the sake of testing the xcontent parser + */ + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + if (mappingsOverride.containsKey("_doc")) { + builder.field(MAPPINGS_OVERRIDE_FIELD.getPreferredName(), mappingsOverride.get("_doc")); + } + + if (settingsOverride.isEmpty() == false) { + builder.startObject(SETTINGS_OVERRIDE_FIELD.getPreferredName()); + settingsOverride.toXContent(builder, params); + builder.endObject(); + } + + return builder; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java index 7739a9346d47c..09f9d48b62558 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java @@ -69,31 +69,31 @@ public CreateIndexFromSourceTransportAction( @Override protected void doExecute(Task task, CreateIndexFromSourceAction.Request request, ActionListener listener) { - IndexMetadata sourceIndex = clusterService.state().getMetadata().index(request.getSourceIndex()); + IndexMetadata sourceIndex = clusterService.state().getMetadata().index(request.sourceIndex()); if (sourceIndex == null) { - listener.onFailure(new IndexNotFoundException(request.getSourceIndex())); + listener.onFailure(new IndexNotFoundException(request.sourceIndex())); return; } - logger.debug("Creating destination index [{}] for source index [{}]", request.getDestIndex(), request.getSourceIndex()); + logger.debug("Creating destination index [{}] for source index [{}]", request.destIndex(), request.sourceIndex()); Settings settings = Settings.builder() // add source settings .put(filterSettings(sourceIndex)) // add override settings from request - .put(request.getSettingsOverride()) + .put(request.settingsOverride()) .build(); Map mergeMappings; try { - mergeMappings = mergeMappings(sourceIndex.mapping(), request.getMappingsOverride()); + mergeMappings = mergeMappings(sourceIndex.mapping(), request.mappingsOverride()); } catch (IOException e) { listener.onFailure(e); return; } - var createIndexRequest = new CreateIndexRequest(request.getDestIndex()).settings(settings); + var createIndexRequest = new CreateIndexRequest(request.destIndex()).settings(settings); if (mergeMappings.isEmpty() == false) { createIndexRequest.mapping(mergeMappings); } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java index 8cba57240ad66..6dd38d35ca9f7 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java @@ -156,7 +156,7 @@ private void createIndex( Map.of() ); request.setParentTask(parentTaskId); - var errorMessage = String.format(Locale.ROOT, "Could not create index [%s]", request.getDestIndex()); + var errorMessage = String.format(Locale.ROOT, "Could not create index [%s]", request.destIndex()); client.execute(CreateIndexFromSourceAction.INSTANCE, request, failIfNotAcknowledged(listener, errorMessage)); } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/rest/RestCreateIndexFromSourceAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/rest/RestCreateIndexFromSourceAction.java new file mode 100644 index 0000000000000..15306f65499e9 --- /dev/null +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/rest/RestCreateIndexFromSourceAction.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.rest; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceAction; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.rest.RestRequest.Method.PUT; + +public class RestCreateIndexFromSourceAction extends BaseRestHandler { + + @Override + public String getName() { + return "create_index_from_source_action"; + } + + @Override + public List routes() { + return List.of(new Route(PUT, "/_create_from/{source}/{dest}"), new Route(POST, "/_create_from/{source}/{dest}")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + var createRequest = new CreateIndexFromSourceAction.Request(request.param("source"), request.param("dest")); + request.applyContentParser(createRequest::fromXContent); + return channel -> client.execute(CreateIndexFromSourceAction.INSTANCE, createRequest, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java index 85995d5f9575f..20cdd27c72917 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.migrate.task; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.rollover.RolloverAction; @@ -19,6 +21,8 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamAction; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.persistent.AllocatedPersistentTask; @@ -37,6 +41,19 @@ import static org.elasticsearch.xpack.core.deprecation.DeprecatedIndexPredicate.getReindexRequiredPredicate; public class ReindexDataStreamPersistentTaskExecutor extends PersistentTasksExecutor { + /* + * This setting controls how many indices we reindex concurrently for a single data stream. This is not an overall limit -- if five + * data streams are being reindexed, then each of them can have this many indices being reindexed at once. This setting is dynamic, + * but changing it does not have an impact if the task is already running (unless the task is restarted or moves to another node). + */ + public static final Setting MAX_CONCURRENT_INDICES_REINDEXED_PER_DATA_STREAM_SETTING = Setting.intSetting( + "migrate.max_concurrent_indices_reindexed_per_data_stream", + 1, + 1, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + private static final Logger logger = LogManager.getLogger(ReindexDataStreamPersistentTaskExecutor.class); private static final TimeValue TASK_KEEP_ALIVE_TIME = TimeValue.timeValueDays(1); private final Client client; private final ClusterService clusterService; @@ -60,6 +77,7 @@ protected ReindexDataStreamTask createTask( ) { ReindexDataStreamTaskParams params = taskInProgress.getParams(); return new ReindexDataStreamTask( + clusterService, params.startTime(), params.totalIndices(), params.totalIndicesToBeUpgraded(), @@ -73,7 +91,12 @@ protected ReindexDataStreamTask createTask( } @Override - protected void nodeOperation(AllocatedPersistentTask task, ReindexDataStreamTaskParams params, PersistentTaskState state) { + protected void nodeOperation( + AllocatedPersistentTask task, + ReindexDataStreamTaskParams params, + PersistentTaskState persistentTaskState + ) { + ReindexDataStreamPersistentTaskState state = (ReindexDataStreamPersistentTaskState) persistentTaskState; String sourceDataStream = params.getSourceDataStream(); TaskId taskId = new TaskId(clusterService.localNode().getId(), task.getId()); GetDataStreamAction.Request request = new GetDataStreamAction.Request(TimeValue.MAX_VALUE, new String[] { sourceDataStream }); @@ -92,35 +115,74 @@ protected void nodeOperation(AllocatedPersistentTask task, ReindexDataStreamTask RolloverAction.INSTANCE, rolloverRequest, ActionListener.wrap( - rolloverResponse -> reindexIndices(dataStream, reindexDataStreamTask, reindexClient, sourceDataStream, taskId), - e -> completeFailedPersistentTask(reindexDataStreamTask, e) + rolloverResponse -> reindexIndices( + dataStream, + dataStream.getIndices().size() + 1, + reindexDataStreamTask, + params, + state, + reindexClient, + sourceDataStream, + taskId + ), + e -> completeFailedPersistentTask(reindexDataStreamTask, state, e) ) ); } else { - reindexIndices(dataStream, reindexDataStreamTask, reindexClient, sourceDataStream, taskId); + reindexIndices( + dataStream, + dataStream.getIndices().size(), + reindexDataStreamTask, + params, + state, + reindexClient, + sourceDataStream, + taskId + ); } } else { - completeFailedPersistentTask(reindexDataStreamTask, new ElasticsearchException("data stream does not exist")); + completeFailedPersistentTask(reindexDataStreamTask, state, new ElasticsearchException("data stream does not exist")); } - }, exception -> completeFailedPersistentTask(reindexDataStreamTask, exception))); + }, exception -> completeFailedPersistentTask(reindexDataStreamTask, state, exception))); } private void reindexIndices( DataStream dataStream, + int totalIndicesInDataStream, ReindexDataStreamTask reindexDataStreamTask, + ReindexDataStreamTaskParams params, + ReindexDataStreamPersistentTaskState state, ExecuteWithHeadersClient reindexClient, String sourceDataStream, TaskId parentTaskId ) { List indices = dataStream.getIndices(); List indicesToBeReindexed = indices.stream().filter(getReindexRequiredPredicate(clusterService.state().metadata())).toList(); + final ReindexDataStreamPersistentTaskState updatedState; + if (params.totalIndices() != totalIndicesInDataStream + || params.totalIndicesToBeUpgraded() != indicesToBeReindexed.size() + || (state != null + && (state.totalIndices() != null + && state.totalIndicesToBeUpgraded() != null + && (state.totalIndices() != totalIndicesInDataStream + || state.totalIndicesToBeUpgraded() != indicesToBeReindexed.size())))) { + updatedState = new ReindexDataStreamPersistentTaskState( + totalIndicesInDataStream, + indicesToBeReindexed.size(), + state == null ? null : state.completionTime() + ); + reindexDataStreamTask.updatePersistentTaskState(updatedState, ActionListener.noop()); + } else { + updatedState = state; + } reindexDataStreamTask.setPendingIndicesCount(indicesToBeReindexed.size()); // The CountDownActionListener is 1 more than the number of indices so that the count is not 0 if we have no indices CountDownActionListener listener = new CountDownActionListener(indicesToBeReindexed.size() + 1, ActionListener.wrap(response1 -> { - completeSuccessfulPersistentTask(reindexDataStreamTask); - }, exception -> { completeFailedPersistentTask(reindexDataStreamTask, exception); })); + completeSuccessfulPersistentTask(reindexDataStreamTask, updatedState); + }, exception -> { completeFailedPersistentTask(reindexDataStreamTask, updatedState, exception); })); + final int maxConcurrentIndices = clusterService.getClusterSettings().get(MAX_CONCURRENT_INDICES_REINDEXED_PER_DATA_STREAM_SETTING); List indicesRemaining = Collections.synchronizedList(new ArrayList<>(indicesToBeReindexed)); - final int maxConcurrentIndices = 1; + logger.debug("Reindexing {} indices, with up to {} handled concurrently", indicesRemaining.size(), maxConcurrentIndices); for (int i = 0; i < maxConcurrentIndices; i++) { maybeProcessNextIndex(indicesRemaining, reindexDataStreamTask, reindexClient, sourceDataStream, listener, parentTaskId); } @@ -190,15 +252,25 @@ public void onFailure(Exception e) { }); } - private void completeSuccessfulPersistentTask(ReindexDataStreamTask persistentTask) { - persistentTask.allReindexesCompleted(threadPool, getTimeToLive(persistentTask)); + private void completeSuccessfulPersistentTask( + ReindexDataStreamTask persistentTask, + @Nullable ReindexDataStreamPersistentTaskState state + ) { + persistentTask.allReindexesCompleted(threadPool, updateCompletionTimeAndGetTimeToLive(persistentTask, state)); } - private void completeFailedPersistentTask(ReindexDataStreamTask persistentTask, Exception e) { - persistentTask.taskFailed(threadPool, getTimeToLive(persistentTask), e); + private void completeFailedPersistentTask( + ReindexDataStreamTask persistentTask, + @Nullable ReindexDataStreamPersistentTaskState state, + Exception e + ) { + persistentTask.taskFailed(threadPool, updateCompletionTimeAndGetTimeToLive(persistentTask, state), e); } - private TimeValue getTimeToLive(ReindexDataStreamTask reindexDataStreamTask) { + private TimeValue updateCompletionTimeAndGetTimeToLive( + ReindexDataStreamTask reindexDataStreamTask, + @Nullable ReindexDataStreamPersistentTaskState state + ) { PersistentTasksCustomMetadata persistentTasksCustomMetadata = clusterService.state() .getMetadata() .custom(PersistentTasksCustomMetadata.TYPE); @@ -208,16 +280,23 @@ private TimeValue getTimeToLive(ReindexDataStreamTask reindexDataStreamTask) { if (persistentTask == null) { return TimeValue.timeValueMillis(0); } - PersistentTaskState state = persistentTask.getState(); final long completionTime; if (state == null) { completionTime = threadPool.absoluteTimeInMillis(); reindexDataStreamTask.updatePersistentTaskState( - new ReindexDataStreamPersistentTaskState(completionTime), + new ReindexDataStreamPersistentTaskState(null, null, completionTime), ActionListener.noop() ); } else { - completionTime = ((ReindexDataStreamPersistentTaskState) state).completionTime(); + if (state.completionTime() == null) { + completionTime = threadPool.absoluteTimeInMillis(); + reindexDataStreamTask.updatePersistentTaskState( + new ReindexDataStreamPersistentTaskState(state.totalIndices(), state.totalIndicesToBeUpgraded(), completionTime), + ActionListener.noop() + ); + } else { + completionTime = state.completionTime(); + } } return TimeValue.timeValueMillis(TASK_KEEP_ALIVE_TIME.millis() - (threadPool.absoluteTimeInMillis() - completionTime)); } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskState.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskState.java index 130a8f7ce372b..8ab22674082e6 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskState.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskState.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; import org.elasticsearch.persistent.PersistentTaskState; import org.elasticsearch.tasks.Task; import org.elasticsearch.xcontent.ConstructingObjectParser; @@ -18,22 +19,31 @@ import java.io.IOException; -import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public record ReindexDataStreamPersistentTaskState( + @Nullable Integer totalIndices, + @Nullable Integer totalIndicesToBeUpgraded, + @Nullable Long completionTime +) implements Task.Status, PersistentTaskState { -public record ReindexDataStreamPersistentTaskState(long completionTime) implements Task.Status, PersistentTaskState { public static final String NAME = ReindexDataStreamTask.TASK_NAME; + private static final String TOTAL_INDICES_FIELD = "total_indices_in_data_stream"; + private static final String TOTAL_INDICES_REQUIRING_UPGRADE_FIELD = "total_indices_requiring_upgrade"; private static final String COMPLETION_TIME_FIELD = "completion_time"; private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, true, - args -> new ReindexDataStreamPersistentTaskState((long) args[0]) + args -> new ReindexDataStreamPersistentTaskState((Integer) args[0], (Integer) args[1], (Long) args[2]) ); static { - PARSER.declareLong(constructorArg(), new ParseField(COMPLETION_TIME_FIELD)); + PARSER.declareInt(optionalConstructorArg(), new ParseField(TOTAL_INDICES_FIELD)); + PARSER.declareInt(optionalConstructorArg(), new ParseField(TOTAL_INDICES_REQUIRING_UPGRADE_FIELD)); + PARSER.declareLong(optionalConstructorArg(), new ParseField(COMPLETION_TIME_FIELD)); } public ReindexDataStreamPersistentTaskState(StreamInput in) throws IOException { - this(in.readLong()); + this(in.readOptionalInt(), in.readOptionalInt(), in.readOptionalLong()); } @Override @@ -43,13 +53,23 @@ public String getWriteableName() { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeLong(completionTime); + out.writeOptionalInt(totalIndices); + out.writeOptionalInt(totalIndicesToBeUpgraded); + out.writeOptionalLong(completionTime); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(COMPLETION_TIME_FIELD, completionTime); + if (totalIndices != null) { + builder.field(TOTAL_INDICES_FIELD, totalIndices); + } + if (totalIndicesToBeUpgraded != null) { + builder.field(TOTAL_INDICES_REQUIRING_UPGRADE_FIELD, totalIndicesToBeUpgraded); + } + if (completionTime != null) { + builder.field(COMPLETION_TIME_FIELD, completionTime); + } builder.endObject(); return builder; } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java index 7a2b759dfd17a..04295a7521479 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamTask.java @@ -7,10 +7,12 @@ package org.elasticsearch.xpack.migrate.task; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.RunOnce; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.persistent.AllocatedPersistentTask; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; @@ -24,9 +26,10 @@ public class ReindexDataStreamTask extends AllocatedPersistentTask { public static final String TASK_NAME = "reindex-data-stream"; + private final ClusterService clusterService; private final long persistentTaskStartTime; - private final int totalIndices; - private final int totalIndicesToBeUpgraded; + private final int initialTotalIndices; + private final int initialTotalIndicesToBeUpgraded; private volatile boolean complete = false; private volatile Exception exception; private final Set inProgress = Collections.synchronizedSet(new HashSet<>()); @@ -36,9 +39,10 @@ public class ReindexDataStreamTask extends AllocatedPersistentTask { @SuppressWarnings("this-escape") public ReindexDataStreamTask( + ClusterService clusterService, long persistentTaskStartTime, - int totalIndices, - int totalIndicesToBeUpgraded, + int initialTotalIndices, + int initialTotalIndicesToBeUpgraded, long id, String type, String action, @@ -47,9 +51,10 @@ public ReindexDataStreamTask( Map headers ) { super(id, type, action, description, parentTask, headers); + this.clusterService = clusterService; this.persistentTaskStartTime = persistentTaskStartTime; - this.totalIndices = totalIndices; - this.totalIndicesToBeUpgraded = totalIndicesToBeUpgraded; + this.initialTotalIndices = initialTotalIndices; + this.initialTotalIndicesToBeUpgraded = initialTotalIndicesToBeUpgraded; this.completeTask = new RunOnce(() -> { if (exception == null) { markAsCompleted(); @@ -61,6 +66,19 @@ public ReindexDataStreamTask( @Override public ReindexDataStreamStatus getStatus() { + PersistentTasksCustomMetadata persistentTasksCustomMetadata = clusterService.state() + .getMetadata() + .custom(PersistentTasksCustomMetadata.TYPE); + int totalIndices = initialTotalIndices; + int totalIndicesToBeUpgraded = initialTotalIndicesToBeUpgraded; + PersistentTasksCustomMetadata.PersistentTask persistentTask = persistentTasksCustomMetadata.getTask(getPersistentTaskId()); + if (persistentTask != null) { + ReindexDataStreamPersistentTaskState state = (ReindexDataStreamPersistentTaskState) persistentTask.getState(); + if (state != null && state.totalIndices() != null && state.totalIndicesToBeUpgraded() != null) { + totalIndices = Math.toIntExact(state.totalIndices()); + totalIndicesToBeUpgraded = Math.toIntExact(state.totalIndicesToBeUpgraded()); + } + } return new ReindexDataStreamStatus( persistentTaskStartTime, totalIndices, diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/CreateFromSourceIndexRequestTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/CreateFromSourceIndexRequestTests.java new file mode 100644 index 0000000000000..21dab66fccd5e --- /dev/null +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/CreateFromSourceIndexRequestTests.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.action; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceAction.Request; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; + +public class CreateFromSourceIndexRequestTests extends AbstractWireSerializingTestCase { + + public void testToAndFromXContent() throws IOException { + var request = createTestInstance(); + + boolean humanReadable = randomBoolean(); + final XContentType xContentType = randomFrom(XContentType.values()); + BytesReference originalBytes = toShuffledXContent(request, xContentType, EMPTY_PARAMS, humanReadable); + + var parsedRequest = new Request( + randomValueOtherThan(request.sourceIndex(), () -> randomAlphanumericOfLength(30)), + randomValueOtherThan(request.sourceIndex(), () -> randomAlphanumericOfLength(30)) + ); + try (XContentParser xParser = createParser(xContentType.xContent(), originalBytes)) { + parsedRequest.fromXContent(xParser); + } + + // source and dest won't be equal + assertNotEquals(request, parsedRequest); + assertNotEquals(request.sourceIndex(), parsedRequest.sourceIndex()); + assertNotEquals(request.destIndex(), parsedRequest.destIndex()); + + // but fields in xcontent will be equal + assertEquals(request.settingsOverride(), parsedRequest.settingsOverride()); + assertEquals(request.mappingsOverride(), parsedRequest.mappingsOverride()); + + BytesReference finalBytes = toShuffledXContent(parsedRequest, xContentType, EMPTY_PARAMS, humanReadable); + ElasticsearchAssertions.assertToXContentEquivalent(originalBytes, finalBytes, xContentType); + } + + @Override + protected Writeable.Reader instanceReader() { + return Request::new; + } + + @Override + protected Request createTestInstance() { + String source = randomAlphaOfLength(30); + String dest = randomAlphaOfLength(30); + if (randomBoolean()) { + return new Request(source, dest); + } else { + return new Request(source, dest, randomSettings(), randomMappings()); + } + } + + @Override + protected Request mutateInstance(Request instance) throws IOException { + + String sourceIndex = instance.sourceIndex(); + String destIndex = instance.destIndex(); + Settings settingsOverride = instance.settingsOverride(); + Map mappingsOverride = instance.mappingsOverride(); + + switch (between(0, 3)) { + case 0 -> sourceIndex = randomValueOtherThan(sourceIndex, () -> randomAlphaOfLength(30)); + case 1 -> destIndex = randomValueOtherThan(destIndex, () -> randomAlphaOfLength(30)); + case 2 -> settingsOverride = randomValueOtherThan(settingsOverride, CreateFromSourceIndexRequestTests::randomSettings); + case 3 -> mappingsOverride = randomValueOtherThan(mappingsOverride, CreateFromSourceIndexRequestTests::randomMappings); + } + return new Request(sourceIndex, destIndex, settingsOverride, mappingsOverride); + } + + public static Map randomMappings() { + var randMappings = Map.of("properties", Map.of(randomAlphaOfLength(5), Map.of("type", "keyword"))); + return randomBoolean() ? Map.of() : Map.of("_doc", randMappings); + } + + public static Settings randomSettings() { + return randomBoolean() ? Settings.EMPTY : indexSettings(randomIntBetween(1, 10), randomIntBetween(0, 5)).build(); + } +} diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskStateTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskStateTests.java index 140558b997e4b..c2dee83a91dfe 100644 --- a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskStateTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskStateTests.java @@ -26,11 +26,32 @@ protected Writeable.Reader instanceReader( @Override protected ReindexDataStreamPersistentTaskState createTestInstance() { - return new ReindexDataStreamPersistentTaskState(randomNonNegativeLong()); + return new ReindexDataStreamPersistentTaskState( + randomBoolean() ? null : randomNonNegativeInt(), + randomBoolean() ? null : randomNonNegativeInt(), + randomBoolean() ? null : randomNonNegativeLong() + ); } @Override protected ReindexDataStreamPersistentTaskState mutateInstance(ReindexDataStreamPersistentTaskState instance) throws IOException { - return new ReindexDataStreamPersistentTaskState(instance.completionTime() + 1); + return switch (randomInt(2)) { + case 0 -> new ReindexDataStreamPersistentTaskState( + instance.totalIndices() == null ? randomNonNegativeInt() : instance.totalIndices() + 1, + instance.totalIndicesToBeUpgraded(), + instance.completionTime() + ); + case 1 -> new ReindexDataStreamPersistentTaskState( + instance.totalIndices(), + instance.totalIndicesToBeUpgraded() == null ? randomNonNegativeInt() : instance.totalIndicesToBeUpgraded() + 1, + instance.completionTime() + ); + case 2 -> new ReindexDataStreamPersistentTaskState( + instance.totalIndices(), + instance.totalIndicesToBeUpgraded(), + instance.completionTime() == null ? randomNonNegativeLong() : instance.completionTime() + 1 + ); + default -> throw new IllegalArgumentException("Should never get here"); + }; } } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java index cb4dde4e1cf0b..9d72b9eb5c526 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java @@ -93,7 +93,6 @@ protected boolean transportSSLEnabled() { } public void testThatSSLConfigurationReloadsOnModification() throws Exception { - assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm()); Path keyPath = createTempDir().resolve("testnode_updated.pem"); Path certPath = createTempDir().resolve("testnode_updated.crt"); Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.pem"), keyPath); @@ -127,6 +126,9 @@ public void testThatSSLConfigurationReloadsOnModification() throws Exception { fail("handshake should not have been successful!"); } catch (SSLException | SocketException expected) { logger.trace("expected exception", expected); + } catch (IOException e) { + // BouncyCastle throws a different exception + assertThat(e.getClass().getName(), is("org.bouncycastle.tls.TlsFatalAlertReceived")); } // Copy testnode_updated.crt to the placeholder updateable.crt so that the nodes will start trusting it now try { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageCertificateVerificationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageCertificateVerificationTests.java index ad8e15db6f032..ccb73c9ead210 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageCertificateVerificationTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageCertificateVerificationTests.java @@ -39,7 +39,6 @@ import java.util.regex.Pattern; import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLHandshakeException; import javax.net.ssl.SSLParameters; import javax.net.ssl.SSLSocket; import javax.net.ssl.SSLSocketFactory; @@ -104,7 +103,6 @@ public void testMessageForRestClientHostnameVerificationFailure() throws IOExcep } public void testDiagnosticTrustManagerForHostnameVerificationFailure() throws Exception { - assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm()); final Settings settings = getPemSSLSettings( HTTP_SERVER_SSL, "not-this-host.crt", @@ -133,7 +131,7 @@ public void testDiagnosticTrustManagerForHostnameVerificationFailure() throws Ex DiagnosticTrustManager.class.getName(), Level.WARN, "failed to establish trust with server at \\[" - + Pattern.quote(webServer.getHostName()) + + (inFipsJvm() ? "" : Pattern.quote(webServer.getHostName())) + "\\];" + " the server provided a certificate with subject name \\[CN=not-this-host\\]," + " fingerprint \\[[0-9a-f]{40}\\], no keyUsage and no extendedKeyUsage;" @@ -154,13 +152,12 @@ public void testDiagnosticTrustManagerForHostnameVerificationFailure() throws Ex enableHttpsHostnameChecking(clientSocket); connect(clientSocket, webServer); assertThat(clientSocket.isConnected(), is(true)); - final SSLHandshakeException handshakeException = expectThrows( - SSLHandshakeException.class, - () -> clientSocket.getInputStream().read() - ); - assertThat(handshakeException, throwableWithMessage(containsStringIgnoringCase("subject alternative names"))); - assertThat(handshakeException, throwableWithMessage(containsString(webServer.getHostName()))); - + final Exception handshakeException = expectThrows(Exception.class, () -> clientSocket.getInputStream().read()); + // Bouncy Castle throws a different exception message + if (inFipsJvm() == false) { + assertThat(handshakeException, throwableWithMessage(containsStringIgnoringCase("subject alternative names"))); + assertThat(handshakeException, throwableWithMessage(containsString(webServer.getHostName()))); + } // Logging message failures are tricky to debug because you just get a "didn't find match" assertion failure. // You should be able to check the log output for the text that was logged and compare to the regex above. mockLog.assertAllExpectationsMatched(); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index b6d75048591e5..35e5d66e91d97 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -92,7 +92,7 @@ setup: - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} # Testing for the entire function set isn't feasbile, so we just check that we return the correct count as an approximation. - - length: {esql.functions: 130} # check the "sister" test below for a likely update to the same esql.functions length check + - length: {esql.functions: 133} # check the "sister" test below for a likely update to the same esql.functions length check --- "Basic ESQL usage output (telemetry) non-snapshot version": @@ -163,4 +163,4 @@ setup: - match: {esql.functions.cos: $functions_cos} - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} - - length: {esql.functions: 126} # check the "sister" test above for a likely update to the same esql.functions length check + - length: {esql.functions: 129} # check the "sister" test above for a likely update to the same esql.functions length check diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/30_create_from.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/30_create_from.yml new file mode 100644 index 0000000000000..dfffea9a049a1 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/30_create_from.yml @@ -0,0 +1,121 @@ +--- +setup: + - do: + cluster.health: + wait_for_status: yellow + +--- +teardown: + - do: + indices.delete: + index: source-index-1 + ignore_unavailable: true + - do: + indices.delete: + index: dest-index-1 + ignore_unavailable: true + +--- +"Test create from with nonexistent source index": + - requires: + reason: "migration reindex is behind a feature flag" + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_migration/reindex + capabilities: [migration_reindex] + - do: + catch: /index_not_found_exception/ + migrate.create_from: + source: "does_not_exist" + dest: "dest1" + +--- +"Test create_from with existing source index": + - requires: + reason: "migration reindex is behind a feature flag" + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_migration/reindex + capabilities: [migration_reindex] + - do: + indices.create: + index: source-index-1 + body: + settings: + index: + number_of_shards: 3 + number_of_replicas: 5 + mappings: + dynamic: strict + properties: + bar: + type: text + - do: + migrate.create_from: + source: "source-index-1" + dest: "dest-index-1" + - do: + indices.get_settings: + index: dest-index-1 + - match: { dest-index-1.settings.index.number_of_shards: "3" } + - match: { dest-index-1.settings.index.number_of_replicas: "5" } + - do: + indices.get_mapping: + index: dest-index-1 + - match: {dest-index-1.mappings.properties.bar.type: text} + +--- +"Test create_from with existing source index and overrides": + - requires: + reason: "migration reindex is behind a feature flag" + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_migration/reindex + capabilities: [migration_reindex] + - do: + indices.create: + index: source-index-1 + body: + settings: + index: + number_of_shards: 3 + number_of_replicas: 5 + mappings: + dynamic: strict + properties: + bar: + type: text + foo: + type: boolean + - do: + migrate.create_from: + source: "source-index-1" + dest: "dest-index-1" + body: + settings_override: + index: + number_of_shards: 1 + default_pipeline: "pipeline-1" + mappings_override: + dynamic: strict + properties: + bar: + type: keyword + baz: + type: integer + - do: + indices.get_settings: + index: dest-index-1 + - match: { dest-index-1.settings.index.number_of_shards: "1" } + - match: { dest-index-1.settings.index.number_of_replicas: "5" } + - match: { dest-index-1.settings.index.default_pipeline: "pipeline-1" } + - do: + indices.get_mapping: + index: dest-index-1 + - match: {dest-index-1.mappings.properties.bar.type: keyword} + - match: {dest-index-1.mappings.properties.foo.type: boolean} + - match: {dest-index-1.mappings.properties.baz.type: integer} + diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java index ab478dc16f224..84519166eddb6 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformStats; import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfig; import org.elasticsearch.xpack.core.transform.transforms.pivot.SingleGroupSource; import org.elasticsearch.xpack.core.transform.transforms.pivot.TermsGroupSource; @@ -152,7 +153,7 @@ public void testBasicTransformStats() throws Exception { public void testContinuousTransformCrud() throws Exception { var transformId = "transform-continuous-crud"; var indexName = "continuous-crud-reviews"; - createContinuousTransform(indexName, transformId); + createContinuousTransform(indexName, transformId, "reviews-by-user-business-day"); var transformStats = getBasicTransformStats(transformId); assertThat(transformStats.get("state"), equalTo("started")); @@ -181,7 +182,7 @@ public void testContinuousTransformCrud() throws Exception { deleteTransform(transformId); } - private void createContinuousTransform(String indexName, String transformId) throws Exception { + private void createContinuousTransform(String indexName, String transformId, String destinationIndex) throws Exception { createReviewsIndex(indexName, 100, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow); var groups = Map.of( @@ -197,8 +198,9 @@ private void createContinuousTransform(String indexName, String transformId) thr .addAggregator(AggregationBuilders.avg("review_score").field("stars")) .addAggregator(AggregationBuilders.max("timestamp").field("timestamp")); - var config = createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryConfig.matchAll(), indexName) - .setPivotConfig(createPivotConfig(groups, aggs)) + var config = createTransformConfigBuilder(transformId, destinationIndex, QueryConfig.matchAll(), indexName).setPivotConfig( + createPivotConfig(groups, aggs) + ) .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1))) .setSettings(new SettingsConfig.Builder().setAlignCheckpoints(false).build()) .build(); @@ -216,7 +218,7 @@ private void createContinuousTransform(String indexName, String transformId) thr @SuppressWarnings("unchecked") public void testBasicContinuousTransformStats() throws Exception { var transformId = "transform-continuous-basic-stats"; - createContinuousTransform("continuous-basic-stats-reviews", transformId); + createContinuousTransform("continuous-basic-stats-reviews", transformId, "reviews-by-user-business-day"); var transformStats = getBasicTransformStats(transformId); assertEquals("started", XContentMapValues.extractValue("state", transformStats)); @@ -230,6 +232,40 @@ public void testBasicContinuousTransformStats() throws Exception { deleteTransform(transformId); } + public void testDestinationIndexBlocked() throws Exception { + var transformId = "transform-continuous-blocked-destination"; + var sourceIndexName = "source-reviews"; + var destIndexName = "destination-reviews"; + + // create transform & indices, wait until 1st checkpoint is finished + createContinuousTransform(sourceIndexName, transformId, destIndexName); + + // block destination index + Request request = new Request("PUT", destIndexName + "/_block/write"); + assertAcknowledged(adminClient().performRequest(request)); + + // index more docs so the checkpoint tries to run, wait until transform stops + assertBusy(() -> { + indexDoc(42, sourceIndexName); + assertEquals(TransformStats.State.WAITING.value(), getTransformState(transformId)); + }, 30, TimeUnit.SECONDS); + + // unblock index + request = new Request("PUT", destIndexName + "/_settings"); + request.setJsonEntity(""" + { "blocks.write": false } + """); + assertAcknowledged(adminClient().performRequest(request)); + + assertBusy(() -> { + indexDoc(42, sourceIndexName); + assertEquals(TransformStats.State.STARTED.value(), getTransformState(transformId)); + }, 30, TimeUnit.SECONDS); + + stopTransform(transformId); + deleteTransform(transformId); + } + public void testTransformLifecycleInALoop() throws Exception { String transformId = "lifecycle-in-a-loop"; String indexName = transformId + "-src"; @@ -652,4 +688,17 @@ private void indexMoreDocs(long timestamp, long userId, String index) throws Exc bulkBuilder.append("\r\n"); doBulk(bulkBuilder.toString(), true); } + + private void indexDoc(long userId, String index) throws Exception { + StringBuilder bulkBuilder = new StringBuilder(); + bulkBuilder.append(format(""" + {"create":{"_index":"%s"}} + """, index)); + String source = format(""" + {"user_id":"user_%s","count":%s,"business_id":"business_%s","stars":%s,"timestamp":%s} + """, userId, 1, 2, 5, Instant.now().toEpochMilli()); + bulkBuilder.append(source); + bulkBuilder.append("\r\n"); + doBulk(bulkBuilder.toString(), true); + } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java index ae0eb000e70e5..7de774224541c 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java @@ -270,6 +270,9 @@ static TransformStats deriveStats(TransformTask transformTask, @Nullable Transfo && derivedState.equals(TransformStats.State.FAILED) == false) { derivedState = TransformStats.State.STOPPING; reason = Strings.isNullOrEmpty(reason) ? "transform is set to stop at the next checkpoint" : reason; + } else if (derivedState.equals(TransformStats.State.STARTED) && transformTask.getContext().isWaitingForIndexToUnblock()) { + derivedState = TransformStats.State.WAITING; + reason = Strings.isNullOrEmpty(reason) ? "transform is paused while destination index is blocked" : reason; } return new TransformStats( transformTask.getTransformId(), diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java index 62582e885c338..fc25d7fbba878 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java @@ -128,6 +128,10 @@ protected void taskOperation( TransformTask transformTask, ActionListener listener ) { + if (transformTask.getContext().isWaitingForIndexToUnblock()) { + logger.debug("[{}] Destination index is blocked. User requested a retry.", transformTask.getTransformId()); + transformTask.getContext().setIsWaitingForIndexToUnblock(false); + } transformScheduler.scheduleNow(request.getId()); listener.onResponse(Response.TRUE); } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java index 4a0c4c2921ba6..c7fd6df4467dd 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java @@ -25,8 +25,10 @@ import org.elasticsearch.action.search.TransportClosePointInTimeAction; import org.elasticsearch.action.search.TransportOpenPointInTimeAction; import org.elasticsearch.action.search.TransportSearchAction; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.client.internal.ParentTaskAssigningClient; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; @@ -443,9 +445,36 @@ public boolean maybeTriggerAsyncJob(long now) { logger.debug("[{}] schedule was triggered but the Transform is upgrading. Ignoring trigger.", getJobId()); return false; } + if (context.isWaitingForIndexToUnblock()) { + if (destinationIndexHasWriteBlock()) { + logger.debug("[{}] schedule was triggered but the destination index has a write block. Ignoring trigger.", getJobId()); + return false; + } + logger.debug("[{}] destination index is no longer blocked.", getJobId()); + context.setIsWaitingForIndexToUnblock(false); + } + return super.maybeTriggerAsyncJob(now); } + private boolean destinationIndexHasWriteBlock() { + var clusterState = clusterService.state(); + if (clusterState == null) { + // if we can't determine if the index is blocked, we assume it isn't, even though the bulk request may fail again + return false; + } + + var destinationIndexName = transformConfig.getDestination().getIndex(); + var destinationIndex = indexNameExpressionResolver.concreteWriteIndex( + clusterState, + IndicesOptions.lenientExpandOpen(), + destinationIndexName, + true, + false + ); + return destinationIndex != null && clusterState.blocks().indexBlocked(ClusterBlockLevel.WRITE, destinationIndex.getName()); + } + @Override protected void onStop() { closePointInTime(); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java index 53c3ff4091a99..73872fac097c1 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java @@ -49,6 +49,14 @@ public interface Listener { private volatile AuthorizationState authState; private volatile int pageSize = 0; + /** + * If the destination index is blocked (e.g. during a reindex), the Transform will fail to write to it. + * {@link TransformFailureHandler} will silence the error so the Transform automatically retries. + * Every time the Transform runs, it will check if the index is unblocked and reset this to false. + * Users can override this via the `_schedule_now` API. + */ + private volatile boolean isWaitingForIndexToUnblock = false; + // the checkpoint of this transform, storing the checkpoint until data indexing from source to dest is _complete_ // Note: Each indexer run creates a new future checkpoint which becomes the current checkpoint only after the indexer run finished private final AtomicLong currentCheckpoint; @@ -183,6 +191,14 @@ public void setShouldRecreateDestinationIndex(boolean shouldRecreateDestinationI this.shouldRecreateDestinationIndex = shouldRecreateDestinationIndex; } + public boolean isWaitingForIndexToUnblock() { + return isWaitingForIndexToUnblock; + } + + public void setIsWaitingForIndexToUnblock(boolean isWaitingForIndexToUnblock) { + this.isWaitingForIndexToUnblock = isWaitingForIndexToUnblock; + } + public AuthorizationState getAuthState() { return authState; } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandler.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandler.java index 24586e5f36337..eeeabff9efd77 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandler.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandler.java @@ -170,6 +170,7 @@ private void handleScriptException(ScriptException scriptException, boolean unat */ private void handleBulkIndexingException(BulkIndexingException bulkIndexingException, boolean unattended, int numFailureRetries) { if (bulkIndexingException.getCause() instanceof ClusterBlockException) { + context.setIsWaitingForIndexToUnblock(true); retryWithoutIncrementingFailureCount( bulkIndexingException, bulkIndexingException.getDetailedMessage(), diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsActionTests.java index a3f56efa9e138..52ea738b03313 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsActionTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsActionTests.java @@ -343,6 +343,46 @@ public void testDeriveStats() { ); } + public void testDeriveStatsWithIndexBlock() { + String transformId = "transform-with-stats"; + String reason = "transform is paused while destination index is blocked"; + TransformIndexerStats stats = TransformIndexerStatsTests.randomStats(); + TransformState runningState = new TransformState( + TransformTaskState.STARTED, + IndexerState.STARTED, + null, + 0, + null, + null, + null, + false, + null + ); + + var context = new TransformContext(TransformTaskState.STARTED, "", 0, mock()); + context.setIsWaitingForIndexToUnblock(true); + var task = mock(TransformTask.class); + when(task.getContext()).thenReturn(context); + when(task.getTransformId()).thenReturn(transformId); + when(task.getState()).thenReturn(runningState); + when(task.getStats()).thenReturn(stats); + + assertThat( + TransportGetTransformStatsAction.deriveStats(task, null), + equalTo( + new TransformStats( + transformId, + TransformStats.State.WAITING, + reason, + null, + stats, + TransformCheckpointingInfo.EMPTY, + TransformHealth.GREEN + ) + ) + ); + } + private void withIdStateAndStats(String transformId, TransformState state, TransformIndexerStats stats) { when(task.getTransformId()).thenReturn(transformId); when(task.getState()).thenReturn(state); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java index c8677c2816fc9..1fbe5c53caccb 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java @@ -22,13 +22,18 @@ import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.client.internal.ParentTaskAssigningClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.CompositeBytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.search.SearchContextMissingException; import org.elasticsearch.search.SearchHit; @@ -44,6 +49,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ActionNotFoundTransportException; import org.elasticsearch.xpack.core.indexing.IndexerState; +import org.elasticsearch.xpack.core.transform.TransformMetadata; import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; @@ -53,6 +59,7 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition; import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; import org.elasticsearch.xpack.core.transform.transforms.TransformProgress; +import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants; import org.elasticsearch.xpack.transform.TransformExtension; import org.elasticsearch.xpack.transform.TransformNode; @@ -77,6 +84,10 @@ import java.util.function.Consumer; import java.util.stream.IntStream; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -467,6 +478,53 @@ public void testHandlePitIndexNotFound() throws InterruptedException { } } + public void testIndexBlocked() { + var service = serviceWithBlockCheck(true); + var context = new TransformContext(TransformTaskState.STARTED, "", 0, mock()); + + var indexer = createTestIndexer(mock(), service, resolver(), context); + context.setIsWaitingForIndexToUnblock(true); + + assertFalse(indexer.maybeTriggerAsyncJob(Instant.now().toEpochMilli())); + assertTrue(context.isWaitingForIndexToUnblock()); + } + + public void testIndexUnblocked() { + var service = serviceWithBlockCheck(false); + // set state to failed so that TransformIndexer returns false + var context = new TransformContext(TransformTaskState.FAILED, "", 0, mock()); + + var indexer = createTestIndexer(mock(), service, resolver(), context); + context.setIsWaitingForIndexToUnblock(true); + + assertFalse(indexer.maybeTriggerAsyncJob(Instant.now().toEpochMilli())); + // ClientTransformIndexer's maybeTriggerAsyncJob should reset isWaitingForIndexToUnblock to false + assertFalse(context.isWaitingForIndexToUnblock()); + } + + private ClusterService serviceWithBlockCheck(boolean checkResponse) { + var clusterBlocks = mock(ClusterBlocks.class); + when(clusterBlocks.indexBlocked(eq(ClusterBlockLevel.WRITE), anyString())).thenReturn(checkResponse); + var metadata = mock(Metadata.class); + when(metadata.custom(eq(TransformMetadata.TYPE))).thenReturn(TransformMetadata.EMPTY_METADATA); + var clusterState = mock(ClusterState.class); + when(clusterState.blocks()).thenReturn(clusterBlocks); + when(clusterState.getMetadata()).thenReturn(metadata); + var clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(clusterState); + return clusterService; + } + + private IndexNameExpressionResolver resolver() { + var resolver = mock(IndexNameExpressionResolver.class); + when(resolver.concreteWriteIndex(any(), any(), any(), anyBoolean(), anyBoolean())).thenAnswer(ans -> { + Index destIndex = mock(); + when(destIndex.getName()).thenReturn(ans.getArgument(2)); + return destIndex; + }); + return resolver; + } + private static class MockClientTransformIndexer extends ClientTransformIndexer { MockClientTransformIndexer( @@ -627,13 +685,22 @@ private ClientTransformIndexer createTestIndexer() { } private ClientTransformIndexer createTestIndexer(ParentTaskAssigningClient client) { + return createTestIndexer(client, mock(), mock(), mock(TransformContext.class)); + } + + private ClientTransformIndexer createTestIndexer( + ParentTaskAssigningClient client, + ClusterService service, + IndexNameExpressionResolver resolver, + TransformContext context + ) { ThreadPool threadPool = mock(ThreadPool.class); when(threadPool.executor("generic")).thenReturn(mock(ExecutorService.class)); return new ClientTransformIndexer( mock(ThreadPool.class), - mock(ClusterService.class), - mock(IndexNameExpressionResolver.class), + service, + resolver, mock(TransformExtension.class), new TransformServices( mock(IndexBasedTransformConfigManager.class), @@ -652,7 +719,7 @@ private ClientTransformIndexer createTestIndexer(ParentTaskAssigningClient clien new TransformCheckpoint("transform", Instant.now().toEpochMilli(), 0L, Collections.emptyMap(), Instant.now().toEpochMilli()), new TransformCheckpoint("transform", Instant.now().toEpochMilli(), 2L, Collections.emptyMap(), Instant.now().toEpochMilli()), new SeqNoPrimaryTermAndIndex(1, 1, TransformInternalIndexConstants.LATEST_INDEX_NAME), - mock(TransformContext.class), + context, false ); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandlerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandlerTests.java index 3894ff3043ccd..9f16e0d078b3c 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandlerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandlerTests.java @@ -93,7 +93,7 @@ public void testHandleIndexerFailure_BulkIndexExceptionWrappingClusterBlockExcep randomBoolean() ); - List.of(true, false).forEach((unattended) -> { assertRetryFailureCountNotIncremented(bulkIndexingException, unattended); }); + List.of(true, false).forEach((unattended) -> { assertClusterBlockHandled(bulkIndexingException, unattended); }); } public void testHandleIndexerFailure_IrrecoverableBulkIndexException() { @@ -197,7 +197,7 @@ private void assertRetry(Exception e, boolean unattended) { } } - private void assertRetryFailureCountNotIncremented(Exception e, boolean unattended) { + private void assertClusterBlockHandled(Exception e, boolean unattended) { String transformId = randomAlphaOfLength(10); SettingsConfig settings = new SettingsConfig.Builder().setNumFailureRetries(2).setUnattended(unattended).build(); @@ -211,6 +211,7 @@ private void assertRetryFailureCountNotIncremented(Exception e, boolean unattend assertNoFailure(handler, e, contextListener, settings, false); assertNoFailure(handler, e, contextListener, settings, false); assertNoFailure(handler, e, contextListener, settings, false); + assertTrue(context.isWaitingForIndexToUnblock()); } private void assertFailure(Exception e) { diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailSslTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailSslTests.java index b00c54bf500fd..50101d08b711e 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailSslTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailSslTests.java @@ -38,6 +38,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import javax.mail.MessagingException; import javax.mail.internet.MimeMessage; @@ -74,7 +75,6 @@ public void stopSmtpServer() { } public void testFailureSendingMessageToSmtpServerWithUntrustedCertificateAuthority() throws Exception { - assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm()); final Settings.Builder settings = Settings.builder(); final MockSecureSettings secureSettings = new MockSecureSettings(); final ExecutableEmailAction emailAction = buildEmailAction(settings, secureSettings); @@ -84,7 +84,14 @@ public void testFailureSendingMessageToSmtpServerWithUntrustedCertificateAuthori () -> emailAction.execute("my_action_id", ctx, Payload.EMPTY) ); final List allCauses = getAllCauses(exception); - assertThat(allCauses, Matchers.hasItem(Matchers.instanceOf(SSLException.class))); + if (inFipsJvm()) { + assertThat( + allCauses.stream().map(c -> c.getClass().getCanonicalName()).collect(Collectors.toSet()), + Matchers.hasItem("org.bouncycastle.tls.TlsFatalAlert") + ); + } else { + assertThat(allCauses, Matchers.hasItem(Matchers.instanceOf(SSLException.class))); + } } public void testCanSendMessageToSmtpServerUsingTrustStore() throws Exception { @@ -149,7 +156,6 @@ public void testCanSendMessageToSmtpServerUsingSmtpSslTrust() throws Exception { * over the account level "smtp.ssl.trust" setting) but smtp.ssl.trust was ignored for a period of time (see #52153) * so this is the least breaking way to resolve that. */ - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/49094") public void testNotificationSslSettingsOverrideSmtpSslTrust() throws Exception { List messages = new ArrayList<>(); server.addListener(messages::add); @@ -165,9 +171,16 @@ public void testNotificationSslSettingsOverrideSmtpSslTrust() throws Exception { MessagingException.class, () -> emailAction.execute("my_action_id", ctx, Payload.EMPTY) ); - final List allCauses = getAllCauses(exception); - assertThat(allCauses, Matchers.hasItem(Matchers.instanceOf(SSLException.class))); + if (inFipsJvm()) { + assertThat( + allCauses.stream().map(c -> c.getClass().getCanonicalName()).collect(Collectors.toSet()), + Matchers.hasItem("org.bouncycastle.tls.TlsFatalAlert") + ); + } else { + assertThat(allCauses, Matchers.hasItem(Matchers.instanceOf(SSLException.class))); + } + } finally { server.clearListeners(); } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java index b3e5f557b83ef..def3b467157e7 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java @@ -250,7 +250,11 @@ private static void createAndRolloverDataStream(String dataStreamName, int numRo } } - private void upgradeDataStream(String dataStreamName, int numRollovers) throws Exception { + private void upgradeDataStream(String dataStreamName, int numRolloversOnOldCluster) throws Exception { + final int explicitRolloverOnNewClusterCount = randomIntBetween(0, 2); + for (int i = 0; i < explicitRolloverOnNewClusterCount; i++) { + rollover(dataStreamName); + } Request reindexRequest = new Request("POST", "/_migration/reindex"); reindexRequest.setJsonEntity(Strings.format(""" { @@ -271,18 +275,31 @@ private void upgradeDataStream(String dataStreamName, int numRollovers) throws E ); assertOK(statusResponse); assertThat(statusResponseMap.get("complete"), equalTo(true)); - /* - * total_indices_in_data_stream is determined at the beginning of the reindex, and does not take into account the write - * index being rolled over - */ - assertThat(statusResponseMap.get("total_indices_in_data_stream"), equalTo(numRollovers + 1)); + final int originalWriteIndex = 1; if (isOriginalClusterSameMajorVersionAsCurrent()) { + assertThat( + statusResponseMap.get("total_indices_in_data_stream"), + equalTo(originalWriteIndex + numRolloversOnOldCluster + explicitRolloverOnNewClusterCount) + ); // If the original cluster was the same as this one, we don't want any indices reindexed: assertThat(statusResponseMap.get("total_indices_requiring_upgrade"), equalTo(0)); assertThat(statusResponseMap.get("successes"), equalTo(0)); } else { - assertThat(statusResponseMap.get("total_indices_requiring_upgrade"), equalTo(numRollovers + 1)); - assertThat(statusResponseMap.get("successes"), equalTo(numRollovers + 1)); + // The number of rollovers that will have happened when we call reindex: + final int rolloversPerformedByReindex = explicitRolloverOnNewClusterCount == 0 ? 1 : 0; + assertThat( + statusResponseMap.get("total_indices_in_data_stream"), + equalTo(originalWriteIndex + numRolloversOnOldCluster + explicitRolloverOnNewClusterCount + rolloversPerformedByReindex) + ); + /* + * total_indices_requiring_upgrade is made up of: (the original write index) + numRolloversOnOldCluster. The number of + * rollovers on the upgraded cluster is irrelevant since those will not be reindexed. + */ + assertThat( + statusResponseMap.get("total_indices_requiring_upgrade"), + equalTo(originalWriteIndex + numRolloversOnOldCluster) + ); + assertThat(statusResponseMap.get("successes"), equalTo(numRolloversOnOldCluster + 1)); } }, 60, TimeUnit.SECONDS); Request cancelRequest = new Request("POST", "_migration/reindex/" + dataStreamName + "/_cancel"); diff --git a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java index eeec67a0580fa..addcc328e8c2a 100644 --- a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java +++ b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java @@ -35,11 +35,14 @@ public class ExampleSecurityExtension implements SecurityExtension { static { - // check that the extension's policy works. - AccessController.doPrivileged((PrivilegedAction) () -> { - System.getSecurityManager().checkPropertyAccess("myproperty"); - return null; - }); + final boolean useEntitlements = Boolean.parseBoolean(System.getProperty("es.entitlements.enabled")); + if (useEntitlements == false) { + // check that the extension's policy works. + AccessController.doPrivileged((PrivilegedAction) () -> { + System.getSecurityManager().checkPropertyAccess("myproperty"); + return null; + }); + } } @Override diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle index da2d095c001d4..26919a21cb009 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle +++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle @@ -49,6 +49,14 @@ project(':plugins').getChildProjects().each { pluginName, pluginProject -> testClusters.matching { it.name == "yamlRestTest" }.configureEach { testDistribution = 'DEFAULT' setting 'xpack.monitoring.collection.interval', '1s' + setting 'xpack.monitoring.collection.enabled', 'false' //will enable in test + setting 'xpack.monitoring.exporters._http.enabled', 'false'//will enable in test + setting 'xpack.monitoring.exporters._http.type', "http" + setting 'xpack.monitoring.exporters._http.host', "https://example.com" //will be replaced in test + setting 'xpack.monitoring.exporters._http.auth.username', 'monitoring_agent' + setting 'xpack.monitoring.exporters._http.ssl.verification_mode', 'full' + setting 'xpack.monitoring.exporters._http.ssl.certificate_authorities', 'testnode.crt' + keystore 'xpack.monitoring.exporters._http.auth.secure_password', 'x-pack-test-password' setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' diff --git a/x-pack/qa/smoke-test-plugins-ssl/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java b/x-pack/qa/smoke-test-plugins-ssl/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java index f5247cad12d70..ac394b81ed5e7 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java +++ b/x-pack/qa/smoke-test-plugins-ssl/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java @@ -11,7 +11,6 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -53,26 +52,25 @@ public class SmokeTestMonitoringWithSecurityIT extends ESRestTestCase { private static final String USER = "test_user"; private static final SecureString PASS = new SecureString("x-pack-test-password".toCharArray()); - private static final String KEYSTORE_PASS = "testnode"; private static final String MONITORING_PATTERN = ".monitoring-*"; - static Path keyStore; + static Path trustStore; @BeforeClass public static void getKeyStore() { try { - keyStore = PathUtils.get(SmokeTestMonitoringWithSecurityIT.class.getResource("/testnode.jks").toURI()); + trustStore = PathUtils.get(SmokeTestMonitoringWithSecurityIT.class.getResource("/testnode.crt").toURI()); } catch (URISyntaxException e) { - throw new ElasticsearchException("exception while reading the store", e); + throw new ElasticsearchException("exception while reading the truststore", e); } - if (Files.exists(keyStore) == false) { - throw new IllegalStateException("Keystore file [" + keyStore + "] does not exist."); + if (Files.exists(trustStore) == false) { + throw new IllegalStateException("Truststore file [" + trustStore + "] does not exist."); } } @AfterClass public static void clearKeyStore() { - keyStore = null; + trustStore = null; } @Override @@ -85,24 +83,17 @@ protected Settings restClientSettings() { String token = basicAuthHeaderValue(USER, PASS); return Settings.builder() .put(ThreadContext.PREFIX + ".Authorization", token) - .put(ESRestTestCase.TRUSTSTORE_PATH, keyStore) - .put(ESRestTestCase.TRUSTSTORE_PASSWORD, KEYSTORE_PASS) + .put(ESRestTestCase.CERTIFICATE_AUTHORITIES, trustStore) .build(); } @Before public void enableExporter() throws Exception { - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString("xpack.monitoring.exporters._http.auth.secure_password", "x-pack-test-password"); Settings exporterSettings = Settings.builder() .put("xpack.monitoring.collection.enabled", true) .put("xpack.monitoring.exporters._http.enabled", true) .put("xpack.monitoring.exporters._http.type", "http") .put("xpack.monitoring.exporters._http.host", "https://" + randomNodeHttpAddress()) - .put("xpack.monitoring.exporters._http.auth.username", "monitoring_agent") - .put("xpack.monitoring.exporters._http.ssl.verification_mode", "full") - .put("xpack.monitoring.exporters._http.ssl.certificate_authorities", "testnode.crt") - .setSecureSettings(secureSettings) .build(); updateClusterSettingsIgnoringWarnings(client(), exporterSettings); } @@ -140,7 +131,6 @@ private boolean getMonitoringUsageExportersDefined() throws Exception { return exporters != null && exporters.isEmpty() == false; } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/49094") public void testHTTPExporterWithSSL() throws Exception { // Ensures that the exporter is actually on assertBusy(() -> assertThat("[_http] exporter is not defined", getMonitoringUsageExportersDefined(), is(true))); @@ -193,12 +183,11 @@ public void testHTTPExporterWithSSL() throws Exception { }); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/49094") public void testSettingsFilter() throws IOException { final Request request = new Request("GET", "/_cluster/settings"); final Response response = client().performRequest(request); final ObjectPath path = ObjectPath.createFromResponse(response); - final Map settings = path.evaluate("transient.xpack.monitoring.exporters._http"); + final Map settings = path.evaluate("persistent.xpack.monitoring.exporters._http"); assertThat(settings, hasKey("type")); assertThat(settings, not(hasKey("auth"))); assertThat(settings, not(hasKey("ssl")));