diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java new file mode 100644 index 0000000000000..708cf74bceee7 --- /dev/null +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.ccq; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.apache.http.HttpHost; +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.test.TestClustersThreadFilter; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.xpack.esql.qa.rest.RequestIndexFilteringTestCase; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; + +@ThreadLeakFilters(filters = TestClustersThreadFilter.class) +public class RequestIndexFilteringIT extends RequestIndexFilteringTestCase { + + static ElasticsearchCluster remoteCluster = Clusters.remoteCluster(); + static ElasticsearchCluster localCluster = Clusters.localCluster(remoteCluster); + + @ClassRule + public static TestRule clusterRule = RuleChain.outerRule(remoteCluster).around(localCluster); + private static RestClient remoteClient; + + @Override + protected String getTestRestCluster() { + return localCluster.getHttpAddresses(); + } + + @Before + public void setRemoteClient() throws IOException { + if (remoteClient == null) { + var clusterHosts = parseClusterHosts(remoteCluster.getHttpAddresses()); + remoteClient = buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0])); + } + } + + @BeforeClass + public static void checkVersion() { + assumeTrue("skip if version before 8.18", Clusters.localClusterVersion().onOrAfter(Version.V_8_18_0)); + } + + @AfterClass + public static void closeRemoteClients() throws IOException { + try { + IOUtils.close(remoteClient); + } finally { + remoteClient = null; + } + } + + @Override + protected void indexTimestampData(int docs, String indexName, String date, String differentiatorFieldName) throws IOException { + indexTimestampDataForClient(client(), docs, indexName, date, differentiatorFieldName); + indexTimestampDataForClient(remoteClient, docs, indexName, date, differentiatorFieldName); + } + + @Override + protected String from(String... indexName) { + if (randomBoolean()) { + return "FROM *:" + String.join(",*:", indexName); + } else { + return "FROM " + String.join(",", indexName); + } + } + + @After + public void wipeRemoteTestData() throws IOException { + try { + var response = remoteClient.performRequest(new Request("DELETE", "/test*")); + assertEquals(200, response.getStatusLine().getStatusCode()); + } catch (ResponseException re) { + assertEquals(404, re.getResponse().getStatusLine().getStatusCode()); + } + } +} diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java index ba93e9b31bb09..ac5a3d4be27f3 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java @@ -11,6 +11,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.esql.AssertWarnings; @@ -30,12 +31,13 @@ import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.entityToMap; import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.requestObjectBuilder; import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.oneOf; public abstract class RequestIndexFilteringTestCase extends ESRestTestCase { @@ -49,6 +51,10 @@ public void wipeTestData() throws IOException { } } + protected String from(String... indexName) { + return "FROM " + String.join(",", indexName); + } + public void testTimestampFilterFromQuery() throws IOException { int docsTest1 = 50; int docsTest2 = 30; @@ -56,7 +62,7 @@ public void testTimestampFilterFromQuery() throws IOException { indexTimestampData(docsTest2, "test2", "2023-11-26", "id2"); // filter includes both indices in the result (all columns, all rows) - RestEsqlTestCase.RequestObjectBuilder builder = timestampFilter("gte", "2023-01-01").query("FROM test*"); + RestEsqlTestCase.RequestObjectBuilder builder = timestampFilter("gte", "2023-01-01").query(from("test*")); Map result = runEsql(builder); assertMap( result, @@ -70,7 +76,7 @@ public void testTimestampFilterFromQuery() throws IOException { ); // filter includes only test1. Columns from test2 are filtered out, as well (not only rows)! - builder = timestampFilter("gte", "2024-01-01").query("FROM test*"); + builder = timestampFilter("gte", "2024-01-01").query(from("test*")); assertMap( runEsql(builder), matchesMap().entry( @@ -83,7 +89,7 @@ public void testTimestampFilterFromQuery() throws IOException { // filter excludes both indices (no rows); the first analysis step fails because there are no columns, a second attempt succeeds // after eliminating the index filter. All columns are returned. - builder = timestampFilter("gte", "2025-01-01").query("FROM test*"); + builder = timestampFilter("gte", "2025-01-01").query(from("test*")); assertMap( runEsql(builder), matchesMap().entry( @@ -103,7 +109,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException { indexTimestampData(docsTest2, "test2", "2023-11-26", "id2"); // filter includes only test1. Columns and rows of test2 are filtered out - RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query("FROM test*"); + RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query(from("test*")); Map result = runEsql(builder); assertMap( result, @@ -116,7 +122,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException { ); // filter includes only test1. Columns from test2 are filtered out, as well (not only rows)! - builder = existsFilter("id1").query("FROM test* METADATA _index | KEEP _index, id*"); + builder = existsFilter("id1").query(from("test*") + " METADATA _index | KEEP _index, id*"); result = runEsql(builder); assertMap( result, @@ -129,7 +135,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException { @SuppressWarnings("unchecked") var values = (List>) result.get("values"); for (List row : values) { - assertThat(row.get(0), equalTo("test1")); + assertThat(row.get(0), oneOf("test1", "remote_cluster:test1")); assertThat(row.get(1), instanceOf(Integer.class)); } } @@ -142,7 +148,7 @@ public void testFieldExistsFilter_With_ExplicitUseOfDiscardedIndexFields() throw // test2 is explicitly used in a query with "SORT id2" even if the index filter should discard test2 RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query( - "FROM test* METADATA _index | SORT id2 | KEEP _index, id*" + from("test*") + " METADATA _index | SORT id2 | KEEP _index, id*" ); Map result = runEsql(builder); assertMap( @@ -157,7 +163,7 @@ public void testFieldExistsFilter_With_ExplicitUseOfDiscardedIndexFields() throw @SuppressWarnings("unchecked") var values = (List>) result.get("values"); for (List row : values) { - assertThat(row.get(0), equalTo("test1")); + assertThat(row.get(0), oneOf("test1", "remote_cluster:test1")); assertThat(row.get(1), instanceOf(Integer.class)); assertThat(row.get(2), nullValue()); } @@ -172,59 +178,59 @@ public void testFieldNameTypo() throws IOException { // idx field name is explicitly used, though it doesn't exist in any of the indices. First test - without filter ResponseException e = expectThrows( ResponseException.class, - () -> runEsql(requestObjectBuilder().query("FROM test* | WHERE idx == 123")) + () -> runEsql(requestObjectBuilder().query(from("test*") + " | WHERE idx == 123")) ); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("verification_exception")); assertThat(e.getMessage(), containsString("Found 1 problem")); - assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]")); + assertThat(e.getMessage(), containsString("Unknown column [idx]")); - e = expectThrows(ResponseException.class, () -> runEsql(requestObjectBuilder().query("FROM test1 | WHERE idx == 123"))); + e = expectThrows(ResponseException.class, () -> runEsql(requestObjectBuilder().query(from("test1") + " | WHERE idx == 123"))); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("verification_exception")); assertThat(e.getMessage(), containsString("Found 1 problem")); - assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]")); + assertThat(e.getMessage(), containsString("Unknown column [idx]")); e = expectThrows( ResponseException.class, - () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM test* | WHERE idx == 123")) + () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test*") + " | WHERE idx == 123")) ); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("Found 1 problem")); - assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]")); + assertThat(e.getMessage(), containsString("Unknown column [idx]")); e = expectThrows( ResponseException.class, - () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM test2 | WHERE idx == 123")) + () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test2") + " | WHERE idx == 123")) ); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("Found 1 problem")); - assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]")); + assertThat(e.getMessage(), containsString("Unknown column [idx]")); } public void testIndicesDontExist() throws IOException { int docsTest1 = 0; // we are interested only in the created index, not necessarily that it has data indexTimestampData(docsTest1, "test1", "2024-11-26", "id1"); - ResponseException e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo"))); + ResponseException e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo")))); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("verification_exception")); - assertThat(e.getMessage(), containsString("Unknown index [foo]")); + assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo]"), containsString("Unknown index [remote_cluster:foo]"))); - e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo*"))); + e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo*")))); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("verification_exception")); - assertThat(e.getMessage(), containsString("Unknown index [foo*]")); + assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo*]"), containsString("Unknown index [remote_cluster:foo*]"))); - e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo,test1"))); + e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo", "test1")))); assertEquals(404, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("index_not_found_exception")); - assertThat(e.getMessage(), containsString("no such index [foo]")); + assertThat(e.getMessage(), anyOf(containsString("no such index [foo]"), containsString("no such index [remote_cluster:foo]"))); if (EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled()) { e = expectThrows( ResponseException.class, - () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM test1 | LOOKUP JOIN foo ON id1")) + () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test1") + " | LOOKUP JOIN foo ON id1")) ); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("verification_exception")); @@ -251,6 +257,11 @@ public Map runEsql(RestEsqlTestCase.RequestObjectBuilder request } protected void indexTimestampData(int docs, String indexName, String date, String differentiatorFieldName) throws IOException { + indexTimestampDataForClient(client(), docs, indexName, date, differentiatorFieldName); + } + + protected void indexTimestampDataForClient(RestClient client, int docs, String indexName, String date, String differentiatorFieldName) + throws IOException { Request createIndex = new Request("PUT", indexName); createIndex.setJsonEntity(""" { @@ -273,7 +284,7 @@ protected void indexTimestampData(int docs, String indexName, String date, Strin } } }""".replace("%differentiator_field_name%", differentiatorFieldName)); - Response response = client().performRequest(createIndex); + Response response = client.performRequest(createIndex); assertThat( entityToMap(response.getEntity(), XContentType.JSON), matchesMap().entry("shards_acknowledged", true).entry("index", indexName).entry("acknowledged", true) @@ -291,7 +302,7 @@ protected void indexTimestampData(int docs, String indexName, String date, Strin bulk.addParameter("refresh", "true"); bulk.addParameter("filter_path", "errors"); bulk.setJsonEntity(b.toString()); - response = client().performRequest(bulk); + response = client.performRequest(bulk); Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); } }