diff --git a/presto-elasticsearch/pom.xml b/presto-elasticsearch/pom.xml index aef412f672b5e..374c84db49f82 100644 --- a/presto-elasticsearch/pom.xml +++ b/presto-elasticsearch/pom.xml @@ -243,6 +243,30 @@ test + + net.java.dev.jna + jna + 5.5.0 + test + + + + org.testcontainers + testcontainers + 1.15.2 + test + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.core + jackson-annotations + + + + org.testcontainers elasticsearch @@ -256,10 +280,6 @@ org.apache.commons commons-compress - - net.java.dev.jna - jna - diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConfig.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConfig.java index 4822bd7b6756f..43cfd6ba33d8a 100644 --- a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConfig.java +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConfig.java @@ -32,7 +32,8 @@ public class ElasticsearchConfig { public enum Security { - AWS + AWS, + PASSWORD } private String host; diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorModule.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorModule.java index 09762c6aad7d9..4cd973f38793e 100644 --- a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorModule.java +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorModule.java @@ -30,6 +30,7 @@ import static com.facebook.airlift.json.JsonBinder.jsonBinder; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.elasticsearch.ElasticsearchConfig.Security.AWS; +import static com.facebook.presto.elasticsearch.ElasticsearchConfig.Security.PASSWORD; import static com.google.common.base.Preconditions.checkArgument; import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; import static java.util.Objects.requireNonNull; @@ -55,6 +56,7 @@ protected void setup(Binder binder) binder.install(new DecoderModule()); newOptionalBinder(binder, AwsSecurityConfig.class); + newOptionalBinder(binder, PasswordConfig.class); install(installModuleIf( ElasticsearchConfig.class, @@ -62,6 +64,13 @@ protected void setup(Binder binder) .filter(isEqual(AWS)) .isPresent(), conditionalBinder -> configBinder(conditionalBinder).bindConfig(AwsSecurityConfig.class))); + + install(installModuleIf( + ElasticsearchConfig.class, + config -> config.getSecurity() + .filter(isEqual(PASSWORD)) + .isPresent(), + conditionalBinder -> configBinder(conditionalBinder).bindConfig(PasswordConfig.class))); } private static final class TypeDeserializer diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/PasswordConfig.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/PasswordConfig.java new file mode 100644 index 0000000000000..d2def0d8a3e40 --- /dev/null +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/PasswordConfig.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.elasticsearch; + +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigSecuritySensitive; + +import javax.validation.constraints.NotNull; + +public class PasswordConfig +{ + private String user; + private String password; + + @NotNull + public String getUser() + { + return user; + } + + @Config("elasticsearch.auth.user") + public PasswordConfig setUser(String user) + { + this.user = user; + return this; + } + + @NotNull + public String getPassword() + { + return password; + } + + @Config("elasticsearch.auth.password") + @ConfigSecuritySensitive + public PasswordConfig setPassword(String password) + { + this.password = password; + return this; + } +} diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/client/ElasticsearchClient.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/client/ElasticsearchClient.java index abf46d151d820..55caaf8fef42b 100644 --- a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/client/ElasticsearchClient.java +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/client/ElasticsearchClient.java @@ -24,6 +24,7 @@ import com.facebook.airlift.security.pem.PemReader; import com.facebook.presto.elasticsearch.AwsSecurityConfig; import com.facebook.presto.elasticsearch.ElasticsearchConfig; +import com.facebook.presto.elasticsearch.PasswordConfig; import com.facebook.presto.spi.PrestoException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -35,10 +36,14 @@ import io.airlift.units.Duration; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.RequestConfig; import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.impl.nio.reactor.IOReactorConfig; import org.apache.http.message.BasicHeader; @@ -129,9 +134,12 @@ public class ElasticsearchClient private final boolean ignorePublishAddress; @Inject - public ElasticsearchClient(ElasticsearchConfig config, Optional awsSecurityConfig) + public ElasticsearchClient( + ElasticsearchConfig config, + Optional awsSecurityConfig, + Optional passwordConfig) { - client = createClient(config, awsSecurityConfig); + client = createClient(config, awsSecurityConfig, passwordConfig); this.ignorePublishAddress = config.isIgnorePublishAddress(); this.scrollSize = config.getScrollSize(); @@ -183,7 +191,10 @@ private void refreshNodes() } } - private static RestHighLevelClient createClient(ElasticsearchConfig config, Optional awsSecurityConfig) + private static RestHighLevelClient createClient( + ElasticsearchConfig config, + Optional awsSecurityConfig, + Optional passwordConfig) { RestClientBuilder builder = RestClient.builder( new HttpHost(config.getHost(), config.getPort(), config.isTlsEnabled() ? "https" : "http")) @@ -216,6 +227,12 @@ private static RestHighLevelClient createClient(ElasticsearchConfig config, Opti } } + passwordConfig.ifPresent(securityConfig -> { + CredentialsProvider credentials = new BasicCredentialsProvider(); + credentials.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(securityConfig.getUser(), securityConfig.getPassword())); + clientBuilder.setDefaultCredentialsProvider(credentials); + }); + awsSecurityConfig.ifPresent(securityConfig -> clientBuilder.addInterceptorLast(new AwsRequestSigner( securityConfig.getRegion(), getAwsCredentialsProvider(securityConfig)))); diff --git a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorTest.java b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorTest.java index c718dcc31d879..6690cd0afc7fc 100644 --- a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorTest.java +++ b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorTest.java @@ -13,25 +13,47 @@ */ package com.facebook.presto.elasticsearch; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.MaterializedRow; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestIntegrationSmokeTest; +import com.google.common.collect.ImmutableMap; +import com.google.common.net.HostAndPort; import io.airlift.tpch.TpchTable; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; import org.intellij.lang.annotations.Language; +import org.testng.annotations.AfterClass; import org.testng.annotations.Test; +import java.io.IOException; + +import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.elasticsearch.ElasticsearchQueryRunner.createElasticsearchQueryRunner; -import static com.facebook.presto.elasticsearch.EmbeddedElasticsearchNode.createEmbeddedElasticsearchNode; -import static org.elasticsearch.client.Requests.indexAliasesRequest; -import static org.elasticsearch.client.Requests.refreshRequest; +import static com.facebook.presto.testing.MaterializedResult.resultBuilder; +import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static java.lang.String.format; +@Test(singleThreaded = true) public class ElasticsearchConnectorTest extends AbstractTestIntegrationSmokeTest { - private EmbeddedElasticsearchNode embeddedElasticsearchNode; + private final String elasticsearchServer = "docker.elastic.co/elasticsearch/elasticsearch-oss:6.0.0"; + private ElasticsearchServer elasticsearch; + private RestHighLevelClient client; + + @AfterClass(alwaysRun = true) + public final void destroy() + throws IOException + { + elasticsearch.stop(); + client.close(); + } @Test public void testSelectInformationSchemaForMultiIndexAlias() + throws IOException { addAlias("nation", "multi_alias"); addAlias("region", "multi_alias"); @@ -132,25 +154,71 @@ public void testSelectInformationSchemaColumns() protected QueryRunner createQueryRunner() throws Exception { - embeddedElasticsearchNode = createEmbeddedElasticsearchNode(); - return createElasticsearchQueryRunner(embeddedElasticsearchNode, TpchTable.getTables()); + elasticsearch = new ElasticsearchServer(elasticsearchServer, ImmutableMap.of()); + HostAndPort address = elasticsearch.getAddress(); + client = new RestHighLevelClient(RestClient.builder(new HttpHost(address.getHost(), address.getPort()))); + + return createElasticsearchQueryRunner(elasticsearch.getAddress(), + TpchTable.getTables(), + ImmutableMap.of(), + ImmutableMap.of()); + } + + @Test + @Override + public void testDescribeTable() + { + MaterializedResult actualColumns = computeActual("DESC orders").toTestTypes(); + MaterializedResult.Builder builder = resultBuilder(getQueryRunner().getDefaultSession(), VARCHAR, VARCHAR, VARCHAR, VARCHAR); + for (MaterializedRow row : actualColumns.getMaterializedRows()) { + builder.row(row.getField(0), row.getField(1), "", ""); + } + MaterializedResult actualResult = builder.build(); + builder = resultBuilder(getQueryRunner().getDefaultSession(), VARCHAR, VARCHAR, VARCHAR, VARCHAR); + MaterializedResult expectedColumns = builder + .row("clerk", "varchar", "", "") + .row("comment", "varchar", "", "") + .row("custkey", "bigint", "", "") + .row("orderdate", "timestamp", "", "") + .row("orderkey", "bigint", "", "") + .row("orderpriority", "varchar", "", "") + .row("orderstatus", "varchar", "", "") + .row("shippriority", "bigint", "", "") + .row("totalprice", "real", "", "") + .build(); + assertEquals(actualResult, expectedColumns, format("%s != %s", actualResult, expectedColumns)); + } + + @Test + public void testMultipleRangesPredicate() + { + assertQuery("" + + "SELECT orderkey, custkey, orderstatus, totalprice, orderdate, orderpriority, clerk, shippriority, comment " + + "FROM orders " + + "WHERE orderkey BETWEEN 10 AND 50 OR orderkey BETWEEN 100 AND 150"); + } + + @Test + public void testRangePredicate() + { + // List columns explicitly, as there's no defined order in Elasticsearch + assertQuery("" + + "SELECT orderkey, custkey, orderstatus, totalprice, orderdate, orderpriority, clerk, shippriority, comment " + + "FROM orders " + + "WHERE orderkey BETWEEN 10 AND 50"); + } + + @Test + public void testSelectAll() + { + // List columns explicitly, as there's no defined order in Elasticsearch + assertQuery("SELECT orderkey, custkey, orderstatus, totalprice, orderdate, orderpriority, clerk, shippriority, comment FROM orders"); } private void addAlias(String index, String alias) + throws IOException { - embeddedElasticsearchNode.getClient() - .admin() - .indices() - .aliases(indexAliasesRequest() - .addAliasAction(IndicesAliasesRequest.AliasActions.add() - .index(index) - .alias(alias))) - .actionGet(); - - embeddedElasticsearchNode.getClient() - .admin() - .indices() - .refresh(refreshRequest(alias)) - .actionGet(); + client.getLowLevelClient() + .performRequest("PUT", format("/%s/_alias/%s", index, alias)); } } diff --git a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchLoader.java b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchLoader.java index fb2042a37261a..80545c6488a9b 100644 --- a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchLoader.java +++ b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchLoader.java @@ -24,7 +24,6 @@ import com.facebook.presto.tests.ResultsSession; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -43,6 +42,7 @@ import static com.facebook.presto.common.type.Varchars.isVarcharType; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; public class ElasticsearchLoader @@ -108,7 +108,7 @@ public void addResults(QueryStatusInfo statusInfo, QueryData data) throw new UncheckedIOException("Error loading data into Elasticsearch index: " + tableName, e); } } - request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + request.setRefreshPolicy(IMMEDIATE); try { restClient.bulk(request); } diff --git a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchQueryRunner.java b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchQueryRunner.java index 6bd8b26390a1b..dfb273c52eb1f 100644 --- a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchQueryRunner.java +++ b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchQueryRunner.java @@ -46,7 +46,11 @@ private ElasticsearchQueryRunner() {} private static final String TPCH_SCHEMA = "tpch"; private static final int NODE_COUNT = 2; - public static DistributedQueryRunner createElasticsearchQueryRunner(HostAndPort address, Iterable> tables) + public static DistributedQueryRunner createElasticsearchQueryRunner( + HostAndPort address, + Iterable> tables, + Map extraProperties, + Map extraConnectorProperties) throws Exception { RestHighLevelClient client = null; @@ -54,6 +58,7 @@ public static DistributedQueryRunner createElasticsearchQueryRunner(HostAndPort try { queryRunner = DistributedQueryRunner.builder(createSession()) .setNodeCount(NODE_COUNT) + .setExtraProperties(extraProperties) .build(); queryRunner.installPlugin(new TpchPlugin()); @@ -61,7 +66,7 @@ public static DistributedQueryRunner createElasticsearchQueryRunner(HostAndPort TestingElasticsearchConnectorFactory testFactory = new TestingElasticsearchConnectorFactory(); - installElasticsearchPlugin(address, queryRunner, testFactory); + installElasticsearchPlugin(address, queryRunner, testFactory, extraConnectorProperties); TestingPrestoClient prestoClient = queryRunner.getRandomClient(); @@ -83,8 +88,11 @@ public static DistributedQueryRunner createElasticsearchQueryRunner(HostAndPort } } - private static void installElasticsearchPlugin(HostAndPort address, QueryRunner queryRunner, TestingElasticsearchConnectorFactory factory) - throws Exception + private static void installElasticsearchPlugin( + HostAndPort address, + QueryRunner queryRunner, + TestingElasticsearchConnectorFactory factory, + Map extraConnectorProperties) { queryRunner.installPlugin(new ElasticsearchPlugin(factory)); Map config = ImmutableMap.builder() @@ -98,6 +106,7 @@ private static void installElasticsearchPlugin(HostAndPort address, QueryRunner .put("elasticsearch.scroll-timeout", "1m") .put("elasticsearch.max-hits", "1000000") .put("elasticsearch.request-timeout", "2m") + .putAll(extraConnectorProperties) .build(); queryRunner.createCatalog("elasticsearch", "elasticsearch", config); @@ -124,8 +133,12 @@ public static void main(String[] args) // docker run -p 9200:9200 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.6.2 Logging.initialize(); - HostAndPort address = HostAndPort.fromParts("localhost", 9200); - DistributedQueryRunner queryRunner = createElasticsearchQueryRunner(address, TpchTable.getTables()); + + DistributedQueryRunner queryRunner = createElasticsearchQueryRunner( + HostAndPort.fromParts("localhost", 9200), + TpchTable.getTables(), + ImmutableMap.of("http-server.http.port", "8080"), + ImmutableMap.of()); Logger log = Logger.get(ElasticsearchQueryRunner.class); log.info("======== SERVER STARTED ========"); log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); diff --git a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchServer.java b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchServer.java index cc80677cad743..d4b5dcae621e7 100644 --- a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchServer.java +++ b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchServer.java @@ -16,19 +16,46 @@ import com.google.common.net.HostAndPort; import org.testcontainers.elasticsearch.ElasticsearchContainer; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import static com.google.common.io.Files.createTempDir; +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.testcontainers.utility.MountableFile.forHostPath; + public class ElasticsearchServer { + private final String containerPath = "/usr/share/elasticsearch/config/"; + private final Path configurationPath; private final ElasticsearchContainer container; - public ElasticsearchServer() + public ElasticsearchServer(String image, Map configurationFiles) + throws IOException { - container = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch-oss:6.0.0"); + container = new ElasticsearchContainer(image); + + configurationPath = createTempDir().toPath(); + for (Map.Entry entry : configurationFiles.entrySet()) { + String name = entry.getKey(); + byte[] contents = entry.getValue().getBytes(UTF_8); + + Path path = configurationPath.resolve(name); + Files.write(path, contents); + container.withCopyFileToContainer(forHostPath(path), containerPath + name); + } + container.start(); } public void stop() + throws IOException { container.close(); + deleteRecursively(configurationPath, ALLOW_INSECURE); } public HostAndPort getAddress() diff --git a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestElasticsearchIntegrationSmokeTest.java b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestElasticsearchIntegrationSmokeTest.java index 6f1a3a21aa314..fc15c6daf5d5e 100644 --- a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestElasticsearchIntegrationSmokeTest.java +++ b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestElasticsearchIntegrationSmokeTest.java @@ -50,6 +50,7 @@ public class TestElasticsearchIntegrationSmokeTest extends AbstractTestIntegrationSmokeTest { + private final String elasticsearchServer = "docker.elastic.co/elasticsearch/elasticsearch-oss:6.0.0"; private ElasticsearchServer elasticsearch; private RestHighLevelClient client; @@ -57,12 +58,15 @@ public class TestElasticsearchIntegrationSmokeTest protected QueryRunner createQueryRunner() throws Exception { - elasticsearch = new ElasticsearchServer(); + elasticsearch = new ElasticsearchServer(elasticsearchServer, ImmutableMap.of()); HostAndPort address = elasticsearch.getAddress(); client = new RestHighLevelClient(RestClient.builder(new HttpHost(address.getHost(), address.getPort()))); - return createElasticsearchQueryRunner(elasticsearch.getAddress(), TpchTable.getTables()); + return createElasticsearchQueryRunner(elasticsearch.getAddress(), + TpchTable.getTables(), + ImmutableMap.of(), + ImmutableMap.of()); } @AfterClass(alwaysRun = true) @@ -138,15 +142,15 @@ public void testShowCreateTable() { assertThat(computeActual("SHOW CREATE TABLE orders").getOnlyValue()) .isEqualTo("CREATE TABLE elasticsearch.tpch.orders (\n" + - " clerk varchar,\n" + - " comment varchar,\n" + - " custkey bigint,\n" + - " orderdate timestamp,\n" + - " orderkey bigint,\n" + - " orderpriority varchar,\n" + - " orderstatus varchar,\n" + - " shippriority bigint,\n" + - " totalprice real\n" + + " \"clerk\" varchar,\n" + + " \"comment\" varchar,\n" + + " \"custkey\" bigint,\n" + + " \"orderdate\" timestamp,\n" + + " \"orderkey\" bigint,\n" + + " \"orderpriority\" varchar,\n" + + " \"orderstatus\" varchar,\n" + + " \"shippriority\" bigint,\n" + + " \"totalprice\" real\n" + ")"); } @@ -730,14 +734,7 @@ public void testAlias() "SELECT count(*) FROM orders_alias", "SELECT count(*) FROM orders"); - embeddedElasticsearchNode.getClient() - .admin() - .indices() - .aliases(indexAliasesRequest() - .addAliasAction(IndicesAliasesRequest.AliasActions.remove() - .index("orders") - .alias("orders_alias"))) - .actionGet(); + removeAlias("orders", "orders_alias"); } @Test(enabled = false) @@ -795,6 +792,13 @@ private void addAlias(String index, String alias) .performRequest("PUT", format("/%s/_alias/%s", index, alias)); } + private void removeAlias(String index, String alias) + throws IOException + { + client.getLowLevelClient() + .performRequest("DELETE", format("/%s/_alias/%s", index, alias)); + } + private void createIndex(String indexName, @Language("JSON") String mapping) throws IOException { diff --git a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestPasswordAuthentication.java b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestPasswordAuthentication.java new file mode 100644 index 0000000000000..f13c60aaa3d7e --- /dev/null +++ b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestPasswordAuthentication.java @@ -0,0 +1,112 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.elasticsearch; + +import com.amazonaws.util.Base64; +import com.facebook.presto.sql.query.QueryAssertions; +import com.facebook.presto.tests.DistributedQueryRunner; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.io.Resources; +import com.google.common.net.HostAndPort; +import org.apache.http.HttpHost; +import org.apache.http.entity.ContentType; +import org.apache.http.message.BasicHeader; +import org.apache.http.nio.entity.NStringEntity; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.testcontainers.shaded.com.google.common.collect.ImmutableList; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static com.facebook.presto.elasticsearch.ElasticsearchQueryRunner.createElasticsearchQueryRunner; +import static com.google.common.io.Resources.getResource; +import static java.lang.String.format; +import static java.nio.charset.StandardCharsets.UTF_8; + +public class TestPasswordAuthentication +{ + // We use 7.8.0 because security became a non-commercial feature in recent versions + private final String elasticsearchImage = "elasticsearch:7.8.0"; + private static final String USER = "elastic_user"; + private static final String PASSWORD = "123456"; + + private final ElasticsearchServer elasticsearch; + private final RestHighLevelClient client; + private final QueryAssertions assertions; + + public TestPasswordAuthentication() + throws Exception + { + elasticsearch = new ElasticsearchServer(elasticsearchImage, ImmutableMap.builder() + .put("elasticsearch.yml", loadResource("elasticsearch.yml")) + .put("users", loadResource("users")) + .put("users_roles", loadResource("users_roles")) + .put("roles.yml", loadResource("roles.yml")) + .build()); + + HostAndPort address = elasticsearch.getAddress(); + client = new RestHighLevelClient(RestClient.builder(new HttpHost(address.getHost(), address.getPort()))); + + DistributedQueryRunner runner = createElasticsearchQueryRunner( + elasticsearch.getAddress(), + ImmutableList.of(), + ImmutableMap.of(), + ImmutableMap.builder() + .put("elasticsearch.security", "PASSWORD") + .put("elasticsearch.auth.user", USER) + .put("elasticsearch.auth.password", PASSWORD) + .build()); + + assertions = new QueryAssertions(runner); + } + + @AfterClass(alwaysRun = true) + public final void destroy() + throws IOException + { + assertions.close(); + elasticsearch.stop(); + client.close(); + } + + @Test + public void test() + throws IOException + { + String json = new ObjectMapper().writeValueAsString(ImmutableMap.builder() + .put("value", 42L) + .build()); + + client.getLowLevelClient() + .performRequest( + "POST", + "/test/_doc?refresh", + ImmutableMap.of(), + new NStringEntity(json, ContentType.APPLICATION_JSON), + new BasicHeader("Authorization", format("Basic %s", Base64.encodeAsString(format("%s:%s", USER, PASSWORD).getBytes(StandardCharsets.UTF_8))))); + + assertions.assertQuery("SELECT * FROM test", + "VALUES BIGINT '42'"); + } + + private static String loadResource(String file) + throws IOException + { + return Resources.toString(getResource(file), UTF_8); + } +} diff --git a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestPasswordConfig.java b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestPasswordConfig.java new file mode 100644 index 0000000000000..e4fa4fccc5190 --- /dev/null +++ b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestPasswordConfig.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.elasticsearch; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestPasswordConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(PasswordConfig.class) + .setUser(null) + .setPassword(null)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("elasticsearch.auth.user", "user") + .put("elasticsearch.auth.password", "password") + .build(); + + PasswordConfig expected = new PasswordConfig() + .setUser("user") + .setPassword("password"); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-elasticsearch/src/test/resources/elasticsearch.yml b/presto-elasticsearch/src/test/resources/elasticsearch.yml new file mode 100644 index 0000000000000..dab862554754d --- /dev/null +++ b/presto-elasticsearch/src/test/resources/elasticsearch.yml @@ -0,0 +1,4 @@ +cluster.name: "docker-cluster" +network.host: 0.0.0.0 + +xpack.security.enabled: true \ No newline at end of file diff --git a/presto-elasticsearch/src/test/resources/roles.yml b/presto-elasticsearch/src/test/resources/roles.yml new file mode 100644 index 0000000000000..ee8750c9b0384 --- /dev/null +++ b/presto-elasticsearch/src/test/resources/roles.yml @@ -0,0 +1,6 @@ +admin: + cluster: + - all + indices: + - names: '*' + privileges: [ all ] \ No newline at end of file diff --git a/presto-elasticsearch/src/test/resources/users b/presto-elasticsearch/src/test/resources/users new file mode 100644 index 0000000000000..6cdaed94971e3 --- /dev/null +++ b/presto-elasticsearch/src/test/resources/users @@ -0,0 +1 @@ +elastic_user:$2a$10$tbO62EbOfqMezJDBDWlxbuvIleeYeNlw30F5OgWMXzi1R8aXqnVni \ No newline at end of file diff --git a/presto-elasticsearch/src/test/resources/users_roles b/presto-elasticsearch/src/test/resources/users_roles new file mode 100644 index 0000000000000..3e263d0cc4882 --- /dev/null +++ b/presto-elasticsearch/src/test/resources/users_roles @@ -0,0 +1 @@ +admin:elastic_user \ No newline at end of file diff --git a/presto-main/src/test/java/com/facebook/presto/sql/query/QueryAssertions.java b/presto-main/src/test/java/com/facebook/presto/sql/query/QueryAssertions.java index 3dd141a67b223..8f765ed6010bf 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/query/QueryAssertions.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/query/QueryAssertions.java @@ -22,27 +22,18 @@ import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.MaterializedRow; import com.facebook.presto.testing.QueryRunner; -import org.assertj.core.api.AbstractAssert; -import org.assertj.core.api.AssertProvider; -import org.assertj.core.api.ListAssert; -import org.assertj.core.presentation.Representation; -import org.assertj.core.presentation.StandardRepresentation; import org.intellij.lang.annotations.Language; import java.io.Closeable; import java.util.List; import java.util.Map; -import java.util.function.BiFunction; import java.util.function.Consumer; -import java.util.stream.Collectors; import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; -import static com.facebook.presto.sql.query.QueryAssertions.QueryAssert.newQueryAssert; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.google.common.base.Strings.nullToEmpty; import static java.lang.String.format; import static java.util.Objects.requireNonNull; -import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; @@ -70,7 +61,7 @@ public QueryAssertions(Map systemProperties) public QueryAssertions(Session session) { - runner = new LocalQueryRunner(session); + this(new LocalQueryRunner(requireNonNull(session, "session is null"))); } public QueryAssertions(QueryRunner runner) @@ -83,26 +74,6 @@ public QueryRunner getQueryRunner() return runner; } - public Session.SessionBuilder sessionBuilder() - { - return Session.builder(runner.getDefaultSession()); - } - - public Session getDefaultSession() - { - return runner.getDefaultSession(); - } - - public AssertProvider query(@Language("SQL") String query) - { - return query(query, runner.getDefaultSession()); - } - - public AssertProvider query(@Language("SQL") String query, Session session) - { - return newQueryAssert(query, runner, session); - } - public void assertFails(@Language("SQL") String sql, @Language("RegExp") String expectedMessageRegExp) { try { @@ -133,7 +104,7 @@ public void assertQuery(@Language("SQL") String actual, @Language("SQL") String assertQuery(actual, expected, false); } - private void assertQuery(@Language("SQL") String actual, @Language("SQL") String expected, boolean ensureOrdering) + public void assertQuery(@Language("SQL") String actual, @Language("SQL") String expected, boolean ensureOrdering) { MaterializedResult actualResults = null; try { @@ -171,83 +142,4 @@ public void close() { runner.close(); } - - public static class QueryAssert - extends AbstractAssert - { - private static final Representation ROWS_REPRESENTATION = new StandardRepresentation() - { - @Override - public String toStringOf(Object object) - { - if (object instanceof List) { - List list = (List) object; - return list.stream() - .map(this::toStringOf) - .collect(Collectors.joining(", ")); - } - if (object instanceof MaterializedRow) { - MaterializedRow row = (MaterializedRow) object; - - return row.getFields().stream() - .map(Object::toString) - .collect(Collectors.joining(", ", "(", ")")); - } - else { - return super.toStringOf(object); - } - } - }; - - private final QueryRunner runner; - private final Session session; - private boolean ordered; - - static AssertProvider newQueryAssert(String query, QueryRunner runner, Session session) - { - MaterializedResult result = runner.execute(session, query); - return () -> new QueryAssert(runner, session, result); - } - - public QueryAssert(QueryRunner runner, Session session, MaterializedResult actual) - { - super(actual, Object.class); - this.runner = runner; - this.session = session; - } - - public QueryAssert matches(BiFunction evaluator) - { - MaterializedResult expected = evaluator.apply(session, runner); - return isEqualTo(expected); - } - - public QueryAssert ordered() - { - ordered = true; - return this; - } - - public QueryAssert matches(@Language("SQL") String query) - { - MaterializedResult expected = runner.execute(session, query); - - return satisfies(actual -> { - assertThat(actual.getTypes()) - .as("Output types") - .isEqualTo(expected.getTypes()); - - ListAssert assertion = assertThat(actual.getMaterializedRows()) - .as("Rows") - .withRepresentation(ROWS_REPRESENTATION); - - if (ordered) { - assertion.containsExactlyElementsOf(expected.getMaterializedRows()); - } - else { - assertion.containsExactlyInAnyOrder(expected.getMaterializedRows().toArray(new MaterializedRow[0])); - } - }); - } - } }