diff --git a/presto-docs/src/main/sphinx/connector/iceberg.rst b/presto-docs/src/main/sphinx/connector/iceberg.rst index 86f83bb3674c6..504a37d31dfb2 100644 --- a/presto-docs/src/main/sphinx/connector/iceberg.rst +++ b/presto-docs/src/main/sphinx/connector/iceberg.rst @@ -224,6 +224,11 @@ Property Name Description ``iceberg.rest.auth.oauth2.credential``. Example: ``PRINCIPAL_ROLE:ALL`` +``iceberg.rest.nested.namespace.enabled`` In REST Catalogs, tables are grouped into namespaces, that can be + nested. But if a large number of recursive namespaces result in + lower performance, querying nested namespaces can be disabled. + Defaults to ``true``. + ``iceberg.rest.session.type`` The session type to use when communicating with the REST catalog. Available values are ``NONE`` or ``USER`` (default: ``NONE``). diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeCatalogFactory.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeCatalogFactory.java index d0a361184af13..4d23a97a3ff71 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeCatalogFactory.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeCatalogFactory.java @@ -99,6 +99,11 @@ public SupportsNamespaces getNamespaces(ConnectorSession session) throw new PrestoException(NOT_SUPPORTED, "Iceberg catalog of type " + catalogType + " does not support namespace operations"); } + public boolean isNestedNamespaceEnabled() + { + return false; + } + protected String getCacheKey(ConnectorSession session) { StringBuilder sb = new StringBuilder(); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeMetadata.java index 04f2282264341..a29c877b22f77 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeMetadata.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeMetadata.java @@ -54,6 +54,7 @@ import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.stream.Stream; import static com.facebook.presto.iceberg.IcebergSessionProperties.getCompressionCodec; import static com.facebook.presto.iceberg.IcebergTableProperties.getFileFormat; @@ -71,6 +72,8 @@ import static com.facebook.presto.iceberg.SchemaConverter.toPrestoSchema; import static com.facebook.presto.iceberg.util.IcebergPrestoModelConverters.toIcebergNamespace; import static com.facebook.presto.iceberg.util.IcebergPrestoModelConverters.toIcebergTableIdentifier; +import static com.facebook.presto.iceberg.util.IcebergPrestoModelConverters.toPrestoSchemaName; +import static com.facebook.presto.iceberg.util.IcebergPrestoModelConverters.toPrestoSchemaTableName; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.StandardErrorCode.SCHEMA_NOT_EMPTY; import static com.google.common.base.Verify.verify; @@ -137,12 +140,26 @@ protected boolean tableExists(ConnectorSession session, SchemaTableName schemaTa public List listSchemaNames(ConnectorSession session) { SupportsNamespaces supportsNamespaces = catalogFactory.getNamespaces(session); + if (catalogFactory.isNestedNamespaceEnabled()) { + return listNestedNamespaces(supportsNamespaces, Namespace.empty()); + } + return supportsNamespaces.listNamespaces() .stream() .map(IcebergPrestoModelConverters::toPrestoSchemaName) .collect(toList()); } + private List listNestedNamespaces(SupportsNamespaces supportsNamespaces, Namespace parentNamespace) + { + return supportsNamespaces.listNamespaces(parentNamespace) + .stream() + .flatMap(childNamespace -> Stream.concat( + Stream.of(toPrestoSchemaName(childNamespace)), + listNestedNamespaces(supportsNamespaces, childNamespace).stream())) + .collect(toList()); + } + @Override public List listTables(ConnectorSession session, Optional schemaName) { @@ -152,16 +169,16 @@ public List listTables(ConnectorSession session, Optional toPrestoSchemaTableName(tableIdentifier, catalogFactory.isNestedNamespaceEnabled())) .collect(toList()); } @Override public void createSchema(ConnectorSession session, String schemaName, Map properties) { - catalogFactory.getNamespaces(session).createNamespace(toIcebergNamespace(Optional.of(schemaName)), + catalogFactory.getNamespaces(session).createNamespace(toIcebergNamespace(Optional.of(schemaName), catalogFactory.isNestedNamespaceEnabled()), properties.entrySet().stream() .collect(toMap(Map.Entry::getKey, e -> e.getValue().toString()))); } @@ -170,7 +187,7 @@ public void createSchema(ConnectorSession session, String schemaName, Map listViews(ConnectorSession session, Optional getViews(ConnectorSession s for (SchemaTableName schemaTableName : tableNames) { try { - if (((ViewCatalog) catalog).viewExists(TableIdentifier.of(schemaTableName.getSchemaName(), schemaTableName.getTableName()))) { - View view = ((ViewCatalog) catalog).loadView(TableIdentifier.of(schemaTableName.getSchemaName(), schemaTableName.getTableName())); + TableIdentifier viewIdentifier = toIcebergTableIdentifier(schemaTableName, catalogFactory.isNestedNamespaceEnabled()); + if (((ViewCatalog) catalog).viewExists(viewIdentifier)) { + View view = ((ViewCatalog) catalog).loadView(viewIdentifier); verifyAndPopulateViews(view, schemaTableName, view.sqlFor(VIEW_DIALECT).sql(), views); } } @@ -268,7 +287,7 @@ public void dropView(ConnectorSession session, SchemaTableName viewName) if (!(catalog instanceof ViewCatalog)) { throw new PrestoException(NOT_SUPPORTED, "This connector does not support dropping views"); } - ((ViewCatalog) catalog).dropView(TableIdentifier.of(viewName.getSchemaName(), viewName.getTableName())); + ((ViewCatalog) catalog).dropView(toIcebergTableIdentifier(viewName, catalogFactory.isNestedNamespaceEnabled())); } private void verifyAndPopulateViews(View view, SchemaTableName schemaTableName, String viewData, ImmutableMap.Builder views) @@ -295,7 +314,10 @@ public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, Con try { transaction = catalogFactory.getCatalog(session).newCreateTableTransaction( - toIcebergTableIdentifier(schemaTableName), schema, partitionSpec, populateTableProperties(tableMetadata, fileFormat, session)); + toIcebergTableIdentifier(schemaTableName, catalogFactory.isNestedNamespaceEnabled()), + schema, + partitionSpec, + populateTableProperties(tableMetadata, fileFormat, session)); } catch (AlreadyExistsException e) { throw new TableAlreadyExistsException(schemaTableName); @@ -319,7 +341,7 @@ public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle { IcebergTableHandle icebergTableHandle = (IcebergTableHandle) tableHandle; verify(icebergTableHandle.getIcebergTableName().getTableType() == DATA, "only the data table can be dropped"); - TableIdentifier tableIdentifier = toIcebergTableIdentifier(icebergTableHandle.getSchemaTableName()); + TableIdentifier tableIdentifier = toIcebergTableIdentifier(icebergTableHandle.getSchemaTableName(), catalogFactory.isNestedNamespaceEnabled()); catalogFactory.getCatalog(session).dropTable(tableIdentifier); } @@ -328,20 +350,20 @@ public void renameTable(ConnectorSession session, ConnectorTableHandle tableHand { IcebergTableHandle icebergTableHandle = (IcebergTableHandle) tableHandle; verify(icebergTableHandle.getIcebergTableName().getTableType() == DATA, "only the data table can be renamed"); - TableIdentifier from = toIcebergTableIdentifier(icebergTableHandle.getSchemaTableName()); - TableIdentifier to = toIcebergTableIdentifier(newTable); + TableIdentifier from = toIcebergTableIdentifier(icebergTableHandle.getSchemaTableName(), catalogFactory.isNestedNamespaceEnabled()); + TableIdentifier to = toIcebergTableIdentifier(newTable, catalogFactory.isNestedNamespaceEnabled()); catalogFactory.getCatalog(session).renameTable(from, to); } @Override public void registerTable(ConnectorSession clientSession, SchemaTableName schemaTableName, Path metadataLocation) { - catalogFactory.getCatalog(clientSession).registerTable(toIcebergTableIdentifier(schemaTableName), metadataLocation.toString()); + catalogFactory.getCatalog(clientSession).registerTable(toIcebergTableIdentifier(schemaTableName, catalogFactory.isNestedNamespaceEnabled()), metadataLocation.toString()); } @Override public void unregisterTable(ConnectorSession clientSession, SchemaTableName schemaTableName) { - catalogFactory.getCatalog(clientSession).dropTable(toIcebergTableIdentifier(schemaTableName), false); + catalogFactory.getCatalog(clientSession).dropTable(toIcebergTableIdentifier(schemaTableName, catalogFactory.isNestedNamespaceEnabled()), false); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java index bb3139fba3e67..95fcc4fd0d497 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java @@ -254,7 +254,7 @@ public static Table getHiveIcebergTable(ExtendedHiveMetastore metastore, HdfsEnv public static Table getNativeIcebergTable(IcebergNativeCatalogFactory catalogFactory, ConnectorSession session, SchemaTableName table) { - return catalogFactory.getCatalog(session).loadTable(toIcebergTableIdentifier(table)); + return catalogFactory.getCatalog(session).loadTable(toIcebergTableIdentifier(table, catalogFactory.isNestedNamespaceEnabled())); } public static View getNativeIcebergView(IcebergNativeCatalogFactory catalogFactory, ConnectorSession session, SchemaTableName table) @@ -263,7 +263,7 @@ public static View getNativeIcebergView(IcebergNativeCatalogFactory catalogFacto if (!(catalog instanceof ViewCatalog)) { throw new PrestoException(NOT_SUPPORTED, "This connector does not support get views"); } - return ((ViewCatalog) catalog).loadView(toIcebergTableIdentifier(table)); + return ((ViewCatalog) catalog).loadView(toIcebergTableIdentifier(table, catalogFactory.isNestedNamespaceEnabled())); } public static List getPartitionKeyColumnHandles(IcebergTableHandle tableHandle, Table table, TypeManager typeManager) diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/rest/IcebergRestCatalogFactory.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/rest/IcebergRestCatalogFactory.java index ca363e793ace3..3f4d9422d0146 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/rest/IcebergRestCatalogFactory.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/rest/IcebergRestCatalogFactory.java @@ -59,6 +59,7 @@ public class IcebergRestCatalogFactory private final IcebergRestConfig catalogConfig; private final NodeVersion nodeVersion; private final String catalogName; + private final boolean nestedNamespaceEnabled; @Inject public IcebergRestCatalogFactory( @@ -73,6 +74,7 @@ public IcebergRestCatalogFactory( this.catalogConfig = requireNonNull(catalogConfig, "catalogConfig is null"); this.nodeVersion = requireNonNull(nodeVersion, "nodeVersion is null"); this.catalogName = requireNonNull(catalogName, "catalogName is null").getCatalogName(); + this.nestedNamespaceEnabled = catalogConfig.isNestedNamespaceEnabled(); } @Override @@ -135,6 +137,12 @@ protected Map getCatalogProperties(ConnectorSession session) return properties.build(); } + @Override + public boolean isNestedNamespaceEnabled() + { + return this.nestedNamespaceEnabled; + } + protected SessionContext convertSession(ConnectorSession session) { RestSessionBuilder sessionContextBuilder = catalogConfig.getSessionType() diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/rest/IcebergRestConfig.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/rest/IcebergRestConfig.java index e46dfda2109c9..613a48d02f13a 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/rest/IcebergRestConfig.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/rest/IcebergRestConfig.java @@ -29,6 +29,7 @@ public class IcebergRestConfig private String credential; private String token; private String scope; + private boolean nestedNamespaceEnabled = true; @NotNull public Optional getServerUri() @@ -122,6 +123,19 @@ public IcebergRestConfig setScope(String scope) return this; } + public boolean isNestedNamespaceEnabled() + { + return nestedNamespaceEnabled; + } + + @Config("iceberg.rest.nested.namespace.enabled") + @ConfigDescription("Allows querying nested namespaces. Default: true") + public IcebergRestConfig setNestedNamespaceEnabled(boolean nestedNamespaceEnabled) + { + this.nestedNamespaceEnabled = nestedNamespaceEnabled; + return this; + } + public boolean credentialOrTokenExists() { return credential != null || token != null; diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/util/IcebergPrestoModelConverters.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/util/IcebergPrestoModelConverters.java index d1b9a9a551360..efb830185af78 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/util/IcebergPrestoModelConverters.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/util/IcebergPrestoModelConverters.java @@ -13,14 +13,22 @@ */ package com.facebook.presto.iceberg.util; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; +import com.google.common.base.Splitter; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import java.util.Optional; +import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static java.lang.String.format; + public class IcebergPrestoModelConverters { + private static final String NAMESPACE_SEPARATOR = "."; + private static final Splitter NAMESPACE_SPLITTER = Splitter.on(NAMESPACE_SEPARATOR).omitEmptyStrings().trimResults(); + private IcebergPrestoModelConverters() { } @@ -30,23 +38,38 @@ public static String toPrestoSchemaName(Namespace icebergNamespace) return icebergNamespace.toString(); } - public static Namespace toIcebergNamespace(Optional prestoSchemaName) + public static Namespace toIcebergNamespace(Optional prestoSchemaName, boolean nestedNamespaceEnabled) + { + if (prestoSchemaName.isPresent()) { + checkNestedNamespaceSupport(prestoSchemaName.get(), nestedNamespaceEnabled); + return Namespace.of(NAMESPACE_SPLITTER.splitToList(prestoSchemaName.get()).toArray(new String[0])); + } + return Namespace.empty(); + } + + public static SchemaTableName toPrestoSchemaTableName(TableIdentifier icebergTableIdentifier, boolean nestedNamespaceEnabled) { - return prestoSchemaName.map(Namespace::of).orElse(Namespace.empty()); + String schemaName = icebergTableIdentifier.namespace().toString(); + checkNestedNamespaceSupport(schemaName, nestedNamespaceEnabled); + return new SchemaTableName(schemaName, icebergTableIdentifier.name()); } - public static SchemaTableName toPrestoSchemaTableName(TableIdentifier icebergTableIdentifier) + public static TableIdentifier toIcebergTableIdentifier(SchemaTableName prestoSchemaTableName, boolean nestedNamespaceEnabled) { - return new SchemaTableName(icebergTableIdentifier.namespace().toString(), icebergTableIdentifier.name()); + return toIcebergTableIdentifier(toIcebergNamespace(Optional.ofNullable(prestoSchemaTableName.getSchemaName()), nestedNamespaceEnabled), + prestoSchemaTableName.getTableName(), nestedNamespaceEnabled); } - public static TableIdentifier toIcebergTableIdentifier(SchemaTableName prestoSchemaTableName) + public static TableIdentifier toIcebergTableIdentifier(Namespace icebergNamespace, String prestoTableName, boolean nestedNamespaceEnabled) { - return toIcebergTableIdentifier(prestoSchemaTableName.getSchemaName(), prestoSchemaTableName.getTableName()); + checkNestedNamespaceSupport(icebergNamespace.toString(), nestedNamespaceEnabled); + return TableIdentifier.of(icebergNamespace, prestoTableName); } - public static TableIdentifier toIcebergTableIdentifier(String prestoSchemaName, String prestoTableName) + private static void checkNestedNamespaceSupport(String schemaName, boolean nestedNamespaceEnabled) { - return TableIdentifier.of(prestoSchemaName, prestoTableName); + if (!nestedNamespaceEnabled && schemaName.contains(NAMESPACE_SEPARATOR)) { + throw new PrestoException(NOT_SUPPORTED, format("Nested namespaces are disabled. Schema %s is not valid", schemaName)); + } } } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedSmokeTestBase.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedSmokeTestBase.java index 57a8acfae41a6..38fd38b056267 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedSmokeTestBase.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedSmokeTestBase.java @@ -125,8 +125,9 @@ public void testDescribeTable() @Test public void testShowCreateTable() { + String schemaName = getSession().getSchema().get(); assertThat(computeActual("SHOW CREATE TABLE orders").getOnlyValue()) - .isEqualTo(format("CREATE TABLE iceberg.tpch.orders (\n" + + .isEqualTo(format("CREATE TABLE iceberg.%s.orders (\n" + " \"orderkey\" bigint,\n" + " \"custkey\" bigint,\n" + " \"orderstatus\" varchar,\n" + @@ -145,7 +146,7 @@ public void testShowCreateTable() " metadata_delete_after_commit = false,\n" + " metadata_previous_versions_max = 100,\n" + " metrics_max_inferred_column = 100\n" + - ")", getLocation("tpch", "orders"))); + ")", schemaName, getLocation(schemaName, "orders"))); } @Test @@ -397,7 +398,7 @@ public void testCreatePartitionedTableAs() testWithAllFileFormats(this::testCreatePartitionedTableAs); } - private void testCreatePartitionedTableAs(Session session, FileFormat fileFormat) + protected void testCreatePartitionedTableAs(Session session, FileFormat fileFormat) { @Language("SQL") String createTable = "" + "CREATE TABLE test_create_partitioned_table_as_" + fileFormat.toString().toLowerCase(ENGLISH) + " " + @@ -602,9 +603,10 @@ public void testColumnComments() public void testTableComments() { Session session = getSession(); + String schemaName = session.getSchema().get(); @Language("SQL") String createTable = "" + - "CREATE TABLE iceberg.tpch.test_table_comments (\n" + + "CREATE TABLE iceberg.%s.test_table_comments (\n" + " \"_x\" bigint\n" + ")\n" + "COMMENT '%s'\n" + @@ -613,10 +615,10 @@ public void testTableComments() " format_version = '2'\n" + ")"; - assertUpdate(format(createTable, "test table comment")); + assertUpdate(format(createTable, schemaName, "test table comment")); String createTableTemplate = "" + - "CREATE TABLE iceberg.tpch.test_table_comments (\n" + + "CREATE TABLE iceberg.%s.test_table_comments (\n" + " \"_x\" bigint\n" + ")\n" + "COMMENT '%s'\n" + @@ -629,7 +631,7 @@ public void testTableComments() " metadata_previous_versions_max = 100,\n" + " metrics_max_inferred_column = 100\n" + ")"; - String createTableSql = format(createTableTemplate, "test table comment", getLocation("tpch", "test_table_comments")); + String createTableSql = format(createTableTemplate, schemaName, "test table comment", getLocation(schemaName, "test_table_comments")); MaterializedResult resultOfCreate = computeActual("SHOW CREATE TABLE test_table_comments"); assertEquals(getOnlyElement(resultOfCreate.getOnlyColumnAsSet()), createTableSql); @@ -651,11 +653,11 @@ public void testRollbackSnapshot() assertQuery(session, "SELECT * FROM test_rollback ORDER BY col0", "VALUES (123, CAST(987 AS BIGINT)), (456, CAST(654 AS BIGINT)), (123, CAST(321 AS BIGINT))"); - assertUpdate(format("CALL system.rollback_to_snapshot('tpch', 'test_rollback', %s)", afterFirstInsertId)); + assertUpdate(format("CALL system.rollback_to_snapshot('%s', 'test_rollback', %s)", session.getSchema().get(), afterFirstInsertId)); assertQuery(session, "SELECT * FROM test_rollback ORDER BY col0", "VALUES (123, CAST(987 AS BIGINT)), (123, CAST(321 AS BIGINT))"); - assertUpdate(format("CALL system.rollback_to_snapshot('tpch', 'test_rollback', %s)", afterCreateTableId)); + assertUpdate(format("CALL system.rollback_to_snapshot('%s', 'test_rollback', %s)", session.getSchema().get(), afterCreateTableId)); assertEquals((long) computeActual(session, "SELECT COUNT(*) FROM test_rollback").getOnlyValue(), 1); dropTable(session, "test_rollback"); @@ -708,6 +710,7 @@ private void testSchemaEvolution(Session session, FileFormat fileFormat) private void testCreateTableLike() { Session session = getSession(); + String schemaName = session.getSchema().get(); assertUpdate(session, "CREATE TABLE test_create_table_like_original (col1 INTEGER, aDate DATE) WITH(format = 'PARQUET', partitioning = ARRAY['aDate'])"); assertEquals(getTablePropertiesString("test_create_table_like_original"), format("WITH (\n" + @@ -719,7 +722,7 @@ private void testCreateTableLike() " metadata_previous_versions_max = 100,\n" + " metrics_max_inferred_column = 100,\n" + " partitioning = ARRAY['adate']\n" + - ")", getLocation("tpch", "test_create_table_like_original"))); + ")", getLocation(schemaName, "test_create_table_like_original"))); assertUpdate(session, "CREATE TABLE test_create_table_like_copy0 (LIKE test_create_table_like_original, col2 INTEGER)"); assertUpdate(session, "INSERT INTO test_create_table_like_copy0 (col1, aDate, col2) VALUES (1, CAST('1950-06-28' AS DATE), 3)", 1); @@ -735,7 +738,7 @@ private void testCreateTableLike() " metadata_delete_after_commit = false,\n" + " metadata_previous_versions_max = 100,\n" + " metrics_max_inferred_column = 100\n" + - ")", getLocation("tpch", "test_create_table_like_copy1"))); + ")", getLocation(schemaName, "test_create_table_like_copy1"))); dropTable(session, "test_create_table_like_copy1"); assertUpdate(session, "CREATE TABLE test_create_table_like_copy2 (LIKE test_create_table_like_original EXCLUDING PROPERTIES)"); @@ -747,7 +750,7 @@ private void testCreateTableLike() " metadata_delete_after_commit = false,\n" + " metadata_previous_versions_max = 100,\n" + " metrics_max_inferred_column = 100\n" + - ")", getLocation("tpch", "test_create_table_like_copy2"))); + ")", getLocation(schemaName, "test_create_table_like_copy2"))); dropTable(session, "test_create_table_like_copy2"); assertUpdate(session, "CREATE TABLE test_create_table_like_copy3 (LIKE test_create_table_like_original INCLUDING PROPERTIES)"); @@ -761,8 +764,8 @@ private void testCreateTableLike() " metrics_max_inferred_column = 100,\n" + " partitioning = ARRAY['adate']\n" + ")", catalogType.equals(CatalogType.HIVE) ? - getLocation("tpch", "test_create_table_like_original") : - getLocation("tpch", "test_create_table_like_copy3"))); + getLocation(schemaName, "test_create_table_like_original") : + getLocation(schemaName, "test_create_table_like_copy3"))); dropTable(session, "test_create_table_like_copy3"); assertUpdate(session, "CREATE TABLE test_create_table_like_copy4 (LIKE test_create_table_like_original INCLUDING PROPERTIES) WITH (format = 'ORC')"); @@ -776,8 +779,8 @@ private void testCreateTableLike() " metrics_max_inferred_column = 100,\n" + " partitioning = ARRAY['adate']\n" + ")", catalogType.equals(CatalogType.HIVE) ? - getLocation("tpch", "test_create_table_like_original") : - getLocation("tpch", "test_create_table_like_copy4"))); + getLocation(schemaName, "test_create_table_like_original") : + getLocation(schemaName, "test_create_table_like_copy4"))); dropTable(session, "test_create_table_like_copy4"); dropTable(session, "test_create_table_like_original"); @@ -1168,11 +1171,12 @@ protected void cleanupTableWithMergeOnRead(String tableName) @Test public void testMergeOnReadEnabled() { + String schemaName = getSession().getSchema().get(); String tableName = "test_merge_on_read_enabled"; try { Session session = getSession(); - createTableWithMergeOnRead(session, "tpch", tableName); + createTableWithMergeOnRead(session, schemaName, tableName); assertUpdate(session, "INSERT INTO " + tableName + " VALUES (1, 1)", 1); assertUpdate(session, "INSERT INTO " + tableName + " VALUES (2, 2)", 1); assertQuery(session, "SELECT * FROM " + tableName, "VALUES (1, 1), (2, 2)"); @@ -1192,7 +1196,7 @@ public void testMergeOnReadDisabled() .setCatalogSessionProperty(ICEBERG_CATALOG, "merge_on_read_enabled", "false") .build(); - createTableWithMergeOnRead(session, "tpch", tableName); + createTableWithMergeOnRead(session, session.getSchema().get(), tableName); assertQueryFails(session, "INSERT INTO " + tableName + " VALUES (1, 1)", errorMessage); assertQueryFails(session, "INSERT INTO " + tableName + " VALUES (2, 2)", errorMessage); assertQueryFails(session, "SELECT * FROM " + tableName, errorMessage); @@ -1849,6 +1853,7 @@ public void testUpdatingCommitRetries() public void testUpdateNonExistentTable() { assertQuerySucceeds("ALTER TABLE IF EXISTS non_existent_test_table1 SET PROPERTIES (commit_retries = 6)"); - assertQueryFails("ALTER TABLE non_existent_test_table2 SET PROPERTIES (commit_retries = 6)", "Table does not exist: iceberg.tpch.non_existent_test_table2"); + assertQueryFails("ALTER TABLE non_existent_test_table2 SET PROPERTIES (commit_retries = 6)", + format("Table does not exist: iceberg.%s.non_existent_test_table2", getSession().getSchema().get())); } } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergQueryRunner.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergQueryRunner.java index 09a300011c643..11be7a0e9857c 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergQueryRunner.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergQueryRunner.java @@ -146,7 +146,7 @@ public static DistributedQueryRunner createIcebergQueryRunner( Optional dataDirectory) throws Exception { - return createIcebergQueryRunner(extraProperties, extraConnectorProperties, format, createTpchTables, addJmxPlugin, nodeCount, externalWorkerLauncher, dataDirectory, false); + return createIcebergQueryRunner(extraProperties, extraConnectorProperties, format, createTpchTables, addJmxPlugin, nodeCount, externalWorkerLauncher, dataDirectory, false, Optional.empty()); } public static DistributedQueryRunner createIcebergQueryRunner( @@ -160,12 +160,28 @@ public static DistributedQueryRunner createIcebergQueryRunner( Optional dataDirectory, boolean addStorageFormatToPath) throws Exception + { + return createIcebergQueryRunner(extraProperties, extraConnectorProperties, format, createTpchTables, addJmxPlugin, nodeCount, externalWorkerLauncher, dataDirectory, addStorageFormatToPath, Optional.empty()); + } + + public static DistributedQueryRunner createIcebergQueryRunner( + Map extraProperties, + Map extraConnectorProperties, + FileFormat format, + boolean createTpchTables, + boolean addJmxPlugin, + OptionalInt nodeCount, + Optional> externalWorkerLauncher, + Optional dataDirectory, + boolean addStorageFormatToPath, + Optional schemaName) + throws Exception { setupLogging(); Session session = testSessionBuilder() .setCatalog(ICEBERG_CATALOG) - .setSchema("tpch") + .setSchema(schemaName.orElse("tpch")) .build(); DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(session) diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/rest/TestIcebergRestConfig.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/rest/TestIcebergRestConfig.java index e21f70992e57c..e500a429e1b61 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/rest/TestIcebergRestConfig.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/rest/TestIcebergRestConfig.java @@ -36,7 +36,8 @@ public void testDefaults() .setCredential(null) .setToken(null) .setScope(null) - .setSessionType(null)); + .setSessionType(null) + .setNestedNamespaceEnabled(true)); } @Test @@ -50,6 +51,7 @@ public void testExplicitPropertyMappings() .put("iceberg.rest.auth.oauth2.token", "SXVLUXUhIExFQ0tFUiEK") .put("iceberg.rest.auth.oauth2.scope", "PRINCIPAL_ROLE:ALL") .put("iceberg.rest.session.type", "USER") + .put("iceberg.rest.nested.namespace.enabled", "false") .build(); IcebergRestConfig expected = new IcebergRestConfig() @@ -59,7 +61,8 @@ public void testExplicitPropertyMappings() .setCredential("key:secret") .setToken("SXVLUXUhIExFQ0tFUiEK") .setScope("PRINCIPAL_ROLE:ALL") - .setSessionType(USER); + .setSessionType(USER) + .setNestedNamespaceEnabled(false); assertFullMapping(properties, expected); } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/rest/TestIcebergSmokeRestNestedNamespace.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/rest/TestIcebergSmokeRestNestedNamespace.java new file mode 100644 index 0000000000000..6660515ae6811 --- /dev/null +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/rest/TestIcebergSmokeRestNestedNamespace.java @@ -0,0 +1,364 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg.rest; + +import com.facebook.airlift.http.server.testing.TestingHttpServer; +import com.facebook.presto.Session; +import com.facebook.presto.hive.NodeVersion; +import com.facebook.presto.hive.gcs.HiveGcsConfig; +import com.facebook.presto.hive.gcs.HiveGcsConfigurationInitializer; +import com.facebook.presto.hive.s3.HiveS3Config; +import com.facebook.presto.hive.s3.PrestoS3ConfigurationUpdater; +import com.facebook.presto.iceberg.FileFormat; +import com.facebook.presto.iceberg.IcebergCatalogName; +import com.facebook.presto.iceberg.IcebergConfig; +import com.facebook.presto.iceberg.IcebergNativeCatalogFactory; +import com.facebook.presto.iceberg.IcebergQueryRunner; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.DistributedQueryRunner; +import com.google.common.collect.ImmutableMap; +import org.apache.iceberg.Table; +import org.assertj.core.util.Files; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; + +import static com.facebook.presto.iceberg.CatalogType.REST; +import static com.facebook.presto.iceberg.FileFormat.PARQUET; +import static com.facebook.presto.iceberg.IcebergQueryRunner.ICEBERG_CATALOG; +import static com.facebook.presto.iceberg.IcebergUtil.getNativeIcebergTable; +import static com.facebook.presto.iceberg.rest.IcebergRestTestUtil.getRestServer; +import static com.facebook.presto.iceberg.rest.IcebergRestTestUtil.restConnectorProperties; +import static com.google.common.collect.Iterables.getOnlyElement; +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static java.lang.String.format; +import static java.util.Locale.ENGLISH; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.testng.Assert.assertEquals; + +@Test +public class TestIcebergSmokeRestNestedNamespace + extends TestIcebergSmokeRest +{ + private static final String ICEBERG_NESTED_NAMESPACE_DISABLED_CATALOG = "iceberg_without_nested_namespaces"; + + private File warehouseLocation; + private TestingHttpServer restServer; + private String serverUri; + + public TestIcebergSmokeRestNestedNamespace() + { + } + + @BeforeClass + @Override + public void init() + throws Exception + { + warehouseLocation = Files.newTemporaryFolder(); + + restServer = getRestServer(warehouseLocation.getAbsolutePath()); + restServer.start(); + + serverUri = restServer.getBaseUrl().toString(); + super.init(); + } + + @AfterClass + public void tearDown() + throws Exception + { + if (restServer != null) { + restServer.stop(); + } + deleteRecursively(warehouseLocation.toPath(), ALLOW_INSECURE); + } + + @Override + protected String getLocation(String schema, String table) + { + String namespaceSeparatorEscaped = "\\."; + String schemaPath = schema.replaceAll(namespaceSeparatorEscaped, "/"); + + return format("%s/%s/%s", warehouseLocation, schemaPath, table); + } + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + Map restConnectorProperties = restConnectorProperties(serverUri); + DistributedQueryRunner icebergQueryRunner = IcebergQueryRunner.createIcebergQueryRunner( + ImmutableMap.of(), + restConnectorProperties, + PARQUET, + true, + false, + OptionalInt.empty(), + Optional.empty(), + Optional.of(warehouseLocation.toPath()), + false, + Optional.of("ns1.ns2")); + + // additional catalog for testing nested namespace disabled + icebergQueryRunner.createCatalog(ICEBERG_NESTED_NAMESPACE_DISABLED_CATALOG, "iceberg", + new ImmutableMap.Builder() + .putAll(restConnectorProperties) + .put("iceberg.rest.nested.namespace.enabled", "false") + .build()); + + return icebergQueryRunner; + } + + protected IcebergNativeCatalogFactory getCatalogFactory(IcebergRestConfig restConfig) + { + IcebergConfig icebergConfig = new IcebergConfig() + .setCatalogType(REST) + .setCatalogWarehouse(warehouseLocation.getAbsolutePath()); + + return new IcebergRestCatalogFactory( + icebergConfig, + restConfig, + new IcebergCatalogName(ICEBERG_CATALOG), + new PrestoS3ConfigurationUpdater(new HiveS3Config()), + new HiveGcsConfigurationInitializer(new HiveGcsConfig()), + new NodeVersion("test_version")); + } + + @Override + protected Table getIcebergTable(ConnectorSession session, String schema, String tableName) + { + IcebergRestConfig restConfig = new IcebergRestConfig().setServerUri(serverUri); + return getNativeIcebergTable(getCatalogFactory(restConfig), + session, + new SchemaTableName(schema, tableName)); + } + + @Test + @Override // override due to double quotes around nested namespace + public void testShowCreateTable() + { + String schemaName = getSession().getSchema().get(); + assertThat(computeActual("SHOW CREATE TABLE orders").getOnlyValue()) + .isEqualTo(format("CREATE TABLE iceberg.\"%s\".orders (\n" + + " \"orderkey\" bigint,\n" + + " \"custkey\" bigint,\n" + + " \"orderstatus\" varchar,\n" + + " \"totalprice\" double,\n" + + " \"orderdate\" date,\n" + + " \"orderpriority\" varchar,\n" + + " \"clerk\" varchar,\n" + + " \"shippriority\" integer,\n" + + " \"comment\" varchar\n" + + ")\n" + + "WITH (\n" + + " delete_mode = 'merge-on-read',\n" + + " format = 'PARQUET',\n" + + " format_version = '2',\n" + + " location = '%s',\n" + + " metadata_delete_after_commit = false,\n" + + " metadata_previous_versions_max = 100,\n" + + " metrics_max_inferred_column = 100\n" + + ")", schemaName, getLocation(schemaName, "orders"))); + } + + @Test + @Override // override due to double quotes around nested namespace + public void testTableComments() + { + Session session = getSession(); + String schemaName = session.getSchema().get(); + + @Language("SQL") String createTable = "" + + "CREATE TABLE iceberg.\"%s\".test_table_comments (\n" + + " \"_x\" bigint\n" + + ")\n" + + "COMMENT '%s'\n" + + "WITH (\n" + + " format = 'ORC',\n" + + " format_version = '2'\n" + + ")"; + + assertUpdate(format(createTable, schemaName, "test table comment")); + + String createTableTemplate = "" + + "CREATE TABLE iceberg.\"%s\".test_table_comments (\n" + + " \"_x\" bigint\n" + + ")\n" + + "COMMENT '%s'\n" + + "WITH (\n" + + " delete_mode = 'merge-on-read',\n" + + " format = 'ORC',\n" + + " format_version = '2',\n" + + " location = '%s',\n" + + " metadata_delete_after_commit = false,\n" + + " metadata_previous_versions_max = 100,\n" + + " metrics_max_inferred_column = 100\n" + + ")"; + String createTableSql = format(createTableTemplate, schemaName, "test table comment", getLocation(schemaName, "test_table_comments")); + + MaterializedResult resultOfCreate = computeActual("SHOW CREATE TABLE test_table_comments"); + assertEquals(getOnlyElement(resultOfCreate.getOnlyColumnAsSet()), createTableSql); + + dropTable(session, "test_table_comments"); + } + + @Override // override due to double quotes around nested namespace + protected void testCreatePartitionedTableAs(Session session, FileFormat fileFormat) + { + String fileFormatString = fileFormat.toString().toLowerCase(ENGLISH); + @Language("SQL") String createTable = "" + + "CREATE TABLE test_create_partitioned_table_as_%s " + + "WITH (" + + "format = '%s', " + + "partitioning = ARRAY['ORDER_STATUS', 'Ship_Priority', 'Bucket(order_key,9)']" + + ") " + + "AS " + + "SELECT orderkey AS order_key, shippriority AS ship_priority, orderstatus AS order_status " + + "FROM tpch.tiny.orders"; + + assertUpdate(session, format(createTable, fileFormatString, fileFormat), "SELECT count(*) from orders"); + + String createTableSql = format("" + + "CREATE TABLE %s.\"%s\".%s (\n" + + " \"order_key\" bigint,\n" + + " \"ship_priority\" integer,\n" + + " \"order_status\" varchar\n" + + ")\n" + + "WITH (\n" + + " delete_mode = 'merge-on-read',\n" + + " format = '%s',\n" + + " format_version = '2',\n" + + " location = '%s',\n" + + " metadata_delete_after_commit = false,\n" + + " metadata_previous_versions_max = 100,\n" + + " metrics_max_inferred_column = 100,\n" + + " partitioning = ARRAY['order_status','ship_priority','bucket(order_key, 9)']\n" + + ")", + getSession().getCatalog().get(), + getSession().getSchema().get(), + "test_create_partitioned_table_as_" + fileFormatString, + fileFormat, + getLocation(getSession().getSchema().get(), "test_create_partitioned_table_as_" + fileFormatString)); + + MaterializedResult actualResult = computeActual("SHOW CREATE TABLE test_create_partitioned_table_as_" + fileFormatString); + assertEquals(getOnlyElement(actualResult.getOnlyColumnAsSet()), createTableSql); + + assertQuery(session, "SELECT * from test_create_partitioned_table_as_" + fileFormatString, + "SELECT orderkey, shippriority, orderstatus FROM orders"); + + dropTable(session, "test_create_partitioned_table_as_" + fileFormatString); + } + + @Test + @Override + public void testCreateTableWithFormatVersion() + { + // v1 table create fails due to Iceberg REST catalog bug (see: https://github.com/apache/iceberg/issues/8756) + assertThatThrownBy(() -> testCreateTableWithFormatVersion("1", "copy-on-write")) + .hasCauseInstanceOf(RuntimeException.class) + .hasStackTraceContaining("Cannot downgrade v2 table to v1"); + + // v2 succeeds + testCreateTableWithFormatVersion("2", "merge-on-read"); + } + + @Override // override due to double quotes around nested namespace + protected void testCreateTableWithFormatVersion(String formatVersion, String defaultDeleteMode) + { + @Language("SQL") String createTable = "" + + "CREATE TABLE test_create_table_with_format_version_%s " + + "WITH (" + + "format = 'PARQUET', " + + "format_version = '%s'" + + ") " + + "AS " + + "SELECT orderkey AS order_key, shippriority AS ship_priority, orderstatus AS order_status " + + "FROM tpch.tiny.orders"; + + Session session = getSession(); + + assertUpdate(session, format(createTable, formatVersion, formatVersion), "SELECT count(*) from orders"); + + String createTableSql = format("" + + "CREATE TABLE %s.\"%s\".%s (\n" + + " \"order_key\" bigint,\n" + + " \"ship_priority\" integer,\n" + + " \"order_status\" varchar\n" + + ")\n" + + "WITH (\n" + + " delete_mode = '%s',\n" + + " format = 'PARQUET',\n" + + " format_version = '%s',\n" + + " location = '%s',\n" + + " metadata_delete_after_commit = false,\n" + + " metadata_previous_versions_max = 100,\n" + + " metrics_max_inferred_column = 100\n" + + ")", + getSession().getCatalog().get(), + getSession().getSchema().get(), + "test_create_table_with_format_version_" + formatVersion, + defaultDeleteMode, + formatVersion, + getLocation(getSession().getSchema().get(), "test_create_table_with_format_version_" + formatVersion)); + + MaterializedResult actualResult = computeActual("SHOW CREATE TABLE test_create_table_with_format_version_" + formatVersion); + assertEquals(getOnlyElement(actualResult.getOnlyColumnAsSet()), createTableSql); + + dropTable(session, "test_create_table_with_format_version_" + formatVersion); + } + + @Test + public void testView() + { + Session session = getSession(); + String schemaName = getSession().getSchema().get(); + + assertUpdate(session, "CREATE VIEW view_orders AS SELECT * from orders"); + assertQuery(session, "SELECT * FROM view_orders", "SELECT * from orders"); + assertThat(computeActual("SHOW CREATE VIEW view_orders").getOnlyValue()) + .isEqualTo(format("CREATE VIEW iceberg.\"%s\".view_orders AS\n" + + "SELECT *\n" + + "FROM\n" + + " orders", schemaName)); + assertUpdate(session, "DROP VIEW view_orders"); + } + + @Test + void testNestedNamespaceDisabled() + { + assertQuery(format("SHOW SCHEMAS FROM %s", ICEBERG_NESTED_NAMESPACE_DISABLED_CATALOG), + "VALUES 'ns1', 'tpch', 'tpcds', 'information_schema'"); + + assertQueryFails(format("CREATE SCHEMA %s.\"ns1.ns2.ns3\"", ICEBERG_NESTED_NAMESPACE_DISABLED_CATALOG), + "Nested namespaces are disabled. Schema ns1.ns2.ns3 is not valid"); + assertQueryFails(format("CREATE TABLE %s.\"ns1.ns2\".test_table(a int)", ICEBERG_NESTED_NAMESPACE_DISABLED_CATALOG), + "Nested namespaces are disabled. Schema ns1.ns2 is not valid"); + assertQueryFails(format("SELECT * FROM %s.\"ns1.ns2\".orders", ICEBERG_NESTED_NAMESPACE_DISABLED_CATALOG), + "Nested namespaces are disabled. Schema ns1.ns2 is not valid"); + assertQueryFails(format("SHOW TABLES IN %s.\"ns1.ns2\"", ICEBERG_NESTED_NAMESPACE_DISABLED_CATALOG), + "line 1:1: Schema 'ns1.ns2' does not exist"); + } +}