Skip to content

Commit

Permalink
Move isManagedTable and deleteDirectoryRecursively helper methods to …
Browse files Browse the repository at this point in the history 
…MetastoreUtil class
  • Loading branch information
@imjalpreet @highker
imjalpreet authored and highker committed Apr 20, 2022
1 parent bab2ea3 commit ce5e322
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 46 deletions.
20 changes: 20 additions & 0 deletions presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
*/
package com.facebook.presto.hive.metastore;

import com.facebook.airlift.log.Logger;
import com.facebook.presto.common.Page;
import com.facebook.presto.common.block.Block;
import com.facebook.presto.common.predicate.Domain;
Expand Down Expand Up @@ -100,6 +101,7 @@
import static com.facebook.presto.common.type.Varchars.isVarcharType;
import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR;
import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE;
import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE;
import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED;
import static com.facebook.presto.spi.statistics.ColumnStatisticType.MAX_VALUE;
import static com.facebook.presto.spi.statistics.ColumnStatisticType.MAX_VALUE_SIZE_IN_BYTES;
Expand Down Expand Up @@ -136,6 +138,8 @@

public class MetastoreUtil
{
private static final Logger log = Logger.get(MetastoreUtil.class);

public static final String METASTORE_HEADERS = "metastore_headers";
public static final String PRESTO_OFFLINE = "presto_offline";
public static final String AVRO_SCHEMA_URL_KEY = "avro.schema.url";
Expand Down Expand Up @@ -948,4 +952,20 @@ public static boolean isUserDefinedTypeEncodingEnabled(ConnectorSession session)
return false;
}
}

public static boolean isManagedTable(String tableType)
{
return tableType.equals(MANAGED_TABLE.name());
}

public static void deleteDirectoryRecursively(HdfsContext context, HdfsEnvironment hdfsEnvironment, Path path, boolean recursive)
{
try {
hdfsEnvironment.getFileSystem(context, path).delete(path, recursive);
}
catch (IOException | RuntimeException e) {
// don't fail if unable to delete path
log.warn(e, "Failed to delete path: " + path.toString());
}
}
}
30 changes: 6 additions & 24 deletions ...ve-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@
import com.amazonaws.services.glue.model.UpdateDatabaseRequest;
import com.amazonaws.services.glue.model.UpdatePartitionRequest;
import com.amazonaws.services.glue.model.UpdateTableRequest;
import com.facebook.airlift.log.Logger;
import com.facebook.presto.common.predicate.Domain;
import com.facebook.presto.common.type.Type;
import com.facebook.presto.hive.HdfsContext;
Expand Down Expand Up @@ -123,12 +122,13 @@
import static com.facebook.presto.hive.metastore.MetastoreOperationResult.EMPTY_RESULT;
import static com.facebook.presto.hive.metastore.MetastoreUtil.convertPredicateToParts;
import static com.facebook.presto.hive.metastore.MetastoreUtil.createDirectory;
import static com.facebook.presto.hive.metastore.MetastoreUtil.deleteDirectoryRecursively;
import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveBasicStatistics;
import static com.facebook.presto.hive.metastore.MetastoreUtil.isManagedTable;
import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName;
import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues;
import static com.facebook.presto.hive.metastore.MetastoreUtil.updateStatisticsParameters;
import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyCanDropColumn;
import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE;
import static com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW;
import static com.facebook.presto.hive.metastore.glue.GlueExpressionUtil.buildGlueExpression;
import static com.facebook.presto.hive.metastore.glue.converter.GlueInputConverter.convertColumn;
Expand All @@ -148,8 +148,6 @@
public class GlueHiveMetastore
implements ExtendedHiveMetastore
{
private static final Logger log = Logger.get(GlueHiveMetastore.class);

private static final String PUBLIC_ROLE_NAME = "public";
private static final String DEFAULT_METASTORE_USER = "presto";
private static final String WILDCARD_EXPRESSION = "";
Expand Down Expand Up @@ -537,24 +535,8 @@ public void dropTable(MetastoreContext metastoreContext, String databaseName, St
}

String tableLocation = table.getStorage().getLocation();
if (deleteData && isManagedTable(table) && !isNullOrEmpty(tableLocation)) {
deleteDir(hdfsContext, hdfsEnvironment, new Path(tableLocation), true);
}
}

private static boolean isManagedTable(Table table)
{
return table.getTableType().equals(MANAGED_TABLE);
}

private static void deleteDir(HdfsContext context, HdfsEnvironment hdfsEnvironment, Path path, boolean recursive)
{
try {
hdfsEnvironment.getFileSystem(context, path).delete(path, recursive);
}
catch (Exception e) {
// don't fail if unable to delete path
log.warn(e, "Failed to delete path: " + path.toString());
if (deleteData && isManagedTable(table.getTableType().name()) && !isNullOrEmpty(tableLocation)) {
deleteDirectoryRecursively(hdfsContext, hdfsEnvironment, new Path(tableLocation), true);
}
}

Expand Down Expand Up @@ -924,8 +906,8 @@ public void dropPartition(MetastoreContext metastoreContext, String databaseName
}

String partLocation = partition.getStorage().getLocation();
if (deleteData && isManagedTable(table) && !isNullOrEmpty(partLocation)) {
deleteDir(hdfsContext, hdfsEnvironment, new Path(partLocation), true);
if (deleteData && isManagedTable(table.getTableType().name()) && !isNullOrEmpty(partLocation)) {
deleteDirectoryRecursively(hdfsContext, hdfsEnvironment, new Path(partLocation), true);
}
}

Expand Down
26 changes: 4 additions & 22 deletions ...etastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
*/
package com.facebook.presto.hive.metastore.thrift;

import com.facebook.airlift.log.Logger;
import com.facebook.presto.common.predicate.Domain;
import com.facebook.presto.common.type.ArrayType;
import com.facebook.presto.common.type.MapType;
Expand Down Expand Up @@ -113,7 +112,9 @@
import static com.facebook.presto.hive.metastore.MetastoreOperationResult.EMPTY_RESULT;
import static com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_VIEW_FLAG;
import static com.facebook.presto.hive.metastore.MetastoreUtil.convertPredicateToParts;
import static com.facebook.presto.hive.metastore.MetastoreUtil.deleteDirectoryRecursively;
import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveBasicStatistics;
import static com.facebook.presto.hive.metastore.MetastoreUtil.isManagedTable;
import static com.facebook.presto.hive.metastore.MetastoreUtil.updateStatisticsParameters;
import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.createMetastoreColumnStatistics;
import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreApiPrincipalType;
Expand Down Expand Up @@ -144,7 +145,6 @@
import static java.util.function.Function.identity;
import static java.util.stream.Collectors.toSet;
import static org.apache.hadoop.hive.common.FileUtils.makePartName;
import static org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE;
import static org.apache.hadoop.hive.metastore.api.HiveObjectType.TABLE;
import static org.apache.hadoop.hive.metastore.api.LockState.ACQUIRED;
import static org.apache.hadoop.hive.metastore.api.LockState.WAITING;
Expand All @@ -155,8 +155,6 @@
public class ThriftHiveMetastore
implements HiveMetastore
{
private static final Logger log = Logger.get(ThriftHiveMetastore.class);

private static final String DEFAULT_METASTORE_USER = "presto";
private static final HdfsContext hdfsContext = new HdfsContext(new ConnectorIdentity(DEFAULT_METASTORE_USER, Optional.empty(), Optional.empty()));

Expand Down Expand Up @@ -916,8 +914,8 @@ public void dropTable(MetastoreContext metastoreContext, String databaseName, St
Table table = client.getTable(databaseName, tableName);
client.dropTable(databaseName, tableName, deleteData);
String tableLocation = table.getSd().getLocation();
if (deleteData && isManagedTable(table) && !isNullOrEmpty(tableLocation)) {
deleteDirectoryRecursively(hdfsContext, hdfsEnvironment, new Path(tableLocation));
if (deleteData && isManagedTable(table.getTableType()) && !isNullOrEmpty(tableLocation)) {
deleteDirectoryRecursively(hdfsContext, hdfsEnvironment, new Path(tableLocation), true);
}
}
return null;
Expand All @@ -934,22 +932,6 @@ public void dropTable(MetastoreContext metastoreContext, String databaseName, St
}
}

private static void deleteDirectoryRecursively(HdfsContext context, HdfsEnvironment hdfsEnvironment, Path path)
{
try {
hdfsEnvironment.getFileSystem(context, path).delete(path, true);
}
catch (IOException | RuntimeException e) {
// don't fail if unable to delete path
log.warn(e, "Failed to delete path: " + path.toString());
}
}

private static boolean isManagedTable(Table table)
{
return table.getTableType().equals(MANAGED_TABLE.name());
}

@Override
public MetastoreOperationResult alterTable(MetastoreContext metastoreContext, String databaseName, String tableName, Table table)
{
Expand Down

0 comments on commit ce5e322

Please sign in to comment.