Skip to content

Commit

Permalink
Add config property 'tpcds.use-varchar-type'
Browse files Browse the repository at this point in the history 
Co-authored-by: Pramod Satya <pramod.satya@ibm.com>
  • Loading branch information
@pdabre12 @pramodsatya
2 people authored and Pratik Joseph Dabre committed Feb 17, 2025
1 parent 700fdc7 commit 7cd4ce2
Show file tree
Hide file tree
Showing 20 changed files with 257 additions and 181 deletions.
2 changes: 2 additions & 0 deletions presto-docs/src/main/sphinx/connector/tpcds.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,4 +66,6 @@ Property Name Description
================================================== ========================================================================== ==============================
``tpcds.splits-per-node`` Number of data splits generated per Presto worker node when querying Number of available processors
data from the TPCDS connector.

``tpcds.use-varchar-type`` Toggle all char columns to varchar in the TPC-DS connector. false
================================================== ========================================================================== ==============================
45 changes: 34 additions & 11 deletions presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,26 +98,32 @@ public static DistributedQueryRunner createQueryRunner(Iterable<TpchTable<?>> ta
return createQueryRunner(tables, ImmutableMap.of(), Optional.empty());
}

public static DistributedQueryRunner createQueryRunner(Iterable<TpchTable<?>> tpchTables, Map<String, String> extraProperties, Map<String, String> tpcdsProperties)
throws Exception
{
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, ImmutableMap.of(), "sql-standard", ImmutableMap.of(), Optional.empty(), Optional.empty(), Optional.empty(), tpcdsProperties);
}

public static DistributedQueryRunner createQueryRunner(
Iterable<TpchTable<?>> tpchTables,
Map<String, String> extraProperties,
Map<String, String> extraCoordinatorProperties,
Optional<Path> dataDirectory)
throws Exception
{
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, extraCoordinatorProperties, "sql-standard", ImmutableMap.of(), Optional.empty(), dataDirectory, Optional.empty());
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, extraCoordinatorProperties, "sql-standard", ImmutableMap.of(), Optional.empty(), dataDirectory, Optional.empty(), ImmutableMap.of());
}

public static DistributedQueryRunner createQueryRunner(Iterable<TpchTable<?>> tpchTables, Map<String, String> extraProperties, Optional<Path> dataDirectory)
throws Exception
{
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, ImmutableMap.of(), "sql-standard", ImmutableMap.of(), Optional.empty(), dataDirectory, Optional.empty());
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, ImmutableMap.of(), "sql-standard", ImmutableMap.of(), Optional.empty(), dataDirectory, Optional.empty(), ImmutableMap.of());
}

public static DistributedQueryRunner createQueryRunner(Iterable<TpchTable<?>> tpchTables, List<String> tpcdsTableNames, Map<String, String> extraProperties, Optional<Path> dataDirectory)
throws Exception
{
return createQueryRunner(tpchTables, tpcdsTableNames, extraProperties, ImmutableMap.of(), "sql-standard", ImmutableMap.of(), Optional.empty(), dataDirectory, Optional.empty());
return createQueryRunner(tpchTables, tpcdsTableNames, extraProperties, ImmutableMap.of(), "sql-standard", ImmutableMap.of(), Optional.empty(), dataDirectory, Optional.empty(), ImmutableMap.of());
}

public static DistributedQueryRunner createQueryRunner(
Expand All @@ -128,7 +134,19 @@ public static DistributedQueryRunner createQueryRunner(
Optional<Path> dataDirectory)
throws Exception
{
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, ImmutableMap.of(), security, extraHiveProperties, Optional.empty(), dataDirectory, Optional.empty());
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, ImmutableMap.of(), security, extraHiveProperties, Optional.empty(), dataDirectory, Optional.empty(), ImmutableMap.of());
}

public static DistributedQueryRunner createQueryRunner(
Iterable<TpchTable<?>> tpchTables,
Map<String, String> extraProperties,
String security,
Map<String, String> extraHiveProperties,
Optional<Path> dataDirectory,
Map<String, String> tpcdsProperties)
throws Exception
{
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, ImmutableMap.of(), security, extraHiveProperties, Optional.empty(), dataDirectory, Optional.empty(), tpcdsProperties);
}

public static DistributedQueryRunner createQueryRunner(
Expand All @@ -140,10 +158,11 @@ public static DistributedQueryRunner createQueryRunner(
Map<String, String> extraHiveProperties,
Optional<Integer> workerCount,
Optional<Path> dataDirectory,
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher)
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher,
Map<String, String> tpcdsProperties)
throws Exception
{
return createQueryRunner(tpchTables, tpcdsTableNames, extraProperties, extraCoordinatorProperties, security, extraHiveProperties, workerCount, dataDirectory, externalWorkerLauncher, Optional.empty());
return createQueryRunner(tpchTables, tpcdsTableNames, extraProperties, extraCoordinatorProperties, security, extraHiveProperties, workerCount, dataDirectory, externalWorkerLauncher, Optional.empty(), tpcdsProperties);
}

public static DistributedQueryRunner createQueryRunner(
Expand All @@ -156,7 +175,8 @@ public static DistributedQueryRunner createQueryRunner(
Optional<Integer> workerCount,
Optional<Path> dataDirectory,
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher,
Optional<ExtendedHiveMetastore> externalMetastore)
Optional<ExtendedHiveMetastore> externalMetastore,
Map<String, String> tpcdsProperties)
throws Exception
{
return createQueryRunner(
Expand All @@ -170,7 +190,8 @@ public static DistributedQueryRunner createQueryRunner(
dataDirectory,
externalWorkerLauncher,
externalMetastore,
false);
false,
tpcdsProperties);
}

public static DistributedQueryRunner createQueryRunner(
Expand All @@ -184,7 +205,8 @@ public static DistributedQueryRunner createQueryRunner(
Optional<Path> dataDirectory,
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher,
Optional<ExtendedHiveMetastore> externalMetastore,
boolean addJmxPlugin)
boolean addJmxPlugin,
Map<String, String> tpcdsProperties)
throws Exception
{
assertEquals(DateTimeZone.getDefault(), TIME_ZONE, "Timezone not configured correctly. Add -Duser.timezone=America/Bahia_Banderas to your JVM arguments");
Expand All @@ -211,7 +233,7 @@ public static DistributedQueryRunner createQueryRunner(
queryRunner.installPlugin(new TpcdsPlugin());
queryRunner.installPlugin(new TestingHiveEventListenerPlugin());
queryRunner.createCatalog("tpch", "tpch");
queryRunner.createCatalog("tpcds", "tpcds");
queryRunner.createCatalog("tpcds", "tpcds", tpcdsProperties);
Map<String, String> tpchProperties = ImmutableMap.<String, String>builder()
.put("tpch.column-naming", "standard")
.build();
Expand Down Expand Up @@ -326,7 +348,8 @@ public static DistributedQueryRunner createMaterializingQueryRunner(Iterable<Tpc
"grouped-execution-enabled", "true"),
"sql-standard",
ImmutableMap.of("hive.create-empty-bucket-files-for-temporary-table", "false"),
Optional.empty());
Optional.empty(),
ImmutableMap.of());
}

public static DistributedQueryRunner createMaterializingAndSpillingQueryRunner(Iterable<TpchTable<?>> tables)
Expand Down
4 changes: 1 addition & 3 deletions presto-hive/src/test/java/com/facebook/presto/hive/TestHiveNativeLogicalPlanner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@
import com.google.common.collect.ImmutableMap;
import org.testng.annotations.Test;

import java.util.Optional;

import static com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG;
import static com.facebook.presto.hive.HiveSessionProperties.PARTIAL_AGGREGATION_PUSHDOWN_ENABLED;
import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree;
Expand All @@ -47,7 +45,7 @@ protected QueryRunner createQueryRunner()
return HiveQueryRunner.createQueryRunner(
ImmutableList.of(ORDERS),
ImmutableMap.of("native-execution-enabled", "true"),
Optional.empty());
ImmutableMap.of("tpcds.use-varchar-type", "true"));
}

@Test
Expand Down
3 changes: 2 additions & 1 deletion presto-hive/src/test/java/com/facebook/presto/hive/s3/S3HiveQueryRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ public static DistributedQueryRunner create(
hiveEndpoint.getPort()),
new MetastoreClientConfig(),
HDFS_ENVIRONMENT),
new HivePartitionMutator())));
new HivePartitionMutator())),
ImmutableMap.of());
}
}
12 changes: 7 additions & 5 deletions presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergQueryRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ public static DistributedQueryRunner createIcebergQueryRunner(
Optional<Path> dataDirectory)
throws Exception
{
return createIcebergQueryRunner(extraProperties, extraConnectorProperties, format, createTpchTables, addJmxPlugin, nodeCount, externalWorkerLauncher, dataDirectory, false, Optional.empty());
return createIcebergQueryRunner(extraProperties, extraConnectorProperties, format, createTpchTables, addJmxPlugin, nodeCount, externalWorkerLauncher, dataDirectory, false, Optional.empty(), ImmutableMap.of());
}

public static DistributedQueryRunner createIcebergQueryRunner(
Expand All @@ -161,10 +161,11 @@ public static DistributedQueryRunner createIcebergQueryRunner(
OptionalInt nodeCount,
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher,
Optional<Path> dataDirectory,
boolean addStorageFormatToPath)
boolean addStorageFormatToPath,
Map<String, String> tpcdsProperties)
throws Exception
{
return createIcebergQueryRunner(extraProperties, extraConnectorProperties, format, createTpchTables, addJmxPlugin, nodeCount, externalWorkerLauncher, dataDirectory, addStorageFormatToPath, Optional.empty());
return createIcebergQueryRunner(extraProperties, extraConnectorProperties, format, createTpchTables, addJmxPlugin, nodeCount, externalWorkerLauncher, dataDirectory, addStorageFormatToPath, Optional.empty(), tpcdsProperties);
}

public static DistributedQueryRunner createIcebergQueryRunner(
Expand All @@ -177,7 +178,8 @@ public static DistributedQueryRunner createIcebergQueryRunner(
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher,
Optional<Path> dataDirectory,
boolean addStorageFormatToPath,
Optional<String> schemaName)
Optional<String> schemaName,
Map<String, String> tpcdsProperties)
throws Exception
{
setupLogging();
Expand All @@ -198,7 +200,7 @@ public static DistributedQueryRunner createIcebergQueryRunner(
queryRunner.createCatalog("tpch", "tpch");

queryRunner.installPlugin(new TpcdsPlugin());
queryRunner.createCatalog("tpcds", "tpcds");
queryRunner.createCatalog("tpcds", "tpcds", tpcdsProperties);

queryRunner.getServers().forEach(server -> {
MBeanServer mBeanServer = MBeanServerFactory.newMBeanServer();
Expand Down
3 changes: 2 additions & 1 deletion ...g/src/test/java/com/facebook/presto/iceberg/rest/TestIcebergSmokeRestNestedNamespace.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,8 @@ protected QueryRunner createQueryRunner()
Optional.empty(),
Optional.of(warehouseLocation.toPath()),
false,
Optional.of("ns1.ns2"));
Optional.of("ns1.ns2"),
ImmutableMap.of());

// additional catalog for testing nested namespace disabled
icebergQueryRunner.createCatalog(ICEBERG_NESTED_NAMESPACE_DISABLED_CATALOG, "iceberg",
Expand Down
Loading

0 comments on commit 7cd4ce2

Please sign in to comment.