Skip to content

Commit 84efc90

Browse files
pdabre12pramodsatya
authored and
aditi-pandit
committed
Add config property 'tpcds.use-varchar-type'
Co-authored-by: Pramod Satya <pramod.satya@ibm.com>
1 parent 0c546f7 commit 84efc90

File tree

20 files changed

+258
-182
lines changed

20 files changed

+258
-182
lines changed

‎presto-docs/src/main/sphinx/connector/tpcds.rst

+2
Original file line numberDiff line numberDiff line change
@@ -66,4 +66,6 @@ Property Name Description
6666
================================================== ========================================================================== ==============================
6767
``tpcds.splits-per-node`` Number of data splits generated per Presto worker node when querying Number of available processors
6868
data from the TPCDS connector.
69+
70+
``tpcds.use-varchar-type`` Toggle all char columns to varchar in the TPC-DS connector. false
6971
================================================== ========================================================================== ==============================

‎presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java

+34-11
Original file line numberDiff line numberDiff line change
@@ -98,26 +98,32 @@ public static DistributedQueryRunner createQueryRunner(Iterable<TpchTable<?>> ta
9898
return createQueryRunner(tables, ImmutableMap.of(), Optional.empty());
9999
}
100100

101+
public static DistributedQueryRunner createQueryRunner(Iterable<TpchTable<?>> tpchTables, Map<String, String> extraProperties, Map<String, String> tpcdsProperties)
102+
throws Exception
103+
{
104+
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, ImmutableMap.of(), "sql-standard", ImmutableMap.of(), Optional.empty(), Optional.empty(), Optional.empty(), tpcdsProperties);
105+
}
106+
101107
public static DistributedQueryRunner createQueryRunner(
102108
Iterable<TpchTable<?>> tpchTables,
103109
Map<String, String> extraProperties,
104110
Map<String, String> extraCoordinatorProperties,
105111
Optional<Path> dataDirectory)
106112
throws Exception
107113
{
108-
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, extraCoordinatorProperties, "sql-standard", ImmutableMap.of(), Optional.empty(), dataDirectory, Optional.empty());
114+
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, extraCoordinatorProperties, "sql-standard", ImmutableMap.of(), Optional.empty(), dataDirectory, Optional.empty(), ImmutableMap.of());
109115
}
110116

111117
public static DistributedQueryRunner createQueryRunner(Iterable<TpchTable<?>> tpchTables, Map<String, String> extraProperties, Optional<Path> dataDirectory)
112118
throws Exception
113119
{
114-
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, ImmutableMap.of(), "sql-standard", ImmutableMap.of(), Optional.empty(), dataDirectory, Optional.empty());
120+
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, ImmutableMap.of(), "sql-standard", ImmutableMap.of(), Optional.empty(), dataDirectory, Optional.empty(), ImmutableMap.of());
115121
}
116122

117123
public static DistributedQueryRunner createQueryRunner(Iterable<TpchTable<?>> tpchTables, List<String> tpcdsTableNames, Map<String, String> extraProperties, Optional<Path> dataDirectory)
118124
throws Exception
119125
{
120-
return createQueryRunner(tpchTables, tpcdsTableNames, extraProperties, ImmutableMap.of(), "sql-standard", ImmutableMap.of(), Optional.empty(), dataDirectory, Optional.empty());
126+
return createQueryRunner(tpchTables, tpcdsTableNames, extraProperties, ImmutableMap.of(), "sql-standard", ImmutableMap.of(), Optional.empty(), dataDirectory, Optional.empty(), ImmutableMap.of());
121127
}
122128

123129
public static DistributedQueryRunner createQueryRunner(
@@ -128,7 +134,19 @@ public static DistributedQueryRunner createQueryRunner(
128134
Optional<Path> dataDirectory)
129135
throws Exception
130136
{
131-
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, ImmutableMap.of(), security, extraHiveProperties, Optional.empty(), dataDirectory, Optional.empty());
137+
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, ImmutableMap.of(), security, extraHiveProperties, Optional.empty(), dataDirectory, Optional.empty(), ImmutableMap.of());
138+
}
139+
140+
public static DistributedQueryRunner createQueryRunner(
141+
Iterable<TpchTable<?>> tpchTables,
142+
Map<String, String> extraProperties,
143+
String security,
144+
Map<String, String> extraHiveProperties,
145+
Optional<Path> dataDirectory,
146+
Map<String, String> tpcdsProperties)
147+
throws Exception
148+
{
149+
return createQueryRunner(tpchTables, ImmutableList.of(), extraProperties, ImmutableMap.of(), security, extraHiveProperties, Optional.empty(), dataDirectory, Optional.empty(), tpcdsProperties);
132150
}
133151

134152
public static DistributedQueryRunner createQueryRunner(
@@ -140,10 +158,11 @@ public static DistributedQueryRunner createQueryRunner(
140158
Map<String, String> extraHiveProperties,
141159
Optional<Integer> workerCount,
142160
Optional<Path> dataDirectory,
143-
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher)
161+
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher,
162+
Map<String, String> tpcdsProperties)
144163
throws Exception
145164
{
146-
return createQueryRunner(tpchTables, tpcdsTableNames, extraProperties, extraCoordinatorProperties, security, extraHiveProperties, workerCount, dataDirectory, externalWorkerLauncher, Optional.empty());
165+
return createQueryRunner(tpchTables, tpcdsTableNames, extraProperties, extraCoordinatorProperties, security, extraHiveProperties, workerCount, dataDirectory, externalWorkerLauncher, Optional.empty(), tpcdsProperties);
147166
}
148167

149168
public static DistributedQueryRunner createQueryRunner(
@@ -156,7 +175,8 @@ public static DistributedQueryRunner createQueryRunner(
156175
Optional<Integer> workerCount,
157176
Optional<Path> dataDirectory,
158177
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher,
159-
Optional<ExtendedHiveMetastore> externalMetastore)
178+
Optional<ExtendedHiveMetastore> externalMetastore,
179+
Map<String, String> tpcdsProperties)
160180
throws Exception
161181
{
162182
return createQueryRunner(
@@ -170,7 +190,8 @@ public static DistributedQueryRunner createQueryRunner(
170190
dataDirectory,
171191
externalWorkerLauncher,
172192
externalMetastore,
173-
false);
193+
false,
194+
tpcdsProperties);
174195
}
175196

176197
public static DistributedQueryRunner createQueryRunner(
@@ -184,7 +205,8 @@ public static DistributedQueryRunner createQueryRunner(
184205
Optional<Path> dataDirectory,
185206
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher,
186207
Optional<ExtendedHiveMetastore> externalMetastore,
187-
boolean addJmxPlugin)
208+
boolean addJmxPlugin,
209+
Map<String, String> tpcdsProperties)
188210
throws Exception
189211
{
190212
assertEquals(DateTimeZone.getDefault(), TIME_ZONE, "Timezone not configured correctly. Add -Duser.timezone=America/Bahia_Banderas to your JVM arguments");
@@ -211,7 +233,7 @@ public static DistributedQueryRunner createQueryRunner(
211233
queryRunner.installPlugin(new TpcdsPlugin());
212234
queryRunner.installPlugin(new TestingHiveEventListenerPlugin());
213235
queryRunner.createCatalog("tpch", "tpch");
214-
queryRunner.createCatalog("tpcds", "tpcds");
236+
queryRunner.createCatalog("tpcds", "tpcds", tpcdsProperties);
215237
Map<String, String> tpchProperties = ImmutableMap.<String, String>builder()
216238
.put("tpch.column-naming", "standard")
217239
.build();
@@ -326,7 +348,8 @@ public static DistributedQueryRunner createMaterializingQueryRunner(Iterable<Tpc
326348
"grouped-execution-enabled", "true"),
327349
"sql-standard",
328350
ImmutableMap.of("hive.create-empty-bucket-files-for-temporary-table", "false"),
329-
Optional.empty());
351+
Optional.empty(),
352+
ImmutableMap.of());
330353
}
331354

332355
public static DistributedQueryRunner createMaterializingAndSpillingQueryRunner(Iterable<TpchTable<?>> tables)

‎presto-hive/src/test/java/com/facebook/presto/hive/TestHiveNativeLogicalPlanner.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,6 @@
2525
import com.google.common.collect.ImmutableMap;
2626
import org.testng.annotations.Test;
2727

28-
import java.util.Optional;
29-
3028
import static com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG;
3129
import static com.facebook.presto.hive.HiveSessionProperties.PARTIAL_AGGREGATION_PUSHDOWN_ENABLED;
3230
import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree;
@@ -47,7 +45,7 @@ protected QueryRunner createQueryRunner()
4745
return HiveQueryRunner.createQueryRunner(
4846
ImmutableList.of(ORDERS),
4947
ImmutableMap.of("native-execution-enabled", "true"),
50-
Optional.empty());
48+
ImmutableMap.of("tpcds.use-varchar-type", "true"));
5149
}
5250

5351
@Test

‎presto-hive/src/test/java/com/facebook/presto/hive/s3/S3HiveQueryRunner.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ public static DistributedQueryRunner create(
6060
hiveEndpoint.getPort()),
6161
new MetastoreClientConfig(),
6262
HDFS_ENVIRONMENT),
63-
new HivePartitionMutator())));
63+
new HivePartitionMutator())),
64+
ImmutableMap.of());
6465
}
6566
}

‎presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergQueryRunner.java

+7-5
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ public static DistributedQueryRunner createIcebergQueryRunner(
149149
Optional<Path> dataDirectory)
150150
throws Exception
151151
{
152-
return createIcebergQueryRunner(extraProperties, extraConnectorProperties, format, createTpchTables, addJmxPlugin, nodeCount, externalWorkerLauncher, dataDirectory, false, Optional.empty());
152+
return createIcebergQueryRunner(extraProperties, extraConnectorProperties, format, createTpchTables, addJmxPlugin, nodeCount, externalWorkerLauncher, dataDirectory, false, Optional.empty(), ImmutableMap.of());
153153
}
154154

155155
public static DistributedQueryRunner createIcebergQueryRunner(
@@ -161,10 +161,11 @@ public static DistributedQueryRunner createIcebergQueryRunner(
161161
OptionalInt nodeCount,
162162
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher,
163163
Optional<Path> dataDirectory,
164-
boolean addStorageFormatToPath)
164+
boolean addStorageFormatToPath,
165+
Map<String, String> tpcdsProperties)
165166
throws Exception
166167
{
167-
return createIcebergQueryRunner(extraProperties, extraConnectorProperties, format, createTpchTables, addJmxPlugin, nodeCount, externalWorkerLauncher, dataDirectory, addStorageFormatToPath, Optional.empty());
168+
return createIcebergQueryRunner(extraProperties, extraConnectorProperties, format, createTpchTables, addJmxPlugin, nodeCount, externalWorkerLauncher, dataDirectory, addStorageFormatToPath, Optional.empty(), tpcdsProperties);
168169
}
169170

170171
public static DistributedQueryRunner createIcebergQueryRunner(
@@ -177,7 +178,8 @@ public static DistributedQueryRunner createIcebergQueryRunner(
177178
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher,
178179
Optional<Path> dataDirectory,
179180
boolean addStorageFormatToPath,
180-
Optional<String> schemaName)
181+
Optional<String> schemaName,
182+
Map<String, String> tpcdsProperties)
181183
throws Exception
182184
{
183185
setupLogging();
@@ -198,7 +200,7 @@ public static DistributedQueryRunner createIcebergQueryRunner(
198200
queryRunner.createCatalog("tpch", "tpch");
199201

200202
queryRunner.installPlugin(new TpcdsPlugin());
201-
queryRunner.createCatalog("tpcds", "tpcds");
203+
queryRunner.createCatalog("tpcds", "tpcds", tpcdsProperties);
202204

203205
queryRunner.getServers().forEach(server -> {
204206
MBeanServer mBeanServer = MBeanServerFactory.newMBeanServer();

‎presto-iceberg/src/test/java/com/facebook/presto/iceberg/rest/TestIcebergSmokeRestNestedNamespace.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,8 @@ protected QueryRunner createQueryRunner()
120120
Optional.empty(),
121121
Optional.of(warehouseLocation.toPath()),
122122
false,
123-
Optional.of("ns1.ns2"));
123+
Optional.of("ns1.ns2"),
124+
ImmutableMap.of());
124125

125126
// additional catalog for testing nested namespace disabled
126127
icebergQueryRunner.createCatalog(ICEBERG_NESTED_NAMESPACE_DISABLED_CATALOG, "iceberg",

0 commit comments

Comments
 (0)