Skip to content

Commit

Permalink
Replace random test values by discrete ones (prestodb#23956)
Browse files Browse the repository at this point in the history 
The use of Random() function in the parquet.batchreader.decoders tests may cause flakiness.
Add TestMode.java to parameterize tests with an arbitrary value and an upper-/lower-bounded value.

Resolves: prestodb#23840
  • Loading branch information
@ges1227
ges1227 authored Dec 12, 2024
1 parent e8cca76 commit 5c9fdb7
Show file tree
Hide file tree
Showing 3 changed files with 136 additions and 75 deletions.
10 changes: 4 additions & 6 deletions ...java/com/facebook/presto/parquet/batchreader/decoders/TestFlatDefinitionLevelDecoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,10 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;

import static com.facebook.presto.parquet.batchreader.decoders.TestParquetUtils.addDLRLEBlock;
import static com.facebook.presto.parquet.batchreader.decoders.TestParquetUtils.addDLValues;
import static com.facebook.presto.parquet.batchreader.decoders.TestParquetUtils.randomValues;
import static com.facebook.presto.parquet.batchreader.decoders.TestParquetUtils.fillValues;
import static java.lang.Math.min;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.fail;
Expand All @@ -35,19 +34,18 @@ public class TestFlatDefinitionLevelDecoder
private static int valueCount;
private static int nonNullCount;
private static byte[] pageBytes;
private static List<Integer> expectedValues = new ArrayList<>();
private static final List<Integer> expectedValues = new ArrayList<>();

@BeforeClass
public void setup()
throws IOException
{
Random random = new Random(200);
RunLengthBitPackingHybridEncoder encoder = TestParquetUtils.getSimpleDLEncoder();

addDLRLEBlock(1, 50, encoder, expectedValues);
addDLValues(randomValues(random, 457, 1), encoder, expectedValues);
addDLValues(fillValues(457, 1), encoder, expectedValues);
addDLRLEBlock(0, 37, encoder, expectedValues);
addDLValues(randomValues(random, 186, 1), encoder, expectedValues);
addDLValues(fillValues(186, 1), encoder, expectedValues);

valueCount = expectedValues.size();
for (Integer value : expectedValues) {
Expand Down
49 changes: 21 additions & 28 deletions ...quet/src/test/java/com/facebook/presto/parquet/batchreader/decoders/TestParquetUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
*/
package com.facebook.presto.parquet.batchreader.decoders;

import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime;
import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils;
import org.apache.parquet.bytes.BytesUtils;
Expand All @@ -31,7 +30,6 @@
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Random;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
Expand Down Expand Up @@ -80,16 +78,16 @@ public static void addDLValues(Iterator<Integer> values, RunLengthBitPackingHybr
}
}

public static Iterator<Integer> randomValues(Random random, int numValues, int maxValue)
public static Iterator<Integer> fillValues(int numValues, int maxValue)
{
List<Integer> values = new ArrayList<>();
for (int i = 0; i < numValues; i++) {
values.add(random.nextInt(maxValue + 1));
values.add(maxValue);
}
return values.iterator();
}

public static byte[] generatePlainValuesPage(int valueCount, int valueSizeBits, Random random, List<Object> addedValues)
public static byte[] generatePlainValuesPage(int valueCount, int valueSizeBits, List<Object> addedValues, int valueInt, long valueLong, int positiveUpperBoundedInt)
{
ValuesWriter writer;

Expand All @@ -103,15 +101,14 @@ public static byte[] generatePlainValuesPage(int valueCount, int valueSizeBits,
switch (valueSizeBits) {
case 1: {
for (int i = 0; i < valueCount; i++) {
int value = random.nextInt(2);
writer.writeInteger(value);
addedValues.add(value);
writer.writeInteger(positiveUpperBoundedInt);
addedValues.add(positiveUpperBoundedInt);
}
break;
}
case -1: {
for (int i = 0; i < valueCount; i++) {
String valueStr = RandomStringUtils.random(random.nextInt(10), 0, 0, true, true, null, random);
String valueStr = "4nY" + valueCount;
byte[] valueUtf8 = valueStr.getBytes(StandardCharsets.UTF_8);
writer.writeBytes(Binary.fromConstantByteArray(valueUtf8, 0, valueUtf8.length));
addedValues.add(valueStr);
Expand All @@ -120,23 +117,21 @@ public static byte[] generatePlainValuesPage(int valueCount, int valueSizeBits,
}
case 32: {
for (int i = 0; i < valueCount; i++) {
int value = random.nextInt();
writer.writeInteger(value);
addedValues.add(value);
writer.writeInteger(valueInt);
addedValues.add(valueInt);
}
break;
}
case 64: {
for (int i = 0; i < valueCount; i++) {
long value = random.nextLong();
writer.writeLong(value);
addedValues.add(value);
writer.writeLong(valueLong);
addedValues.add(valueLong);
}
break;
}
case 96: {
for (int i = 0; i < valueCount; i++) {
long millisValue = Long.valueOf(random.nextInt(1572281176) * 1000);
long millisValue = positiveUpperBoundedInt * 1000L;
NanoTime nanoTime = NanoTimeUtils.getNanoTime(new Timestamp(millisValue), false);
writer.writeLong(nanoTime.getTimeOfDayNanos());
writer.writeInteger(nanoTime.getJulianDay());
Expand All @@ -146,12 +141,10 @@ public static byte[] generatePlainValuesPage(int valueCount, int valueSizeBits,
}
case 128:
for (int i = 0; i < valueCount; i++) {
long value = random.nextLong();
writer.writeLong(value);
addedValues.add(value);
value = random.nextLong();
writer.writeLong(value);
addedValues.add(value);
writer.writeLong(valueLong);
addedValues.add(valueLong);
writer.writeLong(valueLong);
addedValues.add(valueLong);
}
break;
default:
Expand All @@ -166,19 +159,19 @@ public static byte[] generatePlainValuesPage(int valueCount, int valueSizeBits,
}
}

public static byte[] generateDictionaryIdPage2048(int maxValue, Random random, List<Integer> addedValues)
public static byte[] generateDictionaryIdPage2048(int maxValue, List<Integer> addedValues, int fillerValue)
{
RunLengthBitPackingHybridEncoder encoder = getDictionaryDataPageEncoder(maxValue);

addDLRLEBlock(maxValue / 2, 50, encoder, addedValues);
addDLValues(randomValues(random, 457, maxValue), encoder, addedValues);
addDLValues(fillValues(457, fillerValue), encoder, addedValues);
addDLRLEBlock(0, 37, encoder, addedValues);
addDLValues(randomValues(random, 186, maxValue), encoder, addedValues);
addDLValues(randomValues(random, 289, maxValue), encoder, addedValues);
addDLValues(fillValues(186, fillerValue), encoder, addedValues);
addDLValues(fillValues(289, fillerValue), encoder, addedValues);
addDLRLEBlock(maxValue - 1, 76, encoder, addedValues);
addDLValues(randomValues(random, 789, maxValue), encoder, addedValues);
addDLValues(fillValues(789, fillerValue), encoder, addedValues);
addDLRLEBlock(maxValue - 1, 137, encoder, addedValues);
addDLValues(randomValues(random, 27, maxValue), encoder, addedValues);
addDLValues(fillValues(27, fillerValue), encoder, addedValues);

checkState(addedValues.size() == 2048);

Expand Down
Loading

0 comments on commit 5c9fdb7

Please sign in to comment.