diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/AbstractOrcRecordReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/AbstractOrcRecordReader.java index bfcae98d8f7ef..db35cbd036a1a 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/AbstractOrcRecordReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/AbstractOrcRecordReader.java @@ -315,10 +315,10 @@ private static void includeOrcColumnsRecursive(List types, Set List subfields = ImmutableList.of(); if (requiredFields.isPresent()) { String fieldName = type.getFieldNames().get(i).toLowerCase(Locale.ENGLISH); - if (!requiredFields.get().containsKey(fieldName)) { + if (!requiredFields.orElseThrow().containsKey(fieldName)) { continue; } - subfields = requiredFields.get().get(fieldName); + subfields = requiredFields.orElseThrow().get(fieldName); } includeOrcColumnsRecursive(types, result, type.getFieldTypeIndex(i), subfields); @@ -436,7 +436,7 @@ private static boolean isStripeIncluded( if (!stripeStats.isPresent()) { return true; } - return predicate.matches(stripe.getNumberOfRows(), getStatisticsByColumnOrdinal(rootStructType, stripeStats.get().getColumnStatistics())); + return predicate.matches(stripe.getNumberOfRows(), getStatisticsByColumnOrdinal(rootStructType, stripeStats.orElseThrow().getColumnStatistics())); } @VisibleForTesting @@ -511,7 +511,7 @@ public void close() } rowGroups = null; if (writeChecksumBuilder.isPresent()) { - OrcWriteValidation.WriteChecksum actualChecksum = writeChecksumBuilder.get().build(); + OrcWriteValidation.WriteChecksum actualChecksum = writeChecksumBuilder.orElseThrow().build(); validateWrite(validation -> validation.getChecksum().getTotalRowCount() == actualChecksum.getTotalRowCount(), "Invalid row count"); List columnHashes = actualChecksum.getColumnHashes(); for (int i = 0; i < columnHashes.size(); i++) { @@ -522,8 +522,8 @@ public void close() validateWrite(validation -> validation.getChecksum().getStripeHash() == actualChecksum.getStripeHash(), "Invalid stripes checksum"); } if (fileStatisticsValidation.isPresent()) { - List columnStatistics = fileStatisticsValidation.get().build(); - writeValidation.get().validateFileStatistics(orcDataSource.getId(), columnStatistics); + List columnStatistics = fileStatisticsValidation.orElseThrow().build(); + writeValidation.orElseThrow().validateFileStatistics(orcDataSource.getId(), columnStatistics); } } @@ -544,9 +544,9 @@ private boolean advanceToNextRowGroup() if (currentRowGroup >= 0) { if (rowGroupStatisticsValidation.isPresent()) { - OrcWriteValidation.StatisticsValidation statisticsValidation = rowGroupStatisticsValidation.get(); + OrcWriteValidation.StatisticsValidation statisticsValidation = rowGroupStatisticsValidation.orElseThrow(); long offset = stripes.get(currentStripe).getOffset(); - writeValidation.get().validateRowGroupStatistics(orcDataSource.getId(), offset, currentRowGroup, statisticsValidation.build()); + writeValidation.orElseThrow().validateRowGroupStatistics(orcDataSource.getId(), offset, currentRowGroup, statisticsValidation.build()); statisticsValidation.reset(); } } @@ -637,9 +637,9 @@ private void advanceToNextStripe() if (currentStripe >= 0) { if (stripeStatisticsValidation.isPresent()) { - OrcWriteValidation.StatisticsValidation statisticsValidation = stripeStatisticsValidation.get(); + OrcWriteValidation.StatisticsValidation statisticsValidation = stripeStatisticsValidation.orElseThrow(); long offset = stripes.get(currentStripe).getOffset(); - writeValidation.get().validateStripeStatistics(orcDataSource.getId(), offset, statisticsValidation.build()); + writeValidation.orElseThrow().validateStripeStatistics(orcDataSource.getId(), offset, statisticsValidation.build()); statisticsValidation.reset(); } } @@ -661,9 +661,9 @@ private void advanceToNextStripe() // or it has been set, but we have new decryption keys, // set dwrfEncryptionInfo if ((!stripeDecryptionKeyMetadata.isEmpty() && !dwrfEncryptionInfo.isPresent()) - || (dwrfEncryptionInfo.isPresent() && !stripeDecryptionKeyMetadata.equals(dwrfEncryptionInfo.get().getEncryptedKeyMetadatas()))) { + || (dwrfEncryptionInfo.isPresent() && !stripeDecryptionKeyMetadata.equals(dwrfEncryptionInfo.orElseThrow().getEncryptedKeyMetadatas()))) { verify(encryptionLibrary.isPresent(), "encryptionLibrary is absent"); - dwrfEncryptionInfo = Optional.of(createDwrfEncryptionInfo(encryptionLibrary.get(), stripeDecryptionKeyMetadata, intermediateKeyMetadata, dwrfEncryptionGroupMap)); + dwrfEncryptionInfo = Optional.of(createDwrfEncryptionInfo(encryptionLibrary.orElseThrow(), stripeDecryptionKeyMetadata, intermediateKeyMetadata, dwrfEncryptionGroupMap)); } SharedBuffer sharedDecompressionBuffer = new SharedBuffer(currentStripeSystemMemoryContext.newOrcLocalMemoryContext("sharedDecompressionBuffer")); @@ -698,7 +698,7 @@ public static List getDecryptionKeyMetadata(int currentStripe, List test, String messageFormat, Object... args) throws OrcCorruptionException { - if (writeValidation.isPresent() && !test.apply(writeValidation.get())) { + if (writeValidation.isPresent() && !test.apply(writeValidation.orElseThrow())) { throw new OrcCorruptionException(orcDataSource.getId(), "Write validation failed: " + messageFormat, args); } } @@ -706,7 +706,7 @@ private void validateWrite(Predicate test, String messageFor private void validateWriteStripe(long rowCount) { if (writeChecksumBuilder.isPresent()) { - writeChecksumBuilder.get().addStripe(rowCount); + writeChecksumBuilder.orElseThrow().addStripe(rowCount); } } @@ -775,10 +775,10 @@ protected boolean shouldValidateWritePageChecksum() protected void validateWritePageChecksum(Page page) { if (writeChecksumBuilder.isPresent()) { - writeChecksumBuilder.get().addPage(page); - rowGroupStatisticsValidation.get().addPage(page); - stripeStatisticsValidation.get().addPage(page); - fileStatisticsValidation.get().addPage(page); + writeChecksumBuilder.orElseThrow().addPage(page); + rowGroupStatisticsValidation.orElseThrow().addPage(page); + stripeStatisticsValidation.orElseThrow().addPage(page); + fileStatisticsValidation.orElseThrow().addPage(page); } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/CachingStripeMetadataSource.java b/presto-orc/src/main/java/com/facebook/presto/orc/CachingStripeMetadataSource.java index 1aede72f6c803..ca6c08e81936f 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/CachingStripeMetadataSource.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/CachingStripeMetadataSource.java @@ -130,7 +130,7 @@ public List getRowIndexes( throws IOException { if (rowGroupIndexCache.isPresent()) { - List rowGroupIndices = rowGroupIndexCache.get().getIfPresent(new StripeStreamId(stripId, streamId)); + List rowGroupIndices = rowGroupIndexCache.orElseThrow().getIfPresent(new StripeStreamId(stripId, streamId)); if (rowGroupIndices != null) { runtimeStats.addMetricValue("OrcRowGroupIndexCacheHit", NONE, 1); runtimeStats.addMetricValue("OrcRowGroupIndexInMemoryBytesRead", BYTE, rowGroupIndices.stream().mapToLong(RowGroupIndex::getRetainedSizeInBytes).sum()); @@ -143,7 +143,7 @@ public List getRowIndexes( } List rowGroupIndices = delegate.getRowIndexes(metadataReader, hiveWriterVersion, stripId, streamId, inputStream, bloomFilters, runtimeStats); if (rowGroupIndexCache.isPresent()) { - rowGroupIndexCache.get().put(new StripeStreamId(stripId, streamId), rowGroupIndices); + rowGroupIndexCache.orElseThrow().put(new StripeStreamId(stripId, streamId), rowGroupIndices); } return rowGroupIndices; } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/DwrfAwareStripeMetadataSource.java b/presto-orc/src/main/java/com/facebook/presto/orc/DwrfAwareStripeMetadataSource.java index c2c0ad56f9dba..c1c4ff47d07ef 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/DwrfAwareStripeMetadataSource.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/DwrfAwareStripeMetadataSource.java @@ -57,7 +57,7 @@ public Slice getStripeFooterSlice(OrcDataSource orcDataSource, StripeId stripeId { Optional stripeFooterSlice = stripeCache.getStripeFooterSlice(stripeId, footerLength); if (stripeFooterSlice.isPresent()) { - return stripeFooterSlice.get(); + return stripeFooterSlice.orElseThrow(); } return delegate.getStripeFooterSlice(orcDataSource, stripeId, footerOffset, footerLength, cacheable); } @@ -71,7 +71,7 @@ public Map getInputs(OrcDataSource orcDataSource, return delegate.getInputs(orcDataSource, stripeId, diskRanges, cacheable); } - Slice cacheSlice = stripeCacheIndexStreamsSlice.get(); + Slice cacheSlice = stripeCacheIndexStreamsSlice.orElseThrow(); ImmutableMap.Builder inputsBuilder = ImmutableMap.builder(); ImmutableMap.Builder dataStreamsBuilder = ImmutableMap.builder(); diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/DwrfAwareStripeMetadataSourceFactory.java b/presto-orc/src/main/java/com/facebook/presto/orc/DwrfAwareStripeMetadataSourceFactory.java index 1bf473e56b6ab..1e87914e792c8 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/DwrfAwareStripeMetadataSourceFactory.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/DwrfAwareStripeMetadataSourceFactory.java @@ -34,7 +34,7 @@ public StripeMetadataSource create(Optional dwrfStripeCache) { StripeMetadataSource delegate = requireNonNull(delegateFactory.create(dwrfStripeCache), "created delegate is null"); if (dwrfStripeCache.isPresent()) { - return new DwrfAwareStripeMetadataSource(delegate, dwrfStripeCache.get()); + return new DwrfAwareStripeMetadataSource(delegate, dwrfStripeCache.orElseThrow()); } return delegate; } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcOutputBuffer.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcOutputBuffer.java index 60b5dd97f955c..cf1e3866205d8 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/OrcOutputBuffer.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcOutputBuffer.java @@ -504,7 +504,7 @@ private void writeChunkToOutputStream(byte[] chunk, int offset, int length) } } if (dwrfEncryptor.isPresent()) { - chunk = dwrfEncryptor.get().encrypt(chunk, offset, length); + chunk = dwrfEncryptor.orElseThrow().encrypt(chunk, offset, length); length = chunk.length; offset = 0; // size after encryption should not exceed what the 3 byte header can hold (2^23) diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcReader.java index b3227dff84e9d..86d8b30a80eb6 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/OrcReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcReader.java @@ -201,11 +201,11 @@ public OrcReader( requireNonNull(dwrfKeyProvider, "dwrfKeyProvider is null"); validateEncryption(footer, this.orcDataSource.getId()); this.dwrfEncryptionGroupMap = createNodeToGroupMap( - encryption.get().getEncryptionGroups().stream() + encryption.orElseThrow().getEncryptionGroups().stream() .map(EncryptionGroup::getNodes) .collect(toImmutableList()), footer.getTypes()); - this.encryptionLibrary = Optional.of(dwrfEncryptionProvider.getEncryptionLibrary(encryption.get().getKeyProvider())); + this.encryptionLibrary = Optional.of(dwrfEncryptionProvider.getEncryptionLibrary(encryption.orElseThrow().getKeyProvider())); this.columnsToIntermediateKeys = ImmutableMap.copyOf(dwrfKeyProvider.getIntermediateKeys(footer.getTypes())); } else { @@ -228,17 +228,17 @@ public OrcReader( validateWrite(writeValidation, orcDataSource, validation -> validation.getColumnNames().equals(footer.getTypes().get(0).getFieldNames()), "Unexpected column names"); validateWrite(writeValidation, orcDataSource, validation -> validation.getRowGroupMaxRowCount() == footer.getRowsInRowGroup(), "Unexpected rows in group"); if (writeValidation.isPresent()) { - writeValidation.get().validateMetadata(orcDataSource.getId(), footer.getUserMetadata()); - writeValidation.get().validateFileStatistics(orcDataSource.getId(), footer.getFileStats()); - writeValidation.get().validateStripeStatistics(orcDataSource.getId(), footer.getStripes(), metadata.getStripeStatsList()); + writeValidation.orElseThrow().validateMetadata(orcDataSource.getId(), footer.getUserMetadata()); + writeValidation.orElseThrow().validateFileStatistics(orcDataSource.getId(), footer.getFileStats()); + writeValidation.orElseThrow().validateStripeStatistics(orcDataSource.getId(), footer.getStripes(), metadata.getStripeStatsList()); } this.cacheable = requireNonNull(cacheable, "cacheable is null"); Optional dwrfStripeCache = Optional.empty(); if (orcFileTail.getDwrfStripeCacheData().isPresent() && footer.getDwrfStripeCacheOffsets().isPresent()) { - DwrfStripeCacheData dwrfStripeCacheData = orcFileTail.getDwrfStripeCacheData().get(); - DwrfStripeCache cache = dwrfStripeCacheData.buildDwrfStripeCache(footer.getStripes(), footer.getDwrfStripeCacheOffsets().get()); + DwrfStripeCacheData dwrfStripeCacheData = orcFileTail.getDwrfStripeCacheData().orElseThrow(); + DwrfStripeCache cache = dwrfStripeCacheData.buildDwrfStripeCache(footer.getStripes(), footer.getDwrfStripeCacheOffsets().orElseThrow()); dwrfStripeCache = Optional.of(cache); } @@ -252,7 +252,7 @@ public static void validateEncryption(Footer footer, OrcDataSourceId dataSourceI if (!footer.getEncryption().isPresent()) { return; } - DwrfEncryption dwrfEncryption = footer.getEncryption().get(); + DwrfEncryption dwrfEncryption = footer.getEncryption().orElseThrow(); int encryptionGroupSize = dwrfEncryption.getEncryptionGroups().size(); List stripes = footer.getStripes(); if (!stripes.isEmpty() && encryptionGroupSize > 0 && stripes.get(0).getKeyMetadata().isEmpty()) { @@ -483,7 +483,7 @@ static void validateFile( public static void validateWrite(Optional writeValidation, OrcDataSource orcDataSource, Predicate test, String messageFormat, Object... args) throws OrcCorruptionException { - if (writeValidation.isPresent() && !test.test(writeValidation.get())) { + if (writeValidation.isPresent() && !test.test(writeValidation.orElseThrow())) { throw new OrcCorruptionException(orcDataSource.getId(), "Write validation failed: " + messageFormat, args); } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcSelectiveRecordReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcSelectiveRecordReader.java index 9defe56b2da04..8860d23764dc9 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/OrcSelectiveRecordReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcSelectiveRecordReader.java @@ -40,7 +40,6 @@ import com.facebook.presto.orc.reader.SelectiveStreamReader; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; import io.airlift.slice.Slice; @@ -84,6 +83,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.common.collect.MoreCollectors.onlyElement; import static io.airlift.slice.SizeOf.sizeOf; import static java.lang.Math.max; import static java.lang.Math.min; @@ -470,7 +470,7 @@ private static Optional getFilterFunctionWithoutInputs(List columnFilters) @@ -487,7 +487,7 @@ private static int scoreFilter(Map filters) return 1000; } - Map.Entry filterEntry = Iterables.getOnlyElement(filters.entrySet()); + Map.Entry filterEntry = filters.entrySet().stream().collect(onlyElement()); if (!filterEntry.getKey().getPath().isEmpty()) { // Complex type column. Complex types are expensive! return 1000; @@ -785,7 +785,7 @@ private int applyFilterFunctionWithNoInputs(int positionCount) { initializeOutputPositions(positionCount); Page page = new Page(positionCount); - return filterFunctionWithoutInput.get().filter(page, outputPositions, positionCount, errors); + return filterFunctionWithoutInput.orElseThrow().filter(page, outputPositions, positionCount, errors); } private int applyFilterFunctions(List filterFunctions, Set filterFunctionInputs, int[] positions, int positionCount) diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriteValidation.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriteValidation.java index d11edb867132f..ef5e8ccbacdda 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriteValidation.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriteValidation.java @@ -48,7 +48,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; -import com.google.common.collect.Iterables; import io.airlift.slice.Slice; import io.airlift.slice.Slices; import io.airlift.slice.XxHash64; @@ -94,6 +93,7 @@ import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.MoreCollectors.onlyElement; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -846,7 +846,7 @@ else if (type instanceof DecimalType) { else if (type.getTypeSignature().getBase().equals(ARRAY)) { statisticsBuilder = new CountStatisticsBuilder(); fieldExtractor = block -> ImmutableList.of(toColumnarArray(block).getElementsBlock()); - fieldBuilders = ImmutableList.of(new ColumnStatisticsValidation(Iterables.getOnlyElement(type.getTypeParameters()))); + fieldBuilders = ImmutableList.of(new ColumnStatisticsValidation(type.getTypeParameters().stream().collect(onlyElement()))); } else if (type.getTypeSignature().getBase().equals(MAP)) { statisticsBuilder = new CountStatisticsBuilder(); diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriter.java index 869ecf23aede6..c8f790ee930e7 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriter.java @@ -47,7 +47,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; import io.airlift.slice.DynamicSliceOutput; import io.airlift.slice.Slice; @@ -261,14 +260,14 @@ public OrcWriter( dwrfWriterEncryption = requireNonNull(encryption, "encryption is null"); this.dwrfEncryptionProvider = requireNonNull(dwrfEncryptionProvider, "dwrfEncryptionProvider is null"); if (dwrfWriterEncryption.isPresent()) { - List writerEncryptionGroups = dwrfWriterEncryption.get().getWriterEncryptionGroups(); + List writerEncryptionGroups = dwrfWriterEncryption.orElseThrow().getWriterEncryptionGroups(); Map nodeToGroupMap = createNodeToGroupMap( writerEncryptionGroups .stream() .map(WriterEncryptionGroup::getNodes) .collect(toImmutableList()), orcTypes); - EncryptionLibrary encryptionLibrary = dwrfEncryptionProvider.getEncryptionLibrary(dwrfWriterEncryption.get().getKeyProvider()); + EncryptionLibrary encryptionLibrary = dwrfEncryptionProvider.getEncryptionLibrary(dwrfWriterEncryption.orElseThrow().getKeyProvider()); List dataEncryptionKeys = writerEncryptionGroups.stream() .map(group -> encryptionLibrary.generateDataEncryptionKey(group.getIntermediateKeyMetadata().getBytes())) .collect(toImmutableList()); @@ -466,7 +465,7 @@ private void writeChunk(Page chunk) boolean dictionaryIsFull = dictionaryCompressionOptimizer.isFull(bufferedBytes); Optional flushReason = flushPolicy.shouldFlushStripe(stripeRowCount, bufferedBytes, dictionaryIsFull); if (flushReason.isPresent()) { - flushStripe(flushReason.get()); + flushStripe(flushReason.orElseThrow()); } columnWritersRetainedBytes = columnWriters.stream().mapToLong(ColumnWriter::getRetainedBytes).sum(); } @@ -558,9 +557,9 @@ private List bufferStripeData(long stripeStartOffset, FlushReason fl // if the previous stream was part of a different encryption group, need to specify an offset so we know the column order Optional encryptionGroup = dwrfEncryptionInfo.getGroupByNodeId(indexStream.getStream().getColumn()); if (encryptionGroup.isPresent()) { - Stream stream = previousEncryptionGroup == encryptionGroup.get() ? indexStream.getStream() : indexStream.getStream().withOffset(offset); - encryptedStreams.put(encryptionGroup.get(), stream); - previousEncryptionGroup = encryptionGroup.get(); + Stream stream = previousEncryptionGroup == encryptionGroup.orElseThrow() ? indexStream.getStream() : indexStream.getStream().withOffset(offset); + encryptedStreams.put(encryptionGroup.orElseThrow(), stream); + previousEncryptionGroup = encryptionGroup.orElseThrow(); } else { Stream stream = previousEncryptionGroup == -1 ? indexStream.getStream() : indexStream.getStream().withOffset(offset); @@ -573,7 +572,7 @@ private List bufferStripeData(long stripeStartOffset, FlushReason fl } if (dwrfStripeCacheWriter.isPresent()) { - dwrfStripeCacheWriter.get().addIndexStreams(ImmutableList.copyOf(indexStreams), indexLength); + dwrfStripeCacheWriter.orElseThrow().addIndexStreams(ImmutableList.copyOf(indexStreams), indexLength); } // data streams (sorted by size) @@ -594,7 +593,7 @@ private List bufferStripeData(long stripeStartOffset, FlushReason fl // reorder data streams streamLayout.reorder(dataStreams, nodeIdToColumn, columnEncodings); - streamSizeHelper.collectStreamSizes(Iterables.concat(indexStreams, dataStreams), columnEncodings); + streamSizeHelper.collectStreamSizes(java.util.stream.Stream.concat(indexStreams.stream(), dataStreams.stream()).toList(), columnEncodings); // add data streams for (StreamDataOutput dataStream : dataStreams) { @@ -602,9 +601,9 @@ private List bufferStripeData(long stripeStartOffset, FlushReason fl // if the previous stream was part of a different encryption group, need to specify an offset so we know the column order Optional encryptionGroup = dwrfEncryptionInfo.getGroupByNodeId(dataStream.getStream().getColumn()); if (encryptionGroup.isPresent()) { - Stream stream = previousEncryptionGroup == encryptionGroup.get() ? dataStream.getStream() : dataStream.getStream().withOffset(offset); - encryptedStreams.put(encryptionGroup.get(), stream); - previousEncryptionGroup = encryptionGroup.get(); + Stream stream = previousEncryptionGroup == encryptionGroup.orElseThrow() ? dataStream.getStream() : dataStream.getStream().withOffset(offset); + encryptedStreams.put(encryptionGroup.orElseThrow(), stream); + previousEncryptionGroup = encryptionGroup.orElseThrow(); } else { Stream stream = previousEncryptionGroup == -1 ? dataStream.getStream() : dataStream.getStream().withOffset(offset); @@ -737,7 +736,7 @@ private List bufferFileFooter() Optional dwrfEncryption; if (dwrfWriterEncryption.isPresent()) { ImmutableList.Builder encryptionGroupBuilder = ImmutableList.builder(); - List writerEncryptionGroups = dwrfWriterEncryption.get().getWriterEncryptionGroups(); + List writerEncryptionGroups = dwrfWriterEncryption.orElseThrow().getWriterEncryptionGroups(); for (int i = 0; i < writerEncryptionGroups.size(); i++) { WriterEncryptionGroup group = writerEncryptionGroups.get(i); Map groupStats = encryptedStats.get(i); @@ -751,7 +750,7 @@ private List bufferFileFooter() } dwrfEncryption = Optional.of( new DwrfEncryption( - dwrfWriterEncryption.get().getKeyProvider(), + dwrfWriterEncryption.orElseThrow().getKeyProvider(), encryptionGroupBuilder.build())); } else { @@ -802,8 +801,8 @@ private void addStatsRecursive(List allStats, int index, Map new ArrayList<>()).add(columnStatistics); @@ -854,7 +853,7 @@ public void validate(OrcDataSource input) checkState(validationBuilder != null, "validation is not enabled"); ImmutableMap.Builder intermediateKeyMetadata = ImmutableMap.builder(); if (dwrfWriterEncryption.isPresent()) { - List writerEncryptionGroups = dwrfWriterEncryption.get().getWriterEncryptionGroups(); + List writerEncryptionGroups = dwrfWriterEncryption.orElseThrow().getWriterEncryptionGroups(); for (int i = 0; i < writerEncryptionGroups.size(); i++) { for (Integer node : writerEncryptionGroups.get(i).getNodes()) { intermediateKeyMetadata.put(node, writerEncryptionGroups.get(i).getIntermediateKeyMetadata()); diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/StreamSizeHelper.java b/presto-orc/src/main/java/com/facebook/presto/orc/StreamSizeHelper.java index f0a73b15f556d..323c1a4119c54 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/StreamSizeHelper.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/StreamSizeHelper.java @@ -123,7 +123,7 @@ public void collectStreamSizes(Iterable streamDataOutputs, Map checkArgument(columnEncoding != null, "columnEncoding for flat map node %s is null", flatMapNode); checkArgument(columnEncoding.getAdditionalSequenceEncodings().isPresent(), "columnEncoding for flat map node %s does not have keys", flatMapNode); - SortedMap sequenceToKey = columnEncoding.getAdditionalSequenceEncodings().get(); + SortedMap sequenceToKey = columnEncoding.getAdditionalSequenceEncodings().orElseThrow(); Int2LongMap sequenceToSize = flatMapNodeSizes.get(flatMapNode); Object2LongMap keyToSize = keySizes.computeIfAbsent(flatMapNode, (ignore) -> new Object2LongOpenHashMap<>()); diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/StripeReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/StripeReader.java index 73ef45aca8498..362cac4ca6663 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/StripeReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/StripeReader.java @@ -78,7 +78,7 @@ import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.Iterables.getOnlyElement; +import static com.google.common.collect.MoreCollectors.onlyElement; import static java.lang.Math.multiplyExact; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; @@ -172,8 +172,8 @@ public Stripe readStripe( // included columns may be encrypted if (decryptors.isPresent()) { List encryptedEncryptionGroups = stripeFooter.getStripeEncryptionGroups(); - for (Integer groupId : decryptors.get().getEncryptorGroupIds()) { - StripeEncryptionGroup stripeEncryptionGroup = getStripeEncryptionGroup(decryptors.get().getEncryptorByGroupId(groupId), encryptedEncryptionGroups.get(groupId), dwrfEncryptionGroupColumns.get(groupId), systemMemoryUsage); + for (Integer groupId : decryptors.orElseThrow().getEncryptorGroupIds()) { + StripeEncryptionGroup stripeEncryptionGroup = getStripeEncryptionGroup(decryptors.orElseThrow().getEncryptorByGroupId(groupId), encryptedEncryptionGroups.get(groupId), dwrfEncryptionGroupColumns.get(groupId), systemMemoryUsage); allStreams.add(stripeEncryptionGroup.getStreams()); columnEncodings.putAll(stripeEncryptionGroup.getColumnEncodings()); boolean encryptedHasRowGroupDictionary = addIncludedStreams(stripeEncryptionGroup.getColumnEncodings(), stripeEncryptionGroup.getStreams(), includedStreams); @@ -195,7 +195,7 @@ public Stripe readStripe( Map> columnIndexes = readColumnIndexes(includedStreams, streamsData, stripeId); fileIntrospector.ifPresent(introspector -> introspector.onRowGroupIndexes(stripe, columnIndexes)); if (writeValidation.isPresent()) { - writeValidation.get().validateRowGroupStatistics(orcDataSource.getId(), stripe.getOffset(), columnIndexes); + writeValidation.orElseThrow().validateRowGroupStatistics(orcDataSource.getId(), stripe.getOffset(), columnIndexes); } // select the row groups matching the tuple domain @@ -322,7 +322,7 @@ private boolean addIncludedStreams(Map columnEncodings, Optional> additionalSequenceEncodings = columnEncoding.getAdditionalSequenceEncodings(); if (additionalSequenceEncodings.isPresent() - && additionalSequenceEncodings.get().values().stream() + && additionalSequenceEncodings.orElseThrow().values().stream() .map(DwrfSequenceEncoding::getValueEncoding) .anyMatch(encoding -> encoding.getColumnEncodingKind() == DICTIONARY)) { hasRowGroupDictionary = true; @@ -370,7 +370,7 @@ private Optional createDwrfDecryptor(StreamId id, Optional> createValueStreams(Map streams, Map streamsData, Map columnEncodings) @@ -576,7 +576,7 @@ private static Map getRowGroupStatistics(OrcType root List columnStatistics = groupedColumnStatistics.get(rootStructType.getFieldTypeIndex(ordinal)); if (columnStatistics != null) { if (columnStatistics.size() == 1) { - statistics.put(ordinal, getOnlyElement(columnStatistics)); + statistics.put(ordinal, columnStatistics.stream().collect(onlyElement())); } else { // Merge statistics from different streams @@ -602,7 +602,7 @@ public static Map getDiskRanges(List> streams) for (Stream stream : groupStreams) { int streamLength = toIntExact(stream.getLength()); if (stream.getOffset().isPresent()) { - stripeOffset = stream.getOffset().get(); + stripeOffset = stream.getOffset().orElseThrow(); } // ignore zero byte streams if (streamLength > 0) { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainOrcPredicate.java b/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainOrcPredicate.java index f605dabd507c5..c1fe020f59b90 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainOrcPredicate.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainOrcPredicate.java @@ -80,7 +80,7 @@ public boolean matches(long numberOfRows, Map statist // effective predicate is none, so skip this section return false; } - Map effectivePredicateDomains = optionalEffectivePredicateDomains.get(); + Map effectivePredicateDomains = optionalEffectivePredicateDomains.orElseThrow(); for (ColumnReference columnReference : columnReferences) { Domain predicateDomain = effectivePredicateDomains.get(columnReference.getColumn()); @@ -135,7 +135,7 @@ private boolean columnOverlaps(ColumnReference columnReference, Domain predic } // if none of the discrete predicate values are found in the bloom filter, there is no overlap and the section should be skipped - if (discreteValues.get().stream().noneMatch(value -> checkInBloomFilter(bloomFilter, value, stripeDomain.getType()))) { + if (discreteValues.orElseThrow().stream().noneMatch(value -> checkInBloomFilter(bloomFilter, value, stripeDomain.getType()))) { return false; } return true; diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/cache/StorageOrcFileTailSource.java b/presto-orc/src/main/java/com/facebook/presto/orc/cache/StorageOrcFileTailSource.java index 3406c1320a916..05c7912a2b213 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/cache/StorageOrcFileTailSource.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/cache/StorageOrcFileTailSource.java @@ -121,7 +121,7 @@ public OrcFileTail getOrcFileTail(OrcDataSource orcDataSource, MetadataReader me boolean readDwrfStripeCache = dwrfStripeCacheEnabled && postScript.getDwrfStripeCacheLength().isPresent() && postScript.getDwrfStripeCacheMode().isPresent() - && postScript.getDwrfStripeCacheMode().get() != DwrfStripeCacheMode.NONE; + && postScript.getDwrfStripeCacheMode().orElseThrow() != DwrfStripeCacheMode.NONE; int dwrfStripeCacheSize = 0; if (readDwrfStripeCache) { dwrfStripeCacheSize = postScript.getDwrfStripeCacheLength().getAsInt(); @@ -158,7 +158,7 @@ public OrcFileTail getOrcFileTail(OrcDataSource orcDataSource, MetadataReader me Optional dwrfStripeCacheData = Optional.empty(); if (readDwrfStripeCache) { Slice dwrfStripeCacheSlice = completeFooterSlice.slice(0, dwrfStripeCacheSize); - DwrfStripeCacheMode stripeCacheMode = postScript.getDwrfStripeCacheMode().get(); + DwrfStripeCacheMode stripeCacheMode = postScript.getDwrfStripeCacheMode().orElseThrow(); dwrfStripeCacheData = Optional.of(new DwrfStripeCacheData(dwrfStripeCacheSlice, dwrfStripeCacheSize, stripeCacheMode)); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/ColumnEncoding.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/ColumnEncoding.java index 5d7fe302f064d..d27965211b644 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/ColumnEncoding.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/ColumnEncoding.java @@ -83,13 +83,13 @@ public ColumnEncoding getColumnEncoding(int sequence) additionalSequenceEncodings.isPresent(), "Got non-zero sequence: %s, but there are no additional sequence encodings: %s", sequence, this); - DwrfSequenceEncoding sequenceEncoding = additionalSequenceEncodings.get().get(sequence); + DwrfSequenceEncoding sequenceEncoding = additionalSequenceEncodings.orElseThrow().get(sequence); checkState( sequenceEncoding != null, "Non-zero sequence %s is not present in the ColumnEncoding's additional sequences: %s", sequence, - additionalSequenceEncodings.get().keySet()); + additionalSequenceEncodings.orElseThrow().keySet()); return sequenceEncoding.getValueEncoding(); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/DwrfMetadataReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/DwrfMetadataReader.java index bf07ef9cc4fff..21707f4e9a07c 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/DwrfMetadataReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/DwrfMetadataReader.java @@ -152,8 +152,8 @@ public Footer readFooter(HiveWriterVersion hiveWriterVersion, if (encryption.isPresent()) { Map keys = dwrfKeyProvider.getIntermediateKeys(types); - EncryptionLibrary encryptionLibrary = dwrfEncryptionProvider.getEncryptionLibrary(encryption.get().getKeyProvider()); - fileStats = decryptAndCombineFileStatistics(hiveWriterVersion, encryption.get(), encryptionLibrary, fileStats, fileStripes, keys, orcDataSource, decompressor); + EncryptionLibrary encryptionLibrary = dwrfEncryptionProvider.getEncryptionLibrary(encryption.orElseThrow().getKeyProvider()); + fileStats = decryptAndCombineFileStatistics(hiveWriterVersion, encryption.orElseThrow(), encryptionLibrary, fileStats, fileStripes, keys, orcDataSource, decompressor); } runtimeStats.addMetricValue("DwrfReadFooterTimeNanos", RuntimeUnit.NANO, THREAD_MX_BEAN.getCurrentThreadCpuTime() - cpuStart); @@ -214,7 +214,7 @@ private List decryptAndCombineFileStatistics(HiveWriterVersion // The key in the footer takes priority over the key in the first stripe. byte[] encryptedDataKeyWithMeta = null; if (encryptionGroup.getKeyMetadata().isPresent()) { - encryptedDataKeyWithMeta = encryptionGroup.getKeyMetadata().get().byteArray(); + encryptedDataKeyWithMeta = encryptionGroup.getKeyMetadata().orElseThrow().byteArray(); } else if (stripeKeys != null) { encryptedDataKeyWithMeta = stripeKeys.get(groupIdx); diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/DwrfMetadataWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/DwrfMetadataWriter.java index af925dc44ff3b..1c5451e0dac05 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/DwrfMetadataWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/DwrfMetadataWriter.java @@ -94,7 +94,7 @@ public int writeDwrfStripeCache(SliceOutput output, Optional toColumnEncodings(Map sequences = columnEncoding.getAdditionalSequenceEncodings().get(); + Map sequences = columnEncoding.getAdditionalSequenceEncodings().orElseThrow(); for (Entry sequenceEntry : sequences.entrySet()) { int sequence = sequenceEntry.getKey(); DwrfSequenceEncoding sequenceEncoding = sequenceEntry.getValue(); diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcFileTail.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcFileTail.java index d639449df08a4..297884e500959 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcFileTail.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcFileTail.java @@ -95,7 +95,7 @@ private int getDwrfStripeCacheSize() { int dwrfStripeCacheSize = 0; if (dwrfStripeCacheData.isPresent()) { - dwrfStripeCacheSize = dwrfStripeCacheData.get().getDwrfStripeCacheSize(); + dwrfStripeCacheSize = dwrfStripeCacheData.orElseThrow().getDwrfStripeCacheSize(); } return dwrfStripeCacheSize; } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcMetadataWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcMetadataWriter.java index 0465f0a2e8992..566eca118c0bf 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcMetadataWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcMetadataWriter.java @@ -151,13 +151,13 @@ private static Type toType(OrcType type) .addAllAttributes(toStringPairList(type.getAttributes())); if (type.getLength().isPresent()) { - builder.setMaximumLength(type.getLength().get()); + builder.setMaximumLength(type.getLength().orElseThrow()); } if (type.getPrecision().isPresent()) { - builder.setPrecision(type.getPrecision().get()); + builder.setPrecision(type.getPrecision().orElseThrow()); } if (type.getScale().isPresent()) { - builder.setScale(type.getScale().get()); + builder.setScale(type.getScale().orElseThrow()); } return builder.build(); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/BinaryStatisticsBuilder.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/BinaryStatisticsBuilder.java index 67a692b9afd34..32490e00137de 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/BinaryStatisticsBuilder.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/BinaryStatisticsBuilder.java @@ -59,7 +59,7 @@ public ColumnStatistics buildColumnStatistics() Optional binaryStatistics = buildBinaryStatistics(); if (binaryStatistics.isPresent()) { verify(nonNullValueCount > 0); - return new BinaryColumnStatistics(nonNullValueCount, null, rawSize, storageSize, binaryStatistics.get()); + return new BinaryColumnStatistics(nonNullValueCount, null, rawSize, storageSize, binaryStatistics.orElseThrow()); } return new ColumnStatistics(nonNullValueCount, null, rawSize, storageSize); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/BooleanStatisticsBuilder.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/BooleanStatisticsBuilder.java index 92150ab764d28..f426212b52b44 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/BooleanStatisticsBuilder.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/BooleanStatisticsBuilder.java @@ -68,7 +68,7 @@ public ColumnStatistics buildColumnStatistics() { Optional booleanStatistics = buildBooleanStatistics(); if (booleanStatistics.isPresent()) { - return new BooleanColumnStatistics(nonNullValueCount, null, rawSize, storageSize, booleanStatistics.get()); + return new BooleanColumnStatistics(nonNullValueCount, null, rawSize, storageSize, booleanStatistics.orElseThrow()); } return new ColumnStatistics(nonNullValueCount, null, rawSize, storageSize); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/DateStatisticsBuilder.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/DateStatisticsBuilder.java index b43c2a86e8169..376acc5bdc8f5 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/DateStatisticsBuilder.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/DateStatisticsBuilder.java @@ -62,7 +62,7 @@ public ColumnStatistics buildColumnStatistics() { Optional dateStatistics = buildDateStatistics(); if (dateStatistics.isPresent()) { - return new DateColumnStatistics(nonNullValueCount, null, rawSize, storageSize, dateStatistics.get()); + return new DateColumnStatistics(nonNullValueCount, null, rawSize, storageSize, dateStatistics.orElseThrow()); } return new ColumnStatistics(nonNullValueCount, null, rawSize, storageSize); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/DoubleStatisticsBuilder.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/DoubleStatisticsBuilder.java index e5f79a900fc74..f5719acdbabed 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/DoubleStatisticsBuilder.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/DoubleStatisticsBuilder.java @@ -84,7 +84,7 @@ public ColumnStatistics buildColumnStatistics() { Optional doubleStatistics = buildDoubleStatistics(); if (doubleStatistics.isPresent()) { - return new DoubleColumnStatistics(nonNullValueCount, null, rawSize, storageSize, doubleStatistics.get()); + return new DoubleColumnStatistics(nonNullValueCount, null, rawSize, storageSize, doubleStatistics.orElseThrow()); } return new ColumnStatistics(nonNullValueCount, null, rawSize, storageSize); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/IntegerStatisticsBuilder.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/IntegerStatisticsBuilder.java index 0b0b9288ddbaf..c9d4f7396f568 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/IntegerStatisticsBuilder.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/IntegerStatisticsBuilder.java @@ -90,7 +90,7 @@ public ColumnStatistics buildColumnStatistics() { Optional integerStatistics = buildIntegerStatistics(); if (integerStatistics.isPresent()) { - return new IntegerColumnStatistics(nonNullValueCount, null, rawSize, storageSize, integerStatistics.get()); + return new IntegerColumnStatistics(nonNullValueCount, null, rawSize, storageSize, integerStatistics.orElseThrow()); } return new ColumnStatistics(nonNullValueCount, null, rawSize, storageSize); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/LongDecimalStatisticsBuilder.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/LongDecimalStatisticsBuilder.java index 65b7de6ec91ae..0629105140dde 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/LongDecimalStatisticsBuilder.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/LongDecimalStatisticsBuilder.java @@ -97,7 +97,7 @@ public ColumnStatistics buildColumnStatistics() { Optional decimalStatistics = buildDecimalStatistics(LONG_DECIMAL_VALUE_BYTES); if (decimalStatistics.isPresent()) { - return new DecimalColumnStatistics(nonNullValueCount, null, rawSize, storageSize, decimalStatistics.get()); + return new DecimalColumnStatistics(nonNullValueCount, null, rawSize, storageSize, decimalStatistics.orElseThrow()); } return new ColumnStatistics(nonNullValueCount, null, rawSize, storageSize); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/ShortDecimalStatisticsBuilder.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/ShortDecimalStatisticsBuilder.java index 36ea276b6229f..17f97b1e0a288 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/ShortDecimalStatisticsBuilder.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/ShortDecimalStatisticsBuilder.java @@ -60,7 +60,7 @@ public ColumnStatistics buildColumnStatistics() { Optional decimalStatistics = buildDecimalStatistics(); if (decimalStatistics.isPresent()) { - return new DecimalColumnStatistics(nonNullValueCount, null, rawSize, storageSize, decimalStatistics.get()); + return new DecimalColumnStatistics(nonNullValueCount, null, rawSize, storageSize, decimalStatistics.orElseThrow()); } return new ColumnStatistics(nonNullValueCount, null, rawSize, storageSize); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/StringStatisticsBuilder.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/StringStatisticsBuilder.java index 83007e640fbda..d1e5c1f19e8d0 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/StringStatisticsBuilder.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/statistics/StringStatisticsBuilder.java @@ -128,7 +128,7 @@ public ColumnStatistics buildColumnStatistics() Optional stringStatistics = buildStringStatistics(); if (stringStatistics.isPresent()) { verify(nonNullValueCount > 0); - return new StringColumnStatistics(nonNullValueCount, null, rawSize, storageSize, stringStatistics.get()); + return new StringColumnStatistics(nonNullValueCount, null, rawSize, storageSize, stringStatistics.orElseThrow()); } return new ColumnStatistics(nonNullValueCount, null, rawSize, storageSize); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/AbstractDecimalSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/AbstractDecimalSelectiveStreamReader.java index 1eb18cbfda5fb..6c5f291a85e21 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/AbstractDecimalSelectiveStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/AbstractDecimalSelectiveStreamReader.java @@ -93,7 +93,7 @@ public AbstractDecimalSelectiveStreamReader( this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); this.nonDeterministicFilter = this.filter != null && !this.filter.isDeterministic(); this.nullsAllowed = this.filter == null || this.nonDeterministicFilter || this.filter.testNull(); - this.scale = streamDescriptor.getOrcType().getScale().get(); + this.scale = streamDescriptor.getOrcType().getScale().orElseThrow(); this.nullBlock = outputType.map(type -> type.createBlockBuilder(null, 1).appendNull().build()).orElse(null); this.valuesPerPosition = valuesPerPosition; } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListSelectiveStreamReader.java index 31504b638db2a..d23768e9cb8ae 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListSelectiveStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListSelectiveStreamReader.java @@ -33,7 +33,6 @@ import com.facebook.presto.orc.stream.InputStreamSources; import com.facebook.presto.orc.stream.LongInputStream; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import org.joda.time.DateTimeZone; import org.openjdk.jol.info.ClassLayout; @@ -59,6 +58,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.MoreCollectors.onlyElement; import static io.airlift.slice.SizeOf.sizeOf; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; @@ -167,7 +167,7 @@ public ListSelectiveStreamReader( else if (!filters.isEmpty()) { Optional topLevelFilter = getTopLevelFilter(filters); if (topLevelFilter.isPresent()) { - nullsAllowed = topLevelFilter.get() == IS_NULL; + nullsAllowed = topLevelFilter.orElseThrow() == IS_NULL; nonNullsAllowed = !nullsAllowed; } else { @@ -223,7 +223,7 @@ private static Optional getTopLevelFilter(Map getTopLevelFilter(Map additionalSequenceEncodings = Collections.emptySortedMap(); // encoding or encoding.getAdditionalSequenceEncodings() may not be present when every map is empty or null if (encoding != null && encoding.getAdditionalSequenceEncodings().isPresent()) { - additionalSequenceEncodings = encoding.getAdditionalSequenceEncodings().get(); + additionalSequenceEncodings = encoding.getAdditionalSequenceEncodings().orElseThrow(); } // The ColumnEncoding with sequence ID 0 doesn't have any data associated with it diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapFlatSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapFlatSelectiveStreamReader.java index ec40e08bd520d..8252704677dbe 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapFlatSelectiveStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapFlatSelectiveStreamReader.java @@ -42,7 +42,6 @@ import com.facebook.presto.orc.stream.InputStreamSources; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import io.airlift.slice.SizeOf; import io.airlift.slice.Slice; @@ -74,6 +73,7 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.common.collect.MoreCollectors.onlyElement; import static io.airlift.slice.SizeOf.sizeOf; import static java.util.Objects.requireNonNull; @@ -209,7 +209,7 @@ private static Optional getTopLevelFilter(Map additionalSequenceEncodings = Collections.emptySortedMap(); // encoding or encoding.getAdditionalSequenceEncodings() may not be present when every map is empty or null if (encoding != null && encoding.getAdditionalSequenceEncodings().isPresent()) { - additionalSequenceEncodings = encoding.getAdditionalSequenceEncodings().get(); + additionalSequenceEncodings = encoding.getAdditionalSequenceEncodings().orElseThrow(); } keyIndices = ensureCapacity(keyIndices, additionalSequenceEncodings.size()); keyCount = 0; diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SelectiveStreamReaders.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SelectiveStreamReaders.java index dbb3271e6f481..abb22b2d4b04e 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SelectiveStreamReaders.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SelectiveStreamReaders.java @@ -37,7 +37,6 @@ import com.facebook.presto.orc.StreamDescriptor; import com.facebook.presto.orc.metadata.OrcType.OrcTypeKind; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import org.joda.time.DateTimeZone; import java.util.List; @@ -49,6 +48,7 @@ import static com.facebook.presto.common.type.Decimals.MAX_SHORT_PRECISION; import static com.facebook.presto.common.type.TimestampType.TIMESTAMP_MICROSECONDS; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.MoreCollectors.onlyElement; import static java.lang.String.format; public final class SelectiveStreamReaders @@ -103,7 +103,7 @@ public static SelectiveStreamReader createStreamReader( return new SliceSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext, isLowMemory); case TIMESTAMP: case TIMESTAMP_MICROSECONDS: { - boolean enableMicroPrecision = outputType.isPresent() && outputType.get() == TIMESTAMP_MICROSECONDS; + boolean enableMicroPrecision = outputType.isPresent() && outputType.orElseThrow() == TIMESTAMP_MICROSECONDS; checkArgument(requiredSubfields.isEmpty(), "Timestamp stream reader doesn't support subfields"); verifyStreamType(streamDescriptor, outputType, TimestampType.class::isInstance); return new TimestampSelectiveStreamReader( @@ -125,7 +125,7 @@ public static SelectiveStreamReader createStreamReader( return new MapSelectiveStreamReader(streamDescriptor, filters, requiredSubfields, outputType, hiveStorageTimeZone, options, systemMemoryContext, isLowMemory); case DECIMAL: { verifyStreamType(streamDescriptor, outputType, DecimalType.class::isInstance); - if (streamDescriptor.getOrcType().getPrecision().get() <= MAX_SHORT_PRECISION) { + if (streamDescriptor.getOrcType().getPrecision().orElseThrow() <= MAX_SHORT_PRECISION) { return new ShortDecimalSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName())); } else { @@ -141,7 +141,7 @@ public static SelectiveStreamReader createStreamReader( private static void verifyStreamType(StreamDescriptor streamDescriptor, Optional outputType, Predicate predicate) { if (outputType.isPresent()) { - ReaderUtils.verifyStreamType(streamDescriptor, outputType.get(), predicate); + ReaderUtils.verifyStreamType(streamDescriptor, outputType.orElseThrow(), predicate); } } @@ -152,7 +152,7 @@ private static Optional getOptionalOnlyFilter(OrcTypeKind typ } checkArgument(filters.size() == 1, format("Stream reader for %s doesn't support multiple range filters", type)); - return Optional.of(Iterables.getOnlyElement(filters.values())); + return Optional.of(filters.values().stream().collect(onlyElement())); } public static SelectiveStreamReader createNestedStreamReader( @@ -184,7 +184,7 @@ public static SelectiveStreamReader createNestedStreamReader( case DECIMAL: Map elementFilters = ImmutableMap.of(); if (parentFilter.isPresent()) { - TupleDomainFilter.PositionalFilter positionalFilter = parentFilter.get().getPositionalFilter(); + TupleDomainFilter.PositionalFilter positionalFilter = parentFilter.orElseThrow().getPositionalFilter(); if (positionalFilter != null) { elementFilters = ImmutableMap.of(new Subfield("c"), positionalFilter); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/StructSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/StructSelectiveStreamReader.java index 69c052881e7c7..26b9cac690613 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/StructSelectiveStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/StructSelectiveStreamReader.java @@ -33,7 +33,6 @@ import com.facebook.presto.orc.stream.InputStreamSources; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import org.joda.time.DateTimeZone; import org.openjdk.jol.info.ClassLayout; @@ -63,6 +62,7 @@ import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.common.collect.MoreCollectors.onlyElement; import static io.airlift.slice.SizeOf.sizeOf; import static java.util.Objects.requireNonNull; @@ -124,7 +124,7 @@ public StructSelectiveStreamReader( else { Optional topLevelFilter = getTopLevelFilter(filters); if (topLevelFilter.isPresent()) { - nullsAllowed = topLevelFilter.get() == IS_NULL; + nullsAllowed = topLevelFilter.orElseThrow() == IS_NULL; nonNullsAllowed = !nullsAllowed; } else { @@ -171,7 +171,7 @@ else if (outputRequired || !fieldsWithFilters.isEmpty()) { if (nestedStream == null) { verify(fieldOutputType.isPresent(), "Missing output type for subfield " + fieldName); - nestedReaders.put(fieldName, new MissingFieldStreamReader(fieldOutputType.get())); + nestedReaders.put(fieldName, new MissingFieldStreamReader(fieldOutputType.orElseThrow())); } else { if (requiredField || fieldsWithFilters.contains(fieldName)) { @@ -678,7 +678,7 @@ private static Optional getTopLevelFilter(Map keysPerColumn = columnToKeySet.get(column); - for (Map.Entry sequenceToEncoding : entry.getValue().getAdditionalSequenceEncodings().get().entrySet()) { + for (Map.Entry sequenceToEncoding : entry.getValue().getAdditionalSequenceEncodings().orElseThrow().entrySet()) { Integer sequence = sequenceToEncoding.getKey(); DwrfProto.KeyInfo key = sequenceToEncoding.getValue().getKey(); // add the stream only if it is present in the stream ordering config diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/AbstractTestOrcReader.java b/presto-orc/src/test/java/com/facebook/presto/orc/AbstractTestOrcReader.java index db80111decebd..29d96b9e8dc63 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/AbstractTestOrcReader.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/AbstractTestOrcReader.java @@ -36,6 +36,7 @@ import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Range; +import com.google.common.collect.Streams; import io.airlift.slice.Slice; import io.airlift.units.DataSize; import io.airlift.units.Duration; @@ -53,12 +54,14 @@ import java.io.IOException; import java.math.BigInteger; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Random; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; @@ -81,9 +84,6 @@ import static com.facebook.presto.orc.OrcTester.createSettableStructObjectInspector; import static com.facebook.presto.testing.DateTimeTestingUtils.sqlTimestampOf; import static com.facebook.presto.testing.TestingConnectorSession.SESSION; -import static com.google.common.collect.Iterables.concat; -import static com.google.common.collect.Iterables.cycle; -import static com.google.common.collect.Iterables.limit; import static com.google.common.collect.Lists.newArrayList; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.lang.Math.toIntExact; @@ -91,6 +91,7 @@ import static java.util.Collections.nCopies; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.function.Function.identity; import static java.util.stream.Collectors.toList; import static org.testng.Assert.assertEquals; @@ -123,7 +124,7 @@ public void setUp() public void testBooleanSequence() throws Exception { - tester.testRoundTrip(BOOLEAN, newArrayList(limit(cycle(ImmutableList.of(true, false, false)), 30_000))); + tester.testRoundTrip(BOOLEAN, Stream.generate(() -> ImmutableList.of(true, false, false)).flatMap(Collection::stream).limit(30_000).toList()); } @Test @@ -154,7 +155,8 @@ public void testLongSequenceWithHoles() public void testLongDirect() throws Exception { - testRoundTripNumeric(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), 30_000)); + testRoundTripNumeric(Stream.generate(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)::stream) + .flatMap(identity()).limit(30_000).toList()); } @Test @@ -173,21 +175,33 @@ public void testLongDirect2() public void testLongShortRepeat() throws Exception { - testRoundTripNumeric(limit(repeatEach(4, cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17))), 30_000)); + testRoundTripNumeric( + Stream.generate(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)::stream) + .flatMap(identity()) + .flatMap(item -> repeat(4, item)) + .limit(30_000) + .toList()); } @Test public void testLongPatchedBase() throws Exception { - testRoundTripNumeric(limit(cycle(concat(intsBetween(0, 18), intsBetween(0, 18), ImmutableList.of(30_000, 20_000, 400_000, 30_000, 20_000))), 30_000)); + testRoundTripNumeric(Stream.generate(() -> Streams.concat(intsBetween(0, 18).stream(), + intsBetween(0, 18).stream(), + ImmutableList.of(30_000, 20_000, 400_000, 30_000, 20_000).stream())) + .flatMap(identity()) + .limit(30_000).toList()); } @Test public void testLongStrideDictionary() throws Exception { - testRoundTripNumeric(concat(ImmutableList.of(1), nCopies(9999, 123), ImmutableList.of(2), nCopies(9999, 123))); + testRoundTripNumeric(Streams.concat( + ImmutableList.of(1).stream(), nCopies(9999, 123).stream(), + ImmutableList.of(2).stream(), nCopies(9999, 123).stream()) + .toList()); } @Test @@ -240,15 +254,15 @@ public void testCaching() assertEquals(stripeFootercache.stats().hitCount(), 0); assertEquals(stripeStreamCache.stats().missCount(), 2); assertEquals(stripeStreamCache.stats().hitCount(), 0); - assertEquals(rowGroupIndexCache.get().stats().missCount(), 1); - assertEquals(rowGroupIndexCache.get().stats().hitCount(), 0); + assertEquals(rowGroupIndexCache.orElseThrow().stats().missCount(), 1); + assertEquals(rowGroupIndexCache.orElseThrow().stats().hitCount(), 0); cacheReader.nextBatch(); assertEquals(stripeFootercache.stats().missCount(), 1); assertEquals(stripeFootercache.stats().hitCount(), 1); assertEquals(stripeStreamCache.stats().missCount(), 2); assertEquals(stripeStreamCache.stats().hitCount(), 2); - assertEquals(rowGroupIndexCache.get().stats().missCount(), 1); - assertEquals(rowGroupIndexCache.get().stats().hitCount(), 1); + assertEquals(rowGroupIndexCache.orElseThrow().stats().missCount(), 1); + assertEquals(rowGroupIndexCache.orElseThrow().stats().hitCount(), 1); assertEquals(storageReader.readBlock(0).getInt(0), cacheReader.readBlock(0).getInt(0)); } } @@ -372,7 +386,7 @@ public void testDoubleNaNInfinity() public void testStringUnicode() throws Exception { - tester.testRoundTrip(VARCHAR, newArrayList(limit(cycle(ImmutableList.of("apple", "apple pie", "apple\uD835\uDC03", "apple\uFFFD")), 30_000))); + tester.testRoundTrip(VARCHAR, Stream.generate(() -> ImmutableList.of("apple", "apple pie", "apple\uD835\uDC03", "apple\uFFFD")).flatMap(Collection::stream).limit(30_000).toList()); } @Test @@ -392,7 +406,9 @@ public void testStringDictionarySequence() { tester.testRoundTrip( VARCHAR, - newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), 30_000)).stream() + Stream.generate(() -> ImmutableList.of(1, 3, 5, 7, 11, 13, 17)) + .flatMap(Collection::stream) + .limit(30_000) .map(Object::toString) .collect(toList())); } @@ -401,14 +417,17 @@ public void testStringDictionarySequence() public void testStringStrideDictionary() throws Exception { - tester.testRoundTrip(VARCHAR, newArrayList(concat(ImmutableList.of("a"), nCopies(9999, "123"), ImmutableList.of("b"), nCopies(9999, "123")))); + tester.testRoundTrip(VARCHAR, Streams.concat( + ImmutableList.of("a").stream(), nCopies(9999, "123").stream(), + ImmutableList.of("b").stream(), nCopies(9999, "123").stream()) + .toList()); } @Test public void testEmptyStringSequence() throws Exception { - tester.testRoundTrip(VARCHAR, newArrayList(limit(cycle(""), 30_000))); + tester.testRoundTrip(VARCHAR, Stream.generate(() -> "").limit(30_000).toList()); } @Test @@ -428,7 +447,9 @@ public void testCharDictionarySequence() { tester.testRoundTrip( CHAR, - newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), 30_000)).stream() + Stream.generate(() -> ImmutableList.of(1, 3, 5, 7, 11, 13, 17)) + .flatMap(Collection::stream) + .limit(30_000) .map(this::toCharValue) .collect(toList())); } @@ -437,7 +458,7 @@ public void testCharDictionarySequence() public void testEmptyCharSequence() throws Exception { - tester.testRoundTrip(CHAR, newArrayList(limit(cycle(" "), 30_000))); + tester.testRoundTrip(CHAR, Stream.generate(() -> " ").limit(30_000).toList()); } private String toCharValue(Object value) @@ -463,7 +484,9 @@ public void testBinaryDictionarySequence() throws Exception { tester.testRoundTrip( - VARBINARY, ImmutableList.copyOf(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), 30_000)).stream() + VARBINARY, Stream.generate(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)::stream) + .flatMap(identity()) + .limit(30_000) .map(Object::toString) .map(string -> string.getBytes(UTF_8)) .map(SqlVarbinary::new) @@ -481,13 +504,14 @@ public void testEmptyBinarySequence() public void testDwrfInvalidCheckpointsForRowGroupDictionary() throws Exception { - List values = newArrayList(limit( - cycle(concat( - ImmutableList.of(1), nCopies(9999, 123), - ImmutableList.of(2), nCopies(9999, 123), - ImmutableList.of(3), nCopies(9999, 123), - nCopies(1_000_000, null))), - 200_000)); + List values = Stream.generate(() -> Streams.concat( + ImmutableList.of(1).stream(), nCopies(9999, 123).stream(), + ImmutableList.of(2).stream(), nCopies(9999, 123).stream(), + ImmutableList.of(3).stream(), nCopies(9999, 123).stream(), + nCopies(1_000_000, (Integer) null).stream())) + .flatMap(identity()) + .limit(200_000) + .toList(); tester.assertRoundTrip(INTEGER, values, false); @@ -504,7 +528,9 @@ public void testDwrfInvalidCheckpointsForStripeDictionary() { tester.testRoundTrip( VARCHAR, - newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), 200_000)).stream() + Stream.generate(() -> ImmutableList.of(1, 3, 5, 7, 11, 13, 17)) + .flatMap(Collection::stream) + .limit(200_000) .map(Object::toString) .collect(toList())); } @@ -546,31 +572,10 @@ protected T computeNext() }; } - private static Iterable repeatEach(int n, Iterable iterable) + private static Stream repeat(int n, T item) { - return () -> new AbstractIterator() - { - private final Iterator delegate = iterable.iterator(); - private int position; - private T value; - - @Override - protected T computeNext() - { - if (position == 0) { - if (!delegate.hasNext()) { - return endOfData(); - } - value = delegate.next(); - } - - position++; - if (position >= n) { - position = 0; - } - return value; - } - }; + return Stream.generate(() -> item) + .limit(n); } private static List doubleSequence(double start, double step, int items) diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/OrcTester.java b/presto-orc/src/test/java/com/facebook/presto/orc/OrcTester.java index 3c6a2497b6b78..faeef4233cbe7 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/OrcTester.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/OrcTester.java @@ -55,9 +55,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Streams; import io.airlift.slice.Slice; import io.airlift.slice.Slices; import io.airlift.units.DataSize; @@ -477,7 +477,7 @@ private void testMapRoundTrip(Type type, List readValues) Type mapType = mapType(type, type); // maps can not have a null key, so select a value to use for the map key when the value is null - Object readNullKeyValue = Iterables.getLast(readValues); + Object readNullKeyValue = Streams.findLast(readValues.stream()).orElseThrow(); // values in simple map testRoundTripType( @@ -1240,7 +1240,7 @@ else if (nestedType instanceof RowType) { int index = -1; List fields = ((RowType) nestedType).getFields(); for (int i = 0; i < fields.size(); i++) { - if (fieldName.equalsIgnoreCase(fields.get(i).getName().get())) { + if (fieldName.equalsIgnoreCase(fields.get(i).getName().orElseThrow())) { index = i; nestedType = fields.get(i).getType(); break; @@ -2251,7 +2251,7 @@ private static ObjectInspector getJavaObjectInspector(Type type) if (type.getTypeSignature().getBase().equals(StandardTypes.ROW)) { return getStandardStructObjectInspector( type.getTypeSignature().getParameters().stream() - .map(parameter -> parameter.getNamedTypeSignature().getName().get()) + .map(parameter -> parameter.getNamedTypeSignature().getName().orElseThrow()) .collect(toList()), type.getTypeParameters().stream() .map(OrcTester::getJavaObjectInspector) diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestColumnStatistics.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestColumnStatistics.java index 810cc5ac011bc..b35e6aa7e9a9f 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestColumnStatistics.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestColumnStatistics.java @@ -1127,7 +1127,7 @@ private static void assertCommonRawAndStorageSize(CapturingOrcFileIntrospector i // additionalSequence will be absent if map statistics are not enabled if (flatMapColumnEncoding.getAdditionalSequenceEncodings().isPresent()) { Object2LongMap fileLevelKeySizes = flatMapKeySizes.computeIfAbsent(flatMapNode.intValue(), (ignore) -> new Object2LongOpenHashMap<>()); - Map stripeSequenceToKey = flatMapColumnEncoding.getAdditionalSequenceEncodings().get(); + Map stripeSequenceToKey = flatMapColumnEncoding.getAdditionalSequenceEncodings().orElseThrow(); for (Map.Entry entry : stripeSequenceToKey.entrySet()) { Integer sequence = entry.getKey(); KeyInfo flatMapKey = entry.getValue().getKey(); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestDecryption.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestDecryption.java index 4bee9227222b5..bdb749c4fa936 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestDecryption.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestDecryption.java @@ -651,7 +651,7 @@ private static void validateFileStatistics(TempFile tempFile, Optional fileStatsNoKey = readerNoKeys.getFooter().getFileStats(); assertEquals(fileStatsNoKey.size(), types.size()); - Set allEncryptedNodes = dwrfWriterEncryption.get().getWriterEncryptionGroups().stream() + Set allEncryptedNodes = dwrfWriterEncryption.orElseThrow().getWriterEncryptionGroups().stream() .flatMap(group -> group.getNodes().stream()) .flatMap(node -> collectNodeTree(types, node).stream()) .collect(Collectors.toSet()); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestDefaultOrcWriterFlushPolicy.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestDefaultOrcWriterFlushPolicy.java index b06cf4c7500be..4dddeb2bac4eb 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestDefaultOrcWriterFlushPolicy.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestDefaultOrcWriterFlushPolicy.java @@ -42,7 +42,7 @@ public void testFlushMaxStripeRowCount() actual = flushPolicy.shouldFlushStripe(10, 0, false); assertTrue(actual.isPresent()); - assertEquals(actual.get(), MAX_ROWS); + assertEquals(actual.orElseThrow(), MAX_ROWS); actual = flushPolicy.shouldFlushStripe(20, 0, false); assertFalse(actual.isPresent()); @@ -67,7 +67,7 @@ public void testFlushMaxStripeSize() actual = flushPolicy.shouldFlushStripe(1, 200, false); assertTrue(actual.isPresent()); - assertEquals(actual.get(), MAX_BYTES); + assertEquals(actual.orElseThrow(), MAX_BYTES); } @Test @@ -80,6 +80,6 @@ public void testFlushDictionaryFull() actual = flushPolicy.shouldFlushStripe(1, 1, true); assertTrue(actual.isPresent()); - assertEquals(actual.get(), DICTIONARY_FULL); + assertEquals(actual.orElseThrow(), DICTIONARY_FULL); } } diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestDictionaryColumnWriter.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestDictionaryColumnWriter.java index ece5108ebfffb..d5f62c2f592e5 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestDictionaryColumnWriter.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestDictionaryColumnWriter.java @@ -36,6 +36,7 @@ import java.util.Optional; import java.util.Random; import java.util.UUID; +import java.util.stream.Stream; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.IntegerType.INTEGER; @@ -56,9 +57,6 @@ import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; import static com.facebook.presto.orc.metadata.CompressionKind.ZSTD; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.Iterables.cycle; -import static com.google.common.collect.Iterables.limit; -import static com.google.common.collect.Lists.newArrayList; import static io.airlift.slice.SizeOf.SIZE_OF_BYTE; import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.units.DataSize.Unit.MEGABYTE; @@ -104,7 +102,7 @@ public void testStringNoRows() public void testStringAllNullsWithDirectConversion() throws Exception { - List values = newArrayList(limit(cycle(new String[] {null}), 90_000)); + List values = Stream.generate(() -> null).limit(90_000).toList(); for (StringDictionaryInput input : StringDictionaryInput.values()) { DirectConversionTester directConversionTester = new DirectConversionTester(); directConversionTester.add(7, megabytes(1), true); @@ -295,7 +293,7 @@ public void testDisableStringDictionaryEncoding() public void testDisableStringOnlyNulls() throws IOException { - List values = newArrayList(limit(cycle(new String[] {null}), 3 * STRIPE_MAX_ROWS)); + List values = Stream.generate(() -> null).limit(3 * STRIPE_MAX_ROWS).toList(); testStringDirectColumn(values); } @@ -356,7 +354,7 @@ public void testIntegerDictionaryAllNulls() directConversionTester.add(7, megabytes(1), true); directConversionTester.add(14, megabytes(1), true); directConversionTester.add(32, megabytes(1), true); - List values = newArrayList(limit(cycle(new Integer[] {null}), 60_000)); + List values = Stream.generate(() -> null).limit(60_000).toList(); List stripeFooters = testIntegerDictionary(directConversionTester, values); verifyDwrfDirectEncoding(getStripeSize(values.size()), stripeFooters); @@ -367,7 +365,10 @@ public void testIntegerDictionaryAlternatingNulls() throws IOException { DirectConversionTester directConversionTester = new DirectConversionTester(); - List values = newArrayList(limit(cycle(Integer.MAX_VALUE, null, Integer.MIN_VALUE), 60_000)); + List values = Stream.generate(() -> new Integer[] {Integer.MAX_VALUE, null, Integer.MIN_VALUE}) + .flatMap(Arrays::stream) + .limit(60_000) + .toList(); List stripeFooters = testIntegerDictionary(directConversionTester, values); verifyDictionaryEncoding(getStripeSize(values.size()), DWRF, stripeFooters, ImmutableList.of(2, 2, 2, 2)); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestDwrfStripeCacheOptions.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestDwrfStripeCacheOptions.java index 408d9e6b098e5..699979bd825ea 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestDwrfStripeCacheOptions.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestDwrfStripeCacheOptions.java @@ -31,7 +31,7 @@ public void testToString() .withDwrfStripeCacheMaxSize(new DataSize(27, MEGABYTE)) .build(); - DwrfStripeCacheOptions dwrfStripeCacheOptions = options.getDwrfStripeCacheOptions().get(); + DwrfStripeCacheOptions dwrfStripeCacheOptions = options.getDwrfStripeCacheOptions().orElseThrow(); String expectedString = "DwrfStripeCacheOptions{stripeCacheMode=INDEX_AND_FOOTER, stripeCacheMaxSize=27MB}"; assertEquals(dwrfStripeCacheOptions.toString(), expectedString); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestFlatMapWriter.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestFlatMapWriter.java index b24c02b4b5d4b..af537d5868285 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestFlatMapWriter.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestFlatMapWriter.java @@ -23,14 +23,18 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Streams; import org.testng.annotations.Test; import java.time.LocalDate; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Optional; import java.util.Random; +import java.util.stream.Stream; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; @@ -46,8 +50,6 @@ import static com.facebook.presto.orc.OrcTester.createOrcWriter; import static com.facebook.presto.orc.OrcTester.mapType; import static com.facebook.presto.orc.OrcTester.rowType; -import static com.google.common.collect.Iterables.cycle; -import static com.google.common.collect.Iterables.limit; import static com.google.common.collect.Lists.newArrayList; import static java.nio.charset.StandardCharsets.UTF_8; import static org.testng.Assert.assertEquals; @@ -181,9 +183,9 @@ public void testMixedNullMapAndEmptyMap() Type mapType = mapType(INTEGER, DOUBLE); OrcTester tester = OrcTester.quickDwrfFlatMapTester(); - tester.testRoundTrip(mapType, newArrayList(limit(cycle((Map) null), ROWS))); - tester.testRoundTrip(mapType, newArrayList(limit(cycle((Map) null, EMPTY_MAP), ROWS))); - tester.testRoundTrip(mapType, newArrayList(limit(cycle(EMPTY_MAP, (Map) null), ROWS))); + tester.testRoundTrip(mapType, Stream.generate(() -> (Map) null).limit(ROWS).toList()); + tester.testRoundTrip(mapType, Stream.generate(() -> new Object[] {null, EMPTY_MAP}).flatMap(Arrays::stream).limit(ROWS).toList()); + tester.testRoundTrip(mapType, Stream.generate(() -> new Object[] {EMPTY_MAP, null}).flatMap(Arrays::stream).limit(ROWS).toList()); } private static void runTest(Type mapType, K key1, K key2, K key3, V value1, V value2, V value3, V value4, V value5) @@ -200,26 +202,28 @@ private static void runTest(Type mapType, K key1, K key2, K key3, V value Map map5 = ImmutableMap.of(key1, value5, key2, value4, key3, value2); // empty and null value maps - tester.testRoundTrip(mapType, newArrayList(limit(cycle(EMPTY_MAP), ROWS))); - tester.testRoundTrip(mapType, newArrayList(limit(cycle(nullValue), ROWS))); - tester.testRoundTrip(mapType, newArrayList(limit(cycle(nullValue, EMPTY_MAP), ROWS))); - tester.testRoundTrip(mapType, newArrayList(limit(cycle(EMPTY_MAP, nullValue), ROWS))); + tester.testRoundTrip(mapType, Stream.generate(() -> ImmutableList.of(EMPTY_MAP)).flatMap(Collection::stream).limit(ROWS).toList()); + tester.testRoundTrip(mapType, Stream.generate(() -> ImmutableList.of(nullValue)).flatMap(Collection::stream).limit(ROWS).toList()); + tester.testRoundTrip(mapType, Stream.generate(() -> ImmutableList.of(nullValue, EMPTY_MAP)).flatMap(Collection::stream).limit(ROWS).toList()); + tester.testRoundTrip(mapType, Stream.generate(() -> ImmutableList.of(EMPTY_MAP, nullValue)).flatMap(Collection::stream).limit(ROWS).toList()); // same keys - tester.testRoundTrip(mapType, newArrayList(limit(cycle(map1, map2), ROWS))); - tester.testRoundTrip(mapType, newArrayList(limit(cycle(map1, map2, EMPTY_MAP), ROWS))); - tester.testRoundTrip(mapType, newArrayList(limit(cycle(map5, EMPTY_MAP), ROWS))); - tester.testRoundTrip(mapType, newArrayList(limit(cycle(map1, map2, nullValue), ROWS))); - tester.testRoundTrip(mapType, newArrayList(limit(cycle(map1, map2, nullValue, EMPTY_MAP), ROWS))); + tester.testRoundTrip(mapType, Stream.generate(() -> ImmutableList.of(map1, map2)).flatMap(Collection::stream).limit(ROWS).toList()); + tester.testRoundTrip(mapType, Stream.generate(() -> ImmutableList.of(map1, map2, EMPTY_MAP)).flatMap(Collection::stream).limit(ROWS).toList()); + tester.testRoundTrip(mapType, Stream.generate(() -> ImmutableList.of(map5, EMPTY_MAP)).flatMap(Collection::stream).limit(ROWS).toList()); + tester.testRoundTrip(mapType, Stream.generate(() -> ImmutableList.of(map1, map2, nullValue)).flatMap(Collection::stream).limit(ROWS).toList()); + tester.testRoundTrip(mapType, Stream.generate(() -> ImmutableList.of(map1, map2, nullValue, EMPTY_MAP)).flatMap(Collection::stream).limit(ROWS).toList()); // zig-zag keys - tester.testRoundTrip(mapType, newArrayList(limit(cycle(map1, map3, map4, map5), ROWS))); - tester.testRoundTrip(mapType, newArrayList(limit(cycle(map1, map3, map4, map5, EMPTY_MAP), ROWS))); - tester.testRoundTrip(mapType, newArrayList(limit(cycle(map1, map3, map4, map5, nullValue), ROWS))); - tester.testRoundTrip(mapType, newArrayList(limit(cycle(map1, map3, map4, map5, nullValue, EMPTY_MAP), ROWS))); + tester.testRoundTrip(mapType, Stream.generate(() -> ImmutableList.of(map1, map3, map4, map5)).flatMap(Collection::stream).limit(ROWS).toList()); + tester.testRoundTrip(mapType, Stream.generate(() -> ImmutableList.of(map1, map3, map4, map5, EMPTY_MAP)).flatMap(Collection::stream).limit(ROWS).toList()); + tester.testRoundTrip(mapType, Stream.generate(() -> ImmutableList.of(map1, map3, map4, map5, nullValue)).flatMap(Collection::stream).limit(ROWS).toList()); + tester.testRoundTrip(mapType, Stream.generate(() -> ImmutableList.of(map1, map3, map4, map5, nullValue, EMPTY_MAP)).flatMap(Collection::stream).limit(ROWS).toList()); // randomize keys - tester.testRoundTrip(mapType, newArrayList(limit(random(map1, map2, map3, map4, map5, nullValue, EMPTY_MAP), ROWS))); + tester.testRoundTrip(mapType, Stream.generate(() -> + TestFlatMapWriter.>random(map1, map2, map3, map4, map5, nullValue, EMPTY_MAP)) + .flatMap(Streams::stream).limit(ROWS).toList()); } @Test diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcBloomFilters.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcBloomFilters.java index c696416ee92fe..ec4655acc3efb 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcBloomFilters.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcBloomFilters.java @@ -251,7 +251,7 @@ public void testExtractValuesFromSingleDomain() Domain predicateDomain = Domain.singleValue(testValue.getKey(), testValue.getValue()); Optional> discreteValues = extractDiscreteValues(predicateDomain.getValues()); assertTrue(discreteValues.isPresent()); - Collection objects = discreteValues.get(); + Collection objects = discreteValues.orElseThrow(); assertEquals(objects.size(), 1); assertEquals(objects.iterator().next(), testValue.getValue()); } diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcOutputBuffer.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcOutputBuffer.java index c63fa0cda73e8..5aafd44e8658a 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcOutputBuffer.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcOutputBuffer.java @@ -346,7 +346,7 @@ private void assertCompressedContent(OrcOutputBuffer orcOutputBuffer, byte[] exp private DecompressionResult decompress(Slice slice) { - OrcDecompressor decompressor = OrcDecompressor.createOrcDecompressor(DATA_SOURCE_ID, ZSTD, 256 * 1024).get(); + OrcDecompressor decompressor = OrcDecompressor.createOrcDecompressor(DATA_SOURCE_ID, ZSTD, 256 * 1024).orElseThrow(); SharedBuffer decompressionBuffer = new SharedBuffer(NOOP_ORC_LOCAL_MEMORY_CONTEXT); ImmutableList.Builder sizes = ImmutableList.builder(); BasicSliceInput input = slice.getInput(); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderDwrfStripeCaching.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderDwrfStripeCaching.java index 821a5a0355ec6..0a6c9db88f8e8 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderDwrfStripeCaching.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderDwrfStripeCaching.java @@ -49,7 +49,7 @@ public void testBothAllStripes(File orcFile) { Optional optionalDwrfStripeCache = getDwrfStripeCache(orcFile); assertTrue(optionalDwrfStripeCache.isPresent()); - DwrfStripeCache dwrfStripeCache = optionalDwrfStripeCache.get(); + DwrfStripeCache dwrfStripeCache = optionalDwrfStripeCache.orElseThrow(); DwrfProto.Footer footer = readFileFooter(orcFile); List stripes = footer.getStripesList(); @@ -70,7 +70,7 @@ public void testBothHalfStripes(File orcFile) { Optional optionalDwrfStripeCache = getDwrfStripeCache(orcFile); assertTrue(optionalDwrfStripeCache.isPresent()); - DwrfStripeCache dwrfStripeCache = optionalDwrfStripeCache.get(); + DwrfStripeCache dwrfStripeCache = optionalDwrfStripeCache.orElseThrow(); DwrfProto.Footer footer = readFileFooter(orcFile); List stripes = footer.getStripesList(); @@ -97,7 +97,7 @@ public void testIndexAllStripes(File orcFile) { Optional optionalDwrfStripeCache = getDwrfStripeCache(orcFile); assertTrue(optionalDwrfStripeCache.isPresent()); - DwrfStripeCache dwrfStripeCache = optionalDwrfStripeCache.get(); + DwrfStripeCache dwrfStripeCache = optionalDwrfStripeCache.orElseThrow(); DwrfProto.Footer footer = readFileFooter(orcFile); List stripes = footer.getStripesList(); @@ -118,7 +118,7 @@ public void testIndexHalfStripes(File orcFile) { Optional optionalDwrfStripeCache = getDwrfStripeCache(orcFile); assertTrue(optionalDwrfStripeCache.isPresent()); - DwrfStripeCache dwrfStripeCache = optionalDwrfStripeCache.get(); + DwrfStripeCache dwrfStripeCache = optionalDwrfStripeCache.orElseThrow(); DwrfProto.Footer footer = readFileFooter(orcFile); List stripes = footer.getStripesList(); @@ -144,7 +144,7 @@ public void testFooterAllStripes(File orcFile) { Optional optionalDwrfStripeCache = getDwrfStripeCache(orcFile); assertTrue(optionalDwrfStripeCache.isPresent()); - DwrfStripeCache dwrfStripeCache = optionalDwrfStripeCache.get(); + DwrfStripeCache dwrfStripeCache = optionalDwrfStripeCache.orElseThrow(); DwrfProto.Footer footer = readFileFooter(orcFile); List stripes = footer.getStripesList(); @@ -165,7 +165,7 @@ public void testFooterHalfStripes(File orcFile) { Optional optionalDwrfStripeCache = getDwrfStripeCache(orcFile); assertTrue(optionalDwrfStripeCache.isPresent()); - DwrfStripeCache dwrfStripeCache = optionalDwrfStripeCache.get(); + DwrfStripeCache dwrfStripeCache = optionalDwrfStripeCache.orElseThrow(); DwrfProto.Footer footer = readFileFooter(orcFile); List stripes = footer.getStripesList(); @@ -207,7 +207,7 @@ private void assertStripeIndexCachePresent(DwrfStripeCache dwrfStripeCache, Rand StripeId stripeId = new StripeId(TEST_DATA_SOURCE_ID, stripe.getOffset()); Optional stripeIndexSlice = dwrfStripeCache.getIndexStreamsSlice(stripeId); assertTrue(stripeIndexSlice.isPresent()); - assertEquals(stripeIndexSlice.get().getBytes(), readBytes(file, stripe.getOffset(), stripe.getIndexLength())); + assertEquals(stripeIndexSlice.orElseThrow().getBytes(), readBytes(file, stripe.getOffset(), stripe.getIndexLength())); } private void assertStripeIndexCacheAbsent(DwrfStripeCache dwrfStripeCache, DwrfProto.StripeInformation stripe) @@ -224,7 +224,7 @@ private void assertStripeFooterCachePresent(DwrfStripeCache dwrfStripeCache, Ran Optional stripeFooterSlice = dwrfStripeCache.getStripeFooterSlice(stripeId, toIntExact(stripe.getFooterLength())); assertTrue(stripeFooterSlice.isPresent()); long footerOffset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(); - assertEquals(stripeFooterSlice.get().getBytes(), readBytes(file, footerOffset, stripe.getFooterLength())); + assertEquals(stripeFooterSlice.orElseThrow().getBytes(), readBytes(file, footerOffset, stripe.getFooterLength())); } private void assertStripeFooterCacheAbsent(DwrfStripeCache dwrfStripeCache, DwrfProto.StripeInformation stripe) diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcWriterOptions.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcWriterOptions.java index 9544a275eb4e8..e46780dbd3301 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcWriterOptions.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcWriterOptions.java @@ -49,7 +49,7 @@ public void testDwrfWriterOptionsProperties() assertEquals(options.getDwrfStripeCacheOptions().isPresent(), value); if (value) { - DwrfStripeCacheOptions dwrfStripeCacheOptions = options.getDwrfStripeCacheOptions().get(); + DwrfStripeCacheOptions dwrfStripeCacheOptions = options.getDwrfStripeCacheOptions().orElseThrow(); assertEquals(dwrfStripeCacheOptions.getStripeCacheMode(), DWRF_STRIPE_CACHE_MODE); assertEquals(dwrfStripeCacheOptions.getStripeCacheMaxSize(), DWRF_STRIPE_CACHE_MAX_SIZE); } diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestReadBloomFilter.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestReadBloomFilter.java index ff99f871fdd49..d3529799c6883 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestReadBloomFilter.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestReadBloomFilter.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.stream.Stream; import static com.facebook.presto.common.predicate.TupleDomain.fromFixedValues; import static com.facebook.presto.common.type.BigintType.BIGINT; @@ -44,12 +45,10 @@ import static com.facebook.presto.orc.OrcTester.writeOrcColumnHive; import static com.facebook.presto.orc.TupleDomainOrcPredicate.ColumnReference; import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; -import static com.google.common.collect.Iterables.cycle; -import static com.google.common.collect.Iterables.limit; -import static com.google.common.collect.Lists.newArrayList; import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.function.Function.identity; import static org.testng.Assert.assertEquals; public class TestReadBloomFilter @@ -75,7 +74,9 @@ public void test() private static void testType(Type type, List uniqueValues, T inBloomFilter, T notInBloomFilter) throws Exception { - List writeValues = newArrayList(limit(cycle(uniqueValues), 30_000)); + List writeValues = Stream.generate(uniqueValues::stream) + .flatMap(identity()) + .limit(30_000).toList(); try (TempFile tempFile = new TempFile()) { writeOrcColumnHive(tempFile.getFile(), ORC_12, SNAPPY, type, writeValues); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestSelectiveOrcReader.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestSelectiveOrcReader.java index ae7c9dcc9a128..a9017c1800296 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestSelectiveOrcReader.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestSelectiveOrcReader.java @@ -53,6 +53,7 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -61,6 +62,7 @@ import java.util.Random; import java.util.function.Function; import java.util.stream.IntStream; +import java.util.stream.Stream; import static com.facebook.airlift.testing.Assertions.assertBetweenInclusive; import static com.facebook.presto.common.predicate.TupleDomainFilter.IS_NOT_NULL; @@ -97,12 +99,10 @@ import static com.facebook.presto.testing.TestingConnectorSession.SESSION; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static com.google.common.collect.Iterables.concat; -import static com.google.common.collect.Iterables.cycle; -import static com.google.common.collect.Iterables.limit; import static com.google.common.collect.Lists.newArrayList; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.nCopies; +import static java.util.function.Function.identity; import static java.util.stream.Collectors.toList; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -133,13 +133,13 @@ public void testBooleanSequence() { tester.testRoundTrip( BOOLEAN, - newArrayList(limit(cycle(ImmutableList.of(true, false, false)), NUM_ROWS)), + Stream.generate(() -> ImmutableList.of(true, false, false)).flatMap(Collection::stream).limit(NUM_ROWS).toList(), BooleanValue.of(true, false), TupleDomainFilter.IS_NULL); tester.testRoundTripTypes(ImmutableList.of(BOOLEAN, BOOLEAN), ImmutableList.of( - newArrayList(limit(cycle(ImmutableList.of(true, false, false)), NUM_ROWS)), - newArrayList(limit(cycle(ImmutableList.of(true, true, false)), NUM_ROWS))), + Stream.generate(() -> ImmutableList.of(true, false, false)).flatMap(Collection::stream).limit(NUM_ROWS).toList(), + Stream.generate(() -> ImmutableList.of(true, true, false)).flatMap(Collection::stream).limit(NUM_ROWS).toList()), toSubfieldFilters( ImmutableMap.of(0, BooleanValue.of(true, false)), ImmutableMap.of(0, TupleDomainFilter.IS_NULL), @@ -193,7 +193,10 @@ public void testByteValuesRepeat() .collect(toList()); tester.testRoundTrip( TINYINT, - newArrayList(limit(repeatEach(4, cycle(byteValues)), NUM_ROWS)), + Stream.generate(() -> byteValues) + .flatMap(Collection::stream) + .flatMap(item -> repeat(4, item)) + .limit(NUM_ROWS).toList(), BigintRange.of(1, 14, true)); } @@ -201,14 +204,15 @@ public void testByteValuesRepeat() public void testByteValuesPatchedBase() throws Exception { - List byteValues = newArrayList( - limit(cycle(concat( - intsBetween(0, 18), - intsBetween(0, 18), - ImmutableList.of(NUM_ROWS, 20_000, 400_000, NUM_ROWS, 20_000))), NUM_ROWS)) - .stream() - .map(Integer::byteValue) - .collect(toList()); + List byteValues = + Stream.generate(() -> Streams.concat( + intsBetween(0, 18).stream(), + intsBetween(0, 18).stream(), + ImmutableList.of(NUM_ROWS, 20_000, 400_000, NUM_ROWS, 20_000).stream())) + .flatMap(identity()) + .limit(NUM_ROWS) + .map(Integer::byteValue) + .collect(toList()); tester.testRoundTrip( TINYINT, byteValues, @@ -285,7 +289,10 @@ public void testLongSequenceWithHoles() public void testLongDirect() throws Exception { - testRoundTripNumeric(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), NUM_ROWS), BigintRange.of(4, 14, false)); + testRoundTripNumeric(Stream.generate(() -> ImmutableList.of(1, 3, 5, 7, 11, 13, 17)) + .flatMap(Collection::stream) + .limit(NUM_ROWS).toList(), + BigintRange.of(4, 14, false)); Random random = new Random(0); @@ -319,14 +326,25 @@ public void testLongDirectVarintScale() public void testLongShortRepeat() throws Exception { - testRoundTripNumeric(limit(repeatEach(4, cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17))), NUM_ROWS), BigintRange.of(4, 14, true)); + testRoundTripNumeric( + Stream.generate(() -> ImmutableList.of(1, 3, 5, 7, 11, 13, 17)) + .flatMap(Collection::stream) + .flatMap(item -> repeat(4, item)) + .limit(NUM_ROWS).toList(), + BigintRange.of(4, 14, true)); } @Test public void testLongPatchedBase() throws Exception { - testRoundTripNumeric(limit(cycle(concat(intsBetween(0, 18), intsBetween(0, 18), ImmutableList.of(NUM_ROWS, 20_000, 400_000, NUM_ROWS, 20_000))), NUM_ROWS), + testRoundTripNumeric(Stream.generate(() -> Streams.concat( + intsBetween(0, 18).stream(), + intsBetween(0, 18).stream(), + ImmutableList.of(NUM_ROWS, 20_000, 400_000, NUM_ROWS, 20_000).stream())) + .flatMap(identity()) + .limit(NUM_ROWS) + .toList(), toBigintValues(new long[] {0, 5, 10, 15, 20_000}, true)); } @@ -334,7 +352,12 @@ public void testLongPatchedBase() public void testLongStrideDictionary() throws Exception { - testRoundTripNumeric(concat(ImmutableList.of(1), nCopies(9999, 123), ImmutableList.of(2), nCopies(9999, 123)), BigintRange.of(123, 123, true)); + testRoundTripNumeric( + Streams.concat( + ImmutableList.of(1).stream(), nCopies(9999, 123).stream(), + ImmutableList.of(2).stream(), nCopies(9999, 123).stream()) + .toList(), + BigintRange.of(123, 123, true)); } @Test @@ -346,15 +369,31 @@ public void testFloats() FloatRange.of(-100.0f, false, true, 0.0f, false, true, false), IS_NULL); - tester.testRoundTrip(REAL, ImmutableList.copyOf(repeatEach(10, ImmutableList.of(-100.0f, 0.0f, 100.0f))), filters); - tester.testRoundTrip(REAL, ImmutableList.copyOf(repeatEach(10, ImmutableList.of(1000.0f, -1.23f, Float.POSITIVE_INFINITY)))); + tester.testRoundTrip(REAL, + Stream.generate(() -> ImmutableList.of(-100.0f, 0.0f, 100.0f)) + .flatMap(Collection::stream) + .flatMap(item -> repeat(10, item)) + .limit(30).toList(), + filters); + tester.testRoundTrip(REAL, + Stream.generate(() -> ImmutableList.of(1000.0f, -1.23f, Float.POSITIVE_INFINITY)) + .flatMap(Collection::stream) + .flatMap(item -> repeat(10, item)) + .limit(30).toList()); List floatValues = ImmutableList.of(1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f); tester.testRoundTripTypes( ImmutableList.of(REAL, REAL), - ImmutableList.of( - ImmutableList.copyOf(limit(repeatEach(4, cycle(floatValues)), 100)), - ImmutableList.copyOf(limit(repeatEach(4, cycle(floatValues)), 100))), + ImmutableList.of(Stream.generate(() -> floatValues) + .flatMap(Collection::stream) + .flatMap(item -> repeat(4, item)) + .limit(100) + .toList(), + Stream.generate(() -> floatValues) + .flatMap(Collection::stream) + .flatMap(item -> repeat(4, item)) + .limit(100) + .toList()), toSubfieldFilters( ImmutableMap.of( 0, FloatRange.of(1.0f, false, true, 7.0f, false, true, true), @@ -378,7 +417,9 @@ public void testFilterOrder() Random random = new Random(0); tester.testRoundTripTypesWithOrder(ImmutableList.of(INTEGER, INTEGER), - ImmutableList.of(newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11)), NUM_ROWS)), randomIntegers(NUM_ROWS, random)), + ImmutableList.of( + Stream.generate(() -> ImmutableList.of(1, 3, 5, 7, 11)).flatMap(Collection::stream).limit(NUM_ROWS).toList(), + randomIntegers(NUM_ROWS, random)), toSubfieldFilters( ImmutableMap.of( 0, BigintRange.of(1, 1, true), @@ -386,7 +427,9 @@ public void testFilterOrder() ImmutableList.of(ImmutableList.of(0, 1))); tester.testRoundTripTypesWithOrder(ImmutableList.of(INTEGER, INTEGER), - ImmutableList.of(newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11)), NUM_ROWS)), randomIntegers(NUM_ROWS, random)), + ImmutableList.of( + Stream.generate(() -> ImmutableList.of(1, 3, 5, 7, 11)).flatMap(Collection::stream).limit(NUM_ROWS).toList(), + randomIntegers(NUM_ROWS, random)), toSubfieldFilters( ImmutableMap.of( 0, BigintRange.of(100, 100, false), @@ -395,7 +438,10 @@ public void testFilterOrder() tester.testRoundTripTypesWithOrder( ImmutableList.of(INTEGER, INTEGER, DOUBLE, arrayType(INTEGER)), - ImmutableList.of(newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11)), NUM_ROWS)), createList(NUM_ROWS, i -> random.nextInt(200)), doubleSequence(0, 0.1, NUM_ROWS), nCopies(NUM_ROWS, randomIntegers(5, random))), + ImmutableList.of( + Stream.generate(() -> ImmutableList.of(1, 3, 5, 7, 11)).flatMap(Collection::stream).limit(NUM_ROWS).toList(), + createList(NUM_ROWS, i -> random.nextInt(200)), doubleSequence(0, 0.1, NUM_ROWS), + nCopies(NUM_ROWS, randomIntegers(5, random))), ImmutableList.of( ImmutableMap.of( 0, toSubfieldFilter(BigintRange.of(1, 1, true)), @@ -732,7 +778,7 @@ public void testMapsWithSubfieldPruning() private static Map createMap(int seed) { int mapSize = Math.abs(seed) % 7 + 1; - return IntStream.range(0, mapSize).boxed().collect(toImmutableMap(Function.identity(), i -> i + seed)); + return IntStream.range(0, mapSize).boxed().collect(toImmutableMap(identity(), i -> i + seed)); } private static List createList(int size, Function createElement) @@ -789,7 +835,7 @@ public void testVarchars() 2, stringIn(true, "def", "abc")))); // direct and dictionary - tester.testRoundTrip(VARCHAR, newArrayList(limit(cycle(ImmutableList.of("apple", "apple pie", "apple\uD835\uDC03", "apple\uFFFD")), NUM_ROWS)), + tester.testRoundTrip(VARCHAR, Stream.generate(() -> ImmutableList.of("apple", "apple pie", "apple\uD835\uDC03", "apple\uFFFD")).flatMap(Collection::stream).limit(NUM_ROWS).toList(), stringIn(false, "apple", "apple pie")); // direct and dictionary materialized @@ -804,25 +850,28 @@ public void testVarchars() ImmutableList.of(VARCHAR, VARCHAR), ImmutableList.of( intsBetween(0, NUM_ROWS).stream().map(Object::toString).collect(toList()), - newArrayList(limit(cycle(ImmutableList.of("A", "B", "C")), NUM_ROWS))), + Stream.generate(() -> ImmutableList.of("A", "B", "C")).flatMap(Collection::stream).limit(NUM_ROWS).toList()), toSubfieldFilters(ImmutableMap.of( 0, stringBetween(true, "16", "10"), 1, stringBetween(false, "B", "A")))); //stripe dictionary - tester.testRoundTrip(VARCHAR, newArrayList(concat(ImmutableList.of("a"), nCopies(9999, "123"), ImmutableList.of("b"), nCopies(9999, "123")))); + tester.testRoundTrip(VARCHAR, Streams.concat( + ImmutableList.of("a").stream(), nCopies(9999, "123").stream(), + ImmutableList.of("b").stream(), nCopies(9999, "123").stream()).toList()); //empty sequence tester.testRoundTrip(VARCHAR, nCopies(NUM_ROWS, ""), stringEquals(false, "")); // copy of AbstractOrcTester::testDwrfInvalidCheckpointsForRowGroupDictionary - List values = newArrayList(limit( - cycle(concat( - ImmutableList.of(1), nCopies(9999, 123), - ImmutableList.of(2), nCopies(9999, 123), - ImmutableList.of(3), nCopies(9999, 123), - nCopies(1_000_000, null))), - 200_000)); + List values = Stream.generate(() -> Streams.concat( + ImmutableList.of(1).stream(), nCopies(9999, 123).stream(), + ImmutableList.of(2).stream(), nCopies(9999, 123).stream(), + ImmutableList.of(3).stream(), nCopies(9999, 123).stream(), + nCopies(1_000_000, (Integer) null).stream())) + .flatMap(identity()) + .limit(200_000) + .toList(); tester.assertRoundTrip( VARCHAR, @@ -833,7 +882,8 @@ public void testVarchars() //copy of AbstractOrcTester::testDwrfInvalidCheckpointsForStripeDictionary tester.testRoundTrip( VARCHAR, - newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), 200_000)).stream() + Stream.generate(() -> ImmutableList.of(1, 3, 5, 7, 11, 13, 17)).flatMap(Collection::stream) + .limit(200_000) .map(Object::toString) .collect(toList())); @@ -857,15 +907,15 @@ public void testChars() // multiple columns filter on not null tester.testRoundTripTypes(ImmutableList.of(VARCHAR, createCharType(5)), ImmutableList.of( - newArrayList(limit(cycle(ImmutableList.of("123456789", "23456789", "3456789")), NUM_ROWS)), - newArrayList(limit(cycle(ImmutableList.of("12345", "23456", "34567")), NUM_ROWS))), + Stream.generate(() -> ImmutableList.of("123456789", "23456789", "3456789")).flatMap(Collection::stream).limit(NUM_ROWS).toList(), + Stream.generate(() -> ImmutableList.of("12345", "23456", "34567")).flatMap(Collection::stream).limit(NUM_ROWS).toList()), toSubfieldFilters(ImmutableMap.of(0, IS_NOT_NULL))); - tester.testRoundTrip(createCharType(2), newArrayList(limit(cycle(ImmutableList.of("aa", "bb", "cc", "dd")), NUM_ROWS)), IS_NULL); + tester.testRoundTrip(createCharType(2), Stream.generate(() -> ImmutableList.of("aa", "bb", "cc", "dd")).flatMap(Collection::stream).limit(NUM_ROWS).toList(), IS_NULL); tester.testRoundTrip( createCharType(1), - newArrayList(limit(cycle(ImmutableList.of("a", "b", "c", "d")), NUM_ROWS)), + Stream.generate(() -> ImmutableList.of("a", "b", "c", "d")).flatMap(Collection::stream).limit(NUM_ROWS).toList(), stringIn(false, "a", "b"), stringIn(true, "a", "b")); @@ -878,7 +928,9 @@ public void testChars() // char with filter // The values are padded with trailing spaces so that they are all 10 characters long - List values = newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), NUM_ROWS)).stream() + List values = Stream.generate(() -> ImmutableList.of(1, 3, 5, 7, 11, 13, 17)) + .flatMap(Collection::stream) + .limit(NUM_ROWS) .map(i -> toCharValue(i, 10)) .collect(toList()); // Test with filter c IN {'1','3'}. Note that the values should not have any paddings since the filter passed to the reader is without paddings for CHAR(n). @@ -890,11 +942,13 @@ public void testChars() ImmutableList.of(ImmutableMap.of(new Subfield("c"), stringIn(true, toCharValue(1, 10), toCharValue(3, 10))))); // char with 0 truncated length - tester.testRoundTrip(CHAR_10, newArrayList(limit(cycle(toCharValue("", 10)), NUM_ROWS))); + tester.testRoundTrip(CHAR_10, Stream.generate(() -> toCharValue("", 10)).limit(NUM_ROWS).toList()); tester.testRoundTrip( VARCHAR, - newArrayList(concat(ImmutableList.of("a"), nCopies(9999, "123"), ImmutableList.of("b"), nCopies(9999, "123"))), + Streams.concat( + ImmutableList.of("a").stream(), nCopies(9999, "123").stream(), + ImmutableList.of("b").stream(), nCopies(9999, "123").stream()).toList(), ImmutableList.of(ImmutableMap.of(new Subfield("c"), stringIn(false, "a", "b"))), ImmutableList.of(ImmutableMap.of(new Subfield("c"), stringIn(false, "a", "b")))); } @@ -912,7 +966,9 @@ public void testVarBinaries() ImmutableList.of(VARBINARY, VARBINARY), ImmutableList.of( createList(NUM_ROWS, i -> new SqlVarbinary(Ints.toByteArray(i))), - Streams.stream(limit(cycle(ImmutableList.of("A", "B", "C")), NUM_ROWS)) + Stream.generate(() -> ImmutableList.of("A", "B", "C")) + .flatMap(Collection::stream) + .limit(NUM_ROWS) .map(String::getBytes) .map(SqlVarbinary::new) .collect(toImmutableList())), @@ -932,7 +988,9 @@ public void testVarBinaries() tester.testRoundTrip( VARBINARY, - ImmutableList.copyOf(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), NUM_ROWS)).stream() + Stream.generate(() -> ImmutableList.of(1, 3, 5, 7, 11, 13, 17)) + .flatMap(Collection::stream) + .limit(NUM_ROWS) .map(String::valueOf) .map(string -> string.getBytes(UTF_8)) .map(SqlVarbinary::new) @@ -954,15 +1012,16 @@ private void testMemoryTracking(CompressionKind compression, long lowerRetainedM { List types = ImmutableList.of(INTEGER, VARCHAR, VARCHAR); TempFile tempFile = new TempFile(); - List intValues = newArrayList(limit( - cycle(concat( - ImmutableList.of(1), nCopies(9999, 123), - ImmutableList.of(2), nCopies(9999, 123), - ImmutableList.of(3), nCopies(9999, 123), - nCopies(1_000_000, null))), - NUM_ROWS)); - List varcharDirectValues = newArrayList(limit(cycle(ImmutableList.of("A", "B", "C")), NUM_ROWS)); - List varcharDictionaryValues = newArrayList(limit(cycle(ImmutableList.of("apple", "apple pie", "apple\uD835\uDC03", "apple\uFFFD")), NUM_ROWS)); + List intValues = Stream.generate(() -> Streams.concat( + ImmutableList.of(1).stream(), nCopies(9999, 123).stream(), + ImmutableList.of(2).stream(), nCopies(9999, 123).stream(), + ImmutableList.of(3).stream(), nCopies(9999, 123).stream(), + nCopies(1_000_000, (Integer) null).stream())) + .flatMap(identity()) + .limit(NUM_ROWS) + .toList(); + List varcharDirectValues = Stream.generate(() -> ImmutableList.of("A", "B", "C")).flatMap(Collection::stream).limit(NUM_ROWS).toList(); + List varcharDictionaryValues = Stream.generate(() -> ImmutableList.of("apple", "apple pie", "apple\uD835\uDC03", "apple\uFFFD")).flatMap(Collection::stream).limit(NUM_ROWS).toList(); List> values = ImmutableList.of(intValues, varcharDirectValues, varcharDictionaryValues); writeOrcColumnsPresto(tempFile.getFile(), DWRF, compression, Optional.empty(), types, values, NOOP_WRITER_STATS); @@ -970,7 +1029,7 @@ private void testMemoryTracking(CompressionKind compression, long lowerRetainedM OrcPredicate orcPredicate = createOrcPredicate(types, values, DWRF, false); Map includedColumns = IntStream.range(0, types.size()) .boxed() - .collect(toImmutableMap(Function.identity(), types::get)); + .collect(toImmutableMap(identity(), types::get)); List outputColumns = IntStream.range(0, types.size()) .boxed() .collect(toImmutableList()); @@ -1018,7 +1077,8 @@ private void testMemoryTracking(CompressionKind compression, long lowerRetainedM } @Test - public void testToZeroBasedColumnIndex() throws Exception + public void testToZeroBasedColumnIndex() + throws Exception { List types = ImmutableList.of(INTEGER, INTEGER, INTEGER); TempFile tempFile = new TempFile(); @@ -1060,7 +1120,7 @@ public void testToZeroBasedColumnIndex() throws Exception public void testOutputNotRequired() throws Exception { - List inputValues = newArrayList(limit(cycle(ImmutableList.of("A", "B", "C")), NUM_ROWS)); + List inputValues = Stream.generate(() -> ImmutableList.of("A", "B", "C")).flatMap(Collection::stream).limit(NUM_ROWS).toList(); Map filters = ImmutableMap.of(new Subfield("c"), stringIn(true, "A", "B", "C")); verifyOutputNotRequired(inputValues, filters, inputValues); } @@ -1069,8 +1129,11 @@ public void testOutputNotRequired() public void testOutputNotRequiredNonNullFilterNulls() throws Exception { - List inputValues = newArrayList(limit(cycle(Arrays.asList("A", null)), NUM_ROWS)); - List expectedValues = newArrayList(limit(cycle(Arrays.asList("A")), (NUM_ROWS + 1) / 2)); + List inputValues = Stream.generate(() -> Arrays.asList("A", null)) + .flatMap(Collection::stream) + .limit(NUM_ROWS) + .toList(); + List expectedValues = Stream.generate(() -> "A").limit((NUM_ROWS + 1) / 2).toList(); Map filters = ImmutableMap.of(new Subfield("c"), IS_NOT_NULL); verifyOutputNotRequired(inputValues, filters, expectedValues); @@ -1080,7 +1143,10 @@ public void testOutputNotRequiredNonNullFilterNulls() public void testOutputNotRequiredNonNullFilterNoNulls() throws Exception { - List inputValues = newArrayList(limit(cycle(Arrays.asList("A", "B", "C")), NUM_ROWS)); + List inputValues = Stream.generate(() -> Arrays.asList("A", "B", "C")) + .flatMap(Collection::stream) + .limit(NUM_ROWS) + .toList(); Map filters = ImmutableMap.of(new Subfield("c"), IS_NOT_NULL); verifyOutputNotRequired(inputValues, filters, inputValues); @@ -1098,7 +1164,7 @@ private void verifyOutputNotRequired(List inputValues, Map includedColumns = IntStream.range(0, types.size()) .boxed() - .collect(toImmutableMap(Function.identity(), types::get)); + .collect(toImmutableMap(identity(), types::get)); // Do not output column 0 but only column 1 List outputColumns = ImmutableList.of(1); @@ -1376,31 +1442,9 @@ protected T computeNext() }; } - private static Iterable repeatEach(int n, Iterable iterable) + private static Stream repeat(int n, T item) { - return () -> new AbstractIterator() - { - private final Iterator delegate = iterable.iterator(); - private int position; - private T value; - - @Override - protected T computeNext() - { - if (position == 0) { - if (!delegate.hasNext()) { - return endOfData(); - } - value = delegate.next(); - } - - position++; - if (position >= n) { - position = 0; - } - return value; - } - }; + return Stream.generate(() -> item).limit(n); } private static String toCharValue(Object value, int minLength) diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestStorageOrcFileTailSource.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestStorageOrcFileTailSource.java index 582d17b850bd4..16266fb87f88d 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestStorageOrcFileTailSource.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestStorageOrcFileTailSource.java @@ -168,7 +168,7 @@ public void testReadDwrfStripeCacheIfEnabled() // make sure the stripe cache is loaded correctly assertTrue(orcFileTail.getDwrfStripeCacheData().isPresent()); - DwrfStripeCacheData dwrfStripeCacheData = orcFileTail.getDwrfStripeCacheData().get(); + DwrfStripeCacheData dwrfStripeCacheData = orcFileTail.getDwrfStripeCacheData().orElseThrow(); assertEquals(dwrfStripeCacheData.getDwrfStripeCacheMode(), INDEX_AND_FOOTER); assertEquals(dwrfStripeCacheData.getDwrfStripeCacheSize(), stripeCache.length); assertEquals(dwrfStripeCacheData.getDwrfStripeCacheSlice().getBytes(), stripeCache); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestStreamLayout.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestStreamLayout.java index d148898233ed4..16e63cb7c6a50 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestStreamLayout.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestStreamLayout.java @@ -682,7 +682,7 @@ private static void assertFileStreamsOrder(Object[] orderedKeys, MapType mapType // get sequence to key mapping for flat map value node int node = 3; // map value node ColumnEncoding columnEncoding = stripeFooter.getColumnEncodings().get(node); - SortedMap nodeSequences = columnEncoding.getAdditionalSequenceEncodings().get(); + SortedMap nodeSequences = columnEncoding.getAdditionalSequenceEncodings().orElseThrow(); ImmutableMap.Builder keyToSequenceBuilder = ImmutableMap.builder(); for (Map.Entry entry : nodeSequences.entrySet()) { long key = entry.getValue().getKey().getIntKey(); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestTimestampWriteAndRead.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestTimestampWriteAndRead.java index 96e9325d91826..f5ce245e6e626 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestTimestampWriteAndRead.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestTimestampWriteAndRead.java @@ -91,7 +91,7 @@ public void testMicroWriteAndMilliRead() List microSecondValuesInMilli = MICROSECOND_VALUES.stream() .map(microTimestamp -> new SqlTimestamp( floorDiv(microTimestamp.getMicrosUtc(), 1000), - microTimestamp.getSessionTimeZoneKey().get(), + microTimestamp.getSessionTimeZoneKey().orElseThrow(), TimeUnit.MILLISECONDS)) .collect(toList()); @@ -213,7 +213,7 @@ private List getMilliTimestampsInMicros(List millise return millisecondValues.stream() .map(milliTimestamp -> new SqlTimestamp( milliTimestamp.getMillisUtc() * 1000, - milliTimestamp.getSessionTimeZoneKey().get(), + milliTimestamp.getSessionTimeZoneKey().orElseThrow(), MICROSECONDS)) .collect(toList()); } diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestingOrcPredicate.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestingOrcPredicate.java index a1ec15c68d775..86de366cdbca1 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestingOrcPredicate.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestingOrcPredicate.java @@ -24,7 +24,6 @@ import com.facebook.presto.orc.OrcTester.Format; import com.facebook.presto.orc.metadata.statistics.ColumnStatistics; import com.facebook.presto.orc.metadata.statistics.HiveBloomFilter; -import com.google.common.collect.Iterables; import com.google.common.collect.Ordering; import io.airlift.slice.Slice; import io.airlift.slice.Slices; @@ -54,7 +53,6 @@ import static com.google.common.base.Predicates.equalTo; import static com.google.common.base.Predicates.notNull; import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.Iterables.filter; import static com.google.common.collect.Lists.newArrayList; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; @@ -236,7 +234,7 @@ private void assertChunkStats(List chunk, ColumnStatistics columnStatistics) protected boolean chunkMatchesStats(List chunk, ColumnStatistics columnStatistics) { // verify non null count - if (columnStatistics.getNumberOfValues() != Iterables.size(filter(chunk, notNull()))) { + if (columnStatistics.getNumberOfValues() != chunk.stream().filter(notNull()).count()) { return false; } @@ -267,7 +265,7 @@ protected boolean chunkMatchesStats(List chunk, ColumnStatistics column // statistics can be missing for any reason if (columnStatistics.getBooleanStatistics() != null) { - if (columnStatistics.getBooleanStatistics().getTrueValueCount() != Iterables.size(filter(chunk, equalTo(Boolean.TRUE)))) { + if (columnStatistics.getBooleanStatistics().getTrueValueCount() != chunk.stream().filter(equalTo(Boolean.TRUE)).count()) { return false; } } diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/metadata/TestDwrfMetadataReader.java b/presto-orc/src/test/java/com/facebook/presto/orc/metadata/TestDwrfMetadataReader.java index dad519788cb9c..93eee1805b932 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/metadata/TestDwrfMetadataReader.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/metadata/TestDwrfMetadataReader.java @@ -88,7 +88,7 @@ public void testReadPostScript() assertEquals(postScript.getCompression(), CompressionKind.ZSTD); assertEquals(postScript.getCompressionBlockSize(), compressionBlockSize); assertEquals(postScript.getDwrfStripeCacheLength().getAsInt(), 12); - assertEquals(postScript.getDwrfStripeCacheMode().get(), DwrfStripeCacheMode.INDEX_AND_FOOTER); + assertEquals(postScript.getDwrfStripeCacheMode().orElseThrow(), DwrfStripeCacheMode.INDEX_AND_FOOTER); } @Test @@ -195,7 +195,7 @@ public void testReadFooter() assertEquals(footer.getNumberOfRows(), numberOfRows); assertEquals(footer.getRowsInRowGroup(), rowIndexStride); - assertEquals(footer.getDwrfStripeCacheOffsets().get(), stripeCacheOffsets); + assertEquals(footer.getDwrfStripeCacheOffsets().orElseThrow(), stripeCacheOffsets); assertEquals(footer.getRawSize(), rawDataSize); assertEquals(footer.getStripes(), Collections.emptyList()); } diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/metadata/statistics/TestMapColumnStatisticsBuilder.java b/presto-orc/src/test/java/com/facebook/presto/orc/metadata/statistics/TestMapColumnStatisticsBuilder.java index 9b2d028d3d7e9..29801f2b0dee3 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/metadata/statistics/TestMapColumnStatisticsBuilder.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/metadata/statistics/TestMapColumnStatisticsBuilder.java @@ -128,7 +128,7 @@ public void testMergeMapStatistics(KeyInfo[] keys) builder2.increaseValueCount(16); ColumnStatistics columnStatistics2 = builder2.buildColumnStatistics(); - MapStatistics mergedMapStatistics = MapColumnStatisticsBuilder.mergeMapStatistics(ImmutableList.of(columnStatistics1, columnStatistics2), null).get(); + MapStatistics mergedMapStatistics = MapColumnStatisticsBuilder.mergeMapStatistics(ImmutableList.of(columnStatistics1, columnStatistics2), null).orElseThrow(); assertMergedMapStatistics(keys, mergedMapStatistics); } diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/writer/TestDictionaryRowGroupBuilder.java b/presto-orc/src/test/java/com/facebook/presto/orc/writer/TestDictionaryRowGroupBuilder.java index 819b6ec8e2dcb..42b82edae1d5f 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/writer/TestDictionaryRowGroupBuilder.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/writer/TestDictionaryRowGroupBuilder.java @@ -90,7 +90,7 @@ public void testIntegerIndexes() } } - @Test(expectedExceptions = {IllegalStateException.class}) + @Test(expectedExceptions = IllegalStateException.class) public void testDecreasingMaxThrows() { DictionaryRowGroupBuilder rowGroupBuilder = new DictionaryRowGroupBuilder();