Skip to content

Commit

Permalink
Remove size tracking using ehcache
Browse files Browse the repository at this point in the history
We added detailed memory tracking in #19194 that makes size tracking using ehcache redundant
  • Loading branch information
aaneja committed Jan 9, 2025
1 parent 50200dd commit b4fcb0b
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 15 deletions.
5 changes: 0 additions & 5 deletions presto-parquet/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -192,11 +192,6 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.ehcache</groupId>
<artifactId>sizeof</artifactId>
<version>0.4.3</version>
</dependency>
</dependencies>

<build>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import org.apache.parquet.io.MessageColumnIO;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.PrimitiveType;
import org.ehcache.sizeof.SizeOf;
import org.testng.annotations.Test;

import java.io.IOException;
Expand Down Expand Up @@ -103,7 +102,6 @@ public void testMemoryUsedIsNotAFunctionOfDataPageCount()
//We set the expected max to 30% more than MAX_DATA_SOURCE_BUFFER_SIZE
long expectedMaxMemoryUsage = (long) (MAX_DATA_SOURCE_BUFFER_SIZE.toBytes() * 1.30);

SizeOf sizeOf = SizeOf.newInstance();
for (Integer testPageCount : Arrays.asList(1000, 5000, 10000, 20000)) {
//Create an input file with 1 row group, 1 column and a pre-determined number of data pages
TestParquetFileProperties parquetFile = createTestParquetFile(schema, testPageCount);
Expand All @@ -114,23 +112,16 @@ public void testMemoryUsedIsNotAFunctionOfDataPageCount()
long maxSystemMemoryUsed = 0;

ParquetReader parquetReader = parquetFile.getParquetReader();
long parquetReaderDeepSize = sizeOf.deepSizeOf(parquetReader);
long totalRowsRead = 0;
while (totalRowsRead < expectedRowCount) {
parquetReader.nextBatch();
int rowsRead = parquetReader.readBlock(col1).getPositionCount();
totalRowsRead += rowsRead;
//Check the max memory used during each batch read
parquetReaderDeepSize = Math.max(parquetReaderDeepSize, sizeOf.deepSizeOf(parquetReader));
//Same check will work on system memory context; the upper bound here can be 'tightened' more
maxSystemMemoryUsed = Math.max(maxSystemMemoryUsed, parquetReader.getSystemMemoryUsage());
}

String testAssertFormat = "[%d] pages :: %s :: actual [%d], expected < [%d]";
assertTrue(maxSystemMemoryUsed < expectedMaxMemoryUsage,
format(testAssertFormat, testPageCount, "maxSystemMemoryUsed", maxSystemMemoryUsed, expectedMaxMemoryUsage));
assertTrue(parquetReaderDeepSize < expectedMaxMemoryUsage,
format(testAssertFormat, testPageCount, "parquetReaderDeepSize", parquetReaderDeepSize, expectedMaxMemoryUsage));
format("[%d] pages :: %s :: actual [%d], expected < [%d]", testPageCount, "maxSystemMemoryUsed", maxSystemMemoryUsed, expectedMaxMemoryUsage));

parquetReader.close();
}
Expand Down

0 comments on commit b4fcb0b

Please sign in to comment.