Skip to content

Commit

Permalink
Modify TestDirectoryListCacheInvalidation to fix flakiness
Browse files Browse the repository at this point in the history
  • Loading branch information
agrawalreetika authored and tdcmeehan committed Apr 10, 2024
1 parent af58662 commit 0619997
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 18 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#
# WARNING
# ^^^^^^^
# This configuration file is for development only and should NOT be used be
# used in production. For example configuration, see the Presto documentation.
#

connector.name=hive-hadoop2
hive.metastore.uri=thrift://hadoop-master:9083
hive.metastore.thrift.client.socks-proxy=hadoop-master:1180
hive.config.resources=/docker/volumes/conf/presto/etc/hive-default-fs-site.xml
hive.allow-add-column=true
hive.allow-drop-column=true
hive.allow-rename-column=true
hive.allow-drop-table=true
hive.allow-rename-table=true
hive.metastore-cache-ttl=0s
hive.fs.cache.max-size=10
hive.max-partitions-per-scan=100
hive.collect-column-statistics-on-write=true

# List file cache
hive.file-status-cache-expire-time=24h
hive.file-status-cache-size=100000000
hive.file-status-cache-tables=*
Original file line number Diff line number Diff line change
Expand Up @@ -29,21 +29,20 @@ public class TestDirectoryListCacheInvalidation
@BeforeTestWithContext
public void setUp()
{
query("DROP TABLE IF EXISTS hivecached.default.region_cache");
query("CREATE TABLE hivecached.default.region_cache AS SELECT * FROM tpch.tiny.region");
query("DROP TABLE IF EXISTS hive_listcache.default.region_cache");
query("CREATE TABLE hive_listcache.default.region_cache AS SELECT * FROM tpch.tiny.region");
}

// Flaky test: https://github.com/prestodb/presto/issues/22425
@Test(groups = {HIVE_LIST_CACHING}, enabled = false)
@Test(groups = {HIVE_LIST_CACHING})
public void testDirectoryListCacheInvalidation()
{
String jmxMetricsQuery = "SELECT sum(hitcount), sum(misscount) from jmx.current.\"com.facebook.presto.hive:name=hivecached,type=cachingdirectorylister\"";
String regionQuery = "SELECT * FROM hivecached.default.region_cache";
String jmxMetricsQuery = "SELECT sum(hitcount), sum(misscount) from jmx.current.\"com.facebook.presto.hive:name=hive_listcache,type=cachingdirectorylister\"";
String regionQuery = "SELECT * FROM hive_listcache.default.region_cache";

// Initial cache entries, hitcount, misscount will all be zero
// Initial hitcount, misscount based on cache entries,
QueryResult queryResult = query(jmxMetricsQuery);
assertEquals((long) queryResult.row(0).get(0), 0L);
assertEquals((long) queryResult.row(0).get(1), 0L);
long hitCountInitial = (long) queryResult.row(0).get(0);
long missCountInitial = (long) queryResult.row(0).get(1);

for (int i = 0; i < 2; i++) {
query(regionQuery);
Expand All @@ -54,25 +53,25 @@ public void testDirectoryListCacheInvalidation()
long hitCount = (long) result.row(0).get(0);
long missCount = (long) result.row(0).get(1);

assertEquals(hitCount, 1L);
assertEquals(missCount, 1L);
assertEquals(hitCount, hitCountInitial + 1);
assertEquals(missCount, missCountInitial + 1);

// Invalidate directory list cache
query("CALL hivecached.system.invalidate_directory_list_cache()");
query("CALL hive_listcache.system.invalidate_directory_list_cache()");

query(regionQuery);
result = query(jmxMetricsQuery);

hitCount = (long) result.row(0).get(0);
missCount = (long) result.row(0).get(1);
// No results are cached, miss count would increase
assertEquals(hitCount, 1L);
assertEquals(missCount, 2L);
long hitCountAfter = (long) result.row(0).get(0);
long missCountAfter = (long) result.row(0).get(1);
// No results are cached, miss count would increase but hit count remains same
assertEquals(hitCountAfter, hitCount);
assertEquals(missCountAfter, missCount + 1);
}

@AfterTestWithContext
public void tearDown()
{
query("DROP TABLE IF EXISTS hivecached.default.region_cache");
query("DROP TABLE IF EXISTS hive_listcache.default.region_cache");
}
}

0 comments on commit 0619997

Please sign in to comment.