Add fuzzer for async data cache

velox/common/caching/AsyncDataCache.cpp

@@ -25,6 +25,15 @@
 #include "velox/common/base/SuccinctPrinter.h"
 #include "velox/common/caching/FileIds.h"
 
+#define VELOX_CACHE_ERROR(errorMessage)                             \
+  _VELOX_THROW(                                                     \
+      ::facebook::velox::VeloxRuntimeError,                         \
+      ::facebook::velox::error_source::kErrorSourceRuntime.c_str(), \
+      ::facebook::velox::error_code::kNoCacheSpace.c_str(),         \
+      /* isRetriable */ true,                                       \
+      "{}",                                                         \
+      errorMessage);
+
 namespace facebook::velox::cache {
 
 using memory::MachinePageCount;
@@ -120,13 +129,10 @@ void AsyncDataCacheEntry::initialize(FileCacheKey key) {
     } else {
       // No memory to cover 'this'.
       release();
-      _VELOX_THROW(
-          VeloxRuntimeError,
-          error_source::kErrorSourceRuntime.c_str(),
-          error_code::kNoCacheSpace.c_str(),
-          /* isRetriable */ true,
-          "Failed to allocate {} bytes for cache",
-          size_);
+      VELOX_CACHE_ERROR(fmt::format(
+          "Failed to allocate {} bytes for cache: {}",
+          size_,
+          cache->allocator()->getAndClearFailureMessage()));
     }
   }
 }

velox/docs/develop/testing/async-data-cache-fuzzer.rst

@@ -0,0 +1,63 @@
+============
+Cache Fuzzer
+============
+
+Cache fuzzer is designed to test the correctness and the reliability of the
+in-memory async data cache and the durable local SSD cache, and their
+interactions such as staging from async data cache to SSD cache, and load the
+cache miss data from SSD cache into async data cache.
+
+During each iteration, the fuzzer performs the following actions steps by steps:
+1. Creating a set of data files on local file system with varying sizes as source data files.
+2. Setting up the async data cache with and without SSD using a specific configuration.
+3. Performing parallel random reads from the source data files created in step1.
+
+How to run
+----------
+
+Use velox_cache_fuzzer_test binary to run cache fuzzer:
+
+::
+
+    velox/exec/tests/velox_cache_fuzzer_test
+
+By default, the fuzzer will go through 10 interations. Use --steps
+or --duration-sec flag to run fuzzer for longer. Use --seed to
+reproduce fuzzer failures.
+
+Here is a full list of supported command line arguments.
+
+* ``–-steps``: How many iterations to run. Each iteration generates and
+  evaluates one tale writer plan. Default is 10.
+
+* ``–-duration_sec``: For how long to run in seconds. If both ``-–steps``
+  and ``-–duration_sec`` are specified, –duration_sec takes precedence.
+
+* ``–-seed``: The seed to generate random expressions and input vectors with.
+
+* ``–-num_threads``: Number of read threads.
+
+* ``–-read_iteration_sec``: For how long each read thread should run (in seconds).
+
+* ``–-num_source_files``: Number of data files to be created.
+
+* ``–-min_source_file_bytes``: Minimum source file size in bytes.
+
+* ``–-max_source_file_bytes``: Maximum source file size in bytes.
+
+* ``–-memory_cache_bytes``: Memory cache size in bytes.
+
+* ``–-ssd_cache_bytes``: Ssd cache size in bytes.
+
+* ``–-num_ssd_cache_shards``: Number of SSD cache shards.
+
+* ``–-ssd_checkpoint_interval_bytes``: Checkpoint after every
+  ``--ssd_checkpoint_interval_bytes``/``--num_ssd_cache_shards`` written into
+  each file. 0 means no checkpointing.
+
+* ``–-enable_checksum``: Enable checksum write to SSD.
+
+* ``–-enable_checksum_read_verification``: Enable checksum read verification
+  from SSD.
+
+If running from CLion IDE, add ``--logtostderr=1`` to see the full output.

velox/exec/fuzzer/CMakeLists.txt

@@ -104,3 +104,8 @@ target_link_libraries(
   velox_expression_test_utility
   velox_functions_prestosql
   velox_aggregates)
+
+add_library(velox_cache_fuzzer CacheFuzzer.cpp)
+
+target_link_libraries(velox_cache_fuzzer velox_dwio_common velox_temp_path
+                      velox_vector_test_lib)