Adjust size estimate for encrypted entries

Add 170 bytes to the estimated size of each entry when encryption is
enabled. This assumes that the default context properties are used.

Rakefile

@@ -23,11 +23,7 @@ def run_without_aborting(*tasks)
 end
 
 def configs
-<<<<<<< HEAD
-  [ :default, :connects_to, :database, :no_database, :shards, :unprepared_statements ]
-=======
   [ :default, :connects_to, :database, :encrypted, :encrypted_custom, :no_database, :shards, :unprepared_statements ]
->>>>>>> e7305dc (Encrypt cache values via Solid Cache config)
 end
 
 task :test do

app/models/solid_cache/entry.rb

@@ -8,6 +8,10 @@ class Entry < Record
     # Based on experimentation on SQLite, MySQL and Postgresql.
     # A bit high for SQLite (more like 90 bytes), but about right for MySQL/Postgresql.
     ESTIMATED_ROW_OVERHEAD = 140
+
+    # Assuming MessagePack serialization
+    ESTIMATED_ENCRYPTION_OVERHEAD = 170
+
     KEY_HASH_ID_RANGE = -(2**63)..(2**63 - 1)
 
     class << self
@@ -149,7 +153,15 @@ def key_hashes_for(keys)
         end
 
         def byte_size_for(payload)
-          payload[:key].to_s.bytesize + payload[:value].to_s.bytesize + ESTIMATED_ROW_OVERHEAD
+          payload[:key].to_s.bytesize + payload[:value].to_s.bytesize + estimated_row_overhead
+        end
+
+        def estimated_row_overhead
+          if SolidCache.configuration.encrypt?
+            ESTIMATED_ROW_OVERHEAD + ESTIMATED_ENCRYPTION_OVERHEAD
+          else
+            ESTIMATED_ROW_OVERHEAD
+          end
         end
     end
   end

gemfiles/rails_main.gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/rails/rails.git
-  revision: d7f93473682d9b64ab268a6736d5af2853e97ce2
+  revision: abc4538300e8a2897f350611666d7e64021e6d16
   branch: main
   specs:
     actionpack (8.0.0.alpha)
@@ -66,15 +66,9 @@ GEM
       thor (>= 0.14.0)
     ast (2.4.2)
     base64 (0.2.0)
-<<<<<<< HEAD
     bigdecimal (3.1.8)
     builder (3.3.0)
     concurrent-ruby (1.3.4)
-=======
-    bigdecimal (3.1.7)
-    builder (3.2.4)
-    concurrent-ruby (1.2.3)
->>>>>>> e7305dc (Encrypt cache values via Solid Cache config)
     connection_pool (2.4.1)
     crass (1.0.6)
     debug (1.9.2)
@@ -96,47 +90,32 @@ GEM
     loofah (2.22.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
-<<<<<<< HEAD
     minitest (5.24.1)
     mocha (2.4.5)
       ruby2_keywords (>= 0.0.5)
     msgpack (1.7.2)
     mysql2 (0.5.6)
+    nokogiri (1.16.7-aarch64-linux)
+      racc (~> 1.4)
+    nokogiri (1.16.7-arm-linux)
+      racc (~> 1.4)
     nokogiri (1.16.7-arm64-darwin)
       racc (~> 1.4)
+    nokogiri (1.16.7-x86-linux)
+      racc (~> 1.4)
     nokogiri (1.16.7-x86_64-darwin)
       racc (~> 1.4)
     nokogiri (1.16.7-x86_64-linux)
       racc (~> 1.4)
     parallel (1.26.2)
     parser (3.3.4.2)
-=======
-    minitest (5.21.2)
-    mocha (2.2.0)
-      ruby2_keywords (>= 0.0.5)
-    msgpack (1.7.2)
-    mysql2 (0.5.6)
-    nokogiri (1.16.4-arm64-darwin)
-      racc (~> 1.4)
-    nokogiri (1.16.4-x86_64-darwin)
-      racc (~> 1.4)
-    nokogiri (1.16.4-x86_64-linux)
-      racc (~> 1.4)
-    parallel (1.24.0)
-    parser (3.3.1.0)
->>>>>>> e7305dc (Encrypt cache values via Solid Cache config)
       ast (~> 2.4.1)
       racc
     pg (1.5.7)
     psych (5.1.2)
       stringio
-<<<<<<< HEAD
     racc (1.8.1)
     rack (3.1.7)
-=======
-    racc (1.7.3)
-    rack (3.0.10)
->>>>>>> e7305dc (Encrypt cache values via Solid Cache config)
     rack-session (2.0.0)
       rack (>= 3.0.0)
     rack-test (2.1.0)
@@ -153,7 +132,6 @@ GEM
       nokogiri (~> 1.14)
     rainbow (3.1.1)
     rake (13.2.1)
-<<<<<<< HEAD
     rdoc (6.7.0)
       psych (>= 4.0.0)
     regexp_parser (2.9.2)
@@ -162,15 +140,6 @@ GEM
     rexml (3.3.5)
       strscan
     rubocop (1.65.1)
-=======
-    rdoc (6.6.3.1)
-      psych (>= 4.0.0)
-    regexp_parser (2.9.0)
-    reline (0.5.4)
-      io-console (~> 0.5)
-    rexml (3.2.6)
-    rubocop (1.63.4)
->>>>>>> e7305dc (Encrypt cache values via Solid Cache config)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -181,34 +150,19 @@ GEM
       rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-<<<<<<< HEAD
     rubocop-ast (1.32.0)
       parser (>= 3.3.1.0)
     rubocop-md (1.2.2)
       rubocop (>= 1.0)
     rubocop-minitest (0.35.1)
-=======
-    rubocop-ast (1.31.3)
-      parser (>= 3.3.1.0)
-    rubocop-md (1.2.2)
-      rubocop (>= 1.0)
-    rubocop-minitest (0.35.0)
->>>>>>> e7305dc (Encrypt cache values via Solid Cache config)
       rubocop (>= 1.61, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-<<<<<<< HEAD
     rubocop-performance (1.21.1)
       rubocop (>= 1.48.1, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-rails (2.25.1)
-=======
-    rubocop-performance (1.21.0)
-      rubocop (>= 1.48.1, < 2.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
-    rubocop-rails (2.24.1)
->>>>>>> e7305dc (Encrypt cache values via Solid Cache config)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
       rubocop (>= 1.33.0, < 2.0)
@@ -223,18 +177,18 @@ GEM
       actionpack (>= 6.1)
       activesupport (>= 6.1)
       sprockets (>= 3.0.0)
-<<<<<<< HEAD
+    sqlite3 (2.0.3-aarch64-linux-gnu)
+    sqlite3 (2.0.3-aarch64-linux-musl)
+    sqlite3 (2.0.3-arm-linux-gnu)
+    sqlite3 (2.0.3-arm-linux-musl)
     sqlite3 (2.0.3-arm64-darwin)
+    sqlite3 (2.0.3-x86-linux-gnu)
+    sqlite3 (2.0.3-x86-linux-musl)
     sqlite3 (2.0.3-x86_64-darwin)
     sqlite3 (2.0.3-x86_64-linux-gnu)
+    sqlite3 (2.0.3-x86_64-linux-musl)
     stringio (3.1.1)
     strscan (3.1.0)
-=======
-    sqlite3 (2.0.1-arm64-darwin)
-    sqlite3 (2.0.1-x86_64-darwin)
-    sqlite3 (2.0.1-x86_64-linux-gnu)
-    stringio (3.1.0)
->>>>>>> e7305dc (Encrypt cache values via Solid Cache config)
     thor (1.3.1)
     timeout (0.4.1)
     tzinfo (2.0.6)
@@ -245,12 +199,20 @@ GEM
     zeitwerk (2.6.17)
 
 PLATFORMS
-  arm64-darwin-21
-  arm64-darwin-22
-  arm64-darwin-23
-  x86_64-darwin-20
-  x86_64-darwin-22
+  aarch64-linux
+  aarch64-linux-gnu
+  aarch64-linux-musl
+  arm-linux
+  arm-linux-gnu
+  arm-linux-musl
+  arm64-darwin
+  x86-linux
+  x86-linux-gnu
+  x86-linux-musl
+  x86_64-darwin
   x86_64-linux
+  x86_64-linux-gnu
+  x86_64-linux-musl
 
 DEPENDENCIES
   appraisal
@@ -271,4 +233,4 @@ DEPENDENCIES
   sqlite3
 
 BUNDLED WITH
-   2.5.9
+   2.5.17

test/models/solid_cache/entry/size/estimate_test.rb

@@ -4,28 +4,32 @@
 
 module SolidCache
   class EntrySizeEstimateTest < ActiveSupport::TestCase
+    setup do
+      @encrypted = SolidCache.configuration.encrypt?
+    end
+
     test "write and read cache entries" do
       assert_equal 0, estimate(samples: 10)
     end
 
     test "gets exact estimate when samples sizes are big enough" do
       write_entries(value_lengths: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ])
 
-      assert_equal 1535, estimate(samples: 12)
-      assert_equal 1535, estimate(samples: 10)
-      assert_equal 1688, estimate(samples: 6)
-      assert_equal 1689, estimate(samples: 5)
+      assert_equal @encrypted ? 3235 : 1535, estimate(samples: 12)
+      assert_equal @encrypted ? 3235 : 1535, estimate(samples: 10)
+      assert_equal @encrypted ? 3558 : 1688, estimate(samples: 6)
+      assert_equal @encrypted ? 3559 : 1689, estimate(samples: 5)
     end
 
     test "test larger sample estimates" do
       values_lengths = with_fixed_srand(1) { 1000.times.map { (rand**2 * 1000).to_i } }
       write_entries(value_lengths: values_lengths)
 
-      assert_equal 481257, estimate(samples: 1000)
-      assert_equal 481662, estimate(samples: 500)
-      with_fixed_srand(1) { assert_equal 501624, estimate(samples: 100) }
-      with_fixed_srand(1) { assert_equal 477621, estimate(samples: 50) }
-      with_fixed_srand(1) { assert_equal 471878, estimate(samples: 10) }
+      assert_equal @encrypted ? 651257 : 481257, estimate(samples: 1000)
+      assert_equal @encrypted ? 651832 : 481662, estimate(samples: 500)
+      with_fixed_srand(1) { assert_equal @encrypted ? 680804 : 501624, estimate(samples: 100) }
+      with_fixed_srand(1) { assert_equal @encrypted ? 660541 : 477621, estimate(samples: 50) }
+      with_fixed_srand(1) { assert_equal @encrypted ? 692368 : 471878, estimate(samples: 10) }
     end
 
     test "test with gaps in records estimates" do
@@ -34,12 +38,12 @@ class EntrySizeEstimateTest < ActiveSupport::TestCase
       first_mod = Entry.first.id % 3
       Entry.where("id % 3 = #{first_mod}").delete_all
 
-      assert_equal 324532, estimate(samples: 1000)
-      assert_equal 324741, estimate(samples: 500)
-      with_fixed_srand(1) { assert_equal 323946, estimate(samples: 334) }
-      with_fixed_srand(1) { assert_equal 345103, estimate(samples: 100) }
-      with_fixed_srand(1) { assert_equal 335770, estimate(samples: 50) }
-      with_fixed_srand(1) { assert_equal 281944, estimate(samples: 10) }
+      assert_equal @encrypted ? 437752 : 324532, estimate(samples: 1000)
+      assert_equal @encrypted ? 438131 : 324741, estimate(samples: 500)
+      with_fixed_srand(1) { assert_equal @encrypted ? 437166 : 323946, estimate(samples: 334) }
+      with_fixed_srand(1) { assert_equal @encrypted ? 462859 : 345103, estimate(samples: 100) }
+      with_fixed_srand(1) { assert_equal @encrypted ? 453859 : 335770, estimate(samples: 50) }
+      with_fixed_srand(1) { assert_equal @encrypted ? 401216 : 281944, estimate(samples: 10) }
     end
 
     test "test with more gaps in records estimates" do
@@ -48,12 +52,12 @@ class EntrySizeEstimateTest < ActiveSupport::TestCase
       first_mod = Entry.first.id % 4
       Entry.where("id % 4 != #{first_mod}").delete_all
 
-      assert_equal 120304, estimate(samples: 1000)
-      assert_equal 121488, estimate(samples: 500)
-      with_fixed_srand(1) { assert_equal 121188, estimate(samples: 250) }
-      with_fixed_srand(1) { assert_equal 126768, estimate(samples: 100) }
-      with_fixed_srand(1) { assert_equal 132657, estimate(samples: 50) }
-      with_fixed_srand(1) { assert_equal 25537, estimate(samples: 10) }
+      assert_equal @encrypted ? 162804 : 120304, estimate(samples: 1000)
+      assert_equal @encrypted ? 165348 : 121488, estimate(samples: 500)
+      with_fixed_srand(1) { assert_equal @encrypted ? 164704 : 121188, estimate(samples: 250) }
+      with_fixed_srand(1) { assert_equal @encrypted ? 174266 : 126768, estimate(samples: 100) }
+      with_fixed_srand(1) { assert_equal @encrypted ? 179794 : 132657, estimate(samples: 50) }
+      with_fixed_srand(1) { assert_equal @encrypted ? 44016 : 25537, estimate(samples: 10) }
     end
 
     test "overestimate when all samples sizes are the same" do
@@ -62,11 +66,11 @@ class EntrySizeEstimateTest < ActiveSupport::TestCase
       # estimate in this case.
       write_entries(value_lengths: [1] * 1000)
 
-      assert_equal 149000, estimate(samples: 1000)
-      assert_equal 297851, estimate(samples: 999)
-      assert_equal 223500, estimate(samples: 500)
-      with_fixed_srand(1) { assert_equal 272422, estimate(samples: 6) }
-      with_fixed_srand(1) { assert_equal 326906, estimate(samples: 5) }
+      assert_equal @encrypted ? 319000 : 149000, estimate(samples: 1000)
+      assert_equal @encrypted ? 637681 : 297851, estimate(samples: 999)
+      assert_equal @encrypted ? 478500 : 223500, estimate(samples: 500)
+      with_fixed_srand(1) { assert_equal @encrypted ? 583238 : 272422, estimate(samples: 6) }
+      with_fixed_srand(1) { assert_equal @encrypted ? 699886 : 326906, estimate(samples: 5) }
     end
 
     private

test/models/solid_cache/entry/size/moving_average_estimate_test.rb

@@ -4,6 +4,10 @@
 
 module SolidCache
   class EntrySizeMovingAverageEstimateTest < ActiveSupport::TestCase
+    setup do
+      @encrypted = SolidCache.configuration.encrypt?
+    end
+
     test "write and read cache entries" do
       assert_equal 0, estimate(samples: 10)
     end
@@ -13,7 +17,7 @@ class EntrySizeMovingAverageEstimateTest < ActiveSupport::TestCase
 
       estimate = Entry::Size::MovingAverageEstimate.new(samples: 12)
       assert_predicate estimate, :exact?
-      assert_equal 1535, estimate.size
+      assert_equal @encrypted ? 3235 : 1535, estimate.size
     end
 
     test "tracks moving average" do
@@ -22,24 +26,24 @@ class EntrySizeMovingAverageEstimateTest < ActiveSupport::TestCase
       Entry.write Entry::Size::MovingAverageEstimate::ESTIMATES_KEY, "4637774|4754378|7588547"
 
       with_fixed_srand(1) do
-        assert_equal 10449357, estimate(samples: 1)
+        assert_equal @encrypted ? 11016081 : 10449357, estimate(samples: 1)
       end
 
-      assert_equal "4754378|7588547|19005147", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
+      assert_equal @encrypted ? "4754378|7588547|20705317" : "4754378|7588547|19005147", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
     end
 
     test "appends to moving average when less than required items" do
       write_entries(value_lengths: 5000.times.to_a)
 
       assert_nil Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
 
-      with_fixed_srand(1) { assert_equal 20991897, estimate(samples: 2) }
+      with_fixed_srand(1) { assert_equal @encrypted ? 22691557 : 20991897, estimate(samples: 2) }
 
-      assert_equal "20991897", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
+      assert_equal @encrypted ? "22691557" : "20991897", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
 
-      with_fixed_srand(2) { assert_equal 11917062, estimate(samples: 2) }
+      with_fixed_srand(2) { assert_equal @encrypted ? 13191977 : 11917062, estimate(samples: 2) }
 
-      assert_equal "20991897|2842227", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
+      assert_equal @encrypted ? "22691557|3692397" : "20991897|2842227", Entry.read(Entry::Size::MovingAverageEstimate::ESTIMATES_KEY)
     end
 
     private