Add option to disable GC (#85)

src/benchmarking.jl

@@ -17,7 +17,7 @@ maybecall(::Nothing, x::Tuple{}) = x
 maybecall(f, x::Tuple{Any}) = (f(only(x)),)
 maybecall(f::Function, ::Tuple{}) = (f(),)
 maybecall(x, ::Tuple{}) = (x,)
-function benchmark(init, setup, f, teardown; evals::Union{Int, Nothing}=nothing, samples::Union{Int, Nothing}=nothing, seconds::Union{Real, Nothing}=samples===nothing ? .1 : 1, checksum::Bool=true, _map=(checksum ? default_map : Returns(nothing)), _reduction=default_reduction)
+function benchmark(init, setup, f, teardown; evals::Union{Int, Nothing}=nothing, samples::Union{Int, Nothing}=nothing, seconds::Union{Real, Nothing}=samples===nothing ? .1 : 1, gc::Bool=true, checksum::Bool=true, _map=(checksum ? default_map : Returns(nothing)), _reduction=default_reduction)
     @nospecialize
     samples !== nothing && evals === nothing && throw(ArgumentError("Sorry, we don't support specifying samples but not evals"))
     samples === seconds === nothing && throw(ArgumentError("Must specify either samples or seconds"))
@@ -29,7 +29,12 @@ function benchmark(init, setup, f, teardown; evals::Union{Int, Nothing}=nothing,
 
     function bench(evals, warmup=true)
         args2 = maybecall(setup, args1)
-        sample, t, args3 = _benchmark(f, _map, _reduction, args2, evals, warmup)
+        old_gc = gc || GC.enable(false)
+        sample, t, args3 = try
+            _benchmark(f, _map, _reduction, args2, evals, warmup)
+        finally
+            gc || GC.enable(old_gc)
+        end
         maybecall(teardown, (args3,))
         sample, t
     end

src/public.jl

@@ -99,6 +99,13 @@ arguments to ordinary functions. Keyword arguments to control executions are
     A reasonable effort is made to respect the time limit, but it is always exceeded by a
     small about (less than 1%) and can be significantly exceeded when benchmarking long
     running functions.
+  - `gc::Bool` An experimental option to disable garbage collection during benchmarking.
+    Defaults to `true`. Set to `false` to garbage collection during benchmarking. Disabling
+    garbage collection may cause out of memory errors during a benchmark that requires
+    garbage collection, but should not result in memory leaks that survive past the end of
+    the benchmark. As an experimental option, this may be removed in the future or its
+    semantics may change. This option also depends on Julia internals and so it may break in
+    future versions of Julia.
 
 # Interpolation
 

test/runtests.jl

@@ -72,6 +72,16 @@ using Chairmarks: Sample, Benchmark
             @test a.checksum == b.checksum
         end
 
+        @testset "gc=false" begin
+            a = @b rand(100, 10000, 100)
+            b = @b rand(100, 10000, 100) gc=true
+            c = @b rand(100, 10000, 100) gc=false
+            @test a.gc_fraction != 0
+            @test b.gc_fraction != 0
+            @test c.gc_fraction == 0
+            @test GC.enable(true)
+        end
+
         @testset "no warmup" begin
             runtime = @elapsed res = @be sleep(.1) seconds=.05
             @test runtime < .2 # hopefully this is not going to get too many false positives