Misc improvements to the @resumable macro

.gitignore

@@ -3,3 +3,4 @@
 *.jl.mem
 examples/.ipynb_checkpoints
 docs/build
+Manifest.toml

CHANGELOG.md

@@ -0,0 +1,53 @@
+# News
+
+## v0.6.4 - 2023-08-08
+
+- Docstrings now work with `@resumable` functions.
+- Generated types in `@resumable` functions have clearer names for better debugging.
+- Line-number nodes are preserved in `@resumable` functions for better debugging and code coverage.
+
+## Changelog before moving to JuliaDynamics
+
+* 2023: v0.6.3
+  * Julia 1.6 or newer is required
+  * introduction of `@yieldfrom` to delegate to another resumable function or iterator (similar to [Python's `yield from`](https://peps.python.org/pep-0380/))
+  * resumable functions are now allowed to return values, so that `r = @yieldfrom f` also stores the return value of `f` in `r`
+
+* 2023: v0.6.2 
+  * Julia v1.10 compatibility fix
+  * resumable functions can now dispatch on types
+
+* 2021: v0.6.1
+  * `continue` in loop works
+
+* 2021: v0.6.0
+  * introduction of `@nosave` to keep a variable out of the saved structure.
+  * optimized `for` loop.
+
+* 2020: v0.5.2 is Julia v1.6 compatible.
+
+* 2019: v0.5.1
+  * inference problem solved: force iterator next value to be of type `Union` of `Tuple` and `Nothing`.
+
+* 2019: v0.5.0 is Julia v1.2 compatible.
+
+* 2018: v0.4.2 prepare for Julia v1.1
+  * better inference caused a problem;).
+  * iterator with a specified `rtype` is fixed.
+
+* 2018: v0.4.0 is Julia v1.0 compatible.
+
+* 2018: v0.3.1 uses the new iteration protocol.
+  * the new iteration protocol is used for a `@resumable function` based iterator.
+  * the `for` loop transformation implements also the new iteration protocol.
+
+* 2018: v0.3 is Julia v0.7 compatible.
+  * introduction of `let` block to allow variables not te be persisted between `@resumable function` calls (EXPERIMENTAL).
+  * the `eltype` of a `@resumable function` based iterator is its return type if specified, otherwise `Any`.
+
+* 2018: v0.2 the iterator now behaves as a Python generator: only values that are explicitely yielded are generated; the return value is ignored and a warning is generated.
+
+* 2017: v0.1 initial release that is Julia v0.6 compatible:
+  * Introduction of the `@resumable` and the `@yield` macros.
+  * A `@resumable function` generates a type that implements the [iterator](https://docs.julialang.org/en/stable/manual/interfaces/#man-interface-iteration-1) interface.
+  * Parametric `@resumable functions` are supported.

Project.toml

@@ -5,7 +5,7 @@ license = "MIT"
 desc = "C# sharp style generators a.k.a. semi-coroutines for Julia."
 authors = ["Ben Lauwens <ben.lauwens@gmail.com>"]
 repo = "https://github.com/BenLauwens/ResumableFunctions.jl.git"
-version = "0.6.3"
+version = "0.6.4"
 
 [deps]
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"

README.md

@@ -29,6 +29,9 @@
     </tr>
 </table>
 
+[![status](http://joss.theoj.org/papers/889b2faed426b978ee705689c8f8440b/status.svg)](http://joss.theoj.org/papers/889b2faed426b978ee705689c8f8440b)
+[![DOI](https://zenodo.org/badge/100050892.svg)](https://zenodo.org/badge/latestdoi/100050892)
+
 [C#](https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/) has a convenient way to create iterators using the `yield return` statement. The package `ResumableFunctions` provides the same functionality for the [Julia language](https://julialang.org) by introducing the `@resumable` and the `@yield` macros. These macros can be used to replace the `Task` switching functions `produce` and `consume` which were deprecated in Julia v0.6. `Channels` are the preferred way for inter-task communication in julia v0.6+, but their performance is subpar for iterator applications. See [the benchmarks section below](#Benchmarks).
 
 ## Installation
@@ -115,12 +118,6 @@ Iteration protocol:
   5.822 μs (190 allocations: 4.44 KiB)
 ```
 
-## Licence & References
-
-[![License](http://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](LICENSE.md)
-[![status](http://joss.theoj.org/papers/889b2faed426b978ee705689c8f8440b/status.svg)](http://joss.theoj.org/papers/889b2faed426b978ee705689c8f8440b)
-[![DOI](https://zenodo.org/badge/100050892.svg)](https://zenodo.org/badge/latestdoi/100050892)
-
 ## Authors
 
 * Ben Lauwens, [Royal Military Academy](http://www.rma.ac.be), Brussels, Belgium.
@@ -131,54 +128,13 @@ Iteration protocol:
 * To discuss problems or feature requests, file an issue. For bugs, please include as much information as possible, including operating system, julia version, and version of [MacroTools](https://github.com/MikeInnes/MacroTools.jl.git).
 * To contribute, make a pull request. Contributions should include tests for any new features/bug fixes.
 
-## Release notes
-
-* 2023: v0.6.3
-  * Julia 1.6 or newer is required
-  * introduction of `@yieldfrom` to delegate to another resumable function or iterator (similar to [Python's `yield from`](https://peps.python.org/pep-0380/))
-  * resumable functions are now allowed to return values, so that `r = @yieldfrom f` also stores the return value of `f` in `r`
-
-* 2023: v0.6.2 
-  * Julia v1.10 compatibility fix
-  * resumable functions can now dispatch on types
-
-* 2021: v0.6.1
-  * `continue` in loop works
-
-* 2021: v0.6.0
-  * introduction of `@nosave` to keep a variable out of the saved structure.
-  * optimized `for` loop.
-
-* 2020: v0.5.2 is Julia v1.6 compatible.
-
-* 2019: v0.5.1
-  * inference problem solved: force iterator next value to be of type `Union` of `Tuple` and `Nothing`.
-
-* 2019: v0.5.0 is Julia v1.2 compatible.
-
-* 2018: v0.4.2 prepare for Julia v1.1
-  * better inference caused a problem;).
-  * iterator with a specified `rtype` is fixed.
-
-* 2018: v0.4.0 is Julia v1.0 compatible.
-
-* 2018: v0.3.1 uses the new iteration protocol.
-  * the new iteration protocol is used for a `@resumable function` based iterator.
-  * the `for` loop transformation implements also the new iteration protocol.
-
-* 2018: v0.3 is Julia v0.7 compatible.
-  * introduction of `let` block to allow variables not te be persisted between `@resumable function` calls (EXPERIMENTAL).
-  * the `eltype` of a `@resumable function` based iterator is its return type if specified, otherwise `Any`.
-
-* 2018: v0.2 the iterator now behaves as a Python generator: only values that are explicitely yielded are generated; the return value is ignored and a warning is generated.
+## Release Notes
 
-* 2017: v0.1 initial release that is Julia v0.6 compatible:
-  * Introduction of the `@resumable` and the `@yield` macros.
-  * A `@resumable function` generates a type that implements the [iterator](https://docs.julialang.org/en/stable/manual/interfaces/#man-interface-iteration-1) interface.
-  * Parametric `@resumable functions` are supported.
+A [detailed change log is kept](https://github.com/JuliaDynamics/ResumableFunctions.jl/blob/master/CHANGELOG.md).
 
 ## Caveats
 
 * In a `try` block only top level `@yield` statements are allowed.
 * In a `finally` block a `@yield` statement is not allowed.
 * An anonymous function can not contain a `@yield` statement.
+* Many more restrictions.

benchmark/benchmarks.jl

@@ -3,7 +3,7 @@ using ResumableFunctions
 
 const n = 93
 
-const N = BigInt
+const N = Int
 
 function direct(a::N, b::N)
   b, a+b
@@ -121,8 +121,8 @@ end
 @noinline function test_closure_opt(n::Int)
   fib_closure = fibonacci_closure_opt()
   a = 0
-  for _ in 1:n 
-    a = fib_closure() 
+  for _ in 1:n
+    a = fib_closure()
   end
   a
 end

src/macro.jl

@@ -40,7 +40,7 @@ macro resumable(expr::Expr)
   func_def[:body] = postwalk(transform_yieldfrom, func_def[:body])
   func_def[:body] = postwalk(x->transform_for(x, ui8), func_def[:body])
   slots = get_slots(copy(func_def), arg_dict, __module__)
-  type_name = gensym()
+  type_name = gensym(Symbol(func_def[:name], :_FSMI))
   constr_def = copy(func_def)
   if isempty(params)
     struct_name = :($type_name <: ResumableFunctions.FiniteStateMachineIterator{$rtype})
@@ -113,5 +113,9 @@ macro resumable(expr::Expr)
   func_def[:kwargs] = []
   func_expr = combinedef(func_def) |> flatten
   @debug func_expr|>MacroTools.striplines
-  esc(:($type_expr; $func_expr; $call_expr))
+  esc(quote
+    $type_expr
+    $func_expr
+    Base.@__doc__($call_expr)
+  end)
 end

src/transforms.jl

@@ -1,5 +1,5 @@
 """
-Function that replaces a variable 
+Function that replaces a variable
 """
 function transform_nosave(expr, nosaves::Set{Symbol})
   @capture(expr, @nosave var_ = body_) || return expr
@@ -69,13 +69,13 @@ end
 Function that replaces a `for` loop by a corresponding `while` loop saving explicitely the *iterator* and its *state*.
 """
 function transform_for(expr, ui8::BoxedUInt8)
-  @capture(expr, for element_ in iterator_ body__ end) || return expr
+  @capture(expr, for element_ in iterator_ body_ end) || return expr
   ui8.n += one(UInt8)
   next = Symbol("_iteratornext_", ui8.n)
   state = Symbol("_iterstate_", ui8.n)
   iterator_value = Symbol("_iterator_", ui8.n)
   label = Symbol("_iteratorlabel_", ui8.n)
-  body = postwalk(x->transform_continue(x, label), :(begin $(body...) end))
+  body = postwalk(x->transform_continue(x, label), :(begin $(body) end))
   quote
     $iterator_value = $iterator
     @nosave $next = iterate($iterator_value)
@@ -115,15 +115,14 @@ end
 Function that handles `let` block
 """
 function transform_slots_let(expr::Expr, symbols::Base.KeySet{Symbol, Dict{Symbol,Any}})
-  @capture(expr, let vars__; body__ end)
+  @capture(expr, let vars_; body_ end)
   locals = Set{Symbol}()
-  for var in vars
-    sym = var.args[1].args[2].value
-    push!(locals, sym)
-    var.args[1] = sym
-  end
-  body = postwalk(x->transform_let(x, locals), :(begin $(body...) end))
-  :(let $((:($var) for var in vars)...); $body end)
+  (isa(vars, Expr) && vars.head==:(=))  || error("@resumable currently supports only single variable declarations in let blocks, i.e. only let blocks exactly of the form `let i=j; ...; end`. If you need multiple variables, please submit an issue on the issue tracker and consider contributing a patch.")
+  sym = vars.args[1].args[2].value
+  push!(locals, sym)
+  vars.args[1] = sym
+  body = postwalk(x->transform_let(x, locals), :(begin $(body) end))
+  :(let $vars; $body end)
 end
 
 """
@@ -202,26 +201,25 @@ with a sequence of `try`-`catch`-`end` expressions:
 ```
 """
 function transform_try(expr, ui8::BoxedUInt8)
-  @capture(expr, (try body__ catch exc_; handling__ end) | (try body__ catch exc_; handling__ finally always__ end)) || return expr
+  @capture(expr, (try body_ catch exc_; handling_ end) | (try body_ catch exc_; handling_ finally always_ end)) || return expr
   ui8.n += one(UInt8)
   new_body = []
   segment = []
-  handling = handling === nothing ? [nothing] : handling
-  for ex in body
+  for ex in body.args
     if _is_yield(ex)
       ret = length(ex.args) > 2 ? ex.args[3:end] : [nothing]
-      exc === nothing ? push!(new_body, :(try $(segment...) catch; $(handling...); @goto $(Symbol("_TRY_", :($(ui8.n)))) end)) : push!(new_body, :(try $(segment...) catch $exc; $(handling...) ; @goto $(Symbol("_TRY_", :($(ui8.n)))) end))
+      exc === nothing ? push!(new_body, :(try $(segment...) catch; $(handling); @goto $(Symbol("_TRY_", :($(ui8.n)))) end)) : push!(new_body, :(try $(segment...) catch $exc; $(handling) ; @goto $(Symbol("_TRY_", :($(ui8.n)))) end))
       push!(new_body, quote @yield $(ret...) end)
       segment = []
     else
       push!(segment, ex)
     end
   end
   if segment != []
-    exc === nothing ? push!(new_body, :(try $(segment...) catch; $(handling...) end)) : push!(new_body, :(try $(segment...) catch $exc; $(handling...) end))
+    exc === nothing ? push!(new_body, :(try $(segment...) catch; $(handling) end)) : push!(new_body, :(try $(segment...) catch $exc; $(handling) end))
   end
   push!(new_body, :(@label $(Symbol("_TRY_", :($(ui8.n))))))
-  always === nothing || push!(new_body, quote $(always...) end)
+  always === nothing || push!(new_body, quote $(always) end)
   quote $(new_body...) end
 end
 

test/Project.toml

@@ -2,6 +2,7 @@
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
+MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

test/runtests.jl

@@ -29,6 +29,7 @@ println("Starting tests with $(Threads.nthreads()) threads out of `Sys.CPU_THREA
 @doset "main"
 @doset "yieldfrom"
 @doset "typeparams"
+@doset "coverage_preservation"
 VERSION >= v"1.8" && @doset "doctests"
 VERSION >= v"1.8" && @doset "aqua"
 get(ENV,"JET_TEST","")=="true" && @doset "jet"

test/test_coverage_preservation.jl

@@ -0,0 +1,43 @@
+using ResumableFunctions
+using Test
+using MacroTools: postwalk
+
+before = :(function f(arg, arg2::Int)
+        @yield arg
+        for i in 1:10
+            let i=i
+                @yield arg2
+                arg2 = arg2 + i
+            end
+        end
+        while true
+            try
+                @yield arg2
+                arg2 = arg2 + 1
+            catch e
+                @yield e
+                arg2 = arg2 + 1
+            end
+            break
+        end
+        arg2+arg
+        @yield arg2
+    end
+)
+
+after = eval(quote @macroexpand @resumable $before end)
+
+function get_all_linenodes(expr)
+    nodes = Set()
+    postwalk(expr) do x
+        if x isa LineNumberNode
+            push!(nodes, x)
+        end
+        x
+    end
+    return nodes
+end
+
+@testset "all line numbers are preserved" begin
+@test get_all_linenodes(before) ⊆ get_all_linenodes(after)
+end

test/test_jet.jl

@@ -26,6 +26,6 @@ end
         )
     )
     @show rep
-    @test length(JET.get_reports(rep)) <= 3
+    @test length(JET.get_reports(rep)) <= 4
     @test_broken length(JET.get_reports(rep)) == 0
 end