From e4ae3dc736821b150566801df2fe5fa446dd65ba Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 12 Jul 2019 14:31:32 -0700
Subject: [PATCH 01/32] new `macros.genAst`: fixes all issues with `quote do`

---
 lib/core/macros.nim | 55 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 795909c6b4887..4643cda7e1d09 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -539,6 +539,7 @@ proc parseStmt*(s: string): NimNode {.noSideEffect.} =
 
 proc getAst*(macroOrTemplate: untyped): NimNode {.magic: "ExpandToAst", noSideEffect.}
   ## Obtains the AST nodes returned from a macro or template invocation.
+  ## See also `genAst`.
   ## Example:
   ##
   ## .. code-block:: nim
@@ -547,6 +548,9 @@ proc getAst*(macroOrTemplate: untyped): NimNode {.magic: "ExpandToAst", noSideEf
   ##     var ast = getAst(BarTemplate())
 
 proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} =
+  ## .. warning:: `quote` has many caveats, see https://github.com/nim-lang/RFCs/issues/122
+  ##   Consider using the new `genAst` instead, which fixes the issues with `quote`.
+  ##
   ## Quasi-quoting operator.
   ## Accepts an expression or a block and returns the AST that represents it.
   ## Within the quoted AST, you are able to interpolate NimNode expressions
@@ -1419,6 +1423,57 @@ proc expectIdent*(n: NimNode, name: string) {.since: (1,1).} =
   if not eqIdent(n, name):
     error("Expected identifier to be `" & name & "` here", n)
 
+macro genAst*(args: varargs[untyped]): untyped =
+  ## Accepts a list of captured `variables = value` and a block and returns the
+  ## AST that represents it. Only the captured variables are interpolated.
+  runnableExamples:
+    type Foo = enum kfoo0, kfoo1, kfoo2, kfoo3, kfoo4
+    macro bar(x0: static Foo, x1: Foo, x2: Foo, xignored: Foo): untyped =
+      let s0 = "not captured!" ## does not override `s0` from caller scope
+      let s1 = "not captured!" ## does not override `s1=2`
+      let xignoredLocal = kfoo4
+      let x3 = newLit kfoo4
+      result = genAst(s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3) do:
+        ## only captures variables from `genAst` argument list
+        ## uncaptured variables will be set from caller scope (Eg `s0`)
+        ## `x2` is shortcut for the common `x2=x2`
+        doAssert not declared(xignored)      # not in param list!
+        doAssert not declared(xignoredLocal) # ditto
+        (s1, s2, s0, x0, x1, x2, x3)
+    let s0 = "caller scope!"
+    doAssert bar(kfoo1, kfoo2, kfoo3, kfoo4) ==
+      (2, "asdf", "caller scope!", kfoo1, kfoo2, kfoo3, kfoo4)
+
+  let params = newTree(nnkFormalParams, newEmptyNode())
+  let name = genSym(nskTemplate, "fun")
+  let call = newCall(name)
+  for a in args[0..^2]:
+    var varName: NimNode
+    var varVal: NimNode
+    case a.kind
+    of nnkExprEqExpr:
+      varName = a[0]
+      varVal = a[1]
+    of nnkIdent:
+      varName = a
+      varVal = a
+    else: error("invalid argument kind: " & $a.kind, a)
+
+    params.add newTree(nnkIdentDefs, [varName, newEmptyNode(), newEmptyNode()])
+    call.add varVal
+
+  result = newStmtList()
+  result.add nnkTemplateDef.newTree(
+      name,
+      newEmptyNode(),
+      newEmptyNode(),
+      params,
+      nnkPragma.newTree(ident"dirty"),
+        # dirty is needed to allow binding to caller scope
+      newEmptyNode(),
+      args[^1])
+  result.add newCall(bindSym"getAst", call)
+
 proc hasArgOfName*(params: NimNode; name: string): bool=
   ## Search `nnkFormalParams` for an argument.
   expectKind(params, nnkFormalParams)

From ff40adb3b7663d966507aae0abe76ce48e3ee431 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 12 Jul 2019 15:38:48 -0700
Subject: [PATCH 02/32] add tests

---
 lib/core/macros.nim      |  1 +
 tests/macros/mgenast.nim |  9 ++++
 tests/macros/tgenast.nim | 89 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 99 insertions(+)
 create mode 100644 tests/macros/mgenast.nim
 create mode 100644 tests/macros/tgenast.nim

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 4643cda7e1d09..5eeecbea6d547 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1433,6 +1433,7 @@ macro genAst*(args: varargs[untyped]): untyped =
       let s1 = "not captured!" ## does not override `s1=2`
       let xignoredLocal = kfoo4
       let x3 = newLit kfoo4
+      ## use `result = genAst do` if there are 0 captures
       result = genAst(s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3) do:
         ## only captures variables from `genAst` argument list
         ## uncaptured variables will be set from caller scope (Eg `s0`)
diff --git a/tests/macros/mgenast.nim b/tests/macros/mgenast.nim
new file mode 100644
index 0000000000000..3aadbdbc18d53
--- /dev/null
+++ b/tests/macros/mgenast.nim
@@ -0,0 +1,9 @@
+from std/streams import newStringStream, readData, writeData
+import std/macros
+
+macro bindme*(): untyped =
+  genAst(newStringStream, writeData, readData) do:
+    var tst = "sometext"
+    var ss = newStringStream("anothertext")
+    writeData(ss, tst[0].addr, 2)
+    discard readData(ss, tst[0].addr, 2)
diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
new file mode 100644
index 0000000000000..a593ba0b84f03
--- /dev/null
+++ b/tests/macros/tgenast.nim
@@ -0,0 +1,89 @@
+import std/macros
+
+block:
+  type Foo = enum kfoo0, kfoo1, kfoo2, kfoo3, kfoo4
+
+  macro bar(x0: static Foo, x1: Foo, x2: Foo, xignored: Foo): untyped =
+    let s0 = "not captured!"
+    let s1 = "not captured!"
+    let xignoredLocal = kfoo4
+    let x3 = newLit kfoo4
+    result = genAst(s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3) do:
+      doAssert not declared(xignored)
+      doAssert not declared(xignoredLocal)
+      (s1, s2, s0, x0, x1, x2, x3)
+
+  let s0 = "caller scope!"
+
+  doAssert bar(kfoo1, kfoo2, kfoo3, kfoo4) ==
+    (2, "asdf", "caller scope!", kfoo1, kfoo2, kfoo3, kfoo4)
+
+block:
+  # doesn't have limitation mentioned in https://github.com/nim-lang/RFCs/issues/122#issue-401636535
+  macro abc(name: untyped): untyped =
+    result = genAst(name):
+      type name = object
+
+  abc(Bar)
+  doAssert Bar.default == Bar()
+
+import std/strformat
+
+block:
+  # fix https://github.com/nim-lang/Nim/issues/8220
+  macro foo(): untyped =
+    result = genAst do:
+      let bar = "Hello, World"
+      &"Let's interpolate {bar} in the string"
+  doAssert foo() == "Let's interpolate Hello, World in the string"
+
+block:
+  # backticks parser limitations / ambiguities not an issue with `genAst`:
+  # fix https://github.com/nim-lang/Nim/issues/10326
+  # fix https://github.com/nim-lang/Nim/issues/9745
+  type Foo = object
+    a: int
+
+  macro m1(): untyped =
+    # result = quote do: # Error: undeclared identifier: 'a1'
+    result = genAst do:
+      template `a1=`(x: var Foo, val: int) =
+        x.a = val
+
+  m1()
+  var x0: Foo
+  x0.a1 = 10
+  doAssert x0 == Foo(a: 10)
+
+block:
+  # fix https://github.com/nim-lang/Nim/issues/7375
+  macro fun(b: static[bool], b2: bool): untyped =
+    result = newStmtList()
+  macro foo(c: bool): untyped =
+    var b = false
+    result = genAst(b = newLit b, c) do:
+      fun(b, c)
+
+  foo(true)
+
+when true:
+  # fix https://github.com/nim-lang/Nim/issues/7889
+  from mgenast import bindme
+  bindme()
+
+block:
+  # fix https://github.com/nim-lang/Nim/issues/7589
+  # since `==` works with genAst, the problem goes away
+  macro foo2(): untyped =
+    # result = quote do: # Error: '==' cannot be passed to a procvar
+    result = genAst do:
+      `==`(3,4)
+  doAssert not foo2()
+
+block:
+  # fix https://github.com/nim-lang/Nim/issues/7726
+  macro foo(): untyped =
+    let a = @[1, 2, 3, 4, 5]
+    result = genAst(a, b = a.len) do: # shows 2 ways to get a.len
+      (a.len, b)
+  doAssert foo() == (5, 5)

From d0c691b24e1bacb38c751efd14184e737a3227f1 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 12 Jul 2019 16:00:40 -0700
Subject: [PATCH 03/32] add changelog entry

---
 changelog.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/changelog.md b/changelog.md
index 53d30bf36085e..a70fe85ca046b 100644
--- a/changelog.md
+++ b/changelog.md
@@ -40,6 +40,9 @@
 - Added `BackwardsIndex` overload for `JsonNode`.
 
 - added `jsonutils.jsonTo` overload with `opt = Joptions()` param.
+- Added `macros.genAst` that fixes all issues with `quote do` (#11722)
+
+## Library changes
 
 - `json.%`,`json.to`, `jsonutils.formJson`,`jsonutils.toJson` now work with `uint|uint64`
   instead of raising (as in 1.4) or giving wrong results (as in 1.2).
@@ -51,6 +54,7 @@
 
 - Added `randState` template that exposes the default random number generator.
   Useful for library authors.
+## Language additions
 
 - Added `std/enumutils` module. Added `genEnumCaseStmt` macro that generates case statement to parse string to enum.
   Added `items` for enums with holes.
@@ -327,6 +331,11 @@
 
 
 ## Tool changes
+- VM FFI now works with {.importc, dynlib.}, when using -d:nimHasLibFFI (#11635)
+
+- importc procs with a body are now executed in the VM as if importc wasn't specified,
+  this allows using {.rtl.} procs at CT, making -d:useNimRtl work in more cases,
+  e.g. compiling nim itself (#11635)
 
 - The rst parser now supports markdown table syntax.
   Known limitations:

From 5e0bcdad5ec659e7b0794514bf6f21ff22f013da Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 12 Jul 2019 18:49:20 -0700
Subject: [PATCH 04/32] add workaround for
 https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669

---
 lib/core/macros.nim | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 5eeecbea6d547..d5a178b2ae3bf 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1446,7 +1446,8 @@ macro genAst*(args: varargs[untyped]): untyped =
       (2, "asdf", "caller scope!", kfoo1, kfoo2, kfoo3, kfoo4)
 
   let params = newTree(nnkFormalParams, newEmptyNode())
-  let name = genSym(nskTemplate, "fun")
+  # using `_` as workaround, see https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669
+  let name = genSym(nskTemplate, "_fun")
   let call = newCall(name)
   for a in args[0..^2]:
     var varName: NimNode

From 12b42c45671e8150981ecd6851f459903191e293 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 12 Jul 2019 21:48:27 -0700
Subject: [PATCH 05/32] add test for #9607

---
 tests/macros/tgenast.nim | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index a593ba0b84f03..ed35bf1069b25 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -87,3 +87,17 @@ block:
     result = genAst(a, b = a.len) do: # shows 2 ways to get a.len
       (a.len, b)
   doAssert foo() == (5, 5)
+
+block:
+  # fix https://github.com/nim-lang/Nim/issues/9607
+  proc fun1(info:LineInfo): string = "bar1"
+  proc fun2(info:int): string = "bar2"
+  macro bar(args: varargs[untyped]): untyped =
+    let info = args.lineInfoObj
+    let fun1 = bindSym"fun1"
+    let fun2 = bindSym"fun2"
+    result = genAst(info = newLit info) do:
+      (fun1(info), fun2(info.line))
+  doAssert bar() == ("bar1", "bar2")
+
+

From ba2d73d1649d5e23a4803674dc4e88aba7841211 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 12 Jul 2019 21:48:50 -0700
Subject: [PATCH 06/32] add kNoExposeLocalInjects option

---
 lib/core/macros.nim      |  54 +++++++++++++++++----
 tests/macros/mgenast.nim |  40 ++++++++++++++--
 tests/macros/tgenast.nim | 100 +++++++++++++++++++++++++++++----------
 3 files changed, 157 insertions(+), 37 deletions(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index d5a178b2ae3bf..2cd0597c019f8 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1423,29 +1423,64 @@ proc expectIdent*(n: NimNode, name: string) {.since: (1,1).} =
   if not eqIdent(n, name):
     error("Expected identifier to be `" & name & "` here", n)
 
-macro genAst*(args: varargs[untyped]): untyped =
+type GenAstOpt* = enum
+  kNoExposeLocalInjects,
+    # when unset, inject'd symbols (including implicit ones such as local procs
+    # in scope) are exposed implicitly;
+    # gensym'd symbols will generate a CT internal error: `environment misses:`
+    # when set, local symbols are not exposed unless captured explicitly in
+    # `genAst` argument list. The default is unset, to avoid surprising hijacking
+    # of local symbols by symbols in caller scope.
+
+macro genAst*(options: static set[GenAstOpt] = {}, args: varargs[untyped]): untyped =
   ## Accepts a list of captured `variables = value` and a block and returns the
-  ## AST that represents it. Only the captured variables are interpolated.
+  ## AST that represents it. Local `{.inject.}` symbols are captured (eg
+  ## local procs) unless `kNoExposeLocalInjects in options`; additional variables
+  ## are captured as subsequent parameters.
   runnableExamples:
     type Foo = enum kfoo0, kfoo1, kfoo2, kfoo3, kfoo4
-    macro bar(x0: static Foo, x1: Foo, x2: Foo, xignored: Foo): untyped =
+
+    macro bar1(x0: static Foo, x1: Foo, x2: Foo, xignored: Foo): untyped =
+      let s0 = "not captured!" ## does not override `s0` from caller scope
+      let s1 = "not captured!" ## does not override `s1=2`
+      let xignoredLocal = kfoo4
+      proc localExposed(): auto = kfoo4 # implicitly captured
+      let x3 = newLit kfoo4
+      result = genAst({}, s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3) do:
+        # echo xignored # would give: Error: undeclared identifier
+        # echo s0 # would give: Error: internal error: expr: var not init s0_237159
+        (s1, s2, x0, x1, x2, x3, localExposed())
+
+    macro bar2(x0: static Foo, x1: Foo, x2: Foo, xignored: Foo): untyped =
       let s0 = "not captured!" ## does not override `s0` from caller scope
       let s1 = "not captured!" ## does not override `s1=2`
       let xignoredLocal = kfoo4
       let x3 = newLit kfoo4
-      ## use `result = genAst do` if there are 0 captures
-      result = genAst(s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3) do:
+      result = genAst({kNoExposeLocalInjects}, s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3) do:
         ## only captures variables from `genAst` argument list
         ## uncaptured variables will be set from caller scope (Eg `s0`)
         ## `x2` is shortcut for the common `x2=x2`
         doAssert not declared(xignored)      # not in param list!
         doAssert not declared(xignoredLocal) # ditto
         (s1, s2, s0, x0, x1, x2, x3)
-    let s0 = "caller scope!"
-    doAssert bar(kfoo1, kfoo2, kfoo3, kfoo4) ==
-      (2, "asdf", "caller scope!", kfoo1, kfoo2, kfoo3, kfoo4)
+
+    block:
+      let s0 = "caller scope!"
+      doAssert bar1(kfoo1, kfoo2, kfoo3, kfoo4) ==
+        (2, "asdf", kfoo1, kfoo2, kfoo3, kfoo4, kfoo4)
+
+    block:
+      let s0 = "caller scope!"
+      doAssert bar2(kfoo1, kfoo2, kfoo3, kfoo4) ==
+        (2, "asdf", "caller scope!", kfoo1, kfoo2, kfoo3, kfoo4)
 
   let params = newTree(nnkFormalParams, newEmptyNode())
+  let pragmas =
+    if kNoExposeLocalInjects in options:
+      nnkPragma.newTree(ident"dirty")
+    else:
+      newEmptyNode()
+
   # using `_` as workaround, see https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669
   let name = genSym(nskTemplate, "_fun")
   let call = newCall(name)
@@ -1470,8 +1505,7 @@ macro genAst*(args: varargs[untyped]): untyped =
       newEmptyNode(),
       newEmptyNode(),
       params,
-      nnkPragma.newTree(ident"dirty"),
-        # dirty is needed to allow binding to caller scope
+      pragmas,
       newEmptyNode(),
       args[^1])
   result.add newCall(bindSym"getAst", call)
diff --git a/tests/macros/mgenast.nim b/tests/macros/mgenast.nim
index 3aadbdbc18d53..7e860bbd2b0a1 100644
--- a/tests/macros/mgenast.nim
+++ b/tests/macros/mgenast.nim
@@ -1,8 +1,42 @@
-from std/streams import newStringStream, readData, writeData
 import std/macros
 
-macro bindme*(): untyped =
-  genAst(newStringStream, writeData, readData) do:
+## using a enum instead of, say, int, to make apparent potential bugs related to
+## forgetting converting to NimNode via newLit, see https://github.com/nim-lang/Nim/issues/9607
+
+type Foo* = enum kfoo0, kfoo1, kfoo2, kfoo3, kfoo4
+
+proc myLocalPriv(): auto = kfoo1
+proc myLocalPriv2(): auto = kfoo1
+macro bindme2*(): untyped =
+  genAst({}) do: myLocalPriv()
+macro bindme3*(): untyped =
+  ## myLocalPriv must be captured explicitly
+  genAst({kNoExposeLocalInjects}, myLocalPriv) do: myLocalPriv()
+
+macro bindme4*(): untyped =
+  ## calling this won't compile because `myLocalPriv` isn't captured
+  genAst({kNoExposeLocalInjects}) do: myLocalPriv()
+
+macro bindme5UseExpose*(): untyped =
+  genAst({}) do: myLocalPriv2()
+
+macro bindme5UseExposeFalse*(): untyped =
+  genAst({kNoExposeLocalInjects}) do: myLocalPriv2()
+
+## example from https://github.com/nim-lang/Nim/issues/7889
+from std/streams import newStringStream, readData, writeData
+
+macro bindme6UseExpose*(): untyped =
+  genAst({}) do:
+    var tst = "sometext"
+    var ss = newStringStream("anothertext")
+    writeData(ss, tst[0].addr, 2)
+    discard readData(ss, tst[0].addr, 2)
+
+macro bindme6UseExposeFalse*(): untyped =
+  ## without kexposeLocalInjects, requires passing all referenced symbols
+  ## which can be tedious
+  genAst({kNoExposeLocalInjects}, newStringStream, writeData, readData) do:
     var tst = "sometext"
     var ss = newStringStream("anothertext")
     writeData(ss, tst[0].addr, 2)
diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index ed35bf1069b25..a105f9148ab29 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -1,14 +1,14 @@
 import std/macros
+from std/strformat import `&`
+import ./mgenast
 
 block:
-  type Foo = enum kfoo0, kfoo1, kfoo2, kfoo3, kfoo4
-
   macro bar(x0: static Foo, x1: Foo, x2: Foo, xignored: Foo): untyped =
     let s0 = "not captured!"
     let s1 = "not captured!"
     let xignoredLocal = kfoo4
     let x3 = newLit kfoo4
-    result = genAst(s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3) do:
+    result = genAst({kNoExposeLocalInjects}, s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3) do:
       doAssert not declared(xignored)
       doAssert not declared(xignoredLocal)
       (s1, s2, s0, x0, x1, x2, x3)
@@ -21,22 +21,12 @@ block:
 block:
   # doesn't have limitation mentioned in https://github.com/nim-lang/RFCs/issues/122#issue-401636535
   macro abc(name: untyped): untyped =
-    result = genAst(name):
+    result = genAst({}, name):
       type name = object
 
   abc(Bar)
   doAssert Bar.default == Bar()
 
-import std/strformat
-
-block:
-  # fix https://github.com/nim-lang/Nim/issues/8220
-  macro foo(): untyped =
-    result = genAst do:
-      let bar = "Hello, World"
-      &"Let's interpolate {bar} in the string"
-  doAssert foo() == "Let's interpolate Hello, World in the string"
-
 block:
   # backticks parser limitations / ambiguities not an issue with `genAst`:
   # fix https://github.com/nim-lang/Nim/issues/10326
@@ -46,7 +36,7 @@ block:
 
   macro m1(): untyped =
     # result = quote do: # Error: undeclared identifier: 'a1'
-    result = genAst do:
+    result = genAst({}) do:
       template `a1=`(x: var Foo, val: int) =
         x.a = val
 
@@ -61,22 +51,17 @@ block:
     result = newStmtList()
   macro foo(c: bool): untyped =
     var b = false
-    result = genAst(b = newLit b, c) do:
+    result = genAst({}, b = newLit b, c) do:
       fun(b, c)
 
   foo(true)
 
-when true:
-  # fix https://github.com/nim-lang/Nim/issues/7889
-  from mgenast import bindme
-  bindme()
-
 block:
   # fix https://github.com/nim-lang/Nim/issues/7589
   # since `==` works with genAst, the problem goes away
   macro foo2(): untyped =
     # result = quote do: # Error: '==' cannot be passed to a procvar
-    result = genAst do:
+    result = genAst({}) do:
       `==`(3,4)
   doAssert not foo2()
 
@@ -84,7 +69,7 @@ block:
   # fix https://github.com/nim-lang/Nim/issues/7726
   macro foo(): untyped =
     let a = @[1, 2, 3, 4, 5]
-    result = genAst(a, b = a.len) do: # shows 2 ways to get a.len
+    result = genAst({}, a, b = a.len) do: # shows 2 ways to get a.len
       (a.len, b)
   doAssert foo() == (5, 5)
 
@@ -92,12 +77,79 @@ block:
   # fix https://github.com/nim-lang/Nim/issues/9607
   proc fun1(info:LineInfo): string = "bar1"
   proc fun2(info:int): string = "bar2"
+
+  macro bar2(args: varargs[untyped]): untyped =
+    let info = args.lineInfoObj
+    let fun1 = bindSym"fun1" # optional; we can remove this and also the
+    # capture of fun1
+    result = genAst({}, info = newLit info, fun1) do:
+      (fun1(info), fun2(info.line))
+  doAssert bar2() == ("bar1", "bar2")
+
   macro bar(args: varargs[untyped]): untyped =
     let info = args.lineInfoObj
     let fun1 = bindSym"fun1"
     let fun2 = bindSym"fun2"
-    result = genAst(info = newLit info) do:
+    result = genAst({kNoExposeLocalInjects}, info = newLit info) do:
       (fun1(info), fun2(info.line))
   doAssert bar() == ("bar1", "bar2")
 
+block:
+  # fix https://github.com/nim-lang/Nim/issues/7889
+  doAssert bindme2() == kfoo1
+  doAssert bindme3() == kfoo1
+  doAssert not compiles(bindme4()) # correctly gives Error: undeclared identifier: 'myLocalPriv'
+  proc myLocalPriv2(): auto = kfoo2
+
+  doAssert bindme5UseExpose() == kfoo1
+  doAssert bindme5UseExposeFalse() == kfoo2
+    # local `myLocalPriv2` hijacks symbol, probably not what user wants
+    # by default as it's surprising for the macro writer
+
+  bindme6UseExpose()
+  bindme6UseExposeFalse()
+
+block:
+  macro mbar(x3: Foo, x3b: static Foo): untyped =
+    var x1=kfoo3
+    var x2=newLit kfoo3
+    var x4=kfoo3
+    var xLocal=kfoo3
+
+    proc funLocal(): auto = kfoo4
+
+    result = genAst({}, x1=newLit x1, x2, x3, x4 = newLit x4) do:
+      # local x1 overrides remote x1
+      when false:
+        # one advantage of using `kNoExposeLocalInjects` is that these would hold:
+        doAssert not declared xLocal
+        doAssert not compiles(echo xLocal)
+        # however, even without it, we at least correctly generate CT error
+        # if trying to use un-captured symbol; this correctly gives:
+        # Error: internal error: environment misses: xLocal
+        echo xLocal
+
+      proc foo1(): auto =
+        # note that `funLocal` is captured implicitly, according to hygienic
+        # template rules; with `kNoExposeLocalInjects` it would not unless
+        # captured in `genAst` capture list explicitly
+        (a0: xRemote, a1: x1, a2: x2, a3: x3, a4: x4, a5: funLocal())
+
+    return result
+
+  proc main()=
+    var xRemote=kfoo1
+    var x1=kfoo2
+    mbar(kfoo4, kfoo4)
+    doAssert foo1() == (a0: kfoo1, a1: kfoo3, a2: kfoo3, a3: kfoo4, a4: kfoo3, a5: kfoo4)
+
+  main()
 
+block:
+  # fix https://github.com/nim-lang/Nim/issues/8220
+  macro foo(): untyped =
+    # kNoExposeLocalInjects needed here
+    result = genAst({kNoExposeLocalInjects}) do:
+      let bar = "Hello, World"
+      &"Let's interpolate {bar} in the string"
+  doAssert foo() == "Let's interpolate Hello, World in the string"

From 26fcfdc67669ed44e292812fc1fdab19a1f686ac Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sun, 14 Jul 2019 11:26:14 -0700
Subject: [PATCH 07/32] simplify: `do:`=>`:`

---
 lib/core/macros.nim      |  4 ++--
 tests/macros/mgenast.nim | 14 +++++++-------
 tests/macros/tgenast.nim | 18 +++++++++---------
 3 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 2cd0597c019f8..75d1fd5459889 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1446,7 +1446,7 @@ macro genAst*(options: static set[GenAstOpt] = {}, args: varargs[untyped]): unty
       let xignoredLocal = kfoo4
       proc localExposed(): auto = kfoo4 # implicitly captured
       let x3 = newLit kfoo4
-      result = genAst({}, s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3) do:
+      result = genAst({}, s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3):
         # echo xignored # would give: Error: undeclared identifier
         # echo s0 # would give: Error: internal error: expr: var not init s0_237159
         (s1, s2, x0, x1, x2, x3, localExposed())
@@ -1456,7 +1456,7 @@ macro genAst*(options: static set[GenAstOpt] = {}, args: varargs[untyped]): unty
       let s1 = "not captured!" ## does not override `s1=2`
       let xignoredLocal = kfoo4
       let x3 = newLit kfoo4
-      result = genAst({kNoExposeLocalInjects}, s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3) do:
+      result = genAst({kNoExposeLocalInjects}, s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3):
         ## only captures variables from `genAst` argument list
         ## uncaptured variables will be set from caller scope (Eg `s0`)
         ## `x2` is shortcut for the common `x2=x2`
diff --git a/tests/macros/mgenast.nim b/tests/macros/mgenast.nim
index 7e860bbd2b0a1..650a6c92c23a1 100644
--- a/tests/macros/mgenast.nim
+++ b/tests/macros/mgenast.nim
@@ -8,26 +8,26 @@ type Foo* = enum kfoo0, kfoo1, kfoo2, kfoo3, kfoo4
 proc myLocalPriv(): auto = kfoo1
 proc myLocalPriv2(): auto = kfoo1
 macro bindme2*(): untyped =
-  genAst({}) do: myLocalPriv()
+  genAst({}): myLocalPriv()
 macro bindme3*(): untyped =
   ## myLocalPriv must be captured explicitly
-  genAst({kNoExposeLocalInjects}, myLocalPriv) do: myLocalPriv()
+  genAst({kNoExposeLocalInjects}, myLocalPriv): myLocalPriv()
 
 macro bindme4*(): untyped =
   ## calling this won't compile because `myLocalPriv` isn't captured
-  genAst({kNoExposeLocalInjects}) do: myLocalPriv()
+  genAst({kNoExposeLocalInjects}): myLocalPriv()
 
 macro bindme5UseExpose*(): untyped =
-  genAst({}) do: myLocalPriv2()
+  genAst({}): myLocalPriv2()
 
 macro bindme5UseExposeFalse*(): untyped =
-  genAst({kNoExposeLocalInjects}) do: myLocalPriv2()
+  genAst({kNoExposeLocalInjects}): myLocalPriv2()
 
 ## example from https://github.com/nim-lang/Nim/issues/7889
 from std/streams import newStringStream, readData, writeData
 
 macro bindme6UseExpose*(): untyped =
-  genAst({}) do:
+  genAst({}):
     var tst = "sometext"
     var ss = newStringStream("anothertext")
     writeData(ss, tst[0].addr, 2)
@@ -36,7 +36,7 @@ macro bindme6UseExpose*(): untyped =
 macro bindme6UseExposeFalse*(): untyped =
   ## without kexposeLocalInjects, requires passing all referenced symbols
   ## which can be tedious
-  genAst({kNoExposeLocalInjects}, newStringStream, writeData, readData) do:
+  genAst({kNoExposeLocalInjects}, newStringStream, writeData, readData):
     var tst = "sometext"
     var ss = newStringStream("anothertext")
     writeData(ss, tst[0].addr, 2)
diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index a105f9148ab29..df798d91ac2fa 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -8,7 +8,7 @@ block:
     let s1 = "not captured!"
     let xignoredLocal = kfoo4
     let x3 = newLit kfoo4
-    result = genAst({kNoExposeLocalInjects}, s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3) do:
+    result = genAst({kNoExposeLocalInjects}, s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3):
       doAssert not declared(xignored)
       doAssert not declared(xignoredLocal)
       (s1, s2, s0, x0, x1, x2, x3)
@@ -36,7 +36,7 @@ block:
 
   macro m1(): untyped =
     # result = quote do: # Error: undeclared identifier: 'a1'
-    result = genAst({}) do:
+    result = genAst({}):
       template `a1=`(x: var Foo, val: int) =
         x.a = val
 
@@ -51,7 +51,7 @@ block:
     result = newStmtList()
   macro foo(c: bool): untyped =
     var b = false
-    result = genAst({}, b = newLit b, c) do:
+    result = genAst({}, b = newLit b, c):
       fun(b, c)
 
   foo(true)
@@ -61,7 +61,7 @@ block:
   # since `==` works with genAst, the problem goes away
   macro foo2(): untyped =
     # result = quote do: # Error: '==' cannot be passed to a procvar
-    result = genAst({}) do:
+    result = genAst({}):
       `==`(3,4)
   doAssert not foo2()
 
@@ -69,7 +69,7 @@ block:
   # fix https://github.com/nim-lang/Nim/issues/7726
   macro foo(): untyped =
     let a = @[1, 2, 3, 4, 5]
-    result = genAst({}, a, b = a.len) do: # shows 2 ways to get a.len
+    result = genAst({}, a, b = a.len): # shows 2 ways to get a.len
       (a.len, b)
   doAssert foo() == (5, 5)
 
@@ -82,7 +82,7 @@ block:
     let info = args.lineInfoObj
     let fun1 = bindSym"fun1" # optional; we can remove this and also the
     # capture of fun1
-    result = genAst({}, info = newLit info, fun1) do:
+    result = genAst({}, info = newLit info, fun1):
       (fun1(info), fun2(info.line))
   doAssert bar2() == ("bar1", "bar2")
 
@@ -90,7 +90,7 @@ block:
     let info = args.lineInfoObj
     let fun1 = bindSym"fun1"
     let fun2 = bindSym"fun2"
-    result = genAst({kNoExposeLocalInjects}, info = newLit info) do:
+    result = genAst({kNoExposeLocalInjects}, info = newLit info):
       (fun1(info), fun2(info.line))
   doAssert bar() == ("bar1", "bar2")
 
@@ -118,7 +118,7 @@ block:
 
     proc funLocal(): auto = kfoo4
 
-    result = genAst({}, x1=newLit x1, x2, x3, x4 = newLit x4) do:
+    result = genAst({}, x1=newLit x1, x2, x3, x4 = newLit x4):
       # local x1 overrides remote x1
       when false:
         # one advantage of using `kNoExposeLocalInjects` is that these would hold:
@@ -149,7 +149,7 @@ block:
   # fix https://github.com/nim-lang/Nim/issues/8220
   macro foo(): untyped =
     # kNoExposeLocalInjects needed here
-    result = genAst({kNoExposeLocalInjects}) do:
+    result = genAst({kNoExposeLocalInjects}):
       let bar = "Hello, World"
       &"Let's interpolate {bar} in the string"
   doAssert foo() == "Let's interpolate Hello, World in the string"

From 150610ac797d8035d617912ed91ca23a41f3f700 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sun, 21 Jul 2019 00:15:06 -0700
Subject: [PATCH 08/32] add test case for nested application of genAst

---
 tests/macros/tgenast.nim | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index df798d91ac2fa..8743e544dc590 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -153,3 +153,16 @@ block:
       let bar = "Hello, World"
       &"Let's interpolate {bar} in the string"
   doAssert foo() == "Let's interpolate Hello, World in the string"
+
+
+block: # nested application of genAst
+  macro createMacro(name, obj, field: untyped): untyped =
+    result = genAst({}, obj = newDotExpr(obj, field), lit = newLit(10), name, field):
+      # can't reuse `result` here, would clash
+      macro name(arg: untyped): untyped =
+        genAst({}, arg2=arg): # somehow `arg2` rename is needed
+          (obj, astToStr(field), lit, arg2)
+
+  var x = @[1, 2, 3]
+  createMacro foo, x, len
+  doAssert (foo 20) == (3, "len", 10, 20)

From 42769b8f0b5d77fbd1b331925d6eb368fa68f126 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Tue, 23 Jul 2019 17:36:00 -0700
Subject: [PATCH 09/32] genAst: automatically call newLit when needed

---
 lib/core/macros.nim      | 24 ++++++++++++++++--------
 tests/macros/tgenast.nim | 28 ++++++++++++++++++++--------
 2 files changed, 36 insertions(+), 16 deletions(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 75d1fd5459889..8e1161ce6fa34 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1431,6 +1431,8 @@ type GenAstOpt* = enum
     # when set, local symbols are not exposed unless captured explicitly in
     # `genAst` argument list. The default is unset, to avoid surprising hijacking
     # of local symbols by symbols in caller scope.
+  kNoAutoNewLit,
+    # don't call call newLit automatically in `genAst` capture parameters
 
 macro genAst*(options: static set[GenAstOpt] = {}, args: varargs[untyped]): untyped =
   ## Accepts a list of captured `variables = value` and a block and returns the
@@ -1440,23 +1442,24 @@ macro genAst*(options: static set[GenAstOpt] = {}, args: varargs[untyped]): unty
   runnableExamples:
     type Foo = enum kfoo0, kfoo1, kfoo2, kfoo3, kfoo4
 
-    macro bar1(x0: static Foo, x1: Foo, x2: Foo, xignored: Foo): untyped =
+    ## simple example
+    macro bar1(x0: static Foo, x1: Foo, xignored: Foo): untyped =
       let s0 = "not captured!" ## does not override `s0` from caller scope
       let s1 = "not captured!" ## does not override `s1=2`
       let xignoredLocal = kfoo4
       proc localExposed(): auto = kfoo4 # implicitly captured
-      let x3 = newLit kfoo4
-      result = genAst({}, s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3):
+      result = genAst({}, s1=true, s2="asdf", x0, x1):
         # echo xignored # would give: Error: undeclared identifier
         # echo s0 # would give: Error: internal error: expr: var not init s0_237159
-        (s1, s2, x0, x1, x2, x3, localExposed())
+        (s1, s2, x0, x1, localExposed())
 
+    ## more complex example
     macro bar2(x0: static Foo, x1: Foo, x2: Foo, xignored: Foo): untyped =
       let s0 = "not captured!" ## does not override `s0` from caller scope
       let s1 = "not captured!" ## does not override `s1=2`
       let xignoredLocal = kfoo4
       let x3 = newLit kfoo4
-      result = genAst({kNoExposeLocalInjects}, s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3):
+      result = genAst({kNoExposeLocalInjects}, s1=true, s2="asdf", x0, x1=x1, x2, x3):
         ## only captures variables from `genAst` argument list
         ## uncaptured variables will be set from caller scope (Eg `s0`)
         ## `x2` is shortcut for the common `x2=x2`
@@ -1466,13 +1469,13 @@ macro genAst*(options: static set[GenAstOpt] = {}, args: varargs[untyped]): unty
 
     block:
       let s0 = "caller scope!"
-      doAssert bar1(kfoo1, kfoo2, kfoo3, kfoo4) ==
-        (2, "asdf", kfoo1, kfoo2, kfoo3, kfoo4, kfoo4)
+      doAssert bar1(kfoo1, kfoo2, kfoo4) ==
+        (true, "asdf", kfoo1, kfoo2, kfoo4)
 
     block:
       let s0 = "caller scope!"
       doAssert bar2(kfoo1, kfoo2, kfoo3, kfoo4) ==
-        (2, "asdf", "caller scope!", kfoo1, kfoo2, kfoo3, kfoo4)
+        (true, "asdf", "caller scope!", kfoo1, kfoo2, kfoo3, kfoo4)
 
   let params = newTree(nnkFormalParams, newEmptyNode())
   let pragmas =
@@ -1481,6 +1484,10 @@ macro genAst*(options: static set[GenAstOpt] = {}, args: varargs[untyped]): unty
     else:
       newEmptyNode()
 
+  proc newLitMaybe[T](a: T): auto =
+    when compiles(newLit(a)): newLit(a)
+    else: a
+
   # using `_` as workaround, see https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669
   let name = genSym(nskTemplate, "_fun")
   let call = newCall(name)
@@ -1495,6 +1502,7 @@ macro genAst*(options: static set[GenAstOpt] = {}, args: varargs[untyped]): unty
       varName = a
       varVal = a
     else: error("invalid argument kind: " & $a.kind, a)
+    if kNoAutoNewLit notin options: varVal = newCall(bindSym"newLitMaybe", varVal)
 
     params.add newTree(nnkIdentDefs, [varName, newEmptyNode(), newEmptyNode()])
     call.add varVal
diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index 8743e544dc590..5dc3d3bc8e6ab 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -7,16 +7,20 @@ block:
     let s0 = "not captured!"
     let s1 = "not captured!"
     let xignoredLocal = kfoo4
+
+    # newLit optional:
     let x3 = newLit kfoo4
-    result = genAst({kNoExposeLocalInjects}, s1=2, s2="asdf", x0=newLit x0, x1=x1, x2, x3):
+    let x3b = kfoo4
+
+    result = genAst({kNoExposeLocalInjects}, s1=true, s2="asdf", x0, x1=x1, x2, x3, x3b):
       doAssert not declared(xignored)
       doAssert not declared(xignoredLocal)
-      (s1, s2, s0, x0, x1, x2, x3)
+      (s1, s2, s0, x0, x1, x2, x3, x3b)
 
   let s0 = "caller scope!"
 
   doAssert bar(kfoo1, kfoo2, kfoo3, kfoo4) ==
-    (2, "asdf", "caller scope!", kfoo1, kfoo2, kfoo3, kfoo4)
+    (true, "asdf", "caller scope!", kfoo1, kfoo2, kfoo3, kfoo4, kfoo4)
 
 block:
   # doesn't have limitation mentioned in https://github.com/nim-lang/RFCs/issues/122#issue-401636535
@@ -51,7 +55,7 @@ block:
     result = newStmtList()
   macro foo(c: bool): untyped =
     var b = false
-    result = genAst({}, b = newLit b, c):
+    result = genAst({}, b, c):
       fun(b, c)
 
   foo(true)
@@ -82,7 +86,7 @@ block:
     let info = args.lineInfoObj
     let fun1 = bindSym"fun1" # optional; we can remove this and also the
     # capture of fun1
-    result = genAst({}, info = newLit info, fun1):
+    result = genAst({}, info, fun1):
       (fun1(info), fun2(info.line))
   doAssert bar2() == ("bar1", "bar2")
 
@@ -90,7 +94,7 @@ block:
     let info = args.lineInfoObj
     let fun1 = bindSym"fun1"
     let fun2 = bindSym"fun2"
-    result = genAst({kNoExposeLocalInjects}, info = newLit info):
+    result = genAst({kNoExposeLocalInjects}, info):
       (fun1(info), fun2(info.line))
   doAssert bar() == ("bar1", "bar2")
 
@@ -118,7 +122,7 @@ block:
 
     proc funLocal(): auto = kfoo4
 
-    result = genAst({}, x1=newLit x1, x2, x3, x4 = newLit x4):
+    result = genAst({}, x1, x2, x3, x4):
       # local x1 overrides remote x1
       when false:
         # one advantage of using `kNoExposeLocalInjects` is that these would hold:
@@ -157,7 +161,7 @@ block:
 
 block: # nested application of genAst
   macro createMacro(name, obj, field: untyped): untyped =
-    result = genAst({}, obj = newDotExpr(obj, field), lit = newLit(10), name, field):
+    result = genAst({}, obj = newDotExpr(obj, field), lit = 10, name, field):
       # can't reuse `result` here, would clash
       macro name(arg: untyped): untyped =
         genAst({}, arg2=arg): # somehow `arg2` rename is needed
@@ -166,3 +170,11 @@ block: # nested application of genAst
   var x = @[1, 2, 3]
   createMacro foo, x, len
   doAssert (foo 20) == (3, "len", 10, 20)
+
+block: # test with kNoAutoNewLit
+  macro bar(): untyped =
+    let s1 = true
+    template boo(x): untyped =
+      fun(x)
+    result = genAst({kNoAutoNewLit}, s1=newLit(s1), s1b=s1): (s1, s1b)
+  doAssert bar() == (true, 1)

From e29025761e9c7f117afedc919b2ab85aa8a819e3 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Tue, 23 Jul 2019 18:03:30 -0700
Subject: [PATCH 10/32] allow skipping `{}`: genAst: foo

---
 lib/core/macros.nim      | 10 +++++++---
 tests/macros/mgenast.nim | 14 +++++++-------
 tests/macros/tgenast.nim | 31 ++++++++++++++++++-------------
 3 files changed, 32 insertions(+), 23 deletions(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 8e1161ce6fa34..f9ce4938bc590 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1434,7 +1434,7 @@ type GenAstOpt* = enum
   kNoAutoNewLit,
     # don't call call newLit automatically in `genAst` capture parameters
 
-macro genAst*(options: static set[GenAstOpt] = {}, args: varargs[untyped]): untyped =
+macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untyped =
   ## Accepts a list of captured `variables = value` and a block and returns the
   ## AST that represents it. Local `{.inject.}` symbols are captured (eg
   ## local procs) unless `kNoExposeLocalInjects in options`; additional variables
@@ -1448,7 +1448,7 @@ macro genAst*(options: static set[GenAstOpt] = {}, args: varargs[untyped]): unty
       let s1 = "not captured!" ## does not override `s1=2`
       let xignoredLocal = kfoo4
       proc localExposed(): auto = kfoo4 # implicitly captured
-      result = genAst({}, s1=true, s2="asdf", x0, x1):
+      result = genAst(s1=true, s2="asdf", x0, x1):
         # echo xignored # would give: Error: undeclared identifier
         # echo s0 # would give: Error: internal error: expr: var not init s0_237159
         (s1, s2, x0, x1, localExposed())
@@ -1459,7 +1459,7 @@ macro genAst*(options: static set[GenAstOpt] = {}, args: varargs[untyped]): unty
       let s1 = "not captured!" ## does not override `s1=2`
       let xignoredLocal = kfoo4
       let x3 = newLit kfoo4
-      result = genAst({kNoExposeLocalInjects}, s1=true, s2="asdf", x0, x1=x1, x2, x3):
+      result = genAstOpt({kNoExposeLocalInjects}, s1=true, s2="asdf", x0, x1=x1, x2, x3):
         ## only captures variables from `genAst` argument list
         ## uncaptured variables will be set from caller scope (Eg `s0`)
         ## `x2` is shortcut for the common `x2=x2`
@@ -1518,6 +1518,10 @@ macro genAst*(options: static set[GenAstOpt] = {}, args: varargs[untyped]): unty
       args[^1])
   result.add newCall(bindSym"getAst", call)
 
+template genAst*(args: varargs[untyped]): untyped =
+  ## convenience wrapper around `genAstOpt`
+  genAstOpt({}, args)
+
 proc hasArgOfName*(params: NimNode; name: string): bool=
   ## Search `nnkFormalParams` for an argument.
   expectKind(params, nnkFormalParams)
diff --git a/tests/macros/mgenast.nim b/tests/macros/mgenast.nim
index 650a6c92c23a1..f309fee819b6e 100644
--- a/tests/macros/mgenast.nim
+++ b/tests/macros/mgenast.nim
@@ -8,26 +8,26 @@ type Foo* = enum kfoo0, kfoo1, kfoo2, kfoo3, kfoo4
 proc myLocalPriv(): auto = kfoo1
 proc myLocalPriv2(): auto = kfoo1
 macro bindme2*(): untyped =
-  genAst({}): myLocalPriv()
+  genAst: myLocalPriv()
 macro bindme3*(): untyped =
   ## myLocalPriv must be captured explicitly
-  genAst({kNoExposeLocalInjects}, myLocalPriv): myLocalPriv()
+  genAstOpt({kNoExposeLocalInjects}, myLocalPriv): myLocalPriv()
 
 macro bindme4*(): untyped =
   ## calling this won't compile because `myLocalPriv` isn't captured
-  genAst({kNoExposeLocalInjects}): myLocalPriv()
+  genAstOpt({kNoExposeLocalInjects}): myLocalPriv()
 
 macro bindme5UseExpose*(): untyped =
-  genAst({}): myLocalPriv2()
+  genAst: myLocalPriv2()
 
 macro bindme5UseExposeFalse*(): untyped =
-  genAst({kNoExposeLocalInjects}): myLocalPriv2()
+  genAstOpt({kNoExposeLocalInjects}): myLocalPriv2()
 
 ## example from https://github.com/nim-lang/Nim/issues/7889
 from std/streams import newStringStream, readData, writeData
 
 macro bindme6UseExpose*(): untyped =
-  genAst({}):
+  genAst:
     var tst = "sometext"
     var ss = newStringStream("anothertext")
     writeData(ss, tst[0].addr, 2)
@@ -36,7 +36,7 @@ macro bindme6UseExpose*(): untyped =
 macro bindme6UseExposeFalse*(): untyped =
   ## without kexposeLocalInjects, requires passing all referenced symbols
   ## which can be tedious
-  genAst({kNoExposeLocalInjects}, newStringStream, writeData, readData):
+  genAstOpt({kNoExposeLocalInjects}, newStringStream, writeData, readData):
     var tst = "sometext"
     var ss = newStringStream("anothertext")
     writeData(ss, tst[0].addr, 2)
diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index 5dc3d3bc8e6ab..16266d1a02e3b 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -12,7 +12,7 @@ block:
     let x3 = newLit kfoo4
     let x3b = kfoo4
 
-    result = genAst({kNoExposeLocalInjects}, s1=true, s2="asdf", x0, x1=x1, x2, x3, x3b):
+    result = genAstOpt({kNoExposeLocalInjects}, s1=true, s2="asdf", x0, x1=x1, x2, x3, x3b):
       doAssert not declared(xignored)
       doAssert not declared(xignoredLocal)
       (s1, s2, s0, x0, x1, x2, x3, x3b)
@@ -25,7 +25,7 @@ block:
 block:
   # doesn't have limitation mentioned in https://github.com/nim-lang/RFCs/issues/122#issue-401636535
   macro abc(name: untyped): untyped =
-    result = genAst({}, name):
+    result = genAst(name):
       type name = object
 
   abc(Bar)
@@ -40,7 +40,7 @@ block:
 
   macro m1(): untyped =
     # result = quote do: # Error: undeclared identifier: 'a1'
-    result = genAst({}):
+    result = genAst:
       template `a1=`(x: var Foo, val: int) =
         x.a = val
 
@@ -55,7 +55,7 @@ block:
     result = newStmtList()
   macro foo(c: bool): untyped =
     var b = false
-    result = genAst({}, b, c):
+    result = genAst(b, c):
       fun(b, c)
 
   foo(true)
@@ -65,7 +65,7 @@ block:
   # since `==` works with genAst, the problem goes away
   macro foo2(): untyped =
     # result = quote do: # Error: '==' cannot be passed to a procvar
-    result = genAst({}):
+    result = genAst:
       `==`(3,4)
   doAssert not foo2()
 
@@ -73,7 +73,7 @@ block:
   # fix https://github.com/nim-lang/Nim/issues/7726
   macro foo(): untyped =
     let a = @[1, 2, 3, 4, 5]
-    result = genAst({}, a, b = a.len): # shows 2 ways to get a.len
+    result = genAst(a, b = a.len): # shows 2 ways to get a.len
       (a.len, b)
   doAssert foo() == (5, 5)
 
@@ -86,7 +86,7 @@ block:
     let info = args.lineInfoObj
     let fun1 = bindSym"fun1" # optional; we can remove this and also the
     # capture of fun1
-    result = genAst({}, info, fun1):
+    result = genAst(info, fun1):
       (fun1(info), fun2(info.line))
   doAssert bar2() == ("bar1", "bar2")
 
@@ -94,7 +94,7 @@ block:
     let info = args.lineInfoObj
     let fun1 = bindSym"fun1"
     let fun2 = bindSym"fun2"
-    result = genAst({kNoExposeLocalInjects}, info):
+    result = genAstOpt({kNoExposeLocalInjects}, info):
       (fun1(info), fun2(info.line))
   doAssert bar() == ("bar1", "bar2")
 
@@ -122,7 +122,7 @@ block:
 
     proc funLocal(): auto = kfoo4
 
-    result = genAst({}, x1, x2, x3, x4):
+    result = genAst(x1, x2, x3, x4):
       # local x1 overrides remote x1
       when false:
         # one advantage of using `kNoExposeLocalInjects` is that these would hold:
@@ -153,7 +153,7 @@ block:
   # fix https://github.com/nim-lang/Nim/issues/8220
   macro foo(): untyped =
     # kNoExposeLocalInjects needed here
-    result = genAst({kNoExposeLocalInjects}):
+    result = genAstOpt({kNoExposeLocalInjects}):
       let bar = "Hello, World"
       &"Let's interpolate {bar} in the string"
   doAssert foo() == "Let's interpolate Hello, World in the string"
@@ -161,10 +161,10 @@ block:
 
 block: # nested application of genAst
   macro createMacro(name, obj, field: untyped): untyped =
-    result = genAst({}, obj = newDotExpr(obj, field), lit = 10, name, field):
+    result = genAst(obj = newDotExpr(obj, field), lit = 10, name, field):
       # can't reuse `result` here, would clash
       macro name(arg: untyped): untyped =
-        genAst({}, arg2=arg): # somehow `arg2` rename is needed
+        genAst(arg2=arg): # somehow `arg2` rename is needed
           (obj, astToStr(field), lit, arg2)
 
   var x = @[1, 2, 3]
@@ -176,5 +176,10 @@ block: # test with kNoAutoNewLit
     let s1 = true
     template boo(x): untyped =
       fun(x)
-    result = genAst({kNoAutoNewLit}, s1=newLit(s1), s1b=s1): (s1, s1b)
+    result = genAstOpt({kNoAutoNewLit}, s1=newLit(s1), s1b=s1): (s1, s1b)
   doAssert bar() == (true, 1)
+
+block: # sanity check: check passing `{}` also works
+  macro bar(): untyped =
+    result = genAstOpt({}, s1=true): s1
+  doAssert bar() == true

From 232a0baf2337bac7b19424dbc18949b860f6a6dd Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Tue, 23 Jul 2019 18:14:50 -0700
Subject: [PATCH 11/32] fixup

---
 lib/core/macros.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index f9ce4938bc590..a460ad6b86874 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1504,7 +1504,7 @@ macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untype
     else: error("invalid argument kind: " & $a.kind, a)
     if kNoAutoNewLit notin options: varVal = newCall(bindSym"newLitMaybe", varVal)
 
-    params.add newTree(nnkIdentDefs, [varName, newEmptyNode(), newEmptyNode()])
+    params.add newTree(nnkIdentDefs, varName, newEmptyNode(), newEmptyNode())
     call.add varVal
 
   result = newStmtList()

From a8839d44572730ce10500d941f790263cd73925c Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Tue, 23 Jul 2019 18:16:51 -0700
Subject: [PATCH 12/32] fixup

---
 lib/core/macros.nim      |  8 ++++----
 tests/macros/mgenast.nim |  8 ++++----
 tests/macros/tgenast.nim | 12 ++++++------
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index a460ad6b86874..3d0b1ef4039df 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1424,7 +1424,7 @@ proc expectIdent*(n: NimNode, name: string) {.since: (1,1).} =
     error("Expected identifier to be `" & name & "` here", n)
 
 type GenAstOpt* = enum
-  kNoExposeLocalInjects,
+  kDirtyTemplate,
     # when unset, inject'd symbols (including implicit ones such as local procs
     # in scope) are exposed implicitly;
     # gensym'd symbols will generate a CT internal error: `environment misses:`
@@ -1437,7 +1437,7 @@ type GenAstOpt* = enum
 macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untyped =
   ## Accepts a list of captured `variables = value` and a block and returns the
   ## AST that represents it. Local `{.inject.}` symbols are captured (eg
-  ## local procs) unless `kNoExposeLocalInjects in options`; additional variables
+  ## local procs) unless `kDirtyTemplate in options`; additional variables
   ## are captured as subsequent parameters.
   runnableExamples:
     type Foo = enum kfoo0, kfoo1, kfoo2, kfoo3, kfoo4
@@ -1459,7 +1459,7 @@ macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untype
       let s1 = "not captured!" ## does not override `s1=2`
       let xignoredLocal = kfoo4
       let x3 = newLit kfoo4
-      result = genAstOpt({kNoExposeLocalInjects}, s1=true, s2="asdf", x0, x1=x1, x2, x3):
+      result = genAstOpt({kDirtyTemplate}, s1=true, s2="asdf", x0, x1=x1, x2, x3):
         ## only captures variables from `genAst` argument list
         ## uncaptured variables will be set from caller scope (Eg `s0`)
         ## `x2` is shortcut for the common `x2=x2`
@@ -1479,7 +1479,7 @@ macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untype
 
   let params = newTree(nnkFormalParams, newEmptyNode())
   let pragmas =
-    if kNoExposeLocalInjects in options:
+    if kDirtyTemplate in options:
       nnkPragma.newTree(ident"dirty")
     else:
       newEmptyNode()
diff --git a/tests/macros/mgenast.nim b/tests/macros/mgenast.nim
index f309fee819b6e..41ab473ce5e28 100644
--- a/tests/macros/mgenast.nim
+++ b/tests/macros/mgenast.nim
@@ -11,17 +11,17 @@ macro bindme2*(): untyped =
   genAst: myLocalPriv()
 macro bindme3*(): untyped =
   ## myLocalPriv must be captured explicitly
-  genAstOpt({kNoExposeLocalInjects}, myLocalPriv): myLocalPriv()
+  genAstOpt({kDirtyTemplate}, myLocalPriv): myLocalPriv()
 
 macro bindme4*(): untyped =
   ## calling this won't compile because `myLocalPriv` isn't captured
-  genAstOpt({kNoExposeLocalInjects}): myLocalPriv()
+  genAstOpt({kDirtyTemplate}): myLocalPriv()
 
 macro bindme5UseExpose*(): untyped =
   genAst: myLocalPriv2()
 
 macro bindme5UseExposeFalse*(): untyped =
-  genAstOpt({kNoExposeLocalInjects}): myLocalPriv2()
+  genAstOpt({kDirtyTemplate}): myLocalPriv2()
 
 ## example from https://github.com/nim-lang/Nim/issues/7889
 from std/streams import newStringStream, readData, writeData
@@ -36,7 +36,7 @@ macro bindme6UseExpose*(): untyped =
 macro bindme6UseExposeFalse*(): untyped =
   ## without kexposeLocalInjects, requires passing all referenced symbols
   ## which can be tedious
-  genAstOpt({kNoExposeLocalInjects}, newStringStream, writeData, readData):
+  genAstOpt({kDirtyTemplate}, newStringStream, writeData, readData):
     var tst = "sometext"
     var ss = newStringStream("anothertext")
     writeData(ss, tst[0].addr, 2)
diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index 16266d1a02e3b..19edb90f35b1e 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -12,7 +12,7 @@ block:
     let x3 = newLit kfoo4
     let x3b = kfoo4
 
-    result = genAstOpt({kNoExposeLocalInjects}, s1=true, s2="asdf", x0, x1=x1, x2, x3, x3b):
+    result = genAstOpt({kDirtyTemplate}, s1=true, s2="asdf", x0, x1=x1, x2, x3, x3b):
       doAssert not declared(xignored)
       doAssert not declared(xignoredLocal)
       (s1, s2, s0, x0, x1, x2, x3, x3b)
@@ -94,7 +94,7 @@ block:
     let info = args.lineInfoObj
     let fun1 = bindSym"fun1"
     let fun2 = bindSym"fun2"
-    result = genAstOpt({kNoExposeLocalInjects}, info):
+    result = genAstOpt({kDirtyTemplate}, info):
       (fun1(info), fun2(info.line))
   doAssert bar() == ("bar1", "bar2")
 
@@ -125,7 +125,7 @@ block:
     result = genAst(x1, x2, x3, x4):
       # local x1 overrides remote x1
       when false:
-        # one advantage of using `kNoExposeLocalInjects` is that these would hold:
+        # one advantage of using `kDirtyTemplate` is that these would hold:
         doAssert not declared xLocal
         doAssert not compiles(echo xLocal)
         # however, even without it, we at least correctly generate CT error
@@ -135,7 +135,7 @@ block:
 
       proc foo1(): auto =
         # note that `funLocal` is captured implicitly, according to hygienic
-        # template rules; with `kNoExposeLocalInjects` it would not unless
+        # template rules; with `kDirtyTemplate` it would not unless
         # captured in `genAst` capture list explicitly
         (a0: xRemote, a1: x1, a2: x2, a3: x3, a4: x4, a5: funLocal())
 
@@ -152,8 +152,8 @@ block:
 block:
   # fix https://github.com/nim-lang/Nim/issues/8220
   macro foo(): untyped =
-    # kNoExposeLocalInjects needed here
-    result = genAstOpt({kNoExposeLocalInjects}):
+    # kDirtyTemplate needed here
+    result = genAstOpt({kDirtyTemplate}):
       let bar = "Hello, World"
       &"Let's interpolate {bar} in the string"
   doAssert foo() == "Let's interpolate Hello, World in the string"

From 88d7c9113fd3078dc9ac4c8e0bcc8686a4dbd99b Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Tue, 23 Jul 2019 18:27:17 -0700
Subject: [PATCH 13/32] improve comment

---
 lib/core/macros.nim | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 3d0b1ef4039df..069884befcdeb 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1425,12 +1425,11 @@ proc expectIdent*(n: NimNode, name: string) {.since: (1,1).} =
 
 type GenAstOpt* = enum
   kDirtyTemplate,
-    # when unset, inject'd symbols (including implicit ones such as local procs
-    # in scope) are exposed implicitly;
-    # gensym'd symbols will generate a CT internal error: `environment misses:`
-    # when set, local symbols are not exposed unless captured explicitly in
-    # `genAst` argument list. The default is unset, to avoid surprising hijacking
-    # of local symbols by symbols in caller scope.
+    # When set, uses a dirty template in implementation of `genAst`. This
+    # is occasionally useful as workaround for issues such as #8220, see
+    # `strformat limitations <strformat.html#limitations>`_ for details.
+    # Default is unset, to avoid surprising hijacking of local symbols by
+    # symbols in caller scope.
   kNoAutoNewLit,
     # don't call call newLit automatically in `genAst` capture parameters
 

From e40b5bae2f2e389093f2dc242782495072f49085 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Tue, 23 Jul 2019 18:31:40 -0700
Subject: [PATCH 14/32] fixup

---
 tests/macros/tgenast.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index 19edb90f35b1e..7af2228e69ed4 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -152,7 +152,7 @@ block:
 block:
   # fix https://github.com/nim-lang/Nim/issues/8220
   macro foo(): untyped =
-    # kDirtyTemplate needed here
+    # kDirtyTemplate needed here, see https://nim-lang.github.io/Nim/strformat.html#limitations
     result = genAstOpt({kDirtyTemplate}):
       let bar = "Hello, World"
       &"Let's interpolate {bar} in the string"

From c7d05e0efa3a179dc5db66e8184bd2580cae75f0 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Thu, 25 Jul 2019 12:31:54 -0700
Subject: [PATCH 15/32] clarifications

---
 tests/macros/mgenast.nim |  2 +-
 tests/macros/tgenast.nim | 34 ++++++++++++++++++++++++----------
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/tests/macros/mgenast.nim b/tests/macros/mgenast.nim
index 41ab473ce5e28..c88d97c472d75 100644
--- a/tests/macros/mgenast.nim
+++ b/tests/macros/mgenast.nim
@@ -34,7 +34,7 @@ macro bindme6UseExpose*(): untyped =
     discard readData(ss, tst[0].addr, 2)
 
 macro bindme6UseExposeFalse*(): untyped =
-  ## without kexposeLocalInjects, requires passing all referenced symbols
+  ## with `kDirtyTemplate`, requires passing all referenced symbols
   ## which can be tedious
   genAstOpt({kDirtyTemplate}, newStringStream, writeData, readData):
     var tst = "sometext"
diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index 7af2228e69ed4..0595e380d5e7a 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -32,9 +32,9 @@ block:
   doAssert Bar.default == Bar()
 
 block:
-  # backticks parser limitations / ambiguities not an issue with `genAst`:
-  # fix https://github.com/nim-lang/Nim/issues/10326
-  # fix https://github.com/nim-lang/Nim/issues/9745
+  # backticks parser limitations / ambiguities not are an issue with `genAst`:
+  # (#10326 #9745 are fixed but `quote do` still has underlying ambiguity issue
+  # with backticks)
   type Foo = object
     a: int
 
@@ -71,6 +71,8 @@ block:
 
 block:
   # fix https://github.com/nim-lang/Nim/issues/7726
+  # expressions such as `a.len` are just passed as arguments to `genAst`, and
+  # caller scope is not polluted with definitions such as `let b = newLit a.len`
   macro foo(): untyped =
     let a = @[1, 2, 3, 4, 5]
     result = genAst(a, b = a.len): # shows 2 ways to get a.len
@@ -85,11 +87,17 @@ block:
   macro bar2(args: varargs[untyped]): untyped =
     let info = args.lineInfoObj
     let fun1 = bindSym"fun1" # optional; we can remove this and also the
-    # capture of fun1
+    # capture of fun1, as show in next example
     result = genAst(info, fun1):
       (fun1(info), fun2(info.line))
   doAssert bar2() == ("bar1", "bar2")
 
+  macro bar3(args: varargs[untyped]): untyped =
+    let info = args.lineInfoObj
+    result = genAst(info):
+      (fun1(info), fun2(info.line))
+  doAssert bar3() == ("bar1", "bar2")
+
   macro bar(args: varargs[untyped]): untyped =
     let info = args.lineInfoObj
     let fun1 = bindSym"fun1"
@@ -99,16 +107,21 @@ block:
   doAssert bar() == ("bar1", "bar2")
 
 block:
-  # fix https://github.com/nim-lang/Nim/issues/7889
+  # example from https://github.com/nim-lang/Nim/issues/7889 works
+  # after changing method call syntax to regular call syntax; this is a
+  # limitation described in https://github.com/nim-lang/Nim/issues/7085
+  # note that `quote do` would also work after that change in this example.
   doAssert bindme2() == kfoo1
   doAssert bindme3() == kfoo1
   doAssert not compiles(bindme4()) # correctly gives Error: undeclared identifier: 'myLocalPriv'
   proc myLocalPriv2(): auto = kfoo2
-
   doAssert bindme5UseExpose() == kfoo1
+
+  # example showing hijacking behavior when using `kDirtyTemplate`
   doAssert bindme5UseExposeFalse() == kfoo2
-    # local `myLocalPriv2` hijacks symbol, probably not what user wants
-    # by default as it's surprising for the macro writer
+    # local `myLocalPriv2` hijacks symbol `mgenast.myLocalPriv2`. In most
+    # use cases this is probably not what macro writer intends as it's
+    # surprising; hence `kDirtyTemplate` is not the default.
 
   bindme6UseExpose()
   bindme6UseExposeFalse()
@@ -150,9 +163,10 @@ block:
   main()
 
 block:
-  # fix https://github.com/nim-lang/Nim/issues/8220
+  # With `kDirtyTemplate`, the example from #8220 works.
+  # See https://nim-lang.github.io/Nim/strformat.html#limitations for
+  # an explanation of why {.dirty.} is needed.
   macro foo(): untyped =
-    # kDirtyTemplate needed here, see https://nim-lang.github.io/Nim/strformat.html#limitations
     result = genAstOpt({kDirtyTemplate}):
       let bar = "Hello, World"
       &"Let's interpolate {bar} in the string"

From d54517aba8015e74ea7cefdfd5a00a14d5388e1b Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 26 Jul 2019 15:22:19 -0700
Subject: [PATCH 16/32] fixup

---
 tests/macros/mgenast.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/macros/mgenast.nim b/tests/macros/mgenast.nim
index c88d97c472d75..cfb6cdda9541e 100644
--- a/tests/macros/mgenast.nim
+++ b/tests/macros/mgenast.nim
@@ -1,6 +1,6 @@
 import std/macros
 
-## using a enum instead of, say, int, to make apparent potential bugs related to
+## Using a enum instead of, say, int, to make apparent potential bugs related to
 ## forgetting converting to NimNode via newLit, see https://github.com/nim-lang/Nim/issues/9607
 
 type Foo* = enum kfoo0, kfoo1, kfoo2, kfoo3, kfoo4

From 2e8f5b22018cadaac93631bc6c48b3600ed5dbf4 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 27 Jul 2019 11:49:01 -0700
Subject: [PATCH 17/32] improve comments, simplify runnableExamples

---
 changelog.md             | 18 +++++++++++++
 lib/core/macros.nim      | 55 ++++++++++------------------------------
 tests/macros/tgenast.nim |  8 +++---
 3 files changed, 35 insertions(+), 46 deletions(-)

diff --git a/changelog.md b/changelog.md
index a70fe85ca046b..4df2cd9fea13f 100644
--- a/changelog.md
+++ b/changelog.md
@@ -46,6 +46,24 @@
 
 - `json.%`,`json.to`, `jsonutils.formJson`,`jsonutils.toJson` now work with `uint|uint64`
   instead of raising (as in 1.4) or giving wrong results (as in 1.2).
+- `asyncdispatch.drain` now properly takes into account `selector.hasPendingOperations`
+  and only returns once all pending async operations are guaranteed to have completed.
+- `asyncdispatch.drain` now consistently uses the passed timeout value for all
+  iterations of the event loop, and not just the first iteration.
+  This is more consistent with the other asyncdispatch apis, and allows
+  `asyncdispatch.drain` to be more efficient.
+- `base64.encode` and `base64.decode` was made faster by about 50%.
+- `htmlgen` adds [MathML](https://wikipedia.org/wiki/MathML) support
+  (ISO 40314).
+- `macros.eqIdent` is now invariant to export markers and backtick quotes.
+- `htmlgen.html` allows `lang` on the `<html>` tag and common valid attributes.
+- `macros.basename` and `basename=` got support for `PragmaExpr`,
+  so that an expression like `MyEnum {.pure.}` is handled correctly.
+- `httpclient.maxredirects` changed from `int` to `Natural`, because negative values serve no purpose whatsoever.
+- `httpclient.newHttpClient` and `httpclient.newAsyncHttpClient` added `headers` argument to set initial HTTP Headers,
+  instead of a hardcoded empty `newHttpHeader()`.
+- Added `macros.genAst` that avoids the problems inherent with `quote do` and can
+  be used as a replacement (#11722)
 
 - Added an overload for the `collect` macro that inferes the container type based
   on the syntax of the last expression. Works with std seqs, tables and sets.
diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 069884befcdeb..2d9e535a992ff 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -549,7 +549,7 @@ proc getAst*(macroOrTemplate: untyped): NimNode {.magic: "ExpandToAst", noSideEf
 
 proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} =
   ## .. warning:: `quote` has many caveats, see https://github.com/nim-lang/RFCs/issues/122
-  ##   Consider using the new `genAst` instead, which fixes the issues with `quote`.
+  ##   Consider using the new `genAst` instead, which avoids those issues.
   ##
   ## Quasi-quoting operator.
   ## Accepts an expression or a block and returns the AST that represents it.
@@ -1428,53 +1428,24 @@ type GenAstOpt* = enum
     # When set, uses a dirty template in implementation of `genAst`. This
     # is occasionally useful as workaround for issues such as #8220, see
     # `strformat limitations <strformat.html#limitations>`_ for details.
-    # Default is unset, to avoid surprising hijacking of local symbols by
+    # Default is unset, to avoid hijacking of uncaptured local symbols by
     # symbols in caller scope.
   kNoAutoNewLit,
     # don't call call newLit automatically in `genAst` capture parameters
 
 macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untyped =
-  ## Accepts a list of captured `variables = value` and a block and returns the
-  ## AST that represents it. Local `{.inject.}` symbols are captured (eg
-  ## local procs) unless `kDirtyTemplate in options`; additional variables
-  ## are captured as subsequent parameters.
+  ## Accepts a list of captured variables `a=b` or `a` and a block and returns the
+  ## AST that represents it. Local `{.inject.}` symbols (e.g. procs) are captured
+  ## unless `kDirtyTemplate in options`.
   runnableExamples:
-    type Foo = enum kfoo0, kfoo1, kfoo2, kfoo3, kfoo4
-
-    ## simple example
-    macro bar1(x0: static Foo, x1: Foo, xignored: Foo): untyped =
-      let s0 = "not captured!" ## does not override `s0` from caller scope
-      let s1 = "not captured!" ## does not override `s1=2`
-      let xignoredLocal = kfoo4
-      proc localExposed(): auto = kfoo4 # implicitly captured
-      result = genAst(s1=true, s2="asdf", x0, x1):
-        # echo xignored # would give: Error: undeclared identifier
-        # echo s0 # would give: Error: internal error: expr: var not init s0_237159
-        (s1, s2, x0, x1, localExposed())
-
-    ## more complex example
-    macro bar2(x0: static Foo, x1: Foo, x2: Foo, xignored: Foo): untyped =
-      let s0 = "not captured!" ## does not override `s0` from caller scope
-      let s1 = "not captured!" ## does not override `s1=2`
-      let xignoredLocal = kfoo4
-      let x3 = newLit kfoo4
-      result = genAstOpt({kDirtyTemplate}, s1=true, s2="asdf", x0, x1=x1, x2, x3):
-        ## only captures variables from `genAst` argument list
-        ## uncaptured variables will be set from caller scope (Eg `s0`)
-        ## `x2` is shortcut for the common `x2=x2`
-        doAssert not declared(xignored)      # not in param list!
-        doAssert not declared(xignoredLocal) # ditto
-        (s1, s2, s0, x0, x1, x2, x3)
-
-    block:
-      let s0 = "caller scope!"
-      doAssert bar1(kfoo1, kfoo2, kfoo4) ==
-        (true, "asdf", kfoo1, kfoo2, kfoo4)
-
-    block:
-      let s0 = "caller scope!"
-      doAssert bar2(kfoo1, kfoo2, kfoo3, kfoo4) ==
-        (true, "asdf", "caller scope!", kfoo1, kfoo2, kfoo3, kfoo4)
+    macro fun(a: bool, b: static bool): untyped =
+      let c = false # doesn't override parameter `c`
+      var d = 11 # var => gensym'd
+      proc localFun(): auto = 12 # proc => inject'd
+      genAst(a, b, c = true):
+        # echo d # not captured => gives `var not init`
+        (a, b, c, localFun())
+    doAssert fun(true, false) == (true, false, true, 12)
 
   let params = newTree(nnkFormalParams, newEmptyNode())
   let pragmas =
diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index 0595e380d5e7a..5ba2d0f97b235 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -50,7 +50,7 @@ block:
   doAssert x0 == Foo(a: 10)
 
 block:
-  # fix https://github.com/nim-lang/Nim/issues/7375
+  # avoids https://github.com/nim-lang/Nim/issues/7375
   macro fun(b: static[bool], b2: bool): untyped =
     result = newStmtList()
   macro foo(c: bool): untyped =
@@ -61,7 +61,7 @@ block:
   foo(true)
 
 block:
-  # fix https://github.com/nim-lang/Nim/issues/7589
+  # avoids https://github.com/nim-lang/Nim/issues/7589
   # since `==` works with genAst, the problem goes away
   macro foo2(): untyped =
     # result = quote do: # Error: '==' cannot be passed to a procvar
@@ -70,7 +70,7 @@ block:
   doAssert not foo2()
 
 block:
-  # fix https://github.com/nim-lang/Nim/issues/7726
+  # avoids https://github.com/nim-lang/Nim/issues/7726
   # expressions such as `a.len` are just passed as arguments to `genAst`, and
   # caller scope is not polluted with definitions such as `let b = newLit a.len`
   macro foo(): untyped =
@@ -80,7 +80,7 @@ block:
   doAssert foo() == (5, 5)
 
 block:
-  # fix https://github.com/nim-lang/Nim/issues/9607
+  # avoids https://github.com/nim-lang/Nim/issues/9607
   proc fun1(info:LineInfo): string = "bar1"
   proc fun2(info:int): string = "bar2"
 

From c9dfedd03206d4099cec5a8179da90bb4aac1bca Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Mon, 29 Jul 2019 23:09:15 -0700
Subject: [PATCH 18/32] kNoAutoNewLit => kNoNewLit

---
 lib/core/macros.nim      | 4 ++--
 tests/macros/tgenast.nim | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 2d9e535a992ff..d9ace3da3f5d1 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1430,7 +1430,7 @@ type GenAstOpt* = enum
     # `strformat limitations <strformat.html#limitations>`_ for details.
     # Default is unset, to avoid hijacking of uncaptured local symbols by
     # symbols in caller scope.
-  kNoAutoNewLit,
+  kNoNewLit,
     # don't call call newLit automatically in `genAst` capture parameters
 
 macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untyped =
@@ -1472,7 +1472,7 @@ macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untype
       varName = a
       varVal = a
     else: error("invalid argument kind: " & $a.kind, a)
-    if kNoAutoNewLit notin options: varVal = newCall(bindSym"newLitMaybe", varVal)
+    if kNoNewLit notin options: varVal = newCall(bindSym"newLitMaybe", varVal)
 
     params.add newTree(nnkIdentDefs, varName, newEmptyNode(), newEmptyNode())
     call.add varVal
diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index 5ba2d0f97b235..80b5ae5a10f45 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -185,12 +185,12 @@ block: # nested application of genAst
   createMacro foo, x, len
   doAssert (foo 20) == (3, "len", 10, 20)
 
-block: # test with kNoAutoNewLit
+block: # test with kNoNewLit
   macro bar(): untyped =
     let s1 = true
     template boo(x): untyped =
       fun(x)
-    result = genAstOpt({kNoAutoNewLit}, s1=newLit(s1), s1b=s1): (s1, s1b)
+    result = genAstOpt({kNoNewLit}, s1=newLit(s1), s1b=s1): (s1, s1b)
   doAssert bar() == (true, 1)
 
 block: # sanity check: check passing `{}` also works

From bc19a037a62275f0cf28a44b3942afbe86b0d635 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Tue, 30 Jul 2019 01:29:03 -0700
Subject: [PATCH 19/32] add example

---
 lib/core/macros.nim      |  5 +++--
 tests/macros/tgenast.nim | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index d9ace3da3f5d1..b40bba07b7ee5 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1454,8 +1454,9 @@ macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untype
     else:
       newEmptyNode()
 
-  proc newLitMaybe[T](a: T): auto =
-    when compiles(newLit(a)): newLit(a)
+  template newLitMaybe(a): untyped =
+    when type(a) is NimNode: a
+    elif compiles(newLit(a)): newLit(a)
     else: a
 
   # using `_` as workaround, see https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669
diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index 80b5ae5a10f45..36da0c9b62c4e 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -197,3 +197,22 @@ block: # sanity check: check passing `{}` also works
   macro bar(): untyped =
     result = genAstOpt({}, s1=true): s1
   doAssert bar() == true
+
+block: # test passing function and type symbols
+  proc z1(): auto = 41
+  type Z4 = type(1'i8)
+  macro bar(Z1: typedesc): untyped =
+    proc z2(): auto = 42
+    proc z3[T](a: T): auto = 43
+    let Z2 = genAst():
+      type(true)
+    let z4 = genAst():
+      proc myfun(): auto = 44
+      myfun
+    type Z3 = type(1'u8)
+    result = genAst(z4, Z1, Z2):
+      # z1, z2, z3, Z3, Z4 are captured automatically
+      # z1, z2, z3 can optionally be specified in capture list
+      (z1(), z2(), z3('a'), z4(), $Z1, $Z2, $Z3, $Z4)
+  type Z1 = type('c')
+  doAssert bar(Z1) == (41, 42, 43, 44, "char", "bool", "uint8", "int8")

From e2a90d426d9c3d65cf9ecef7e82208d9a241b48f Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 23 Aug 2019 11:29:28 -0700
Subject: [PATCH 20/32] add test that shows this fixes #11986

---
 tests/macros/tgenast.nim | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index 36da0c9b62c4e..f43d03c32ecb9 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -216,3 +216,22 @@ block: # test passing function and type symbols
       (z1(), z2(), z3('a'), z4(), $Z1, $Z2, $Z3, $Z4)
   type Z1 = type('c')
   doAssert bar(Z1) == (41, 42, 43, 44, "char", "bool", "uint8", "int8")
+
+block: # fix https://github.com/nim-lang/Nim/issues/11986
+  proc foo(): auto =
+    var s = { 'a', 'b' }
+    # var n = quote do: `s` # would print {97, 98}
+    var n = genAst(s): s
+    n.repr
+  static: doAssert foo() == "{'a', 'b'}"
+
+block: # also from #11986
+  macro foo(): untyped =
+    var s = { 'a', 'b' }
+    # quote do:
+    #   let t = `s`
+    #   $typeof(t) # set[range 0..65535(int)]
+    genAst(s):
+      let t = s
+      $typeof(t)
+  doAssert foo() == "set[char]"

From af8ebc18b3f8c3cd5457758ff853b198f3a2a387 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 24 Aug 2019 18:00:35 -0700
Subject: [PATCH 21/32] add examples showing mixin; add examples showing
 passing types, macros, templates

---
 lib/core/macros.nim      |  6 +++---
 tests/macros/mgenast.nim |  9 +++++++++
 tests/macros/tgenast.nim | 22 ++++++++++++++++++++++
 3 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index b40bba07b7ee5..9e1e5a9fc4bb6 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1455,9 +1455,9 @@ macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untype
       newEmptyNode()
 
   template newLitMaybe(a): untyped =
-    when type(a) is NimNode: a
-    elif compiles(newLit(a)): newLit(a)
-    else: a
+    when (a is type) or (typeof(a) is (proc | iterator | func | NimNode)):
+      a # `proc` actually also covers template, macro
+    else: newLit(a)
 
   # using `_` as workaround, see https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669
   let name = genSym(nskTemplate, "_fun")
diff --git a/tests/macros/mgenast.nim b/tests/macros/mgenast.nim
index cfb6cdda9541e..7fbee8a31d625 100644
--- a/tests/macros/mgenast.nim
+++ b/tests/macros/mgenast.nim
@@ -41,3 +41,12 @@ macro bindme6UseExposeFalse*(): untyped =
     var ss = newStringStream("anothertext")
     writeData(ss, tst[0].addr, 2)
     discard readData(ss, tst[0].addr, 2)
+
+
+proc locafun1(): auto = "in locafun1"
+proc locafun2(): auto = "in locafun2"
+# locafun3 in caller scope only
+macro mixinExample*(): untyped =
+  genAst:
+    mixin locafun1
+    (locafun1(), locafun2(), locafun3())
diff --git a/tests/macros/tgenast.nim b/tests/macros/tgenast.nim
index f43d03c32ecb9..d707605934393 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/macros/tgenast.nim
@@ -235,3 +235,25 @@ block: # also from #11986
       let t = s
       $typeof(t)
   doAssert foo() == "set[char]"
+
+block:
+  macro foo(): untyped =
+    type Foo = object
+    template baz2(a: int): untyped = a*10
+    macro baz3(a: int): untyped = newLit 13
+    result = newStmtList()
+
+    result.add genAst(Foo, baz2, baz3) do: # shows you can pass types, templates etc
+      var x: Foo
+      $($typeof(x), baz2(3), baz3(4))
+
+    let ret = genAst() do: # shows you don't have to, since they're inject'd
+      var x: Foo
+      $($typeof(x), baz2(3), baz3(4))
+  doAssert foo() == """("Foo", 30, 13)"""
+
+block: # illustrates how symbol visiblity can be controlled precisely using `mixin`
+  proc locafun1(): auto = "in locafun1 (caller scope)" # this will be used because of `mixin locafun1` => explicit hijacking is ok
+  proc locafun2(): auto = "in locafun2 (caller scope)" # this won't be used => no hijacking
+  proc locafun3(): auto = "in locafun3 (caller scope)"
+  doAssert mixinExample() == ("in locafun1 (caller scope)", "in locafun2", "in locafun3 (caller scope)")

From d3ebf15d9625119ac2adbc4214c5abb63d177a35 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 20 Mar 2021 01:31:25 -0700
Subject: [PATCH 22/32] move to std/genasts

---
 changelog.md                         |  3 +-
 lib/core/macros.nim                  | 72 +---------------------------
 lib/std/genasts.nim                  | 70 +++++++++++++++++++++++++++
 tests/{macros => stdlib}/mgenast.nim |  1 +
 tests/{macros => stdlib}/tgenast.nim |  1 +
 5 files changed, 74 insertions(+), 73 deletions(-)
 create mode 100644 lib/std/genasts.nim
 rename tests/{macros => stdlib}/mgenast.nim (98%)
 rename tests/{macros => stdlib}/tgenast.nim (99%)

diff --git a/changelog.md b/changelog.md
index 4df2cd9fea13f..7c22dd39869c5 100644
--- a/changelog.md
+++ b/changelog.md
@@ -40,7 +40,6 @@
 - Added `BackwardsIndex` overload for `JsonNode`.
 
 - added `jsonutils.jsonTo` overload with `opt = Joptions()` param.
-- Added `macros.genAst` that fixes all issues with `quote do` (#11722)
 
 ## Library changes
 
@@ -62,7 +61,7 @@
 - `httpclient.maxredirects` changed from `int` to `Natural`, because negative values serve no purpose whatsoever.
 - `httpclient.newHttpClient` and `httpclient.newAsyncHttpClient` added `headers` argument to set initial HTTP Headers,
   instead of a hardcoded empty `newHttpHeader()`.
-- Added `macros.genAst` that avoids the problems inherent with `quote do` and can
+- Added `genasts.genAst` that avoids the problems inherent with `quote do` and can
   be used as a replacement (#11722)
 
 - Added an overload for the `collect` macro that inferes the container type based
diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 9e1e5a9fc4bb6..98de78ffd5c75 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1423,77 +1423,7 @@ proc expectIdent*(n: NimNode, name: string) {.since: (1,1).} =
   if not eqIdent(n, name):
     error("Expected identifier to be `" & name & "` here", n)
 
-type GenAstOpt* = enum
-  kDirtyTemplate,
-    # When set, uses a dirty template in implementation of `genAst`. This
-    # is occasionally useful as workaround for issues such as #8220, see
-    # `strformat limitations <strformat.html#limitations>`_ for details.
-    # Default is unset, to avoid hijacking of uncaptured local symbols by
-    # symbols in caller scope.
-  kNoNewLit,
-    # don't call call newLit automatically in `genAst` capture parameters
-
-macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untyped =
-  ## Accepts a list of captured variables `a=b` or `a` and a block and returns the
-  ## AST that represents it. Local `{.inject.}` symbols (e.g. procs) are captured
-  ## unless `kDirtyTemplate in options`.
-  runnableExamples:
-    macro fun(a: bool, b: static bool): untyped =
-      let c = false # doesn't override parameter `c`
-      var d = 11 # var => gensym'd
-      proc localFun(): auto = 12 # proc => inject'd
-      genAst(a, b, c = true):
-        # echo d # not captured => gives `var not init`
-        (a, b, c, localFun())
-    doAssert fun(true, false) == (true, false, true, 12)
-
-  let params = newTree(nnkFormalParams, newEmptyNode())
-  let pragmas =
-    if kDirtyTemplate in options:
-      nnkPragma.newTree(ident"dirty")
-    else:
-      newEmptyNode()
-
-  template newLitMaybe(a): untyped =
-    when (a is type) or (typeof(a) is (proc | iterator | func | NimNode)):
-      a # `proc` actually also covers template, macro
-    else: newLit(a)
-
-  # using `_` as workaround, see https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669
-  let name = genSym(nskTemplate, "_fun")
-  let call = newCall(name)
-  for a in args[0..^2]:
-    var varName: NimNode
-    var varVal: NimNode
-    case a.kind
-    of nnkExprEqExpr:
-      varName = a[0]
-      varVal = a[1]
-    of nnkIdent:
-      varName = a
-      varVal = a
-    else: error("invalid argument kind: " & $a.kind, a)
-    if kNoNewLit notin options: varVal = newCall(bindSym"newLitMaybe", varVal)
-
-    params.add newTree(nnkIdentDefs, varName, newEmptyNode(), newEmptyNode())
-    call.add varVal
-
-  result = newStmtList()
-  result.add nnkTemplateDef.newTree(
-      name,
-      newEmptyNode(),
-      newEmptyNode(),
-      params,
-      pragmas,
-      newEmptyNode(),
-      args[^1])
-  result.add newCall(bindSym"getAst", call)
-
-template genAst*(args: varargs[untyped]): untyped =
-  ## convenience wrapper around `genAstOpt`
-  genAstOpt({}, args)
-
-proc hasArgOfName*(params: NimNode; name: string): bool=
+proc hasArgOfName*(params: NimNode; name: string): bool =
   ## Search `nnkFormalParams` for an argument.
   expectKind(params, nnkFormalParams)
   for i in 1..<params.len:
diff --git a/lib/std/genasts.nim b/lib/std/genasts.nim
new file mode 100644
index 0000000000000..0dec5690de42f
--- /dev/null
+++ b/lib/std/genasts.nim
@@ -0,0 +1,70 @@
+import std/macros
+type GenAstOpt* = enum
+  kDirtyTemplate,
+    # When set, uses a dirty template in implementation of `genAst`. This
+    # is occasionally useful as workaround for issues such as #8220, see
+    # `strformat limitations <strformat.html#limitations>`_ for details.
+    # Default is unset, to avoid hijacking of uncaptured local symbols by
+    # symbols in caller scope.
+  kNoNewLit,
+    # don't call call newLit automatically in `genAst` capture parameters
+
+macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untyped =
+  ## Accepts a list of captured variables `a=b` or `a` and a block and returns the
+  ## AST that represents it. Local `{.inject.}` symbols (e.g. procs) are captured
+  ## unless `kDirtyTemplate in options`.
+  runnableExamples:
+    macro fun(a: bool, b: static bool): untyped =
+      let c = false # doesn't override parameter `c`
+      var d = 11 # var => gensym'd
+      proc localFun(): auto = 12 # proc => inject'd
+      genAst(a, b, c = true):
+        # echo d # not captured => gives `var not init`
+        (a, b, c, localFun())
+    doAssert fun(true, false) == (true, false, true, 12)
+
+  let params = newTree(nnkFormalParams, newEmptyNode())
+  let pragmas =
+    if kDirtyTemplate in options:
+      nnkPragma.newTree(ident"dirty")
+    else:
+      newEmptyNode()
+
+  template newLitMaybe(a): untyped =
+    when (a is type) or (typeof(a) is (proc | iterator | func | NimNode)):
+      a # `proc` actually also covers template, macro
+    else: newLit(a)
+
+  # using `_` as workaround, see https://github.com/nim-lang/Nim/issues/2465#issuecomment-511076669
+  let name = genSym(nskTemplate, "_fun")
+  let call = newCall(name)
+  for a in args[0..^2]:
+    var varName: NimNode
+    var varVal: NimNode
+    case a.kind
+    of nnkExprEqExpr:
+      varName = a[0]
+      varVal = a[1]
+    of nnkIdent:
+      varName = a
+      varVal = a
+    else: error("invalid argument kind: " & $a.kind, a)
+    if kNoNewLit notin options: varVal = newCall(bindSym"newLitMaybe", varVal)
+
+    params.add newTree(nnkIdentDefs, varName, newEmptyNode(), newEmptyNode())
+    call.add varVal
+
+  result = newStmtList()
+  result.add nnkTemplateDef.newTree(
+      name,
+      newEmptyNode(),
+      newEmptyNode(),
+      params,
+      pragmas,
+      newEmptyNode(),
+      args[^1])
+  result.add newCall(bindSym"getAst", call)
+
+template genAst*(args: varargs[untyped]): untyped =
+  ## convenience wrapper around `genAstOpt`
+  genAstOpt({}, args)
diff --git a/tests/macros/mgenast.nim b/tests/stdlib/mgenast.nim
similarity index 98%
rename from tests/macros/mgenast.nim
rename to tests/stdlib/mgenast.nim
index 7fbee8a31d625..74fe8736daf5f 100644
--- a/tests/macros/mgenast.nim
+++ b/tests/stdlib/mgenast.nim
@@ -1,3 +1,4 @@
+import std/genasts
 import std/macros
 
 ## Using a enum instead of, say, int, to make apparent potential bugs related to
diff --git a/tests/macros/tgenast.nim b/tests/stdlib/tgenast.nim
similarity index 99%
rename from tests/macros/tgenast.nim
rename to tests/stdlib/tgenast.nim
index d707605934393..164c9e52ab2a3 100644
--- a/tests/macros/tgenast.nim
+++ b/tests/stdlib/tgenast.nim
@@ -1,3 +1,4 @@
+import std/genasts
 import std/macros
 from std/strformat import `&`
 import ./mgenast

From fa763a38d15426e23560472406633462c1fc8df9 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 20 Mar 2021 01:40:08 -0700
Subject: [PATCH 23/32] fixup

---
 changelog.md | 27 +--------------------------
 1 file changed, 1 insertion(+), 26 deletions(-)

diff --git a/changelog.md b/changelog.md
index 7c22dd39869c5..1a22e34e730c1 100644
--- a/changelog.md
+++ b/changelog.md
@@ -41,28 +41,8 @@
 
 - added `jsonutils.jsonTo` overload with `opt = Joptions()` param.
 
-## Library changes
-
 - `json.%`,`json.to`, `jsonutils.formJson`,`jsonutils.toJson` now work with `uint|uint64`
   instead of raising (as in 1.4) or giving wrong results (as in 1.2).
-- `asyncdispatch.drain` now properly takes into account `selector.hasPendingOperations`
-  and only returns once all pending async operations are guaranteed to have completed.
-- `asyncdispatch.drain` now consistently uses the passed timeout value for all
-  iterations of the event loop, and not just the first iteration.
-  This is more consistent with the other asyncdispatch apis, and allows
-  `asyncdispatch.drain` to be more efficient.
-- `base64.encode` and `base64.decode` was made faster by about 50%.
-- `htmlgen` adds [MathML](https://wikipedia.org/wiki/MathML) support
-  (ISO 40314).
-- `macros.eqIdent` is now invariant to export markers and backtick quotes.
-- `htmlgen.html` allows `lang` on the `<html>` tag and common valid attributes.
-- `macros.basename` and `basename=` got support for `PragmaExpr`,
-  so that an expression like `MyEnum {.pure.}` is handled correctly.
-- `httpclient.maxredirects` changed from `int` to `Natural`, because negative values serve no purpose whatsoever.
-- `httpclient.newHttpClient` and `httpclient.newAsyncHttpClient` added `headers` argument to set initial HTTP Headers,
-  instead of a hardcoded empty `newHttpHeader()`.
-- Added `genasts.genAst` that avoids the problems inherent with `quote do` and can
-  be used as a replacement (#11722)
 
 - Added an overload for the `collect` macro that inferes the container type based
   on the syntax of the last expression. Works with std seqs, tables and sets.
@@ -71,7 +51,6 @@
 
 - Added `randState` template that exposes the default random number generator.
   Useful for library authors.
-## Language additions
 
 - Added `std/enumutils` module. Added `genEnumCaseStmt` macro that generates case statement to parse string to enum.
   Added `items` for enums with holes.
@@ -276,6 +255,7 @@
 - Added `hasClosure` to `std/typetraits`.
 
 
+
 ## Language changes
 
 - `nimscript` now handles `except Exception as e`.
@@ -348,11 +328,6 @@
 
 
 ## Tool changes
-- VM FFI now works with {.importc, dynlib.}, when using -d:nimHasLibFFI (#11635)
-
-- importc procs with a body are now executed in the VM as if importc wasn't specified,
-  this allows using {.rtl.} procs at CT, making -d:useNimRtl work in more cases,
-  e.g. compiling nim itself (#11635)
 
 - The rst parser now supports markdown table syntax.
   Known limitations:

From 5f0d6d3dcfc4e6e0946ab532bc516fbf1f055cb0 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 20 Mar 2021 01:41:22 -0700
Subject: [PATCH 24/32] fixup

---
 changelog.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/changelog.md b/changelog.md
index 1a22e34e730c1..020d8a350a81e 100644
--- a/changelog.md
+++ b/changelog.md
@@ -255,6 +255,8 @@
 - Added `hasClosure` to `std/typetraits`.
 
 
+- Added `genasts.genAst` that avoids the problems inherent with `quote do` and can
+  be used as a replacement.
 
 ## Language changes
 

From 0e3c5d7f8c6513518b6a705f367317f437c09557 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 20 Mar 2021 01:42:56 -0700
Subject: [PATCH 25/32] fixup

---
 lib/core/macros.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 98de78ffd5c75..dade47a4ae9b0 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -548,8 +548,8 @@ proc getAst*(macroOrTemplate: untyped): NimNode {.magic: "ExpandToAst", noSideEf
   ##     var ast = getAst(BarTemplate())
 
 proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} =
-  ## .. warning:: `quote` has many caveats, see https://github.com/nim-lang/RFCs/issues/122
-  ##   Consider using the new `genAst` instead, which avoids those issues.
+  ## .. warning:: `quote` comes with many caveats, consider using the new
+  ##   `genasts.genAst` instead.
   ##
   ## Quasi-quoting operator.
   ## Accepts an expression or a block and returns the AST that represents it.

From ef088710203616404bbaf68e4ff7a9a03c58c2bc Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 20 Mar 2021 01:43:23 -0700
Subject: [PATCH 26/32] _

---
 lib/core/macros.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index dade47a4ae9b0..d9bf07b4cc207 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -539,7 +539,7 @@ proc parseStmt*(s: string): NimNode {.noSideEffect.} =
 
 proc getAst*(macroOrTemplate: untyped): NimNode {.magic: "ExpandToAst", noSideEffect.}
   ## Obtains the AST nodes returned from a macro or template invocation.
-  ## See also `genAst`.
+  ## See also `genasts.genAst`.
   ## Example:
   ##
   ## .. code-block:: nim

From 6e7fcc4d0d8521085813da704a3592f1c6faa691 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 20 Mar 2021 01:44:31 -0700
Subject: [PATCH 27/32] _

---
 lib/std/genasts.nim | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/std/genasts.nim b/lib/std/genasts.nim
index 0dec5690de42f..0add31b867638 100644
--- a/lib/std/genasts.nim
+++ b/lib/std/genasts.nim
@@ -1,4 +1,5 @@
 import std/macros
+
 type GenAstOpt* = enum
   kDirtyTemplate,
     # When set, uses a dirty template in implementation of `genAst`. This
@@ -21,7 +22,7 @@ macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untype
       genAst(a, b, c = true):
         # echo d # not captured => gives `var not init`
         (a, b, c, localFun())
-    doAssert fun(true, false) == (true, false, true, 12)
+    assert fun(true, false) == (true, false, true, 12)
 
   let params = newTree(nnkFormalParams, newEmptyNode())
   let pragmas =

From f160df574bea82d40af9acf928153a0be58cfcf5 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 20 Mar 2021 01:48:25 -0700
Subject: [PATCH 28/32] cleanups

---
 tests/stdlib/mgenast.nim |  6 +++---
 tests/stdlib/tgenast.nim | 14 +++++++-------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/stdlib/mgenast.nim b/tests/stdlib/mgenast.nim
index 74fe8736daf5f..2b5381891fcbe 100644
--- a/tests/stdlib/mgenast.nim
+++ b/tests/stdlib/mgenast.nim
@@ -1,8 +1,8 @@
 import std/genasts
 import std/macros
 
-## Using a enum instead of, say, int, to make apparent potential bugs related to
-## forgetting converting to NimNode via newLit, see https://github.com/nim-lang/Nim/issues/9607
+# Using a enum instead of, say, int, to make apparent potential bugs related to
+# forgetting converting to NimNode via newLit, see bug #9607
 
 type Foo* = enum kfoo0, kfoo1, kfoo2, kfoo3, kfoo4
 
@@ -24,7 +24,7 @@ macro bindme5UseExpose*(): untyped =
 macro bindme5UseExposeFalse*(): untyped =
   genAstOpt({kDirtyTemplate}): myLocalPriv2()
 
-## example from https://github.com/nim-lang/Nim/issues/7889
+# example from bug #7889
 from std/streams import newStringStream, readData, writeData
 
 macro bindme6UseExpose*(): untyped =
diff --git a/tests/stdlib/tgenast.nim b/tests/stdlib/tgenast.nim
index 164c9e52ab2a3..8c420b4057f50 100644
--- a/tests/stdlib/tgenast.nim
+++ b/tests/stdlib/tgenast.nim
@@ -51,7 +51,7 @@ block:
   doAssert x0 == Foo(a: 10)
 
 block:
-  # avoids https://github.com/nim-lang/Nim/issues/7375
+  # avoids bug #7375
   macro fun(b: static[bool], b2: bool): untyped =
     result = newStmtList()
   macro foo(c: bool): untyped =
@@ -62,7 +62,7 @@ block:
   foo(true)
 
 block:
-  # avoids https://github.com/nim-lang/Nim/issues/7589
+  # avoids bug #7589
   # since `==` works with genAst, the problem goes away
   macro foo2(): untyped =
     # result = quote do: # Error: '==' cannot be passed to a procvar
@@ -71,7 +71,7 @@ block:
   doAssert not foo2()
 
 block:
-  # avoids https://github.com/nim-lang/Nim/issues/7726
+  # avoids bug #7726
   # expressions such as `a.len` are just passed as arguments to `genAst`, and
   # caller scope is not polluted with definitions such as `let b = newLit a.len`
   macro foo(): untyped =
@@ -81,7 +81,7 @@ block:
   doAssert foo() == (5, 5)
 
 block:
-  # avoids https://github.com/nim-lang/Nim/issues/9607
+  # avoids bug #9607
   proc fun1(info:LineInfo): string = "bar1"
   proc fun2(info:int): string = "bar2"
 
@@ -108,9 +108,9 @@ block:
   doAssert bar() == ("bar1", "bar2")
 
 block:
-  # example from https://github.com/nim-lang/Nim/issues/7889 works
+  # example from bug #7889 works
   # after changing method call syntax to regular call syntax; this is a
-  # limitation described in https://github.com/nim-lang/Nim/issues/7085
+  # limitation described in bug #7085
   # note that `quote do` would also work after that change in this example.
   doAssert bindme2() == kfoo1
   doAssert bindme3() == kfoo1
@@ -218,7 +218,7 @@ block: # test passing function and type symbols
   type Z1 = type('c')
   doAssert bar(Z1) == (41, 42, 43, 44, "char", "bool", "uint8", "int8")
 
-block: # fix https://github.com/nim-lang/Nim/issues/11986
+block: # fix bug #11986
   proc foo(): auto =
     var s = { 'a', 'b' }
     # var n = quote do: `s` # would print {97, 98}

From 9198b36e20c5ad11e03b33d10075edbf5422cf64 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 20 Mar 2021 01:56:15 -0700
Subject: [PATCH 29/32] test in vm + rt

---
 tests/stdlib/tgenast.nim | 517 ++++++++++++++++++++-------------------
 1 file changed, 263 insertions(+), 254 deletions(-)

diff --git a/tests/stdlib/tgenast.nim b/tests/stdlib/tgenast.nim
index 8c420b4057f50..0904b83dd1a0f 100644
--- a/tests/stdlib/tgenast.nim
+++ b/tests/stdlib/tgenast.nim
@@ -1,260 +1,269 @@
+# xxx also test on js
+
 import std/genasts
 import std/macros
 from std/strformat import `&`
 import ./mgenast
 
-block:
-  macro bar(x0: static Foo, x1: Foo, x2: Foo, xignored: Foo): untyped =
-    let s0 = "not captured!"
-    let s1 = "not captured!"
-    let xignoredLocal = kfoo4
-
-    # newLit optional:
-    let x3 = newLit kfoo4
-    let x3b = kfoo4
-
-    result = genAstOpt({kDirtyTemplate}, s1=true, s2="asdf", x0, x1=x1, x2, x3, x3b):
-      doAssert not declared(xignored)
-      doAssert not declared(xignoredLocal)
-      (s1, s2, s0, x0, x1, x2, x3, x3b)
-
-  let s0 = "caller scope!"
-
-  doAssert bar(kfoo1, kfoo2, kfoo3, kfoo4) ==
-    (true, "asdf", "caller scope!", kfoo1, kfoo2, kfoo3, kfoo4, kfoo4)
-
-block:
-  # doesn't have limitation mentioned in https://github.com/nim-lang/RFCs/issues/122#issue-401636535
-  macro abc(name: untyped): untyped =
-    result = genAst(name):
-      type name = object
-
-  abc(Bar)
-  doAssert Bar.default == Bar()
-
-block:
-  # backticks parser limitations / ambiguities not are an issue with `genAst`:
-  # (#10326 #9745 are fixed but `quote do` still has underlying ambiguity issue
-  # with backticks)
-  type Foo = object
-    a: int
-
-  macro m1(): untyped =
-    # result = quote do: # Error: undeclared identifier: 'a1'
-    result = genAst:
-      template `a1=`(x: var Foo, val: int) =
-        x.a = val
-
-  m1()
-  var x0: Foo
-  x0.a1 = 10
-  doAssert x0 == Foo(a: 10)
-
-block:
-  # avoids bug #7375
-  macro fun(b: static[bool], b2: bool): untyped =
-    result = newStmtList()
-  macro foo(c: bool): untyped =
-    var b = false
-    result = genAst(b, c):
-      fun(b, c)
-
-  foo(true)
-
-block:
-  # avoids bug #7589
-  # since `==` works with genAst, the problem goes away
-  macro foo2(): untyped =
-    # result = quote do: # Error: '==' cannot be passed to a procvar
-    result = genAst:
-      `==`(3,4)
-  doAssert not foo2()
-
-block:
-  # avoids bug #7726
-  # expressions such as `a.len` are just passed as arguments to `genAst`, and
-  # caller scope is not polluted with definitions such as `let b = newLit a.len`
-  macro foo(): untyped =
-    let a = @[1, 2, 3, 4, 5]
-    result = genAst(a, b = a.len): # shows 2 ways to get a.len
-      (a.len, b)
-  doAssert foo() == (5, 5)
-
-block:
-  # avoids bug #9607
-  proc fun1(info:LineInfo): string = "bar1"
-  proc fun2(info:int): string = "bar2"
-
-  macro bar2(args: varargs[untyped]): untyped =
-    let info = args.lineInfoObj
-    let fun1 = bindSym"fun1" # optional; we can remove this and also the
-    # capture of fun1, as show in next example
-    result = genAst(info, fun1):
-      (fun1(info), fun2(info.line))
-  doAssert bar2() == ("bar1", "bar2")
-
-  macro bar3(args: varargs[untyped]): untyped =
-    let info = args.lineInfoObj
-    result = genAst(info):
-      (fun1(info), fun2(info.line))
-  doAssert bar3() == ("bar1", "bar2")
-
-  macro bar(args: varargs[untyped]): untyped =
-    let info = args.lineInfoObj
-    let fun1 = bindSym"fun1"
-    let fun2 = bindSym"fun2"
-    result = genAstOpt({kDirtyTemplate}, info):
-      (fun1(info), fun2(info.line))
-  doAssert bar() == ("bar1", "bar2")
-
-block:
-  # example from bug #7889 works
-  # after changing method call syntax to regular call syntax; this is a
-  # limitation described in bug #7085
-  # note that `quote do` would also work after that change in this example.
-  doAssert bindme2() == kfoo1
-  doAssert bindme3() == kfoo1
-  doAssert not compiles(bindme4()) # correctly gives Error: undeclared identifier: 'myLocalPriv'
-  proc myLocalPriv2(): auto = kfoo2
-  doAssert bindme5UseExpose() == kfoo1
-
-  # example showing hijacking behavior when using `kDirtyTemplate`
-  doAssert bindme5UseExposeFalse() == kfoo2
-    # local `myLocalPriv2` hijacks symbol `mgenast.myLocalPriv2`. In most
-    # use cases this is probably not what macro writer intends as it's
-    # surprising; hence `kDirtyTemplate` is not the default.
-
-  bindme6UseExpose()
-  bindme6UseExposeFalse()
-
-block:
-  macro mbar(x3: Foo, x3b: static Foo): untyped =
-    var x1=kfoo3
-    var x2=newLit kfoo3
-    var x4=kfoo3
-    var xLocal=kfoo3
-
-    proc funLocal(): auto = kfoo4
-
-    result = genAst(x1, x2, x3, x4):
-      # local x1 overrides remote x1
-      when false:
-        # one advantage of using `kDirtyTemplate` is that these would hold:
-        doAssert not declared xLocal
-        doAssert not compiles(echo xLocal)
-        # however, even without it, we at least correctly generate CT error
-        # if trying to use un-captured symbol; this correctly gives:
-        # Error: internal error: environment misses: xLocal
-        echo xLocal
-
-      proc foo1(): auto =
-        # note that `funLocal` is captured implicitly, according to hygienic
-        # template rules; with `kDirtyTemplate` it would not unless
-        # captured in `genAst` capture list explicitly
-        (a0: xRemote, a1: x1, a2: x2, a3: x3, a4: x4, a5: funLocal())
-
-    return result
-
-  proc main()=
-    var xRemote=kfoo1
-    var x1=kfoo2
-    mbar(kfoo4, kfoo4)
-    doAssert foo1() == (a0: kfoo1, a1: kfoo3, a2: kfoo3, a3: kfoo4, a4: kfoo3, a5: kfoo4)
-
-  main()
-
-block:
-  # With `kDirtyTemplate`, the example from #8220 works.
-  # See https://nim-lang.github.io/Nim/strformat.html#limitations for
-  # an explanation of why {.dirty.} is needed.
-  macro foo(): untyped =
-    result = genAstOpt({kDirtyTemplate}):
-      let bar = "Hello, World"
-      &"Let's interpolate {bar} in the string"
-  doAssert foo() == "Let's interpolate Hello, World in the string"
-
-
-block: # nested application of genAst
-  macro createMacro(name, obj, field: untyped): untyped =
-    result = genAst(obj = newDotExpr(obj, field), lit = 10, name, field):
-      # can't reuse `result` here, would clash
-      macro name(arg: untyped): untyped =
-        genAst(arg2=arg): # somehow `arg2` rename is needed
-          (obj, astToStr(field), lit, arg2)
-
-  var x = @[1, 2, 3]
-  createMacro foo, x, len
-  doAssert (foo 20) == (3, "len", 10, 20)
-
-block: # test with kNoNewLit
-  macro bar(): untyped =
-    let s1 = true
-    template boo(x): untyped =
-      fun(x)
-    result = genAstOpt({kNoNewLit}, s1=newLit(s1), s1b=s1): (s1, s1b)
-  doAssert bar() == (true, 1)
-
-block: # sanity check: check passing `{}` also works
-  macro bar(): untyped =
-    result = genAstOpt({}, s1=true): s1
-  doAssert bar() == true
-
-block: # test passing function and type symbols
-  proc z1(): auto = 41
-  type Z4 = type(1'i8)
-  macro bar(Z1: typedesc): untyped =
-    proc z2(): auto = 42
-    proc z3[T](a: T): auto = 43
-    let Z2 = genAst():
-      type(true)
-    let z4 = genAst():
-      proc myfun(): auto = 44
-      myfun
-    type Z3 = type(1'u8)
-    result = genAst(z4, Z1, Z2):
-      # z1, z2, z3, Z3, Z4 are captured automatically
-      # z1, z2, z3 can optionally be specified in capture list
-      (z1(), z2(), z3('a'), z4(), $Z1, $Z2, $Z3, $Z4)
-  type Z1 = type('c')
-  doAssert bar(Z1) == (41, 42, 43, 44, "char", "bool", "uint8", "int8")
-
-block: # fix bug #11986
-  proc foo(): auto =
-    var s = { 'a', 'b' }
-    # var n = quote do: `s` # would print {97, 98}
-    var n = genAst(s): s
-    n.repr
-  static: doAssert foo() == "{'a', 'b'}"
-
-block: # also from #11986
-  macro foo(): untyped =
-    var s = { 'a', 'b' }
-    # quote do:
-    #   let t = `s`
-    #   $typeof(t) # set[range 0..65535(int)]
-    genAst(s):
-      let t = s
-      $typeof(t)
-  doAssert foo() == "set[char]"
-
-block:
-  macro foo(): untyped =
+proc main =
+  block:
+    macro bar(x0: static Foo, x1: Foo, x2: Foo, xignored: Foo): untyped =
+      let s0 = "not captured!"
+      let s1 = "not captured!"
+      let xignoredLocal = kfoo4
+
+      # newLit optional:
+      let x3 = newLit kfoo4
+      let x3b = kfoo4
+
+      result = genAstOpt({kDirtyTemplate}, s1=true, s2="asdf", x0, x1=x1, x2, x3, x3b):
+        doAssert not declared(xignored)
+        doAssert not declared(xignoredLocal)
+        (s1, s2, s0, x0, x1, x2, x3, x3b)
+
+    let s0 = "caller scope!"
+
+    doAssert bar(kfoo1, kfoo2, kfoo3, kfoo4) ==
+      (true, "asdf", "caller scope!", kfoo1, kfoo2, kfoo3, kfoo4, kfoo4)
+
+  block:
+    # doesn't have limitation mentioned in https://github.com/nim-lang/RFCs/issues/122#issue-401636535
+    macro abc(name: untyped): untyped =
+      result = genAst(name):
+        type name = object
+
+    abc(Bar)
+    doAssert Bar.default == Bar()
+
+  block:
+    # backticks parser limitations / ambiguities not are an issue with `genAst`:
+    # (#10326 #9745 are fixed but `quote do` still has underlying ambiguity issue
+    # with backticks)
     type Foo = object
-    template baz2(a: int): untyped = a*10
-    macro baz3(a: int): untyped = newLit 13
-    result = newStmtList()
-
-    result.add genAst(Foo, baz2, baz3) do: # shows you can pass types, templates etc
-      var x: Foo
-      $($typeof(x), baz2(3), baz3(4))
-
-    let ret = genAst() do: # shows you don't have to, since they're inject'd
-      var x: Foo
-      $($typeof(x), baz2(3), baz3(4))
-  doAssert foo() == """("Foo", 30, 13)"""
-
-block: # illustrates how symbol visiblity can be controlled precisely using `mixin`
-  proc locafun1(): auto = "in locafun1 (caller scope)" # this will be used because of `mixin locafun1` => explicit hijacking is ok
-  proc locafun2(): auto = "in locafun2 (caller scope)" # this won't be used => no hijacking
-  proc locafun3(): auto = "in locafun3 (caller scope)"
-  doAssert mixinExample() == ("in locafun1 (caller scope)", "in locafun2", "in locafun3 (caller scope)")
+      a: int
+
+    macro m1(): untyped =
+      # result = quote do: # Error: undeclared identifier: 'a1'
+      result = genAst:
+        template `a1=`(x: var Foo, val: int) =
+          x.a = val
+
+    m1()
+    var x0: Foo
+    x0.a1 = 10
+    doAssert x0 == Foo(a: 10)
+
+  block:
+    # avoids bug #7375
+    macro fun(b: static[bool], b2: bool): untyped =
+      result = newStmtList()
+    macro foo(c: bool): untyped =
+      var b = false
+      result = genAst(b, c):
+        fun(b, c)
+
+    foo(true)
+
+  block:
+    # avoids bug #7589
+    # since `==` works with genAst, the problem goes away
+    macro foo2(): untyped =
+      # result = quote do: # Error: '==' cannot be passed to a procvar
+      result = genAst:
+        `==`(3,4)
+    doAssert not foo2()
+
+  block:
+    # avoids bug #7726
+    # expressions such as `a.len` are just passed as arguments to `genAst`, and
+    # caller scope is not polluted with definitions such as `let b = newLit a.len`
+    macro foo(): untyped =
+      let a = @[1, 2, 3, 4, 5]
+      result = genAst(a, b = a.len): # shows 2 ways to get a.len
+        (a.len, b)
+    doAssert foo() == (5, 5)
+
+  block:
+    # avoids bug #9607
+    proc fun1(info:LineInfo): string = "bar1"
+    proc fun2(info:int): string = "bar2"
+
+    macro bar2(args: varargs[untyped]): untyped =
+      let info = args.lineInfoObj
+      let fun1 = bindSym"fun1" # optional; we can remove this and also the
+      # capture of fun1, as show in next example
+      result = genAst(info, fun1):
+        (fun1(info), fun2(info.line))
+    doAssert bar2() == ("bar1", "bar2")
+
+    macro bar3(args: varargs[untyped]): untyped =
+      let info = args.lineInfoObj
+      result = genAst(info):
+        (fun1(info), fun2(info.line))
+    doAssert bar3() == ("bar1", "bar2")
+
+    macro bar(args: varargs[untyped]): untyped =
+      let info = args.lineInfoObj
+      let fun1 = bindSym"fun1"
+      let fun2 = bindSym"fun2"
+      result = genAstOpt({kDirtyTemplate}, info):
+        (fun1(info), fun2(info.line))
+    doAssert bar() == ("bar1", "bar2")
+
+  block:
+    # example from bug #7889 works
+    # after changing method call syntax to regular call syntax; this is a
+    # limitation described in bug #7085
+    # note that `quote do` would also work after that change in this example.
+    doAssert bindme2() == kfoo1
+    doAssert bindme3() == kfoo1
+    doAssert not compiles(bindme4()) # correctly gives Error: undeclared identifier: 'myLocalPriv'
+    proc myLocalPriv2(): auto = kfoo2
+    doAssert bindme5UseExpose() == kfoo1
+
+    # example showing hijacking behavior when using `kDirtyTemplate`
+    doAssert bindme5UseExposeFalse() == kfoo2
+      # local `myLocalPriv2` hijacks symbol `mgenast.myLocalPriv2`. In most
+      # use cases this is probably not what macro writer intends as it's
+      # surprising; hence `kDirtyTemplate` is not the default.
+
+    when nimvm: # disabled because `newStringStream` is used
+      discard
+    else:
+      bindme6UseExpose()
+      bindme6UseExposeFalse()
+
+  block:
+    macro mbar(x3: Foo, x3b: static Foo): untyped =
+      var x1=kfoo3
+      var x2=newLit kfoo3
+      var x4=kfoo3
+      var xLocal=kfoo3
+
+      proc funLocal(): auto = kfoo4
+
+      result = genAst(x1, x2, x3, x4):
+        # local x1 overrides remote x1
+        when false:
+          # one advantage of using `kDirtyTemplate` is that these would hold:
+          doAssert not declared xLocal
+          doAssert not compiles(echo xLocal)
+          # however, even without it, we at least correctly generate CT error
+          # if trying to use un-captured symbol; this correctly gives:
+          # Error: internal error: environment misses: xLocal
+          echo xLocal
+
+        proc foo1(): auto =
+          # note that `funLocal` is captured implicitly, according to hygienic
+          # template rules; with `kDirtyTemplate` it would not unless
+          # captured in `genAst` capture list explicitly
+          (a0: xRemote, a1: x1, a2: x2, a3: x3, a4: x4, a5: funLocal())
+
+      return result
+
+    proc main()=
+      var xRemote=kfoo1
+      var x1=kfoo2
+      mbar(kfoo4, kfoo4)
+      doAssert foo1() == (a0: kfoo1, a1: kfoo3, a2: kfoo3, a3: kfoo4, a4: kfoo3, a5: kfoo4)
+
+    main()
+
+  block:
+    # With `kDirtyTemplate`, the example from #8220 works.
+    # See https://nim-lang.github.io/Nim/strformat.html#limitations for
+    # an explanation of why {.dirty.} is needed.
+    macro foo(): untyped =
+      result = genAstOpt({kDirtyTemplate}):
+        let bar = "Hello, World"
+        &"Let's interpolate {bar} in the string"
+    doAssert foo() == "Let's interpolate Hello, World in the string"
+
+
+  block: # nested application of genAst
+    macro createMacro(name, obj, field: untyped): untyped =
+      result = genAst(obj = newDotExpr(obj, field), lit = 10, name, field):
+        # can't reuse `result` here, would clash
+        macro name(arg: untyped): untyped =
+          genAst(arg2=arg): # somehow `arg2` rename is needed
+            (obj, astToStr(field), lit, arg2)
+
+    var x = @[1, 2, 3]
+    createMacro foo, x, len
+    doAssert (foo 20) == (3, "len", 10, 20)
+
+  block: # test with kNoNewLit
+    macro bar(): untyped =
+      let s1 = true
+      template boo(x): untyped =
+        fun(x)
+      result = genAstOpt({kNoNewLit}, s1=newLit(s1), s1b=s1): (s1, s1b)
+    doAssert bar() == (true, 1)
+
+  block: # sanity check: check passing `{}` also works
+    macro bar(): untyped =
+      result = genAstOpt({}, s1=true): s1
+    doAssert bar() == true
+
+  block: # test passing function and type symbols
+    proc z1(): auto = 41
+    type Z4 = type(1'i8)
+    macro bar(Z1: typedesc): untyped =
+      proc z2(): auto = 42
+      proc z3[T](a: T): auto = 43
+      let Z2 = genAst():
+        type(true)
+      let z4 = genAst():
+        proc myfun(): auto = 44
+        myfun
+      type Z3 = type(1'u8)
+      result = genAst(z4, Z1, Z2):
+        # z1, z2, z3, Z3, Z4 are captured automatically
+        # z1, z2, z3 can optionally be specified in capture list
+        (z1(), z2(), z3('a'), z4(), $Z1, $Z2, $Z3, $Z4)
+    type Z1 = type('c')
+    doAssert bar(Z1) == (41, 42, 43, 44, "char", "bool", "uint8", "int8")
+
+  block: # fix bug #11986
+    proc foo(): auto =
+      var s = { 'a', 'b' }
+      # var n = quote do: `s` # would print {97, 98}
+      var n = genAst(s): s
+      n.repr
+    static: doAssert foo() == "{'a', 'b'}"
+
+  block: # also from #11986
+    macro foo(): untyped =
+      var s = { 'a', 'b' }
+      # quote do:
+      #   let t = `s`
+      #   $typeof(t) # set[range 0..65535(int)]
+      genAst(s):
+        let t = s
+        $typeof(t)
+    doAssert foo() == "set[char]"
+
+  block:
+    macro foo(): untyped =
+      type Foo = object
+      template baz2(a: int): untyped = a*10
+      macro baz3(a: int): untyped = newLit 13
+      result = newStmtList()
+
+      result.add genAst(Foo, baz2, baz3) do: # shows you can pass types, templates etc
+        var x: Foo
+        $($typeof(x), baz2(3), baz3(4))
+
+      let ret = genAst() do: # shows you don't have to, since they're inject'd
+        var x: Foo
+        $($typeof(x), baz2(3), baz3(4))
+    doAssert foo() == """("Foo", 30, 13)"""
+
+  block: # illustrates how symbol visiblity can be controlled precisely using `mixin`
+    proc locafun1(): auto = "in locafun1 (caller scope)" # this will be used because of `mixin locafun1` => explicit hijacking is ok
+    proc locafun2(): auto = "in locafun2 (caller scope)" # this won't be used => no hijacking
+    proc locafun3(): auto = "in locafun3 (caller scope)"
+    doAssert mixinExample() == ("in locafun1 (caller scope)", "in locafun2", "in locafun3 (caller scope)")
+
+static: main()
+main()

From 71dbcb79dd3d0a4417ce025f8fdafde0a200093f Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 20 Mar 2021 12:01:55 -0700
Subject: [PATCH 30/32] address comments

---
 lib/core/macros.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index d9bf07b4cc207..d80e7366ff434 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -548,8 +548,8 @@ proc getAst*(macroOrTemplate: untyped): NimNode {.magic: "ExpandToAst", noSideEf
   ##     var ast = getAst(BarTemplate())
 
 proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} =
-  ## .. warning:: `quote` comes with many caveats, consider using the new
-  ##   `genasts.genAst` instead.
+  ## .. warning:: `quote` comes with many caveats, use `genasts <genasts.html>`_
+  ##   instead.
   ##
   ## Quasi-quoting operator.
   ## Accepts an expression or a block and returns the AST that represents it.

From 1fafc0aca55342c07579eceb13eddddc434b3868 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sun, 21 Mar 2021 18:26:30 -0700
Subject: [PATCH 31/32] address comment

---
 lib/std/genasts.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/std/genasts.nim b/lib/std/genasts.nim
index 0add31b867638..a6a2a3f91bd39 100644
--- a/lib/std/genasts.nim
+++ b/lib/std/genasts.nim
@@ -1,4 +1,4 @@
-import std/macros
+import macros
 
 type GenAstOpt* = enum
   kDirtyTemplate,

From 429a2f456d6d05c3ea914223516c37b91c287425 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 2 Apr 2021 01:50:55 -0700
Subject: [PATCH 32/32] improve docs

---
 lib/core/macros.nim |  5 ++---
 lib/std/genasts.nim | 32 ++++++++++++++++++++++++--------
 2 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index d80e7366ff434..49c9a999c120e 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -548,9 +548,6 @@ proc getAst*(macroOrTemplate: untyped): NimNode {.magic: "ExpandToAst", noSideEf
   ##     var ast = getAst(BarTemplate())
 
 proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} =
-  ## .. warning:: `quote` comes with many caveats, use `genasts <genasts.html>`_
-  ##   instead.
-  ##
   ## Quasi-quoting operator.
   ## Accepts an expression or a block and returns the AST that represents it.
   ## Within the quoted AST, you are able to interpolate NimNode expressions
@@ -563,6 +560,8 @@ proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} =
   ##
   ## A custom operator interpolation needs accent quoted (``) whenever it resolves
   ## to a symbol.
+  ##
+  ## See also `genasts <genasts.html>`_ which avoids some issues with `quote`.
   runnableExamples:
     macro check(ex: untyped) =
       # this is a simplified version of the check macro from the
diff --git a/lib/std/genasts.nim b/lib/std/genasts.nim
index a6a2a3f91bd39..c5f51e5d99c2e 100644
--- a/lib/std/genasts.nim
+++ b/lib/std/genasts.nim
@@ -15,14 +15,30 @@ macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untype
   ## AST that represents it. Local `{.inject.}` symbols (e.g. procs) are captured
   ## unless `kDirtyTemplate in options`.
   runnableExamples:
-    macro fun(a: bool, b: static bool): untyped =
-      let c = false # doesn't override parameter `c`
-      var d = 11 # var => gensym'd
-      proc localFun(): auto = 12 # proc => inject'd
+    # This example shows how one could write a simplified version of `unittest.check`.
+    import std/[macros, strutils]
+    macro check2(cond: bool): untyped =
+      assert cond.kind == nnkInfix, "$# not implemented" % $cond.kind
+      result = genAst(cond, s = repr(cond), lhs = cond[1], rhs = cond[2]):
+        # each local symbol we access must be explicitly captured
+        if not cond:
+          doAssert false, "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs]
+    let a = 3
+    check2 a*2 == a+3
+    if false: check2 a*2 < a+1 # would error with: 'a * 2 < a + 1'' failed: lhs: '6', rhs: '4'
+
+  runnableExamples:
+    # This example goes in more details about the capture semantics.
+    macro fun(a: string, b: static bool): untyped =
+      let c = 'z'
+      var d = 11 # implicitly {.gensym.} and needs to be captured for use in `genAst`.
+      proc localFun(): auto = 12 # implicitly {.inject.}, doesn't need to be captured.
       genAst(a, b, c = true):
-        # echo d # not captured => gives `var not init`
-        (a, b, c, localFun())
-    assert fun(true, false) == (true, false, true, 12)
+        # `a`, `b` are captured explicitly, `c` is a local definition masking `c = 'z'`.
+        const b2 = b # macro static param `b` is forwarded here as a static param.
+        # `echo d` would give: `var not init` because `d` is not captured.
+        (a & a, b, c, localFun()) # localFun can be called without capture.
+    assert fun("ab", false) == ("abab", false, true, 12)
 
   let params = newTree(nnkFormalParams, newEmptyNode())
   let pragmas =
@@ -67,5 +83,5 @@ macro genAstOpt*(options: static set[GenAstOpt], args: varargs[untyped]): untype
   result.add newCall(bindSym"getAst", call)
 
 template genAst*(args: varargs[untyped]): untyped =
-  ## convenience wrapper around `genAstOpt`
+  ## Convenience wrapper around `genAstOpt`.
   genAstOpt({}, args)