store full definition AST for consts, fix noRewrite (nim-lang#20115)

* continue nim-lang#9582 for consts, close nim-lang#9331, fix nim-lang#20114

also move extractPragma to ast to pave the way for things like {.strdefine: "abc".} etc

* changelog correctly

* fix jsgen

* update tgetimpl

* fix sighashes

* fix nim-lang#19766, add comment about postfix

* fix noRewrite LOL

refs nim-lang#16620

* fix changelog

* fix destructors

changelog.md

@@ -51,6 +51,9 @@
 
 - Static linking against OpenSSL versions below 1.1, previously done by
   setting `-d:openssl10`, is no longer supported.
+
+- `macros.getImpl` for `const` symbols now returns the full definition node
+  (as `nnkConstDef`) rather than the AST of the constant value.
 
 - ORC is now the default memory management strategy. Use
   `--mm:refc` for a transition period.

compiler/ast.nim

@@ -1233,6 +1233,25 @@ proc getDeclPragma*(n: PNode): PNode =
   if result != nil:
     assert result.kind == nkPragma, $(result.kind, n.kind)
 
+proc extractPragma*(s: PSym): PNode =
+  ## gets the pragma node of routine/type/var/let/const symbol `s`
+  if s.kind in routineKinds:
+    result = s.ast[pragmasPos]
+  elif s.kind in {skType, skVar, skLet, skConst}:
+    if s.ast != nil and s.ast.len > 0:
+      if s.ast[0].kind == nkPragmaExpr and s.ast[0].len > 1:
+        # s.ast = nkTypedef / nkPragmaExpr / [nkSym, nkPragma]
+        result = s.ast[0][1]
+  assert result == nil or result.kind == nkPragma
+
+proc skipPragmaExpr*(n: PNode): PNode =
+  ## if pragma expr, give the node the pragmas are applied to,
+  ## otherwise give node itself
+  if n.kind == nkPragmaExpr:
+    result = n[0]
+  else:
+    result = n
+
 when defined(useNodeIds):
   const nodeIdToDebug* = -1 # 2322968
   var gNodeId: int
@@ -1321,7 +1340,7 @@ proc newSym*(symKind: TSymKind, name: PIdent, id: ItemId, owner: PSym,
 
 proc astdef*(s: PSym): PNode =
   # get only the definition (initializer) portion of the ast
-  if s.ast != nil and s.ast.kind == nkIdentDefs:
+  if s.ast != nil and s.ast.kind in {nkIdentDefs, nkConstDef}:
     s.ast[2]
   else:
     s.ast

compiler/ccgexprs.nim

@@ -2856,7 +2856,7 @@ proc genConstSetup(p: BProc; sym: PSym): bool =
   useHeader(m, sym)
   if sym.loc.k == locNone:
     fillBackendName(p.module, sym)
-    fillLoc(sym.loc, locData, sym.ast, OnStatic)
+    fillLoc(sym.loc, locData, sym.astdef, OnStatic)
   if m.hcrOn: incl(sym.loc.flags, lfIndirect)
   result = lfNoDecl notin sym.loc.flags
 
@@ -2882,7 +2882,7 @@ proc genConstDefinition(q: BModule; p: BProc; sym: PSym) =
   var data = newRopeAppender()
   data.addf("N_LIB_PRIVATE NIM_CONST $1 $2 = ",
            [getTypeDesc(q, sym.typ), actualConstName])
-  genBracedInit(q.initProc, sym.ast, isConst = true, sym.typ, data)
+  genBracedInit(q.initProc, sym.astdef, isConst = true, sym.typ, data)
   data.addf(";$n", [])
   q.s[cfsData].add data
   if q.hcrOn:
@@ -2942,7 +2942,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
     of skConst:
       if isSimpleConst(sym.typ):
         var lit = newRopeAppender()
-        genLiteral(p, sym.ast, sym.typ, lit)
+        genLiteral(p, sym.astdef, sym.typ, lit)
         putIntoDest(p, d, n, lit, OnStatic)
       elif useAliveDataFromDce in p.module.flags:
         genConstHeader(p.module, p.module, p, sym)

compiler/ccgliterals.nim

@@ -21,7 +21,7 @@ template detectVersion(field, corename) =
     if core == nil or core.kind != skConst:
       m.g.field = 1
     else:
-      m.g.field = toInt(ast.getInt(core.ast))
+      m.g.field = toInt(ast.getInt(core.astdef))
   result = m.g.field
 
 proc detectStrVersion(m: BModule): int =

compiler/guards.nim

@@ -200,7 +200,7 @@ proc highBound*(conf: ConfigRef; x: PNode; o: Operators): PNode =
              nkIntLit.newIntNode(lastOrd(conf, typ))
            elif typ.kind == tySequence and x.kind == nkSym and
                x.sym.kind == skConst:
-             nkIntLit.newIntNode(x.sym.ast.len-1)
+             nkIntLit.newIntNode(x.sym.astdef.len-1)
            else:
              o.opAdd.buildCall(o.opLen.buildCall(x), minusOne())
   result.info = x.info

compiler/injectdestructors.nim

@@ -866,9 +866,9 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
         if it.kind == nkVarTuple and hasDestructor(c, ri.typ):
           let x = lowerTupleUnpacking(c.graph, it, c.idgen, c.owner)
           result.add p(x, c, s, consumed)
-        elif it.kind == nkIdentDefs and hasDestructor(c, it[0].typ):
+        elif it.kind == nkIdentDefs and hasDestructor(c, skipPragmaExpr(it[0]).typ):
           for j in 0..<it.len-2:
-            let v = it[j]
+            let v = skipPragmaExpr(it[j])
             if v.kind == nkSym:
               if sfCompileTime in v.sym.flags: continue
               pVarTopLevel(v, c, s, result)
@@ -1125,7 +1125,7 @@ proc injectDefaultCalls(n: PNode, c: var Con) =
       if it.kind == nkIdentDefs and it[^1].kind == nkEmpty:
         computeUninit(c)
         for j in 0..<it.len-2:
-          let v = it[j]
+          let v = skipPragmaExpr(it[j])
           doAssert v.kind == nkSym
           if c.uninit.contains(v.sym.id):
             it[^1] = genDefaultCall(v.sym.typ, c, v.info)

compiler/jsgen.nim

@@ -1484,7 +1484,7 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
     if s.loc.r == "":
       internalError(p.config, n.info, "symbol has no generated name: " & s.name.s)
     if sfCompileTime in s.flags:
-      genVarInit(p, s, if s.ast != nil: s.ast else: newNodeI(nkEmpty, s.info))
+      genVarInit(p, s, if s.astdef != nil: s.astdef else: newNodeI(nkEmpty, s.info))
     if s.kind == skParam:
       genCopyForParamIfNeeded(p, n)
     let k = mapType(p, s.typ)
@@ -1945,8 +1945,8 @@ proc genVarStmt(p: PProc, n: PNode) =
 proc genConstant(p: PProc, c: PSym) =
   if lfNoDecl notin c.loc.flags and not p.g.generatedSyms.containsOrIncl(c.id):
     let oldBody = move p.body
-    #genLineDir(p, c.ast)
-    genVarInit(p, c, c.ast)
+    #genLineDir(p, c.astdef)
+    genVarInit(p, c, c.astdef)
     p.g.constants.add(p.body)
     p.body = oldBody
 

compiler/parampatterns.nim

@@ -224,7 +224,7 @@ proc isAssignable*(owner: PSym, n: PNode): TAssignableResult =
     const kinds = {skVar, skResult, skTemp, skParam, skLet, skForVar}
     if n.sym.kind == skParam:
       result = if n.sym.typ.kind in {tyVar, tySink}: arLValue else: arAddressableConst
-    elif n.sym.kind == skConst and dontInlineConstant(n, n.sym.ast):
+    elif n.sym.kind == skConst and dontInlineConstant(n, n.sym.astdef):
       result = arAddressableConst
     elif n.sym.kind in kinds:
       if n.sym.kind in {skParam, skLet, skForVar}:

compiler/patterns.nim

@@ -146,7 +146,7 @@ proc matches(c: PPatternContext, p, n: PNode): bool =
   elif n.kind == nkSym and n.sym.kind == skConst:
     # try both:
     if p.kind == nkSym: result = p.sym == n.sym
-    elif matches(c, p, n.sym.ast): result = true
+    elif matches(c, p, n.sym.astdef): result = true
   elif p.kind == nkPattern:
     # pattern operators: | *
     let opr = p[0].ident.s

compiler/pragmas.nim

@@ -313,7 +313,7 @@ proc expectDynlibNode(c: PContext, n: PNode): PNode =
     # {.dynlib: myGetProcAddr(...).}
     result = c.semExpr(c, n[1])
     if result.kind == nkSym and result.sym.kind == skConst:
-      result = result.sym.ast # look it up
+      result = result.sym.astdef # look it up
     if result.typ == nil or result.typ.kind notin {tyPointer, tyString, tyProc}:
       localError(c.config, n.info, errStringLiteralExpected)
       result = newEmptyStrNode(c, n)
@@ -998,7 +998,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of wExplain:
         sym.flags.incl sfExplain
       of wDeprecated:
-        if sym != nil and sym.kind in routineKinds + {skType, skVar, skLet}:
+        if sym != nil and sym.kind in routineKinds + {skType, skVar, skLet, skConst}:
           if it.kind in nkPragmaCallKinds: discard getStrLitNode(c, it)
           incl(sym.flags, sfDeprecated)
         elif sym != nil and sym.kind != skModule:
@@ -1247,7 +1247,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
     elif comesFromPush and whichKeyword(ident) != wInvalid:
       discard "ignore the .push pragma; it doesn't apply"
     else:
-      if sym == nil or (sym.kind in {skVar, skLet, skParam, skIterator,
+      if sym == nil or (sym.kind in {skVar, skLet, skConst, skParam, skIterator,
                         skField, skProc, skFunc, skConverter, skMethod, skType}):
         n[i] = semCustomPragma(c, it)
       elif sym != nil:

compiler/semcall.nim

@@ -544,8 +544,8 @@ proc semResolvedCall(c: PContext, x: TCandidate,
       for s in instantiateGenericParamList(c, gp, x.bindings):
         case s.kind
         of skConst:
-          if not s.ast.isNil:
-            x.call.add s.ast
+          if not s.astdef.isNil:
+            x.call.add s.astdef
           else:
             x.call.add c.graph.emptyNode
         of skType:

compiler/semexprs.nim

@@ -108,7 +108,7 @@ proc semSymGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
   result = symChoice(c, n, s, scClosed)
 
 proc inlineConst(c: PContext, n: PNode, s: PSym): PNode {.inline.} =
-  result = copyTree(s.ast)
+  result = copyTree(s.astdef)
   if result.isNil:
     localError(c.config, n.info, "constant of type '" & typeToString(s.typ) & "' has no value")
     result = newSymNode(s)
@@ -1230,7 +1230,7 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
       # It is clear that ``[]`` means two totally different things. Thus, we
       # copy `x`'s AST into each context, so that the type fixup phase can
       # deal with two different ``[]``.
-      if s.ast.safeLen == 0: result = inlineConst(c, n, s)
+      if s.astdef.safeLen == 0: result = inlineConst(c, n, s)
       else: result = newSymNode(s, n.info)
     of tyStatic:
       if typ.n != nil:

compiler/semfold.nim

@@ -517,12 +517,12 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
               "{.intdefine.} const was set to an invalid integer: '" &
                 g.config.symbols[s.name.s] & "'")
         else:
-          result = copyTree(s.ast)
+          result = copyTree(s.astdef)
       of mStrDefine:
         if isDefined(g.config, s.name.s):
           result = newStrNodeT(g.config.symbols[s.name.s], n, g)
         else:
-          result = copyTree(s.ast)
+          result = copyTree(s.astdef)
       of mBoolDefine:
         if isDefined(g.config, s.name.s):
           try:
@@ -532,9 +532,9 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
               "{.booldefine.} const was set to an invalid bool: '" &
                 g.config.symbols[s.name.s] & "'")
         else:
-          result = copyTree(s.ast)
+          result = copyTree(s.astdef)
       else:
-        result = copyTree(s.ast)
+        result = copyTree(s.astdef)
     of skProc, skFunc, skMethod:
       result = n
     of skParam:

compiler/sempass2.nim

@@ -1101,12 +1101,13 @@ proc track(tracked: PEffects, n: PNode) =
           for i in 0..<child.len-2:
             createTypeBoundOps(tracked, child[i].typ, child.info)
         else:
-          createTypeBoundOps(tracked, child[0].typ, child.info)
+          createTypeBoundOps(tracked, skipPragmaExpr(child[0]).typ, child.info)
       if child.kind == nkIdentDefs and last.kind != nkEmpty:
         for i in 0..<child.len-2:
-          initVar(tracked, child[i], volatileCheck=false)
-          addAsgnFact(tracked.guards, child[i], last)
-          notNilCheck(tracked, last, child[i].typ)
+          let a = skipPragmaExpr(child[i])
+          initVar(tracked, a, volatileCheck=false)
+          addAsgnFact(tracked.guards, a, last)
+          notNilCheck(tracked, last, a.typ)
       elif child.kind == nkVarTuple and last.kind != nkEmpty:
         for i in 0..<child.len-1:
           if child[i].kind == nkEmpty or

compiler/semstmts.nim

@@ -694,28 +694,25 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
         if def.kind != nkEmpty:
           if sfThread in v.flags: localError(c.config, def.info, errThreadvarCannotInit)
         setVarType(c, v, typ)
+        # this is needed for the evaluation pass, guard checking
+        #  and custom pragmas:
         b = newNodeI(nkIdentDefs, a.info)
         if importantComments(c.config):
           # keep documentation information:
           b.comment = a.comment
-        b.add newSymNode(v)
-        # keep type desc for doc generator
-        b.add a[^2]
-        b.add copyTree(def)
-        addToVarSection(c, result, n, b)
-        # this is needed for the evaluation pass, guard checking
-        #  and custom pragmas:
-        var ast = newNodeI(nkIdentDefs, a.info)
+        # postfix not generated here (to generate, get rid of it in transf)
         if a[j].kind == nkPragmaExpr:
           var p = newNodeI(nkPragmaExpr, a.info)
           p.add newSymNode(v)
-          p.add a[j][1].copyTree
-          ast.add p
+          p.add a[j][1]
+          b.add p
         else:
-          ast.add newSymNode(v)
-        ast.add a[^2].copyTree
-        ast.add def
-        v.ast = ast
+          b.add newSymNode(v)
+        # keep type desc for doc generator
+        b.add a[^2]
+        b.add copyTree(def)
+        addToVarSection(c, result, n, b)
+        v.ast = b
       else:
         if def.kind in {nkPar, nkTupleConstr}: v.ast = def[j]
         # bug #7663, for 'nim check' this can be a non-tuple:
@@ -811,12 +808,21 @@ proc semConst(c: PContext, n: PNode): PNode =
 
       if a.kind != nkVarTuple:
         setVarType(c, v, typ)
-        v.ast = def               # no need to copy
+        when false:
+          v.ast = def               # no need to copy
         b = newNodeI(nkConstDef, a.info)
         if importantComments(c.config): b.comment = a.comment
-        b.add newSymNode(v)
+        # postfix not generated here (to generate, get rid of it in transf)
+        if a[j].kind == nkPragmaExpr:
+          var p = newNodeI(nkPragmaExpr, a.info)
+          p.add newSymNode(v)
+          p.add a[j][1].copyTree
+          b.add p
+        else:
+          b.add newSymNode(v)
         b.add a[1]
         b.add copyTree(def)
+        v.ast = b
       else:
         setVarType(c, v, typ[j])
         v.ast = if def[j].kind != nkExprColonExpr: def[j]
@@ -2330,6 +2336,11 @@ proc setLine(n: PNode, info: TLineInfo) =
     for i in 0..<n.safeLen: setLine(n[i], info)
     n.info = info
 
+proc recursiveSetFlag(n: PNode, flag: TNodeFlag) =
+  if n != nil:
+    for i in 0..<n.safeLen: recursiveSetFlag(n[i], flag)
+    incl(n.flags, flag)
+
 proc semPragmaBlock(c: PContext, n: PNode; expectedType: PType = nil): PNode =
   checkSonsLen(n, 2, c.config)
   let pragmaList = n[0]
@@ -2354,7 +2365,7 @@ proc semPragmaBlock(c: PContext, n: PNode; expectedType: PType = nil): PNode =
   for i in 0..<pragmaList.len:
     case whichPragma(pragmaList[i])
     of wLine: setLine(result, pragmaList[i].info)
-    of wNoRewrite: incl(result.flags, nfNoRewrite)
+    of wNoRewrite: recursiveSetFlag(result, nfNoRewrite)
     else: discard
 
 proc semStaticStmt(c: PContext, n: PNode): PNode =

compiler/sighashes.nim

@@ -333,7 +333,7 @@ proc hashVarSymBody(graph: ModuleGraph, c: var MD5Context, s: PSym) =
     c &= hashNonProc(s)
     # this one works for let and const but not for var. True variables can change value
     # later on. it is user resposibility to hash his global state if required
-    if s.ast != nil and s.ast.kind == nkIdentDefs:
+    if s.ast != nil and s.ast.kind in {nkIdentDefs, nkConstDef}:
       hashBodyTree(graph, c, s.ast[^1])
     else:
       hashBodyTree(graph, c, s.ast)

compiler/suggest.nim

@@ -535,16 +535,6 @@ proc suggestSym*(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym; i
       if parentFileIndex == conf.m.trackPos.fileIndex:
         suggestResult(conf, symToSuggest(g, s, isLocal=false, ideOutline, info, 100, PrefixMatch.None, false, 0))
 
-proc extractPragma(s: PSym): PNode =
-  if s.kind in routineKinds:
-    result = s.ast[pragmasPos]
-  elif s.kind in {skType, skVar, skLet}:
-    if s.ast != nil and s.ast.len > 0:
-      if s.ast[0].kind == nkPragmaExpr and s.ast[0].len > 1:
-        # s.ast = nkTypedef / nkPragmaExpr / [nkSym, nkPragma]
-        result = s.ast[0][1]
-  doAssert result == nil or result.kind == nkPragma
-
 proc warnAboutDeprecated(conf: ConfigRef; info: TLineInfo; s: PSym) =
   var pragmaNode: PNode
   pragmaNode = if s.kind == skEnumField: extractPragma(s.owner) else: extractPragma(s)

compiler/transf.nim

@@ -184,10 +184,12 @@ proc transformVarSection(c: PTransf, v: PNode): PNode =
     if it.kind == nkCommentStmt:
       result[i] = it
     elif it.kind == nkIdentDefs:
-      if it[0].kind == nkSym:
+      var vn = it[0]
+      if vn.kind == nkPragmaExpr: vn = vn[0]
+      if vn.kind == nkSym:
         internalAssert(c.graph.config, it.len == 3)
-        let x = freshVar(c, it[0].sym)
-        idNodeTablePut(c.transCon.mapping, it[0].sym, x)
+        let x = freshVar(c, vn.sym)
+        idNodeTablePut(c.transCon.mapping, vn.sym, x)
         var defs = newTransNode(nkIdentDefs, it.info, 3)
         if importantComments(c.graph.config):
           # keep documentation information:
@@ -1036,7 +1038,7 @@ proc transform(c: PTransf, n: PNode): PNode =
     result = transformAsgn(c, n)
   of nkIdentDefs, nkConstDef:
     result = newTransNode(n)
-    result[0] = transform(c, n[0])
+    result[0] = transform(c, skipPragmaExpr(n[0]))
     # Skip the second son since it only contains an unsemanticized copy of the
     # variable type used by docgen
     let last = n.len-1

compiler/vmgen.nim

@@ -2000,7 +2000,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
       elif importcCond(c, s): c.importcSym(n.info, s)
       genLit(c, n, dest)
     of skConst:
-      let constVal = if s.ast != nil: s.ast else: s.typ.n
+      let constVal = if s.astdef != nil: s.astdef else: s.typ.n
       gen(c, constVal, dest)
     of skEnumField:
       # we never reach this case - as of the time of this comment,