From d6a16020fe98b467459544ee4cfafe61d681f918 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 23 Mar 2021 08:52:00 +0100 Subject: [PATCH] IC: backend: remember produced type information (#17440) --- compiler/ast.nim | 15 ++++---------- compiler/ccgtypes.nim | 14 ++++++++++--- compiler/ic/cbackend.nim | 43 ++++++++++++++++++++++++++++++++------- compiler/ic/ic.nim | 11 ++++++++-- compiler/ic/rodfiles.nim | 1 + compiler/modulegraphs.nim | 29 ++++++++++++++++++++++---- compiler/passes.nim | 5 +++++ compiler/sem.nim | 2 +- compiler/semdata.nim | 15 +------------- 9 files changed, 93 insertions(+), 42 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 50a2fb58c83d2..1f3d5f129d7ff 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1229,17 +1229,10 @@ proc newSym*(symKind: TSymKind, name: PIdent, id: ItemId, owner: PSym, result = PSym(name: name, kind: symKind, flags: {}, info: info, itemId: id, options: options, owner: owner, offset: defaultOffset) when false: - if id.item > 2141: - let s = getStackTrace() - const words = ["createTypeBoundOps", - "initOperators", - "generateInstance", - "semIdentDef", "addLocalDecl"] - for w in words: - if w in s: - x.inc w - return - x.inc "" + if id.module == 48 and id.item == 39: + writeStackTrace() + echo "kind ", symKind, " ", name.s + if owner != nil: echo owner.name.s proc astdef*(s: PSym): PNode = # get only the definition (initializer) portion of the ast diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 73ee9cb8a26ce..e17a555426e25 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1386,7 +1386,6 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope = let owner = t.skipTypes(typedescPtrs).itemId.module if owner != m.module.position and moduleOpenForCodegen(m, owner): # make sure the type info is created in the owner module - assert m.g.modules[owner] != nil discard genTypeInfoV2(m.g.modules[owner], origType, info) # reference the type info as extern here discard cgsym(m, "TNimTypeV2") @@ -1456,18 +1455,27 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope = result = "NTI$1$2_" % [rope(typeToC(t)), rope($sig)] m.typeInfoMarker[sig] = result - let owner = t.skipTypes(typedescPtrs).itemId.module + let old = m.g.graph.emittedTypeInfo.getOrDefault($result) + if old != FileIndex(0): + discard cgsym(m, "TNimType") + discard cgsym(m, "TNimNode") + declareNimType(m, "TNimType", result, old.int) + return prefixTI.rope & result & ")".rope + + var owner = t.skipTypes(typedescPtrs).itemId.module if owner != m.module.position and moduleOpenForCodegen(m, owner): # make sure the type info is created in the owner module - assert m.g.modules[owner] != nil discard genTypeInfoV1(m.g.modules[owner], origType, info) # reference the type info as extern here discard cgsym(m, "TNimType") discard cgsym(m, "TNimNode") declareNimType(m, "TNimType", result, owner) return prefixTI.rope & result & ")".rope + else: + owner = m.module.position.int32 m.g.typeInfoMarker[sig] = (str: result, owner: owner) + rememberEmittedTypeInfo(m.g.graph, FileIndex(owner), $result) case t.kind of tyEmpty, tyVoid: result = rope"0" diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index 88b2a9477224c..34ee59d525727 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -18,7 +18,7 @@ ## also doing cross-module dependency tracking and DCE that we don't need ## anymore. DCE is now done as prepass over the entire packed module graph. -import std/[packedsets, algorithm] +import std/[packedsets, algorithm, tables] # std/intsets would give `UnusedImport`, pending https://github.com/nim-lang/Nim/issues/14246 import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen, pathutils, extccomp, msgs] @@ -45,6 +45,10 @@ proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var Alive finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info)) +proc replayTypeInfo(g: ModuleGraph; m: var LoadedModule; origin: FileIndex) = + for x in mitems(m.fromDisk.emittedTypeInfo): + g.emittedTypeInfo[x] = origin + proc addFileToLink(config: ConfigRef; m: PSym) = let filename = AbsoluteFile toFullPath(config, m.position.FileIndex) let ext = @@ -59,11 +63,28 @@ proc addFileToLink(config: ConfigRef; m: PSym) = flags: {CfileFlag.Cached}) addFileToCompile(config, cf) -proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool = +when defined(debugDce): + import std / [os, packedsets] + +proc storeAliveSymsImpl(asymFile: AbsoluteFile; s: seq[int32]) = + var f = rodfiles.create(asymFile.string) + f.storeHeader() + f.storeSection aliveSymsSection + f.storeSeq(s) + close f + +template prepare {.dirty.} = let asymFile = toRodFile(config, AbsoluteFile toFullPath(config, position.FileIndex), ".alivesyms") var s = newSeqOfCap[int32](alive[position].len) for a in items(alive[position]): s.add int32(a) sort(s) + +proc storeAliveSyms(config: ConfigRef; position: int; alive: AliveSyms) = + prepare() + storeAliveSymsImpl(asymFile, s) + +proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool = + prepare() var f2 = rodfiles.open(asymFile.string) f2.loadHeader() f2.loadSection aliveSymsSection @@ -73,12 +94,17 @@ proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool if f2.err == ok and oldData == s: result = false else: + when defined(debugDce): + let oldAsSet = toPackedSet[int32](oldData) + let newAsSet = toPackedSet[int32](s) + echo "set of live symbols changed ", asymFile.changeFileExt("rod"), " ", position, " ", f2.err + echo "in old but not in new ", oldAsSet.difference(newAsSet) + echo "in new but not in old ", newAsSet.difference(oldAsSet) + + if execShellCmd(getAppFilename() & " rod " & quoteShell(asymFile.changeFileExt("rod"))) != 0: + echo "command failed" result = true - var f = rodfiles.create(asymFile.string) - f.storeHeader() - f.storeSection aliveSymsSection - f.storeSeq(s) - close f + storeAliveSymsImpl(asymFile, s) proc generateCode*(g: ModuleGraph) = ## The single entry point, generate C(++) code for the entire @@ -95,6 +121,8 @@ proc generateCode*(g: ModuleGraph) = assert false of storing, outdated: generateCodeForModule(g, g.packed[i], alive) + closeRodFile(g, g.packed[i].module) + storeAliveSyms(g.config, g.packed[i].module.position, alive) of loaded: # Even though this module didn't change, DCE might trigger a change. # Consider this case: Module A uses symbol S from B and B does not use @@ -104,3 +132,4 @@ proc generateCode*(g: ModuleGraph) = generateCodeForModule(g, g.packed[i], alive) else: addFileToLink(g.config, g.packed[i].module) + replayTypeInfo(g, g.packed[i], FileIndex(i)) diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 99a68e0f03a92..230b4d087de48 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -42,6 +42,8 @@ type methodsPerType*: seq[(PackedItemId, int, PackedItemId)] enumToStringProcs*: seq[(PackedItemId, PackedItemId)] + emittedTypeInfo*: seq[string] + sh*: Shared cfg: PackedConfig @@ -58,7 +60,7 @@ type config*: ConfigRef proc isActive*(e: PackedEncoder): bool = e.config != nil -proc disable*(e: var PackedEncoder) = e.config = nil +proc disable(e: var PackedEncoder) = e.config = nil template primConfigFields(fn: untyped) {.dirty.} = fn backend @@ -552,6 +554,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef loadSeqSection attachedOpsSection, m.attachedOps loadSeqSection methodsPerTypeSection, m.methodsPerType loadSeqSection enumToStringProcsSection, m.enumToStringProcs + loadSeqSection typeInfoSection, m.emittedTypeInfo close(f) result = f.err @@ -614,6 +617,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac storeSeqSection attachedOpsSection, m.attachedOps storeSeqSection methodsPerTypeSection, m.methodsPerType storeSeqSection enumToStringProcsSection, m.enumToStringProcs + storeSeqSection typeInfoSection, m.emittedTypeInfo close(f) encoder.disable() @@ -1139,7 +1143,10 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) = echo "all symbols" for i in 0..high(m.sh.syms): - echo " ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i + if m.sh.syms[i].name != LitId(0): + echo " ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i, " kind ", m.sh.syms[i].kind + else: + echo " local ID: ", i, " kind ", m.sh.syms[i].kind echo "symbols: ", m.sh.syms.len, " types: ", m.sh.types.len, " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len, diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index fa0a7c73428ac..a518870f8db5d 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -36,6 +36,7 @@ type attachedOpsSection methodsPerTypeSection enumToStringProcsSection + typeInfoSection # required by the backend aliveSymsSection # beware, this is stored in a `.alivesyms` file. RodFileError* = enum diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index de3773ca5ea97..9ac76457c1de7 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -11,9 +11,8 @@ ## represents a complete Nim project. Single modules can either be kept in RAM ## or stored in a rod-file. -import ast, astalgo, intsets, tables, options, lineinfos, hashes, idents, - btrees, md5, ropes, msgs - +import std / [intsets, tables, hashes, md5] +import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils import ic / [packed_ast, ic] type @@ -60,6 +59,7 @@ type attachedOps*: array[TTypeAttachedOp, Table[ItemId, PSym]] # Type ID, destructors, etc. methodsPerType*: Table[ItemId, seq[(int, LazySym)]] # Type ID, attached methods enumToStringProcs*: Table[ItemId, LazySym] + emittedTypeInfo*: Table[string, FileIndex] startupPackedConfig*: PackedConfig packageSyms*: TStrTable @@ -68,7 +68,6 @@ type importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies suggestMode*: bool # whether we are in nimsuggest mode or not. invalidTransitiveClosure: bool - systemModuleComplete*: bool inclToMod*: Table[FileIndex, FileIndex] # mapping of include file to the # first module that included it importStack*: seq[FileIndex] # The current import stack. Used for detecting recursive @@ -435,6 +434,7 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = result.canonTypes = initTable[SigHash, PType]() result.symBodyHashes = initTable[int, SigHash]() result.operators = initOperators(result) + result.emittedTypeInfo = initTable[string, FileIndex]() proc resetAllModules*(g: ModuleGraph) = initStrTable(g.packageSyms) @@ -455,6 +455,27 @@ proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym = elif fileIdx.int32 < g.ifaces.len: result = g.ifaces[fileIdx.int32].module +proc rememberEmittedTypeInfo*(g: ModuleGraph; m: FileIndex; ti: string) = + #assert(not isCachedModule(g, m.int32)) + if g.config.symbolFiles != disabledSf: + #assert g.encoders[m.int32].isActive + g.packed[m.int32].fromDisk.emittedTypeInfo.add ti + +proc closeRodFile*(g: ModuleGraph; m: PSym) = + if g.config.symbolFiles in {readOnlySf, v2Sf}: + # For stress testing we seek to reload the symbols from memory. This + # way much of the logic is tested but the test is reproducible as it does + # not depend on the hard disk contents! + let mint = m.position + saveRodFile(toRodFile(g.config, AbsoluteFile toFullPath(g.config, FileIndex(mint))), + g.encoders[mint], g.packed[mint].fromDisk) + elif g.config.symbolFiles == stressTest: + # debug code, but maybe a good idea for production? Could reduce the compiler's + # memory consumption considerably at the cost of more loads from disk. + let mint = m.position + simulateCachedModule(g, m, g.packed[mint].fromDisk) + g.packed[mint].status = loaded + proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) = diff --git a/compiler/passes.nim b/compiler/passes.nim index 11799b1222bd1..3debce1f65bd9 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -187,4 +187,9 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; closeParser(p) if s.kind != llsStdIn: break closePasses(graph, a) + if graph.config.backend notin {backendC, backendCpp, backendObjc}: + # We only write rod files here if no C-like backend is active. + # The C-like backends have been patched to support the IC mechanism. + # They are responsible for closing the rod files. See `cbackend.nim`. + closeRodFile(graph, module) result = true diff --git a/compiler/sem.nim b/compiler/sem.nim index bb42b5c1ba5c9..6b3f0c80dd51f 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -661,7 +661,7 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = result.add(c.module.ast) popOwner(c) popProcCon(c) - saveRodFile(c) + sealRodFile(c) const semPass* = makePass(myOpen, myProcess, myClose, isFrontend = true) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 5cd440cc12f0b..c655047e2dcfc 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -560,23 +560,10 @@ proc addToGenericCache*(c: PContext; s: PSym; inst: PType) = if c.config.symbolFiles != disabledSf: storeTypeInst(c.encoder, c.packedRepr, s, inst) -proc saveRodFile*(c: PContext) = +proc sealRodFile*(c: PContext) = if c.config.symbolFiles != disabledSf: if c.graph.vm != nil: for (m, n) in PCtx(c.graph.vm).vmstateDiff: if m == c.module: addPragmaComputation(c, n) - if sfSystemModule in c.module.flags: - c.graph.systemModuleComplete = true c.idgen.sealed = true # no further additions are allowed - if c.config.symbolFiles != stressTest: - # For stress testing we seek to reload the symbols from memory. This - # way much of the logic is tested but the test is reproducible as it does - # not depend on the hard disk contents! - saveRodFile(toRodFile(c.config, AbsoluteFile toFullPath(c.config, FileIndex c.module.position)), - c.encoder, c.packedRepr) - else: - # debug code, but maybe a good idea for production? Could reduce the compiler's - # memory consumption considerably at the cost of more loads from disk. - simulateCachedModule(c.graph, c.module, c.packedRepr) - c.graph.packed[c.module.position].status = loaded