From b3e804a3fbff3374eaf7b1121b51d02812f386d1 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 25 May 2021 10:40:06 -0700 Subject: [PATCH 01/11] simplify extccomp.nim json logic via jsonutils --- compiler/extccomp.nim | 117 +++++++++++++----------------------------- lib/std/jsonutils.nim | 8 ++- 2 files changed, 42 insertions(+), 83 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 1cea9edebe642..fa45de20c0e06 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -942,101 +942,56 @@ proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile = # works out of the box with `hashMainCompilationParams`. result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json") +import std/jsonutils proc writeJsonBuildInstructions*(conf: ConfigRef) = - # xxx use std/json instead, will result in simpler, more maintainable code. - template lit(x: string) = f.write x - template str(x: string) = - buf.setLen 0 - escapeJson(x, buf) - f.write buf - - proc cfiles(conf: ConfigRef; f: File; buf: var string; clist: CfileList, isExternal: bool) = - var comma = false + proc cfiles(clist: CfileList): JsonNode = + result = newJArray() for i, it in clist: + echo (it.cname, it.flags, "D20210525T094642") if CfileFlag.Cached in it.flags: continue - let compileCmd = getCompileCFileCmd(conf, it) - if comma: lit ",\L" else: comma = true - lit "[" - str it.cname.string - lit ", " - str compileCmd - lit "]" - - proc linkfiles(conf: ConfigRef; f: File; buf, objfiles: var string; clist: CfileList; - llist: seq[string]) = - var pastStart = false + result.add toJson([it.cname.string, getCompileCFileCmd(conf, it)]) + + proc linkfiles(objfiles: var string; clist: CfileList; llist: seq[string]): JsonNode = + result = newJArray() template impl(path) = - let path2 = quoteShell(path) - objfiles.add(' ') - objfiles.add(path2) - if pastStart: lit ",\L" - str path2 - pastStart = true + objfiles.add ' ' & quoteShell(path) + result.add path.toJson # xxx WAS:quoteShell for it in llist: let objfile = if noAbsolutePaths(conf): it.extractFilename else: it - impl(addFileExt(objfile, CC[conf.cCompiler].objExt)) + impl(objfile.addFileExt(CC[conf.cCompiler].objExt)) for it in clist: impl(it.obj) - lit "\L" - proc depfiles(conf: ConfigRef; f: File; buf: var string) = - var i = 0 + proc depfiles(): JsonNode = + result = newJArray() for it in conf.m.fileInfos: let path = it.fullPath.string if isAbsolute(path): # TODO: else? - if i > 0: lit "],\L" - lit "[" - str path - lit ", " - str $secureHashFile(path) - inc i - lit "]\L" - + result.add toJson((path, $secureHashFile(path))) - var buf = newStringOfCap(50) - let jsonFile = conf.jsonBuildInstructionsFile - conf.jsonBuildFile = jsonFile let output = conf.absOutFile - - var f: File - if open(f, jsonFile.string, fmWrite): - lit "{\L" - lit "\"outputFile\": " - str $output - - lit ",\L\"compile\":[\L" - cfiles(conf, f, buf, conf.toCompile, false) - lit "],\L\"link\":[\L" - var objfiles = "" - # XXX add every file here that is to link - linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink) - - lit "],\L\"linkcmd\": " - str getLinkCmd(conf, output, objfiles) - - lit ",\L\"extraCmds\": " - lit $(%* getExtraCmds(conf, conf.absOutFile)) - - lit ",\L\"stdinInput\": " - lit $(%* conf.projectIsStdin) - lit ",\L\"projectIsCmd\": " - lit $(%* conf.projectIsCmd) - lit ",\L\"cmdInput\": " - lit $(%* conf.cmdInput) - lit ",\L\"currentDir\": " - lit $(%* getCurrentDir()) - - if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"): - lit ",\L\"cmdline\": " - str conf.commandLine - lit ",\L\"depfiles\":[\L" - depfiles(conf, f, buf) - lit "],\L\"nimexe\": \L" - str hashNimExe() - lit "\L" - - lit "\L}\L" - close(f) + var objfiles = "" + let j = ( + outputFile: $output, + compile: cfiles(conf.toCompile), + link: linkfiles(objfiles, conf.toCompile, conf.externalToLink), + # XXX add every file here that is to link + linkcmd: getLinkCmd(conf, output, objfiles), + extraCmds: getExtraCmds(conf, conf.absOutFile), + stdinInput: conf.projectIsStdin, + projectIsCmd: conf.projectIsCmd, + cmdInput: conf.cmdInput, + currentDir: getCurrentDir(), + ).toJson + if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"): + let j2 = ( + cmdline: conf.commandLine, + depfiles: depfiles(), + nimexe: hashNimExe(), + ).toJson + for k,v in j2: j[k] = v + conf.jsonBuildFile = conf.jsonBuildInstructionsFile + conf.jsonBuildFile.string.writeFile(j.pretty) proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool = if not fileExists(jsonFile): return true diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 1e222e3a254fe..18cc2fe96f0e8 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -286,8 +286,12 @@ proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode = result = newJArray() for v in a.fields: result.add toJson(v, opt) elif T is ref | ptr: - if system.`==`(a, nil): result = newJNull() - else: result = toJson(a[], opt) + when T is JsonNode: + # xxx we could customize this by ignoring json special case + result = a + else: + if system.`==`(a, nil): result = newJNull() + else: result = toJson(a[], opt) elif T is array | seq | set: result = newJArray() for ai in a: result.add toJson(ai, opt) From b7fb8b94d6e9841cb4dad958c4b05a2e430491e1 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 25 May 2021 17:01:47 -0700 Subject: [PATCH 02/11] works --- compiler/extccomp.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index fa45de20c0e06..e52f9fe31009c 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -943,11 +943,11 @@ proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile = result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json") import std/jsonutils + proc writeJsonBuildInstructions*(conf: ConfigRef) = proc cfiles(clist: CfileList): JsonNode = result = newJArray() for i, it in clist: - echo (it.cname, it.flags, "D20210525T094642") if CfileFlag.Cached in it.flags: continue result.add toJson([it.cname.string, getCompileCFileCmd(conf, it)]) From 85310797a8a6f8e0de6d3bb0a01c14542b863783 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 25 May 2021 17:41:02 -0700 Subject: [PATCH 03/11] BuildCache --- compiler/extccomp.nim | 47 ++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index e52f9fe31009c..24da02009ceb6 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -944,34 +944,46 @@ proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile = import std/jsonutils +type BuildCache = object + outputFile: string + compile: seq[(string, string)] + link: seq[string] + linkcmd: string + extraCmds: seq[string] + stdinInput: bool + projectIsCmd: bool + cmdInput: string + currentDir: string + cmdline: string + depfiles: seq[(string, string)] + nimexe: string + proc writeJsonBuildInstructions*(conf: ConfigRef) = - proc cfiles(clist: CfileList): JsonNode = - result = newJArray() + proc cfiles(clist: CfileList): seq[(string, string)] = for i, it in clist: - if CfileFlag.Cached in it.flags: continue - result.add toJson([it.cname.string, getCompileCFileCmd(conf, it)]) + if CfileFlag.Cached notin it.flags: + result.add (it.cname.string, getCompileCFileCmd(conf, it)) - proc linkfiles(objfiles: var string; clist: CfileList; llist: seq[string]): JsonNode = - result = newJArray() + proc linkfiles(objfiles: var string; clist: CfileList; llist: seq[string]): seq[string] = template impl(path) = objfiles.add ' ' & quoteShell(path) - result.add path.toJson # xxx WAS:quoteShell + result.add path.string # xxx WAS:quoteShell for it in llist: let objfile = if noAbsolutePaths(conf): it.extractFilename else: it impl(objfile.addFileExt(CC[conf.cCompiler].objExt)) for it in clist: impl(it.obj) - proc depfiles(): JsonNode = - result = newJArray() + proc depfiles(): seq[(string, string)] = for it in conf.m.fileInfos: let path = it.fullPath.string if isAbsolute(path): # TODO: else? - result.add toJson((path, $secureHashFile(path))) + result.add (path, $secureHashFile(path)) let output = conf.absOutFile var objfiles = "" - let j = ( + + var bcache = BuildCache( outputFile: $output, compile: cfiles(conf.toCompile), link: linkfiles(objfiles, conf.toCompile, conf.externalToLink), @@ -982,16 +994,13 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = projectIsCmd: conf.projectIsCmd, cmdInput: conf.cmdInput, currentDir: getCurrentDir(), - ).toJson + ) if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"): - let j2 = ( - cmdline: conf.commandLine, - depfiles: depfiles(), - nimexe: hashNimExe(), - ).toJson - for k,v in j2: j[k] = v + bcache.cmdline = conf.commandLine + bcache.depfiles = depfiles() + bcache.nimexe = hashNimExe() conf.jsonBuildFile = conf.jsonBuildInstructionsFile - conf.jsonBuildFile.string.writeFile(j.pretty) + conf.jsonBuildFile.string.writeFile(bcache.toJson.pretty) proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool = if not fileExists(jsonFile): return true From 9678a78b5caf1435ffed39ad23834cc88a22a3b1 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 25 May 2021 17:48:27 -0700 Subject: [PATCH 04/11] simpler --- compiler/extccomp.nim | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 24da02009ceb6..2ac55c4647227 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -15,6 +15,7 @@ import ropes, platform, condsyms, options, msgs, lineinfos, pathutils import os, strutils, osproc, std/sha1, streams, sequtils, times, strtabs, json +from sugar import collect type TInfoCCProp* = enum # properties of the C compiler: @@ -959,12 +960,8 @@ type BuildCache = object nimexe: string proc writeJsonBuildInstructions*(conf: ConfigRef) = - proc cfiles(clist: CfileList): seq[(string, string)] = - for i, it in clist: - if CfileFlag.Cached notin it.flags: - result.add (it.cname.string, getCompileCFileCmd(conf, it)) - - proc linkfiles(objfiles: var string; clist: CfileList; llist: seq[string]): seq[string] = + var objfiles = "" + proc linkfiles(clist: CfileList; llist: seq[string]): seq[string] = template impl(path) = objfiles.add ' ' & quoteShell(path) result.add path.string # xxx WAS:quoteShell @@ -974,30 +971,24 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = for it in clist: impl(it.obj) - proc depfiles(): seq[(string, string)] = - for it in conf.m.fileInfos: - let path = it.fullPath.string - if isAbsolute(path): # TODO: else? - result.add (path, $secureHashFile(path)) - let output = conf.absOutFile - var objfiles = "" - var bcache = BuildCache( outputFile: $output, - compile: cfiles(conf.toCompile), - link: linkfiles(objfiles, conf.toCompile, conf.externalToLink), - # XXX add every file here that is to link + compile: collect(for i, it in conf.toCompile: + if CfileFlag.Cached notin it.flags: (it.cname.string, getCompileCFileCmd(conf, it))), + link: linkfiles(conf.toCompile, conf.externalToLink), linkcmd: getLinkCmd(conf, output, objfiles), extraCmds: getExtraCmds(conf, conf.absOutFile), stdinInput: conf.projectIsStdin, projectIsCmd: conf.projectIsCmd, cmdInput: conf.cmdInput, - currentDir: getCurrentDir(), - ) + currentDir: getCurrentDir()) if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"): bcache.cmdline = conf.commandLine - bcache.depfiles = depfiles() + bcache.depfiles = collect(for it in conf.m.fileInfos: + let path = it.fullPath.string + if isAbsolute(path): # TODO: else? + (path, $secureHashFile(path))) bcache.nimexe = hashNimExe() conf.jsonBuildFile = conf.jsonBuildInstructionsFile conf.jsonBuildFile.string.writeFile(bcache.toJson.pretty) From fd1f3706348cbdbf16268228effc61d02db57152 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 25 May 2021 19:10:08 -0700 Subject: [PATCH 05/11] simpler2 --- compiler/extccomp.nim | 56 +++++++++---------------------------------- 1 file changed, 11 insertions(+), 45 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 2ac55c4647227..8417cf15a356a 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -997,52 +997,18 @@ proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: Absolute if not fileExists(jsonFile): return true if not fileExists(conf.absOutFile): return true result = false - try: - let data = json.parseFile(jsonFile.string) - for key in "depfiles cmdline stdinInput currentDir".split: - if not data.hasKey(key): return true - if getCurrentDir() != data["currentDir"].getStr: - # fixes bug #16271 - # Note that simply comparing `expandFilename(projectFile)` would - # not be sufficient in case other flags depend implicitly on `getCurrentDir`, - # and would require much more care. Simply re-compiling is safer for now. - # A better strategy for future work would be to cache (with an LRU cache) - # the N most recent unique build instructions, as done with `rdmd`, - # which is both robust and avoids recompilation when switching back and forth - # between projects, see https://github.com/timotheecour/Nim/issues/199 - return true - let oldCmdLine = data["cmdline"].getStr - if conf.commandLine != oldCmdLine: - return true - if hashNimExe() != data["nimexe"].getStr: - return true - let stdinInput = data["stdinInput"].getBool - let projectIsCmd = data["projectIsCmd"].getBool - if conf.projectIsStdin or stdinInput: - # could optimize by returning false if stdin input was the same, - # but I'm not sure how to get full stding input - return true - - if conf.projectIsCmd or projectIsCmd: - if not (conf.projectIsCmd and projectIsCmd): return true - if not data.hasKey("cmdInput"): return true - let cmdInput = data["cmdInput"].getStr - if cmdInput != conf.cmdInput: return true - - let depfilesPairs = data["depfiles"] - doAssert depfilesPairs.kind == JArray - for p in depfilesPairs: - doAssert p.kind == JArray - # >= 2 for forwards compatibility with potential later .json files: - doAssert p.len >= 2 - let depFilename = p[0].getStr - let oldHashValue = p[1].getStr - let newHashValue = $secureHashFile(depFilename) - if oldHashValue != newHashValue: - return true + var bcache: BuildCache + try: bcache.fromJson(jsonFile.string.parseFile) except IOError, OSError, ValueError: - echo "Warning: JSON processing failed: ", getCurrentExceptionMsg() - result = true + stderr.write "Warning: JSON processing failed: $#\n" % getCurrentExceptionMsg() + return true + if bcache.currentDir != getCurrentDir() or # fixes bug #16271 + bcache.cmdline != conf.commandLine or bcache.nimexe != hashNimExe() or + bcache.projectIsCmd != conf.projectIsCmd or conf.cmdInput != bcache.cmdInput: return true + if bcache.stdinInput or conf.projectIsStdin: return true + # xxx optimize by returning false if stdin input was the same + for (file, hash) in bcache.depfiles: + if $secureHashFile(file) != hash: return true proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) = try: From 4b1713fd08e16a694911e84c936fbb964bc6c5a3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 25 May 2021 19:15:35 -0700 Subject: [PATCH 06/11] simpler3 --- compiler/extccomp.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 8417cf15a356a..263d26d054f9e 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -994,15 +994,14 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = conf.jsonBuildFile.string.writeFile(bcache.toJson.pretty) proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool = - if not fileExists(jsonFile): return true - if not fileExists(conf.absOutFile): return true - result = false + if not fileExists(jsonFile) or not fileExists(conf.absOutFile): return true var bcache: BuildCache try: bcache.fromJson(jsonFile.string.parseFile) except IOError, OSError, ValueError: stderr.write "Warning: JSON processing failed: $#\n" % getCurrentExceptionMsg() return true if bcache.currentDir != getCurrentDir() or # fixes bug #16271 + bcache.outputFile != conf.absOutFile.string or bcache.cmdline != conf.commandLine or bcache.nimexe != hashNimExe() or bcache.projectIsCmd != conf.projectIsCmd or conf.cmdInput != bcache.cmdInput: return true if bcache.stdinInput or conf.projectIsStdin: return true From 23ea41ba106ac934e428c0d9f03abaf26ef19165 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 25 May 2021 19:31:35 -0700 Subject: [PATCH 07/11] simplify4 --- compiler/extccomp.nim | 65 ++++++++++++++++--------------------------- lib/std/jsonutils.nim | 8 ++---- 2 files changed, 26 insertions(+), 47 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 263d26d054f9e..90489d5ac28b0 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -945,7 +945,9 @@ proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile = import std/jsonutils +const cacheVersion = "D20210525T192431" # update when `BuildCache` spec changes type BuildCache = object + cacheVersion: string outputFile: string compile: seq[(string, string)] link: seq[string] @@ -973,6 +975,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = let output = conf.absOutFile var bcache = BuildCache( + cacheVersion: cacheVersion, outputFile: $output, compile: collect(for i, it in conf.toCompile: if CfileFlag.Cached notin it.flags: (it.cname.string, getCompileCFileCmd(conf, it))), @@ -998,10 +1001,10 @@ proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: Absolute var bcache: BuildCache try: bcache.fromJson(jsonFile.string.parseFile) except IOError, OSError, ValueError: - stderr.write "Warning: JSON processing failed: $#\n" % getCurrentExceptionMsg() + stderr.write "Warning: JSON processing failed for $#: $#\n" % [jsonFile.string.parseFile, getCurrentExceptionMsg()] return true if bcache.currentDir != getCurrentDir() or # fixes bug #16271 - bcache.outputFile != conf.absOutFile.string or + bcache.cacheVersion != cacheVersion or bcache.outputFile != conf.absOutFile.string or bcache.cmdline != conf.commandLine or bcache.nimexe != hashNimExe() or bcache.projectIsCmd != conf.projectIsCmd or conf.cmdInput != bcache.cmdInput: return true if bcache.stdinInput or conf.projectIsStdin: return true @@ -1010,47 +1013,27 @@ proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: Absolute if $secureHashFile(file) != hash: return true proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) = - try: - let data = json.parseFile(jsonFile.string) - let output = data["outputFile"].getStr - createDir output.parentDir - let outputCurrent = $conf.absOutFile - if output != outputCurrent: - # previously, any specified output file would be silently ignored; - # simply copying won't work in some cases, for example with `extraCmds`, - # so we just make it an error, user should use same command for jsonscript - # as was used with --compileOnly. - globalError(conf, gCmdLineInfo, "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % [outputCurrent, output, jsonFile.string]) - - let toCompile = data["compile"] - doAssert toCompile.kind == JArray - var cmds: TStringSeq - var prettyCmds: TStringSeq - let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx]) - for c in toCompile: - doAssert c.kind == JArray - doAssert c.len >= 2 - - cmds.add(c[1].getStr) - prettyCmds.add displayProgressCC(conf, c[0].getStr, c[1].getStr) - - execCmdsInParallel(conf, cmds, prettyCb) - - let linkCmd = data["linkcmd"] - doAssert linkCmd.kind == JString - execLinkCmd(conf, linkCmd.getStr) - if data.hasKey("extraCmds"): - let extraCmds = data["extraCmds"] - doAssert extraCmds.kind == JArray - for cmd in extraCmds: - doAssert cmd.kind == JString, $cmd.kind - let cmd2 = cmd.getStr - execExternalProgram(conf, cmd2, hintExecuting) - + var bcache: BuildCache + try: bcache.fromJson(jsonFile.string.parseFile) except: let e = getCurrentException() - conf.quitOrRaise "\ncaught exception:\n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() & - "error evaluating JSON file: " & jsonFile.string + conf.quitOrRaise "\ncaught exception:\n$#\nstacktrace:\n$#error evaluating JSON file: $#" % + [e.msg, e.getStackTrace(), jsonFile.string] + let output = bcache.outputFile + createDir output.parentDir + let outputCurrent = $conf.absOutFile + if output != outputCurrent or bcache.cacheVersion != cacheVersion: + globalError(conf, gCmdLineInfo, + "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % + [outputCurrent, output, jsonFile.string]) + var cmds, prettyCmds: TStringSeq + let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx]) + for (name, cmd) in bcache.compile: + cmds.add cmd + prettyCmds.add displayProgressCC(conf, name, cmd) + execCmdsInParallel(conf, cmds, prettyCb) + execLinkCmd(conf, bcache.linkcmd) + for cmd in bcache.extraCmds: execExternalProgram(conf, cmd, hintExecuting) proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope = for it in list: diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 18cc2fe96f0e8..1e222e3a254fe 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -286,12 +286,8 @@ proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode = result = newJArray() for v in a.fields: result.add toJson(v, opt) elif T is ref | ptr: - when T is JsonNode: - # xxx we could customize this by ignoring json special case - result = a - else: - if system.`==`(a, nil): result = newJNull() - else: result = toJson(a[], opt) + if system.`==`(a, nil): result = newJNull() + else: result = toJson(a[], opt) elif T is array | seq | set: result = newJArray() for ai in a: result.add toJson(ai, opt) From 5780d66581750ecbd85cc49deef2b2b7f20306bb Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 25 May 2021 19:49:54 -0700 Subject: [PATCH 08/11] fix #18084 --- compiler/extccomp.nim | 8 ++++++-- compiler/nimconf.nim | 11 ++++------- compiler/options.nim | 4 ++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 90489d5ac28b0..f6c5f92c141a7 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -945,7 +945,7 @@ proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile = import std/jsonutils -const cacheVersion = "D20210525T192431" # update when `BuildCache` spec changes +const cacheVersion = "D20210525T193831" # update when `BuildCache` spec changes type BuildCache = object cacheVersion: string outputFile: string @@ -953,6 +953,7 @@ type BuildCache = object link: seq[string] linkcmd: string extraCmds: seq[string] + configFiles: seq[string] # the hash shouldn't be needed stdinInput: bool projectIsCmd: bool cmdInput: string @@ -985,6 +986,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = stdinInput: conf.projectIsStdin, projectIsCmd: conf.projectIsCmd, cmdInput: conf.cmdInput, + configFiles: conf.configFiles.mapIt(it.string), currentDir: getCurrentDir()) if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"): bcache.cmdline = conf.commandLine @@ -999,11 +1001,13 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool = if not fileExists(jsonFile) or not fileExists(conf.absOutFile): return true var bcache: BuildCache + # bcache.fromJson(jsonFile.string.parseFile) try: bcache.fromJson(jsonFile.string.parseFile) except IOError, OSError, ValueError: - stderr.write "Warning: JSON processing failed for $#: $#\n" % [jsonFile.string.parseFile, getCurrentExceptionMsg()] + stderr.write "Warning: JSON processing failed for $#: $#\n" % [jsonFile.string, getCurrentExceptionMsg()] return true if bcache.currentDir != getCurrentDir() or # fixes bug #16271 + bcache.configFiles != conf.configFiles.mapIt(it.string) or bcache.cacheVersion != cacheVersion or bcache.outputFile != conf.absOutFile.string or bcache.cmdline != conf.commandLine or bcache.nimexe != hashNimExe() or bcache.projectIsCmd != conf.projectIsCmd or conf.cmdInput != bcache.cmdInput: return true diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index b63e5a0ad28d4..94c93a2838554 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -240,13 +240,10 @@ proc getSystemConfigPath*(conf: ConfigRef; filename: RelativeFile): AbsoluteFile proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) = setDefaultLibpath(conf) - - var configFiles = newSeq[AbsoluteFile]() - template readConfigFile(path) = let configPath = path if readConfigFile(configPath, cache, conf): - configFiles.add(configPath) + conf.configFiles.add(configPath) template runNimScriptIfExists(path: AbsoluteFile, isMain = false) = let p = path # eval once @@ -256,7 +253,7 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: elif conf.projectIsCmd: s = llStreamOpen(conf.cmdInput) if s == nil and fileExists(p): s = llStreamOpen(p, fmRead) if s != nil: - configFiles.add(p) + conf.configFiles.add(p) runNimScript(cache, p, idgen, freshDefines = false, conf, s) if optSkipSystemConfigFile notin conf.globalOptions: @@ -295,12 +292,12 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: let scriptFile = conf.projectFull.changeFileExt("nims") let scriptIsProj = scriptFile == conf.projectFull template showHintConf = - for filename in configFiles: + for filename in conf.configFiles: # delayed to here so that `hintConf` is honored rawMessage(conf, hintConf, filename.string) if conf.cmd == cmdNimscript: showHintConf() - configFiles.setLen 0 + conf.configFiles.setLen 0 if conf.cmd != cmdIdeTools: if conf.cmd == cmdNimscript: runNimScriptIfExists(conf.projectFull, isMain = true) diff --git a/compiler/options.nim b/compiler/options.nim index 9cbb747c97993..88d717950cb0b 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -307,7 +307,7 @@ type ## should be run ideCmd*: IdeCmd oldNewlines*: bool - cCompiler*: TSystemCC + cCompiler*: TSystemCC # the used compiler modifiedyNotes*: TNoteKinds # notes that have been set/unset from either cmdline/configs cmdlineNotes*: TNoteKinds # notes that have been set/unset from cmdline foreignPackageNotes*: TNoteKinds @@ -352,7 +352,7 @@ type docRoot*: string ## see nim --fullhelp for --docRoot docCmd*: string ## see nim --fullhelp for --docCmd - # the used compiler + configFiles*: seq[AbsoluteFile] # config files (cfg,nims) cIncludes*: seq[AbsoluteDir] # directories to search for included files cLibs*: seq[AbsoluteDir] # directories to search for lib files cLinkedLibs*: seq[string] # libraries to link From 3f72dd798a705c56f2741d1ebf5aa922d3abd817 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 25 May 2021 20:41:37 -0700 Subject: [PATCH 09/11] simplify further --- compiler/extccomp.nim | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index f6c5f92c141a7..0cdfe0b325774 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -963,25 +963,18 @@ type BuildCache = object nimexe: string proc writeJsonBuildInstructions*(conf: ConfigRef) = - var objfiles = "" - proc linkfiles(clist: CfileList; llist: seq[string]): seq[string] = - template impl(path) = - objfiles.add ' ' & quoteShell(path) - result.add path.string # xxx WAS:quoteShell - for it in llist: - let objfile = if noAbsolutePaths(conf): it.extractFilename else: it - impl(objfile.addFileExt(CC[conf.cCompiler].objExt)) - for it in clist: - impl(it.obj) - - let output = conf.absOutFile + var linkFiles = collect(for it in conf.externalToLink: + var it = it + if conf.noAbsolutePaths: it = it.extractFilename + it.addFileExt(CC[conf.cCompiler].objExt)) + for it in conf.toCompile: linkFiles.add it.obj.string var bcache = BuildCache( cacheVersion: cacheVersion, - outputFile: $output, + outputFile: conf.absOutFile.string, compile: collect(for i, it in conf.toCompile: if CfileFlag.Cached notin it.flags: (it.cname.string, getCompileCFileCmd(conf, it))), - link: linkfiles(conf.toCompile, conf.externalToLink), - linkcmd: getLinkCmd(conf, output, objfiles), + link: linkFiles, + linkcmd: getLinkCmd(conf, conf.absOutFile, linkFiles.quoteShellCommand), extraCmds: getExtraCmds(conf, conf.absOutFile), stdinInput: conf.projectIsStdin, projectIsCmd: conf.projectIsCmd, From b70267f920d5394a5e1b087678463bc663a65c91 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 25 May 2021 20:43:20 -0700 Subject: [PATCH 10/11] fixup --- compiler/extccomp.nim | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 0cdfe0b325774..ea2b09866d56e 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -14,8 +14,7 @@ import ropes, platform, condsyms, options, msgs, lineinfos, pathutils -import os, strutils, osproc, std/sha1, streams, sequtils, times, strtabs, json -from sugar import collect +import std/[os, strutils, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar] type TInfoCCProp* = enum # properties of the C compiler: @@ -943,8 +942,6 @@ proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile = # works out of the box with `hashMainCompilationParams`. result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json") -import std/jsonutils - const cacheVersion = "D20210525T193831" # update when `BuildCache` spec changes type BuildCache = object cacheVersion: string @@ -994,7 +991,6 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) = proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool = if not fileExists(jsonFile) or not fileExists(conf.absOutFile): return true var bcache: BuildCache - # bcache.fromJson(jsonFile.string.parseFile) try: bcache.fromJson(jsonFile.string.parseFile) except IOError, OSError, ValueError: stderr.write "Warning: JSON processing failed for $#: $#\n" % [jsonFile.string, getCurrentExceptionMsg()] From 168c57a6df6a2016c2d24c8a088a8ff875940a29 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 26 May 2021 00:24:03 -0700 Subject: [PATCH 11/11] workaround for bootstrap that can be removed after updating csources_v1 >= 1.2 --- lib/std/jsonutils.nim | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 1e222e3a254fe..4da37f8c97f3c 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -34,6 +34,23 @@ import macros from enumutils import symbolName from typetraits import OrdinalEnum +when not defined(nimFixedForwardGeneric): + # xxx remove pending csources_v1 update >= 1.2.0 + proc to[T](node: JsonNode, t: typedesc[T]): T = + when T is string: node.getStr + elif T is bool: node.getBool + else: static: doAssert false, $T # support as needed (only needed during bootstrap) + proc isNamedTuple(T: typedesc): bool = # old implementation + when T isnot tuple: result = false + else: + var t: T + for name, _ in t.fieldPairs: + when name == "Field0": return compiles(t.Field0) + else: return true + return false +else: + proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + type Joptions* = object # xxx rename FromJsonOptions ## Options controlling the behavior of `fromJson`. @@ -56,7 +73,6 @@ proc initToJsonOptions*(): ToJsonOptions = ## initializes `ToJsonOptions` with sane options. ToJsonOptions(enumMode: joptEnumOrd) -proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} template distinctBase[T](a: T): untyped = distinctBase(typeof(a))(a)