From e7084c9dc6b4aa05fd90ed034def770258ba52da Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 10 Apr 2022 16:26:11 -0600 Subject: [PATCH 1/4] Use startprocess and async to make building faster --- src/nimibook/builds.nim | 43 +++++++++++++++++++++++++-------------- src/nimibook/commands.nim | 1 + 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/nimibook/builds.nim b/src/nimibook/builds.nim index 2d43f87..e9da55c 100644 --- a/src/nimibook/builds.nim +++ b/src/nimibook/builds.nim @@ -1,19 +1,22 @@ -import std / [os, strutils] +import std / [os, strutils, asyncdispatch, osproc] import nimibook / [types, commands, themes] import nimib -proc buildNim*(entry: Entry, srcDir: string, nimOptions: seq[string]): bool = +proc buildNim*(entry: Entry, srcDir: string, nimOptions: seq[string]): Future[bool] {.async.} = let cmd = "nim" args = @["r"] & nimOptions & @[srcDir / entry.path] # "-d:release", "-f", "--verbosity:0", "--hints:off" debugEcho "[Executing] ", cmd, " ", args.join(" ") - if execShellCmd(cmd & " " & args.join(" ")) != 0: - echo "[nimibook.error] error while processing ", entry.path - return false - return true + let process = startProcess(cmd, args = args, options = {poUsePath, poStdErrToStdOut}) + while process.running(): + await sleepAsync(10) + + result = process.peekexitCode == 0 + process.close() + -proc buildMd*(entry: Entry): bool = +proc buildMd*(entry: Entry): Future[bool] {.async.}= try: nbInit(theme = useNimibook, thisFileRel = entry.path) nbText nb.source @@ -24,25 +27,35 @@ proc buildMd*(entry: Entry): bool = echo "[nimibook.error] error while processing ", entry.path return false -proc build*(entry: Entry, srcDir: string, nimOptions: seq[string]): bool = +proc build*(entry: Entry, srcDir: string, nimOptions: seq[string]): Future[bool] {.async.} = let splitted = entry.path.splitFile() - if splitted.ext == ".nim": - return buildNim(entry, srcDir, nimOptions) - elif splitted.ext == ".md": - return buildMd(entry) + case splitted.ext + of ".nim": + return await buildNim(entry, srcDir, nimOptions) + of ".md": + return await buildMd(entry) else: echo "[nimibook.error] invalid file extension (must be one of .nim, .md): ", splitted.ext return false proc build*(book: Book, nimOptions: seq[string] = @[]) = - var buildErrors: seq[string] + var + buildPaths: seq[string] + buildFutures: seq[Future[bool]] dump book for entry in book.toc.entries: if entry.isDraft: continue echo "[nimibook] build entry: ", entry.path - if not build(entry, book.srcDir, nimOptions): - buildErrors.add entry.path + buildFutures.add build(entry, book.srcDir, nimOptions) + buildPaths.add entry.path + + var buildErrors: seq[string] + let finished = waitfor all buildFutures + for i, success in finished: + if not success: + buildErrors.add buildPaths[i] + if len(buildErrors) > 0: echo "[nimibook.error] ", len(buildErrors), " build errors:" for err in buildErrors: diff --git a/src/nimibook/commands.nim b/src/nimibook/commands.nim index 2d0c153..f53b020 100644 --- a/src/nimibook/commands.nim +++ b/src/nimibook/commands.nim @@ -4,6 +4,7 @@ import jsony proc dump*(book: Book) = var book = book + discard existsOrCreateDir(book.homeDir()) let uri = normalizedPath(book.homeDir / "book.json") echo "[nimibook] dumping ", uri writeFile(uri, book.toJson) From af7a1be291d5532243ab48e14441352a43589d94 Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 10 Apr 2022 16:39:00 -0600 Subject: [PATCH 2/4] write log file on error --- src/nimibook/builds.nim | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/nimibook/builds.nim b/src/nimibook/builds.nim index e9da55c..d757d93 100644 --- a/src/nimibook/builds.nim +++ b/src/nimibook/builds.nim @@ -1,4 +1,4 @@ -import std / [os, strutils, asyncdispatch, osproc] +import std / [os, strutils, asyncdispatch, osproc, streams] import nimibook / [types, commands, themes] import nimib @@ -8,13 +8,21 @@ proc buildNim*(entry: Entry, srcDir: string, nimOptions: seq[string]): Future[bo args = @["r"] & nimOptions & @[srcDir / entry.path] # "-d:release", "-f", "--verbosity:0", "--hints:off" debugEcho "[Executing] ", cmd, " ", args.join(" ") + let process = startProcess(cmd, args = args, options = {poUsePath, poStdErrToStdOut}) + defer: process.close() while process.running(): await sleepAsync(10) result = process.peekexitCode == 0 - process.close() - + if not result: + # Process failed so we write a '.log' + let logPath = entry.path.changeFileExt("log") + discard tryRemoveFile(logPath) + let fs = openFileStream(logPath, fmWrite) + defer: fs.close() + for line in process.lines: + fs.writeLine(line) proc buildMd*(entry: Entry): Future[bool] {.async.}= try: From cd11f32a4681b387cec03f593099a9d00ea5cee7 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 11 Apr 2022 16:02:25 -0600 Subject: [PATCH 3/4] Logged error files should be chronological. --- src/nimibook/builds.nim | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/nimibook/builds.nim b/src/nimibook/builds.nim index d757d93..f795606 100644 --- a/src/nimibook/builds.nim +++ b/src/nimibook/builds.nim @@ -24,7 +24,7 @@ proc buildNim*(entry: Entry, srcDir: string, nimOptions: seq[string]): Future[bo for line in process.lines: fs.writeLine(line) -proc buildMd*(entry: Entry): Future[bool] {.async.}= +proc buildMd*(entry: Entry): bool = try: nbInit(theme = useNimibook, thisFileRel = entry.path) nbText nb.source @@ -41,7 +41,7 @@ proc build*(entry: Entry, srcDir: string, nimOptions: seq[string]): Future[bool] of ".nim": return await buildNim(entry, srcDir, nimOptions) of ".md": - return await buildMd(entry) + return buildMd(entry) else: echo "[nimibook.error] invalid file extension (must be one of .nim, .md): ", splitted.ext return false @@ -59,10 +59,15 @@ proc build*(book: Book, nimOptions: seq[string] = @[]) = buildPaths.add entry.path var buildErrors: seq[string] - let finished = waitfor all buildFutures - for i, success in finished: - if not success: - buildErrors.add buildPaths[i] + while buildFutures.len > 0: + # 'all' from asyndispatch doesnt return them chronologically. + # meaning logged errors would not present the right path + if waitfor buildFutures[^1].withTimeout(300): + if not buildFutures[^1].read: + buildErrors.add buildPaths[^1] + + discard buildPaths.pop + discard buildFutures.pop if len(buildErrors) > 0: echo "[nimibook.error] ", len(buildErrors), " build errors:" From ee8daa58823bae4c4b9f8d58e6b87e6972f7e9b6 Mon Sep 17 00:00:00 2001 From: Jason Date: Sat, 16 Apr 2022 15:46:39 -0600 Subject: [PATCH 4/4] Clean up async dispatch for building --- .gitignore | 3 ++- src/nimibook/builds.nim | 24 ++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 61444d8..917ceae 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ docs/* examplebook/my* nbook nbook.exe -x_* \ No newline at end of file +x_* +*.log diff --git a/src/nimibook/builds.nim b/src/nimibook/builds.nim index f795606..cccc253 100644 --- a/src/nimibook/builds.nim +++ b/src/nimibook/builds.nim @@ -1,4 +1,4 @@ -import std / [os, strutils, asyncdispatch, osproc, streams] +import std / [os, strutils, asyncdispatch, osproc, streams, sugar] import nimibook / [types, commands, themes] import nimib @@ -48,26 +48,22 @@ proc build*(entry: Entry, srcDir: string, nimOptions: seq[string]): Future[bool] proc build*(book: Book, nimOptions: seq[string] = @[]) = var - buildPaths: seq[string] + buildErrors: seq[string] buildFutures: seq[Future[bool]] dump book - for entry in book.toc.entries: + for i in 0..book.toc.entries.high: + let entry = book.toc.entries[i] # use index since `items` returns `lent` in `1.7.x+` if entry.isDraft: continue echo "[nimibook] build entry: ", entry.path buildFutures.add build(entry, book.srcDir, nimOptions) - buildPaths.add entry.path + closureScope: + let path = entry.path + buildFutures[^1].addCallback do (f: Future[bool]): + if not f.read(): + buildErrors.add path - var buildErrors: seq[string] - while buildFutures.len > 0: - # 'all' from asyndispatch doesnt return them chronologically. - # meaning logged errors would not present the right path - if waitfor buildFutures[^1].withTimeout(300): - if not buildFutures[^1].read: - buildErrors.add buildPaths[^1] - - discard buildPaths.pop - discard buildFutures.pop + discard waitFor all buildFutures if len(buildErrors) > 0: echo "[nimibook.error] ", len(buildErrors), " build errors:"