From 34ff2b0aeb8b740c51aa450557066561b1106819 Mon Sep 17 00:00:00 2001 From: ynfle <23086821+ynfle@users.noreply.github.com> Date: Thu, 9 Jun 2022 16:30:42 +0300 Subject: [PATCH] Feature: upgrade macro evaluation strategy to use `nimscripter` (#37) This commit upgrades the representer to use https://github.com/beef331/nimscripter. This module is a branch between Nimscript (a subset of nim that runs in the VM) and the compiled nim. It embeds an interpreter in to the executable, and doesn't require a nim compiler to be in path. The primary advantage is the the bridge is built-in, so there is no need for recompiling to create a representation for every exercise, as the representer uses nim's macros which only function in the VM. Without this, only the VM is used, but the executable has to be recompiled every time. This also greatly simplifies the testing process, as the representation creation doesn't need to happen at compile to and injecting in. Rather, it can happen at runtime with the VM bridge. This also add the use of https://github.com/docopt/docopt.nim form command line parsing. This was not possible before, because everything happened at compile time and there was no code being run. CI: https://github.com/jiro4989/setup-nim-action is used to install nim using choosenim with the desired version. Caching is utilized to not have to redownload the dependent packages again and invalidates the cache when the nimble file changes. This is the example used on the `setup-nim-action` repo `README.md` Closes #9 as not relevant Co-authored-by: Erik Schierboom Co-authored-by: ee7 <45465154+ee7@users.noreply.github.com> --- .github/workflows/ci.yml | 42 +++++++-- bin/run.sh | 3 +- nim.cfg | 1 + nim_representer.nimble | 4 +- src/representer.nim | 56 ++++++++---- src/representer/loader.nims | 11 +++ src/representer/normalizations.nim | 6 +- src/representer/types.nim | 15 +++ tests/tnormalizations.nim | 141 +++++++++++++---------------- 9 files changed, 168 insertions(+), 111 deletions(-) create mode 100644 src/representer/loader.nims create mode 100644 src/representer/types.nim diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc2b143..7c1a57e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,26 +10,54 @@ on: jobs: job1: name: Nim Tests - runs-on: ubuntu-18.04 - container: nimlang/nim:1.2.4-ubuntu-regular@sha256:02a555518a05c354ccb2627940f6138fca1090d198e5a0eb60936c03a4875c69 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + - uses: jiro4989/setup-nim-action@9a5a618a7cccbc7415b2a539f25c9681fafe0ddb + with: + nim-version: '1.6.6' + + - name: Cache nimble + id: cache-nimble + uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d + with: + path: ~/.nimble + key: ${{ runner.os }}-nimble-${{ hashFiles('*.nimble') }} + - name: Compile and run tests with `nimble test` - run: "nimble test" + run: "nimble test -y" + job2: name: Smoke test - runs-on: ubuntu-18.04 - container: nimlang/nim:1.2.4-ubuntu-regular@sha256:02a555518a05c354ccb2627940f6138fca1090d198e5a0eb60936c03a4875c69 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + - uses: jiro4989/setup-nim-action@9a5a618a7cccbc7415b2a539f25c9681fafe0ddb + with: + nim-version: '1.6.6' + + - name: Cache nimble + id: cache-nimble + uses: actions/cache@c3f1317a9e7b1ef106c153ac8c0f00fed3ddbc0d + with: + path: ~/.nimble + key: ${{ runner.os }}-nimble-${{ hashFiles('*.nimble') }} + + - name: "Install nimble dependencies" + if: steps.cache-nimble.outputs.cache-hit != 'true' + run: "nimble install -y -d" + + - name: "Compile representer" + run: "nimble c -d:release src/representer" + - name: "Make representation of `two-fer`" run: "bin/run.sh two-fer ${PWD}/tests/cases/example-two-fer/ ${PWD}/tests/cases/example-two-fer/" - name: "Check diffs" run: | - diff tests/cases/example-two-fer/mapping.json tests/cases/example-two-fer/expected/mapping.json - diff tests/cases/example-two-fer/representation.txt tests/cases/example-two-fer/expected/representation.txt + diff -y tests/cases/example-two-fer/mapping.json tests/cases/example-two-fer/expected/mapping.json + diff -y tests/cases/example-two-fer/representation.txt tests/cases/example-two-fer/expected/representation.txt diff --git a/bin/run.sh b/bin/run.sh index d5e06e6..a541917 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -5,4 +5,5 @@ if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then echo "slug, solution directory and output directory must be present" exit 1 fi -nim c -f --outdir:bin/ -d:slug="$1" -d:inDir="$2" -d:outDir="$3" src/representer + +bin/representer --slug="$1" --input-dir="$2" --output-dir="$3" --print diff --git a/nim.cfg b/nim.cfg index f253b0b..fec2fef 100644 --- a/nim.cfg +++ b/nim.cfg @@ -1,3 +1,4 @@ +--path:"$nim" --verbosity=0 --hint[Processing]:off --styleCheck:hint diff --git a/nim_representer.nimble b/nim_representer.nimble index 46bdfe6..fa697d7 100644 --- a/nim_representer.nimble +++ b/nim_representer.nimble @@ -10,4 +10,6 @@ binDir = "bin" # Dependencies -requires "nim >= 1.0.0" +requires "nim >= 1.6.6" +requires "nimscripter == 1.0.14" +requires "docopt == 0.6.8" diff --git a/src/representer.nim b/src/representer.nim index a2f37ae..bfc2f85 100644 --- a/src/representer.nim +++ b/src/representer.nim @@ -1,26 +1,42 @@ -import macros, os, sequtils, strutils -import representer/[mapping, normalizations] +import std/[json, os, strformat, strutils] +import nimscripter +import representer/[mapping, types] +import docopt -proc switchKeysValues*(map: IdentMap): OrderedTable[string, NormalizedIdent] = - toSeq(map.pairs).mapIt((it[1], it[0])).toOrderedTable +const doc = """ + Exercism nim representation normalizer. -proc createRepresentation*(fileName: string): tuple[tree: NimNode, map: IdentMap] = - var map: IdentMap - let code = parseStmt fileName.staticRead - result = (tree: code.normalizeStmtList(map), map: map) + Usage: + representer --slug= --input-dir= [--output-dir=] [--print] + Options: + -h --help Show this help message. + -v, --version Display version. + -p, --print Print the results. + -s , --slug= The exercise slug. + -i , --input-dir= The directory of the submission and exercise files. + -o , --output-dir= The directory to output to. + If omitted, output will be written to stdout. + """.dedent -const inDir {.strdefine.} = "" -const outDir {.strdefine.} = "" -const slug {.strdefine.} = "" -const underSlug = slug.replace('-', '_') +proc getFileContents(fileName: string): string = readFile fileName + +func kebabToSnakeCase(s: string): string = s.replace('-', '_') + +proc main() = + let args = docopt(doc) + let intr = loadScript(NimScriptPath("src/representer/loader.nims")) + let (tree, map) = intr.invoke( + getTestableRepresentation, + getFileContents($args["--input-dir"] / kebabToSnakeCase($args["--slug"]) & ".nim"), true, + returnType = SerializedRepresentation + ) + if args["--output-dir"]: + let outDir = $args["--output-dir"] + writeFile outDir / "mapping.json", $map.parseJson + writeFile outDir / "representation.txt", tree + if not args["--output-dir"] or args["--print"]: + echo &"{tree = }\n{map.parseJson.pretty = }" when isMainModule: - import json - static: - let (tree, map) = createRepresentation(inDir / underSlug & ".nim") - let finalMapping = map.switchKeysValues - echo (%*{"map": finalMapping, "tree": tree.repr}).pretty - when defined(outDir): - writeFile(outDir / "representation.txt", tree.repr) - writeFile(outDir / "mapping.json", $(%finalMapping)) + main() diff --git a/src/representer/loader.nims b/src/representer/loader.nims new file mode 100644 index 0000000..c09fead --- /dev/null +++ b/src/representer/loader.nims @@ -0,0 +1,11 @@ +import std/[json, macros] +import "."/[mapping, normalizations, types] + +proc createRepresentation(contents: string): tuple[tree: NimNode, map: IdentMap] = + var map: IdentMap + let code = parseStmt(contents) + result = (tree: code.normalizeStmtList(map), map: map) + +proc getTestableRepresentation*(contents: string, switch = false): SerializedRepresentation = + let (tree, map) = createRepresentation(contents) + result = (repr tree, $(if switch: %map.switchKeysValues else: %map)) diff --git a/src/representer/normalizations.nim b/src/representer/normalizations.nim index 9fb36af..74cb083 100644 --- a/src/representer/normalizations.nim +++ b/src/representer/normalizations.nim @@ -1,6 +1,6 @@ ## Create an normalized AST of a submission on exercism.org to provide feedback -import algorithm, macros, strformat, sequtils, strutils, std/with -import mapping +import std/[algorithm, macros, strformat, sequtils, strutils, with] +import "."/mapping proc normalizeStmtList*(code: NimNode, map: var IdentMap): NimNode proc normalizeValue(value: NimNode, map: var IdentMap): NimNode @@ -35,7 +35,7 @@ proc constructFmtStr(ast: NimNode, map: var IdentMap): string = proc normalizeCall(call: NimNode, map: var IdentMap): NimNode = result = if call.kind != nnkInfix and (call[0] == "fmt".ident or call[0] == "&".ident): - let fmtAst = getAst(fmt(call[1])) + let fmtAst = getAst(&(call[1])) let strToFmt = fmtAst[1..^2].mapIt( if $it[0][0] == "add": $it[2] diff --git a/src/representer/types.nim b/src/representer/types.nim new file mode 100644 index 0000000..a4a5e40 --- /dev/null +++ b/src/representer/types.nim @@ -0,0 +1,15 @@ +import std/[sugar, tables] +import mapping + +type + Representation* = tuple + tree: string + map: IdentMap + SerializedRepresentation* = tuple + tree: string + map: string + +proc switchKeysValues*(map: IdentMap): OrderedTable[string, NormalizedIdent] = + result = collect(initOrderedTable): + for key, val in map.pairs: + {val: key} diff --git a/tests/tnormalizations.nim b/tests/tnormalizations.nim index 8beb7ed..8cc2c65 100644 --- a/tests/tnormalizations.nim +++ b/tests/tnormalizations.nim @@ -1,59 +1,24 @@ -import representer/[mapping, normalizations] -import macros, sequtils, strutils, unittest - - -macro setup(test, code: untyped): untyped = - var map: IdentMap - let tree = code.normalizeStmtList(map) - let tableConstr = nnkTableConstr.newTree.add(toSeq(map.pairs).mapIt( - nnkExprColonExpr.newTree( - newDotExpr(it[0].string.newStrLitNode, "NormalizedIdent".ident), - it[1].newStrLitNode - ) - )) - - let tableInit = - if tableConstr.len != 0: - newDotExpr( - tableConstr, - "toOrderedTable".ident - ) - else: - newEmptyNode() - - newStmtList( - nnkLetSection.newTree( - nnkIdentDefs.newTree( - nnkPragmaExpr.newTree( - ident "tree", - nnkPragma.newTree(ident "used") - ), - newEmptyNode(), - newLit tree.repr - ), - ), - nnkVarSection.newTree( - nnkIdentDefs.newTree( - nnkPragmaExpr.newTree( - ident "map", - nnkPragma.newTree(ident "used") - ), - "IdentMap".ident, - tableInit - ) - ), - - newCall("check", test) - ) +import std/[json, strutils, unittest] +import nimscripter +import representer/types + +let + intr = loadScript(NimScriptPath "src/representer/loader.nims") + +proc getRepresentation(t: string): tuple[tree: string, map: JsonNode] = + let (tree, map) = intr.invoke(getTestableRepresentation, t, false, returnType = SerializedRepresentation) + result = (tree, map.parseJson) suite "End to end": test "Just one `let` statement": - setup(map["x".NormalizedIdent] == "placeholder_0" and map.len == 1): - let x = 1 + let (_, map) = getRepresentation """let x = 1""" + check: + map["x"].getStr == "placeholder_0" + map.len == 1 test "All features": + let (_, map) = getRepresentation dedent """ - setup(map.len == 11): type Dollar = distinct int @@ -77,31 +42,39 @@ suite "End to end": macro testMacro(code: untyped): untyped = discard template testTemplate(code: untyped): untyped = discard + """ + + check map.len == 11 test "No params, return type or statements": - setup(tree.strip == "proc placeholder_0*() =\n discard".strip): - proc helloWorld* = discard + let (tree, _) = getRepresentation """proc helloWorld* = discard""" + + check tree == "\nproc placeholder_0*() =\n discard\n" test "All the things": - const expected = """import - algorithm, macros as m, strutils + const expected = dedent """ + + import + algorithm, macros as m, strutils + + let + placeholder_0 = 1 + placeholder_1 = `$`(placeholder_0).strip.replace("\n", `$`(placeholder_0)) + proc placeholder_2*() = + echo("testing stdout") + + placeholder_2() + proc placeholder_5*(placeholder_3: int; placeholder_4 = "seventeen"): string = + let placeholder_6 = `-`(placeholder_3, placeholder_0) + let placeholder_7 = `&`(placeholder_1, placeholder_4) + let placeholder_8 = `&`(`$`(placeholder_6), placeholder_7) + placeholder_8 + + echo(placeholder_0.placeholder_5) + echo(placeholder_5(placeholder_3 = 1, placeholder_4 = "how old am I?"))""" + + let (tree, _) = getRepresentation dedent """ -let - placeholder_0 = 1 - placeholder_1 = `$`(placeholder_0).strip.replace("\n", `$`(placeholder_0)) -proc placeholder_2*() = - echo("testing stdout") - -placeholder_2() -proc placeholder_5*(placeholder_3: int; placeholder_4 = "seventeen"): string = - let placeholder_6 = `-`(placeholder_3, placeholder_0) - let placeholder_7 = `&`(placeholder_1, placeholder_4) - let placeholder_8 = `&`(`$`(placeholder_6), placeholder_7) - placeholder_8 - -echo(placeholder_0.placeholder_5) -echo(placeholder_5(placeholder_3 = 1, placeholder_4 = "how old am I?"))""" - setup(tree.strip == expected.strip): import strutils, algorithm, macros as m let @@ -121,24 +94,34 @@ echo(placeholder_5(placeholder_3 = 1, placeholder_4 = "how old am I?"))""" echo x.helloWorld - echo hELLOWORLD(name = 1, age = "how old am I?") + echo hELLOWORLD(name = 1, age = "how old am I?")""" + + check tree == expected suite "Test specific functionality": - let expected = """import - strformat + const expected = dedent """ + + import + strformat + + proc placeholder_1*(placeholder_0 = "you"): string = + fmt"One for {placeholder_0}, one for me." + """ -proc placeholder_1*(placeholder_0 = "you"): string = - fmt"One for {placeholder_0}, one for me."""" test "fmt strings": - setup(tree.strip == expected.strip): + let (tree, _) = getRepresentation dedent """ import strformat proc twoFer*(name = "you"): string = - fmt"One for {name}, one for me." + fmt"One for {name}, one for me." """ + + check tree == expected test "fmt string with `&`": - setup(tree.strip == expected.strip): + let (tree, _) = getRepresentation dedent """ import strformat proc twoFer*(name = "you"): string = - &"One for {name}, one for me." + &"One for {name}, one for me." """ + + check tree == expected