diff --git a/.gitignore b/.gitignore index df3bc71860d..6f1163a0f02 100644 --- a/.gitignore +++ b/.gitignore @@ -93,7 +93,7 @@ tweeter_test.db /t15148.txt /tests/vm/tfile_rw.txt -/lib/pure/*.js +/lib/pure/**/*.js !/.builds/ diff --git a/compiler/backend/extccomp.nim b/compiler/backend/extccomp.nim index fe1781fabb4..63c04d93ad6 100644 --- a/compiler/backend/extccomp.nim +++ b/compiler/backend/extccomp.nim @@ -853,7 +853,8 @@ proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx: else: tryExceptOSErrorMessage(conf, "invocation of external compiler program failed."): res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams}, - conf.numberOfProcessors, prettyCb, afterRunEvent=runCb) + conf.numberOfProcessors, beforeRunEvent=prettyCb, + afterRunEvent=runCb) if res != 0: if conf.numberOfProcessors <= 1: diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index de0933b7271..73d65b57d43 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -86,7 +86,7 @@ proc execProcess*(command: string, workingDir: string = "", ## See also: ## * `startProcess proc ## <#startProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ - ## * `execProcesses proc <#execProcesses,openArray[string],proc(int),proc(int,Process)>`_ + ## * `execProcesses proc <#execProcesses,openArray[string],proc(int),proc(int,Process),proc(int,Process)>`_ ## * `execCmd proc <#execCmd,string>`_ ## ## Example: @@ -146,7 +146,7 @@ proc startProcess*(command: string, workingDir: string = "", ## but ``OSError`` is raised in case of an error. ## ## See also: - ## * `execProcesses proc <#execProcesses,openArray[string],proc(int),proc(int,Process)>`_ + ## * `execProcesses proc <#execProcesses,openArray[string],proc(int),proc(int,Process),proc(int,Process)>`_ ## * `execProcess proc ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## * `execCmd proc <#execCmd,string>`_ @@ -336,15 +336,18 @@ when not defined(nimHasEffectsOf): proc execProcesses*(cmds: openArray[string], options = {poStdErrToStdOut, poParentStreams}, n = countProcessors(), beforeRunEvent: proc(idx: int) = nil, + startRunEvent: proc(idx: int, p: Process) = nil, afterRunEvent: proc(idx: int, p: Process) = nil): int {.rtl, extern: "nosp$1", tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect], - effectsOf: [beforeRunEvent, afterRunEvent].} = + effectsOf: [beforeRunEvent, startRunEvent, afterRunEvent].} = ## Executes the commands `cmds` in parallel. ## Creates `n` processes that execute in parallel. ## ## The highest (absolute) return value of all processes is returned. - ## Runs `beforeRunEvent` before running each command. + ## Runs `beforeRunEvent` before running each command; then `startRunEvent` + ## immediately after a command has started, useful for input; and then + ## `afterRunEvent` after a command has finished. assert n > 0 if n > 1: @@ -363,6 +366,8 @@ proc execProcesses*(cmds: openArray[string], if beforeRunEvent != nil: beforeRunEvent(i) q[i] = startProcess(cmds[i], options = options + {poEvalCommand}) + if startRunEvent != nil: + startRunEvent(i, q[i]) idxs[i] = i when defined(windows): w[i] = q[i].fProcessHandle @@ -425,9 +430,12 @@ proc execProcesses*(cmds: openArray[string], if afterRunEvent != nil: afterRunEvent(idxs[rexit], q[rexit]) close(q[rexit]) if i < len(cmds): - if beforeRunEvent != nil: beforeRunEvent(i) + if beforeRunEvent != nil: + beforeRunEvent(i) q[rexit] = startProcess(cmds[i], options = options + {poEvalCommand}) + if startRunEvent != nil: + startRunEvent(i, q[rexit]) idxs[rexit] = i when defined(windows): w[rexit] = q[rexit].fProcessHandle @@ -447,8 +455,11 @@ proc execProcesses*(cmds: openArray[string], if beforeRunEvent != nil: beforeRunEvent(i) var p = startProcess(cmds[i], options = options + {poEvalCommand}) + if startRunEvent != nil: + startRunEvent(i, p) result = max(abs(waitForExit(p)), result) - if afterRunEvent != nil: afterRunEvent(i, p) + if afterRunEvent != nil: + afterRunEvent(i, p) close(p) iterator lines*(p: Process): string {.since: (1, 3), tags: [ReadIOEffect].} = diff --git a/testament/categories.nim b/testament/categories.nim index 3a628d5c481..ff2c8960212 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -20,14 +20,11 @@ const "assert", "debugger", "dll", - "examples", "gc", - "io", "js", "ic", "lib", "manyloc", - "niminaction", "threads", "untestable", # see trunner_special "testdata", @@ -41,6 +38,16 @@ proc isTestFile*(file: string): bool = let (_, name, ext) = splitFile(file) result = ext == ".nim" and name.startsWith("t") +func defaultTargets(category: Category): set[TTarget] = + const standardTargets = {low(TTarget)..high(TTarget)} - {targetObjC} + case category.string + of "arc": + standardTargets - {targetJs} + of "js": + {targetJs} + else: + standardTargets + # --------------------- DLL generation tests ---------------------------------- proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = @@ -150,6 +157,65 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "cyclecollector" testWithoutBoehm "trace_globals" +type + GcTestKinds = enum + gcOther, + gcMarkSweep, + gcBoehm + +proc setupGcTests(execState: var Execution, catId: CategoryId) = + ## setup tests for the gc category, requires special handling due to + ## testament limitations. + + const + withoutMs = {gcOther} + withoutBoehm = {gcOther, gcMarkSweep} + noConditions = {gcOther, gcMarkSweep, gcBoehm} + + let testData = [ + ("foreign_thr.nim", withoutBoehm), + ("gcemscripten.nim", noConditions), + ("growobjcrash.nim", noConditions), + ("gcbench.nim", noConditions), + ("gcleak.nim", noConditions), + ("gcleak2.nim", noConditions), + ("gctest.nim", withoutBoehm), + ("gcleak3.nim", noConditions), + ("gcleak4.nim", noConditions), + ("weakrefs.nim", withoutBoehm), + ("cycleleak.nim", noConditions), + ("closureleak.nim", withoutBoehm), + ("refarrayleak.nim", withoutMs), + ("tlists.nim", withoutBoehm), + ("thavlak.nim", withoutBoehm), + ("stackrefleak.nim", noConditions), + ("cyclecollector.nim", noConditions), + ("trace_globals.nim", withoutBoehm) + ] + + for (testFile, gcConditions) in testData: + let testId: TestId = execState.testFiles.len + execState.testFiles.add TestFile(file: "tests/gc" / testFile, catId: catId) + execState.testOpts[testId] = TestOptionData() + + if gcMarkSweep in gcConditions: + execState.testOpts[testId].optMatrix.add "" # run the test as is + execState.testOpts[testId].optMatrix.add " -d:release --gc:useRealtimeGC" + if testFile != "gctest": + execState.testOpts[testId].optMatrix.add " --gc:orc" + execState.testOpts[testId].optMatrix.add " -d:release --gc:orc" + + if gcMarkSweep in gcConditions: + execState.testOpts[testId].optMatrix.add " --gc:markAndSweep" + execState.testOpts[testId].optMatrix.add " -d:release --gc:markAndSweep" + + if gcBoehm in gcConditions: + when not defined(windows) and not defined(android): + # cannot find any boehm.dll on the net, right now, so disabled for + # windows: + execState.testOpts[testId].optMatrix.add " --gc:boehm" + execState.testOpts[testId].optMatrix.add " -d:release --gc:boehm" + # ------------------------- threading tests ----------------------------------- proc threadTests(r: var TResults, cat: Category, options: string) = @@ -160,19 +226,12 @@ proc threadTests(r: var TResults, cat: Category, options: string) = for t in os.walkFiles("tests/threads/t*.nim"): test(t) -# ------------------------- IO tests ------------------------------------------ - -proc ioTests(r: var TResults, cat: Category, options: string) = - # We need readall_echo to be compiled for this test to run. - # dummy compile result: - var c = initResults() - testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat) - # ^- why is this not appended to r? Should this be discarded? - # EDIT: this should be replaced by something like in D20210524T180826, - # likewise in similar instances where `testSpec c` is used, or more generally - # when a test depends on another test, as it makes tests non-independent, - # creating complications for batching and megatest logic. - testSpec r, makeTest("tests/system/tio", options, cat) +proc setupThreadTests(execState: var Execution, catId: CategoryId) = + for t in os.walkFiles("tests/threads/t*.nim"): + let testId: TestId = execState.testFiles.len + execState.testFiles.add TestFile(file: t, catId: catId) + execState.testOpts[testId] = TestOptionData( + optMatrix: @["", "-d:release", "--tlsEmulation:on"]) # ------------------------- debugger tests ------------------------------------ @@ -206,72 +265,6 @@ proc jsTests(r: var TResults, cat: Category, options: string) = for testfile in ["strutils", "json", "random", "times", "logging"]: test "lib/pure/" & testfile & ".nim" -# ------------------------- nim in action ----------- - -proc testNimInAction(r: var TResults, cat: Category, options: string) = - template test(filename: untyped) = - testSpec r, makeTest(filename, options, cat) - - let tests = [ - "niminaction/Chapter1/various1", - "niminaction/Chapter2/various2", - "niminaction/Chapter2/resultaccept", - "niminaction/Chapter2/resultreject", - "niminaction/Chapter2/explicit_discard", - "niminaction/Chapter2/no_def_eq", - "niminaction/Chapter2/no_iterator", - "niminaction/Chapter2/no_seq_type", - "niminaction/Chapter6/WikipediaStats/concurrency_regex", - "niminaction/Chapter6/WikipediaStats/concurrency", - "niminaction/Chapter6/WikipediaStats/naive", - "niminaction/Chapter6/WikipediaStats/parallel_counts", - "niminaction/Chapter6/WikipediaStats/race_condition", - "niminaction/Chapter6/WikipediaStats/sequential_counts", - "niminaction/Chapter6/WikipediaStats/unguarded_access" - ] - - when false: - # Verify that the files have not been modified. Death shall fall upon - # whoever edits these hashes without dom96's permission, j/k. But please only - # edit when making a conscious breaking change, also please try to make your - # commit message clear and notify me so I can easily compile an errata later. - # --------------------------------------------------------- - # Hash-checks are disabled for Nim 1.1 and beyond - # since we needed to fix the deprecated unary '<' operator. - const refHashes = @[ - "51afdfa84b3ca3d810809d6c4e5037ba", - "30f07e4cd5eaec981f67868d4e91cfcf", - "d14e7c032de36d219c9548066a97e846", - "b335635562ff26ec0301bdd86356ac0c", - "6c4add749fbf50860e2f523f548e6b0e", - "76de5833a7cc46f96b006ce51179aeb1", - "705eff79844e219b47366bd431658961", - "a1e87b881c5eb161553d119be8b52f64", - "2d706a6ec68d2973ec7e733e6d5dce50", - "c11a013db35e798f44077bc0763cc86d", - "3e32e2c5e9a24bd13375e1cd0467079c", - "a5452722b2841f0c1db030cf17708955", - "dc6c45eb59f8814aaaf7aabdb8962294", - "69d208d281a2e7bffd3eaf4bab2309b1", - "ec05666cfb60211bedc5e81d4c1caf3d", - "da520038c153f4054cb8cc5faa617714", - "59906c8cd819cae67476baa90a36b8c1", - "9a8fe78c588d08018843b64b57409a02", - "8b5d28e985c0542163927d253a3e4fc9", - "783299b98179cc725f9c46b5e3b5381f", - "1a2b3fba1187c68d6a9bfa66854f3318", - "391ff57b38d9ea6f3eeb3fe69ab539d3" - ] - for i, test in tests: - let filename = testsDir / test.addFileExt("nim") - let testHash = getMD5(readFile(filename).string) - doAssert testHash == refHashes[i], "Nim in Action test " & filename & - " was changed: " & $(i: i, testHash: testHash, refHash: refHashes[i]) - - # Run the tests. - for testfile in tests: - test "tests/" & testfile & ".nim" - # ------------------------- manyloc ------------------------------------------- proc findMainFile(dir: string): string = @@ -300,12 +293,6 @@ proc manyLoc(r: var TResults, cat: Category, options: string) = test.spec.action = actionCompile testSpec r, test -proc compileExample(r: var TResults, pattern, options: string, cat: Category) = - for test in os.walkFiles(pattern): - var test = makeTest(test, options, cat) - test.spec.action = actionCompile - testSpec r, test - proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = var files: seq[string] @@ -331,60 +318,30 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) = testObj.spec.action = actionCompile testSpec r, testObj -# ---------------- IC tests --------------------------------------------- +proc setupStdlibTests(execState: var Execution, catId: CategoryId) = + proc isValid(file: string): bool = + for dir in parentDirs(file, inclusive = false): + if dir.lastPathPart in ["includes", "nimcache"]: + # e.g.: lib/pure/includes/osenv.nim gives: Error: This is an include file for os.nim! + return false + let name = extractFilename(file) + if name.splitFile.ext != ".nim": return false + for namei in disabledFiles: + # because of `LockFreeHash.nim` which has case + if namei.cmpPaths(name) == 0: return false + return true -proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; - isNavigatorTest: bool) = - const - tooltests = ["compiler/nim.nim"] - incrementalOn = " --incremental:on -d:nimIcIntegrityChecks " - navTestConfig = " --ic:on -d:nimIcNavigatorTests --hint:Conf:off --warnings:off " - - template editedTest(x: untyped) = - var test = makeTest(file, x & options, cat) - if isNavigatorTest: - test.spec.action = actionCompile - test.spec.targets = {getTestSpecTarget()} - testSpecWithNimcache(r, test, nimcache) - - template checkTest() = - var test = makeRawTest(file, options, cat) - test.spec.cmd = compilerPrefix & " check --hint:Conf:off --warnings:off --ic:on $options " & file - testSpecWithNimcache(r, test, nimcache) - - if not isNavigatorTest: - for file in tooltests: - let nimcache = nimcacheDir(file, options, getTestSpecTarget()) - removeDir(nimcache) - - let oldPassed = r.passed - checkTest() - - if r.passed == oldPassed+1: - checkTest() - if r.passed == oldPassed+2: - checkTest() - - const tempExt = "_temp.nim" - for it in walkDirRec(testsDir): - # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test - if isTestFile(it) and not it.endsWith(tempExt): - let nimcache = nimcacheDir(it, options, getTestSpecTarget()) - removeDir(nimcache) - - let content = readFile(it) - for fragment in content.split("#!EDIT!#"): - let file = it.replace(".nim", tempExt) - writeFile(file, fragment) - let oldPassed = r.passed - editedTest(if isNavigatorTest: navTestConfig else: incrementalOn) - if r.passed != oldPassed+1: break + for testFile in os.walkDirRec("lib/pure/"): + if isValid(testFile): + let testId: TestId = execState.testFiles.len + execState.testFiles.add TestFile(file: testFile, catId: catId) + execState.testOpts[testId] = TestOptionData(action: some(actionCompile)) # ---------------------------------------------------------------------------- -# const AdditionalCategories = ["debugger", "examples", "lib", "ic", "navigator"] -const AdditionalCategories = ["debugger", "examples", "lib"] -const MegaTestCat = "megatest" +const + AdditionalCategories = ["debugger", "lib"] + MegaTestCat = "megatest" proc `&.?`(a, b: string): string = # candidate for the stdlib? @@ -408,7 +365,7 @@ proc isJoinableSpec(spec: TSpec): bool = not fileExists(parentDir(spec.file) / "nim.cfg") and not fileExists(parentDir(spec.file) / "config.nims") and spec.cmd.len == 0 and - spec.err != reDisabled and + (spec.err != reDisabled) and not spec.unjoinable and spec.exitCode == 0 and spec.input.len == 0 and @@ -564,20 +521,8 @@ proc processCategory(r: var TResults, cat: Category, manyLoc r, cat, options of "threads": threadTests r, cat, options & " --threads:on" - of "io": - ioTests r, cat, options of "lib": testStdlib(r, "lib/pure/", options, cat) - of "examples": - compileExample(r, "examples/*.nim", options, cat) - compileExample(r, "examples/gtk/*.nim", options, cat) - compileExample(r, "examples/talk/*.nim", options, cat) - of "niminaction": - testNimInAction(r, cat, options) - of "ic": - icTests(r, testsDir / cat2, cat, options, isNavigatorTest=false) - of "navigator": - icTests(r, testsDir / cat2, cat, options, isNavigatorTest=true) of "untestable": # These require special treatment e.g. because they depend on a third party # dependency; see `trunner_special` which runs some of those. diff --git a/testament/specs.nim b/testament/specs.nim index e45c785364d..b31b39695b6 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -15,6 +15,9 @@ type TestamentData* = ref object batchArg*: string testamentNumBatch*: int testamentBatch*: int + includeKnownIssues*: bool + ## whether to run knownIssues as part of this test run + withMegaTestRun*: bool # global mutable state for funsies var @@ -42,23 +45,27 @@ type ocSubstr = "substr" TResultEnum* = enum - reNimcCrash, # nim compiler seems to have crashed - reMsgsDiffer, # error messages differ - reFilesDiffer, # expected and given filenames differ - reLinesDiffer, # expected and given line numbers differ + reNimcCrash, ## nim compiler seems to have crashed + reMsgsDiffer, ## error messages differ + reFilesDiffer, ## expected and given filenames differ + reLinesDiffer, ## expected and given line numbers differ reOutputsDiffer, - reExitcodesDiffer, # exit codes of program or of valgrind differ + reExitcodesDiffer, ## exit codes of program or of valgrind differ reTimeout, reInvalidPeg, reCodegenFailure, reCodeNotFound, reExeNotFound, - reInstallFailed # package installation failed - reBuildFailed # package building failed - reDisabled, # test is disabled - reJoined, # test is disabled because it was joined into the megatest - reSuccess # test was successful - reInvalidSpec # test had problems to parse the spec + reInstallFailed ## package installation failed + reBuildFailed ## package building failed + reDisabled, ## test is disabled + reKnownIssue, + ## test is either not run or assumed to not match expected condition, in + ## CI this should cause a "failure" so the test can be updated and marked + ## as working + reJoined, ## disabled test because it was joined into megatest + reSuccess ## test was successful + reInvalidSpec ## test had problems to parse the spec TTarget* = enum targetC = "c" @@ -126,7 +133,7 @@ var retryContainer* = RetryContainer(retry: false) proc getCmd*(s: TSpec): string = ## Get runner command for a given test specification if s.cmd.len == 0: - result = compilerPrefix & " $target --hints:on -d:testing --clearNimblePath --nimblePath:build/deps/pkgs $options $file" + result = compilerPrefix & " $target --hints:on -d:testing --clearNimblePath --nimblePath:build/deps/pkgs $outfileOption $options $file" else: result = s.cmd @@ -272,10 +279,10 @@ proc parseTargets*(value: string): set[TTarget] = ## Get list of allowed run targets for the testament for v in value.normalize.splitWhitespace: case v - of "c": result.incl(targetC) + of "c": result.incl(targetC) of "cpp", "c++": result.incl(targetCpp) - of "objc": result.incl(targetObjC) - of "js": result.incl(targetJS) + of "objc": result.incl(targetObjC) + of "js": result.incl(targetJS) else: raise newException(ValueError, "invalid target: '$#'" % v) proc addLine*(self: var string; a: string) = @@ -319,7 +326,7 @@ proc parseSpec*(filename: string): TSpec = case e.kind of cfgKeyValuePair: let key = e.key.normalize - const allowMultipleOccurences = ["disabled", "ccodecheck" , "knownissue"] + const allowMultipleOccurences = ["disabled", "ccodecheck", "knownissue"] ## list of flags that are correctly handled when passed multiple times ## (instead of being overwritten) if key notin allowMultipleOccurences: @@ -468,7 +475,14 @@ proc parseSpec*(filename: string): TSpec = of "n", "no", "false", "0": discard else: result.knownIssues.add e.value - result.err = reDisabled + # choose here whether we'll run known issues or not + # xxx: better option would be to defer this decisions but this code + # is so far from something so nice. + result.err = + if testamentData0.includeKnownIssues: + reKnownIssue + else: + reDisabled else: result.parseErrors.addLine "invalid key for test spec: ", e.key diff --git a/testament/testament.nim b/testament/testament.nim index ae76e240b45..bace11c1cd8 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -16,24 +16,72 @@ import std/[ ] import backend, azure, htmlgen, specs from std/sugar import dup +from std/sequtils import map import utils/nodejs import lib/stdtest/testutils from lib/stdtest/specialpaths import splitTestFile import experimental/[sexp, sexp_diff, colortext, colordiff] +# TODO - things to do before this can be finalized: +# * get the tests running +# * remove legacy cruft inside testament +# * Test name, exe, printable name generation +# * Update usage and remove unsupported features +# * update docs so testament is for the compiler only + +const + failString* = "FAIL: " + ## ensures all failures can be searched with 1 keyword in CI logs + knownIssueSuccessString* = "KNOWNISSUE: " + ## ensures all now succeeding known issues can be searched with 1 keyword + ## in CI logs + testsDir = "tests" & DirSep + resultsFile = "testresults.html" + Usage = """Usage: + testament [options] command [arguments] + +Command: + p|pat|pattern run all the tests matching the given pattern + all run all tests + c|cat|category run all the tests of a certain category + r|run run single test file + html generate $1 from the database +Arguments: + arguments are passed to the compiler +Options: + --retry runs tests that failed the last run + --print print results to the console + --verbose print commands (compiling and running tests) + --simulate see what tests would be run but don't run them (for debugging) + --failing only show failing/ignored tests + --targets:"c cpp js objc" run tests for specified targets (default: all) + --nim:path use a particular nim executable (default: $$PATH/nim) + --directory:dir Change to directory dir before reading the tests or doing anything else. + --colors:on|off Turn messages coloring on|off. + --backendLogging:on|off Disable or enable backend logging. By default turned on. + --megatest:on|off Enable or disable megatest. Default is on. + --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored. + --includeKnownIssues runs tests that are marked as known issues + +On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API +provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN. + +Experimental: using environment variable `NIM_TESTAMENT_REMOTE_NETWORKING=1` enables +tests with remote networking (as in CI). +""" % resultsFile + proc trimUnitSep(x: var string) = let L = x.len if L > 0 and x[^1] == '\31': setLen x, L-1 -var useColors = true -var backendLogging = true -var simulate = false -var optVerbose = false -var useMegatest = true -var optFailing = false - -import std/sugar +var + useColors = true + backendLogging = true + simulate = false + optVerbose = false + useMegatest = true + optFailing = false type TOutReport = object @@ -52,8 +100,6 @@ type ignoredGiven: seq[int] cantIgnoreGiven: bool - - proc diffStrings*(a, b: string): tuple[output: string, same: bool] = let a = a.split("\n") let b = b.split("\n") @@ -80,9 +126,7 @@ proc diffStrings*(a, b: string): tuple[output: string, same: bool] = proc format(tcmp: TOutCompare): ColText = ## Pretty-print structured output comparison for further printing. - var - conf = diffFormatter() - res: ColText + var conf = diffFormatter() coloredResult() @@ -149,47 +193,10 @@ proc msg(msgType: MessageType; parts: varargs[string, `$`]) = stdout.writeLine parts flushFile stdout - -proc verboseCmd(cmd: string) = +proc verboseCmd(cmd: string) {.inline.} = if optVerbose: msg Undefined: "executing: " & cmd -const - failString* = "FAIL: " # ensures all failures can be searched with 1 keyword in CI logs - testsDir = "tests" & DirSep - resultsFile = "testresults.html" - Usage = """Usage: - testament [options] command [arguments] - -Command: - p|pat|pattern run all the tests matching the given pattern - all run all tests - c|cat|category run all the tests of a certain category - r|run run single test file - html generate $1 from the database -Arguments: - arguments are passed to the compiler -Options: - --retry runs tests that failed the last run - --print print results to the console - --verbose print commands (compiling and running tests) - --simulate see what tests would be run but don't run them (for debugging) - --failing only show failing/ignored tests - --targets:"c cpp js objc" run tests for specified targets (default: all) - --nim:path use a particular nim executable (default: $$PATH/nim) - --directory:dir Change to directory dir before reading the tests or doing anything else. - --colors:on|off Turn messages coloring on|off. - --backendLogging:on|off Disable or enable backend logging. By default turned on. - --megatest:on|off Enable or disable megatest. Default is on. - --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored - -On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API -provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN. - -Experimental: using environment variable `NIM_TESTAMENT_REMOTE_NETWORKING=1` enables -tests with remote networking (as in CI). -""" % resultsFile - proc isNimRepoTests(): bool = # this logic could either be specific to cwd, or to some file derived from # the input file, eg testament r /pathto/tests/foo/tmain.nim; we choose @@ -201,8 +208,7 @@ type Category = distinct string TResults = object - total, passed, failedButAllowed, skipped: int - ## xxx rename passed to passedOrAllowedFailure + total, passed, knownIssuesSucceeded, skipped: int data: string TTest = object @@ -212,6 +218,8 @@ type testArgs: seq[string] spec: TSpec startTime: float + duration: Option[float] + ## allows newer code pass duration to legacy code debugInfo: string # ---------------------------------------------------------------------------- @@ -222,8 +230,9 @@ let pegOtherError = peg"'Error:' \s* {.*}" pegOfInterest = pegLineError / pegOtherError -var gTargets = {low(TTarget)..high(TTarget)} -var targetsSet = false +var + gTargets = {low(TTarget)..high(TTarget)} + targetsSet = false proc isSuccess(input: string): bool = # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in @@ -279,14 +288,24 @@ proc nimcacheDir(filename, options: string, target: TTarget): string = result = "nimcache" / (filename & '_' & hashInput.getMD5) proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string, - target: TTarget, extraOptions = ""): string = + target: TTarget, extraOptions = "", outfile = ""): string = var options = target.defaultOptions & ' ' & options if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell) options.add ' ' & extraOptions + let outfileOption = + if outfile == "": + "" # don't do anything + else: + "--out:" & outfile.quoteShell + # we avoid using `parseCmdLine` which is buggy, refs bug #14343 - result = cmdTemplate % ["target", target.cmd, - "options", options, "file", filename.quoteShell, - "filedir", filename.getFileDir(), "nim", compilerPrefix] + result = cmdTemplate % [ + "target", target.cmd, + "options", options, + "file", filename.quoteShell, + "filedir", filename.getFileDir(), + "nim", compilerPrefix, + "outfileOption", outfileOption] proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, target: TTarget, extraOptions = ""): TSpec = @@ -353,7 +372,7 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, proc initResults: TResults = result.total = 0 result.passed = 0 - result.failedButAllowed = 0 + result.knownIssuesSucceeded = 0 result.skipped = 0 result.data = "" @@ -382,12 +401,12 @@ template maybeStyledEcho(args: varargs[untyped]): untyped = proc `$`(x: TResults): string = result = """ -Tests passed or allowed to fail: $2 / $1
-Tests failed and allowed to fail: $3 / $1
+Tests passed: $2 / $1
+Tests known issues succeeded: $3 / $1
Tests skipped: $4 / $1
-""" % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped] +""" % [$x.total, $x.passed, $x.knownIssuesSucceeded, $x.skipped] -proc getName(test: TTest, target: TTarget, allowFailure: bool): string = +proc printableName(test: TTest, target: TTarget, allowFailure: bool): string = var name = test.name.replace(DirSep, '/') name.add ' ' & $target if allowFailure: @@ -401,19 +420,17 @@ proc getName(test: TTest, target: TTarget, allowFailure: bool): string = type ReportParams = object ## Contains additional data about report execution state. - duration: float name: string + duration: float outCompare: TOutCompare - success: TResultEnum - expected, given: string + success: TResultEnum proc logToConsole( test: TTest, param: ReportParams, givenSpec: ptr TSpec = nil ) = - ## Format test infomation to the console. `test` contains information ## about the test itself, `param` contains additional data about test ## execution. @@ -529,22 +546,24 @@ proc addResult( # given. test.name is easier to find than test.name.extractfilename a bit # hacky but simple and works with tests/testament/tshould_not_work.nim - # Compute test test duration, final success status, prepare formatting variables + # Compute test duration, final success status, prepare formatting variables var param: ReportParams + param.name = test.printableName(target, allowFailure) + param.duration = + if test.duration.isSome: + test.duration.get + else: + epochTime() - test.startTime + param.outCompare = outCompare param.expected = expected param.given = given - param.outCompare = outCompare - param.duration = epochTime() - test.startTime param.success = if test.spec.timeout > 0.0 and param.duration > test.spec.timeout: reTimeout else: successOrig - - param.name = test.getName(target, allowFailure) - if backendLogging: backend.writeTestResult(name = param.name, category = test.cat.string, @@ -557,7 +576,6 @@ proc addResult( # TODO DOC what is this r.data.addf("$#\t$#\t$#\t$#", param.name, expected, given, $param.success) - # Write to console logToConsole(test, param, givenSpec) @@ -628,7 +646,7 @@ proc nimoutCheck(expected, given: TSpec): bool = result = false proc sexpCheck(test: TTest, expected, given: TSpec): TOutCompare = - ## Check if expected nimout values match with specified ones. Thish check + ## Check if expected nimout values match with specified ones. This check ## implements a structured comparison of the data and returns full report ## about all the mismatches that can be formatted as needed. ## This procedure determines whether `given` spec matches `expected` test @@ -654,7 +672,6 @@ proc sexpCheck(test: TTest, expected, given: TSpec): TOutCompare = if 0 < line.len: r.givenReports.add TOutReport(node: parseSexp(line)) - var map = r.diffMap proc reportCmp(a, b: int): int = # Best place for further optimization and configuration - if more # comparison speed is needed, try starting with error kind, file, line @@ -698,7 +715,6 @@ proc cmpMsgs( # If structural comparison is requested - drop directly to it and handle # the success/failure modes in the branch if expected.nimoutSexp: - echo "executing structural comparison" let outCompare = test.sexpCheck(expected, given) # Full match of the output results. if outCompare.match: @@ -766,12 +782,16 @@ proc cmpMsgs( proc generatedFile(test: TTest, target: TTarget): string = ## Get path to the generated file name from the test. - if target == targetJS: - result = test.name.changeFileExt("js") - else: - let (_, name, _) = test.name.splitFile - let ext = target.ext - result = nimcacheDir(test.name, test.options, target) / "@m" & name.changeFileExt(ext) + result = + case target + of targetJS: + test.name.changeFileExt("js") + else: + let + testFile = test.spec.file + (_, name, _) = testFile.splitFile + ext = target.ext + nimcacheDir(testFile, test.options, target) / "@m" & name.changeFileExt(ext) proc needsCodegenCheck(spec: TSpec): bool = ## If there is any checks that need to be performed for a generated code @@ -879,6 +899,27 @@ proc equalModuloLastNewline(a, b: string): bool = # be fixed instead result = a == b or b.endsWith("\n") and a == b[0 ..< ^1] +proc checkCmdHelper(r: var TResults, + test: TTest, + target: TTarget, + expected: TSpec, + given: var TSpec, + cmdOut: string, + exitCode: int) = + ## extracted from `testSpecHelper` to allow for reuse in new testament runner + + case given.err + of reExitcodesDiffer: + r.addResult(test, + target, + "exitcode: " & $expected.exitCode, + "exitcode: " & $exitCode & "\n\nOutput:\n" & cmdOut, + reExitcodesDiffer) + of reOutputsDiffer: + r.addResult(test, target, expected.output, cmdOut, reOutputsDiffer) + else: + compilerOutputTests(test, target, given, expected, r) + proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, target: TTarget, nimcache: string, extraOptions = "") = test.startTime = epochTime() @@ -943,9 +984,10 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, else: buf if exitCode != expected.exitCode: - r.addResult(test, target, "exitcode: " & $expected.exitCode, - "exitcode: " & $exitCode & "\n\nOutput:\n" & - bufB, reExitcodesDiffer) + given.err = reExitcodesDiffer + # r.addResult(test, target, "exitcode: " & $expected.exitCode, + # "exitcode: " & $exitCode & "\n\nOutput:\n" & + # bufB, reExitcodesDiffer) elif ( expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB) @@ -954,9 +996,11 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, expected.output notin bufB ): given.err = reOutputsDiffer - r.addResult(test, target, expected.output, bufB, reOutputsDiffer) + # r.addResult(test, target, expected.output, bufB, reOutputsDiffer) else: - compilerOutputTests(test, target, given, expected, r) + discard + # compilerOutputTests(test, target, given, expected, r) + checkCmdHelper(r, test, target, expected, given, bufB, exitCode) of actionReject: let given = callNimCompilerImpl() # Scan compiler output fully for all mismatches and report if any found @@ -1042,6 +1086,193 @@ const # array of modules disabled from compilation test of stdlib. disabledFiles = disabledFilesDefault +type + TestId = int # xxx: make this a distinct + RunId = int ## test run's id/index # xxx: make this a distinct + EntryId = int ## matrix entry index # xxx: make this a distinct + ActionId = int ## a test action's id # xxx: make this a distinct + CategoryId = int ## a category's id # xxx: make this a distinct + TestTarget = TTarget # xxx: renamed because I dislike the TXxx convention + + Categories = seq[Category] + + GlobPattern = string + + TestFilterKind {.pure.} = enum + tfkAll, ## all tests + tfkCategories, ## one or more categories + tfkGlob, ## glob file pattern + tfkSingle ## single test + + TestFilter = object + restOfCmdLine: string + case kind: TestFilterKind + of tfkAll: + discard + of tfkCategories: + # xxx: currently multiple categories are unsupported + cats: Categories + of tfkGlob: + pattern: GlobPattern + of tfkSingle: + test: string + + ExecutionFlag = enum + outputColour, ## colour the output + outputResults, ## print results to the console + outputFailureOnly, ## only output failures + outputVerbose, ## increase output verbosity + logBackend ## enable backend logging + dryRun, ## do not run the tests, only indicate which would run + rerunFailed, ## only run tests failed in the previous run + runKnownIssues ## also execute tests marked as known issues + + ExecutionFlags = set[ExecutionFlag] + ## track the option flags that got set for an execution + + RetryInfo = object + test: TestId ## which test failed + target: TestTarget ## the specific target + + RetryList = OrderedTable[TestId, RetryInfo] + ## record failures in here so the user can choose to retry them + + DebugInfo = OrderedTable[RunId, string] + # Refactor - debug info should be per action instead of per run + + TestTargets = set[TestTarget] + + # Category is in specs.nim + + TestFile = object + file: string + catId: CategoryId + + # TSpec is in specs.nim + # TSpec is the the spec in a single test file, but we run (`TestRun`) a test + # for every target and matrix entry, which itself is a number of actions and + # checks. + + TestRun = object + testId: TestId ## test id for which this belongs + target: TestTarget ## which target to run for + matrixEntry: EntryId ## which item from the matrix was used + + # xxx: add 'check' to remove `cmd: "nim check"...` from tests + # TestActionKind = enum + # testActionSkip ## skip this test; check the spec for why + # testActionReject, ## reject the compilation + # testActionCompile, ## compile some source + # testActionRun ## run the compiled program + + TestAction = object + runId: RunId + case kind: TTestAction # xxx: might not need `partOfRun` at all + of actionReject: + discard + of actionRun: + compileActionId: ActionId ## id of the preceeding compile action + of actionCompile: + partOfRun: bool + + RunTime = object + ## time tracking for test run activities + compileStart: float ## when the compile process start + compileEnd: float ## when the compile process ends + compileCheckStart: float ## when compile output check started + compileCheckEnd: float ## when compile output check finished + runStart: float ## for run, start of execution + runEnd: float ## for run, end of execution + runCheckStart: float ## start of run output check + runCheckEnd: float ## end of run output check + + RunActual = object + ## actual data for a run + nimout: string ## nimout from compile, empty if not required + nimExit: int ## exit code produced by the compiler + nimMsg: string ## last message, if any, from the compiler + nimFile: string ## filename from last compiler message, if present + nimLine: int ## line from last compiler message, if present + nimColumn: int ## colunn from last compiler message, if present + prgOut: string ## program output, if any + prgOutSorted: string ## sorted version of `prgOut`; kept for legacy + ## reason, remove when no longer necessary + prgExit: int ## program exit, if any + lastAction: ActionId ## last action in this run + runResult: TResultEnum ## current result, invalid if `lastAction` unset + + RunActuals = seq[RunActual] + + TestOptionData = object + optMatrix: seq[string] ## matrix of cli options for this test + action: Option[TTestAction] ## possible action override + + TestOptions = OrderedTable[TestId, TestOptionData] + ## for legacy reasons (eg: `dllTests`) we need to be able to set per test + ## options, ideally this would be done with `matrix`, but it's not + ## sophisticated enough to support spare configuration like this needs + + Execution = object + # user and execution inputs + filter: TestFilter ## filter that was configured + flags: ExecutionFlags ## various options set by the user + skipsFile: string ## test files to skip loaded from `--skipFrom` + targets: TestTargets ## specified targets or `noTargetsSpecified` + workingDir: string ## working directory to begin execution in + nodeJs: string ## path to nodejs binary + nimSpecified: bool ## whether the user specified the nim + testArgs: string ## arguments passed to tests by the user + isCompilerRepo: bool ## whether this is the compiler repository, used + ## to legacy to handle `AdditionalCategories` + + # environment input / setup + compilerPath: string ## compiler command to use + testsDir: string ## where to look for tests + + # test discovery data + categories: Categories ## categories discovered for this execution + ## first one is a default empty category `""` + testFiles: seq[TestFile] ## files for this execution + testSpecs: seq[TSpec] ## spec for each file + testOpts: TestOptions ## per test options, because legacy category magic + + # test execution data + testRuns: seq[TestRun] ## a test run: reject, compile, or compile + run + ## along with time to check; runs for a test must + ## be contiguous and ordered + runTimes: seq[RunTime] ## run timing information for each test run + runActuals: RunActuals ## actual information for a given run + debugInfo: DebugInfo ## debug info related to runs for tests, should be + ## per action instead of per run. + runProgress: RunProgress ## current run progress based on action progress + + actions: seq[TestAction] ## test actions for each run, phases of a run; + ## actions for a run must be continugous and + ## ordered + + # test execution related data + retryList: RetryList ## list of failures to potentially retry later + + # legacy compat stuff -- try to remove + legacyTestResults: TResults ## Legacy compatability for result reporting + ## kept to limit refactor work to running + ## tests and not include reporting. + legacyTestData: seq[string] ## `TResults` had a `data` field of type + ## `string` where we appended a message per + ## test... which seems horribly wasteful. + ## keep it around until we know we can kill it + + RunProgress = object + ## Acts as a tracker for a producer/consumer model, where `lastCheckedRun` + ## is where the consumer left off, and `mostRecentRun` is where the + ## producer left off. + lastCheckedRun: RunId ## run last reviewed for reporting/consumption + mostRecentRun: RunId ## run that completed most recently/production + + ParseCliResult = enum + parseSuccess ## successfully parsed cli params + parseQuitWithUsage ## parsing failed, quit with usage message + include categories proc loadSkipFrom(name: string): seq[string] = @@ -1062,7 +1293,6 @@ proc main() = of "print": optPrintResults = true of "verbose": optVerbose = true of "failing": optFailing = true - of "pedantic": discard # deadcode refs https://github.com/nim-lang/Nim/issues/16731 of "targets": targetsStr = p.val gTargets = parseTargets(targetsStr) @@ -1079,6 +1309,11 @@ proc main() = useColors = false else: quit Usage + of "withMegaTestRun": + # if megatest was included, then all parallel invocations of testatment + # need to know, this way we can tell if pcat should run joinable tests + # or not + testamentData0.withMegaTestRun = true of "batch": testamentData0.batchArg = p.val if p.val != "_" and p.val.len > 0 and p.val[0] in {'0'..'9'}: @@ -1094,6 +1329,7 @@ proc main() = case p.val: of "on": useMegatest = true + testamentData0.withMegaTestRun = true of "off": useMegatest = false else: @@ -1108,6 +1344,12 @@ proc main() = quit Usage of "skipfrom": skipFrom = p.val + of "includeKnownIssues": + testamentData0.includeKnownIssues = + case p.val + of "on": true + of "off": false + else: quit Usage of "retry": retryContainer.retry = true (retryContainer.cats, retryContainer.names) = backend.getRetries() @@ -1123,16 +1365,17 @@ proc main() = azure.init() backend.open() ## Cli options and misc - var optPrintResults = false - var targetsStr = "" - var isMainProcess = true - var skipFrom = "" - # Init cli parser - var p = initOptParser() + var + optPrintResults = false + targetsStr = "" + isMainProcess = true + skipFrom = "" + p = initOptParser() # Init cli parser p.next() ## First parse options parseOptions(p) # template will complete with next option loaded + # Next option should be cmdarg var action: string parseArgs(p) @@ -1179,14 +1422,15 @@ proc main() = if isNimRepoTests(): cats.add AdditionalCategories # User may pass an option to skip the megatest category, default is useMegaTest - if useMegatest: cats.add MegaTestCat + if useMegatest: + cats.add MegaTestCat # We now prepare the command line arguments for our child processes for cat in cats: # Remember that if we are performing the megatest category, then # all joinable tests will be covered in that, so we use the parallel cat # action - let runtype = if useMegatest: " pcat " else: " cat " + let runtype = if useMegatest: " --withMegaTestRun pcat " else: " cat " cmds.add(myself & runtype & quoteShell(cat) & rest) if simulate: # 'see what tests would be run but don't run them (for debugging)' @@ -1208,17 +1452,20 @@ proc main() = of "c", "cat", "category": # Run all tests of a certain category skips = loadSkipFrom(skipFrom) - var cat = Category(p.key) + let cat = Category(p.key) processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = true) of "pcat": # Run cat in parallel # Run all tests of a certain category in parallel; does not include joinable # tests which are covered in the 'megatest' category. skips = loadSkipFrom(skipFrom) isMainProcess = false - var cat = Category(p.key) + let + cat = Category(p.key) + withMegaTestRun = testamentData0.withMegaTestRun p.next - processCategory(r, cat, p.cmdLineRest, testsDir, runJoinableTests = false) - of "p", "pat", "pattern": # Run all tests matching the given pattern + processCategory(r, cat, p.cmdLineRest, testsDir, + runJoinableTests = not withMegaTestRun) + of "p", "pat", "pattern": # Run all tests matching the given pattern skips = loadSkipFrom(skipFrom) let pattern = p.key p.next @@ -1247,4 +1494,928 @@ proc main() = if paramCount() == 0: quit Usage -main() +# main() + +const + testResultsDir = "testresults" + cacheResultsDir = testResultsDir / "cacheresults" + noTargetsSpecified: TestTargets = {} + defaultExecFlags = {outputColour} + defaultBatchSize = 10 + noMatrixEntry: EntryId = -1 + defaultCatId: CategoryId = 0 + +proc parseOpts(execState: var Execution, p: var OptParser): ParseCliResult = + # xxx: create a narrower type to disallow mutation elsewhere + result = parseSuccess + p.next() + while p.kind in {cmdLongOption, cmdShortOption}: + # read options agnostic of casing + case p.key.normalize + of "print": execState.flags.incl outputResults + of "verbose": execState.flags.incl outputVerbose + of "failing": execState.flags.incl outputFailureOnly + of "targets": + var targetStr = p.val + execState.targets = parseTargets(targetStr) + of "nim", "compiler": + execState.compilerPath = addFileExt(p.val.absolutePath, ExeExt) + # xxx: legacy, remove once `prepareTestCmd`, etc are ported + compilerPrefix = execState.compilerPath + of "directory": + execState.workingDir = p.val + of "colors", "colours": + case p.val: + of "on": execState.flags.incl outputColour + of "off": execState.flags.excl outputColour + else: return parseQuitWithUsage + of "simulate": + execState.flags.incl dryRun + of "backendlogging": + case p.val: + of "on": execState.flags.incl logBackend + of "off": execState.flags.excl logBackend + else: return parseQuitWithUsage + of "includeknownissues": + case p.val: + of "on": execState.flags.incl runKnownIssues + of "off": execState.flags.excl runKnownIssues + else: return parseQuitWithUsage + of "retry": + execState.flags.incl rerunFailed + of "skipFrom": + execState.skipsFile = p.val + else: + return parseQuitWithUsage + p.next() + +proc parseArg(execState: var Execution, p: var OptParser): ParseCliResult = + # xxx: create types to avoid accidental mutation + # next part should be the the filter action, eg: cat, r, etc... + result = parseSuccess + + var filterAction: string + if p.kind != cmdArgument: + quit Usage + filterAction = p.key.normalize + p.next() + + case filterAction + of "all": + # filter nothing, run everything + execState.filter = TestFilter(kind: tfkAll, restOfCmdLine: p.cmdLineRest) + of "c", "cat", "category": + # only specified category + execState.filter = TestFilter( + kind: tfkCategories, + cats: @[Category(p.key)], + restOfCmdLine: p.cmdLineRest + ) + of "pcat": + # run category's tests in parallel + # xxx: consider removing pcat concept or make parallel the default + execState.filter = TestFilter( + kind: tfkCategories, + cats: @[Category(p.key)], + restOfCmdLine: p.cmdLineRest + ) + of "p", "pat", "pattern": + # tests files matching the given pattern + execState.filter = TestFilter( + kind: tfkGlob, + pattern: p.key, + restOfCmdLine: p.cmdLineRest + ) + of "r", "run": + # single test + execState.filter = TestFilter( + kind: tfkSingle, + test: p.key, + restOfCmdLine: p.cmdLineRest + ) + else: return parseQuitWithUsage + +func requestedTargets(execState: Execution): set[TTarget] = + ## get the requested targets by the user or the defaults + if execState.targets == noTargetsSpecified: + {targetC, targetJS} + else: + execState.targets + +func `<`(a, b: TestFile): bool {.inline.} = + a.file < b.file +func cmp(a, b: TestFile): int {.inline.} = + cmp(a.file, b.file) + +proc prepareTestFilesAndSpecs(execState: var Execution) = + ## for the filters specified load all the specs + # xxx: create specific type to avoid accidental mutation + + # xxx: read-only state in let to avoid mutation, put into types + let + testsDir = execState.testsDir + filter = execState.filter + isCompilerRepo = execState.isCompilerRepo + + # REFACTOR: legacy set `specs.skips` + skips = loadSkipFrom(execState.skipsFile) + + template testFilesFromCat(execState: var Execution, cat: Category) = + if cat.string notin ["testdata", "nimcache"]: + let catId = execState.categories.len + execState.categories.add cat + + var handled = true + + let normCat = cat.string.normalize + if isCompilerRepo: + case normCat + of "gc": + setupGcTests(execState, catId) + of "threads": + setupThreadTests(execState, catId) + of "lib": + # xxx: implement this proc and all the subsequent handling + setupStdlibTests(execState, catId) + else: + handled = false + + if not handled: + for file in walkDirRec(testsDir & cat.string): + if file.isTestFile: + execState.testFiles.add: + TestFile(file: file, catId: catId) + + case filter.kind + of tfkAll: + let testsDir = testsDir + for kind, dir in walkDir(testsDir): + if kind == pcDir: + # The category name is extracted from the directory + # eg: 'tests/compiler' -> 'compiler' + let cat = dir[testsDir.len .. ^1] + testFilesFromCat(execState, Category(cat)) + if isCompilerRepo: # handle `AdditionalCategories` + for cat in AdditionalCategories: + testFilesFromCat(execState, Category(cat)) + of tfkCategories: + for cat in filter.cats: + testFilesFromCat(execState, cat) + of tfkGlob: + execState.categories = @[Category ""] + let pattern = filter.pattern + if dirExists(pattern): + for kind, name in walkDir(pattern): + if kind in {pcFile, pcLinkToFile} and name.endsWith(".nim"): + execState.testFiles.add TestFile(file: name) + else: + for name in walkPattern(pattern): + execState.testFiles.add TestFile(file: name) + of tfkSingle: + execState.categories = @[Category ""] + let test = filter.test + # xxx: replace with proper error handling + doAssert fileExists(test), test & " test does not exist" + if isTestFile(test): + execState.testFiles.add TestFile(file: test) + + execState.testFiles.sort # ensures we have a reproducible ordering + + # parse all specs + for testId, test in execState.testFiles.pairs: + execState.testSpecs.add parseSpec(addFileExt(test.file, ".nim")) + + if execState.testOpts.hasKey(testId): + # apply additional test matrix, if specified + let optMatrix = execState.testOpts[testId].optMatrix + execState.testSpecs[testId].matrix = + case execState.testSpecs[testId].matrix.len + of 0: + optMatrix + else: + # REFACTOR - this is a hack, we shouldn't modify the spec.matrix + var tmp: seq[string] = @[] + for o in optMatrix.items: + for e in execState.testSpecs[testId].matrix.items: + tmp.add o & " " & e + tmp + + # apply action override, if specified + let actionOverride = execState.testOpts[testId].action + if actionOverride.isSome: + execState.testSpecs[testId].action = actionOverride.get + +proc prepareTestRuns(execState: var Execution) = + ## create a list of necessary testRuns + # xxx: create specific type to avoid accidental mutation + + # xxx: read-only items; only testRuns are written + let + testSpecs = execState.testSpecs + testFiles = execState.testFiles + categories = execState.categories + + for testId, spec in testSpecs.pairs: + let + specTargets = + if spec.targets == noTargetsSpecified: + categories[testFiles[testId].catId].defaultTargets + else: + spec.targets + targetsToRun = specTargets * execState.requestedTargets + + for target in targetsToRun: + # TODO: create a "target matrix" to cover both js release vs non-release + + case spec.matrix.len + of 0: # no tests to run + execState.testRuns.add: + TestRun(testId: testId, target: target, matrixEntry: noMatrixEntry) + execState.runTimes.add RunTime() + execState.runActuals.add RunActual() + else: + for entryId, _ in spec.matrix.pairs: + execState.testRuns.add: + TestRun(testId: testId, target: target, matrixEntry: entryId) + execState.runTimes.add RunTime() + execState.runActuals.add RunActual() + +proc prepareTestActions(execState: var Execution) = + ## create a list of necessary test actions + # xxx: create specific type to avoid accidental mutation + + # xxx: these are what are read, so a dedicate type would offer a read-only + # view of those, but allow actions mutation + let + testRuns = execState.testRuns + testSpecs = execState.testSpecs + + # TODO: handle disabled and known issue + + for runId, run in testRuns.pairs: + let actionKind = testSpecs[run.testId].action + case actionKind + of actionReject, actionCompile: + execState.actions.add: + TestAction(runId: runId, kind: actionKind) + of actionRun: + let compileActionId = execState.actions.len + execState.actions.add: + TestAction(runId: runId, kind: actionCompile, partOfRun: true) + execState.actions.add: + TestAction(runId: runId, + kind: actionRun, + compileActionId: compileActionId) + +proc exeFile(testRun: TestRun, specFilePath: string): string = + let + target = testRun.target + isJsTarget = target == targetJs + (dirPart, specName, _) = splitFile(specFilePath) + matrixEntry = testRun.matrixEntry + exeName = + (if matrixEntry == noMatrixEntry: + @[specName, target.cmd] + else: + @[specName, $matrixEntry, target.cmd]).join("_") + exeExt = + if isJsTarget: + "js" + else: + ExeExt + + changeFileExt(joinPath(dirPart, exeName), exeExt) + +proc makeName(test: TestFile, + testRun: TestRun, + allowFailure: bool + ): string = + let + target = testRun.target + matrixEntry = testRun.matrixEntry + result = test.file.changeFileExt("").replace(DirSep, '/') + result.add '_' & $target + if matrixEntry != noMatrixEntry: + result.add "[$1]" % $matrixEntry + if allowFailure: + result.add " (allowed to fail) " + # if test.options.len > 0: + # result.add ' ' & test.options + +proc reportTestRunResult( + legacyResults: var TResults, + cat: Category, + testFile: TestFile, + spec: TSpec, + testRun: TestRun, + runActual: RunActual, + runTime: RunTime, + action: TestAction, + cmd: string, + debugInfo: string + ) = + ## birdge newer testament internals (`ExecutionState` and friends) to the + ## legacy reporting and generate output accordingly. + + let + duration = runTime.compileEnd - runTime.compileStart + + runTime.runEnd - runTime.runStart + target = testRun.target + allowFailure = spec.err == reKnownIssue + + var + givenAsSpec = TSpec( + cmd: cmd, + nimout: runActual.nimout, + msg: runActual.nimMsg, + file: runActual.nimFile, + output: runActual.prgOut, + line: runActual.nimLine, + column: runActual.nimColumn, + err: runActual.runResult, + debugInfo: debugInfo) + legacyTest = TTest( + cat: cat, + name: testFile.makeName(testRun, allowFailure), + options: + if testRun.matrixEntry == noMatrixEntry: + "" + else: + spec.matrix[testRun.matrixEntry], + spec: spec, + duration: some(duration)) + + case spec.action + of actionCompile: + compilerOutputTests(legacyTest, target, givenAsSpec, spec, legacyResults) + of actionRun: + case action.kind + of actionCompile: + case givenAsSpec.err + of reSuccess: + discard # success will be determined after `actionRun` + of reExeNotFound: + legacyResults.addResult( + legacyTest, + target, + spec.output, + "executable not found: " & testRun.exeFile(givenAsSpec.file), + reExeNotFound + ) + else: + # all other errors + legacyResults.addResult( + legacyTest, + target, + "", + "$ " & givenAsSpec.cmd & '\n' & givenAsSpec.nimout, + givenAsSpec.err, + givenSpec = givenAsSpec.addr + ) + of actionRun: + checkCmdHelper( + legacyResults, + legacyTest, + testRun.target, + spec, + givenAsSpec, + runActual.prgOutSorted, + case runActual.prgExit: + # xxx - sigh... weird legacy + # Treat all failure codes from nodejs as 1. Older versions of + # nodejs used to return other codes, but for us it is sufficient + # to know that it's not 0. + of 0: 0 + else: 1 + ) + of actionReject: + doAssert false, "we should never get here" + of actionReject: + # Scan compiler output fully for all mismatches and report if any found + cmpMsgs(legacyResults, spec, givenAsSpec, legacyTest, target) + +template runTestBatch(execState: var Execution, + testCmds: seq[string], + processOpts: set[ProcessOption], + batchSize: int, + exitCodes: seq[int], + outputs: seq[string], + startTimes, endTimes: seq[float], + onTestRunStart: proc (i: int), + onTestProcess, onTestRunComplete: proc (i: int, p: Process), + cmdIdToActId: var seq[int], + cmdIdToActKind: var seq[TTestAction], + cmdIdToInput: var seq[Option[string]] + ) = + inc batches + + # TODO - handle executable not found issues for `actionRun`, test this before + # the action is swapped in from next to current batch (it should have + # already been created). If it's not, note progress and remove from + # runnable actions. + + # run actions + #echo "execProcesses with: ", testCmds, processOpts, batchSize #, " and map: ", cmdIdToActId + discard osproc.execProcesses( + testCmds, + processOpts, + batchSize, + onTestRunStart, + onTestProcess, + onTestRunComplete + ) + + for id in 0.. - age: int #<2> - -let dog = Dog(age: 3) #<3> - -proc showNumber(num: int | float) = - echo(num) - -showNumber(3.14) -showNumber(42) - -for i in 0 ..< 10: - echo(i) - -block: # Block added due to clash. - type - Dog = object - - proc bark(self: Dog) = #<1> - echo("Woof!") - - let dog = Dog() - dog.bark() #<2> - -import sequtils, sugar, strutils -let list = @["Dominik Picheta", "Andreas Rumpf", "Desmond Hume"] -list.map( - (x: string) -> (string, string) => (x.split[0], x.split[1]) -).echo - -import strutils -let list1 = @["Dominik Picheta", "Andreas Rumpf", "Desmond Hume"] -for name in list1: - echo((name.split[0], name.split[1])) - diff --git a/tests/niminaction/Chapter2/explicit_discard.nim b/tests/niminaction/Chapter2/explicit_discard.nim deleted file mode 100644 index 7f3b3395e4a..00000000000 --- a/tests/niminaction/Chapter2/explicit_discard.nim +++ /dev/null @@ -1,7 +0,0 @@ -discard """ - errormsg: "has to be used (or discarded)" - line: 7 -""" - -proc myProc(name: string): string = "Hello " & name -myProc("Dominik") diff --git a/tests/niminaction/Chapter2/no_def_eq.nim b/tests/niminaction/Chapter2/no_def_eq.nim deleted file mode 100644 index b9d62e0362d..00000000000 --- a/tests/niminaction/Chapter2/no_def_eq.nim +++ /dev/null @@ -1,16 +0,0 @@ -discard """ - errormsg: "type mismatch" - line: 16 -""" - -type - Dog = object - name: string - - Cat = object - name: string - -let dog: Dog = Dog(name: "Fluffy") -let cat: Cat = Cat(name: "Fluffy") - -echo(dog == cat) diff --git a/tests/niminaction/Chapter2/no_iterator.nim b/tests/niminaction/Chapter2/no_iterator.nim deleted file mode 100644 index 555fac21acd..00000000000 --- a/tests/niminaction/Chapter2/no_iterator.nim +++ /dev/null @@ -1,7 +0,0 @@ -discard """ - errormsg: "type mismatch" - line: 6 -""" - -for i in 5: - echo i diff --git a/tests/niminaction/Chapter2/no_seq_type.nim b/tests/niminaction/Chapter2/no_seq_type.nim deleted file mode 100644 index f1494124b0f..00000000000 --- a/tests/niminaction/Chapter2/no_seq_type.nim +++ /dev/null @@ -1,6 +0,0 @@ -discard """ - errormsg: "cannot infer the type of the sequence" - line: 6 -""" - -var list = @[] diff --git a/tests/niminaction/Chapter2/resultaccept.nim b/tests/niminaction/Chapter2/resultaccept.nim deleted file mode 100644 index 390f7b32991..00000000000 --- a/tests/niminaction/Chapter2/resultaccept.nim +++ /dev/null @@ -1,28 +0,0 @@ -discard """ - output: "" -""" - -# Page 35. - -proc implicit: string = - "I will be returned" - -proc discarded: string = - discard "I will not be returned" - -proc explicit: string = - return "I will be returned" - -proc resultVar: string = - result = "I will be returned" - -proc resultVar2: string = - result = "" - result.add("I will be ") - result.add("returned") - -doAssert implicit() == "I will be returned" -doAssert discarded().len == 0 -doAssert explicit() == "I will be returned" -doAssert resultVar() == "I will be returned" -doAssert resultVar2() == "I will be returned" \ No newline at end of file diff --git a/tests/niminaction/Chapter2/resultreject.nim b/tests/niminaction/Chapter2/resultreject.nim deleted file mode 100644 index 14534507251..00000000000 --- a/tests/niminaction/Chapter2/resultreject.nim +++ /dev/null @@ -1,33 +0,0 @@ -discard """ - errormsg: "has to be used (or discarded)" - line: 27 -""" - -# Page 35. - -proc implicit: string = - "I will be returned" - -proc discarded: string = - discard "I will not be returned" - -proc explicit: string = - return "I will be returned" - -proc resultVar: string = - result = "I will be returned" - -proc resultVar2: string = - result = "" - result.add("I will be ") - result.add("returned") - -proc resultVar3: string = - result = "I am the result" - "I will cause an error" - -doAssert implicit() == "I will be returned" -doAssert discarded() == nil -doAssert explicit() == "I will be returned" -doAssert resultVar() == "I will be returned" -doAssert resultVar2() == "I will be returned" diff --git a/tests/niminaction/Chapter2/various2.nim b/tests/niminaction/Chapter2/various2.nim deleted file mode 100644 index 921f38c7d13..00000000000 --- a/tests/niminaction/Chapter2/various2.nim +++ /dev/null @@ -1,369 +0,0 @@ -discard """ - exitCode: 0 - outputsub: '''42 is greater than 0''' -""" - -if 42 >= 0: - echo "42 is greater than 0" - - -echo("Output: ", - 5) -echo(5 + - 5) -# --- Removed code that is supposed to fail here. Not going to test those. --- - -# Single-line comment -#[ -Multiline comment -]# -when false: - echo("Commented-out code") - -let decimal = 42 -let hex = 0x42 -let octal = 0o42 -let binary = 0b101010 - -let a: int16 = 42 -let b = 42'i8 - -let c = 1'f32 # --- Changed names here to avoid clashes --- -let d = 1.0e19 - -let e = false -let f = true - -let g = 'A' -let h = '\109' -let i = '\x79' - -let text = "The book title is \"Nim in Action\"" - -let filepath = "C:\\Program Files\\Nim" - -# --- Changed name here to avoid clashes --- -let filepath1 = r"C:\Program Files\Nim" - -let multiLine = """foo - bar - baz -""" -echo multiLine - -import strutils -# --- Changed name here to avoid clashes --- -let multiLine1 = """foo - bar - baz -""" -echo multiLine1.unindent -doAssert multiLine1.unindent == "foo\nbar\nbaz\n" - -proc fillString(): string = - result = "" - echo("Generating string") - for i in 0 .. 4: - result.add($i) #<1> - -const count = fillString() - -var - text1 = "hello" - number: int = 10 - isTrue = false - -var 火 = "Fire" -let ogień = true - -var `var` = "Hello" -echo(`var`) - -proc myProc(name: string): string = "Hello " & name -discard myProc("Dominik") - -proc bar(): int #<1> - -proc foo(): float = bar().float -proc bar(): int = foo().int - -proc noReturn() = echo("Hello") -proc noReturn2(): void = echo("Hello") - -proc noReturn3 = echo("Hello") - -proc message(recipient: string): auto = - "Hello " & recipient - -doAssert message("Dom") == "Hello Dom" - -proc max(a: int, b: int): int = - if a > b: a else: b - -doAssert max(5, 10) == 10 - -proc max2(a, b: int): int = - if a > b: a else: b - -proc genHello(name: string, surname = "Doe"): string = - "Hello " & name & " " & surname - -# -- Leaving these as asserts as that is in the original code, just in case -# -- somehow in the future `assert` is removed :) -assert genHello("Peter") == "Hello Peter Doe" -assert genHello("Peter", "Smith") == "Hello Peter Smith" - -proc genHello2(names: varargs[string]): string = - result = "" - for name in names: - result.add("Hello " & name & "\n") - -doAssert genHello2("John", "Bob") == "Hello John\nHello Bob\n" - -proc getUserCity(firstName, lastName: string): string = - case firstName - of "Damien": return "Tokyo" - of "Alex": return "New York" - else: return "Unknown" - -proc getUserCity(userID: int): string = - case userID - of 1: return "Tokyo" - of 2: return "New York" - else: return "Unknown" - -doAssert getUserCity("Damien", "Lundi") == "Tokyo" -doAssert getUserCity(2) == "New York" # -- Errata here: missing closing " - -import sequtils -let numbers = @[1, 2, 3, 4, 5, 6] -let odd = filter(numbers, proc (x: int): bool = x mod 2 != 0) -doAssert odd == @[1, 3, 5] - -import sequtils, sugar -let numbers1 = @[1, 2, 3, 4, 5, 6] -let odd1 = filter(numbers1, (x: int) -> bool => x mod 2 != 0) -assert odd1 == @[1, 3, 5] - -proc isValid(x: int, validator: proc (x: int): bool) = - if validator(x): echo(x, " is valid") - else: echo(x, " is NOT valid") - -import sugar -proc isValid2(x: int, validator: (x: int) -> bool) = - if validator(x): echo(x, " is valid") - else: echo(x, " is NOT valid") - -var list: array[3, int] -list[0] = 1 -list[1] = 42 -assert list[0] == 1 -assert list[1] == 42 -assert list[2] == 0 #<1> - -echo list.repr #<2> - -# echo list[500] - -var list2: array[-10 .. -9, int] -list2[-10] = 1 -list2[-9] = 2 - -var list3 = ["Hi", "There"] - -var list4 = ["My", "name", "is", "Dominik"] -for item in list4: - echo(item) - -for i in list4.low .. list4.high: - echo(list4[i]) - -var list5: seq[int] = @[] -doAssertRaises(IndexDefect): - list5[0] = 1 - -list5.add(1) - -assert list5[0] == 1 -doAssertRaises(IndexDefect): - echo list5[42] - -# -- Errata: var list: seq[int]; echo(list[0]). This now creates an exception, -# -- not a SIGSEGV. - -block: - var list = newSeq[string](3) - assert list[0].len == 0 - list[0] = "Foo" - list[1] = "Bar" - list[2] = "Baz" - - list.add("Lorem") - -block: - let list = @[4, 8, 15, 16, 23, 42] - for i in 0 ..< list.len: - stdout.write($list[i] & " ") - -var collection: set[int16] -doAssert collection == {} - -block: - let collection = {'a', 'x', 'r'} - doAssert 'a' in collection - -block: - let collection = {'a', 'T', 'z'} - let isAllLowerCase = {'A' .. 'Z'} * collection == {} - doAssert(not isAllLowerCase) - -let age = 10 -if age > 0 and age <= 10: - echo("You're still a child") -elif age > 10 and age < 18: - echo("You're a teenager") -else: - echo("You're an adult") - -let variable = "Arthur" -case variable -of "Arthur", "Zaphod", "Ford": - echo("Male") -of "Marvin": - echo("Robot") -of "Trillian": - echo("Female") -else: - echo("Unknown") - -let ageDesc = if age < 18: "Non-Adult" else: "Adult" - -block: - var i = 0 - while i < 3: - echo(i) - i.inc - -block label: - var i = 0 - while true: - while i < 5: - if i > 3: break label - i.inc - -iterator values(): int = - var i = 0 - while i < 5: - yield i - i.inc - -for value in values(): - echo(value) - -import os -for filename in walkFiles("*.nim"): - echo(filename) - -for item in @[1, 2, 3]: - echo(item) - -for i, value in @[1, 2, 3]: echo("Value at ", i, ": ", value) - -doAssertRaises(IOError): - proc second() = - raise newException(IOError, "Somebody set us up the bomb") - - proc first() = - second() - - first() - -block: - proc second() = - raise newException(IOError, "Somebody set us up the bomb") - - proc first() = - try: - second() - except: - echo("Cannot perform second action because: " & - getCurrentExceptionMsg()) - - first() - -block: - type - Person = object - name: string - age: int - - var person: Person - var person1 = Person(name: "Neo", age: 28) - -block: - type - PersonObj = object - name: string - age: int - PersonRef = ref PersonObj - - # proc setName(person: PersonObj) = - # person.name = "George" - - proc setName(person: PersonRef) = - person.name = "George" - -block: - type - Dog = object - name: string - - Cat = object - name: string - - let dog: Dog = Dog(name: "Fluffy") - let cat: Cat = Cat(name: "Fluffy") - -block: - type - Dog = tuple - name: string - - Cat = tuple - name: string - - let dog: Dog = (name: "Fluffy") - let cat: Cat = (name: "Fluffy") - - echo(dog == cat) - -block: - type - Point = tuple[x, y: int] - Point2 = (int, int) - - let pos: Point = (x: 100, y: 50) - doAssert pos == (100, 50) - - let pos1: Point = (x: 100, y: 50) - let (x, y) = pos1 #<1> - let (left, _) = pos1 - doAssert x == pos1[0] - doAssert y == pos1[1] - doAssert left == x - -block: - type - Color = enum - colRed, - colGreen, - colBlue - - let color: Color = colRed - -block: - type - Color {.pure.} = enum - red, green, blue - - let color = Color.red diff --git a/tests/niminaction/Chapter6/WikipediaStats/concurrency.nim b/tests/niminaction/Chapter6/WikipediaStats/concurrency.nim deleted file mode 100644 index 913cd77db00..00000000000 --- a/tests/niminaction/Chapter6/WikipediaStats/concurrency.nim +++ /dev/null @@ -1,83 +0,0 @@ -discard """ -action: compile -""" - -# See this page for info about the format https://wikitech.wikimedia.org/wiki/Analytics/Data/Pagecounts-all-sites -import tables, parseutils, strutils, threadpool - -const filename = "pagecounts-20160101-050000" - -type - Stats = ref object - projectName, pageTitle: string - requests, contentSize: int - -proc `$`(stats: Stats): string = - "(projectName: $#, pageTitle: $#, requests: $#, contentSize: $#)" % [ - stats.projectName, stats.pageTitle, $stats.requests, $stats.contentSize - ] - -proc parse(chunk: string): Stats = - # Each line looks like: en Main_Page 242332 4737756101 - result = Stats(projectName: "", pageTitle: "", requests: 0, contentSize: 0) - - var projectName = "" - var pageTitle = "" - var requests = "" - var contentSize = "" - for line in chunk.splitLines: - var i = 0 - projectName.setLen(0) - i.inc parseUntil(line, projectName, Whitespace, i) - i.inc skipWhitespace(line, i) - pageTitle.setLen(0) - i.inc parseUntil(line, pageTitle, Whitespace, i) - i.inc skipWhitespace(line, i) - requests.setLen(0) - i.inc parseUntil(line, requests, Whitespace, i) - i.inc skipWhitespace(line, i) - contentSize.setLen(0) - i.inc parseUntil(line, contentSize, Whitespace, i) - i.inc skipWhitespace(line, i) - - if requests.len == 0 or contentSize.len == 0: - # Ignore lines with either of the params that are empty. - continue - - let requestsInt = requests.parseInt - if requestsInt > result.requests and projectName == "en": - result = Stats( - projectName: projectName, - pageTitle: pageTitle, - requests: requestsInt, - contentSize: contentSize.parseInt - ) - -proc readChunks(filename: string, chunksize = 1000000): Stats = - result = Stats(projectName: "", pageTitle: "", requests: 0, contentSize: 0) - var file = open(filename) - var responses = newSeq[FlowVar[Stats]]() - var buffer = newString(chunksize) - var oldBufferLen = 0 - while not endOfFile(file): - let readSize = file.readChars(buffer, oldBufferLen, chunksize - oldBufferLen) + oldBufferLen - var chunkLen = readSize - - while chunkLen >= 0 and buffer[chunkLen - 1] notin NewLines: - # Find where the last line ends - chunkLen.dec - - responses.add(spawn parse(buffer[0 ..< chunkLen])) - oldBufferLen = readSize - chunkLen - buffer[0 ..< oldBufferLen] = buffer[readSize - oldBufferLen .. ^1] - - for resp in responses: - let statistic = ^resp - if statistic.requests > result.requests: - result = statistic - - file.close() - - -when true: - echo readChunks(filename) diff --git a/tests/niminaction/Chapter6/WikipediaStats/concurrency.nim.cfg b/tests/niminaction/Chapter6/WikipediaStats/concurrency.nim.cfg deleted file mode 100644 index aed303eef86..00000000000 --- a/tests/niminaction/Chapter6/WikipediaStats/concurrency.nim.cfg +++ /dev/null @@ -1 +0,0 @@ ---threads:on diff --git a/tests/niminaction/Chapter6/WikipediaStats/concurrency_regex.nim b/tests/niminaction/Chapter6/WikipediaStats/concurrency_regex.nim deleted file mode 100644 index 102313de9d8..00000000000 --- a/tests/niminaction/Chapter6/WikipediaStats/concurrency_regex.nim +++ /dev/null @@ -1,68 +0,0 @@ -discard """ -action: compile -""" - -# See this page for info about the format https://wikitech.wikimedia.org/wiki/Analytics/Data/Pagecounts-all-sites -import tables, parseutils, strutils, threadpool, re - -const filename = "pagecounts-20160101-050000" - -type - Stats = ref object - projectName, pageTitle: string - requests, contentSize: int - -proc `$`(stats: Stats): string = - "(projectName: $#, pageTitle: $#, requests: $#, contentSize: $#)" % [ - stats.projectName, stats.pageTitle, $stats.requests, $stats.contentSize - ] - -proc parse(chunk: string): Stats = - # Each line looks like: en Main_Page 242332 4737756101 - result = Stats(projectName: "", pageTitle: "", requests: 0, contentSize: 0) - - var matches: array[4, string] - var reg = re"([^\s]+)\s([^\s]+)\s(\d+)\s(\d+)" - for line in chunk.splitLines: - - let start = find(line, reg, matches) - if start == -1: continue - - let requestsInt = matches[2].parseInt - if requestsInt > result.requests and matches[0] == "en": - result = Stats( - projectName: matches[0], - pageTitle: matches[1], - requests: requestsInt, - contentSize: matches[3].parseInt - ) - -proc readChunks(filename: string, chunksize = 1000000): Stats = - result = Stats(projectName: "", pageTitle: "", requests: 0, contentSize: 0) - var file = open(filename) - var responses = newSeq[FlowVar[Stats]]() - var buffer = newString(chunksize) - var oldBufferLen = 0 - while not endOfFile(file): - let readSize = file.readChars(buffer, oldBufferLen, chunksize - oldBufferLen) + oldBufferLen - var chunkLen = readSize - - while chunkLen >= 0 and buffer[chunkLen - 1] notin NewLines: - # Find where the last line ends - chunkLen.dec - - responses.add(spawn parse(buffer[0 ..< chunkLen])) - oldBufferLen = readSize - chunkLen - buffer[0 ..< oldBufferLen] = buffer[readSize - oldBufferLen .. ^1] - - echo("Spawns: ", responses.len) - for resp in responses: - let statistic = ^resp - if statistic.requests > result.requests: - result = statistic - - file.close() - - -when true: - echo readChunks(filename) diff --git a/tests/niminaction/Chapter6/WikipediaStats/concurrency_regex.nim.cfg b/tests/niminaction/Chapter6/WikipediaStats/concurrency_regex.nim.cfg deleted file mode 100644 index aed303eef86..00000000000 --- a/tests/niminaction/Chapter6/WikipediaStats/concurrency_regex.nim.cfg +++ /dev/null @@ -1 +0,0 @@ ---threads:on diff --git a/tests/niminaction/Chapter6/WikipediaStats/naive.nim b/tests/niminaction/Chapter6/WikipediaStats/naive.nim deleted file mode 100644 index 687177f7419..00000000000 --- a/tests/niminaction/Chapter6/WikipediaStats/naive.nim +++ /dev/null @@ -1,33 +0,0 @@ -discard """ -action: compile -""" - -# See this page for info about the format https://wikitech.wikimedia.org/wiki/Analytics/Data/Pagecounts-all-sites -import tables, parseutils, strutils - -const filename = "pagecounts-20150101-050000" - -proc parse(filename: string): tuple[projectName, pageTitle: string, - requests, contentSize: int] = - # Each line looks like: en Main_Page 242332 4737756101 - var file = open(filename) - for line in file.lines: - var i = 0 - var projectName = "" - i.inc parseUntil(line, projectName, Whitespace, i) - i.inc - var pageTitle = "" - i.inc parseUntil(line, pageTitle, Whitespace, i) - i.inc - var requests = 0 - i.inc parseInt(line, requests, i) - i.inc - var contentSize = 0 - i.inc parseInt(line, contentSize, i) - if requests > result[2] and projectName == "en": - result = (projectName, pageTitle, requests, contentSize) - - file.close() - -when true: - echo parse(filename) diff --git a/tests/niminaction/Chapter6/WikipediaStats/parallel_counts.nim b/tests/niminaction/Chapter6/WikipediaStats/parallel_counts.nim deleted file mode 100644 index 379ec73649a..00000000000 --- a/tests/niminaction/Chapter6/WikipediaStats/parallel_counts.nim +++ /dev/null @@ -1,76 +0,0 @@ -discard """ -action: compile -""" - -import os, parseutils, threadpool, strutils - -type - Stats = ref object - domainCode, pageTitle: string - countViews, totalSize: int - -proc newStats(): Stats = - Stats(domainCode: "", pageTitle: "", countViews: 0, totalSize: 0) - -proc `$`(stats: Stats): string = - "(domainCode: $#, pageTitle: $#, countViews: $#, totalSize: $#)" % [ - stats.domainCode, stats.pageTitle, $stats.countViews, $stats.totalSize - ] - -proc parse(line: string, domainCode, pageTitle: var string, - countViews, totalSize: var int) = - if line.len == 0: return - var i = 0 - domainCode.setLen(0) - i.inc parseUntil(line, domainCode, {' '}, i) - i.inc - pageTitle.setLen(0) - i.inc parseUntil(line, pageTitle, {' '}, i) - i.inc - countViews = 0 - i.inc parseInt(line, countViews, i) - i.inc - totalSize = 0 - i.inc parseInt(line, totalSize, i) - -proc parseChunk(chunk: string): Stats = - result = newStats() - var domainCode = "" - var pageTitle = "" - var countViews = 0 - var totalSize = 0 - for line in splitLines(chunk): - parse(line, domainCode, pageTitle, countViews, totalSize) - if domainCode == "en" and countViews > result.countViews: - result = Stats(domainCode: domainCode, pageTitle: pageTitle, - countViews: countViews, totalSize: totalSize) - -proc readPageCounts(filename: string, chunkSize = 1_000_000) = - var file = open(filename) - var responses = newSeq[FlowVar[Stats]]() - var buffer = newString(chunksize) - var oldBufferLen = 0 - while not endOfFile(file): - let reqSize = chunksize - oldBufferLen - let readSize = file.readChars(buffer, oldBufferLen, reqSize) + oldBufferLen - var chunkLen = readSize - - while chunkLen >= 0 and buffer[chunkLen - 1] notin NewLines: - chunkLen.dec - - responses.add(spawn parseChunk(buffer[0 ..< chunkLen])) - oldBufferLen = readSize - chunkLen - buffer[0 ..< oldBufferLen] = buffer[readSize - oldBufferLen .. ^1] - - var mostPopular = newStats() - for resp in responses: - let statistic = ^resp - if statistic.countViews > mostPopular.countViews: - mostPopular = statistic - - echo("Most popular is: ", mostPopular) - -when true: - const file = "pagecounts-20160101-050000" - let filename = getCurrentDir() / file - readPageCounts(filename) diff --git a/tests/niminaction/Chapter6/WikipediaStats/parallel_counts.nim.cfg b/tests/niminaction/Chapter6/WikipediaStats/parallel_counts.nim.cfg deleted file mode 100644 index 9d57ecf93d4..00000000000 --- a/tests/niminaction/Chapter6/WikipediaStats/parallel_counts.nim.cfg +++ /dev/null @@ -1 +0,0 @@ ---threads:on \ No newline at end of file diff --git a/tests/niminaction/Chapter6/WikipediaStats/race_condition.nim b/tests/niminaction/Chapter6/WikipediaStats/race_condition.nim deleted file mode 100644 index f4b07220436..00000000000 --- a/tests/niminaction/Chapter6/WikipediaStats/race_condition.nim +++ /dev/null @@ -1,17 +0,0 @@ -discard """ -action: compile -""" - -import threadpool - -var counter = 0 - -proc increment(x: int) = - for i in 0 ..< x: - let value = counter + 1 - counter = value - -spawn increment(10_000) -spawn increment(10_000) -sync() -echo(counter) diff --git a/tests/niminaction/Chapter6/WikipediaStats/race_condition.nim.cfg b/tests/niminaction/Chapter6/WikipediaStats/race_condition.nim.cfg deleted file mode 100644 index 9d57ecf93d4..00000000000 --- a/tests/niminaction/Chapter6/WikipediaStats/race_condition.nim.cfg +++ /dev/null @@ -1 +0,0 @@ ---threads:on \ No newline at end of file diff --git a/tests/niminaction/Chapter6/WikipediaStats/sequential_counts.nim b/tests/niminaction/Chapter6/WikipediaStats/sequential_counts.nim deleted file mode 100644 index f4bae3df541..00000000000 --- a/tests/niminaction/Chapter6/WikipediaStats/sequential_counts.nim +++ /dev/null @@ -1,38 +0,0 @@ -discard """ -action: compile -""" - -import os, parseutils - -proc parse(line: string, domainCode, pageTitle: var string, - countViews, totalSize: var int) = - var i = 0 - domainCode.setLen(0) - i.inc parseUntil(line, domainCode, {' '}, i) - i.inc - pageTitle.setLen(0) - i.inc parseUntil(line, pageTitle, {' '}, i) - i.inc - countViews = 0 - i.inc parseInt(line, countViews, i) - i.inc - totalSize = 0 - i.inc parseInt(line, totalSize, i) - -proc readPageCounts(filename: string) = - var domainCode = "" - var pageTitle = "" - var countViews = 0 - var totalSize = 0 - var mostPopular = ("", "", 0, 0) - for line in filename.lines: - parse(line, domainCode, pageTitle, countViews, totalSize) - if domainCode == "en" and countViews > mostPopular[2]: - mostPopular = (domainCode, pageTitle, countViews, totalSize) - - echo("Most popular is: ", mostPopular) - -when true: - const file = "pagecounts-20160101-050000" - let filename = getCurrentDir() / file - readPageCounts(filename) diff --git a/tests/niminaction/Chapter6/WikipediaStats/unguarded_access.nim b/tests/niminaction/Chapter6/WikipediaStats/unguarded_access.nim deleted file mode 100644 index 7bdde83970e..00000000000 --- a/tests/niminaction/Chapter6/WikipediaStats/unguarded_access.nim +++ /dev/null @@ -1,20 +0,0 @@ -discard """ - errormsg: "unguarded access: counter" - line: 14 -""" - -import threadpool, locks - -var counterLock: Lock -initLock(counterLock) -var counter {.guard: counterLock.} = 0 - -proc increment(x: int) = - for i in 0 ..< x: - let value = counter + 1 - counter = value - -spawn increment(10_000) -spawn increment(10_000) -sync() -echo(counter) diff --git a/tests/niminaction/Chapter6/WikipediaStats/unguarded_access.nim.cfg b/tests/niminaction/Chapter6/WikipediaStats/unguarded_access.nim.cfg deleted file mode 100644 index 9d57ecf93d4..00000000000 --- a/tests/niminaction/Chapter6/WikipediaStats/unguarded_access.nim.cfg +++ /dev/null @@ -1 +0,0 @@ ---threads:on \ No newline at end of file diff --git a/tests/niminaction/Chapter9/configurator/configurator.nim b/tests/niminaction/Chapter9/configurator/configurator.nim deleted file mode 100644 index 0d562788968..00000000000 --- a/tests/niminaction/Chapter9/configurator/configurator.nim +++ /dev/null @@ -1,84 +0,0 @@ -import macros - -proc createRefType(ident: NimIdent, identDefs: seq[NimNode]): NimNode = - result = newTree(nnkTypeSection, - newTree(nnkTypeDef, - newIdentNode(ident), - newEmptyNode(), - newTree(nnkRefTy, - newTree(nnkObjectTy, - newEmptyNode(), - newEmptyNode(), - newTree(nnkRecList, - identDefs - ) - ) - ) - ) - ) - -proc toIdentDefs(stmtList: NimNode): seq[NimNode] = - expectKind(stmtList, nnkStmtList) - result = @[] - - for child in stmtList: - expectKind(child, nnkCall) - result.add(newIdentDefs(child[0], child[1][0])) - -template constructor(ident: untyped): untyped = - proc `new ident`(): `ident` = - new result - -proc createLoadProc(typeName: NimIdent, identDefs: seq[NimNode]): NimNode = - var cfgIdent = newIdentNode("cfg") - var filenameIdent = newIdentNode("filename") - var objIdent = newIdentNode("obj") - - var body = newStmtList() - body.add quote do: - var `objIdent` = parseFile(`filenameIdent`) - - for identDef in identDefs: - let fieldNameIdent = identDef[0] - let fieldName = $fieldNameIdent.ident - case $identDef[1].ident - of "string": - body.add quote do: - `cfgIdent`.`fieldNameIdent` = `objIdent`[`fieldName`].getStr - of "int": - body.add quote do: - `cfgIdent`.`fieldNameIdent` = `objIdent`[`fieldName`].getNum().int - else: - doAssert(false, "Not Implemented") - - return newProc(newIdentNode("load"), - [newEmptyNode(), - newIdentDefs(cfgIdent, newIdentNode(typeName)), - newIdentDefs(filenameIdent, newIdentNode("string"))], - body) - -macro config*(typeName: untyped, fields: untyped): untyped = - result = newStmtList() - - let identDefs = toIdentDefs(fields) - result.add createRefType(typeName.ident, identDefs) - result.add getAst(constructor(typeName.ident)) - result.add createLoadProc(typeName.ident, identDefs) - - echo treeRepr(typeName) - echo treeRepr(fields) - - echo treeRepr(result) - echo toStrLit(result) - # TODO: Verify that we can export fields in config type so that it can be - # used in another module. - -import json -config MyAppConfig: - address: string - port: int - -var myConf = newMyAppConfig() -myConf.load("myappconfig.cfg") -echo("Address: ", myConf.address) -echo("Port: ", myConf.port) diff --git a/tests/system/helpers/readall_echo.nim b/tests/system/helpers/readall_echo.nim deleted file mode 100644 index 2891ef3aeaa..00000000000 --- a/tests/system/helpers/readall_echo.nim +++ /dev/null @@ -1,2 +0,0 @@ -when true: - echo(stdin.readAll) diff --git a/tools/koch/koch.nim b/tools/koch/koch.nim index 9f8257976c2..71ef540f273 100644 --- a/tools/koch/koch.nim +++ b/tools/koch/koch.nim @@ -452,7 +452,7 @@ proc winRelease*() = template `|`(a, b): string = (if a.len > 0: a else: b) proc tests(args: string) = - nimexec "cc --opt:speed testament/testament" + nimexec "--lib:lib cc --opt:speed testament/testament" var testCmd = quoteShell(getCurrentDir() / "testament/testament".exe) testCmd.add " " & quoteShell("--nim:" & findNim()) testCmd.add " " & (args|"all") diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim index a589cfb14ee..2c7515c9335 100644 --- a/tools/nimgrep.nim +++ b/tools/nimgrep.nim @@ -7,6 +7,15 @@ # distribution, for details about the copyright. # +# include a testament spec, because we hack in testing that way +discard """ +action: "compile" +targets: "c" +matrix: "--debugger:on" +""" +# Note: force target to C because of MacOS 10.15 SDK headers bug +# https://github.com/nim-lang/Nim/pull/15612#issuecomment-712471879 + import os, strutils, parseopt, pegs, re, terminal, osproc, tables, algorithm, times