diff --git a/package-lock.json b/package-lock.json index d06cfeb235..2105466fe2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4381,25 +4381,6 @@ "git-up": "^7.0.0" } }, - "node_modules/glob": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", - "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -6316,6 +6297,48 @@ "balanced-match": "^1.0.0" } }, + "node_modules/mocha/node_modules/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/mocha/node_modules/glob/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/mocha/node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/mocha/node_modules/minimatch": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", @@ -6652,6 +6675,26 @@ "node": ">=8" } }, + "node_modules/nyc/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/nyc/node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -7714,6 +7757,26 @@ "npm-normalize-package-bin": "^1.0.0" } }, + "node_modules/read-package-json/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/readable-stream": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.4.2.tgz", @@ -8059,6 +8122,25 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/rimraf/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/rotating-file-stream": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/rotating-file-stream/-/rotating-file-stream-3.1.1.tgz", @@ -8369,6 +8451,26 @@ "node": ">=4" } }, + "node_modules/shelljs/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/shiki": { "version": "0.14.3", "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.14.3.tgz", @@ -8805,6 +8907,26 @@ "node": ">=8" } }, + "node_modules/test-exclude/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", diff --git a/src/util/log.ts b/src/util/log.ts index 54b865c8ec..cd73910001 100644 --- a/src/util/log.ts +++ b/src/util/log.ts @@ -3,7 +3,6 @@ import { createStream, type Options } from 'rotating-file-stream' export class FlowrLogger extends Logger { /** by keeping track of all children we can propagate updates of the settings (e.g., in tests) */ - private readonly childLoggers: Logger[] = [] public getSubLogger( diff --git a/test/functionality/helper/ast-builder.ts b/test/functionality/_helper/ast-builder.ts similarity index 100% rename from test/functionality/helper/ast-builder.ts rename to test/functionality/_helper/ast-builder.ts diff --git a/test/functionality/_helper/collect-tests.ts b/test/functionality/_helper/collect-tests.ts new file mode 100644 index 0000000000..b1dd65fb1d --- /dev/null +++ b/test/functionality/_helper/collect-tests.ts @@ -0,0 +1,18 @@ +import fs from 'node:fs' +import path from 'path' + +/** + * Require all files in a folder that end with a suffix, will never recurse into subfolders + * @param folder - the folder to require all files from + * @param suffix - the suffix which the files of interest must have + */ +export function requireAllTestsInFolder(folder: string, suffix = '-tests.ts'): void { + for(const fileBuff of fs.readdirSync(folder, { recursive: false })) { + const file = fileBuff.toString() + if(file.endsWith(suffix)) { + require(path.join(folder,file)) + } else if(!file.endsWith('.spec.ts')) { + throw new Error(`Unexpected file ${file} in ${folder}, neither matches the import suffix '${suffix}', nor the test suffix '.spec.ts'. This is a sanity check so no files are lost. Please restructure your folders if this is intended.`) + } + } +} diff --git a/test/functionality/_helper/environment-builder.ts b/test/functionality/_helper/environment-builder.ts new file mode 100644 index 0000000000..bf1c1487c5 --- /dev/null +++ b/test/functionality/_helper/environment-builder.ts @@ -0,0 +1,7 @@ +import { NodeId } from '../../../src/r-bridge' +import { IdentifierDefinition } from '../../../src/dataflow' +import { LocalScope } from '../../../src/dataflow/environments/scopes' + +export function variable(name: string, definedAt: NodeId): IdentifierDefinition { + return { name, kind: 'variable', scope: LocalScope, used: 'always', nodeId: '_0', definedAt } +} diff --git a/test/functionality/helper/net.ts b/test/functionality/_helper/net.ts similarity index 100% rename from test/functionality/helper/net.ts rename to test/functionality/_helper/net.ts diff --git a/test/functionality/helper/network.ts b/test/functionality/_helper/network.ts similarity index 100% rename from test/functionality/helper/network.ts rename to test/functionality/_helper/network.ts diff --git a/test/functionality/helper/provider.ts b/test/functionality/_helper/provider.ts similarity index 100% rename from test/functionality/helper/provider.ts rename to test/functionality/_helper/provider.ts diff --git a/test/functionality/helper/shell.ts b/test/functionality/_helper/shell.ts similarity index 100% rename from test/functionality/helper/shell.ts rename to test/functionality/_helper/shell.ts diff --git a/test/functionality/helper/version.ts b/test/functionality/_helper/version.ts similarity index 100% rename from test/functionality/helper/version.ts rename to test/functionality/_helper/version.ts diff --git a/test/functionality/benchmark/slicer.spec.ts b/test/functionality/benchmark-slicer/slicer.spec.ts similarity index 99% rename from test/functionality/benchmark/slicer.spec.ts rename to test/functionality/benchmark-slicer/slicer.spec.ts index ab6cb22cba..efd613b0e9 100644 --- a/test/functionality/benchmark/slicer.spec.ts +++ b/test/functionality/benchmark-slicer/slicer.spec.ts @@ -18,7 +18,7 @@ async function retrieveStatsSafe(slicer: BenchmarkSlicer, request: { request: st return { stats, statInfo } } -describe('The Benchmark Slicer', () => { +describe('Benchmark Slicer', () => { describe('Stats by parsing text-based inputs', function() { this.timeout('15min') it('Simple slice for simple line', async() => { diff --git a/test/functionality/dataflow/dataflow.spec.ts b/test/functionality/dataflow/dataflow.spec.ts index a497772139..2225b867d6 100644 --- a/test/functionality/dataflow/dataflow.spec.ts +++ b/test/functionality/dataflow/dataflow.spec.ts @@ -1,16 +1,15 @@ +import { requireAllTestsInFolder } from '../_helper/collect-tests' +import path from 'path' + describe('Dataflow', () => { - require('./environments/environments') + describe('Environments', () => + requireAllTestsInFolder(path.join(__dirname, 'environments')) + ) + + describe('Graph', () => + requireAllTestsInFolder(path.join(__dirname, 'graph')) + ) - describe('Graph', () => { - require('./graph/equal') - }) - describe('Extraction', () => { - require('./elements/atomic') - require('./elements/expression-lists') - describe('Functions', () => { - require('./elements/functions/function-definition') - require('./elements/functions/function-call') - }) - }) + require('./processing-of-elements/processing-of-elements') }) diff --git a/test/functionality/dataflow/elements/expression-lists.ts b/test/functionality/dataflow/elements/expression-lists.ts deleted file mode 100644 index 4b372ecb00..0000000000 --- a/test/functionality/dataflow/elements/expression-lists.ts +++ /dev/null @@ -1,9 +0,0 @@ - -describe('Working with expression lists', () => { - require('./lists/uninteresting') - - require('./lists/read') - - require('./lists/if-then') - require('./loops/loops') -}) diff --git a/test/functionality/dataflow/elements/loops/loops.ts b/test/functionality/dataflow/elements/loops/loops.ts deleted file mode 100644 index 93ff119292..0000000000 --- a/test/functionality/dataflow/elements/loops/loops.ts +++ /dev/null @@ -1,3 +0,0 @@ -describe('Loops', () => { - require('./for-loop') -}) diff --git a/test/functionality/dataflow/environments/environments.ts b/test/functionality/dataflow/environments/environments.ts deleted file mode 100644 index 05e6538cf3..0000000000 --- a/test/functionality/dataflow/environments/environments.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { NodeId } from '../../../../src/r-bridge' -import { IdentifierDefinition } from '../../../../src/dataflow' -import { LocalScope } from '../../../../src/dataflow/environments/scopes' - -export function variable(name: string, definedAt: NodeId): IdentifierDefinition { - return { name, kind: 'variable', scope: LocalScope, used: 'always', nodeId: '_0', definedAt } -} - -describe('Environments', () => { - require('./initialization') - - require('./modification') - require('./resolve') -}) diff --git a/test/functionality/dataflow/environments/initialization.ts b/test/functionality/dataflow/environments/initialization-tests.ts similarity index 100% rename from test/functionality/dataflow/environments/initialization.ts rename to test/functionality/dataflow/environments/initialization-tests.ts diff --git a/test/functionality/dataflow/environments/modification.ts b/test/functionality/dataflow/environments/modification-tests.ts similarity index 98% rename from test/functionality/dataflow/environments/modification.ts rename to test/functionality/dataflow/environments/modification-tests.ts index 84cd0ab78a..9bcc2faa17 100644 --- a/test/functionality/dataflow/environments/modification.ts +++ b/test/functionality/dataflow/environments/modification-tests.ts @@ -3,8 +3,8 @@ import { DefaultEnvironmentMemory, IEnvironment, initializeCleanEnvironments } f import { guard } from '../../../../src/util/assert' import { expect } from 'chai' import { appendEnvironments, define, overwriteEnvironments } from '../../../../src/dataflow/environments' -import { variable } from './environments' import { GlobalScope, LocalScope } from '../../../../src/dataflow/environments/scopes' +import { variable } from '../../_helper/environment-builder' /** if you pass multiple `definedAt`, this will expect the node to have multiple definitions */ function existsDefinedAt(name: string, definedAt: NodeId[], result: IEnvironment | undefined, message?: string) { @@ -17,7 +17,7 @@ function existsDefinedAt(name: string, definedAt: NodeId[], result: IEnvironment expect(got.map(d => d.definedAt), `${name} should be defined at ${JSON.stringify(definedAt)}. ${message ?? ''}`).to.deep.equal(definedAt) } -describe('Overwrite', () => { +describe('Modification', () => { describe('Global', () => { it('Different variables', () => { let clean = initializeCleanEnvironments() diff --git a/test/functionality/dataflow/environments/resolve.ts b/test/functionality/dataflow/environments/resolve-tests.ts similarity index 95% rename from test/functionality/dataflow/environments/resolve.ts rename to test/functionality/dataflow/environments/resolve-tests.ts index 6be46df217..c2fbb8a2c1 100644 --- a/test/functionality/dataflow/environments/resolve.ts +++ b/test/functionality/dataflow/environments/resolve-tests.ts @@ -1,8 +1,8 @@ import { define, initializeCleanEnvironments, resolveByName } from '../../../../src/dataflow/environments' -import { variable } from './environments' import { expect } from 'chai' import { guard } from '../../../../src/util/assert' import { GlobalScope, LocalScope } from '../../../../src/dataflow/environments/scopes' +import { variable } from '../../_helper/environment-builder' describe('Resolve', () => { describe('ByName', () => { diff --git a/test/functionality/dataflow/graph/equal.ts b/test/functionality/dataflow/graph/equal-tests.ts similarity index 98% rename from test/functionality/dataflow/graph/equal.ts rename to test/functionality/dataflow/graph/equal-tests.ts index 878740a185..a45c9bb6bd 100644 --- a/test/functionality/dataflow/graph/equal.ts +++ b/test/functionality/dataflow/graph/equal-tests.ts @@ -12,7 +12,7 @@ function test(cmp: (x: boolean) => void, a: DataflowGraph, b: DataflowGraph, tex } } -describe('Graph Equality', () => { +describe('Equal', () => { const raw = (name: string, a: DataflowGraph, b: DataflowGraph, text: string, cmp: (x: boolean) => void) => { return it(name, () => { // as the comparison is relatively quick, we allow explicit checks for commutativity diff --git a/test/functionality/dataflow/elements/atomic.ts b/test/functionality/dataflow/processing-of-elements/atomic/atomic-tests.ts similarity index 97% rename from test/functionality/dataflow/elements/atomic.ts rename to test/functionality/dataflow/processing-of-elements/atomic/atomic-tests.ts index 2c7e2a04d7..64ba7a2689 100644 --- a/test/functionality/dataflow/elements/atomic.ts +++ b/test/functionality/dataflow/processing-of-elements/atomic/atomic-tests.ts @@ -3,15 +3,15 @@ * Yet, some constructs (like for-loops) require the combination of statements, they are included as well. * This will not include functions! */ -import { assertDataflow, withShell } from '../../helper/shell' -import { DataflowGraph, EdgeType, initializeCleanEnvironments } from '../../../../src/dataflow' -import { RAssignmentOpPool, RNonAssignmentBinaryOpPool, RUnaryOpPool } from '../../helper/provider' -import { appendEnvironments, define } from '../../../../src/dataflow/environments' -import { UnnamedArgumentPrefix } from '../../../../src/dataflow/internal/process/functions/argument' -import { GlobalScope, LocalScope } from '../../../../src/dataflow/environments/scopes' -import { MIN_VERSION_PIPE } from '../../../../src/r-bridge/lang-4.x/ast/model/versions' +import { assertDataflow, withShell } from '../../../_helper/shell' +import { DataflowGraph, EdgeType, initializeCleanEnvironments } from '../../../../../src/dataflow' +import { RAssignmentOpPool, RNonAssignmentBinaryOpPool, RUnaryOpPool } from '../../../_helper/provider' +import { appendEnvironments, define } from '../../../../../src/dataflow/environments' +import { UnnamedArgumentPrefix } from '../../../../../src/dataflow/internal/process/functions/argument' +import { GlobalScope, LocalScope } from '../../../../../src/dataflow/environments/scopes' +import { MIN_VERSION_PIPE } from '../../../../../src/r-bridge/lang-4.x/ast/model/versions' -describe('Atomic dataflow information', withShell((shell) => { +describe('Atomic (dataflow information)', withShell((shell) => { describe('uninteresting leafs', () => { for(const input of ['42', '"test"', 'TRUE', 'NA', 'NULL']) { assertDataflow(input, shell, input, new DataflowGraph()) diff --git a/test/functionality/dataflow/elements/lists/if-then.ts b/test/functionality/dataflow/processing-of-elements/expression-lists/if-then-tests.ts similarity index 98% rename from test/functionality/dataflow/elements/lists/if-then.ts rename to test/functionality/dataflow/processing-of-elements/expression-lists/if-then-tests.ts index 30ac63a936..5830b9e125 100644 --- a/test/functionality/dataflow/elements/lists/if-then.ts +++ b/test/functionality/dataflow/processing-of-elements/expression-lists/if-then-tests.ts @@ -2,7 +2,7 @@ import { DataflowGraph, EdgeType, initializeCleanEnvironments } from '../../../../../src/dataflow' -import { assertDataflow, withShell } from '../../../helper/shell' +import { assertDataflow, withShell } from '../../../_helper/shell' import { appendEnvironments, define } from '../../../../../src/dataflow/environments' import { GlobalScope, LocalScope } from '../../../../../src/dataflow/environments/scopes' diff --git a/test/functionality/dataflow/elements/lists/read.ts b/test/functionality/dataflow/processing-of-elements/expression-lists/read-tests.ts similarity index 99% rename from test/functionality/dataflow/elements/lists/read.ts rename to test/functionality/dataflow/processing-of-elements/expression-lists/read-tests.ts index 2e7e21908e..d41c70ba04 100644 --- a/test/functionality/dataflow/elements/lists/read.ts +++ b/test/functionality/dataflow/processing-of-elements/expression-lists/read-tests.ts @@ -1,6 +1,6 @@ import { NodeId } from '../../../../../src/r-bridge' import { DataflowGraph, EdgeType, initializeCleanEnvironments } from '../../../../../src/dataflow' -import { assertDataflow, withShell } from '../../../helper/shell' +import { assertDataflow, withShell } from '../../../_helper/shell' import { define } from '../../../../../src/dataflow/environments' import { LocalScope } from '../../../../../src/dataflow/environments/scopes' diff --git a/test/functionality/dataflow/elements/lists/uninteresting.ts b/test/functionality/dataflow/processing-of-elements/expression-lists/uninteresting-tests.ts similarity index 80% rename from test/functionality/dataflow/elements/lists/uninteresting.ts rename to test/functionality/dataflow/processing-of-elements/expression-lists/uninteresting-tests.ts index c673fff8ae..8ab42e0397 100644 --- a/test/functionality/dataflow/elements/lists/uninteresting.ts +++ b/test/functionality/dataflow/processing-of-elements/expression-lists/uninteresting-tests.ts @@ -1,4 +1,4 @@ -import { assertDataflow, withShell } from '../../../helper/shell' +import { assertDataflow, withShell } from '../../../_helper/shell' import { DataflowGraph } from '../../../../../src/dataflow' describe('Lists without variable references ', withShell(shell => { diff --git a/test/functionality/dataflow/elements/functions/function-call.ts b/test/functionality/dataflow/processing-of-elements/functions/function-call-tests.ts similarity index 99% rename from test/functionality/dataflow/elements/functions/function-call.ts rename to test/functionality/dataflow/processing-of-elements/functions/function-call-tests.ts index 082c8f565c..325666febf 100644 --- a/test/functionality/dataflow/elements/functions/function-call.ts +++ b/test/functionality/dataflow/processing-of-elements/functions/function-call-tests.ts @@ -1,4 +1,4 @@ -import { assertDataflow, withShell } from '../../../helper/shell' +import { assertDataflow, withShell } from '../../../_helper/shell' import { DataflowGraph, EdgeType, initializeCleanEnvironments } from '../../../../../src/dataflow' import { define, popLocalEnvironment, pushLocalEnvironment } from '../../../../../src/dataflow/environments' import { UnnamedArgumentPrefix } from '../../../../../src/dataflow/internal/process/functions/argument' diff --git a/test/functionality/dataflow/elements/functions/function-definition.ts b/test/functionality/dataflow/processing-of-elements/functions/function-definition-tests.ts similarity index 99% rename from test/functionality/dataflow/elements/functions/function-definition.ts rename to test/functionality/dataflow/processing-of-elements/functions/function-definition-tests.ts index 529139e121..47bb0758a0 100644 --- a/test/functionality/dataflow/elements/functions/function-definition.ts +++ b/test/functionality/dataflow/processing-of-elements/functions/function-definition-tests.ts @@ -1,4 +1,4 @@ -import { assertDataflow, withShell } from '../../../helper/shell' +import { assertDataflow, withShell } from '../../../_helper/shell' import { BuiltIn, DataflowGraph, EdgeType, initializeCleanEnvironments } from '../../../../../src/dataflow' import { define, diff --git a/test/functionality/dataflow/elements/loops/for-loop.ts b/test/functionality/dataflow/processing-of-elements/loops/for-loop-tests.ts similarity index 99% rename from test/functionality/dataflow/elements/loops/for-loop.ts rename to test/functionality/dataflow/processing-of-elements/loops/for-loop-tests.ts index 346aaddc4d..e30faf14c1 100644 --- a/test/functionality/dataflow/elements/loops/for-loop.ts +++ b/test/functionality/dataflow/processing-of-elements/loops/for-loop-tests.ts @@ -1,4 +1,4 @@ -import { assertDataflow, withShell } from '../../../helper/shell' +import { assertDataflow, withShell } from '../../../_helper/shell' import { DataflowGraph, EdgeType, initializeCleanEnvironments } from '../../../../../src/dataflow' import { appendEnvironments, define } from '../../../../../src/dataflow/environments' import { LocalScope } from '../../../../../src/dataflow/environments/scopes' diff --git a/test/functionality/dataflow/processing-of-elements/processing-of-elements.ts b/test/functionality/dataflow/processing-of-elements/processing-of-elements.ts new file mode 100644 index 0000000000..a010056ef5 --- /dev/null +++ b/test/functionality/dataflow/processing-of-elements/processing-of-elements.ts @@ -0,0 +1,20 @@ +import { requireAllTestsInFolder } from '../../_helper/collect-tests' +import path from 'path' + +describe('Processing of Elements', () => { + describe('atomic', () => + requireAllTestsInFolder(path.join(__dirname, 'atomic')) + ) + + describe('loops', () => + requireAllTestsInFolder(path.join(__dirname, 'loops')) + ) + + describe('expression-lists', () => + requireAllTestsInFolder(path.join(__dirname, 'expression-lists')) + ) + + describe('Functions', () => + requireAllTestsInFolder(path.join(__dirname, 'functions')) + ) +}) diff --git a/test/functionality/flowr/server.spec.ts b/test/functionality/flowr/server.spec.ts index ae293c86e6..45233c40a3 100644 --- a/test/functionality/flowr/server.spec.ts +++ b/test/functionality/flowr/server.spec.ts @@ -1,5 +1,5 @@ -import { withShell } from '../helper/shell' -import { fakeSend, withSocket } from '../helper/net' +import { withShell } from '../_helper/shell' +import { fakeSend, withSocket } from '../_helper/net' import { retrieveVersionInformation } from '../../../src/cli/repl/commands/version' import { FlowrHelloResponseMessage } from '../../../src/cli/repl/server/messages/hello' import { assert } from 'chai' @@ -14,103 +14,105 @@ import { } from '../../../src/cli/repl/server/messages/repl' import { extractCFG } from '../../../src/util/cfg' -describe('FlowR Server', withShell(shell => { - it('Correct Hello Message', withSocket(shell,async socket => { - const messages = socket.getMessages() - assert.strictEqual(messages.length, 1, 'Expected exactly one message to hello the client') - - const hello = messages[0] as FlowrHelloResponseMessage - const knownVersion = await retrieveVersionInformation(shell) - - assert.deepStrictEqual(hello, { - type: 'hello', - clientName: 'client-0', - versions: { - r: knownVersion.r, - flowr: knownVersion.flowr - } - }, 'Expected hello message to have the predefined format') +describe('flowr', () => { + describe('Server', withShell(shell => { + it('Correct Hello Message', withSocket(shell,async socket => { + const messages = socket.getMessages() + assert.strictEqual(messages.length, 1, 'Expected exactly one message to hello the client') + + const hello = messages[0] as FlowrHelloResponseMessage + const knownVersion = await retrieveVersionInformation(shell) + + assert.deepStrictEqual(hello, { + type: 'hello', + clientName: 'client-0', + versions: { + r: knownVersion.r, + flowr: knownVersion.flowr + } + }, 'Expected hello message to have the predefined format') + })) + + it('Process simple REPL Message', withSocket(shell, async socket => { + fakeSend(socket, { + type: 'request-repl-execution', + ansi: false, + id: '0', + expression: '1 + 1' + }) + + await socket.waitForMessage('end-repl-execution') + + const messages = socket.getMessages() + + assert.deepStrictEqual(messages[1] as ExecuteIntermediateResponseMessage, { + type: 'response-repl-execution', + id: '0', + stream: 'stdout', + result: '[1] 2\n' + }, 'response message should contain request id, result should be not in standard error (no failure), and it should be the correct result') + + assert.deepStrictEqual(messages[2] as ExecuteEndMessage, { + type: 'end-repl-execution', + id: '0', + }, 'the end message should have the same id as the response (and come after the response)') + + })) + + + it('Allow to analyze a simple expression', withSocket(shell, async socket => { + fakeSend(socket, { + type: 'request-file-analysis', + id: '42', + filetoken: 'super-token', + filename: 'x', + content: '1 + 1' + }) + await socket.waitForMessage('response-file-analysis') + const messages = socket.getMessages(['hello', 'response-file-analysis']) + + const response = messages[1] as FileAnalysisResponseMessageJson + + // we are testing the server and not the slicer here! + const results = await new SteppingSlicer({ + stepOfInterest: LAST_PER_FILE_STEP, + shell, + request: requestFromInput('1 + 1'), + }).allRemainingSteps() + // hideous + results.normalize.idMap = undefined as unknown as DecoratedAstMap + + // cfg should not be set as we did not request it + assert.isUndefined(response.cfg, 'Expected the cfg to be undefined as we did not request it') + + assert.strictEqual(response.id, '42', 'Expected the second message to have the same id as the request') + + // this is hideous and only to unify the ids + const expected = JSON.stringify(results, jsonReplacer) + .replace(/"\.GlobalEnv","id":"\d+"/g, '') + const got = JSON.stringify(response.results, jsonReplacer) + .replace(/"\.GlobalEnv","id":"\d+"/g, '') + assert.strictEqual(got, expected, 'Expected the second message to have the same results as the slicer') + })) + + it('Analyze with the CFG', withSocket(shell, async socket => { + fakeSend(socket, { + type: 'request-file-analysis', + id: '42', + filetoken: 'super-token', + filename: 'x', + content: 'a;b', + cfg: true + }) + await socket.waitForMessage('response-file-analysis') + const messages = socket.getMessages(['hello', 'response-file-analysis']) + + const response = messages[1] as FileAnalysisResponseMessageJson + + const gotCfg = response.cfg + assert.isDefined(gotCfg, 'Expected the cfg to be defined as we requested it') + const expectedCfg = extractCFG(response.results.normalize) + assert.equal(JSON.stringify(gotCfg?.graph, jsonReplacer), JSON.stringify(expectedCfg.graph, jsonReplacer), 'Expected the cfg to be the same as the one extracted from the results') + })) })) - - it('Process simple REPL Message', withSocket(shell, async socket => { - fakeSend(socket, { - type: 'request-repl-execution', - ansi: false, - id: '0', - expression: '1 + 1' - }) - - await socket.waitForMessage('end-repl-execution') - - const messages = socket.getMessages() - - assert.deepStrictEqual(messages[1] as ExecuteIntermediateResponseMessage, { - type: 'response-repl-execution', - id: '0', - stream: 'stdout', - result: '[1] 2\n' - }, 'response message should contain request id, result should be not in standard error (no failure), and it should be the correct result') - - assert.deepStrictEqual(messages[2] as ExecuteEndMessage, { - type: 'end-repl-execution', - id: '0', - }, 'the end message should have the same id as the response (and come after the response)') - - })) - - - it('Allow to analyze a simple expression', withSocket(shell, async socket => { - fakeSend(socket, { - type: 'request-file-analysis', - id: '42', - filetoken: 'super-token', - filename: 'x', - content: '1 + 1' - }) - await socket.waitForMessage('response-file-analysis') - const messages = socket.getMessages(['hello', 'response-file-analysis']) - - const response = messages[1] as FileAnalysisResponseMessageJson - - // we are testing the server and not the slicer here! - const results = await new SteppingSlicer({ - stepOfInterest: LAST_PER_FILE_STEP, - shell, - request: requestFromInput('1 + 1'), - }).allRemainingSteps() - // hideous - results.normalize.idMap = undefined as unknown as DecoratedAstMap - - // cfg should not be set as we did not request it - assert.isUndefined(response.cfg, 'Expected the cfg to be undefined as we did not request it') - - assert.strictEqual(response.id, '42', 'Expected the second message to have the same id as the request') - - // this is hideous and only to unify the ids - const expected = JSON.stringify(results, jsonReplacer) - .replace(/"\.GlobalEnv","id":"\d+"/g, '') - const got = JSON.stringify(response.results, jsonReplacer) - .replace(/"\.GlobalEnv","id":"\d+"/g, '') - assert.strictEqual(got, expected, 'Expected the second message to have the same results as the slicer') - })) - - it('Analyze with the CFG', withSocket(shell, async socket => { - fakeSend(socket, { - type: 'request-file-analysis', - id: '42', - filetoken: 'super-token', - filename: 'x', - content: 'a;b', - cfg: true - }) - await socket.waitForMessage('response-file-analysis') - const messages = socket.getMessages(['hello', 'response-file-analysis']) - - const response = messages[1] as FileAnalysisResponseMessageJson - - const gotCfg = response.cfg - assert.isDefined(gotCfg, 'Expected the cfg to be defined as we requested it') - const expectedCfg = extractCFG(response.results.normalize) - assert.equal(JSON.stringify(gotCfg?.graph, jsonReplacer), JSON.stringify(expectedCfg.graph, jsonReplacer), 'Expected the cfg to be the same as the one extracted from the results') - })) -})) +}) diff --git a/test/functionality/main.spec.ts b/test/functionality/main.spec.ts index 1f29eb788d..a8a0757639 100644 --- a/test/functionality/main.spec.ts +++ b/test/functionality/main.spec.ts @@ -1,22 +1,33 @@ -// we use this file to configure the logging used when running tests +/** + * We use this file to configure the logging used when running tests + * + * @module + */ import { log, LogLevel } from '../../src/util/log' import { serverLog } from '../../src/cli/repl/server/server' -before(() => { + +/** + * Update the minimum level of all flowr loggers (including the detacthed {@link serverLog}). + * @param minLevel - The new minimum level to show messages from (inclusive) + * @param log2File - Whether to log to a file as well + */ +function setMinLevelOfAllLogs(minLevel: LogLevel, log2File = false) { for(const logger of [log, serverLog]) { + if(log2File) { + logger.logToFile() + } logger.updateSettings(logger => { - if(!process.argv.includes('--verbose')) { - logger.settings.minLevel = LogLevel.Error - } else { - logger.settings.minLevel = LogLevel.Trace - log.logToFile() - } + logger.settings.minLevel = minLevel }) } -}) +} + +export const VERBOSE_TESTS = process.argv.includes('--verbose') +before(() => setMinLevelOfAllLogs(VERBOSE_TESTS ? LogLevel.Trace : LogLevel.Error, VERBOSE_TESTS)) -/** controlled with `--test-installation` */ +/** controlled with the `--test-installation` parameter */ export const RUN_INSTALLATION_TESTS = process.argv.includes('--test-installation') export function isInstallTest(test: Mocha.Context): void { diff --git a/test/functionality/r-bridge/lang/ast/parse-access.ts b/test/functionality/r-bridge/lang/ast/parse-access.ts index 4a5a3c732a..f9f30eb019 100644 --- a/test/functionality/r-bridge/lang/ast/parse-access.ts +++ b/test/functionality/r-bridge/lang/ast/parse-access.ts @@ -1,5 +1,5 @@ -import { assertAst, withShell } from '../../../helper/shell' -import { exprList, numVal } from '../../../helper/ast-builder' +import { assertAst, withShell } from '../../../_helper/shell' +import { exprList, numVal } from '../../../_helper/ast-builder' import { rangeFrom } from '../../../../../src/util/range' import { RType } from '../../../../../src/r-bridge' diff --git a/test/functionality/r-bridge/lang/ast/parse-assignments.ts b/test/functionality/r-bridge/lang/ast/parse-assignments.ts index 8e017b70aa..16dcbd605d 100644 --- a/test/functionality/r-bridge/lang/ast/parse-assignments.ts +++ b/test/functionality/r-bridge/lang/ast/parse-assignments.ts @@ -1,6 +1,6 @@ -import { assertAst, withShell } from '../../../helper/shell' -import { exprList, numVal } from '../../../helper/ast-builder' -import { RAssignmentOpPool } from '../../../helper/provider' +import { assertAst, withShell } from '../../../_helper/shell' +import { exprList, numVal } from '../../../_helper/ast-builder' +import { RAssignmentOpPool } from '../../../_helper/provider' import { rangeFrom } from '../../../../../src/util/range' import { RType } from '../../../../../src/r-bridge' diff --git a/test/functionality/r-bridge/lang/ast/parse-constructs.ts b/test/functionality/r-bridge/lang/ast/parse-constructs.ts index 18100d6e80..9800e24008 100644 --- a/test/functionality/r-bridge/lang/ast/parse-constructs.ts +++ b/test/functionality/r-bridge/lang/ast/parse-constructs.ts @@ -1,5 +1,5 @@ -import { assertAst, withShell } from '../../../helper/shell' -import { exprList, numVal } from '../../../helper/ast-builder' +import { assertAst, withShell } from '../../../_helper/shell' +import { exprList, numVal } from '../../../_helper/ast-builder' import { addRanges, rangeFrom } from '../../../../../src/util/range' import { RType } from '../../../../../src/r-bridge' import { ensureExpressionList } from '../../../../../src/r-bridge/lang-4.x/ast/parser/xml/internal' diff --git a/test/functionality/r-bridge/lang/ast/parse-directives.ts b/test/functionality/r-bridge/lang/ast/parse-directives.ts index adec21072b..68ffc5792f 100644 --- a/test/functionality/r-bridge/lang/ast/parse-directives.ts +++ b/test/functionality/r-bridge/lang/ast/parse-directives.ts @@ -1,5 +1,5 @@ -import { assertAst, withShell } from '../../../helper/shell' -import { exprList } from '../../../helper/ast-builder' +import { assertAst, withShell } from '../../../_helper/shell' +import { exprList } from '../../../_helper/ast-builder' import { rangeFrom } from '../../../../../src/util/range' import { RType } from '../../../../../src/r-bridge' diff --git a/test/functionality/r-bridge/lang/ast/parse-expression-lists.ts b/test/functionality/r-bridge/lang/ast/parse-expression-lists.ts index e215646ba8..b2d92d41b7 100644 --- a/test/functionality/r-bridge/lang/ast/parse-expression-lists.ts +++ b/test/functionality/r-bridge/lang/ast/parse-expression-lists.ts @@ -1,5 +1,5 @@ -import { assertAst, withShell } from '../../../helper/shell' -import { exprList, numVal } from '../../../helper/ast-builder' +import { assertAst, withShell } from '../../../_helper/shell' +import { exprList, numVal } from '../../../_helper/ast-builder' import { rangeFrom } from '../../../../../src/util/range' import { RType } from '../../../../../src/r-bridge' diff --git a/test/functionality/r-bridge/lang/ast/parse-function-call.ts b/test/functionality/r-bridge/lang/ast/parse-function-call.ts index e05a807f12..219c575ec6 100644 --- a/test/functionality/r-bridge/lang/ast/parse-function-call.ts +++ b/test/functionality/r-bridge/lang/ast/parse-function-call.ts @@ -1,5 +1,5 @@ -import { assertAst, withShell } from '../../../helper/shell' -import { exprList, numVal } from '../../../helper/ast-builder' +import { assertAst, withShell } from '../../../_helper/shell' +import { exprList, numVal } from '../../../_helper/ast-builder' import { rangeFrom } from '../../../../../src/util/range' import { RType } from '../../../../../src/r-bridge' import { ensureExpressionList } from '../../../../../src/r-bridge/lang-4.x/ast/parser/xml/internal' diff --git a/test/functionality/r-bridge/lang/ast/parse-function-definitions.ts b/test/functionality/r-bridge/lang/ast/parse-function-definitions.ts index 0dab6eebd7..a4f82643c3 100644 --- a/test/functionality/r-bridge/lang/ast/parse-function-definitions.ts +++ b/test/functionality/r-bridge/lang/ast/parse-function-definitions.ts @@ -1,5 +1,5 @@ -import { assertAst, withShell } from '../../../helper/shell' -import { exprList, numVal, parameter } from '../../../helper/ast-builder' +import { assertAst, withShell } from '../../../_helper/shell' +import { exprList, numVal, parameter } from '../../../_helper/ast-builder' import { rangeFrom } from '../../../../../src/util/range' import { RType } from '../../../../../src/r-bridge' import { ensureExpressionList } from '../../../../../src/r-bridge/lang-4.x/ast/parser/xml/internal' diff --git a/test/functionality/r-bridge/lang/ast/parse-hooks.ts b/test/functionality/r-bridge/lang/ast/parse-hooks.ts index 1e1e2ce5bd..f7e428e72e 100644 --- a/test/functionality/r-bridge/lang/ast/parse-hooks.ts +++ b/test/functionality/r-bridge/lang/ast/parse-hooks.ts @@ -1,4 +1,4 @@ -import { retrieveNormalizedAst, withShell } from '../../../helper/shell' +import { retrieveNormalizedAst, withShell } from '../../../_helper/shell' import { assert } from 'chai' import { requestFromInput } from '../../../../../src/r-bridge' import { SteppingSlicer } from '../../../../../src/core' diff --git a/test/functionality/r-bridge/lang/ast/parse-operations.ts b/test/functionality/r-bridge/lang/ast/parse-operations.ts index d927fbd945..48c9e55a1d 100644 --- a/test/functionality/r-bridge/lang/ast/parse-operations.ts +++ b/test/functionality/r-bridge/lang/ast/parse-operations.ts @@ -1,10 +1,10 @@ -import { assertAst, withShell } from '../../../helper/shell' -import { exprList, numVal } from '../../../helper/ast-builder' +import { assertAst, withShell } from '../../../_helper/shell' +import { exprList, numVal } from '../../../_helper/ast-builder' import { RArithmeticBinaryOpPool, RLogicalBinaryOpPool, RUnaryOpPool, -} from '../../../helper/provider' +} from '../../../_helper/provider' import { type RShell, RType, ComparisonOperators } from '../../../../../src/r-bridge' import { rangeFrom } from '../../../../../src/util/range' diff --git a/test/functionality/r-bridge/lang/ast/parse-pipes.ts b/test/functionality/r-bridge/lang/ast/parse-pipes.ts index 0a8a78c4a7..6fa6bbd62f 100644 --- a/test/functionality/r-bridge/lang/ast/parse-pipes.ts +++ b/test/functionality/r-bridge/lang/ast/parse-pipes.ts @@ -1,5 +1,5 @@ -import { assertAst, withShell } from '../../../helper/shell' -import { exprList } from '../../../helper/ast-builder' +import { assertAst, withShell } from '../../../_helper/shell' +import { exprList } from '../../../_helper/ast-builder' import { rangeFrom } from '../../../../../src/util/range' import { RType } from '../../../../../src/r-bridge' import { MIN_VERSION_PIPE } from '../../../../../src/r-bridge/lang-4.x/ast/model/versions' diff --git a/test/functionality/r-bridge/lang/ast/parse-snippets.ts b/test/functionality/r-bridge/lang/ast/parse-snippets.ts index 980bac80c5..3c9b9ee1ac 100644 --- a/test/functionality/r-bridge/lang/ast/parse-snippets.ts +++ b/test/functionality/r-bridge/lang/ast/parse-snippets.ts @@ -1,5 +1,5 @@ -import { exprList, numVal } from '../../../helper/ast-builder' -import { assertAst, withShell } from '../../../helper/shell' +import { exprList, numVal } from '../../../_helper/ast-builder' +import { assertAst, withShell } from '../../../_helper/shell' import { rangeFrom } from '../../../../../src/util/range' import { RType } from '../../../../../src/r-bridge' import { ensureExpressionList } from '../../../../../src/r-bridge/lang-4.x/ast/parser/xml/internal' diff --git a/test/functionality/r-bridge/lang/ast/parse-symbols.ts b/test/functionality/r-bridge/lang/ast/parse-symbols.ts index 4e281161db..b207127fa2 100644 --- a/test/functionality/r-bridge/lang/ast/parse-symbols.ts +++ b/test/functionality/r-bridge/lang/ast/parse-symbols.ts @@ -1,5 +1,5 @@ -import { assertAst, withShell } from '../../../helper/shell' -import { exprList } from '../../../helper/ast-builder' +import { assertAst, withShell } from '../../../_helper/shell' +import { exprList } from '../../../_helper/ast-builder' import { rangeFrom } from '../../../../../src/util/range' import { RType } from '../../../../../src/r-bridge' diff --git a/test/functionality/r-bridge/lang/ast/parse-values.ts b/test/functionality/r-bridge/lang/ast/parse-values.ts index 875d753c29..0ad6f4a951 100644 --- a/test/functionality/r-bridge/lang/ast/parse-values.ts +++ b/test/functionality/r-bridge/lang/ast/parse-values.ts @@ -1,10 +1,10 @@ -import { assertAst, withShell } from '../../../helper/shell' +import { assertAst, withShell } from '../../../_helper/shell' import { RNumberPool, RStringPool, RSymbolPool, -} from '../../../helper/provider' -import { exprList } from '../../../helper/ast-builder' +} from '../../../_helper/provider' +import { exprList } from '../../../_helper/ast-builder' import { rangeFrom } from '../../../../../src/util/range' import { retrieveXmlFromRCode, RType } from '../../../../../src/r-bridge' import chai, { assert } from 'chai' diff --git a/test/functionality/r-bridge/lang/values.ts b/test/functionality/r-bridge/lang/values.ts index 4394228a92..911a8df16c 100644 --- a/test/functionality/r-bridge/lang/values.ts +++ b/test/functionality/r-bridge/lang/values.ts @@ -1,7 +1,7 @@ import { it } from 'mocha' import { assert } from 'chai' import { boolean2ts, isBoolean, number2ts, string2ts, ts2r } from '../../../../src/r-bridge' -import { RNumberPool, RStringPool } from '../../helper/provider' +import { RNumberPool, RStringPool } from '../../_helper/provider' describe('Bidirectional Value Translation', () => { describe('TS value to R', () => { diff --git a/test/functionality/r-bridge/processing.spec.ts b/test/functionality/r-bridge/processing.spec.ts index e030648b42..a96bcb1d28 100644 --- a/test/functionality/r-bridge/processing.spec.ts +++ b/test/functionality/r-bridge/processing.spec.ts @@ -1,5 +1,5 @@ -import { assertDecoratedAst, retrieveNormalizedAst, withShell } from '../helper/shell' -import { numVal } from '../helper/ast-builder' +import { assertDecoratedAst, retrieveNormalizedAst, withShell } from '../_helper/shell' +import { numVal } from '../_helper/ast-builder' import { rangeFrom } from '../../../src/util/range' import { RType, diff --git a/test/functionality/r-bridge/sessions.ts b/test/functionality/r-bridge/sessions.ts index 427885f46f..a034b21d89 100644 --- a/test/functionality/r-bridge/sessions.ts +++ b/test/functionality/r-bridge/sessions.ts @@ -1,8 +1,8 @@ import chai, { assert } from 'chai' import fs from 'fs' import { randomString } from '../../../src/util/random' -import { testRequiresNetworkConnection } from '../helper/network' -import { testWithShell, withShell } from '../helper/shell' +import { testRequiresNetworkConnection } from '../_helper/network' +import { testWithShell, withShell } from '../_helper/shell' import { isInstallTest } from '../main.spec' import { parseCSV } from '../../../src/r-bridge' import { log, LogLevel } from '../../../src/util/log' diff --git a/test/functionality/slicing/reconstruct/simple.ts b/test/functionality/slicing/reconstruct/simple-tests.ts similarity index 97% rename from test/functionality/slicing/reconstruct/simple.ts rename to test/functionality/slicing/reconstruct/simple-tests.ts index 58223224bb..dacf8dc35f 100644 --- a/test/functionality/slicing/reconstruct/simple.ts +++ b/test/functionality/slicing/reconstruct/simple-tests.ts @@ -1,4 +1,4 @@ -import { assertReconstructed, withShell } from '../../helper/shell' +import { assertReconstructed, withShell } from '../../_helper/shell' describe('Simple', withShell(shell => { describe('Constant assignments', () => { diff --git a/test/functionality/slicing/criterion/collect-all.ts b/test/functionality/slicing/slicing-criterion/collect-all-tests.ts similarity index 96% rename from test/functionality/slicing/criterion/collect-all.ts rename to test/functionality/slicing/slicing-criterion/collect-all-tests.ts index 2c4f17f35b..5c1c2929fa 100644 --- a/test/functionality/slicing/criterion/collect-all.ts +++ b/test/functionality/slicing/slicing-criterion/collect-all-tests.ts @@ -6,7 +6,7 @@ import { SlicingCriteria } from '../../../../src/slicing' import { decorateAst, RShell } from '../../../../src/r-bridge' -import { retrieveNormalizedAst, withShell } from '../../helper/shell' +import { retrieveNormalizedAst, withShell } from '../../_helper/shell' import { assert } from 'chai' function assertRetrievedIdsWith(shell: RShell, name: string, input: string, filter: SlicingCriteriaFilter, ...expected: SlicingCriteria[]) { diff --git a/test/functionality/slicing/slicing.spec.ts b/test/functionality/slicing/slicing.spec.ts index b4dd090e03..62096d9cfd 100644 --- a/test/functionality/slicing/slicing.spec.ts +++ b/test/functionality/slicing/slicing.spec.ts @@ -1,12 +1,14 @@ +import { requireAllTestsInFolder } from '../_helper/collect-tests' +import path from 'path' + describe('Slicing', () => { - describe('Expression Based Code Reconstruction', () => { - require('./reconstruct/simple') + describe('Reconstruct', () => { + requireAllTestsInFolder(path.join(__dirname, 'reconstruct')) }) describe('Static Program Slices', () => { - require('./static/simple') - require('./static/calls') + requireAllTestsInFolder(path.join(__dirname, 'static-program-slices')) }) - describe('Collect All Ids', () => { - require('./criterion/collect-all') + describe('Slicing-Criterion', () => { + requireAllTestsInFolder(path.join(__dirname, 'slicing-criterion')) }) }) diff --git a/test/functionality/slicing/static/calls.ts b/test/functionality/slicing/static-program-slices/calls-tests.ts similarity index 98% rename from test/functionality/slicing/static/calls.ts rename to test/functionality/slicing/static-program-slices/calls-tests.ts index 235bfe63c5..8a746ac4f3 100644 --- a/test/functionality/slicing/static/calls.ts +++ b/test/functionality/slicing/static-program-slices/calls-tests.ts @@ -1,7 +1,7 @@ -import { assertSliced, withShell } from '../../helper/shell' +import { assertSliced, withShell } from '../../_helper/shell' -describe('With Call', withShell(shell => { - describe('Simple', () => { +describe('Calls', withShell(shell => { + describe('Simple Calls', () => { const code = `i <- 4 a <- function(x) { x } a(i)` diff --git a/test/functionality/slicing/static/simple.ts b/test/functionality/slicing/static-program-slices/simple-tests.ts similarity index 98% rename from test/functionality/slicing/static/simple.ts rename to test/functionality/slicing/static-program-slices/simple-tests.ts index 9af30041c7..4487e1f023 100644 --- a/test/functionality/slicing/static/simple.ts +++ b/test/functionality/slicing/static-program-slices/simple-tests.ts @@ -1,4 +1,4 @@ -import { assertSliced, withShell } from '../../helper/shell' +import { assertSliced, withShell } from '../../_helper/shell' describe('Simple', withShell(shell => { describe('Constant assignments', () => { diff --git a/test/functionality/statistics/features/assignments.ts b/test/functionality/statistics/features/assignments-tests.ts similarity index 82% rename from test/functionality/statistics/features/assignments.ts rename to test/functionality/statistics/features/assignments-tests.ts index 034c13c0c7..51e2a0ad17 100644 --- a/test/functionality/statistics/features/assignments.ts +++ b/test/functionality/statistics/features/assignments-tests.ts @@ -1,7 +1,7 @@ -import { withShell } from '../../helper/shell' +import { withShell } from '../../_helper/shell' import { testForFeatureForInput } from '../statistics.spec' -describe('Used Ways to Assign', withShell(shell => { +describe('Assignments', withShell(shell => { testForFeatureForInput(shell, 'assignments', [ { name: 'no assignment', diff --git a/test/functionality/statistics/features/control-flow.ts b/test/functionality/statistics/features/control-flow-tests.ts similarity index 98% rename from test/functionality/statistics/features/control-flow.ts rename to test/functionality/statistics/features/control-flow-tests.ts index f335f40da1..3e387aa1c0 100644 --- a/test/functionality/statistics/features/control-flow.ts +++ b/test/functionality/statistics/features/control-flow-tests.ts @@ -1,4 +1,4 @@ -import { withShell } from '../../helper/shell' +import { withShell } from '../../_helper/shell' import { testForFeatureForInput } from '../statistics.spec' import { RFalse, RTrue } from '../../../../src/r-bridge' diff --git a/test/functionality/statistics/features/data-access.ts b/test/functionality/statistics/features/data-access-tests.ts similarity index 98% rename from test/functionality/statistics/features/data-access.ts rename to test/functionality/statistics/features/data-access-tests.ts index b508e436cb..0c7eb2caa4 100644 --- a/test/functionality/statistics/features/data-access.ts +++ b/test/functionality/statistics/features/data-access-tests.ts @@ -1,8 +1,8 @@ -import { withShell } from '../../helper/shell' +import { withShell } from '../../_helper/shell' import { testForFeatureForInput } from '../statistics.spec' import { emptyCommonSyntaxTypeCounts } from '../../../../src/statistics/features/common-syntax-probability' -describe('Used Ways to Access Data', withShell(shell => { +describe('Data Access', withShell(shell => { testForFeatureForInput(shell, 'dataAccess', [ { name: 'no data access', diff --git a/test/functionality/statistics/features/defined-functions.ts b/test/functionality/statistics/features/defined-functions-tests.ts similarity index 97% rename from test/functionality/statistics/features/defined-functions.ts rename to test/functionality/statistics/features/defined-functions-tests.ts index baf6a55271..8774b656b4 100644 --- a/test/functionality/statistics/features/defined-functions.ts +++ b/test/functionality/statistics/features/defined-functions-tests.ts @@ -1,9 +1,9 @@ -import { withShell } from '../../helper/shell' +import { withShell } from '../../_helper/shell' import { testForFeatureForInput } from '../statistics.spec' import { MIN_VERSION_LAMBDA } from '../../../../src/r-bridge/lang-4.x/ast/model/versions' -describe('Used Function Definitions', withShell(shell => { +describe('Defined Functions', withShell(shell => { testForFeatureForInput(shell, 'definedFunctions', [ { name: 'no definitions', diff --git a/test/functionality/statistics/features/expression-list.ts b/test/functionality/statistics/features/expression-list-tests.ts similarity index 93% rename from test/functionality/statistics/features/expression-list.ts rename to test/functionality/statistics/features/expression-list-tests.ts index 1fad7b6365..c60b2a2e91 100644 --- a/test/functionality/statistics/features/expression-list.ts +++ b/test/functionality/statistics/features/expression-list-tests.ts @@ -1,4 +1,4 @@ -import { withShell } from '../../helper/shell' +import { withShell } from '../../_helper/shell' import { testForFeatureForInput } from '../statistics.spec' diff --git a/test/functionality/statistics/features/loops.ts b/test/functionality/statistics/features/loops-tests.ts similarity index 98% rename from test/functionality/statistics/features/loops.ts rename to test/functionality/statistics/features/loops-tests.ts index 5c7f921aa9..ab03c1b896 100644 --- a/test/functionality/statistics/features/loops.ts +++ b/test/functionality/statistics/features/loops-tests.ts @@ -1,4 +1,4 @@ -import { withShell } from '../../helper/shell' +import { withShell } from '../../_helper/shell' import { testForFeatureForInput } from '../statistics.spec' import { RFalse, RTrue } from '../../../../src/r-bridge' diff --git a/test/functionality/statistics/features/used-functions.ts b/test/functionality/statistics/features/used-function-calls-tests.ts similarity index 98% rename from test/functionality/statistics/features/used-functions.ts rename to test/functionality/statistics/features/used-function-calls-tests.ts index 76e3af6a21..8dde786655 100644 --- a/test/functionality/statistics/features/used-functions.ts +++ b/test/functionality/statistics/features/used-function-calls-tests.ts @@ -1,4 +1,4 @@ -import { withShell } from '../../helper/shell' +import { withShell } from '../../_helper/shell' import { testForFeatureForInput } from '../statistics.spec' import { emptyCommonSyntaxTypeCounts } from '../../../../src/statistics/features/common-syntax-probability' diff --git a/test/functionality/statistics/features/variables.ts b/test/functionality/statistics/features/variables-tests.ts similarity index 96% rename from test/functionality/statistics/features/variables.ts rename to test/functionality/statistics/features/variables-tests.ts index 137afaf20a..796bb316bb 100644 --- a/test/functionality/statistics/features/variables.ts +++ b/test/functionality/statistics/features/variables-tests.ts @@ -1,4 +1,4 @@ -import { withShell } from '../../helper/shell' +import { withShell } from '../../_helper/shell' import { testForFeatureForInput } from '../statistics.spec' diff --git a/test/functionality/statistics/statistics.spec.ts b/test/functionality/statistics/statistics.spec.ts index deafc06bcb..1e5418f51b 100644 --- a/test/functionality/statistics/statistics.spec.ts +++ b/test/functionality/statistics/statistics.spec.ts @@ -10,8 +10,10 @@ import { assert } from 'chai' import { RShell } from '../../../src/r-bridge' import { deepMergeObject } from '../../../src/util/objects' import { jsonReplacer, jsonRetriever } from '../../../src/util/json' -import { ensureConfig, TestConfiguration } from '../helper/shell' +import { ensureConfig, TestConfiguration } from '../_helper/shell' import { DeepPartial } from 'ts-essentials' +import { requireAllTestsInFolder } from '../_helper/collect-tests' +import path from 'path' async function requestFeature(shell: RShell, feature: T, code: string): Promise> { const results = await extractUsageStatistics(shell, () => { /* do nothing */ }, new Set([feature]), staticRequests({ request: 'text', content: code })) @@ -59,11 +61,5 @@ export function testForFeatureForInput(shell: RShell, feat } describe('Statistics', () => { - require('./features/control-flow.ts') - require('./features/used-functions.ts') - require('./features/loops.ts') - require('./features/data-access.ts') - require('./features/defined-functions.ts') - require('./features/variables.ts') - require('./features/assignments.ts') + requireAllTestsInFolder(path.join(__dirname, 'features')) }) diff --git a/test/functionality/util/ags.spec.ts b/test/functionality/util/arguments-tests.ts similarity index 100% rename from test/functionality/util/ags.spec.ts rename to test/functionality/util/arguments-tests.ts diff --git a/test/functionality/util/arrays.spec.ts b/test/functionality/util/arrays-tests.ts similarity index 100% rename from test/functionality/util/arrays.spec.ts rename to test/functionality/util/arrays-tests.ts diff --git a/test/functionality/util/cfg.spec.ts b/test/functionality/util/control-flow-graph-tests.ts similarity index 99% rename from test/functionality/util/cfg.spec.ts rename to test/functionality/util/control-flow-graph-tests.ts index f29ea272af..3e168e8b2a 100644 --- a/test/functionality/util/cfg.spec.ts +++ b/test/functionality/util/control-flow-graph-tests.ts @@ -1,5 +1,5 @@ import { assert } from 'chai' -import { withShell } from '../helper/shell' +import { withShell } from '../_helper/shell' import { ControlFlowGraph, cfg2quads, diff --git a/test/functionality/util/objects.spec.ts b/test/functionality/util/objects-tests.ts similarity index 100% rename from test/functionality/util/objects.spec.ts rename to test/functionality/util/objects-tests.ts diff --git a/test/functionality/util/quads.spec.ts b/test/functionality/util/quads-tests.ts similarity index 93% rename from test/functionality/util/quads.spec.ts rename to test/functionality/util/quads-tests.ts index b0a4c854ce..30edb898b0 100644 --- a/test/functionality/util/quads.spec.ts +++ b/test/functionality/util/quads-tests.ts @@ -1,9 +1,9 @@ -import { retrieveNormalizedAst, withShell } from '../helper/shell' +import { retrieveNormalizedAst, withShell } from '../_helper/shell' import { decorateAst, RType } from '../../../src/r-bridge' import { defaultQuadIdGenerator, serialize2quads } from '../../../src/util/quads' import { assert } from 'chai' -describe('Quad Generation', withShell(shell => { +describe('Quads', withShell(shell => { const context = 'test' const domain = 'https://uni-ulm.de/r-ast/' diff --git a/test/functionality/util/random.spec.ts b/test/functionality/util/random-tests.ts similarity index 100% rename from test/functionality/util/random.spec.ts rename to test/functionality/util/random-tests.ts diff --git a/test/functionality/util/range.spec.ts b/test/functionality/util/range-tests.ts similarity index 99% rename from test/functionality/util/range.spec.ts rename to test/functionality/util/range-tests.ts index 18722faf86..b7df79fec2 100644 --- a/test/functionality/util/range.spec.ts +++ b/test/functionality/util/range-tests.ts @@ -9,7 +9,7 @@ import { import { allPermutations } from '../../../src/util/arrays' import { formatRange } from '../../../src/dataflow' -describe('Ranges', () => { +describe('Range', () => { describe('rangeFrom', () => { it('correct arguments', () => { const pool = [-1, 0, 1, 2, 99] diff --git a/test/functionality/util/set.spec.ts b/test/functionality/util/set-operations-tests.ts similarity index 95% rename from test/functionality/util/set.spec.ts rename to test/functionality/util/set-operations-tests.ts index 2685b1aece..2188282759 100644 --- a/test/functionality/util/set.spec.ts +++ b/test/functionality/util/set-operations-tests.ts @@ -3,7 +3,7 @@ import { setEquals } from '../../../src/util/set' import { jsonReplacer } from '../../../src/util/json' import { Test } from 'mocha' -describe('Set operations', () => { +describe('Set (operations)', () => { describe('setEquals', () => { function test(name: string, should: boolean, a: Set, b: Set): Test { return it(name, () => diff --git a/test/functionality/util/strings.spec.ts b/test/functionality/util/strings-tests.ts similarity index 100% rename from test/functionality/util/strings.spec.ts rename to test/functionality/util/strings-tests.ts diff --git a/test/functionality/util/util.spec.ts b/test/functionality/util/util.spec.ts new file mode 100644 index 0000000000..21a7cb600e --- /dev/null +++ b/test/functionality/util/util.spec.ts @@ -0,0 +1,5 @@ +import { requireAllTestsInFolder } from '../_helper/collect-tests' + +describe('Util', () => { + requireAllTestsInFolder(__dirname) +}) diff --git a/wiki/Test.md b/wiki/Test.md index 5779b7c0d6..7da4f93fb5 100644 --- a/wiki/Test.md +++ b/wiki/Test.md @@ -1,7 +1,9 @@ -***This wiki page is currently under construction*** +For the latest code-coverage information, see [codecov.io](https://codecov.io/gh/Code-Inspect/flowr), for the latest benchmark results, see the [benchmark results](https://code-inspect.github.io/flowr/wiki/stats/benchmark) wiki page. - [Testing Suites](#testing-suites) - [Functionality Tests](#functionality-tests) + - [Test Structure](#test-structure) + - [Writing a Test](#writing-a-test) - [Performance Tests](#performance-tests) - [CI Pipeline](#ci-pipeline) - [Linting](#linting) @@ -10,15 +12,56 @@ ## Testing Suites +Currently, flowR contains two testing suites: one for [functionality](#functionality-tests) and one for [performance](#performance-tests). We explain each of them in the following. + ### Functionality Tests -How to test *flowR* +The functionality tests represent conventional unit (and depending on your terminology component/api) tests. +We use [mocha](https://mochajs.org/) as our testing framework and [chai](https://www.chaijs.com/) as our assertion library. +To run these tests, simply issue: + +```shell +npm run test +``` + +What may be counter-intuitive is that this does not run *all* tests by default but only those that do not try R's package installation (as this can require more time). To run all tests, installation tests included, use: + +```shell +npm run test-full +``` + +However, depending on your local R version, your network connection and potentially other factors, some tests may be skipped automatically as they do not apply to your current system setup (or can't be tested with the current prerequisites). Each test can speciy such requirements as part of the `TestConfiguration`, which is then enforced by the [`ensureConfig`](https://github.com/Code-Inspect/flowr/blob/main/test/functionality/_helper/shell.ts) function. +It is up to the [ci](#ci-pipeline) to run the tests on different systems to ensure that those tests are ensured to run. + +#### Test Structure + +All functionality tests are to be located under [test/functionality](https://github.com/Code-Inspect/flowr/tree/main/test/functionality). + +This folder contains two special elements: + +- `main.spec.ts` which is the entry point if *all* tests are run. It should automatically disable logging statements and configure global variables (e.g., if installation tests should run). +- `_helper` which contains helper functions to be used by other tests. + +Besides folders can (theoretically) arbitrarily structure their tests. We use the following convention: + +- `*.spec.ts` denotes a test file which is to be collected when all tests are run, it may require other files to avoid big testing files and improve structure, but those should never end in `*.spec.ts`. +. `-tests.ts` denotes test files which are required by `*.spec.ts` files. To require them, there is also the helper function `requireAllTestsInFolder` which performs sanity checks to avoid forgotten tests. + + +#### Writing a Test + +Currently this is heavily dependend on what you want to test (normalization, dataflow, quad-export, ...) and it is probably best to have a look at existing tests in that area to get an idea of what comfort functionality is available. ### Performance Tests The performance test suite of *flowR* uses several suites to check for variations in the required times for certain steps. Although we measure wall time in the CI (which is subject to rather large variations), it should give a rough idea of the performance of *flowR*. Furthermore, the respective scripts can be used locally as well. +To run them, issue: + +```shell +npm run performance-test +``` See [test/performance](https://github.com/Code-Inspect/flowr/tree/main/test/performance) for more information on the suites, how to run them, and their results. If you are interested in the results of the benchmarks, see [here](https://code-inspect.github.io/flowr/wiki/stats/benchmark).