Allow Variable Steps in the Dataflow Processor

.githooks/commit-msg

@@ -4,9 +4,19 @@ message_file="$1"
 message=$(cat "$message_file")
 
 # Define the regular expression pattern
-regex="^((\[(no|skip) ci\] )?(((feat|tests?|lint|refactor|ci|git|special|doc|typo|log|ts|fix|wip|docker|dep)(-fix|-fail)?(, ?)?)+: |Merge (remote-tracking )?branch|Auto-merging).+|\[release:(patch|minor|major)\] .+)"
+regex="^((\[(no|skip) ci\] )?"                                                                  # allow to skip ci steps if required
+regex="$regex(Merge (remote-tracking )?branch|Auto-merging|"                                    # allow merge commit messages
+  regex="$regex("                                                                               # allow multiple types
+    regex="$regex(feat|tests?|lint|refactor|ci|git|special|doc|typo|ts|fix|wip|docker|dep)"     # all valid types
+    regex="$regex(-fix|-fail)?"                                                                 # optional fail suffix
+    regex="$regex(\([^)]+\))?"                                                                  # optional scope
+    regex="$regex(, ?)?"                                                                        # optional comma between types
+  regex="$regex)+: "                                                                            # at least one type is required
+regex="$regex)"                                                                                 # allow arbitrary message (no end marker)
+regex="$regex|\[release:(patch|minor|major)\] .+)"                                              # alternatively, allow release commits (only to be done on main)
 
 if ! echo "$message" | grep -qE "$regex"; then
-  echo "[POLICY] Your message is not formatted correctly. Respect the regex: '$regex'!"
+  echo   "[POLICY] Your message is not formatted correctly. Please respect the style defined in '.github/CONTRIBUTING.md'."
+  printf "[POLICY] Your message was (ignoring git comments):\n\n%s\n" "$(echo "$message" | grep -vE "^#")"
   exit 1
 fi

.githooks/pre-push

@@ -2,9 +2,42 @@
 
 set -eu
 
-if ! [ -x "$(command -v npm)" ]; then
-  echo 'Error: npm not found. Make it available to the host shell (e.g., with "nvm use --lts").'
-  exit 2
+
+find_npm_linux() {
+  export NPM_CMD="npm"
+
+  if ! (type $NPM_CMD >> /dev/null); then
+    echo "npm not found, trying to make it available using nvm..."
+    if type nvm >> /dev/null; then
+      echo "nvm found, using it to install the latest lts node"
+      nvm use --lts
+    else
+      echo "nvm not found, trying to make it available using the nvm.sh"
+      # try to make it available based on https://github.com/typicode/husky/issues/912#issuecomment-817522060
+      export NVM_DIR="$HOME/.nvm/nvm.sh"
+      . "$(dirname $NVM_DIR)/nvm.sh"
+
+      export NVM_DIR="$HOME/.nvm"
+      a=$(nvm ls --no-colors | grep 'node')
+      v=$(echo "$a" | sed -E 's/.*\(-> ([^ ]+).*/\1/')
+
+      export PATH="$NVM_DIR/versions/node/$v/bin:$PATH"
+
+      if ! (type $NPM_CMD >> /dev/null); then
+        echo "no variant of npm or nvm found, trying to use the npm.cmd"
+        export NPM_CMD="npm.cmd"
+      fi
+    fi
+  fi
+}
+
+if [ -z "${OSTYPE+x}" ]; then
+  find_npm_linux
+else
+  case "$OSTYPE" in
+    msys*) export NPM_CMD="npm.cmd";;
+    *)     find_npm_linux ;;
+  esac
 fi
 
 
@@ -37,7 +70,7 @@ if [ -n "$(git status --porcelain)" ]; then
 fi
 
 echo "Linting project (local mode)..."
-npm run lint-local
+$NPM_CMD run lint-local
 
 
 # shellcheck disable=SC2124 # we want the argument splitting

.github/CONTRIBUTING.md

@@ -33,8 +33,8 @@ If you have any questions, refer to the [wiki](https://github.com/Code-Inspect/f
 
 ## Commit Messages
 
-We structure our commit messages (enforced by our git-hooks) using the format `<type(s)>: <description>`.
-Currently, the following types are at your disposal (more may have been or are still available, but please restrict yourself to the following):
+We structure our commit messages (enforced by our git-hooks) using the format `<type(s)>(<scope>): <description>` (with `(<scope>)` being optional).
+Currently, the following `<type(s)>` are at your disposal (more may have been or are still available, but please restrict yourself to the following):
 
 
 | name                                            | description                                                                                        |
@@ -49,7 +49,6 @@ Currently, the following types are at your disposal (more may have been or are s
 | `lint`                                          | Adapted or updated linter-issues.                                                                  |
 | `doc`                                           | Updated the documentation of *flowR*.                                                              |
 | `typo`                                          | Dealt with a small typo/a grammatical mistake.                                                     |
-| `log`                                           | Improved or updated the logging of *flowR*.                                                        |
 | `ts`                                            | Performed something typescript-specific (e.g., reconfigured the `tsconfig.json`).                  |
 | `wip`                                           | *Use this only in combination with another type*. It marks the commit to be unfinished.            |
 | `special`                                       | *Use this only if none of the other categories apply*. Explain the details in your commit message. |
@@ -65,9 +64,12 @@ Although you can give the same type repeatedly - if you think you should, please
 
 With this, the artificial message
 
-> `feat, test-fix: Support for branching in dataflow, fixed branching-test`
+> `feat, test-fix: Support for branching in dataflow, fixed branching test`
 
 represents the addition of a new feature and the fix of a corresponding test.
+With scopes, it could look like this:
+
+> `feat, test-fix(dataflow): Support branching & fixed branching test`
 
 To skip the `ci`, you can prefix the commit message with `[skip ci]`.
 

package.json

@@ -18,7 +18,7 @@
     "lint": "npm run license-compat -- --summary && eslint src/ test/",
     "license-compat": "license-checker --onlyAllow 'MIT;MIT OR X11;GPLv2;LGPL;GNUGPL;ISC;Apache-2.0;FreeBSD;BSD-2-Clause;clearbsd;ModifiedBSD;BSD-3-Clause;Python-2.0;Unlicense;WTFPL;CC-BY-4.0;CC-BY-3.0;CC0-1.0;0BSD'",
     "doc": "typedoc",
-    "test": "nyc --no-clean mocha --require ts-node/register --timeout 60000 \"test/**/*.spec.ts\"",
+    "test": "nyc --source-map --produce-source-map mocha --require ts-node/register --timeout 60000 \"test/**/*.spec.ts\"",
     "performance-test": "func() { cd test/performance/ && bash run-all-suites.sh $1 $2; cd ../../; }; func",
     "test-full": "npm run test -- --test-installation"
   },
@@ -29,14 +29,13 @@
     "all": true,
     "per-file": true,
     "check-coverage": false,
-    "skip-full": false,
+    "skip-full": true,
     "lines": 70,
     "extension": [
       ".ts"
     ],
     "include": "src/**/*.ts",
     "reporter": [
-      "html",
       "text",
       "lcov",
       "cobertura"
@@ -173,7 +172,7 @@
       "check-file/filename-naming-convention": [
         "error",
         {
-          "**/*.ts": "?([A-Z])+([a-z])*((-|.)?([A-Z])+([a-z]))"
+          "**/*.ts": "?([0-9]+-)?([A-Z])+([a-z])*((-|.)?([A-Z])+([a-z]))"
         }
       ],
       "check-file/folder-match-with-fex": [

src/benchmark/index.ts

@@ -1,3 +1,3 @@
 export * from './stats'
-export * from './slicer'
+export * from './benchmark-slicer'
 export * from './stopwatch'

src/cli/repl/core.ts

@@ -3,7 +3,7 @@
  *
  * @module
  */
-import { RShell } from '../../r-bridge'
+import { RShell, RShellReviveOptions } from '../../r-bridge'
 import readline from 'readline/promises'
 import { bold } from '../../statistics'
 import { prompt } from './prompt'
@@ -76,7 +76,7 @@ export async function replProcessAnswer(output: ReplOutput, expr: string, shell:
  * For the execution, this function makes use of {@link replProcessAnswer}
  *
  */
-export async function repl(shell = new RShell({ revive: 'always' }), rl = readline.createInterface(DEFAULT_REPL_READLINE_CONFIGURATION), output = standardReplOutput) {
+export async function repl(shell = new RShell({ revive: RShellReviveOptions.Always }), rl = readline.createInterface(DEFAULT_REPL_READLINE_CONFIGURATION), output = standardReplOutput) {
 
 	// the incredible repl :D, we kill it with ':quit'
 	// eslint-disable-next-line no-constant-condition,@typescript-eslint/no-unnecessary-condition

src/cli/repl/server/connection.ts

@@ -1,4 +1,4 @@
-import { LAST_STEP, printStepResult, SteppingSlicer, StepResults, STEPS_PER_SLICE } from '../../../core'
+import { LAST_STEP, SteppingSlicer, StepResults, STEPS_PER_SLICE } from '../../../core'
 import { DEFAULT_XML_PARSER_CONFIG, NormalizedAst, RShell, XmlParserConfig } from '../../../r-bridge'
 import { sendMessage } from './send'
 import { answerForValidationError, validateBaseMessageFormat, validateMessage } from './validate'
@@ -24,8 +24,11 @@ import { cfg2quads, ControlFlowInformation, extractCFG } from '../../../util/cfg
 import { defaultQuadIdGenerator, QuadSerializationConfiguration } from '../../../util/quads'
 import { deepMergeObject } from '../../../util/objects'
 import { LogLevel } from '../../../util/log'
-import { StepOutputFormat } from '../../../core/print/print'
+import { printStepResult, StepOutputFormat } from '../../../core/print/print'
 import { DataflowInformation } from '../../../dataflow/internal/info'
+import { PARSE_WITH_R_SHELL_STEP } from '../../../core/steps/all/core/00-parse'
+import { NORMALIZE } from '../../../core/steps/all/core/10-normalize'
+import { LEGACY_STATIC_DATAFLOW } from '../../../core/steps/all/core/20-dataflow'
 
 /**
  * Each connection handles a single client, answering to its requests.
@@ -134,9 +137,9 @@ export class FlowRServerConnection {
 				id:      message.id,
 				cfg:     cfg ? cfg2quads(cfg, config()) : undefined,
 				results: {
-					parse:     await printStepResult('parse', results.parse as string, StepOutputFormat.RdfQuads, config(), parseConfig),
-					normalize: await printStepResult('normalize', results.normalize as NormalizedAst, StepOutputFormat.RdfQuads, config()),
-					dataflow:  await printStepResult('dataflow', results.dataflow as DataflowInformation, StepOutputFormat.RdfQuads, config()),
+					parse:     await printStepResult(PARSE_WITH_R_SHELL_STEP, results.parse as string, StepOutputFormat.RdfQuads, config(), parseConfig),
+					normalize: await printStepResult(NORMALIZE, results.normalize as NormalizedAst, StepOutputFormat.RdfQuads, config()),
+					dataflow:  await printStepResult(LEGACY_STATIC_DATAFLOW, results.dataflow as DataflowInformation, StepOutputFormat.RdfQuads, config()),
 				}
 			})
 		} else {

src/cli/statistics-helper-app.ts

@@ -12,9 +12,11 @@ import { create } from 'tar'
 import fs from 'fs'
 import { guard } from '../util/assert'
 import { retrieveArchiveName } from './common/features'
-import { printStepResult } from '../core'
-import { StepOutputFormat } from '../core/print/print'
+import { printStepResult, StepOutputFormat } from '../core/print/print'
 import { date2string } from '../util/time'
+import { PARSE_WITH_R_SHELL_STEP } from '../core/steps/all/core/00-parse'
+import { NORMALIZE } from '../core/steps/all/core/10-normalize'
+import { LEGACY_STATIC_DATAFLOW } from '../core/steps/all/core/20-dataflow'
 
 // apps should never depend on other apps when forking (otherwise, they are "run" on load :/)
 
@@ -89,9 +91,9 @@ async function getStatsForSingleFile() {
 		if(options['dump-json']) {
 			const [, output] = [...stats.outputs.entries()][0]
 			const cfg = extractCFG(output.normalize)
-			statisticsFileProvider.append('output-json', 'parse', await printStepResult('parse', output.parse, StepOutputFormat.Json))
-			statisticsFileProvider.append('output-json', 'normalize', await printStepResult('normalize', output.normalize, StepOutputFormat.Json))
-			statisticsFileProvider.append('output-json', 'dataflow', await printStepResult('dataflow', output.dataflow, StepOutputFormat.Json))
+			statisticsFileProvider.append('output-json', 'parse', await printStepResult(PARSE_WITH_R_SHELL_STEP, output.parse, StepOutputFormat.Json))
+			statisticsFileProvider.append('output-json', 'normalize', await printStepResult(NORMALIZE, output.normalize, StepOutputFormat.Json))
+			statisticsFileProvider.append('output-json', 'dataflow', await printStepResult(LEGACY_STATIC_DATAFLOW, output.dataflow, StepOutputFormat.Json))
 			statisticsFileProvider.append('output-json', 'cfg', JSON.stringify(cfg, jsonReplacer))
 		}
 

src/core/index.ts

@@ -1,4 +1,4 @@
-export { SteppingSlicer } from './slicer'
-export * from './steps'
-export * from './input'
-export * from './output'
+export { SteppingSlicer } from './stepping-slicer'
+export * from './steps/steps'
+export * from './steps/input'
+export * from './steps/output'