Dataflow v2

.githooks/check-package-update.sh

@@ -14,3 +14,4 @@ changed() {
 
 
 echo "$changedFiles" | (grep --quiet "package.json" && changed)
+exit 0

.github/workflows/check-broken-links.yaml → .github/workflows/broken-links-and-wiki.yaml

@@ -1,4 +1,4 @@
-name: Check for Broken Links
+name: Check for Broken Links and Publish Wiki
 
 'on':
   push:
@@ -17,10 +17,19 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - name: "Setup the Environment"
+        uses: Code-Inspect/flowr/.github/actions/setup@main
+
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
         with:
           lfs: true
 
+      - name: Update the Capabilities Wiki Page
+        run: |
+          npm ci
+          npm run capabilities-markdown --silent > wiki/Capabilities-New.md
+
       - name: Check the README for broken links
         uses: Wandalen/wretry.action@v1.3.0
         with:

package.json

@@ -22,6 +22,7 @@
     "benchmark": "npm run build && node dist/src/cli/benchmark-app.js",
     "summarizer": "ts-node src/cli/summarizer-app.ts",
     "export-quads": "ts-node src/cli/export-quads-app.ts",
+    "capabilities-markdown": "ts-node src/r-bridge/data/print.ts",
     "build": "tsc --project .",
     "lint-local": "eslint src/ test/ --rule \"no-warning-comments: off\"",
     "lint": "npm run license-compat -- --summary && eslint src/ test/",
@@ -44,20 +45,23 @@
   "mocha": {
     "require": "ts-node/register",
     "timeout": 60000,
-    "spec": "test/**/*.spec.ts"
+    "spec": "test/**/*.spec.ts",
+    "source-map": true,
+    "recursive": true,
+    "exit": true,
+    "produce-source-map": true
   },
   "nyc": {
     "all": true,
     "per-file": true,
     "check-coverage": false,
-    "skip-full": false,
+    "skip-full": true,
     "lines": 70,
     "extension": [
       ".ts"
     ],
     "include": "src/**/*.ts",
     "reporter": [
-      "html",
       "text",
       "lcov",
       "cobertura"
@@ -207,7 +211,7 @@
       "check-file/filename-naming-convention": [
         "error",
         {
-          "**/*.ts": "?([A-Z])+([a-z])*((-|.)?([A-Z])+([a-z]))"
+          "**/*.ts": "?([0-9]+-)?([A-Z])+([a-z])*((-|.)?([A-Z])+([a-z]))"
         }
       ],
       "check-file/folder-match-with-fex": [

scripts/normalize.R

@@ -0,0 +1,84 @@
+# TODO normalize, decorate etc.
+input <- file("../test/testfiles/example-cfg.R")
+exprs <- rlang::parse_exprs(input)
+
+# the conversion code is based on lazyeval::ast_
+
+# currently no json at all :D
+flowr.expr_to_json <- function(x) {
+  if (base::is.expression(x) || base::is.list(x)) {
+    trees <- base::vapply(x, flowr.tree, character(1))
+    out <- base::paste0(trees, collapse = "\n\n")
+  } else {
+    out <- flowr.tree(x)
+  }
+
+  cat(out, "\n")
+}
+
+flowr.is_atomic <- function(x) {
+  typeof(x) %in% c("logical", "integer", "double", "complex", "character", "raw")
+}
+
+flowr.is_name <- function(x) {
+  typeof(x) == "symbol"
+}
+
+flowr.is_call <- function(x) {
+  typeof(x) == "language"
+}
+
+flowr.is_pairlist <- function(x) {
+  typeof(x) == "pairlist"
+}
+
+
+flowr.tree <- function(x, level = 1) {
+  if (flowr.is_atomic(x) && base::length(x) == 1) {
+    label <- base::paste0(" ", base::deparse(x)[1])
+    children <- NULL
+  } else if (flowr.is_name(x)) {
+    x <- base::as.character(x)
+    if (x == "") {
+      # Special case the missing argument
+      label <- "`MISSING"
+    } else {
+      label <- base::paste0("`", base::as.character(x))
+    }
+
+    children <- NULL
+  } else if (flowr.is_call(x)) {
+    label <- "()"
+    children <- base::vapply(base::as.list(x), flowr.tree, character(1), level = level + 1)
+  } else if (flowr.is_pairlist(x)) {
+    label <- "[]"
+
+    branches <- paste("\u2517", format(names(x)), "=")
+    children <- character(length(x))
+    for (i in seq_along(x)) {
+      children[i] <- flowr.tree(x[[i]], level = level + 1)
+    }
+  } else {
+    # Special case for srcrefs, since they're commonly seen
+    if (inherits(x, "srcref")) {
+      label <- "<srcref>"
+    } else {
+      label <- paste0("<", typeof(x), ">")
+    }
+    children <- NULL
+  }
+
+  indent <- paste0(str_dup(" ", level - 1), "> ")
+
+  if (is.null(children)) {
+    paste0(indent, label)
+  } else {
+    paste0(indent, label, "\n", paste0(children, collapse = "\n"))
+  }
+}
+
+str_dup <- function(x, n) {
+  paste0(rep(x, n), collapse = "")
+}
+
+print(flowr.expr_to_json(exprs))

src/abstract-interpretation/handler/binop/binop.ts

@@ -1,13 +1,11 @@
 import type { Handler } from '../handler'
 import type { AINode } from '../../processor'
 import { aiLogger } from '../../processor'
-import type { BinaryOperatorFlavor, ParentInformation, RBinaryOp } from '../../../r-bridge'
+import type { ParentInformation, RBinaryOp } from '../../../r-bridge'
 import { guard } from '../../../util/assert'
 import { operators } from './operators'
 
-export type BinOpOperators = {
-	[key in BinaryOperatorFlavor]: (lhs: AINode, rhs: AINode, node: RBinaryOp<ParentInformation>) => AINode
-}
+export type BinaryOpProcessor = (lhs: AINode, rhs: AINode, node: RBinaryOp<ParentInformation>) => AINode
 
 export class BinOp implements Handler<AINode> {
 	lhs: AINode | undefined
@@ -16,7 +14,7 @@ export class BinOp implements Handler<AINode> {
 	constructor(readonly node: RBinaryOp<ParentInformation>) {}
 
 	getName(): string {
-		return `Bin Op (${this.node.flavor})`
+		return `Bin Op (${this.node.operator})`
 	}
 
 	enter(): void {
@@ -27,7 +25,9 @@ export class BinOp implements Handler<AINode> {
 		aiLogger.trace(`Exited ${this.getName()}`)
 		guard(this.lhs !== undefined, `No LHS found for assignment ${this.node.info.id}`)
 		guard(this.rhs !== undefined, `No RHS found for assignment ${this.node.info.id}`)
-		return operators[this.node.flavor](this.lhs, this.rhs, this.node)
+		const processor: BinaryOpProcessor | undefined = operators[this.node.operator]
+		guard(processor !== undefined, `No processor found for binary operator ${this.node.operator}`)
+		return processor(this.lhs, this.rhs, this.node)
 	}
 
 	next(node: AINode): void {
@@ -40,4 +40,4 @@ export class BinOp implements Handler<AINode> {
 			guard(false, `BinOp ${this.node.info.id} already has both LHS and RHS`)
 		}
 	}
-}
+}

src/abstract-interpretation/handler/binop/operators.ts

@@ -1,40 +1,26 @@
-import { guard } from '../../../util/assert'
-import type { BinOpOperators } from './binop'
+import type { BinaryOpProcessor } from './binop'
 import { addDomains, subtractDomains } from '../../domain'
 
-export const operators: BinOpOperators = {
-	'assignment': (lhs, rhs, node) => {
+export const operators: Record<string, BinaryOpProcessor> = {
+	'<-': (lhs, rhs, node) => {
 		return {
 			id:      lhs.id,
 			domain:  rhs.domain,
 			astNode: node.lhs,
 		}
 	},
-	'arithmetic': (lhs, rhs, node) => {
-		switch(node.operator) {
-			case '+':
-				return {
-					id:      lhs.id,
-					domain:  addDomains(lhs.domain, rhs.domain),
-					astNode: node,
-				}
-			case '-':
-				return {
-					id:      lhs.id,
-					domain:  subtractDomains(lhs.domain, rhs.domain),
-					astNode: node,
-				}
-			default:
-				guard(false, `Unknown binary operator ${node.operator}`)
+	'+': (lhs, rhs, node) => {
+		return {
+			id:      lhs.id,
+			domain:  addDomains(lhs.domain, rhs.domain),
+			astNode: node,
 		}
 	},
-	'logical': () => {
-		guard(false, 'Not implemented yet')
-	},
-	'model formula': () => {
-		guard(false, 'Not implemented yet')
-	},
-	'comparison': () => {
-		guard(false, 'Not implemented yet')
-	},
+	'-': (lhs, rhs, node) => {
+		return {
+			id:      lhs.id,
+			domain:  subtractDomains(lhs.domain, rhs.domain),
+			astNode: node,
+		}
+	}
 }

src/abstract-interpretation/processor.ts

@@ -1,4 +1,4 @@
-import type { DataflowInformation } from '../dataflow/internal/info'
+import type { DataflowInformation } from '../dataflow/info'
 import type { NodeId, NormalizedAst, ParentInformation, RNodeWithParent } from '../r-bridge'
 import { RType } from '../r-bridge'
 import { CfgVertexType, extractCFG } from '../util/cfg/cfg'
@@ -23,13 +23,13 @@ class Stack<ElementType> {
 	private backingStore: ElementType[] = []
 
 	size(): number {
-		return this.backingStore.length 
+		return this.backingStore.length
 	}
 	peek(): ElementType | undefined {
-		return this.backingStore[this.size() - 1] 
+		return this.backingStore[this.size() - 1]
 	}
 	pop(): ElementType | undefined {
-		return this.backingStore.pop() 
+		return this.backingStore.pop()
 	}
 	push(item: ElementType): ElementType {
 		this.backingStore.push(item)

src/benchmark/slicer.ts → src/benchmark/benchmark-slicer.ts

@@ -2,7 +2,6 @@
  * Provides a top-level slicer that can be used to slice code *and* retrieve stats.
  * @module
  */
-
 import type {
 	NormalizedAst,
 	RParseRequestFromFile, RParseRequestFromText
@@ -15,7 +14,7 @@ import {
 import type { IStoppableStopwatch } from './stopwatch'
 import { Measurements } from './stopwatch'
 import { guard } from '../util/assert'
-import type { DataflowInformation } from '../dataflow/internal/info'
+import type { DataflowInformation } from '../dataflow/info'
 import type {
 	SlicingCriteria,
 	SlicingCriteriaFilter,
@@ -32,11 +31,12 @@ import type {
 	SlicerStats
 } from './stats'
 import fs from 'fs'
-import { log, LogLevel } from '../util/log'
+import { expensiveTrace, log, LogLevel } from '../util/log'
 import type { MergeableRecord } from '../util/objects'
-import type { STEPS, StepResult } from '../core'
-import { LAST_STEP, SteppingSlicer } from '../core'
 import { withoutWhitespace } from '../util/strings'
+import { SteppingSlicer } from '../core/stepping-slicer'
+import type { StepResult, STEPS } from '../core/steps/steps'
+import { LAST_STEP } from '../core/steps/steps'
 
 export const benchmarkLogger = log.getSubLogger({ name: 'benchmark' })
 
@@ -81,23 +81,24 @@ export class BenchmarkSlicer {
 	/** Measures all data that is recorded *once* per slicer (complete setup up to the dataflow graph creation) */
 	private readonly commonMeasurements = new Measurements<CommonSlicerMeasurements>()
 	private readonly perSliceMeasurements = new Map<SlicingCriteria, PerSliceStats>()
-	private readonly shell: RShell
-	private stats:          SlicerStats | undefined
-	private loadedXml:      string | undefined
-	private dataflow:       DataflowInformation | undefined
-	private ai:             DataflowInformation | undefined
-	private normalizedAst:  NormalizedAst | undefined
-	private totalStopwatch: IStoppableStopwatch
+	private readonly shell:    RShell
+	private stats:             SlicerStats | undefined
+	private loadedData:        string | undefined
+	private dataflow:          DataflowInformation | undefined
+	private normalizedAst:     NormalizedAst | undefined
+	private totalStopwatch:    IStoppableStopwatch
+	private readonly waitTime: number
 	private finished = false
 	// Yes this is dirty, but we know that we assign the stepper during the initialization and this saves us from having to check for nullability every time
-	private stepper:        SteppingSlicer = null as unknown as SteppingSlicer
+	private stepper:           SteppingSlicer = null as unknown as SteppingSlicer
 
-	constructor() {
+	constructor(waitTime = 750) {
 		this.totalStopwatch = this.commonMeasurements.start('total')
 		this.shell = this.commonMeasurements.measure(
 			'initialize R session',
 			() => new RShell()
 		)
+		this.waitTime = waitTime
 	}
 
 	/**
@@ -106,6 +107,8 @@ export class BenchmarkSlicer {
 	 */
 	public async init(request: RParseRequestFromFile | RParseRequestFromText) {
 		guard(this.stats === undefined, 'cannot initialize the slicer twice')
+		// wait so the R session can catch up and to measure the time	more fairly
+		await new Promise(resolve => setTimeout(resolve, this.waitTime))
 
 		this.stepper = new SteppingSlicer({
 			shell:          this.shell,
@@ -114,7 +117,8 @@ export class BenchmarkSlicer {
 			criterion:      []
 		})
 
-		this.loadedXml = await this.measureCommonStep('parse', 'retrieve AST from R code')
+		this.loadedData = await this.measureCommonStep('parse', 'retrieve AST from R code')
+
 		this.normalizedAst = await this.measureCommonStep('normalize', 'normalize R AST')
 		this.dataflow = await this.measureCommonStep('dataflow', 'produce dataflow information')
 
@@ -174,7 +178,7 @@ export class BenchmarkSlicer {
    * @returns The per slice stats retrieved for this slicing criteria
    */
 	public async slice(...slicingCriteria: SlicingCriteria): Promise<BenchmarkSingleSliceStats> {
-		benchmarkLogger.trace(`try to slice for criteria ${JSON.stringify(slicingCriteria)}`)
+		expensiveTrace(benchmarkLogger, () => `try to slice for criteria ${JSON.stringify(slicingCriteria)}`)
 
 		this.guardActive()
 		guard(!this.perSliceMeasurements.has(slicingCriteria), 'do not slice the same criteria combination twice')
@@ -205,10 +209,10 @@ export class BenchmarkSlicer {
 
 		totalStopwatch.stop()
 
-		benchmarkLogger.debug(`Produced code for ${JSON.stringify(slicingCriteria)}: ${stats.reconstructedCode.code}`)
+		expensiveTrace(benchmarkLogger, () => `Produced code for ${JSON.stringify(slicingCriteria)}: ${stats.reconstructedCode.code}`)
 		const results = this.stepper.getResults(false)
 
-		if(benchmarkLogger.settings.minLevel >= LogLevel.Info) {
+		if(benchmarkLogger.settings.minLevel <= LogLevel.Info) {
 			benchmarkLogger.info(`mapped slicing criteria: ${slicedOutput.decodedCriteria.map(c => {
 				const node = results.normalize.idMap.get(c.id)
 				return `\n-   id: ${c.id}, location: ${JSON.stringify(node?.location)}, lexeme: ${JSON.stringify(node?.lexeme)}`
@@ -290,7 +294,7 @@ export class BenchmarkSlicer {
 		this.stats.commonMeasurements = this.commonMeasurements.get()
 		return {
 			stats:     this.stats,
-			parse:     this.loadedXml as string,
+			parse:     this.loadedData as string,
 			dataflow:  this.dataflow as DataflowInformation,
 			normalize: this.normalizedAst as NormalizedAst
 		}