Feat: Follow up on source calls (#609)

* wip: added testfile & started on environment (??)

* wip: some todos

* wip: start of source argument resolve

* wip: added testfile & started on environment (??)

* wip: some todos

* wip: start of source argument resolve

* refactor: start using sync r shell implementation

* wip: rebase

* refactor: start using sync r shell implementation

* wip: use executeSingleSubStep for parsing sourced code

* wip: added testfile & started on environment (??)

* wip: some todos

* wip: start of source argument resolve

* wip: rebase

* refactor: start using sync r shell implementation

* wip: some todos

* wip: use executeSingleSubStep for parsing sourced code

* wip: fix merge issues

* feat-fix: avoid cyclic dependency when using step executor

* wip: run normalize and dataflow on sourced file

* wip: some work on source dataflowing

* refactor: remove print

* refactor: clean up todos and move source to its own function

* refactor: explicitly as in processSourceCall

* refactor: damn u typescript

* feat-fix: ensure we only parse built-in source calls

* refactor: remove todo

* feat: allow overriding the source file provider

* test: start on source tests

* refactor: overhaul source providers

* refactor: generify source providers to RParseRequestProvider

* test: added test for conditional source

* refactor: properly handle missing/invalid sourced files

* wip: test for recursive sources

* feat: skip dataflow analysis for re-sourced references

* wip: add another todo

* refactor: use parse requests in dataflow processor info

* refactor: first pass of reference chain impl

* feat-fix: also catch normalize and dataflow errors

* test: finished recursive source test

* test: added test for non-constant source argument

* test: added multi-source test

* feat-fix: sourcing multiple files works correctly now

* refactor: resolve review comments

* test: reset the source provider to the default value after each describe

* test-fix: reset the source provider in the source describe instead

---------

Co-authored-by: Florian Sihler <florian.sihler@uni-ulm.de>

src/cli/repl/commands/parse.ts

@@ -135,7 +135,7 @@ export const parseCommand: ReplCommand = {
 		}).allRemainingSteps()
 
 		const config = deepMergeObject<XmlParserConfig>(DEFAULT_XML_PARSER_CONFIG, { tokenMap: await shell.tokenMap() })
-		const object = await xlm2jsonObject(config, result.parse)
+		const object = xlm2jsonObject(config, result.parse)
 
 		output.stdout(depthListToTextTree(toDepthMap(object, config), config, output.formatter))
 	}

src/core/print/parse-printer.ts

@@ -24,8 +24,8 @@ function filterObject(obj: XmlBasedJson, keys: Set<string>): XmlBasedJson[] | Xm
 
 }
 
-export async function parseToQuads(code: string, config: QuadSerializationConfiguration, parseConfig: XmlParserConfig): Promise<string> {
-	const obj = await xlm2jsonObject(parseConfig, code)
+export function parseToQuads(code: string, config: QuadSerializationConfiguration, parseConfig: XmlParserConfig): string{
+	const obj = xlm2jsonObject(parseConfig, code)
 	// recursively filter so that if the object contains one of the keys 'a', 'b' or 'c', all other keys are ignored
 	return serialize2quads(
 		filterObject(obj, new Set([parseConfig.attributeName, parseConfig.childrenName, parseConfig.contentName])) as XmlBasedJson,

src/core/slicer.ts

@@ -204,11 +204,11 @@ export class SteppingSlicer<InterestedIn extends StepName | undefined = typeof L
 				break
 			case 1:
 				step = guardStep('normalize')
-				result = await executeSingleSubStep(step, this.results.parse as string, await this.shell.tokenMap(), this.hooks, this.getId)
+				result = executeSingleSubStep(step, this.results.parse as string, await this.shell.tokenMap(), this.hooks, this.getId)
 				break
 			case 2:
 				step = guardStep('dataflow')
-				result = executeSingleSubStep(step, this.results.normalize as NormalizedAst)
+				result = executeSingleSubStep(step, this.request, this.results.normalize as NormalizedAst)
 				break
 			case 3:
 				step = guardStep('ai')

src/core/steps.ts

@@ -96,7 +96,7 @@ export const STEPS_PER_FILE = {
 	} satisfies IStep<typeof normalize>,
 	'dataflow': {
 		description: 'Construct the dataflow graph',
-		processor:   produceDataFlowGraph,
+		processor:   (r, a) => produceDataFlowGraph(r, a),
 		required:    'once-per-file',
 		printer:     {
 			[StepOutputFormat.Internal]:   internalPrinter,

src/dataflow/environments/environment.ts

@@ -147,6 +147,14 @@ export const DefaultEnvironmentMemory = new Map<Identifier, IdentifierDefinition
 		definedAt: BuiltIn,
 		name:      'print',
 		nodeId:    BuiltIn
+	}]],
+	['source', [{
+		kind:      'built-in-function',
+		scope:     GlobalScope,
+		used:      'always',
+		definedAt: BuiltIn,
+		name:      'source',
+		nodeId:    BuiltIn
 	}]]
 ])
 

src/dataflow/extractor.ts

@@ -1,4 +1,5 @@
-import type { NormalizedAst, ParentInformation, RAssignmentOp, RBinaryOp} from '../r-bridge'
+import type {NormalizedAst, ParentInformation, RAssignmentOp, RBinaryOp, RParseRequest} from '../r-bridge'
+import { requestFingerprint} from '../r-bridge'
 import { RType } from '../r-bridge'
 import type { DataflowInformation } from './internal/info'
 import type { DataflowProcessorInformation, DataflowProcessors} from './processor'
@@ -48,8 +49,15 @@ const processors: DataflowProcessors<any> = {
 	[RType.ExpressionList]:     processExpressionList,
 }
 
-export function produceDataFlowGraph<OtherInfo>(ast: NormalizedAst<OtherInfo & ParentInformation>, initialScope: DataflowScopeName = LocalScope): DataflowInformation {
-	return processDataflowFor<OtherInfo>(ast.ast, { completeAst: ast, activeScope: initialScope, environments: initializeCleanEnvironments(), processors: processors as DataflowProcessors<OtherInfo & ParentInformation> })
+export function produceDataFlowGraph<OtherInfo>(request: RParseRequest, ast: NormalizedAst<OtherInfo & ParentInformation>, initialScope: DataflowScopeName = LocalScope): DataflowInformation {
+	return processDataflowFor<OtherInfo>(ast.ast, {
+		completeAst:    ast,
+		activeScope:    initialScope,
+		environments:   initializeCleanEnvironments(),
+		processors:     processors as DataflowProcessors<OtherInfo & ParentInformation>,
+		currentRequest: request,
+		referenceChain: [requestFingerprint(request)]
+	})
 }
 
 export function processBinaryOp<OtherInfo>(node: RBinaryOp<OtherInfo & ParentInformation>, data: DataflowProcessorInformation<OtherInfo & ParentInformation>) {

src/dataflow/internal/process/functions/function-call.ts

@@ -1,14 +1,15 @@
 import type { DataflowInformation } from '../../info'
-import type { DataflowProcessorInformation} from '../../../processor'
+import type {DataflowProcessorInformation} from '../../../processor'
 import { processDataflowFor } from '../../../processor'
-import { define, overwriteEnvironments, resolveByName } from '../../../environments'
-import type { ParentInformation, RFunctionCall} from '../../../../r-bridge'
-import { RType } from '../../../../r-bridge'
+import {define, overwriteEnvironments, resolveByName} from '../../../environments'
+import type {ParentInformation, RFunctionCall} from '../../../../r-bridge'
+import { RType} from '../../../../r-bridge'
 import { guard } from '../../../../util/assert'
-import type { FunctionArgument } from '../../../index'
+import type {FunctionArgument} from '../../../index'
 import { DataflowGraph, dataflowLogger, EdgeType } from '../../../index'
 import { linkArgumentsOnCall } from '../../linker'
 import { LocalScope } from '../../../environments/scopes'
+import {isSourceCall, processSourceCall} from './source'
 
 export const UnnamedFunctionCallPrefix = 'unnamed-function-call-'
 
@@ -40,7 +41,6 @@ export function processFunctionCall<OtherInfo>(functionCall: RFunctionCall<Other
 		finalGraph.mergeWith(functionName.graph)
 	}
 
-
 	for(const arg of functionCall.arguments) {
 		if(arg === undefined) {
 			callArgs.push('empty')
@@ -107,13 +107,19 @@ export function processFunctionCall<OtherInfo>(functionCall: RFunctionCall<Other
 		inIds.push(...functionName.in, ...functionName.unknownReferences)
 	}
 
-	return {
+	let info: DataflowInformation = {
 		unknownReferences: [],
 		in:                inIds,
 		out:               functionName.out, // we do not keep argument out as it has been linked by the function
 		graph:             finalGraph,
 		environments:      finalEnv,
 		scope:             data.activeScope
 	}
-}
 
+	// parse a source call and analyze the referenced code
+	if(isSourceCall(functionCallName, data.activeScope,finalEnv)) {
+		info = processSourceCall(functionCall, data, info)
+	}
+
+	return info
+}

src/dataflow/internal/process/functions/source.ts

@@ -0,0 +1,80 @@
+import type {IdGenerator, NoInfo, RArgument, RParseRequest, RParseRequestProvider} from '../../../../r-bridge'
+import { requestFingerprint} from '../../../../r-bridge'
+import { sourcedDeterministicCountingIdGenerator} from '../../../../r-bridge'
+import {requestProviderFromFile} from '../../../../r-bridge'
+import {type NormalizedAst, type ParentInformation, removeTokenMapQuotationMarks, type RFunctionCall, RType} from '../../../../r-bridge'
+import {RShellExecutor} from '../../../../r-bridge/shell-executor'
+import {executeSingleSubStep} from '../../../../core'
+import {type DataflowProcessorInformation, processDataflowFor} from '../../../processor'
+import {type DataflowScopeName, type Identifier, overwriteEnvironments, type REnvironmentInformation, resolveByName} from '../../../environments'
+import type {DataflowInformation} from '../../info'
+import {dataflowLogger} from '../../../index'
+
+let sourceProvider = requestProviderFromFile()
+
+export function setSourceProvider(provider: RParseRequestProvider): void {
+	sourceProvider = provider
+}
+
+export function isSourceCall(name: Identifier, scope: DataflowScopeName, environments: REnvironmentInformation): boolean {
+	const definitions = resolveByName(name, scope, environments)
+	if(definitions === undefined) {
+		return false
+	}
+	// fail if there are multiple definitions because then we must treat the complete import as a maybe because it might do something different
+	if(definitions.length !== 1) {
+		return false
+	}
+	const def = definitions[0]
+	return def.name == 'source' && def.kind == 'built-in-function'
+}
+
+export function processSourceCall<OtherInfo>(functionCall: RFunctionCall<OtherInfo & ParentInformation>, data: DataflowProcessorInformation<OtherInfo & ParentInformation>, information: DataflowInformation): DataflowInformation {
+	const sourceFile = functionCall.arguments[0] as RArgument<ParentInformation> | undefined
+	if(sourceFile?.value?.type == RType.String) {
+		const path = removeTokenMapQuotationMarks(sourceFile.lexeme)
+		const request = sourceProvider.createRequest(path)
+
+		// check if the sourced file has already been dataflow analyzed, and if so, skip it
+		if(data.referenceChain.includes(requestFingerprint(request))) {
+			dataflowLogger.info(`Found loop in dataflow analysis for ${JSON.stringify(request)}: ${JSON.stringify(data.referenceChain)}, skipping further dataflow analysis`)
+			return information
+		}
+
+		return sourceRequest(request, data, information, sourcedDeterministicCountingIdGenerator(path, functionCall.location))
+	} else {
+		dataflowLogger.info(`Non-constant argument ${JSON.stringify(sourceFile)} for source is currently not supported, skipping`)
+		return information
+	}
+}
+
+export function sourceRequest<OtherInfo>(request: RParseRequest, data: DataflowProcessorInformation<OtherInfo & ParentInformation>, information: DataflowInformation, getId: IdGenerator<NoInfo>): DataflowInformation {
+	const executor = new RShellExecutor()
+
+	// parse, normalize and dataflow the sourced file
+	let normalized: NormalizedAst<OtherInfo & ParentInformation>
+	let dataflow: DataflowInformation
+	try {
+		const parsed = executeSingleSubStep('parse', request, executor) as string
+		normalized = executeSingleSubStep('normalize', parsed, executor.getTokenMap(), undefined, getId) as NormalizedAst<OtherInfo & ParentInformation>
+		dataflow = processDataflowFor(normalized.ast, {
+			...data,
+			currentRequest: request,
+			environments:   information.environments,
+			referenceChain: [...data.referenceChain, requestFingerprint(request)]
+		})
+	} catch(e) {
+		dataflowLogger.warn(`Failed to analyze sourced file ${JSON.stringify(request)}, skipping: ${(e as Error).message}`)
+		return information
+	}
+
+	// update our graph with the sourced file's information
+	const newInformation = {...information}
+	newInformation.environments = overwriteEnvironments(information.environments, dataflow.environments)
+	newInformation.graph.mergeWith(dataflow.graph)
+	// this can be improved, see issue #628
+	for(const [k, v] of normalized.idMap) {
+		data.completeAst.idMap.set(k, v)
+	}
+	return newInformation
+}

src/dataflow/processor.ts

@@ -4,7 +4,7 @@
 import type {
 	NormalizedAst,
 	ParentInformation, RNode,
-	RNodeWithParent
+	RNodeWithParent, RParseRequest
 } from '../r-bridge'
 import type { DataflowInformation } from './internal/info'
 import type { DataflowScopeName, REnvironmentInformation } from './environments'
@@ -13,20 +13,29 @@ export interface DataflowProcessorInformation<OtherInfo> {
 	/**
    * Initial and frozen ast-information
    */
-	readonly completeAst:  NormalizedAst<OtherInfo>
+	readonly completeAst:    NormalizedAst<OtherInfo>
 	/**
    * Correctly contains pushed local scopes introduced by `function` scopes.
    * Will by default *not* contain any symbol-bindings introduces along the way, they have to be decorated when moving up the tree.
    */
-	readonly environments: REnvironmentInformation
+	readonly environments:   REnvironmentInformation
 	/**
    * Name of the currently active scope, (hopefully) always {@link LocalScope | Local}
    */
-	readonly activeScope:  DataflowScopeName
+	readonly activeScope:    DataflowScopeName
 	/**
    * Other processors to be called by the given functions
    */
-	readonly processors:   DataflowProcessors<OtherInfo>
+	readonly processors:     DataflowProcessors<OtherInfo>
+	/**
+	 * The {@link RParseRequest} that is currently being parsed
+	 */
+	readonly currentRequest: RParseRequest
+	/**
+	 * The chain of {@link RParseRequest} fingerprints ({@link requestFingerprint}) that lead to the {@link currentRequest}.
+	 * The most recent (last) entry is expected to always be the {@link currentRequest}.
+	 */
+	readonly referenceChain: string[]
 }
 
 export type DataflowProcessor<OtherInfo, NodeType extends RNodeWithParent<OtherInfo>> = (node: NodeType, data: DataflowProcessorInformation<OtherInfo>) => DataflowInformation
@@ -55,6 +64,3 @@ export type DataflowProcessors<OtherInfo> = {
 export function processDataflowFor<OtherInfo>(current: RNodeWithParent<OtherInfo>, data: DataflowProcessorInformation<OtherInfo & ParentInformation>): DataflowInformation {
 	return data.processors[current.type](current as never, data)
 }
-
-
-

src/r-bridge/lang-4.x/ast/model/processing/decorate.ts

@@ -46,6 +46,11 @@ export function deterministicCountingIdGenerator(start = 0): () => NodeId {
 	return () => `${id++}`
 }
 
+export function sourcedDeterministicCountingIdGenerator(path: string, location: SourceRange, start = 0): () => NodeId {
+	let id = start
+	return () => `${path}-${loc2Id(location)}-${id++}`
+}
+
 function loc2Id(loc: SourceRange) {
 	return `${loc.start.line}:${loc.start.column}-${loc.end.line}:${loc.end.column}`
 }

src/r-bridge/lang-4.x/ast/parser/xml/internal/xml-to-json.ts

@@ -8,8 +8,11 @@ import type { XmlBasedJson } from '../input-format'
  * @param config    - The configuration to use (i.e., what names should be used for the attributes, children, ...)
  * @param xmlString - The xml input to parse
  */
-export function xlm2jsonObject(config: XmlParserConfig, xmlString: string): Promise<XmlBasedJson> {
-	return xml2js.parseStringPromise(xmlString, {
+export function xlm2jsonObject(config: XmlParserConfig, xmlString: string): XmlBasedJson {
+	let result: XmlBasedJson = {}
+	xml2js.parseString(xmlString, {
+		// we want this to be strictly synchronous so that the result can be returned immediately below!
+		async:                 false,
 		attrkey:               config.attributeName,
 		charkey:               config.contentName,
 		childkey:              config.childrenName,
@@ -22,5 +25,6 @@ export function xlm2jsonObject(config: XmlParserConfig, xmlString: string): Prom
 		includeWhiteChars:     true,
 		normalize:             false,
 		strict:                true
-	}) as Promise<XmlBasedJson>
+	}, (_, r)=> result = r as XmlBasedJson)
+	return result 
 }

src/r-bridge/lang-4.x/ast/parser/xml/parser.ts

@@ -30,12 +30,12 @@ export const parseLog = log.getSubLogger({ name: 'ast-parser' })
  *
  * @returns The normalized and decorated AST (i.e., as a doubly linked tree)
  */
-export async function normalize(xmlString: string, tokenMap: TokenMap, hooks?: DeepPartial<XmlParserHooks>, getId: IdGenerator<NoInfo> = deterministicCountingIdGenerator(0)): Promise<NormalizedAst> {
+export function normalize(xmlString: string, tokenMap: TokenMap, hooks?: DeepPartial<XmlParserHooks>, getId: IdGenerator<NoInfo> = deterministicCountingIdGenerator(0)): NormalizedAst {
 	const config = { ...DEFAULT_XML_PARSER_CONFIG, tokenMap }
 	const hooksWithDefaults = deepMergeObject(DEFAULT_PARSER_HOOKS, hooks) as XmlParserHooks
 
 	const data: ParserData = { config, hooks: hooksWithDefaults, currentRange: undefined, currentLexeme: undefined }
-	const object = await xlm2jsonObject(config, xmlString)
+	const object = xlm2jsonObject(config, xmlString)
 
 	return decorateAst(parseRootObjToAst(data, object), getId)
 }