Document the Normalized AST

.github/workflows/broken-links-and-wiki.yaml

@@ -78,6 +78,7 @@ jobs:
           update_wiki_page "Dataflow Graph" wiki:df-graph
           update_wiki_page "Query API" wiki:query-api
           update_wiki_page "Interface" wiki:interface
+          update_wiki_page "Normalized AST" wiki:normalized-ast
 
           if [ $CHANGED_ANY == "true" ]; then
               git config --local user.email "action@github.com"

package.json

@@ -26,6 +26,7 @@
     "export-quads": "ts-node src/cli/export-quads-app.ts",
     "capabilities-markdown": "ts-node src/documentation/print-capabilities-markdown.ts",
     "wiki:df-graph": "ts-node src/documentation/print-dataflow-graph-wiki.ts",
+    "wiki:normalized-ast": "ts-node src/documentation/print-normalized-ast-wiki.ts",
     "wiki:query-api": "ts-node src/documentation/print-query-wiki.ts",
     "wiki:interface": "ts-node src/documentation/print-interface-wiki.ts",
     "build": "tsc --project .",

src/cli/repl/server/messages/message-analysis.ts

@@ -1,4 +1,4 @@
-import type { IdMessageBase, MessageDefinition } from './messages';
+import type { IdMessageBase, MessageDefinition } from './all-messages';
 import Joi from 'joi';
 import type { ControlFlowInformation } from '../../../../util/cfg/cfg';
 import type { DEFAULT_DATAFLOW_PIPELINE, DEFAULT_SLICING_PIPELINE } from '../../../../core/steps/pipeline/default-pipelines';

src/cli/repl/server/messages/message-error.ts

@@ -1,4 +1,4 @@
-import type { IdMessageBase } from './messages';
+import type { IdMessageBase } from './all-messages';
 
 /**
  * Sent in case of any error (e.g., if the analysis fails, or the message contains syntax errors).

src/cli/repl/server/messages/message-hello.ts

@@ -1,5 +1,5 @@
 import type { VersionInformation } from '../../commands/repl-version';
-import type { IdMessageBase, MessageDefinition } from './messages';
+import type { IdMessageBase, MessageDefinition } from './all-messages';
 import Joi from 'joi';
 
 /**

src/cli/repl/server/messages/message-lineage.ts

@@ -1,5 +1,5 @@
 import type { SingleSlicingCriterion } from '../../../../slicing/criterion/parse';
-import type { IdMessageBase, MessageDefinition } from './messages';
+import type { IdMessageBase, MessageDefinition } from './all-messages';
 import type { NodeId } from '../../../../r-bridge/lang-4.x/ast/model/processing/node-id';
 import Joi from 'joi';
 

src/cli/repl/server/messages/message-query.ts

@@ -1,4 +1,4 @@
-import type { IdMessageBase, MessageDefinition } from './messages';
+import type { IdMessageBase, MessageDefinition } from './all-messages';
 import Joi from 'joi';
 import { QueriesSchema } from '../../../../queries/query-schema';
 import type { Queries,  QueryResults, SupportedQueryTypes } from '../../../../queries/query';

src/cli/repl/server/messages/message-repl.ts

@@ -1,4 +1,4 @@
-import type { IdMessageBase, MessageDefinition } from './messages';
+import type { IdMessageBase, MessageDefinition } from './all-messages';
 import * as Joi from 'joi';
 
 /**

src/cli/repl/server/messages/message-slice.ts

@@ -1,4 +1,4 @@
-import type { IdMessageBase, MessageDefinition } from './messages';
+import type { IdMessageBase, MessageDefinition } from './all-messages';
 import * as Joi from 'joi';
 import type { SlicingCriteria } from '../../../../slicing/criterion/parse';
 import type { PipelineOutput } from '../../../../core/steps/pipeline/pipeline';

src/cli/repl/server/send.ts

@@ -1,4 +1,4 @@
-import type { IdMessageBase } from './messages/messages';
+import type { IdMessageBase } from './messages/all-messages';
 import type { Socket } from './net';
 import { serverLog } from './server';
 import { jsonReplacer } from '../../../util/json';

src/cli/repl/server/validate.ts

@@ -1,7 +1,7 @@
 import type * as Joi from 'joi';
 import { sendMessage } from './send';
-import type { FlowrMessage, IdMessageBase, MessageDefinition } from './messages/messages';
-import { baseMessage } from './messages/messages';
+import type { FlowrMessage, IdMessageBase, MessageDefinition } from './messages/all-messages';
+import { baseMessage } from './messages/all-messages';
 import type { FlowrErrorMessage } from './messages/message-error';
 import type { Socket } from './net';
 

src/documentation/doc-util/doc-normalized-ast.ts

@@ -0,0 +1,86 @@
+import type { DataflowGraph } from '../../dataflow/graph/graph';
+import type { RShell } from '../../r-bridge/shell';
+import { PipelineExecutor } from '../../core/pipeline-executor';
+import { DEFAULT_DATAFLOW_PIPELINE, DEFAULT_NORMALIZE_PIPELINE } from '../../core/steps/pipeline/default-pipelines';
+import { requestFromInput } from '../../r-bridge/retriever';
+import type { RNodeWithParent } from '../../r-bridge/lang-4.x/ast/model/processing/decorate';
+import { deterministicCountingIdGenerator } from '../../r-bridge/lang-4.x/ast/model/processing/decorate';
+import { resolveDataflowGraph } from '../../dataflow/graph/resolve-graph';
+import type { DataflowDifferenceReport } from '../../dataflow/graph/diff';
+import { diffOfDataflowGraphs } from '../../dataflow/graph/diff';
+import { guard } from '../../util/assert';
+import { printAsMs } from './doc-ms';
+import { normalizedAstToMermaid } from '../../util/mermaid/ast';
+
+export function printNormalizedAst(ast: RNodeWithParent, prefix = 'flowchart TD\n') {
+	return `
+\`\`\`mermaid
+${normalizedAstToMermaid(ast, prefix)}
+\`\`\`
+	`;
+}
+
+export interface PrintNormalizedAstOptions {
+	readonly showCode?: boolean;
+	readonly prefix?:   string;
+}
+export async function printNormalizedAstForCode(shell: RShell, code: string, { showCode = true, prefix = 'flowchart TD\n' }: PrintNormalizedAstOptions = {}) {
+	const now = performance.now();
+	const result = await new PipelineExecutor(DEFAULT_NORMALIZE_PIPELINE, {
+		shell,
+		request: requestFromInput(code)
+	}).allRemainingSteps();
+	const duration = performance.now() - now;
+
+	const metaInfo = `The analysis required _${printAsMs(duration)}_ (including parsing) within the generation environment.`;
+
+	return '\n\n' +  printNormalizedAst(result.normalize.ast, prefix) + (showCode ? `
+<details>
+
+<summary style="color:gray">R Code of the Normalized AST</summary>
+
+${metaInfo}
+
+\`\`\`r
+${code}
+\`\`\`
+
+<details>
+
+<summary style="color:gray">Mermaid Code</summary>
+
+\`\`\`
+${normalizedAstToMermaid(result.normalize.ast, prefix)}
+\`\`\`
+
+</details>
+
+</details>
+
+` : '\n(' + metaInfo + ')\n\n')
+	;
+}
+
+
+/** returns resolved expected df graph */
+export async function verifyExpectedSubgraph(shell: RShell, code: string, expectedSubgraph: DataflowGraph): Promise<DataflowGraph> {
+	/* we verify that we get what we want first! */
+	const info = await new PipelineExecutor(DEFAULT_DATAFLOW_PIPELINE, {
+		shell,
+		request: requestFromInput(code),
+		getId:   deterministicCountingIdGenerator(0)
+	}).allRemainingSteps();
+
+	expectedSubgraph.setIdMap(info.normalize.idMap);
+	expectedSubgraph = resolveDataflowGraph(expectedSubgraph);
+	const report: DataflowDifferenceReport = diffOfDataflowGraphs(
+		{ name: 'expected', graph: expectedSubgraph },
+		{ name: 'got',      graph: info.dataflow.graph },
+		{
+			leftIsSubgraph: true
+		}
+	);
+
+	guard(report.isEqual(), () => `report:\n * ${report.comments()?.join('\n * ') ?? ''}`);
+	return expectedSubgraph;
+}

src/documentation/doc-util/doc-server-message.ts

@@ -2,7 +2,7 @@ import type { RShell } from '../../r-bridge/shell';
 import { getFilePathMd } from './doc-files';
 import { describeSchema } from '../../util/schema';
 import { markdownFormatter } from '../../util/ansi';
-import type { FlowrMessage, IdMessageBase, MessageDefinition } from '../../cli/repl/server/messages/messages';
+import type { FlowrMessage, IdMessageBase, MessageDefinition } from '../../cli/repl/server/messages/all-messages';
 import type { FakeServer, FakeSocket } from '../../../test/functionality/_helper/net';
 import { withSocket } from '../../../test/functionality/_helper/net';
 import { codeBlock } from './doc-code';