feat: pre-load all relevant workspace files before executing pipeline

packages/safe-ds-vscode/src/extension/mainClient.ts

@@ -12,12 +12,12 @@ import {
  stopPythonServer,
  tryMapToSafeDSSource,
 } from './pythonServer.js';
-import { createSafeDsServicesWithBuiltins, SafeDsServices } from '@safe-ds/lang';
+import { ast, createSafeDsServicesWithBuiltins, SafeDsServices } from '@safe-ds/lang';
 import { NodeFileSystem } from 'langium/node';
-import { getSafeDSOutputChannel, initializeLog, logOutput, printOutputMessage } from './output.js';
+import { getSafeDSOutputChannel, initializeLog, logError, logOutput, printOutputMessage } from './output.js';
 import { createPlaceholderQueryMessage, RuntimeErrorMessage } from './messages.js';
 import crypto from 'crypto';
-import { URI } from 'langium';
+import { LangiumDocument, URI } from 'langium';
 
 let client: LanguageClient;
 let services: SafeDsServices;
@@ -82,6 +82,14 @@ const startLanguageClient = function (context: vscode.ExtensionContext): Languag
 
 const acceptRunRequests = function (context: vscode.ExtensionContext) {
  // Register logging message callbacks
+ registerMessageLoggingCallbacks();
+ // Register VS Code Entry Points
+ registerVSCodeCommands(context);
+ // Register watchers
+ registerVSCodeWatchers();
+};
+
+const registerMessageLoggingCallbacks = function () {
  addMessageCallback((message) => {
  printOutputMessage(
  `Placeholder value is (${message.id}): ${message.data.name} of type ${message.data.type} = ${message.data.value}`,
@@ -126,32 +134,105 @@ const acceptRunRequests = function (context: vscode.ExtensionContext) {
  .join('\n')}`,
  );
  }, 'runtime_error');
- // Register VS Code Entry Points
+};
+
+const registerVSCodeCommands = function (context: vscode.ExtensionContext) {
  context.subscriptions.push(
- vscode.commands.registerCommand('extension.safe-ds.runPipelineFile', (filePath: vscode.Uri | undefined) => {
- let pipelinePath = filePath;
- // Allow execution via command menu
- if (!pipelinePath && vscode.window.activeTextEditor) {
- pipelinePath = vscode.window.activeTextEditor.document.uri;
- }
- if (
- pipelinePath &&
- !services.LanguageMetaData.fileExtensions.some((extension: string) =>
- pipelinePath!.fsPath.endsWith(extension),
- )
- ) {
- vscode.window.showErrorMessage(`Could not run ${pipelinePath!.fsPath} as it is not a Safe-DS file`);
- return;
- }
- if (!pipelinePath) {
- vscode.window.showErrorMessage('Could not run Safe-DS Pipeline, as no pipeline is currently selected.');
- return;
- }
- const pipelineId = crypto.randomUUID();
- printOutputMessage(`Launching Pipeline (${pipelineId}): ${pipelinePath}`);
- executePipeline(services, pipelinePath.fsPath, pipelineId);
- }),
+ vscode.commands.registerCommand('extension.safe-ds.runPipelineFile', commandRunPipelineFile),
+ );
+};
+
+const commandRunPipelineFile = async function (filePath: vscode.Uri | undefined) {
+ let pipelinePath = filePath;
+ // Allow execution via command menu
+ if (!pipelinePath && vscode.window.activeTextEditor) {
+ pipelinePath = vscode.window.activeTextEditor.document.uri;
+ }
+ if (
+ pipelinePath &&
+ !services.LanguageMetaData.fileExtensions.some((extension: string) => pipelinePath!.fsPath.endsWith(extension))
+ ) {
+ vscode.window.showErrorMessage(`Could not run ${pipelinePath!.fsPath} as it is not a Safe-DS file`);
+ return;
+ }
+ if (!pipelinePath) {
+ vscode.window.showErrorMessage('Could not run Safe-DS Pipeline, as no pipeline is currently selected.');
+ return;
+ }
+ // Refresh workspace
+ // Do not delete builtins
+ services.shared.workspace.LangiumDocuments.all
+ .filter(
+ (document) =>
+ !(
+ ast.isSdsModule(document.parseResult.value) &&
+ (<ast.SdsModule>document.parseResult.value).name === 'safeds.lang'
+ ),
+ )
+ .forEach((oldDocument) => {
+ services.shared.workspace.LangiumDocuments.deleteDocument(oldDocument.uri);
+ });
+ const workspaceSdsFiles = await vscode.workspace.findFiles('**/*.{sdspipe,sdsstub,sdstest}');
+ // Load all documents
+ const unvalidatedSdsDocuments = workspaceSdsFiles.map((newDocumentUri) =>
+ services.shared.workspace.LangiumDocuments.getOrCreateDocument(newDocumentUri),
  );
+ // Validate them
+ const validationErrorMessage = await validateDocuments(services, unvalidatedSdsDocuments);
+ if (validationErrorMessage) {
+ vscode.window.showErrorMessage(validationErrorMessage);
+ return;
+ }
+ // Run it
+ const pipelineId = crypto.randomUUID();
+ printOutputMessage(`Launching Pipeline (${pipelineId}): ${pipelinePath}`);
+ let mainDocument;
+ if (!services.shared.workspace.LangiumDocuments.hasDocument(pipelinePath)) {
+ mainDocument = services.shared.workspace.LangiumDocuments.getOrCreateDocument(pipelinePath);
+ const mainDocumentValidationErrorMessage = await validateDocuments(services, [mainDocument]);
+ if (mainDocumentValidationErrorMessage) {
+ vscode.window.showErrorMessage(mainDocumentValidationErrorMessage);
+ return;
+ }
+ } else {
+ mainDocument = services.shared.workspace.LangiumDocuments.getOrCreateDocument(pipelinePath);
+ }
+ await executePipeline(services, mainDocument, pipelineId);
+};
+
+const validateDocuments = async function (
+ sdsServices: SafeDsServices,
+ documents: LangiumDocument[],
+): Promise<undefined | string> {
+ await sdsServices.shared.workspace.DocumentBuilder.build(documents, { validation: true });
+
+ const errors = documents.flatMap((validatedDocument) => {
+ const validationInfo = {
+ validatedDocument,
+ diagnostics: (validatedDocument.diagnostics ?? []).filter((e) => e.severity === 1),
+ };
+ return validationInfo.diagnostics.length > 0 ? [validationInfo] : [];
+ });
+
+ if (errors.length > 0) {
+ for (const validationInfo of errors) {
+ logError(`File ${validationInfo.validatedDocument.uri.toString()} has errors:`);
+ for (const validationError of validationInfo.diagnostics) {
+ logError(
+ `\tat line ${validationError.range.start.line + 1}: ${
+ validationError.message
+ } [${validationInfo.validatedDocument.textDocument.getText(validationError.range)}]`,
+ );
+ }
+ }
+ return `As file(s) ${errors
+ .map((validationInfo) => validationInfo.validatedDocument.uri.toString())
+ .join(', ')} has errors, the main pipeline cannot be run.`;
+ }
+ return undefined;
+};
+
+const registerVSCodeWatchers = function () {
  vscode.workspace.onDidChangeConfiguration((event) => {
  if (event.affectsConfiguration('safe-ds.runner.command')) {
  // Try starting runner

packages/safe-ds-vscode/src/extension/pythonServer.ts

@@ -146,29 +146,6 @@ export const removeMessageCallback = function <M extends PythonServerMessage['ty
  );
 };
 
-const importPipeline = async function (
- services: SafeDsServices,
- documentUri: URI,
-): Promise<LangiumDocument | undefined> {
- const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(documentUri);
- await services.shared.workspace.DocumentBuilder.build([document], { validation: true });
-
- const errors = (document.diagnostics ?? []).filter((e) => e.severity === 1);
-
- if (errors.length > 0) {
- logError(`The file ${documentUri.toString()} has errors and cannot be run.`);
- for (const validationError of errors) {
- logError(
- `\tat line ${validationError.range.start.line + 1}: ${
- validationError.message
- } [${document.textDocument.getText(validationError.range)}]`,
- );
- }
- return undefined;
- }
- return document;
-};
-
 /**
  * Context containing information about the execution of a pipeline.
  */
@@ -243,13 +220,13 @@ export const tryMapToSafeDSSource = async function (
  * If a valid target placeholder is provided, the pipeline is only executed partially, to calculate the result of the placeholder.
  *
  * @param services SafeDsServices object, used to import the pipeline file.
- * @param pipelinePath Path to a Safe-DS pipeline file to execute.
+ * @param pipelineDocument Document containing the main Safe-DS pipeline to execute.
  * @param id A unique id that is used in further communication with this pipeline.
  * @param targetPlaceholder The name of the target placeholder, used to do partial execution. If no value or undefined is provided, the entire pipeline is run.
  */
 export const executePipeline = async function (
  services: SafeDsServices,
- pipelinePath: string,
+ pipelineDocument: LangiumDocument,
  id: string,
  targetPlaceholder: string | undefined = undefined,
 ) {
@@ -261,67 +238,72 @@ export const executePipeline = async function (
  return;
  }
  }
- // TODO include all relevant files from workspace
- const documentUri = URI.file(pipelinePath);
- const workspaceRoot = path.dirname(documentUri.fsPath); // TODO get actual workspace root
- services.shared.workspace.LangiumDocuments.deleteDocument(documentUri);
- let document = await importPipeline(services, documentUri);
- if (!document) {
- vscode.window.showErrorMessage(`The file ${documentUri.fsPath} has errors and cannot be run.`);
- return;
- }
- const lastExecutedSource = document.textDocument.getText();
-
- const node = document.parseResult.value;
- //
- let mainPipelineName;
- let mainModuleName;
+ const node = pipelineDocument.parseResult.value;
  if (!ast.isSdsModule(node)) {
  return;
  }
+ // Pipeline / Module name handling
  const mainPythonModuleName = services.builtins.Annotations.getPythonModule(node);
- const mainPackage = mainPythonModuleName === undefined ? node?.name.split('.') : [mainPythonModuleName];
+ const mainPackage = mainPythonModuleName === undefined ? node.name.split('.') : [mainPythonModuleName];
  const firstPipeline = getModuleMembers(node).find(ast.isSdsPipeline);
  if (firstPipeline === undefined) {
  logError('Cannot execute: no pipeline found');
  vscode.window.showErrorMessage('The current file cannot be executed, as no pipeline could be found.');
  return;
  }
- mainPipelineName = services.builtins.Annotations.getPythonName(firstPipeline) || firstPipeline.name;
- if (pipelinePath.endsWith('.sdspipe')) {
- mainModuleName = services.generation.PythonGenerator.sanitizeModuleNameForPython(
- path.basename(pipelinePath, '.sdspipe'),
- );
- } else if (pipelinePath.endsWith('.sdstest')) {
- mainModuleName = services.generation.PythonGenerator.sanitizeModuleNameForPython(
- path.basename(pipelinePath, '.sdstest'),
- );
- } else {
- mainModuleName = services.generation.PythonGenerator.sanitizeModuleNameForPython(path.basename(pipelinePath));
- }
- //
- const generatedDocuments = services.generation.PythonGenerator.generate(document, {
- destination: URI.file(path.dirname(documentUri.fsPath)), // actual directory of main module file
+ const mainPipelineName = services.builtins.Annotations.getPythonName(firstPipeline) || firstPipeline.name;
+ const mainModuleName = getMainModuleName(services, pipelineDocument);
+ // Code generation
+ const [codeMap, lastGeneratedSources] = generateCodeForRunner(services, pipelineDocument, targetPlaceholder);
+ // Store information about the run
+ executionInformation.set(id, {
+ generatedSource: lastGeneratedSources,
+ sourceMappings: new Map<string, BasicSourceMapConsumer>(),
+ path: pipelineDocument.uri.fsPath,
+ source: pipelineDocument.textDocument.getText(),
+ calculatedPlaceholders: new Map<string, string>(),
+ });
+ // Code execution
+ sendMessageToPythonServer(
+ createProgramMessage(id, {
+ code: codeMap,
+ main: {
+ modulepath: mainPackage.join('.'),
+ module: mainModuleName,
+ pipeline: mainPipelineName,
+ },
+ }),
+ );
+};
+
+const generateCodeForRunner = function (
+ services: SafeDsServices,
+ pipelineDocument: LangiumDocument,
+ targetPlaceholder: string | undefined,
+): [ProgramCodeMap, Map<string, string>] {
+ const rootGenerationDir = path.parse(pipelineDocument.uri.fsPath).dir;
+ const generatedDocuments = services.generation.PythonGenerator.generate(pipelineDocument, {
+ destination: URI.file(rootGenerationDir), // actual directory of main module file
  createSourceMaps: true,
  targetPlaceholder,
  });
- const lastGeneratedSource = new Map<string, string>();
+ const lastGeneratedSources = new Map<string, string>();
  let codeMap: ProgramCodeMap = {};
  for (const generatedDocument of generatedDocuments) {
  const fsPath = URI.parse(generatedDocument.uri).fsPath;
- const workspaceRelativeFilePath = path.relative(workspaceRoot, path.dirname(fsPath));
+ const workspaceRelativeFilePath = path.relative(rootGenerationDir, path.dirname(fsPath));
  const sdsFileName = path.basename(fsPath);
  const sdsNoExtFilename =
  path.extname(sdsFileName).length > 0
  ? sdsFileName.substring(0, sdsFileName.length - path.extname(sdsFileName).length)
  : sdsFileName;
-
- lastGeneratedSource.set(
+ // Put code in map for further use in the extension (e.g. to remap errors)
+ lastGeneratedSources.set(
  path.join(workspaceRelativeFilePath, sdsFileName).replaceAll('\\', '/'),
  generatedDocument.getText(),
  );
  // Check for sourcemaps after they are already added to the pipeline context
- // This needs to happen after lastGeneratedSource.set, as errors would not get mapped otherwise
+ // This needs to happen after lastGeneratedSources.set, as errors would not get mapped otherwise
  if (fsPath.endsWith('.map')) {
  // exclude sourcemaps from sending to runner
  continue;
@@ -330,26 +312,26 @@ export const executePipeline = async function (
  if (!codeMap.hasOwnProperty(modulePath)) {
  codeMap[modulePath] = {};
  }
- const content = generatedDocument.getText();
- codeMap[modulePath]![sdsNoExtFilename] = content;
+ // Put code in object for runner
+ codeMap[modulePath]![sdsNoExtFilename] = generatedDocument.getText();
+ }
+ return [codeMap, lastGeneratedSources];
+};
+
+const getMainModuleName = function (services: SafeDsServices, pipelineDocument: LangiumDocument): string {
+ if (pipelineDocument.uri.fsPath.endsWith('.sdspipe')) {
+ return services.generation.PythonGenerator.sanitizeModuleNameForPython(
+ path.basename(pipelineDocument.uri.fsPath, '.sdspipe'),
+ );
+ } else if (pipelineDocument.uri.fsPath.endsWith('.sdstest')) {
+ return services.generation.PythonGenerator.sanitizeModuleNameForPython(
+ path.basename(pipelineDocument.uri.fsPath, '.sdstest'),
+ );
+ } else {
+ return services.generation.PythonGenerator.sanitizeModuleNameForPython(
+ path.basename(pipelineDocument.uri.fsPath),
+ );
  }
- executionInformation.set(id, {
- generatedSource: lastGeneratedSource,
- sourceMappings: new Map<string, BasicSourceMapConsumer>(),
- path: pipelinePath,
- source: lastExecutedSource,
- calculatedPlaceholders: new Map<string, string>(),
- });
- sendMessageToPythonServer(
- createProgramMessage(id, {
- code: codeMap,
- main: {
- modulepath: mainPackage.join('.'),
- module: mainModuleName,
- pipeline: mainPipelineName,
- },
- }),
- );
 };
 
 /**
@@ -435,7 +417,11 @@ const connectToWebSocket = async function (): Promise<void> {
  logOutput(`[Runner] Message received: (${event.type}, ${typeof event.data}) ${event.data}`);
  return;
  }
- logOutput(`[Runner] Message received: '${event.data}'`);
+ logOutput(
+ `[Runner] Message received: '${
+ event.data.length > 128 ? event.data.substring(0, 128) + '<truncated>' : event.data
+ }'`,
+ );
  const pythonServerMessage: PythonServerMessage = JSON.parse(<string>event.data);
  if (!pythonServerMessageCallbacks.has(pythonServerMessage.type)) {
  logOutput(`[Runner] Message type '${pythonServerMessage.type}' is not handled`);