Make it possible to dynamically load node packages with dependencies

packages/cli/commands/start.ts

@@ -33,6 +33,7 @@ import {
 } from '../src';
 
 import { getLogger } from '../src/Logger';
+import { getAllInstalledPackages } from '../src/CommunityNodes/packageModel';
 
 // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-var-requires
 const open = require('open');
@@ -60,6 +61,10 @@ export class Start extends Command {
 			description:
 				'runs the webhooks via a hooks.n8n.cloud tunnel server. Use only for testing and development!',
 		}),
+		reinstallMissingPackages: flags.boolean({
+			description:
+				'Attempts to self heal n8n if packages with nodes are missing. Might drastically increase startup times.',
+		}),
 	};
 
 	/**
@@ -206,6 +211,23 @@ export class Start extends Command {
 				// Wait till the database is ready
 				await startDbInitPromise;
 
+				const installedPackages = await getAllInstalledPackages();
+				const missingPackages = new Set<{
+					packageName: string;
+					version: string;
+				}>();
+				installedPackages.forEach((installedpackage) => {
+					installedpackage.installedNodes.forEach((installedNode) => {
+						if (!loadNodesAndCredentials.nodeTypes[installedNode.type]) {
+							// Leave the list ready for installing in case we need.
+							missingPackages.add({
+								packageName: installedpackage.packageName,
+								version: installedpackage.installedVersion,
+							});
+						}
+					});
+				});
+
 				await UserSettings.getEncryptionKey();
 
 				// Load settings from database and set them to config.
@@ -214,6 +236,42 @@ export class Start extends Command {
 					config.set(setting.key, JSON.parse(setting.value));
 				});
 
+				config.set('nodes.packagesMissing', '');
+				if (missingPackages.size) {
+					LoggerProxy.error(
+						'n8n detected that some packages are missing. For more information, visit https://docs.n8n.io/integrations/community-nodes/troubleshooting/',
+					);
+
+					if (flags.reinstallMissingPackages || process.env.N8N_REINSTALL_MISSING_PACKAGES) {
+						LoggerProxy.info('Attempting to reinstall missing packages', { missingPackages });
+						try {
+							// Optimistic approach - stop if any installation fails
+							// eslint-disable-next-line no-restricted-syntax
+							for (const missingPackage of missingPackages) {
+								// eslint-disable-next-line no-await-in-loop
+								void (await loadNodesAndCredentials.loadNpmModule(
+									missingPackage.packageName,
+									missingPackage.version,
+								));
+								missingPackages.delete(missingPackage);
+							}
+							LoggerProxy.info(
+								'Packages reinstalled successfully. Resuming regular intiailization.',
+							);
+						} catch (error) {
+							LoggerProxy.error('n8n was unable to install the missing packages.');
+						}
+					}
+				}
+				if (missingPackages.size) {
+					config.set(
+						'nodes.packagesMissing',
+						Array.from(missingPackages)
+							.map((missingPackage) => `${missingPackage.packageName}@${missingPackage.version}`)
+							.join(' '),
+					);
+				}
+
 				if (config.getEnv('executions.mode') === 'queue') {
 					const redisHost = config.getEnv('queue.bull.redis.host');
 					const redisPassword = config.getEnv('queue.bull.redis.password');

packages/cli/config/schema.ts

@@ -740,6 +740,14 @@ export const schema = {
 			default: 'n8n-nodes-base.errorTrigger',
 			env: 'NODES_ERROR_TRIGGER_TYPE',
 		},
+		communityPackages: {
+			enabled: {
+				doc: 'Allows you to disable the usage of community packages for nodes',
+				format: Boolean,
+				default: true,
+				env: 'N8N_COMMUNITY_PACKAGES_ENABLED',
+			},
+		},
 	},
 
 	logs: {

packages/cli/package.json

@@ -86,7 +86,6 @@
     "@types/supertest": "^2.0.11",
     "@types/uuid": "^8.3.0",
     "@types/validator": "^13.7.0",
-    "axios": "^0.21.1",
     "concurrently": "^5.1.0",
     "jest": "^27.4.7",
     "nodemon": "^2.0.2",
@@ -109,6 +108,7 @@
     "@types/shelljs": "^0.8.11",
     "@types/swagger-ui-express": "^4.1.3",
     "@types/yamljs": "^0.2.31",
+    "axios": "^0.21.1",
     "basic-auth": "^2.0.1",
     "bcryptjs": "^2.4.3",
     "body-parser": "^1.18.3",

packages/cli/src/CommunityNodes/helpers.ts

@@ -0,0 +1,218 @@
+/* eslint-disable import/no-cycle */
+/* eslint-disable @typescript-eslint/naming-convention */
+import { promisify } from 'util';
+import { exec } from 'child_process';
+import { access as fsAccess, mkdir as fsMkdir } from 'fs/promises';
+
+import { UserSettings } from 'n8n-core';
+import { LoggerProxy, PublicInstalledPackage } from 'n8n-workflow';
+import axios from 'axios';
+import {
+	NODE_PACKAGE_PREFIX,
+	NPM_COMMAND_TOKENS,
+	NPM_PACKAGE_STATUS_GOOD,
+	RESPONSE_ERROR_MESSAGES,
+} from '../constants';
+import { NpmPackageStatusCheck, NpmUpdatesAvailable, ParsedNpmPackageName } from '../Interfaces';
+import { InstalledPackages } from '../databases/entities/InstalledPackages';
+import config from '../../config';
+
+const execAsync = promisify(exec);
+
+export const parsePackageName = (originalString: string | undefined): ParsedNpmPackageName => {
+	if (!originalString) {
+		throw new Error('Package name was not provided');
+	}
+
+	if (new RegExp(/[^0-9a-z@\-./]/).test(originalString)) {
+		// Prevent any strings that are not valid npm package names or
+		// could indicate malicous commands
+		throw new Error('Package name must be a single word');
+	}
+
+	const scope = originalString.includes('/') ? originalString.split('/')[0] : undefined;
+
+	const packageNameWithoutScope = scope ? originalString.replace(`${scope}/`, '') : originalString;
+
+	if (!packageNameWithoutScope.startsWith(NODE_PACKAGE_PREFIX)) {
+		throw new Error('Package name must start with n8n-nodes-');
+	}
+
+	const version = packageNameWithoutScope.includes('@')
+		? packageNameWithoutScope.split('@')[1]
+		: undefined;
+
+	const packageName = version ? originalString.replace(`@${version}`, '') : originalString;
+
+	return {
+		packageName,
+		scope,
+		version,
+		originalString,
+	};
+};
+
+export const executeCommand = async (
+	command: string,
+	options?: {
+		doNotHandleError?: boolean;
+	},
+): Promise<string> => {
+	const downloadFolder = UserSettings.getUserN8nFolderDowloadedNodesPath();
+	// Make sure the node-download folder exists
+	try {
+		await fsAccess(downloadFolder);
+		// eslint-disable-next-line no-empty
+	} catch (error) {
+		await fsMkdir(downloadFolder);
+	}
+	const execOptions = {
+		cwd: downloadFolder,
+		env: {
+			NODE_PATH: process.env.NODE_PATH,
+			PATH: process.env.PATH,
+		},
+	};
+
+	try {
+		const commandResult = await execAsync(command, execOptions);
+		return commandResult.stdout;
+	} catch (error) {
+		if (options?.doNotHandleError) {
+			throw error;
+		}
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+		const errorMessage = error.message as string;
+
+		if (
+			errorMessage.includes(NPM_COMMAND_TOKENS.NPM_PACKAGE_NOT_FOUND_ERROR) ||
+			errorMessage.includes(NPM_COMMAND_TOKENS.NPM_NO_VERSION_AVAILABLE)
+		) {
+			throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND);
+		}
+		if (errorMessage.includes(NPM_COMMAND_TOKENS.NPM_PACKAGE_VERSION_NOT_FOUND_ERROR)) {
+			throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_VERSION_NOT_FOUND);
+		}
+		if (
+			errorMessage.includes(NPM_COMMAND_TOKENS.NPM_DISK_NO_SPACE) ||
+			errorMessage.includes(NPM_COMMAND_TOKENS.NPM_DISK_INSUFFICIENT_SPACE)
+		) {
+			throw new Error(RESPONSE_ERROR_MESSAGES.DISK_IS_FULL);
+		}
+
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+		LoggerProxy.warn('npm command failed; see message', { errorMessage });
+
+		throw new Error('Package could not be installed - check logs for details');
+	}
+};
+
+export function matchPackagesWithUpdates(
+	installedPackages: InstalledPackages[],
+	availableUpdates?: NpmUpdatesAvailable,
+): PublicInstalledPackage[] {
+	if (!availableUpdates) {
+		return installedPackages;
+	}
+	const hydratedPackageList = [] as PublicInstalledPackage[];
+
+	for (let i = 0; i < installedPackages.length; i++) {
+		const installedPackage = installedPackages[i];
+		const publicPackage = { ...installedPackage } as PublicInstalledPackage;
+
+		if (availableUpdates[installedPackage.packageName]) {
+			publicPackage.updateAvailable = availableUpdates[installedPackage.packageName].latest;
+		}
+		hydratedPackageList.push(publicPackage);
+	}
+
+	return hydratedPackageList;
+}
+
+export function matchMissingPackages(
+	installedPackages: PublicInstalledPackage[],
+	missingPackages: string,
+): PublicInstalledPackage[] {
+	const missingPackageNames = missingPackages.split(' ');
+
+	const missingPackagesList = missingPackageNames.map((missingPackageName: string) => {
+		// Strip away versions but maintain scope and package name
+		try {
+			const parsedPackageData = parsePackageName(missingPackageName);
+			return parsedPackageData.packageName;
+
+			// eslint-disable-next-line no-empty
+		} catch (_) {}
+		return undefined;
+	});
+
+	const hydratedPackageList = [] as PublicInstalledPackage[];
+	installedPackages.forEach((installedPackage) => {
+		const hydratedInstalledPackage = { ...installedPackage };
+		if (missingPackagesList.includes(hydratedInstalledPackage.packageName)) {
+			hydratedInstalledPackage.failedLoading = true;
+		}
+		hydratedPackageList.push(hydratedInstalledPackage);
+	});
+
+	return hydratedPackageList;
+}
+
+export async function checkPackageStatus(packageName: string): Promise<NpmPackageStatusCheck> {
+	// You can change this URL for testing - the default testing url below
+	// is a postman mock service
+	const n8nBackendServiceUrl = 'https://api.n8n.io/api/package';
+
+	try {
+		const output = await axios.post(
+			n8nBackendServiceUrl,
+			{ name: packageName },
+			{
+				method: 'POST',
+			},
+		);
+
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+		if (output.data.status !== NPM_PACKAGE_STATUS_GOOD) {
+			return output.data as NpmPackageStatusCheck;
+		}
+	} catch (error) {
+		// Do nothing if service is unreachable
+	}
+	return { status: NPM_PACKAGE_STATUS_GOOD };
+}
+
+export function hasPackageLoadedSuccessfully(packageName: string): boolean {
+	try {
+		const failedPackages = (config.get('nodes.packagesMissing') as string).split(' ');
+
+		const packageFailedToLoad = failedPackages.find(
+			(packageNameAndVersion) =>
+				packageNameAndVersion.startsWith(packageName) &&
+				packageNameAndVersion.replace(packageName, '').startsWith('@'),
+		);
+		if (packageFailedToLoad) {
+			return false;
+		}
+		return true;
+	} catch (_error) {
+		// If key doesn't exist it means all packages loaded fine
+		return true;
+	}
+}
+
+export function removePackageFromMissingList(packageName: string): void {
+	try {
+		const failedPackages = (config.get('nodes.packagesMissing') as string).split(' ');
+
+		const packageFailedToLoad = failedPackages.filter(
+			(packageNameAndVersion) =>
+				!packageNameAndVersion.startsWith(packageName) ||
+				!packageNameAndVersion.replace(packageName, '').startsWith('@'),
+		);
+
+		config.set('nodes.packagesMissing', packageFailedToLoad.join(' '));
+	} catch (_error) {
+		// Do nothing
+	}
+}