feat(editor): Add data pinning functionality (#3511)

* feat: Design system color improvements and button component redesign.

* feat: Added button focus state and unit tests.

* refactor: Aligned n8n-button usage inside of editor-ui.

* test: Updated snapshots.

* refactor: Extracted focus outline width into scss variable.

* fix: Fixed select input border-radius.

* refactor: Removed element-ui references in button.

* fix: Fixed scss variable imports.

* feat: Added color-neutral variable story.

* fix: Fixed color-secondary variable definition.

* feat: Added color-white story.

* test: Updated button snapshot.

* feat: Replaced zoom buttons with new n8n-icon-button.

* feat: Added stories for float utilities.

* chore: Updated color shades generation code for later use.

* chore: Removed color-white code.

* chore: Updated story properties for button components.

* fix: Added el-button fallback for places where el-button is not replaceable (messagebox).

* feat: Reverted to css modules. Replaced el-button with n8n-button at application level.

* test: Updated button snapshot.

* fix: Fixed element-ui locally referenced buttons (via components: {}).

* fix: Updated colors. Removed irrelevant validation. Added ElButton override component.

* test: Updated button override snapshot.

* fix: Various button adjustments and fixes.

* fix: Updated button disabled state.

* test: Updated snapshots.

* fix: Consolidated css variables changes.

* Data pinning (#3512)

* refactor: Aligned n8n-button usage inside of editor-ui.

* feat: Added edit data button on json hover.

* feat: Extracted code editor into separate form component.

* feat: Added edit data button on json hover.

* feat: Added pinData and edit mode methods.

* 🔥 Remove conflict markers

* ✏️ Update i18n keys

* ⚡ Add JSON validation

* 🗃️ Add `pinData` column to `workflow_entity`

* 📘 Tighten type

* ⚡ Make `pinData` column nullable

* ⚡ Adjust workflow endpoints for pin data

* 📘 Improve types

* ✏️ Improve wording

* Inject pindata into items flow (#3420)

* ⚡ Inject pin data - Second approach

* 🔥 Remove unneeded lint exception

* feat: Added edit data button on json hover.

* feat: Extracted code editor into separate form component.

* feat: Added edit data button on json hover.

* fix: Fixed rebase conflicts.

* ⏪ Undo button change

* 🐛 Fix runNode call

Adjust per update in bdb8413

* 🧪 Fix workflow tests

* 🐛 More merge conflict fixes

* feat: Added pin/unpin button and store mutations.

* feat: Size check. Various design and ux improvements.

* ⚡ Add transformer

* ⚡ Hoist pin data

* ⚡ Adjust endpoints for hoisted pin data

* 📘 Expand interface

* 🐛 Fix stray array

* 👕 Fix build

* 👕 Add lint exception

* 👕 Fix header

* 🎨 Add color secondary tints

* ✨ Create `HeaderMessage` component

* ⚡ Adjust `InfoTip` component

* ✨ Add `HeaderMessage` to `RunData`

* 🐛 Fix console error

* 👕 Fix lint

* ⚡ Consolidate `HeaderMessage` and `Callout`

* ⏪ Undo `InfoTip` changes

* 🔥 Remove duplicate icons

* ⚡ Simplify template

* 🎨 Change cursor for action text

* 👕 Fix lint

* ⚡ Add URL

* 🐛 Fix handler name

* ⚡ Use constant

* ♻️ Refactor per feedback

* fix: Various fixes after data pinning relocation.

* fix: Added store mutation for setting pinned data.

* feat: Added pinned state for workflow canvas node.

* fix: Fixed workflow saving.

* fix: Removed pinData hoisting (no longer necessary).

* feat: Added canPinData flag to hide for input pane and binary data. Fixed unpin and execute flow.

* ⚡ Fixes for canvas pin data (#3587)

* ⚡ Fixes for canvas pin data

* 📘 Rename type

* 🧪 Fix unrelated Public API test

* 🔥 Remove logging

* feat: Updated pinData mixin to no longer include extra fields.

* ⚡ Output same pindata for every run

* 🎨 Fix cropping

* 🔥 Remove unrelated logging

* feat: Moved edit button next to pin button.

* feat: Changed data to be inserted for empty state.

* chore: Changed invalid editor output translation.

* feat: Added error line reporting on JSON Validation.

* feat: Migrated pinData edit mode to store.

* chore: Merged duplicate node border color condition.

* feat: Moved pin data validation to mixin. Added check before closing ndv modal.

* fix: Changed pinned data size calculation to discard active node pin data.

* feat: Added support for rename and delete node with pin data.

* feat: Simplified editing state. Fixed edit mode in input panel after store migration.

* feat: Various data pinning improvements.

* fix: Fixed callout link underline.

* refactor: Added support for both string and objects for data size check.

* feat: Added disabled node check for input panel. Fixed monaco editor resizing.

* fix: Fixed edit mode footer size.

* ⚡ Fix pindata items per run

* 👕 Remove unneeded exception

* refactor: Added isValidPinData() helper method.

* refactor: Changed how string size in bytes in calculated.g

* refactor: Updated pinData mixin interface.

* refactor: Merged filter and reduce in pinDataSize calculation.

* fix: Changed code-editor to correct type.

* fix: Added insert test data message to trigger nodes.

* feat: Disabled data pinning for multiple output nodes.

* refactor: Updated ndv.input.disabled translation to include node name.

* refactor: Aligned n8n-button usage inside of editor-ui.

* feat: Added edit data button on json hover.

* feat: Extracted code editor into separate form component.

* feat: Added edit data button on json hover.

* feat: Added pinData and edit mode methods.

* 🔥 Remove conflict markers

* ✏️ Update i18n keys

* ⚡ Add JSON validation

* 🗃️ Add `pinData` column to `workflow_entity`

* 📘 Tighten type

* ⚡ Make `pinData` column nullable

* ⚡ Adjust workflow endpoints for pin data

* 📘 Improve types

* ✏️ Improve wording

* Inject pindata into items flow (#3420)

* ⚡ Inject pin data - Second approach

* 🔥 Remove unneeded lint exception

* feat: Added edit data button on json hover.

* feat: Extracted code editor into separate form component.

* feat: Added edit data button on json hover.

* fix: Fixed rebase conflicts.

* ⏪ Undo button change

* 🐛 Fix runNode call

Adjust per update in bdb8413

* 🧪 Fix workflow tests

* 🐛 More merge conflict fixes

* feat: Added pin/unpin button and store mutations.

* feat: Size check. Various design and ux improvements.

* ⚡ Add transformer

* ⚡ Hoist pin data

* ⚡ Adjust endpoints for hoisted pin data

* 📘 Expand interface

* 🐛 Fix stray array

* 👕 Fix build

* 🎨 Add color secondary tints

* ✨ Create `HeaderMessage` component

* ⚡ Adjust `InfoTip` component

* ✨ Add `HeaderMessage` to `RunData`

* 🐛 Fix console error

* 👕 Fix lint

* ⚡ Consolidate `HeaderMessage` and `Callout`

* ⏪ Undo `InfoTip` changes

* 🔥 Remove duplicate icons

* ⚡ Simplify template

* 🎨 Change cursor for action text

* 👕 Fix lint

* ⚡ Add URL

* 🐛 Fix handler name

* ⚡ Use constant

* ♻️ Refactor per feedback

* fix: Various fixes after data pinning relocation.

* fix: Added store mutation for setting pinned data.

* feat: Added pinned state for workflow canvas node.

* ⚡ Fixes for canvas pin data (#3587)

* ⚡ Fixes for canvas pin data

* 📘 Rename type

* 🧪 Fix unrelated Public API test

* 🔥 Remove logging

* feat: Updated pinData mixin to no longer include extra fields.

* fix: Removed pinData hoisting (no longer necessary).

* chore: Merged duplicate node border color condition.

* ⚡ Output same pindata for every run

* 🎨 Fix cropping

* 🐛 Fix excess closing template tag

* fix: Removed rogue template tag after merge.

* fix: Fixed code-editor resizing when moving ndv panel.

* feat: Added node duplication pin data.

* ⚡ Implement telemetry

* ♻️ Add clarifications from call

* fix: Fixed run data header height.

* feat: Removed border from pin data callout.

* feat: Added line-break before 'or insert pin data'.

* feat: Changed enterEditMode to always insert test data if there's no execution data.

* feat: Removed copy output tooltip.

* feat: Removed unpin tooltip.

* fix: Removed thumbtack icon rotation.

* fix: Removed run info from Edit Output title.

* feat: Hid edit and pin buttons when editing.

* feat: Updated monaco code-editor padding and borders.

* feat: Progress on pinData error message format

* feat: Updated copy feature to work without any selected value.

* feat: Moved save and cancel buttons. Cleared notifications on save.

* feat: Changed pin data beforeClosing confirm text.

* feat: Closing ndv when discarding or saving pindata on close.

* feat: Added split in batches node to pin data denylist.

* fix: Added missing margin-bottom to webhook node.

* feat: Moved thumbtack icon to the right, replacing the checkmark.

* fix: Hid pagination while editing.

* feat: Added pin data discovery flow.

* feat: Changed pin data discovery flow to avoid tooltip glitching.

* fix: Changed copy selection to copy all input data.

* feat: Updated pin data validation error message for unexpected single quotes.

* fix: Replaced :manual='true' prop with manual shorthand.

* fix: Removed unused variable.

* chore: Renamed translation key to node.discovery.pinData.

* refactor: Extracted isPinDataNodeType to pinData mixin.

* fix: Updated watch condition to improve performance.

* refactor: Renamed some pin data variables and methods as per review.

* fix: Added partial translation for JSON.parse pin data error messages.

* chore: Temporarily disabled failing unit test.

* 🧪 Fix data pinning workflow retrieval test

* 🔥 Remove unused imports

* 🔥 Remove leftover line

* ⚡ Skip pindata node issues on BE

* ⚡ Skip pindata node issues on FE

* ⚡ Hide `RunInfo` for pindata node

* ⚡ Hide purple banner in edit output mode

* feat: Updated data pinning discoverability flow.

* fix: Fixed paginated data pinning.

* fix: Disabled pin data in read only mode.

* 🐛 Fix runtime error with non-array

* fix: Loading pin data when opening execution.

* ⚡ Adjust stale data warning for pinned data

* ⚡ Skip auth in endpoint

* ⚡ Mark start node for pinned trigger

* ✏️ Comment on passthrough

* 🔥 Remove comment

* Final pindata metrics changes (#3673)

* 🐛 Fix `pinData` tracked as `0`

* ⚡ Add `is_pinned` to `nodesGraph`

* 📘 Extend `IWorkflowBase`

* ⚡ Handle `pinData` being `undefined`

* ⚡ Add `data_pinning_tooltip_presented`

* ♻️ Refactor to remove circular dependency

* fix: Added pin data handling when importing workflow. (#3698)

* 🔥 Remove helper from WorkflowExecute

* ⚡ Add logic for single pinned trigger

* 👕 Remove lint exception

* fix: Added pin data handling in importWorkflowExact.

* N8N-4077 data pinning discoverability part 2 (#3701)

* fix: Fixed pin data discovery tooltip position when moving canvas.

* feat: Updated data pinning discovery tooltip copy.

* Fix data pinning build (#3702)

* ⚡ Disable edit button for disabled node

* ⚡ Ensure disabled pinned nodes are passthrough

* 🐛 Fix JSON key unfurling in edit mode

* ⚡ Improve implementation

* 🐛 Fix console error

* fix: Fixed copying pinned output data. (#3715)

* Fix pinning for webhook responding with output from last node (#3719)

* fix: Fixed entering edit mode after refresh.

* fix: Fixed type error during build.

* fix: RunData import formatting.

* chore: Updated pin data types.

* fix: Added missing type to stringSizeInBytes.

Co-authored-by: Iván Ovejero <ivov.src@gmail.com>

* fix: Showing pin data without executing the node only in output pane.

* fix: Updated no data message when previous node not executed.

* feat: Added expression input and evaluation for pin data nodes without execution.

* chore: Fixed linting issues and removed remnant console.log().

* chore: Undone package-lock changes.

* fix: Removed pin data store changes.

* fix: Created a new object using vuex runExecutionData.

* fix: Fixed bug appearing when adding a new node after executing.

* fix: Fix editor-ui build

* feat: Added green node connectors when having pin data output.

* chore: Fixed linting errors.

* fix: Added pin data eventBus unsubscribe.

* fix: Added pin data color check after adding a connection.

* 🎨 Add pindata styles

Co-authored-by: Iván Ovejero <ivov.src@gmail.com>

packages/cli/src/Interfaces.ts

@@ -15,6 +15,7 @@ import {
 	ITelemetrySettings,
 	ITelemetryTrackProperties,
 	IWorkflowBase as IWorkflowBaseWorkflow,
+	PinData,
 	Workflow,
 	WorkflowExecuteMode,
 } from 'n8n-workflow';
@@ -688,6 +689,7 @@ export interface IWorkflowExecutionDataProcess {
 	executionMode: WorkflowExecuteMode;
 	executionData?: IRunExecutionData;
 	runData?: IRunData;
+	pinData?: PinData;
 	retryOf?: number | string;
 	sessionId?: string;
 	startNodes?: string[];

packages/cli/src/Server.ts

@@ -74,6 +74,7 @@ import {
 	IWorkflowBase,
 	LoggerProxy,
 	NodeHelpers,
+	PinData,
 	WebhookHttpMethod,
 	Workflow,
 	WorkflowExecuteMode,
@@ -130,6 +131,7 @@ import {
 	WorkflowRunner,
 	getCredentialForUser,
 	getCredentialWithoutUser,
+	IWorkflowDb,
 } from '.';
 
 import config from '../config';
@@ -157,9 +159,9 @@ import type {
 } from './requests';
 import { DEFAULT_EXECUTIONS_GET_ALL_LIMIT, validateEntity } from './GenericHelpers';
 import { ExecutionEntity } from './databases/entities/ExecutionEntity';
-import { SharedWorkflow } from './databases/entities/SharedWorkflow';
 import { AUTH_COOKIE_NAME, RESPONSE_ERROR_MESSAGES } from './constants';
 import { credentialsController } from './api/credentials.api';
+import { workflowsController } from './api/workflows.api';
 import { nodesController } from './api/nodes.api';
 import { oauth2CredentialController } from './api/oauth2Credential.api';
 import {
@@ -168,6 +170,7 @@ import {
 	isUserManagementEnabled,
 } from './UserManagement/UserManagementHelper';
 import { loadPublicApiVersions } from './PublicApi';
+import { SharedWorkflow } from './databases/entities/SharedWorkflow';
 
 require('body-parser-xml')(bodyParser);
 
@@ -769,74 +772,6 @@ class App {
 		// Workflow
 		// ----------------------------------------
 
-		// Creates a new workflow
-		this.app.post(
-			`/${this.restEndpoint}/workflows`,
-			ResponseHelper.send(async (req: WorkflowRequest.Create) => {
-				delete req.body.id; // delete if sent
-
-				const newWorkflow = new WorkflowEntity();
-
-				Object.assign(newWorkflow, req.body);
-
-				await validateEntity(newWorkflow);
-
-				await this.externalHooks.run('workflow.create', [newWorkflow]);
-
-				const { tags: tagIds } = req.body;
-
-				if (tagIds?.length && !config.getEnv('workflowTagsDisabled')) {
-					newWorkflow.tags = await Db.collections.Tag.findByIds(tagIds, {
-						select: ['id', 'name'],
-					});
-				}
-
-				await WorkflowHelpers.replaceInvalidCredentials(newWorkflow);
-
-				let savedWorkflow: undefined | WorkflowEntity;
-
-				await getConnection().transaction(async (transactionManager) => {
-					savedWorkflow = await transactionManager.save<WorkflowEntity>(newWorkflow);
-
-					const role = await Db.collections.Role.findOneOrFail({
-						name: 'owner',
-						scope: 'workflow',
-					});
-
-					const newSharedWorkflow = new SharedWorkflow();
-
-					Object.assign(newSharedWorkflow, {
-						role,
-						user: req.user,
-						workflow: savedWorkflow,
-					});
-
-					await transactionManager.save<SharedWorkflow>(newSharedWorkflow);
-				});
-
-				if (!savedWorkflow) {
-					LoggerProxy.error('Failed to create workflow', { userId: req.user.id });
-					throw new ResponseHelper.ResponseError('Failed to save workflow');
-				}
-
-				if (tagIds && !config.getEnv('workflowTagsDisabled')) {
-					savedWorkflow.tags = TagHelpers.sortByRequestOrder(savedWorkflow.tags, {
-						requestOrder: tagIds,
-					});
-				}
-
-				await this.externalHooks.run('workflow.afterCreate', [savedWorkflow]);
-				void InternalHooksManager.getInstance().onWorkflowCreated(req.user.id, newWorkflow, false);
-
-				const { id, ...rest } = savedWorkflow;
-
-				return {
-					id: id.toString(),
-					...rest,
-				};
-			}),
-		);
-
 		// Reads and returns workflow data from an URL
 		this.app.get(
 			`/${this.restEndpoint}/workflows/from-url`,
@@ -962,50 +897,6 @@ class App {
 			}),
 		);
 
-		// Returns a specific workflow
-		this.app.get(
-			`/${this.restEndpoint}/workflows/:id`,
-			ResponseHelper.send(async (req: WorkflowRequest.Get) => {
-				const { id: workflowId } = req.params;
-
-				let relations = ['workflow', 'workflow.tags'];
-
-				if (config.getEnv('workflowTagsDisabled')) {
-					relations = relations.filter((relation) => relation !== 'workflow.tags');
-				}
-
-				const shared = await Db.collections.SharedWorkflow.findOne({
-					relations,
-					where: whereClause({
-						user: req.user,
-						entityType: 'workflow',
-						entityId: workflowId,
-					}),
-				});
-
-				if (!shared) {
-					LoggerProxy.info('User attempted to access a workflow without permissions', {
-						workflowId,
-						userId: req.user.id,
-					});
-					throw new ResponseHelper.ResponseError(
-						`Workflow with ID "${workflowId}" could not be found.`,
-						undefined,
-						404,
-					);
-				}
-
-				const {
-					workflow: { id, ...rest },
-				} = shared;
-
-				return {
-					id: id.toString(),
-					...rest,
-				};
-			}),
-		);
-
 		// Updates an existing workflow
 		this.app.patch(
 			`/${this.restEndpoint}/workflows/:id`,
@@ -1204,19 +1095,23 @@ class App {
 				): Promise<IExecutionPushResponse> => {
 					const { workflowData } = req.body;
 					const { runData } = req.body;
+					const { pinData } = req.body;
 					const { startNodes } = req.body;
 					const { destinationNode } = req.body;
 					const executionMode = 'manual';
 					const activationMode = 'manual';
 
 					const sessionId = GenericHelpers.getSessionId(req);
 
+					const pinnedTrigger = findFirstPinnedTrigger(workflowData, pinData);
+
 					// If webhooks nodes exist and are active we have to wait for till we receive a call
 					if (
-						runData === undefined ||
-						startNodes === undefined ||
-						startNodes.length === 0 ||
-						destinationNode === undefined
+						pinnedTrigger === undefined &&
+						(runData === undefined ||
+							startNodes === undefined ||
+							startNodes.length === 0 ||
+							destinationNode === undefined)
 					) {
 						const additionalData = await WorkflowExecuteAdditionalData.getBase(req.user.id);
 						const nodeTypes = NodeTypes();
@@ -1254,11 +1149,17 @@ class App {
 						destinationNode,
 						executionMode,
 						runData,
+						pinData,
 						sessionId,
 						startNodes,
 						workflowData,
 						userId: req.user.id,
 					};
+
+					if (pinnedTrigger) {
+						data.startNodes = [pinnedTrigger.name];
+					}
+
 					const workflowRunner = new WorkflowRunner();
 					const executionId = await workflowRunner.run(data);
 
@@ -1269,6 +1170,8 @@ class App {
 			),
 		);
 
+		this.app.use(`/${this.restEndpoint}/workflows`, workflowsController);
+
 		// Retrieves all tags, with or without usage count
 		this.app.get(
 			`/${this.restEndpoint}/tags`,
@@ -2927,3 +2830,20 @@ function isOAuth(credType: ICredentialType) {
 		)
 	);
 }
+
+const TRIGGER_NODE_SUFFIXES = ['trigger', 'webhook'];
+
+const isTrigger = (str: string) =>
+	TRIGGER_NODE_SUFFIXES.some((suffix) => str.toLowerCase().includes(suffix));
+
+function findFirstPinnedTrigger(workflow: IWorkflowDb, pinData?: PinData) {
+	if (!pinData) return;
+
+	const firstPinnedTriggerName = Object.keys(pinData).find(isTrigger);
+
+	if (!firstPinnedTriggerName) return;
+
+	return workflow.nodes.find(
+		({ type, name }) => isTrigger(type) && name === firstPinnedTriggerName,
+	);
+}

packages/cli/src/WebhookHelpers.ts

@@ -485,6 +485,10 @@ export async function executeWebhook(
 					return undefined;
 				}
 
+				if (workflowData.pinData) {
+					data.data.resultData.pinData = workflowData.pinData;
+				}
+
 				const returnData = WorkflowHelpers.getDataLastExecutedNodeData(data);
 				if (data.data.resultData.error || returnData?.error !== undefined) {
 					if (!didSendResponse) {

packages/cli/src/WorkflowHelpers.ts

@@ -50,7 +50,7 @@ const ERROR_TRIGGER_TYPE = config.getEnv('nodes.errorTriggerType');
  * @returns {(ITaskData | undefined)}
  */
 export function getDataLastExecutedNodeData(inputData: IRun): ITaskData | undefined {
-	const { runData } = inputData.data.resultData;
+	const { runData, pinData = {} } = inputData.data.resultData;
 	const { lastNodeExecuted } = inputData.data.resultData;
 
 	if (lastNodeExecuted === undefined) {
@@ -61,7 +61,26 @@ export function getDataLastExecutedNodeData(inputData: IRun): ITaskData | undefi
 		return undefined;
 	}
 
-	return runData[lastNodeExecuted][runData[lastNodeExecuted].length - 1];
+	const lastNodeRunData = runData[lastNodeExecuted][runData[lastNodeExecuted].length - 1];
+
+	let lastNodePinData = pinData[lastNodeExecuted];
+
+	if (lastNodePinData) {
+		if (!Array.isArray(lastNodePinData)) lastNodePinData = [lastNodePinData];
+
+		const itemsPerRun = lastNodePinData.map((item, index) => {
+			return { json: item, pairedItem: { item: index } };
+		});
+
+		return {
+			startTime: 0,
+			executionTime: 0,
+			data: { main: [itemsPerRun] },
+			source: lastNodeRunData.source,
+		};
+	}
+
+	return lastNodeRunData;
 }
 
 /**

packages/cli/src/WorkflowRunner.ts

@@ -310,7 +310,12 @@ export class WorkflowRunner {
 
 				// Can execute without webhook so go on
 				const workflowExecute = new WorkflowExecute(additionalData, data.executionMode);
-				workflowExecution = workflowExecute.run(workflow, undefined, data.destinationNode);
+				workflowExecution = workflowExecute.run(
+					workflow,
+					undefined,
+					data.destinationNode,
+					data.pinData,
+				);
 			} else {
 				Logger.debug(`Execution ID ${executionId} is a partial execution.`, { executionId });
 				// Execute only the nodes between start and destination nodes
@@ -320,6 +325,7 @@ export class WorkflowRunner {
 					data.runData,
 					data.startNodes,
 					data.destinationNode,
+					data.pinData,
 				);
 			}
 

packages/cli/src/WorkflowRunnerProcess.ts

@@ -345,9 +345,22 @@ export class WorkflowRunnerProcess {
 		) {
 			// Execute all nodes
 
+			let startNode;
+			if (
+				this.data.startNodes?.length === 1 &&
+				Object.keys(this.data.pinData ?? {}).includes(this.data.startNodes[0])
+			) {
+				startNode = this.workflow.getNode(this.data.startNodes[0]) ?? undefined;
+			}
+
 			// Can execute without webhook so go on
 			this.workflowExecute = new WorkflowExecute(additionalData, this.data.executionMode);
-			return this.workflowExecute.run(this.workflow, undefined, this.data.destinationNode);
+			return this.workflowExecute.run(
+				this.workflow,
+				startNode,
+				this.data.destinationNode,
+				this.data.pinData,
+			);
 		}
 		// Execute only the nodes between start and destination nodes
 		this.workflowExecute = new WorkflowExecute(additionalData, this.data.executionMode);
@@ -356,6 +369,7 @@ export class WorkflowRunnerProcess {
 			this.data.runData,
 			this.data.startNodes,
 			this.data.destinationNode,
+			this.data.pinData,
 		);
 	}