diff --git a/.vscode/launch.json b/.vscode/launch.json index 13c32cd994c9d..255c1f8a9d2cf 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,6 +5,17 @@ "version": "0.2.0", "configurations": [ { + "type": "node", + "request": "launch", + "name": "Jest: current file", + "program": "${workspaceFolder}/node_modules/.bin/jest", + "args": ["${fileBasenameNoExtension}"], + "console": "integratedTerminal", + "windows": { + "program": "${workspaceFolder}/node_modules/jest/bin/jest" + } + }, + { "name": "Attach to running n8n", "processId": "${command:PickProcess}", "request": "attach", diff --git a/packages/editor-ui/tsconfig.json b/packages/editor-ui/tsconfig.json index 8be38644497e2..fd6a19a4cb211 100644 --- a/packages/editor-ui/tsconfig.json +++ b/packages/editor-ui/tsconfig.json @@ -10,6 +10,7 @@ "importHelpers": true, "incremental": false, "allowSyntheticDefaultImports": true, + "resolveJsonModule": true, "baseUrl": ".", "types": ["vitest/globals"], "paths": { diff --git a/packages/editor-ui/vite.config.ts b/packages/editor-ui/vite.config.ts index 80764c6e7cf03..b1270c79e2027 100644 --- a/packages/editor-ui/vite.config.ts +++ b/packages/editor-ui/vite.config.ts @@ -43,6 +43,10 @@ const lodashAliases = ['orderBy', 'camelCase', 'cloneDeep', 'isEqual', 'startCas export default mergeConfig( defineConfig({ + define: { + // This causes test to fail but is required for actually running it + ...(process.env.NODE_ENV !== 'test' ? { global: 'globalThis' } : {}), + }, plugins: [ legacy({ targets: ['defaults', 'not IE 11'], diff --git a/packages/workflow/package.json b/packages/workflow/package.json index 8ff00dae3ba65..86d339e6ef615 100644 --- a/packages/workflow/package.json +++ b/packages/workflow/package.json @@ -39,6 +39,8 @@ "dist/**/*" ], "devDependencies": { + "@n8n_io/eslint-config": "", + "@types/crypto-js": "^4.1.1", "@types/express": "^4.17.6", "@types/jmespath": "^0.15.0", "@types/lodash.get": "^4.4.6", @@ -50,12 +52,16 @@ }, "dependencies": { "@n8n_io/riot-tmpl": "^2.0.0", + "crypto-js": "^4.1.1", "jmespath": "^0.16.0", + "js-base64": "^3.7.2", "lodash.get": "^4.4.2", "lodash.isequal": "^4.5.0", "lodash.merge": "^4.6.2", "lodash.set": "^4.3.2", "luxon": "~2.3.0", + "recast": "^0.21.5", + "transliteration": "^2.3.5", "xml2js": "^0.4.23" } } diff --git a/packages/workflow/src/Expression.ts b/packages/workflow/src/Expression.ts index 82793a6a13382..925105a1454bb 100644 --- a/packages/workflow/src/Expression.ts +++ b/packages/workflow/src/Expression.ts @@ -14,10 +14,21 @@ import { NodeParameterValueType, WorkflowExecuteMode, } from './Interfaces'; -import { ExpressionError } from './ExpressionError'; +import { ExpressionError, ExpressionExtensionError } from './ExpressionError'; import { WorkflowDataProxy } from './WorkflowDataProxy'; import type { Workflow } from './Workflow'; +// eslint-disable-next-line import/no-cycle +import { extend, hasExpressionExtension, hasNativeMethod } from './Extensions'; +import { + ExpressionChunk, + ExpressionCode, + joinExpression, + splitExpression, +} from './Extensions/ExpressionParser'; +import { extendTransform } from './Extensions/ExpressionExtension'; +import { extendedFunctions } from './Extensions/ExtendedFunctions'; + // Set it to use double curly brackets instead of single ones tmpl.brackets.set('{{ }}'); @@ -242,6 +253,11 @@ export class Expression { data.Boolean = Boolean; data.Symbol = Symbol; + // expression extensions + data.extend = extend; + + Object.assign(data, extendedFunctions); + const constructorValidation = new RegExp(/\.\s*constructor/gm); if (parameterValue.match(constructorValidation)) { throw new ExpressionError('Expression contains invalid constructor function call', { @@ -252,7 +268,8 @@ export class Expression { } // Execute the expression - const returnValue = this.renderExpression(parameterValue, data); + const extendedExpression = this.extendSyntax(parameterValue); + const returnValue = this.renderExpression(extendedExpression, data); if (typeof returnValue === 'function') { if (returnValue.name === '$') throw new Error('invalid syntax'); throw new Error('This is a function. Please add ()'); @@ -267,7 +284,10 @@ export class Expression { return returnValue; } - private renderExpression(expression: string, data: IWorkflowDataProxyData): tmpl.ReturnValue { + private renderExpression( + expression: string, + data: IWorkflowDataProxyData, + ): tmpl.ReturnValue | undefined { try { return tmpl.tmpl(expression, data); } catch (error) { @@ -279,10 +299,43 @@ export class Expression { } } } - return null; } + extendSyntax(bracketedExpression: string): string { + if (!hasExpressionExtension(bracketedExpression) || hasNativeMethod(bracketedExpression)) + return bracketedExpression; + + const chunks = splitExpression(bracketedExpression); + + const extendedChunks = chunks.map((chunk): ExpressionChunk => { + if (chunk.type === 'code') { + const output = extendTransform(chunk.text); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + if (!output?.code) { + throw new ExpressionExtensionError('Failed to extend syntax'); + } + + let text = output.code; + // We need to cut off any trailing semicolons. These cause issues + // with certain types of expression and cause the whole expression + // to fail. + if (text.trim().endsWith(';')) { + text = text.trim().slice(0, -1); + } + + return { + ...chunk, + text, + } as ExpressionCode; + } + return chunk; + }); + + return joinExpression(extendedChunks); + } + /** * Resolves value of parameter. But does not work for workflow-data. * @@ -439,6 +492,7 @@ export class Expression { selfData, ); } + return this.resolveSimpleParameterValue( value as NodeParameterValue, siblingParameters, diff --git a/packages/workflow/src/ExpressionError.ts b/packages/workflow/src/ExpressionError.ts index 2fbd972bc9576..f0972aa4f3105 100644 --- a/packages/workflow/src/ExpressionError.ts +++ b/packages/workflow/src/ExpressionError.ts @@ -50,3 +50,10 @@ export class ExpressionError extends ExecutionBaseError { } } } + +export class ExpressionExtensionError extends ExpressionError { + constructor(message: string) { + super(message); + this.context.failExecution = true; + } +} diff --git a/packages/workflow/src/Extensions/ArrayExtensions.ts b/packages/workflow/src/Extensions/ArrayExtensions.ts new file mode 100644 index 0000000000000..f66d412f5b429 --- /dev/null +++ b/packages/workflow/src/Extensions/ArrayExtensions.ts @@ -0,0 +1,392 @@ +import { ExpressionError, ExpressionExtensionError } from '../ExpressionError'; +import type { ExtensionMap } from './Extensions'; +import { compact as oCompact, merge as oMerge } from './ObjectExtensions'; + +function deepCompare(left: unknown, right: unknown): boolean { + if (left === right) { + return true; + } + + // Check to see if they're the basic type + if (typeof left !== typeof right) { + return false; + } + + if (typeof left === 'number' && isNaN(left) && isNaN(right as number)) { + return true; + } + + // Quickly check how many properties each has to avoid checking obviously mismatching + // objects + if (Object.keys(left as object).length !== Object.keys(right as object).length) { + return false; + } + + // Quickly check if they're arrays + if (Array.isArray(left) !== Array.isArray(right)) { + return false; + } + + // Check if arrays are equal, ordering is important + if (Array.isArray(left)) { + if (left.length !== (right as unknown[]).length) { + return false; + } + return left.every((v, i) => deepCompare(v, (right as object[])[i])); + } + + // Check right first quickly. This is to see if we have mismatched properties. + // We'll check the left more indepth later to cover all our bases. + for (const key in right as object) { + if ((left as object).hasOwnProperty(key) !== (right as object).hasOwnProperty(key)) { + return false; + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + } else if (typeof (left as any)[key] !== typeof (right as any)[key]) { + return false; + } + } + + // Check left more in depth + for (const key in left as object) { + if ((left as object).hasOwnProperty(key) !== (right as object).hasOwnProperty(key)) { + return false; + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + } else if (typeof (left as any)[key] !== typeof (right as any)[key]) { + return false; + } + + if ( + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + typeof (left as any)[key] === 'object' + ) { + if ( + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + (left as any)[key] !== (right as any)[key] && + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + !deepCompare((left as any)[key], (right as any)[key]) + ) { + return false; + } + } else { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + if ((left as any)[key] !== (right as any)[key]) { + return false; + } + } + } + + return true; +} + +function first(value: unknown[]): unknown { + return value[0]; +} + +function isBlank(value: unknown[]): boolean { + return value.length === 0; +} + +function isPresent(value: unknown[]): boolean { + return value.length > 0; +} + +function last(value: unknown[]): unknown { + return value[value.length - 1]; +} + +function length(value: unknown[]): number { + return Array.isArray(value) ? value.length : 0; +} + +function pluck(value: unknown[], extraArgs: unknown[]): unknown[] { + if (!Array.isArray(extraArgs)) { + throw new ExpressionError('arguments must be passed to pluck'); + } + const fieldsToPluck = extraArgs; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (value as any[]).map((element: object) => { + const entries = Object.entries(element); + return entries.reduce((p, c) => { + const [key, val] = c as [string, Date | string | number]; + if (fieldsToPluck.includes(key)) { + Object.assign(p, { [key]: val }); + } + return p; + }, {}); + }) as unknown[]; +} + +function random(value: unknown[]): unknown { + const len = value === undefined ? 0 : value.length; + return len ? value[Math.floor(Math.random() * len)] : undefined; +} + +function unique(value: unknown[], extraArgs: string[]): unknown[] { + if (extraArgs.length) { + return value.reduce((l, v) => { + if (typeof v === 'object' && v !== null && extraArgs.every((i) => i in v)) { + const alreadySeen = l.find((i) => + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + extraArgs.every((j) => deepCompare((i as any)[j], (v as any)[j])), + ); + if (!alreadySeen) { + l.push(v); + } + } + return l; + }, []); + } + return value.reduce((l, v) => { + if (l.findIndex((i) => deepCompare(i, v)) === -1) { + l.push(v); + } + return l; + }, []); +} + +function sum(value: unknown[]): number { + return value.reduce((p: number, c: unknown) => { + if (typeof c === 'string') { + return p + parseFloat(c); + } + if (typeof c !== 'number') { + return NaN; + } + return p + c; + }, 0); +} + +function min(value: unknown[]): number { + return Math.min( + ...value.map((v) => { + if (typeof v === 'string') { + return parseFloat(v); + } + if (typeof v !== 'number') { + return NaN; + } + return v; + }), + ); +} + +function max(value: unknown[]): number { + return Math.max( + ...value.map((v) => { + if (typeof v === 'string') { + return parseFloat(v); + } + if (typeof v !== 'number') { + return NaN; + } + return v; + }), + ); +} + +export function average(value: unknown[]) { + // This would usually be NaN but I don't think users + // will expect that + if (value.length === 0) { + return 0; + } + return sum(value) / value.length; +} + +function compact(value: unknown[]): unknown[] { + return value + .filter((v) => v !== null && v !== undefined) + .map((v) => { + if (typeof v === 'object' && v !== null) { + return oCompact(v); + } + return v; + }); +} + +function smartJoin(value: unknown[], extraArgs: string[]): object { + const [keyField, valueField] = extraArgs; + if (!keyField || !valueField || typeof keyField !== 'string' || typeof valueField !== 'string') { + throw new ExpressionExtensionError( + 'smartJoin requires 2 arguments: keyField and nameField. e.g. .smartJoin("name", "value")', + ); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-return + return value.reduce((o, v) => { + if (typeof v === 'object' && v !== null && keyField in v && valueField in v) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + o[(v as any)[keyField]] = (v as any)[valueField]; + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return o; + }, {}); +} + +function chunk(value: unknown[], extraArgs: number[]) { + const [chunkSize] = extraArgs; + if (typeof chunkSize !== 'number') { + throw new ExpressionExtensionError('chunk requires 1 parameter: chunkSize. e.g. .chunk(5)'); + } + const chunks: unknown[][] = []; + for (let i = 0; i < value.length; i += chunkSize) { + // I have no clue why eslint thinks 2 numbers could be anything but that but here we are + // eslint-disable-next-line @typescript-eslint/restrict-plus-operands + chunks.push(value.slice(i, i + chunkSize)); + } + return chunks; +} + +function filter(value: unknown[], extraArgs: unknown[]): unknown[] { + const [field, term] = extraArgs as [string | (() => void), unknown | string]; + if (typeof field !== 'string' && typeof field !== 'function') { + throw new ExpressionExtensionError( + 'filter requires 1 or 2 arguments: (field and term), (term and [optional keepOrRemove "keep" or "remove" default "keep"] (for string arrays)), or function. e.g. .filter("type", "home") or .filter((i) => i.type === "home") or .filter("home", [optional keepOrRemove]) (for string arrays)', + ); + } + if (value.every((i) => typeof i === 'string') && typeof field === 'string') { + return (value as string[]).filter((i) => + term === 'remove' ? !i.includes(field) : i.includes(field), + ); + } else if (typeof field === 'string') { + return value.filter( + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + (v) => typeof v === 'object' && v !== null && field in v && (v as any)[field] === term, + ); + } + return value.filter(field); +} + +function renameKeys(value: unknown[], extraArgs: string[]): unknown[] { + if (extraArgs.length === 0 || extraArgs.length % 2 !== 0) { + throw new ExpressionExtensionError( + 'renameKeys requires an even amount of arguments: from1, to1 [, from2, to2, ...]. e.g. .renameKeys("name", "title")', + ); + } + return value.map((v) => { + if (typeof v !== 'object' || v === null) { + return v; + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any + const newObj = { ...(v as any) }; + const chunkedArgs = chunk(extraArgs, [2]) as string[][]; + chunkedArgs.forEach(([from, to]) => { + if (from in newObj) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + newObj[to] = newObj[from]; + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + delete newObj[from]; + } + }); + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return newObj; + }); +} + +function merge(value: unknown[], extraArgs: unknown[][]): unknown[] { + const [others] = extraArgs; + if (!Array.isArray(others)) { + throw new ExpressionExtensionError( + 'merge requires 1 argument that is an array. e.g. .merge([{ id: 1, otherValue: 3 }])', + ); + } + const listLength = value.length > others.length ? value.length : others.length; + const newList = new Array(listLength); + for (let i = 0; i < listLength; i++) { + if (value[i] !== undefined) { + if (typeof value[i] === 'object' && typeof others[i] === 'object') { + newList[i] = oMerge(value[i] as object, [others[i]]); + } else { + newList[i] = value[i]; + } + } else { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + newList[i] = others[i]; + } + } + return newList; +} + +function union(value: unknown[], extraArgs: unknown[][]): unknown[] { + const [others] = extraArgs; + if (!Array.isArray(others)) { + throw new ExpressionExtensionError( + 'union requires 1 argument that is an array. e.g. .union([1, 2, 3, 4])', + ); + } + const newArr: unknown[] = Array.from(value); + for (const v of others) { + if (newArr.findIndex((w) => deepCompare(w, v)) === -1) { + newArr.push(v); + } + } + return unique(newArr, []); +} + +function difference(value: unknown[], extraArgs: unknown[][]): unknown[] { + const [others] = extraArgs; + if (!Array.isArray(others)) { + throw new ExpressionExtensionError( + 'difference requires 1 argument that is an array. e.g. .difference([1, 2, 3, 4])', + ); + } + const newArr: unknown[] = []; + for (const v of value) { + if (others.findIndex((w) => deepCompare(w, v)) === -1) { + newArr.push(v); + } + } + return unique(newArr, []); +} + +function intersection(value: unknown[], extraArgs: unknown[][]): unknown[] { + const [others] = extraArgs; + if (!Array.isArray(others)) { + throw new ExpressionExtensionError( + 'difference requires 1 argument that is an array. e.g. .difference([1, 2, 3, 4])', + ); + } + const newArr: unknown[] = []; + for (const v of value) { + if (others.findIndex((w) => deepCompare(w, v)) !== -1) { + newArr.push(v); + } + } + for (const v of others) { + if (value.findIndex((w) => deepCompare(w, v)) !== -1) { + newArr.push(v); + } + } + return unique(newArr, []); +} + +export const arrayExtensions: ExtensionMap = { + typeName: 'Array', + functions: { + count: length, + duplicates: unique, + filter, + first, + last, + length, + pluck, + unique, + random, + randomItem: random, + remove: unique, + size: length, + sum, + min, + max, + average, + isPresent, + isBlank, + compact, + smartJoin, + chunk, + renameKeys, + merge, + union, + difference, + intersection, + }, +}; diff --git a/packages/workflow/src/Extensions/DateExtensions.ts b/packages/workflow/src/Extensions/DateExtensions.ts new file mode 100644 index 0000000000000..a34238f1fbd61 --- /dev/null +++ b/packages/workflow/src/Extensions/DateExtensions.ts @@ -0,0 +1,268 @@ +/* eslint-disable @typescript-eslint/unbound-method */ +/* eslint-disable @typescript-eslint/explicit-member-accessibility */ +import { + DateTime, + DateTimeFormatOptions, + DateTimeUnit, + Duration, + DurationObjectUnits, + LocaleOptions, +} from 'luxon'; +import type { ExtensionMap } from './Extensions'; + +type DurationUnit = + | 'milliseconds' + | 'seconds' + | 'minutes' + | 'hours' + | 'days' + | 'weeks' + | 'months' + | 'quarter' + | 'years'; +type DatePart = + | 'day' + | 'month' + | 'year' + | 'hour' + | 'minute' + | 'second' + | 'millisecond' + | 'weekNumber' + | 'yearDayNumber' + | 'weekday'; + +const DURATION_MAP: Record = { + day: 'days', + month: 'months', + year: 'years', + week: 'weeks', + hour: 'hours', + minute: 'minutes', + second: 'seconds', + millisecond: 'milliseconds', + ms: 'milliseconds', + sec: 'seconds', + secs: 'seconds', + hr: 'hours', + hrs: 'hours', + min: 'minutes', + mins: 'minutes', +}; + +const DATETIMEUNIT_MAP: Record = { + days: 'day', + months: 'month', + years: 'year', + hours: 'hour', + minutes: 'minute', + seconds: 'second', + milliseconds: 'millisecond', + hrs: 'hour', + hr: 'hour', + mins: 'minute', + min: 'minute', + secs: 'second', + sec: 'second', + ms: 'millisecond', +}; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function isDateTime(date: any): date is DateTime { + if (date) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + return DateTime.isDateTime(date); + } + return false; +} + +function generateDurationObject(durationValue: number, unit: DurationUnit) { + const convertedUnit = DURATION_MAP[unit] || unit; + return { [`${convertedUnit}`]: durationValue } as DurationObjectUnits; +} + +function beginningOf(date: Date | DateTime, extraArgs: DurationUnit[]): Date { + const [unit = 'week'] = extraArgs; + + if (isDateTime(date)) { + return date.startOf(DATETIMEUNIT_MAP[unit] || unit).toJSDate(); + } + let datetime = DateTime.fromJSDate(date); + if (date.getTimezoneOffset() === 0) { + datetime = datetime.setZone('UTC'); + } + return datetime.startOf(DATETIMEUNIT_MAP[unit] || unit).toJSDate(); +} + +function endOfMonth(date: Date | DateTime): Date { + if (isDateTime(date)) { + return date.endOf('month').toJSDate(); + } + return DateTime.fromJSDate(date).endOf('month').toJSDate(); +} + +function extract(inputDate: Date | DateTime, extraArgs: DatePart[]): number | Date { + const [part] = extraArgs; + let date = inputDate; + if (isDateTime(date)) { + date = date.toJSDate(); + } + if (part === 'yearDayNumber') { + const firstDayOfTheYear = new Date(date.getFullYear(), 0, 0); + const diff = + date.getTime() - + firstDayOfTheYear.getTime() + + (firstDayOfTheYear.getTimezoneOffset() - date.getTimezoneOffset()) * 60 * 1000; + return Math.floor(diff / (1000 * 60 * 60 * 24)); + } + + return DateTime.fromJSDate(date).get((DATETIMEUNIT_MAP[part] as keyof DateTime) || part); +} + +function format(date: Date | DateTime, extraArgs: unknown[]): string { + const [dateFormat, localeOpts = {}] = extraArgs as [string, LocaleOptions]; + if (isDateTime(date)) { + return date.toFormat(dateFormat, { ...localeOpts }); + } + return DateTime.fromJSDate(date).toFormat(dateFormat, { ...localeOpts }); +} + +function isBetween(date: Date | DateTime, extraArgs: unknown[]): boolean { + const [first, second] = extraArgs as string[]; + const firstDate = new Date(first); + const secondDate = new Date(second); + + if (firstDate > secondDate) { + return secondDate < date && date < firstDate; + } + return secondDate > date && date > firstDate; +} + +function isDst(date: Date): boolean { + return DateTime.fromJSDate(date).isInDST; +} + +function isInLast(date: Date | DateTime, extraArgs: unknown[]): boolean { + const [durationValue = 0, unit = 'minutes'] = extraArgs as [number, DurationUnit]; + + const dateInThePast = DateTime.now().minus(generateDurationObject(durationValue, unit)); + let thisDate = date; + if (!isDateTime(thisDate)) { + thisDate = DateTime.fromJSDate(thisDate); + } + return dateInThePast <= thisDate && thisDate <= DateTime.now(); +} + +function isWeekend(date: Date): boolean { + enum DAYS { + saturday = 6, + sunday = 7, + } + return [DAYS.saturday, DAYS.sunday].includes(DateTime.fromJSDate(date).weekday); +} + +function minus(date: Date | DateTime, extraArgs: unknown[]): Date { + const [durationValue = 0, unit = 'minutes'] = extraArgs as [number, DurationUnit]; + + if (isDateTime(date)) { + return date.minus(generateDurationObject(durationValue, unit)).toJSDate(); + } + return DateTime.fromJSDate(date).minus(generateDurationObject(durationValue, unit)).toJSDate(); +} + +function plus(date: Date | DateTime, extraArgs: unknown[]): Date { + const [durationValue = 0, unit = 'minutes'] = extraArgs as [number, DurationUnit]; + + if (isDateTime(date)) { + return date.plus(generateDurationObject(durationValue, unit)).toJSDate(); + } + return DateTime.fromJSDate(date).plus(generateDurationObject(durationValue, unit)).toJSDate(); +} + +function toLocaleString(date: Date | DateTime, extraArgs: unknown[]): string { + const [locale, dateFormat = { timeStyle: 'short', dateStyle: 'short' }] = extraArgs as [ + string | undefined, + DateTimeFormatOptions, + ]; + + if (isDateTime(date)) { + return date.toLocaleString(dateFormat, { locale }); + } + return DateTime.fromJSDate(date).toLocaleString(dateFormat, { locale }); +} + +function toTimeFromNow(date: Date): string { + let diffObj: Duration; + if (isDateTime(date)) { + diffObj = date.diffNow(); + } else { + diffObj = DateTime.fromJSDate(date).diffNow(); + } + + const as = (unit: DurationUnit) => { + return Math.round(Math.abs(diffObj.as(unit))); + }; + + if (as('years')) { + return `${as('years')} years ago`; + } + if (as('months')) { + return `${as('months')} months ago`; + } + if (as('weeks')) { + return `${as('weeks')} weeks ago`; + } + if (as('days')) { + return `${as('days')} days ago`; + } + if (as('hours')) { + return `${as('hours')} hours ago`; + } + if (as('minutes')) { + return `${as('minutes')} minutes ago`; + } + if (as('seconds') && as('seconds') > 10) { + return `${as('seconds')} seconds ago`; + } + return 'just now'; +} + +function timeTo(date: Date | DateTime, extraArgs: unknown[]): Duration { + const [diff = new Date().toISOString(), unit = 'seconds'] = extraArgs as [string, DurationUnit]; + const diffDate = new Date(diff); + if (isDateTime(date)) { + return date.diff(DateTime.fromJSDate(diffDate), DURATION_MAP[unit] || unit); + } + return DateTime.fromJSDate(date).diff(DateTime.fromJSDate(diffDate), DURATION_MAP[unit] || unit); +} + +function toDate(date: Date | DateTime) { + if (isDateTime(date)) { + return date.set({ hour: 0, minute: 0, second: 0, millisecond: 0 }).toJSDate(); + } + let datetime = DateTime.fromJSDate(date); + if (date.getTimezoneOffset() === 0) { + datetime = datetime.setZone('UTC'); + } + return datetime.set({ hour: 0, minute: 0, second: 0, millisecond: 0 }).toJSDate(); +} + +export const dateExtensions: ExtensionMap = { + typeName: 'Date', + functions: { + beginningOf, + endOfMonth, + extract, + isBetween, + isDst, + isInLast, + isWeekend, + minus, + plus, + toTimeFromNow, + timeTo, + format, + toLocaleString, + toDate, + }, +}; diff --git a/packages/workflow/src/Extensions/ExpressionExtension.ts b/packages/workflow/src/Extensions/ExpressionExtension.ts new file mode 100644 index 0000000000000..ecae3f00c2c74 --- /dev/null +++ b/packages/workflow/src/Extensions/ExpressionExtension.ts @@ -0,0 +1,270 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +import { DateTime } from 'luxon'; +import { ExpressionExtensionError } from '../ExpressionError'; +import { parse, visit, types, print } from 'recast'; + +import { arrayExtensions } from './ArrayExtensions'; +import { dateExtensions } from './DateExtensions'; +import { numberExtensions } from './NumberExtensions'; +import { stringExtensions } from './StringExtensions'; +import { objectExtensions } from './ObjectExtensions'; + +const EXPRESSION_EXTENDER = 'extend'; + +function isBlank(value: unknown) { + return value === null || value === undefined || !value; +} + +function isPresent(value: unknown) { + return !isBlank(value); +} + +const EXTENSION_OBJECTS = [ + arrayExtensions, + dateExtensions, + numberExtensions, + objectExtensions, + stringExtensions, +]; + +// eslint-disable-next-line @typescript-eslint/ban-types +const genericExtensions: Record = { + isBlank, + isPresent, +}; + +const EXPRESSION_EXTENSION_METHODS = Array.from( + new Set([ + ...Object.keys(stringExtensions.functions), + ...Object.keys(numberExtensions.functions), + ...Object.keys(dateExtensions.functions), + ...Object.keys(arrayExtensions.functions), + ...Object.keys(objectExtensions.functions), + ...Object.keys(genericExtensions), + '$if', + ]), +); + +const isExpressionExtension = (str: string) => EXPRESSION_EXTENSION_METHODS.some((m) => m === str); + +export const hasExpressionExtension = (str: string): boolean => + EXPRESSION_EXTENSION_METHODS.some((m) => str.includes(m)); + +export const hasNativeMethod = (method: string): boolean => { + if (hasExpressionExtension(method)) { + return false; + } + const methods = method + .replace(/[^\w\s]/gi, ' ') + .split(' ') + .filter(Boolean); // DateTime.now().toLocaleString().format() => [DateTime,now,toLocaleString,format] + return methods.every((methodName) => { + return [String.prototype, Array.prototype, Number.prototype, Date.prototype].some( + (nativeType) => { + if (methodName in nativeType) { + return true; + } + + return false; + }, + ); + }); +}; + +/** + * recast's types aren't great and we need to use a lot of anys + */ + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const findParent = (path: T, matcher: (path: T) => boolean): T | undefined => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + let parent = path.parentPath; + while (parent) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + if (matcher(parent)) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return parent; + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + parent = parent.parentPath; + } + return; +}; + +/** + * A function to inject an extender function call into the AST of an expression. + * This uses recast to do the transform. + * + * ```ts + * 'a'.method('x') // becomes + * extend('a', 'method', ['x']); + * + * 'a'.first('x').second('y') // becomes + * extend(extend('a', 'first', ['x']), 'second', ['y'])); + * ``` + */ +export const extendTransform = (expression: string): { code: string } | undefined => { + try { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const ast = parse(expression); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + visit(ast, { + visitIdentifier(path) { + this.traverse(path); + if (path.node.name === '$if') { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const callPath: any = findParent(path, (p) => p.value?.type === 'CallExpression'); + if (!callPath || callPath.value?.type !== 'CallExpression') { + return; + } + + if (callPath.node.arguments.length < 2) { + throw new ExpressionExtensionError( + '$if requires at least 2 parameters: test, value_if_true[, and value_if_false]', + ); + } + + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const test = callPath.node.arguments[0]; + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const consequent = callPath.node.arguments[1]; + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const alternative = + callPath.node.arguments[2] === undefined + ? types.builders.booleanLiteral(false) + : callPath.node.arguments[2]; + + // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-argument + callPath.replace(types.builders.conditionalExpression(test, consequent, alternative)); + } + }, + }); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + visit(ast, { + visitIdentifier(path) { + this.traverse(path); + if ( + isExpressionExtension(path.node.name) && + // types.namedTypes.MemberExpression.check(path.parent) + path.parentPath?.value?.type === 'MemberExpression' + ) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const callPath: any = findParent(path, (p) => p.value?.type === 'CallExpression'); + + if (!callPath || callPath.value?.type !== 'CallExpression') { + return; + } + + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + callPath.replace( + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + types.builders.callExpression(types.builders.identifier(EXPRESSION_EXTENDER), [ + path.parentPath.value.object, + types.builders.stringLiteral(path.node.name), + // eslint-disable-next-line + types.builders.arrayExpression(callPath.node.arguments), + ]), + ); + } + }, + }); + // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-argument + return print(ast); + } catch (e) { + return; + } +}; + +function isDate(input: unknown): boolean { + if (typeof input !== 'string' || !input.length) { + return false; + } + if (!/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/.test(input)) { + return false; + } + const d = new Date(input); + return d instanceof Date && !isNaN(d.valueOf()) && d.toISOString() === input; +} + +/** + * Extender function injected by expression extension plugin to allow calls to extensions. + * + * ```ts + * extend(input, "functionName", [...args]); + * ``` + */ +export function extend(input: unknown, functionName: string, args: unknown[]) { + // eslint-disable-next-line @typescript-eslint/ban-types + let foundFunction: Function | undefined; + if (Array.isArray(input)) { + foundFunction = arrayExtensions.functions[functionName]; + } else if (isDate(input) && functionName !== 'toDate') { + // If it's a string date (from $json), convert it to a Date object, + // unless that function is `toDate`, since `toDate` does something + // very different on date objects + input = new Date(input as string); + foundFunction = dateExtensions.functions[functionName]; + } else if (typeof input === 'string') { + foundFunction = stringExtensions.functions[functionName]; + } else if (typeof input === 'number') { + foundFunction = numberExtensions.functions[functionName]; + } else if (input && (DateTime.isDateTime(input) || input instanceof Date)) { + foundFunction = dateExtensions.functions[functionName]; + } else if (input !== null && typeof input === 'object') { + foundFunction = objectExtensions.functions[functionName]; + } + + // Look for generic or builtin + if (!foundFunction) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const inputAny: any = input; + // This is likely a builtin we're implementing for another type + // (e.g. toLocaleString). We'll just call it + if ( + inputAny && + functionName && + functionName in inputAny && + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + typeof inputAny[functionName] === 'function' + ) { + // I was having weird issues with eslint not finding rules on this line. + // Just disabling all eslint rules for now. + // eslint-disable-next-line + return inputAny[functionName](...args); + } + + // Use a generic version if available + foundFunction = genericExtensions[functionName]; + } + + // No type specific or generic function found. Check to see if + // any types have a function with that name. Then throw an error + // letting the user know the available types. + if (!foundFunction) { + const haveFunction = EXTENSION_OBJECTS.filter((v) => functionName in v.functions); + if (!haveFunction.length) { + // This shouldn't really be possible but we should cover it anyway + throw new ExpressionExtensionError(`Unknown expression function: ${functionName}`); + } + + if (haveFunction.length > 1) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const lastType = `"${haveFunction.pop()!.typeName}"`; + const typeNames = `${haveFunction.map((v) => `"${v.typeName}"`).join(', ')}, and ${lastType}`; + throw new ExpressionExtensionError( + `${functionName}() is only callable on types ${typeNames}`, + ); + } else { + throw new ExpressionExtensionError( + `${functionName}() is only callable on type "${haveFunction[0].typeName}"`, + ); + } + } + + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return foundFunction(input, args); +} diff --git a/packages/workflow/src/Extensions/ExpressionParser.ts b/packages/workflow/src/Extensions/ExpressionParser.ts new file mode 100644 index 0000000000000..3e94bfb2eb931 --- /dev/null +++ b/packages/workflow/src/Extensions/ExpressionParser.ts @@ -0,0 +1,99 @@ +export interface ExpressionText { + type: 'text'; + text: string; +} + +export interface ExpressionCode { + type: 'code'; + text: string; + // tmpl has different behaviours if the last expression + // doesn't close itself. + hasClosingBrackets: boolean; +} + +export type ExpressionChunk = ExpressionCode | ExpressionText; + +const OPEN_BRACKET = /(?\\|)(?\{\{)/; +const CLOSE_BRACKET = /(?\\|)(?\}\})/; + +export const escapeCode = (text: string): string => { + return text.replace('\\}}', '}}'); +}; + +export const splitExpression = (expression: string): ExpressionChunk[] => { + const chunks: ExpressionChunk[] = []; + let searchingFor: 'open' | 'close' = 'open'; + let activeRegex = OPEN_BRACKET; + + let buffer = ''; + + let index = 0; + + while (index < expression.length) { + const expr = expression.slice(index); + const res = activeRegex.exec(expr); + // No more brackets. If it's a closing bracket + // this is sort of valid so we accept it but mark + // that it has no closing bracket. + if (!res?.groups) { + buffer += expr; + if (searchingFor === 'open') { + chunks.push({ + type: 'text', + text: buffer, + }); + } else { + chunks.push({ + type: 'code', + text: escapeCode(buffer), + hasClosingBrackets: false, + }); + } + break; + } + if (res.groups.escape) { + buffer += expr.slice(0, res.index + 3); + index += res.index + 3; + } else { + buffer += expr.slice(0, res.index); + + if (searchingFor === 'open') { + chunks.push({ + type: 'text', + text: buffer, + }); + searchingFor = 'close'; + activeRegex = CLOSE_BRACKET; + } else { + chunks.push({ + type: 'code', + text: escapeCode(buffer), + hasClosingBrackets: true, + }); + searchingFor = 'open'; + activeRegex = OPEN_BRACKET; + } + + index += res.index + 2; + buffer = ''; + } + } + + return chunks; +}; + +// Expressions only have closing brackets escaped +const escapeTmplExpression = (part: string) => { + return part.replace('}}', '\\}}'); +}; + +export const joinExpression = (parts: ExpressionChunk[]): string => { + return parts + .map((chunk) => { + if (chunk.type === 'code') { + return `{{${escapeTmplExpression(chunk.text)}${chunk.hasClosingBrackets ? '}}' : ''}`; + } + return chunk.text; + }) + .join(''); +}; diff --git a/packages/workflow/src/Extensions/ExtendedFunctions.ts b/packages/workflow/src/Extensions/ExtendedFunctions.ts new file mode 100644 index 0000000000000..73fb21ca90696 --- /dev/null +++ b/packages/workflow/src/Extensions/ExtendedFunctions.ts @@ -0,0 +1,53 @@ +import { ExpressionExtensionError } from '../ExpressionError'; +import { average as aAverage } from './ArrayExtensions'; + +const min = Math.min; +const max = Math.max; + +const numberList = (start: number, end: number): number[] => { + const size = Math.abs(start - end) + 1; + const arr = new Array(size); + + let curr = start; + for (let i = 0; i < size; i++) { + if (start < end) { + arr[i] = curr++; + } else { + arr[i] = curr--; + } + } + + return arr; +}; + +const zip = (keys: unknown[], values: unknown[]): unknown => { + if (keys.length !== values.length) { + throw new ExpressionExtensionError('keys and values not of equal length'); + } + return keys.reduce((p, c, i) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + (p as any)[c as any] = values[i]; + return p; + }, {}); +}; + +const average = (...args: number[]) => { + return aAverage(args); +}; + +const not = (value: unknown): boolean => { + return !value; +}; + +export const extendedFunctions = { + min, + max, + not, + average, + numberList, + zip, + $min: min, + $max: max, + $average: average, + $not: not, +}; diff --git a/packages/workflow/src/Extensions/Extensions.ts b/packages/workflow/src/Extensions/Extensions.ts new file mode 100644 index 0000000000000..460567d0ebb01 --- /dev/null +++ b/packages/workflow/src/Extensions/Extensions.ts @@ -0,0 +1,5 @@ +export interface ExtensionMap { + typeName: string; + // eslint-disable-next-line @typescript-eslint/ban-types + functions: Record; +} diff --git a/packages/workflow/src/Extensions/NumberExtensions.ts b/packages/workflow/src/Extensions/NumberExtensions.ts new file mode 100644 index 0000000000000..afa20c0b652ca --- /dev/null +++ b/packages/workflow/src/Extensions/NumberExtensions.ts @@ -0,0 +1,73 @@ +/** + * @jest-environment jsdom + */ + +import { ExtensionMap } from './Extensions'; + +function format(value: number, extraArgs: unknown[]): string { + const [locales = 'en-US', config = {}] = extraArgs as [ + string | string[], + Intl.NumberFormatOptions, + ]; + + return new Intl.NumberFormat(locales, config).format(value); +} + +function isBlank(value: number): boolean { + return typeof value !== 'number'; +} + +function isPresent(value: number): boolean { + return !isBlank(value); +} + +function random(value: number): number { + return Math.floor(Math.random() * value); +} + +function isTrue(value: number) { + return value === 1; +} + +function isFalse(value: number) { + return value === 0; +} + +function isEven(value: number) { + return value % 2 === 0; +} + +function isOdd(value: number) { + return Math.abs(value) % 2 === 1; +} + +function floor(value: number) { + return Math.floor(value); +} + +function ceil(value: number) { + return Math.ceil(value); +} + +function round(value: number, extraArgs: number[]) { + const [decimalPlaces = 0] = extraArgs; + return +value.toFixed(decimalPlaces); +} + +export const numberExtensions: ExtensionMap = { + typeName: 'Number', + functions: { + ceil, + floor, + format, + random, + round, + isBlank, + isPresent, + isTrue, + isNotTrue: isFalse, + isFalse, + isEven, + isOdd, + }, +}; diff --git a/packages/workflow/src/Extensions/ObjectExtensions.ts b/packages/workflow/src/Extensions/ObjectExtensions.ts new file mode 100644 index 0000000000000..b3ef5dde57b1f --- /dev/null +++ b/packages/workflow/src/Extensions/ObjectExtensions.ts @@ -0,0 +1,104 @@ +import { ExpressionExtensionError } from '../ExpressionError'; +import type { ExtensionMap } from './Extensions'; + +export function merge(value: object, extraArgs: unknown[]): unknown { + const [other] = extraArgs; + if (typeof other !== 'object' || !other) { + throw new ExpressionExtensionError('argument of merge must be an object'); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const newObject: any = { ...value }; + for (const [key, val] of Object.entries(other)) { + if (!(key in newObject)) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + newObject[key] = val; + } + } + return newObject; +} + +function isEmpty(value: object): boolean { + return Object.keys(value).length === 0; +} + +function hasField(value: object, extraArgs: string[]): boolean { + const [name] = extraArgs; + return name in value; +} + +function removeField(value: object, extraArgs: string[]): object { + const [name] = extraArgs; + if (name in value) { + const newObject = { ...value }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + delete (newObject as any)[name]; + return newObject; + } + return value; +} + +function removeFieldsContaining(value: object, extraArgs: string[]): object { + const [match] = extraArgs; + if (typeof match !== 'string') { + throw new ExpressionExtensionError('argument of removeFieldsContaining must be an string'); + } + const newObject = { ...value }; + for (const [key, val] of Object.entries(value)) { + if (typeof val === 'string' && val.includes(match)) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + delete (newObject as any)[key]; + } + } + return newObject; +} + +function keepFieldsContaining(value: object, extraArgs: string[]): object { + const [match] = extraArgs; + if (typeof match !== 'string') { + throw new ExpressionExtensionError('argument of keepFieldsContaining must be an string'); + } + const newObject = { ...value }; + for (const [key, val] of Object.entries(value)) { + if (typeof val === 'string' && !val.includes(match)) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + delete (newObject as any)[key]; + } + } + return newObject; +} + +export function compact(value: object): object { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const newObj: any = {}; + for (const [key, val] of Object.entries(value)) { + if (val !== null && val !== undefined) { + if (typeof val === 'object') { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument + newObj[key] = compact(val); + } else { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + newObj[key] = val; + } + } + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return newObj; +} + +export function urlEncode(value: object) { + return new URLSearchParams(value as Record).toString(); +} + +export const objectExtensions: ExtensionMap = { + typeName: 'Object', + functions: { + isEmpty, + merge, + hasField, + removeField, + removeFieldsContaining, + keepFieldsContaining, + compact, + urlEncode, + }, +}; diff --git a/packages/workflow/src/Extensions/StringExtensions.ts b/packages/workflow/src/Extensions/StringExtensions.ts new file mode 100644 index 0000000000000..e0ce8cb4d3371 --- /dev/null +++ b/packages/workflow/src/Extensions/StringExtensions.ts @@ -0,0 +1,305 @@ +/* eslint-disable @typescript-eslint/unbound-method */ +// import { createHash } from 'crypto'; +import * as ExpressionError from '../ExpressionError'; +import type { ExtensionMap } from './Extensions'; +import CryptoJS from 'crypto-js'; +import { encode } from 'js-base64'; +import { transliterate } from 'transliteration'; + +const hashFunctions: Record = { + md5: CryptoJS.MD5, + sha1: CryptoJS.SHA1, + sha224: CryptoJS.SHA224, + sha256: CryptoJS.SHA256, + sha384: CryptoJS.SHA384, + sha512: CryptoJS.SHA512, + sha3: CryptoJS.SHA3, + ripemd160: CryptoJS.RIPEMD160, +}; + +// All symbols from https://www.xe.com/symbols/ as for 2022/11/09 +const CURRENCY_REGEXP = + /(\u004c\u0065\u006b|\u060b|\u0024|\u0192|\u20bc|\u0042\u0072|\u0042\u005a\u0024|\u0024\u0062|\u004b\u004d|\u0050|\u043b\u0432|\u0052\u0024|\u17db|\u00a5|\u20a1|\u006b\u006e|\u20b1|\u004b\u010d|\u006b\u0072|\u0052\u0044\u0024|\u00a3|\u20ac|\u00a2|\u0051|\u004c|\u0046\u0074|\u20b9|\u0052\u0070|\ufdfc|\u20aa|\u004a\u0024|\u20a9|\u20ad|\u0434\u0435\u043d|\u0052\u004d|\u20a8|\u20ae|\u004d\u0054|\u0043\u0024|\u20a6|\u0042\u002f\u002e|\u0047\u0073|\u0053\u002f\u002e|\u007a\u0142|\u006c\u0065\u0069|\u20bd|\u0414\u0438\u043d\u002e|\u0053|\u0052|\u0043\u0048\u0046|\u004e\u0054\u0024|\u0e3f|\u0054\u0054\u0024|\u20ba|\u20b4|\u0024\u0055|\u0042\u0073|\u20ab|\u005a\u0024)/gu; +const DOMAIN_REGEXP = /^[a-zA-Z0-9][a-zA-Z0-9-]{1,61}[a-zA-Z0-9](?:\.[a-zA-Z]{2,})+$/; + +// This won't validate or catch literally valid email address, just what most people +// would expect +const EMAIL_REGEXP = + /(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@(?(\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))/; + +// This also might not catch every possible URL +const URL_REGEXP = + /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{2,}\b([-a-zA-Z0-9()\[\]@:%_\+.~#?&//=]*)/; + +const CHAR_TEST_REGEXP = /\p{L}/u; +const PUNC_TEST_REGEXP = /[!?.]/; + +const TRUE_VALUES = ['true', '1', 't', 'yes', 'y']; +const FALSE_VALUES = ['false', '0', 'f', 'no', 'n']; + +function encrypt(value: string, extraArgs?: unknown): string { + const [format = 'MD5'] = extraArgs as string[]; + if (format.toLowerCase() === 'base64') { + // We're using a library instead of btoa because btoa only + // works on ASCII + return encode(value); + } + const hashFunction = hashFunctions[format.toLowerCase()]; + if (!hashFunction) { + throw new ExpressionError.ExpressionExtensionError( + `Unknown encrypt type ${format}. Available types are: ${Object.keys(hashFunctions) + .map((s) => s.toUpperCase()) + .join(', ')}, and Base64.`, + ); + } + return hashFunction(value.toString()).toString(); + // return createHash(format).update(value.toString()).digest('hex'); +} + +function getOnlyFirstCharacters(value: string, extraArgs: number[]): string { + const [end] = extraArgs; + + if (typeof end !== 'number') { + throw new ExpressionError.ExpressionExtensionError( + 'getOnlyFirstCharacters() requires a argument', + ); + } + + return value.slice(0, end); +} + +function isBlank(value: string): boolean { + return value === ''; +} + +function isPresent(value: string): boolean { + return !isBlank(value); +} + +function length(value: string): number { + return value.length; +} + +function removeMarkdown(value: string): string { + let output = value; + try { + output = output.replace(/^([\s\t]*)([*\-+]|\d\.)\s+/gm, '$1'); + + output = output + // Header + .replace(/\n={2,}/g, '\n') + // Strikethrough + .replace(/~~/g, '') + // Fenced codeblocks + .replace(/`{3}.*\n/g, ''); + + output = output + // Remove HTML tags + .replace(/<[\w|\s|=|'|"|:|(|)|,|;|/|0-9|.|-]+[>|\\>]/g, '') + // Remove setext-style headers + .replace(/^[=-]{2,}\s*$/g, '') + // Remove footnotes? + .replace(/\[\^.+?\](: .*?$)?/g, '') + .replace(/\s{0,2}\[.*?\]: .*?$/g, '') + // Remove images + .replace(/!\[.*?\][[(].*?[\])]/g, '') + // Remove inline links + .replace(/\[(.*?)\][[(].*?[\])]/g, '$1') + // Remove Blockquotes + .replace(/>/g, '') + // Remove reference-style links? + .replace(/^\s{1,2}\[(.*?)\]: (\S+)( ".*?")?\s*$/g, '') + // Remove atx-style headers + .replace(/^#{1,6}\s*([^#]*)\s*(#{1,6})?/gm, '$1') + .replace(/([*_]{1,3})(\S.*?\S)\1/g, '$2') + .replace(/(`{3,})(.*?)\1/gm, '$2') + .replace(/^-{3,}\s*$/g, '') + .replace(/`(.+?)`/g, '$1') + .replace(/\n{2,}/g, '\n\n'); + } catch (e) { + return value; + } + return output; +} + +function sayHi(value: string) { + return `hi ${value}`; +} + +function stripTags(value: string): string { + return value.replace(/<[^>]*>?/gm, ''); +} + +function toDate(value: string): Date { + return new Date(value.toString()); +} + +function urlDecode(value: string, extraArgs: boolean[]): string { + const [entireString = false] = extraArgs; + if (entireString) { + return decodeURI(value.toString()); + } + return decodeURIComponent(value.toString()); +} + +function urlEncode(value: string, extraArgs: boolean[]): string { + const [entireString = false] = extraArgs; + if (entireString) { + return encodeURI(value.toString()); + } + return encodeURIComponent(value.toString()); +} + +function toInt(value: string, extraArgs: Array) { + const [radix] = extraArgs; + return parseInt(value.replace(CURRENCY_REGEXP, ''), radix); +} + +function toFloat(value: string) { + return parseFloat(value.replace(CURRENCY_REGEXP, '')); +} + +function quote(value: string, extraArgs: string[]) { + const [quoteChar = '"'] = extraArgs; + return `${quoteChar}${value + .replace(/\\/g, '\\\\') + .replace(new RegExp(`\\${quoteChar}`, 'g'), `\\${quoteChar}`)}${quoteChar}`; +} + +function isTrue(value: string) { + return TRUE_VALUES.includes(value.toLowerCase()); +} + +function isFalse(value: string) { + return FALSE_VALUES.includes(value.toLowerCase()); +} + +function isNumeric(value: string) { + return !isNaN(value as unknown as number) && !isNaN(parseFloat(value)); +} + +function isUrl(value: string) { + let url: URL; + try { + url = new URL(value); + } catch (_error) { + return false; + } + return url.protocol === 'http:' || url.protocol === 'https:'; +} + +function isDomain(value: string) { + return DOMAIN_REGEXP.test(value); +} + +function isEmail(value: string) { + return EMAIL_REGEXP.test(value); +} + +function stripSpecialChars(value: string) { + return transliterate(value, { unknown: '?' }); +} + +function toTitleCase(value: string) { + return value.replace(/\w\S*/g, (v) => v.charAt(0).toLocaleUpperCase() + v.slice(1)); +} + +function toSentenceCase(value: string) { + let current = value.slice(); + let buffer = ''; + + while (CHAR_TEST_REGEXP.test(current)) { + const charIndex = current.search(CHAR_TEST_REGEXP); + current = + current.slice(0, charIndex) + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + current[charIndex]!.toLocaleUpperCase() + + current.slice(charIndex + 1).toLocaleLowerCase(); + const puncIndex = current.search(PUNC_TEST_REGEXP); + if (puncIndex === -1) { + buffer += current; + current = ''; + break; + } + buffer += current.slice(0, puncIndex + 1); + current = current.slice(puncIndex + 1); + } + + return buffer; +} + +function toSnakeCase(value: string) { + return value + .toLocaleLowerCase() + .replace(/[ \-]/g, '_') + .replace(/[\u2000-\u206F\u2E00-\u2E7F\\'!"#$%&()*+,.\/:;<=>?@\[\]^`{|}~]/g, ''); +} + +function extractEmail(value: string) { + const matched = EMAIL_REGEXP.exec(value); + if (!matched) { + return undefined; + } + return matched[0]; +} + +function extractDomain(value: string) { + if (isEmail(value)) { + const matched = EMAIL_REGEXP.exec(value); + // This shouldn't happen + if (!matched) { + return undefined; + } + return matched.groups?.domain; + } else if (isUrl(value)) { + return new URL(value).hostname; + } + return undefined; +} + +function extractUrl(value: string) { + const matched = URL_REGEXP.exec(value); + if (!matched) { + return undefined; + } + return matched[0]; +} + +export const stringExtensions: ExtensionMap = { + typeName: 'String', + functions: { + encrypt, + hash: encrypt, + getOnlyFirstCharacters, + removeMarkdown, + sayHi, + stripTags, + toBoolean: isTrue, + toDate, + toDecimalNumber: toFloat, + toFloat, + toInt, + toWholeNumber: toInt, + toSentenceCase, + toSnakeCase, + toTitleCase, + urlDecode, + urlEncode, + quote, + stripSpecialChars, + length, + isDomain, + isEmail, + isTrue, + isFalse, + isNotTrue: isFalse, + isNumeric, + isUrl, + isURL: isUrl, + isBlank, + isPresent, + extractEmail, + extractDomain, + extractUrl, + }, +}; diff --git a/packages/workflow/src/Extensions/index.ts b/packages/workflow/src/Extensions/index.ts new file mode 100644 index 0000000000000..9bc07b1e8ba67 --- /dev/null +++ b/packages/workflow/src/Extensions/index.ts @@ -0,0 +1,7 @@ +// eslint-disable-next-line import/no-cycle +export { + extend, + hasExpressionExtension, + hasNativeMethod, + extendTransform, +} from './ExpressionExtension'; diff --git a/packages/workflow/test/ExpressionExtensions/ArrayExtensions.test.ts b/packages/workflow/test/ExpressionExtensions/ArrayExtensions.test.ts new file mode 100644 index 0000000000000..bd696b3f90d4c --- /dev/null +++ b/packages/workflow/test/ExpressionExtensions/ArrayExtensions.test.ts @@ -0,0 +1,198 @@ +/** + * @jest-environment jsdom + */ + +import { evaluate } from './Helpers'; + +describe('Data Transformation Functions', () => { + describe('Array Data Transformation Functions', () => { + test('.random() should work correctly on an array', () => { + expect(evaluate('={{ [1,2,3].random() }}')).not.toBeUndefined(); + }); + + test('.randomItem() alias should work correctly on an array', () => { + expect(evaluate('={{ [1,2,3].randomItem() }}')).not.toBeUndefined(); + }); + + test('.isPresent() should work correctly on an array', () => { + expect(evaluate('={{ [1,2,3, "imhere"].isPresent() }}')).toEqual(true); + }); + + test('.pluck() should work correctly on an array', () => { + expect( + evaluate(`={{ [ + { value: 1, string: '1' }, + { value: 2, string: '2' }, + { value: 3, string: '3' }, + { value: 4, string: '4' }, + { value: 5, string: '5' }, + { value: 6, string: '6' } + ].pluck("value") }}`), + ).toEqual( + expect.arrayContaining([ + { value: 1 }, + { value: 2 }, + { value: 3 }, + { value: 4 }, + { value: 5 }, + { value: 6 }, + ]), + ); + }); + + test('.unique() should work correctly on an array', () => { + expect(evaluate('={{ ["repeat","repeat","a","b","c"].unique() }}')).toEqual( + expect.arrayContaining(['repeat', 'repeat', 'a', 'b', 'c']), + ); + }); + + test('.isBlank() should work correctly on an array', () => { + expect(evaluate('={{ [].isBlank() }}')).toEqual(true); + }); + + test('.isBlank() should work correctly on an array', () => { + expect(evaluate('={{ [1].isBlank() }}')).toEqual(false); + }); + + test('.length() should work correctly on an array', () => { + expect(evaluate('={{ [].length() }}')).toEqual(0); + }); + + test('.count() should work correctly on an array', () => { + expect(evaluate('={{ [1].count() }}')).toEqual(1); + }); + + test('.size() should work correctly on an array', () => { + expect(evaluate('={{ [1,2].size() }}')).toEqual(2); + }); + + test('.last() should work correctly on an array', () => { + expect(evaluate('={{ ["repeat","repeat","a","b","c"].last() }}')).toEqual('c'); + }); + + test('.first() should work correctly on an array', () => { + expect(evaluate('={{ ["repeat","repeat","a","b","c"].first() }}')).toEqual('repeat'); + }); + + test('.filter() should work correctly on an array', () => { + expect(evaluate('={{ ["repeat","repeat","a","b","c"].filter("repeat") }}')).toEqual( + expect.arrayContaining(['repeat', 'repeat']), + ); + }); + + test('.merge() should work correctly on an array', () => { + expect( + evaluate( + '={{ [{ test1: 1, test2: 2 }, { test1: 1, test3: 3 }].merge([{ test1: 2, test3: 3 }, { test4: 4 }]) }}', + ), + ).toEqual([ + { test1: 1, test2: 2, test3: 3 }, + { test1: 1, test3: 3, test4: 4 }, + ]); + }); + + test('.smartJoin() should work correctly on an array of objects', () => { + expect( + evaluate( + '={{ [{ name: "test1", value: "value1" }, { name: "test2", value: null }].smartJoin("name", "value") }}', + ), + ).toEqual({ + test1: 'value1', + test2: null, + }); + }); + + test('.renameKeys() should work correctly on an array of objects', () => { + expect( + evaluate( + '={{ [{ test1: 1, test2: 2 }, { test1: 1, test3: 3 }].renameKeys("test1", "rename1", "test3", "rename3") }}', + ), + ).toEqual([ + { rename1: 1, test2: 2 }, + { rename1: 1, rename3: 3 }, + ]); + }); + + test('.sum() should work on an array of numbers', () => { + expect(evaluate('={{ [1, 2, 3, 4, 5, 6].sum() }}')).toEqual(21); + expect(evaluate('={{ ["1", 2, 3, 4, 5, 6].sum() }}')).toEqual(21); + expect(evaluate('={{ ["1", 2, 3, 4, 5, "bad"].sum() }}')).toBeNaN(); + }); + + test('.average() should work on an array of numbers', () => { + expect(evaluate('={{ [1, 2, 3, 4, 5, 6].average() }}')).toEqual(3.5); + expect(evaluate('={{ ["1", 2, 3, 4, 5, 6].average() }}')).toEqual(3.5); + expect(evaluate('={{ ["1", 2, 3, 4, 5, "bad"].average() }}')).toBeNaN(); + }); + + test('.min() should work on an array of numbers', () => { + expect(evaluate('={{ [1, 2, 3, 4, 5, 6].min() }}')).toEqual(1); + expect(evaluate('={{ ["1", 2, 3, 4, 5, 6].min() }}')).toEqual(1); + expect(evaluate('={{ ["1", 2, 3, 4, 5, "bad"].min() }}')).toBeNaN(); + }); + + test('.max() should work on an array of numbers', () => { + expect(evaluate('={{ [1, 2, 3, 4, 5, 6].max() }}')).toEqual(6); + expect(evaluate('={{ ["1", 2, 3, 4, 5, 6].max() }}')).toEqual(6); + expect(evaluate('={{ ["1", 2, 3, 4, 5, "bad"].max() }}')).toBeNaN(); + }); + + test('.union() should work on an array of objects', () => { + expect( + evaluate( + '={{ [{ test1: 1 }, { test2: 2 }].union([{ test1: 1, test3: 3 }, { test2: 2 }, { test4: 4 }]) }}', + ), + ).toEqual([{ test1: 1 }, { test2: 2 }, { test1: 1, test3: 3 }, { test4: 4 }]); + }); + + test('.intersection() should work on an array of objects', () => { + expect( + evaluate( + '={{ [{ test1: 1 }, { test2: 2 }].intersection([{ test1: 1, test3: 3 }, { test2: 2 }, { test4: 4 }]) }}', + ), + ).toEqual([{ test2: 2 }]); + }); + + test('.difference() should work on an array of objects', () => { + expect( + evaluate( + '={{ [{ test1: 1 }, { test2: 2 }].difference([{ test1: 1, test3: 3 }, { test2: 2 }, { test4: 4 }]) }}', + ), + ).toEqual([{ test1: 1 }]); + + expect( + evaluate('={{ [{ test1: 1 }, { test2: 2 }].difference([{ test1: 1 }, { test2: 2 }]) }}'), + ).toEqual([]); + }); + + test('.compact() should work on an array', () => { + expect( + evaluate( + '={{ [{ test1: 1, test2: undefined, test3: null }, null, undefined, 1, 2, 0, { test: "asdf" }].compact() }}', + ), + ).toEqual([{ test1: 1 }, 1, 2, 0, { test: 'asdf' }]); + }); + + test('.chunk() should work on an array', () => { + expect(evaluate('={{ numberList(1, 20).chunk(5) }}')).toEqual([ + [1, 2, 3, 4, 5], + [6, 7, 8, 9, 10], + [11, 12, 13, 14, 15], + [16, 17, 18, 19, 20], + ]); + }); + + test('.filter() should work on a list of strings', () => { + expect( + evaluate( + '={{ ["i am a test string", "i should be kept", "i should be removed test"].filter("test", "remove") }}', + ), + ).toEqual(['i should be kept']); + expect( + evaluate( + '={{ ["i am a test string", "i should be kept test", "i should be removed"].filter("test") }}', + ), + ).toEqual(['i am a test string', 'i should be kept test']); + }); + }); +}); diff --git a/packages/workflow/test/ExpressionExtensions/DateExtensions.test.ts b/packages/workflow/test/ExpressionExtensions/DateExtensions.test.ts new file mode 100644 index 0000000000000..5084fca09a538 --- /dev/null +++ b/packages/workflow/test/ExpressionExtensions/DateExtensions.test.ts @@ -0,0 +1,53 @@ +/** + * @jest-environment jsdom + */ + +import { extend } from '@/Extensions'; +import { dateExtensions } from '@/Extensions/DateExtensions'; +import { evaluate } from './Helpers'; + +describe('Data Transformation Functions', () => { + describe('Date Data Transformation Functions', () => { + test('.isWeekend() should work correctly on a date', () => { + expect(evaluate('={{DateTime.now().isWeekend()}}')).toEqual( + extend(new Date(), 'isWeekend', []), + ); + }); + + test('.toTimeFromNow() should work correctly on a date', () => { + const JUST_NOW_STRING_RESULT = 'just now'; + expect(evaluate('={{DateTime.now().toTimeFromNow()}}')).toEqual(JUST_NOW_STRING_RESULT); + }); + + test('.beginningOf("week") should work correctly on a date', () => { + expect(evaluate('={{(new Date).beginningOf("week")}}')).toEqual( + dateExtensions.functions.beginningOf(new Date(), ['week']), + ); + }); + + test('.endOfMonth() should work correctly on a date', () => { + expect(evaluate('={{ DateTime.now().endOfMonth() }}')).toEqual( + dateExtensions.functions.endOfMonth(new Date()), + ); + }); + + test('.extract("day") should work correctly on a date', () => { + expect(evaluate('={{ DateTime.now().extract("day") }}')).toEqual( + dateExtensions.functions.extract(new Date(), ['day']), + ); + }); + + test('.format("yyyy LLL dd") should work correctly on a date', () => { + expect(evaluate('={{ DateTime.now().format("yyyy LLL dd") }}')).toEqual( + dateExtensions.functions.format(new Date(), ['yyyy LLL dd']), + ); + expect(evaluate('={{ DateTime.now().format("yyyy LLL dd") }}')).not.toEqual( + dateExtensions.functions.format(new Date(), ["HH 'hours and' mm 'minutes'"]), + ); + }); + + test('.toDate() should work on a string', () => { + expect(evaluate('={{ "2022-01-03T00:00:00.000+00:00".toDate() }}')).toEqual(new Date(2022, 0, 3)); + }); + }); +}); diff --git a/packages/workflow/test/ExpressionExtensions/ExpressionExtension.test.ts b/packages/workflow/test/ExpressionExtensions/ExpressionExtension.test.ts new file mode 100644 index 0000000000000..72f1bd62c7131 --- /dev/null +++ b/packages/workflow/test/ExpressionExtensions/ExpressionExtension.test.ts @@ -0,0 +1,162 @@ +/** + * @jest-environment jsdom + */ + +import { extendTransform } from '@/Extensions'; +import { joinExpression, splitExpression } from '@/Extensions/ExpressionParser'; +import { evaluate } from './Helpers'; + +describe('Expression Extension Transforms', () => { + describe('extend() transform', () => { + test('Basic transform with .isBlank', () => { + expect(extendTransform('"".isBlank()')!.code).toEqual('extend("", "isBlank", [])'); + }); + + test('Chained transform with .sayHi.getOnlyFirstCharacters', () => { + expect(extendTransform('"".sayHi().getOnlyFirstCharacters(2)')!.code).toEqual( + 'extend(extend("", "sayHi", []), "getOnlyFirstCharacters", [2])', + ); + }); + + test('Chained transform with native functions .sayHi.trim.getOnlyFirstCharacters', () => { + expect(extendTransform('"aaa ".sayHi().trim().getOnlyFirstCharacters(2)')!.code).toEqual( + 'extend(extend("aaa ", "sayHi", []).trim(), "getOnlyFirstCharacters", [2])', + ); + }); + }); +}); + +describe('tmpl Expression Parser', () => { + describe('Compatible splitting', () => { + test('Lone expression', () => { + expect(splitExpression('{{ "" }}')).toEqual([ + { type: 'text', text: '' }, + { type: 'code', text: ' "" ', hasClosingBrackets: true }, + ]); + }); + + test('Multiple expression', () => { + expect(splitExpression('{{ "test".sayHi() }} you have ${{ (100).format() }}.')).toEqual([ + { type: 'text', text: '' }, + { type: 'code', text: ' "test".sayHi() ', hasClosingBrackets: true }, + { type: 'text', text: ' you have $' }, + { type: 'code', text: ' (100).format() ', hasClosingBrackets: true }, + { type: 'text', text: '.' }, + ]); + }); + + test('Unclosed expression', () => { + expect(splitExpression('{{ "test".sayHi() }} you have ${{ (100).format()')).toEqual([ + { type: 'text', text: '' }, + { type: 'code', text: ' "test".sayHi() ', hasClosingBrackets: true }, + { type: 'text', text: ' you have $' }, + { type: 'code', text: ' (100).format()', hasClosingBrackets: false }, + ]); + }); + + test('Escaped opening bracket', () => { + expect(splitExpression('test \\{{ no code }}')).toEqual([ + { type: 'text', text: 'test \\{{ no code }}' }, + ]); + }); + + test('Escaped closinging bracket', () => { + expect(splitExpression('test {{ code.test("\\}}") }}')).toEqual([ + { type: 'text', text: 'test ' }, + { type: 'code', text: ' code.test("}}") ', hasClosingBrackets: true }, + ]); + }); + }); + + describe('Compatible joining', () => { + test('Lone expression', () => { + expect(joinExpression(splitExpression('{{ "" }}'))).toEqual('{{ "" }}'); + }); + + test('Multiple expression', () => { + expect( + joinExpression(splitExpression('{{ "test".sayHi() }} you have ${{ (100).format() }}.')), + ).toEqual('{{ "test".sayHi() }} you have ${{ (100).format() }}.'); + }); + + test('Unclosed expression', () => { + expect( + joinExpression(splitExpression('{{ "test".sayHi() }} you have ${{ (100).format()')), + ).toEqual('{{ "test".sayHi() }} you have ${{ (100).format()'); + }); + + test('Escaped opening bracket', () => { + expect(joinExpression(splitExpression('test \\{{ no code }}'))).toEqual( + 'test \\{{ no code }}', + ); + }); + + test('Escaped closinging bracket', () => { + expect(joinExpression(splitExpression('test {{ code.test("\\}}") }}'))).toEqual( + 'test {{ code.test("\\}}") }}', + ); + }); + }); + + describe('Non dot extensions', () => { + test('min', () => { + expect(evaluate('={{ min(1, 2, 3, 4, 5, 6) }}')).toEqual(1); + expect(evaluate('={{ min(1, NaN, 3, 4, 5, 6) }}')).toBeNaN(); + }); + + test('max', () => { + expect(evaluate('={{ max(1, 2, 3, 4, 5, 6) }}')).toEqual(6); + expect(evaluate('={{ max(1, NaN, 3, 4, 5, 6) }}')).toBeNaN(); + }); + + test('average', () => { + expect(evaluate('={{ average(1, 2, 3, 4, 5, 6) }}')).toEqual(3.5); + expect(evaluate('={{ average(1, NaN, 3, 4, 5, 6) }}')).toBeNaN(); + }); + + test('numberList', () => { + expect(evaluate('={{ numberList(1, 10) }}')).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + expect(evaluate('={{ numberList(1, -10) }}')).toEqual([ + 1, 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, + ]); + }); + + test('zip', () => { + expect(evaluate('={{ zip(["test1", "test2", "test3"], [1, 2, 3]) }}')).toEqual({ + test1: 1, + test2: 2, + test3: 3, + }); + }); + + test('$if', () => { + expect(evaluate('={{ $if("a"==="a", 1, 2) }}')).toEqual(1); + expect(evaluate('={{ $if("a"==="b", 1, 2) }}')).toEqual(2); + expect(evaluate('={{ $if("a"==="a", 1) }}')).toEqual(1); + expect(evaluate('={{ $if("a"==="b", 1) }}')).toEqual(false); + + // This will likely break when sandboxing is implemented but it works for now. + // If you're implementing sandboxing maybe provide a way to add functions to + // sandbox we can check instead? + const mockCallback = jest.fn(() => false); + // @ts-ignore + evaluate('={{ $if("a"==="a", true, $data["cb"]()) }}', [{ cb: mockCallback }]); + expect(mockCallback.mock.calls.length).toEqual(0); + + // @ts-ignore + evaluate('={{ $if("a"==="b", true, $data["cb"]()) }}', [{ cb: mockCallback }]); + expect(mockCallback.mock.calls.length).toEqual(0); + }); + + test('$not', () => { + expect(evaluate('={{ $not(1) }}')).toEqual(false); + expect(evaluate('={{ $not(0) }}')).toEqual(true); + expect(evaluate('={{ $not(true) }}')).toEqual(false); + expect(evaluate('={{ $not(false) }}')).toEqual(true); + expect(evaluate('={{ $not(undefined) }}')).toEqual(true); + expect(evaluate('={{ $not(null) }}')).toEqual(true); + expect(evaluate('={{ $not("") }}')).toEqual(true); + expect(evaluate('={{ $not("a") }}')).toEqual(false); + }); + }); +}); diff --git a/packages/workflow/test/ExpressionExtensions/GenericExtensions.test.ts b/packages/workflow/test/ExpressionExtensions/GenericExtensions.test.ts new file mode 100644 index 0000000000000..0ebebce6e0585 --- /dev/null +++ b/packages/workflow/test/ExpressionExtensions/GenericExtensions.test.ts @@ -0,0 +1,9 @@ +import { evaluate } from './Helpers'; + +describe('Data Transformation Functions', () => { + describe('Genric Data Transformation Functions', () => { + test('.isBlank() should work correctly on undefined', () => { + expect(evaluate('={{(undefined).isBlank()}}')).toEqual(true); + }); + }); +}); diff --git a/packages/workflow/test/ExpressionExtensions/Helpers.ts b/packages/workflow/test/ExpressionExtensions/Helpers.ts new file mode 100644 index 0000000000000..9d5307bdeca72 --- /dev/null +++ b/packages/workflow/test/ExpressionExtensions/Helpers.ts @@ -0,0 +1,33 @@ +import { Expression, INodeExecutionData, Workflow } from '../../src'; +import * as Helpers from '../Helpers'; + +export const nodeTypes = Helpers.NodeTypes(); +export const workflow = new Workflow({ + nodes: [ + { + name: 'node', + typeVersion: 1, + type: 'test.set', + id: 'uuid-1234', + position: [0, 0], + parameters: {}, + }, + ], + connections: {}, + active: false, + nodeTypes, +}); +export const expression = new Expression(workflow); + +export const evaluate = (value: string, values?: INodeExecutionData[]) => + expression.getParameterValue( + value, + null, + 0, + 0, + 'node', + values ?? [], + 'manual', + 'America/New_York', + {}, + ); diff --git a/packages/workflow/test/ExpressionExtensions/NumberExtensions.test.ts b/packages/workflow/test/ExpressionExtensions/NumberExtensions.test.ts new file mode 100644 index 0000000000000..9dbd6c78a9d43 --- /dev/null +++ b/packages/workflow/test/ExpressionExtensions/NumberExtensions.test.ts @@ -0,0 +1,85 @@ +/** + * @jest-environment jsdom + */ + +import { numberExtensions } from '@/Extensions/NumberExtensions'; +import { evaluate } from './Helpers'; + +describe('Data Transformation Functions', () => { + describe('Number Data Transformation Functions', () => { + test('.random() should work correctly on a number', () => { + expect(evaluate('={{ Number(100).random() }}')).not.toBeUndefined(); + }); + + test('.isBlank() should work correctly on a number', () => { + expect(evaluate('={{ Number(100).isBlank() }}')).toEqual(false); + }); + + test('.isPresent() should work correctly on a number', () => { + expect(evaluate('={{ Number(100).isPresent() }}')).toEqual( + numberExtensions.functions.isPresent(100), + ); + }); + + test('.format() should work correctly on a number', () => { + expect(evaluate('={{ Number(100).format() }}')).toEqual( + numberExtensions.functions.format(100, []), + ); + }); + + test('.ceil() should work on a number', () => { + expect(evaluate('={{ (1.2).ceil() }}')).toEqual(2); + expect(evaluate('={{ (1.9).ceil() }}')).toEqual(2); + expect(evaluate('={{ (1.0).ceil() }}')).toEqual(1); + expect(evaluate('={{ (NaN).ceil() }}')).toBeNaN(); + }); + + test('.floor() should work on a number', () => { + expect(evaluate('={{ (1.2).floor() }}')).toEqual(1); + expect(evaluate('={{ (1.9).floor() }}')).toEqual(1); + expect(evaluate('={{ (1.0).floor() }}')).toEqual(1); + expect(evaluate('={{ (NaN).floor() }}')).toBeNaN(); + }); + + test('.round() should work on a number', () => { + expect(evaluate('={{ (1.3333333).round(3) }}')).toEqual(1.333); + expect(evaluate('={{ (1.3333333).round(0) }}')).toEqual(1); + expect(evaluate('={{ (1.5001).round(0) }}')).toEqual(2); + expect(evaluate('={{ (NaN).round(3) }}')).toBeNaN(); + }); + + test('.isTrue() should work on a number', () => { + expect(evaluate('={{ (1).isTrue() }}')).toEqual(true); + expect(evaluate('={{ (0).isTrue() }}')).toEqual(false); + expect(evaluate('={{ (NaN).isTrue() }}')).toEqual(false); + }); + + test('.isFalse() should work on a number', () => { + expect(evaluate('={{ (1).isFalse() }}')).toEqual(false); + expect(evaluate('={{ (0).isFalse() }}')).toEqual(true); + expect(evaluate('={{ (NaN).isFalse() }}')).toEqual(false); + }); + + test('.isOdd() should work on a number', () => { + expect(evaluate('={{ (9).isOdd() }}')).toEqual(true); + expect(evaluate('={{ (8).isOdd() }}')).toEqual(false); + expect(evaluate('={{ (0).isOdd() }}')).toEqual(false); + expect(evaluate('={{ (NaN).isOdd() }}')).toEqual(false); + }); + + test('.isEven() should work on a number', () => { + expect(evaluate('={{ (9).isEven() }}')).toEqual(false); + expect(evaluate('={{ (8).isEven() }}')).toEqual(true); + expect(evaluate('={{ (0).isEven() }}')).toEqual(true); + expect(evaluate('={{ (NaN).isEven() }}')).toEqual(false); + }); + }); + + describe('Multiple expressions', () => { + test('Basic multiple expressions', () => { + expect(evaluate('={{ "Test".sayHi() }} you have ${{ (100).format() }}.')).toEqual( + 'hi Test you have $100.', + ); + }); + }); +}); diff --git a/packages/workflow/test/ExpressionExtensions/ObjectExtensions.test.ts b/packages/workflow/test/ExpressionExtensions/ObjectExtensions.test.ts new file mode 100644 index 0000000000000..67ba3d60ca5c8 --- /dev/null +++ b/packages/workflow/test/ExpressionExtensions/ObjectExtensions.test.ts @@ -0,0 +1,67 @@ +import { evaluate } from './Helpers'; + +describe('Data Transformation Functions', () => { + describe('Object Data Transformation Functions', () => { + test('.isEmpty() should work correctly on an object', () => { + expect(evaluate('={{({}).isEmpty()}}')).toEqual(true); + expect(evaluate('={{({ test1: 1 }).isEmpty()}}')).toEqual(false); + }); + + test('.merge should work on an object', () => { + expect(evaluate('={{ ({ test1: 1, test2: 2 }).merge({ test2: 3, test3: 3 }) }}')).toEqual({ + test1: 1, + test2: 2, + test3: 3, + }); + }); + + test('.hasField should work on an object', () => { + expect(evaluate('={{ ({ test1: 1 }).hasField("test1") }}')).toEqual(true); + expect(evaluate('={{ ({ test1: 1 }).hasField("test2") }}')).toEqual(false); + }); + + test('.removeField should work on an object', () => { + expect(evaluate('={{ ({ test1: 1, test2: 2, test3: 3 }).removeField("test2") }}')).toEqual({ + test1: 1, + test3: 3, + }); + expect( + evaluate('={{ ({ test1: 1, test2: 2, test3: 3 }).removeField("testDoesntExist") }}'), + ).toEqual({ + test1: 1, + test2: 2, + test3: 3, + }); + }); + + test('.removeFieldsContaining should work on an object', () => { + expect( + evaluate( + '={{ ({ test1: "i exist", test2: "i should be removed", test3: "i should also be removed" }).removeFieldsContaining("removed") }}', + ), + ).toEqual({ + test1: 'i exist', + }); + }); + + test('.keepFieldsContaining should work on an object', () => { + expect( + evaluate( + '={{ ({ test1: "i exist", test2: "i should be removed", test3: "i should also be removed" }).keepFieldsContaining("exist") }}', + ), + ).toEqual({ + test1: 'i exist', + }); + }); + + test('.compact should work on an object', () => { + expect( + evaluate('={{ ({ test1: 1, test2: "2", test3: undefined, test4: null }).compact() }}'), + ).toEqual({ test1: 1, test2: '2' }); + }); + + test('.urlEncode should work on an object', () => { + expect(evaluate('={{ ({ test1: 1, test2: "2" }).urlEncode() }}')).toEqual('test1=1&test2=2'); + }); + }); +}); diff --git a/packages/workflow/test/ExpressionExtensions/StringExtensions.test.ts b/packages/workflow/test/ExpressionExtensions/StringExtensions.test.ts new file mode 100644 index 0000000000000..bc80d89c1cca8 --- /dev/null +++ b/packages/workflow/test/ExpressionExtensions/StringExtensions.test.ts @@ -0,0 +1,225 @@ +/** + * @jest-environment jsdom + */ + +import { stringExtensions } from '@/Extensions/StringExtensions'; +import { dateExtensions } from '@/Extensions/DateExtensions'; +import { evaluate } from './Helpers'; + +describe('Data Transformation Functions', () => { + describe('String Data Transformation Functions', () => { + test('.isBlank() should work correctly on a string that is not empty', () => { + expect(evaluate('={{"NotBlank".isBlank()}}')).toEqual(false); + }); + + test('.isBlank() should work correctly on a string that is empty', () => { + expect(evaluate('={{"".isBlank()}}')).toEqual(true); + }); + + test('.getOnlyFirstCharacters() should work correctly on a string', () => { + expect(evaluate('={{"myNewField".getOnlyFirstCharacters(5)}}')).toEqual('myNew'); + + expect(evaluate('={{"myNewField".getOnlyFirstCharacters(10)}}')).toEqual('myNewField'); + + expect( + evaluate('={{"myNewField".getOnlyFirstCharacters(5).length >= "myNewField".length}}'), + ).toEqual(false); + + expect(evaluate('={{DateTime.now().toLocaleString().getOnlyFirstCharacters(2)}}')).toEqual( + stringExtensions.functions.getOnlyFirstCharacters( + // @ts-ignore + dateExtensions.functions.toLocaleString(new Date(), []), + [2], + ), + ); + }); + + test('.sayHi() should work correctly on a string', () => { + expect(evaluate('={{ "abc".sayHi() }}')).toEqual('hi abc'); + }); + + test('.encrypt() should work correctly on a string', () => { + expect(evaluate('={{ "12345".encrypt("sha256") }}')).toEqual( + stringExtensions.functions.encrypt('12345', ['sha256']), + ); + + expect(evaluate('={{ "12345".encrypt("sha256") }}')).not.toEqual( + stringExtensions.functions.encrypt('12345', ['MD5']), + ); + + expect(evaluate('={{ "12345".encrypt("MD5") }}')).toEqual( + stringExtensions.functions.encrypt('12345', ['MD5']), + ); + + expect(evaluate('={{ "12345".hash("sha256") }}')).toEqual( + '5994471abb01112afcc18159f6cc74b4f511b99806da59b3caf5a9c173cacfc5', + ); + }); + + test('.hash() alias should work correctly on a string', () => { + expect(evaluate('={{ "12345".hash("sha256") }}')).toEqual( + '5994471abb01112afcc18159f6cc74b4f511b99806da59b3caf5a9c173cacfc5', + ); + }); + + test('.urlDecode should work correctly on a string', () => { + expect(evaluate('={{ "string%20with%20spaces".urlDecode(false) }}')).toEqual( + 'string with spaces', + ); + }); + + test('.urlEncode should work correctly on a string', () => { + expect(evaluate('={{ "string with spaces".urlEncode(false) }}')).toEqual( + 'string%20with%20spaces', + ); + }); + + test('.stripTags should work correctly on a string', () => { + expect(evaluate('={{ "test".stripTags() }}')).toEqual('test'); + }); + + test('.removeMarkdown should work correctly on a string', () => { + expect(evaluate('={{ "test".removeMarkdown() }}')).toEqual('test'); + }); + + test('.toLowerCase should work correctly on a string', () => { + expect(evaluate('={{ "TEST".toLowerCase() }}')).toEqual('test'); + }); + + test('.toDate should work correctly on a date string', () => { + expect(evaluate('={{ "2022-09-01T19:42:28.164Z".toDate() }}')).toEqual( + new Date('2022-09-01T19:42:28.164Z'), + ); + }); + + test('.toBoolean should work correctly on a string', () => { + const validTrue = ['y', 'yes', 't', 'true', '1', 'YES']; + for (const v of validTrue) { + expect(evaluate(`={{ "${v}".toBoolean() }}`)).toEqual(true); + } + + const validFalse = ['n', 'no', 'f', 'false', '0', 'NO']; + for (const v of validFalse) { + expect(evaluate(`={{ "${v}".toBoolean() }}`)).toEqual(false); + } + + expect(evaluate('={{ "maybe".toBoolean() }}')).toEqual(false); + }); + + test('.isTrue should work correctly on a string', () => { + const validTrue = ['y', 'yes', 't', 'true', '1', 'YES']; + for (const v of validTrue) { + expect(evaluate(`={{ "${v}".isTrue() }}`)).toEqual(true); + } + + const validFalse = ['n', 'no', 'f', 'false', '0', 'NO']; + for (const v of validFalse) { + expect(evaluate(`={{ "${v}".isTrue() }}`)).toEqual(false); + } + + expect(evaluate('={{ "maybe".isTrue() }}')).toEqual(false); + }); + + test('.isFalse should work correctly on a string', () => { + const validTrue = ['y', 'yes', 't', 'true', '1', 'YES']; + for (const v of validTrue) { + expect(evaluate(`={{ "${v}".isFalse() }}`)).toEqual(false); + } + + const validFalse = ['n', 'no', 'f', 'false', '0', 'NO']; + for (const v of validFalse) { + expect(evaluate(`={{ "${v}".isFalse() }}`)).toEqual(true); + } + + expect(evaluate('={{ "maybe".isFalse() }}')).toEqual(false); + }); + + test('.toFloat should work correctly on a string', () => { + expect(evaluate('={{ "1.1".toFloat() }}')).toEqual(1.1); + expect(evaluate('={{ "1.1".toDecimalNumber() }}')).toEqual(1.1); + }); + + test('.toInt should work correctly on a string', () => { + expect(evaluate('={{ "1.1".toInt() }}')).toEqual(1); + expect(evaluate('={{ "1.1".toWholeNumber() }}')).toEqual(1); + expect(evaluate('={{ "1.5".toInt() }}')).toEqual(1); + expect(evaluate('={{ "1.5".toWholeNumber() }}')).toEqual(1); + }); + + test('.quote should work correctly on a string', () => { + expect(evaluate('={{ "test".quote() }}')).toEqual('"test"'); + expect(evaluate('={{ "\\"test\\"".quote() }}')).toEqual('"\\"test\\""'); + }); + + test('.isNumeric should work correctly on a string', () => { + expect(evaluate('={{ "".isNumeric() }}')).toEqual(false); + expect(evaluate('={{ "asdf".isNumeric() }}')).toEqual(false); + expect(evaluate('={{ "1234".isNumeric() }}')).toEqual(true); + expect(evaluate('={{ "4e4".isNumeric() }}')).toEqual(true); + expect(evaluate('={{ "4.4".isNumeric() }}')).toEqual(true); + }); + + test('.isUrl should work on a string', () => { + expect(evaluate('={{ "https://example.com/".isUrl() }}')).toEqual(true); + expect(evaluate('={{ "example.com".isUrl() }}')).toEqual(false); + }); + + test('.isDomain should work on a string', () => { + expect(evaluate('={{ "example.com".isDomain() }}')).toEqual(true); + expect(evaluate('={{ "asdf".isDomain() }}')).toEqual(false); + expect(evaluate('={{ "https://example.com/".isDomain() }}')).toEqual(false); + }); + + test('.toSnakeCase should work on a string', () => { + expect(evaluate('={{ "I am a test!".toSnakeCase() }}')).toEqual('i_am_a_test'); + expect(evaluate('={{ "i_am_a_test".toSnakeCase() }}')).toEqual('i_am_a_test'); + }); + + test('.toSentenceCase should work on a string', () => { + expect( + evaluate( + '={{ "i am a test! i have multiple types of Punctuation. or do i?".toSentenceCase() }}', + ), + ).toEqual('I am a test! I have multiple types of punctuation. Or do i?'); + expect(evaluate('={{ "i am a test!".toSentenceCase() }}')).toEqual('I am a test!'); + expect(evaluate('={{ "i am a test".toSentenceCase() }}')).toEqual('I am a test'); + }); + + test('.toTitleCase should work on a string', () => { + expect( + evaluate( + '={{ "i am a test! i have multiple types of Punctuation. or do i?".toTitleCase() }}', + ), + ).toEqual('I Am A Test! I Have Multiple Types Of Punctuation. Or Do I?'); + expect(evaluate('={{ "i am a test!".toTitleCase() }}')).toEqual('I Am A Test!'); + expect(evaluate('={{ "i am a test".toTitleCase() }}')).toEqual('I Am A Test'); + }); + + test('.extractUrl should work on a string', () => { + expect( + evaluate( + '={{ "I am a test with a url: https://example.net/ and I am a test with an email: test@example.org".extractUrl() }}', + ), + ).toEqual('https://example.net/'); + }); + + test('.extractDomain should work on a string', () => { + expect(evaluate('={{ "test@example.org".extractDomain() }}')).toEqual('example.org'); + expect(evaluate('={{ "https://example.org/".extractDomain() }}')).toEqual('example.org'); + }); + + test('.extractEmail should work on a string', () => { + expect( + evaluate( + '={{ "I am a test with a url: https://example.net/ and I am a test with an email: test@example.org".extractEmail() }}', + ), + ).toEqual('test@example.org'); + }); + + test('.isEmail should work on a string', () => { + expect(evaluate('={{ "test@example.com".isEmail() }}')).toEqual(true); + expect(evaluate('={{ "aaaaaaaa".isEmail() }}')).toEqual(false); + expect(evaluate('={{ "test @ n8n".isEmail() }}')).toEqual(false); + }); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a6ab0c3aa4d7a..59d8b252e3cc9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -882,7 +882,9 @@ importers: packages/workflow: specifiers: + '@n8n_io/eslint-config': '' '@n8n_io/riot-tmpl': ^2.0.0 + '@types/crypto-js': ^4.1.1 '@types/express': ^4.17.6 '@types/jmespath': ^0.15.0 '@types/lodash.get': ^4.4.6 @@ -891,23 +893,33 @@ importers: '@types/lodash.set': ^4.3.6 '@types/luxon': ^2.0.9 '@types/xml2js': ^0.4.3 + crypto-js: ^4.1.1 jmespath: ^0.16.0 + js-base64: ^3.7.2 lodash.get: ^4.4.2 lodash.isequal: ^4.5.0 lodash.merge: ^4.6.2 lodash.set: ^4.3.2 luxon: ~2.3.0 + recast: ^0.21.5 + transliteration: ^2.3.5 xml2js: ^0.4.23 dependencies: '@n8n_io/riot-tmpl': 2.0.0 + crypto-js: 4.1.1 jmespath: 0.16.0 + js-base64: 3.7.2 lodash.get: 4.4.2 lodash.isequal: 4.5.0 lodash.merge: 4.6.2 lodash.set: 4.3.2 luxon: 2.3.2 + recast: 0.21.5 + transliteration: 2.3.5 xml2js: 0.4.23 devDependencies: + '@n8n_io/eslint-config': link:../@n8n_io/eslint-config + '@types/crypto-js': 4.1.1 '@types/express': 4.17.14 '@types/jmespath': 0.15.0 '@types/lodash.get': 4.4.7 @@ -7590,7 +7602,6 @@ packages: engines: {node: '>=4'} dependencies: tslib: 2.4.0 - dev: true /astral-regex/2.0.0: resolution: {integrity: sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==} @@ -14477,6 +14488,10 @@ packages: resolution: {integrity: sha512-opJeO4nCucVnsjiXOE+/PcCgYw9Gwpvs/a6B1LL/lQhwWwpbVEVYDZ1FokFr8PRc7ghYlrFPuyHuiiDNTQxmcw==} dev: false + /js-base64/3.7.2: + resolution: {integrity: sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==} + dev: false + /js-beautify/1.14.6: resolution: {integrity: sha512-GfofQY5zDp+cuHc+gsEXKPpNw2KbPddreEo35O6jT6i0RVK6LhsoYBhq5TvK4/n74wnA0QbK8gGd+jUZwTMKJw==} engines: {node: '>=10'} @@ -18298,6 +18313,16 @@ packages: tslib: 2.4.0 dev: true + /recast/0.21.5: + resolution: {integrity: sha512-hjMmLaUXAm1hIuTqOdeYObMslq/q+Xff6QE3Y2P+uoHAg2nmVlLBps2hzh1UJDdMtDTMXOFewK6ky51JQIeECg==} + engines: {node: '>= 4'} + dependencies: + ast-types: 0.15.2 + esprima: 4.0.1 + source-map: 0.6.1 + tslib: 2.4.0 + dev: false + /rechoir/0.6.2: resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==} engines: {node: '>= 0.10'} @@ -20601,6 +20626,14 @@ packages: dependencies: punycode: 2.1.1 + /transliteration/2.3.5: + resolution: {integrity: sha512-HAGI4Lq4Q9dZ3Utu2phaWgtm3vB6PkLUFqWAScg/UW+1eZ/Tg6Exo4oC0/3VUol/w4BlefLhUUSVBr/9/ZGQOw==} + engines: {node: '>=6.0.0'} + hasBin: true + dependencies: + yargs: 17.6.0 + dev: false + /tree-kill/1.2.2: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true