From 243f3b0d1ff2048f623bea2391373121ca171e05 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Mon, 25 Sep 2023 16:58:31 -0300 Subject: [PATCH 01/45] feat(KNO-4367): add email layout pull, writer and helpers --- src/commands/layout/pull.ts | 129 ++++++++++++++++++++++++ src/lib/marshal/email-layout/helpers.ts | 30 ++++++ src/lib/marshal/email-layout/index.ts | 2 + src/lib/marshal/email-layout/writer.ts | 16 +++ src/lib/run-context/types.ts | 13 ++- 5 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 src/commands/layout/pull.ts create mode 100644 src/lib/marshal/email-layout/helpers.ts create mode 100644 src/lib/marshal/email-layout/writer.ts diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts new file mode 100644 index 0000000..4e13ecb --- /dev/null +++ b/src/commands/layout/pull.ts @@ -0,0 +1,129 @@ +import * as path from "node:path"; + +import { Args, Flags } from "@oclif/core"; + +import * as ApiV1 from "@/lib/api-v1"; +import BaseCommand from "@/lib/base-command"; +import * as CustomFlags from "@/lib/helpers/flag"; +import { merge } from "@/lib/helpers/object"; +import { withSpinner } from "@/lib/helpers/request"; +import { promptToConfirm } from "@/lib/helpers/ux"; +import * as EmailLayout from "@/lib/marshal/email-layout"; +import { WithAnnotation } from "@/lib/marshal/shared/types"; +import { + EmailLayoutDirContext, + ensureResourceDirForTarget, + ResourceTarget, +} from "@/lib/run-context"; + +export default class EmailLayoutPull extends BaseCommand< + typeof EmailLayoutPull +> { + static summary = + "Pull one or more email layouts from an environment into a local file system."; + + static flags = { + environment: Flags.string({ + default: "development", + summary: "The environment to use.", + }), + all: Flags.boolean({ + summary: + "Whether to pull all email layouts from the specified environment.", + }), + "email- layout - dir": CustomFlags.dirPath({ + summary: "The target directory path to pull all email layouts into.", + dependsOn: ["all"], + }), + "hide - uncommitted - changes": Flags.boolean({ + summary: "Hide any uncommitted changes.", + }), + force: Flags.boolean({ + summary: "Remove the confirmation prompt.", + }), + }; + + static args = { + emailLayoutKey: Args.string({ + required: false, + }), + }; + + async run(): Promise { + const { args, flags } = this.props; + if (flags.all && args.emailLayoutKey) { + return this.error( + `emailLayoutKey arg \`${args.emailLayoutKey}\` cannot also be provided when using --all`, + ); + } + + return this.pullOneEmailLayout(); + } + + /* + * Pull one email layout + */ + async pullOneEmailLayout(): Promise { + const { flags } = this.props; + + const dirContext = await this.getEmailLayoutDirContext(); + + if (dirContext.exists) { + this.log(`‣ Found \`${dirContext.key}\` at ${dirContext.abspath}`); + } else { + const prompt = `Create a new email layout directory \`${dirContext.key}\` at ${dirContext.abspath}?`; + const input = flags.force || (await promptToConfirm(prompt)); + if (!input) return; + } + + const resp = await withSpinner>( + () => { + const props = merge(this.props, { + args: { emailLayoutKey: dirContext.key }, + flags: { annotate: true }, + }); + return this.apiV1.getEmailLayout(props); + }, + ); + + await EmailLayout.writeEmailLayoutFile(dirContext.abspath, resp.data); + + const action = dirContext.exists ? "updated" : "created"; + this.log( + `‣ Successfully ${action} \`${dirContext.key}\` at ${dirContext.abspath}`, + ); + } + + async getEmailLayoutDirContext(): Promise { + const { emailLayoutKey } = this.props.args; + const { resourceDir, cwd: runCwd } = this.runContext; + // Inside an existing resource dir, use it if valid for the target email layout. + if (resourceDir) { + const target: ResourceTarget = { + commandId: BaseCommand.id, + type: "email_layout", + key: emailLayoutKey, + }; + return ensureResourceDirForTarget( + resourceDir, + target, + ) as EmailLayoutDirContext; + } + + // Not inside any existing email layout directory, which means either create a + // new email layout directory in the cwd, or update it if there is one already. + if (emailLayoutKey) { + const dirPath = path.resolve(runCwd, emailLayoutKey); + const exists = await EmailLayout.isEmailLayoutDir(dirPath); + return { + type: "email_layout", + key: emailLayoutKey, + abspath: dirPath, + exists, + }; + } + + // Not in any email layout directory, nor a email layout key arg was given so error. + return this.error("Missing 1 required arg: emailLayoutKey"); + } +} diff --git a/src/lib/marshal/email-layout/helpers.ts b/src/lib/marshal/email-layout/helpers.ts new file mode 100644 index 0000000..e1dd583 --- /dev/null +++ b/src/lib/marshal/email-layout/helpers.ts @@ -0,0 +1,30 @@ +import * as path from "node:path"; + +import * as fs from "fs-extra"; + +export const EMAIL_LAYOUT_JSON = "email_layout.json"; + +export type EmailLayoutFileContext = { + key: string; + abspath: string; + exists: boolean; +}; + +/* + * Evaluates whether the given directory path is an email layout directory, by + * checking for the presence of email_layout.json file. + */ +export const isEmailLayoutDir = async (dirPath: string): Promise => + Boolean(await isEmailLayoutJson(dirPath)); + +/* + * Check for email_layout.json file and return the file path if present. + */ +export const isEmailLayoutJson = async ( + dirPath: string, +): Promise => { + const workflowJsonPath = path.resolve(dirPath, EMAIL_LAYOUT_JSON); + + const exists = await fs.pathExists(workflowJsonPath); + return exists ? workflowJsonPath : undefined; +}; diff --git a/src/lib/marshal/email-layout/index.ts b/src/lib/marshal/email-layout/index.ts index eea524d..65171bc 100644 --- a/src/lib/marshal/email-layout/index.ts +++ b/src/lib/marshal/email-layout/index.ts @@ -1 +1,3 @@ +export * from "./helpers"; export * from "./types"; +export * from "./writer"; diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts new file mode 100644 index 0000000..d8ea826 --- /dev/null +++ b/src/lib/marshal/email-layout/writer.ts @@ -0,0 +1,16 @@ +import * as fs from "fs-extra"; + +import { DOUBLE_SPACES } from "@/lib/helpers/json"; + +import { EmailLayoutData } from "./types"; + +/* + * Write a single email layout file. + */ +export const writeEmailLayoutFile = async ( + emailLayoutFilePath: string, + emailLayout: EmailLayoutData, +): Promise => + fs.outputJson(emailLayoutFilePath, JSON.parse(emailLayout.html_layout), { + spaces: DOUBLE_SPACES, + }); diff --git a/src/lib/run-context/types.ts b/src/lib/run-context/types.ts index 00779bc..ec10f0d 100644 --- a/src/lib/run-context/types.ts +++ b/src/lib/run-context/types.ts @@ -16,7 +16,11 @@ export type T = RunContext; * Resource directory context */ -export type ResourceType = "workflow" | "layout" | "translation"; +export type ResourceType = + | "workflow" + | "layout" + | "translation" + | "email_layout"; type ResourceDirContextBase = DirContext & { type: ResourceType; @@ -35,10 +39,15 @@ export type TranslationDirContext = ResourceDirContextBase & { type: "translation"; }; +export type EmailLayoutDirContext = ResourceDirContextBase & { + type: "email_layout"; +}; + export type ResourceDirContext = | WorkflowDirContext | LayoutDirContext - | TranslationDirContext; + | TranslationDirContext + | EmailLayoutDirContext; export type ResourceTarget = { commandId: string; From e7ecf5a355c38a0f9edb860a84bad76e3184184c Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 26 Sep 2023 11:42:38 -0300 Subject: [PATCH 02/45] feat(KNO-4367): pull one layout --- src/commands/layout/pull.ts | 21 ++++++++++++++++----- src/lib/marshal/email-layout/helpers.ts | 21 ++++++++++++--------- src/lib/marshal/email-layout/writer.ts | 7 ++++--- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts index 4e13ecb..a8040de 100644 --- a/src/commands/layout/pull.ts +++ b/src/commands/layout/pull.ts @@ -19,6 +19,8 @@ import { export default class EmailLayoutPull extends BaseCommand< typeof EmailLayoutPull > { + static aliases = ["email-layout:pull", "email_layout:pull"]; + static summary = "Pull one or more email layouts from an environment into a local file system."; @@ -31,11 +33,11 @@ export default class EmailLayoutPull extends BaseCommand< summary: "Whether to pull all email layouts from the specified environment.", }), - "email- layout - dir": CustomFlags.dirPath({ + "layout-dir": CustomFlags.dirPath({ summary: "The target directory path to pull all email layouts into.", dependsOn: ["all"], }), - "hide - uncommitted - changes": Flags.boolean({ + "hide-uncommitted-changes": Flags.boolean({ summary: "Hide any uncommitted changes.", }), force: Flags.boolean({ @@ -51,6 +53,7 @@ export default class EmailLayoutPull extends BaseCommand< async run(): Promise { const { args, flags } = this.props; + if (flags.all && args.emailLayoutKey) { return this.error( `emailLayoutKey arg \`${args.emailLayoutKey}\` cannot also be provided when using --all`, @@ -63,6 +66,7 @@ export default class EmailLayoutPull extends BaseCommand< /* * Pull one email layout */ + async pullOneEmailLayout(): Promise { const { flags } = this.props; @@ -97,6 +101,7 @@ export default class EmailLayoutPull extends BaseCommand< async getEmailLayoutDirContext(): Promise { const { emailLayoutKey } = this.props.args; const { resourceDir, cwd: runCwd } = this.runContext; + // Inside an existing resource dir, use it if valid for the target email layout. if (resourceDir) { const target: ResourceTarget = { @@ -104,6 +109,7 @@ export default class EmailLayoutPull extends BaseCommand< type: "email_layout", key: emailLayoutKey, }; + return ensureResourceDirForTarget( resourceDir, target, @@ -114,16 +120,21 @@ export default class EmailLayoutPull extends BaseCommand< // new email layout directory in the cwd, or update it if there is one already. if (emailLayoutKey) { const dirPath = path.resolve(runCwd, emailLayoutKey); - const exists = await EmailLayout.isEmailLayoutDir(dirPath); + const exists = await EmailLayout.isEmailLayoutDir( + dirPath, + `${emailLayoutKey}.json`, + ); + const abspath = path.resolve(dirPath, `${emailLayoutKey}.json`); + return { type: "email_layout", key: emailLayoutKey, - abspath: dirPath, + abspath: abspath, exists, }; } // Not in any email layout directory, nor a email layout key arg was given so error. - return this.error("Missing 1 required arg: emailLayoutKey"); + return this.error("Missing 1 required arg:\nemailLayoutKey"); } } diff --git a/src/lib/marshal/email-layout/helpers.ts b/src/lib/marshal/email-layout/helpers.ts index e1dd583..1b36627 100644 --- a/src/lib/marshal/email-layout/helpers.ts +++ b/src/lib/marshal/email-layout/helpers.ts @@ -2,8 +2,6 @@ import * as path from "node:path"; import * as fs from "fs-extra"; -export const EMAIL_LAYOUT_JSON = "email_layout.json"; - export type EmailLayoutFileContext = { key: string; abspath: string; @@ -12,19 +10,24 @@ export type EmailLayoutFileContext = { /* * Evaluates whether the given directory path is an email layout directory, by - * checking for the presence of email_layout.json file. + * checking for the presence of `${email_layout_key}.json` file. */ -export const isEmailLayoutDir = async (dirPath: string): Promise => - Boolean(await isEmailLayoutJson(dirPath)); +export const isEmailLayoutDir = async ( + dirPath: string, + emailLayoutJson: string, +): Promise => + Boolean(await isEmailLayoutJson(dirPath, emailLayoutJson)); /* - * Check for email_layout.json file and return the file path if present. + * Check for the existance of`${email_layout_key}.json` file in the directory. */ + export const isEmailLayoutJson = async ( dirPath: string, + emailLayoutJson: string, ): Promise => { - const workflowJsonPath = path.resolve(dirPath, EMAIL_LAYOUT_JSON); + const emailLayoutJsonPath = path.resolve(dirPath, emailLayoutJson); - const exists = await fs.pathExists(workflowJsonPath); - return exists ? workflowJsonPath : undefined; + const exists = await fs.pathExists(emailLayoutJsonPath); + return exists ? emailLayoutJsonPath : undefined; }; diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index d8ea826..c2f839c 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -5,12 +5,13 @@ import { DOUBLE_SPACES } from "@/lib/helpers/json"; import { EmailLayoutData } from "./types"; /* - * Write a single email layout file. + * Write a single email layout file in a given path. */ + export const writeEmailLayoutFile = async ( - emailLayoutFilePath: string, + emailLayoutAbsPath: string, emailLayout: EmailLayoutData, ): Promise => - fs.outputJson(emailLayoutFilePath, JSON.parse(emailLayout.html_layout), { + fs.outputJson(emailLayoutAbsPath, emailLayout, { spaces: DOUBLE_SPACES, }); From 6e71d950db986076740acce411d7797169eaa382 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 26 Sep 2023 15:30:09 -0300 Subject: [PATCH 03/45] feat(KNO-4367): add writer logic --- src/commands/layout/pull.ts | 5 +- src/lib/marshal/email-layout/writer.ts | 151 +++++++++++++++++++++++-- 2 files changed, 145 insertions(+), 11 deletions(-) diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts index a8040de..67e2656 100644 --- a/src/commands/layout/pull.ts +++ b/src/commands/layout/pull.ts @@ -90,7 +90,7 @@ export default class EmailLayoutPull extends BaseCommand< }, ); - await EmailLayout.writeEmailLayoutFile(dirContext.abspath, resp.data); + await EmailLayout.writeWorkflowDirFromData(dirContext, resp.data); const action = dirContext.exists ? "updated" : "created"; this.log( @@ -124,12 +124,11 @@ export default class EmailLayoutPull extends BaseCommand< dirPath, `${emailLayoutKey}.json`, ); - const abspath = path.resolve(dirPath, `${emailLayoutKey}.json`); return { type: "email_layout", key: emailLayoutKey, - abspath: abspath, + abspath: dirPath, exists, }; } diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index c2f839c..7905406 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -1,17 +1,152 @@ +import path from "node:path"; + import * as fs from "fs-extra"; +import { get, set, uniqueId } from "lodash"; +import { sandboxDir } from "@/lib/helpers/const"; import { DOUBLE_SPACES } from "@/lib/helpers/json"; +import { + mapValuesDeep, + ObjKeyOrArrayIdx, + omitDeep, +} from "@/lib/helpers/object"; +import { AnyObj, split } from "@/lib/helpers/object"; +import { EmailLayoutDirContext } from "@/lib/run-context"; +import { ExtractionSettings, WithAnnotation } from "../shared/types"; +import { FILEPATH_MARKED_RE } from "../workflow"; import { EmailLayoutData } from "./types"; +export type EmailLayoutDirBundle = { + [relpath: string]: string; +}; /* - * Write a single email layout file in a given path. + * Sanitize the email latyout content into a format that's appropriate for reading + * and writing, by stripping out any annotation fields and handling readonly + * fields. */ +type CompiledExtractionSettings = Map; + +const compileExtractionSettings = ( + emailLayout: EmailLayoutData, +): CompiledExtractionSettings => { + const extractableFields = get( + emailLayout, + ["__annotation", "extractable_fields"], + {}, + ); + const map: CompiledExtractionSettings = new Map(); + + for (const [key] of Object.entries(emailLayout)) { + // If the field we are on is extractable, then add its extraction + // settings to the map with the current object path. + if (key in extractableFields) { + map.set([key], extractableFields[key]); + } + } + + return map; +}; + +const toEmailLayoutJson = ( + emailLayout: EmailLayoutData, +): AnyObj => { + // Move read only field under the dedicated field "__readonly". + const readonlyFields = emailLayout.__annotation?.readonly_fields || []; + const [readonly, remainder] = split(emailLayout, readonlyFields); + const emailLayoutjson = { ...remainder, __readonly: readonly }; + + // Strip out all schema annotations, so not to expose them to end users. + return omitDeep(emailLayoutjson, ["__annotation"]); +}; + +// Builds an email layout dir bundle and writes it into a layout directory on local file system. +export const writeWorkflowDirFromData = async ( + emailLayoutDirCtx: EmailLayoutDirContext, + emailLayout: EmailLayoutData, +): Promise => { + const backupDirPath = path.resolve(sandboxDir, uniqueId("backup")); + + const bundle = buildEmailLayoutDirBundle(emailLayout); + try { + // We store a backup in case there's an error. + if (emailLayoutDirCtx.exists) { + await fs.copy(emailLayoutDirCtx.abspath, backupDirPath); + await fs.emptyDir(emailLayoutDirCtx.abspath); + } + + const promises = Object.entries(bundle).map(([relpath, fileContent]) => { + const filePath = path.resolve(emailLayoutDirCtx.abspath, relpath); + + return relpath === `${emailLayout.key}.json` + ? fs.outputJson(filePath, fileContent, { spaces: DOUBLE_SPACES }) + : fs.outputFile(filePath, fileContent); + }); + await Promise.all(promises); + } catch (error) { + // In case of any error, wipe the target directory that is likely in a bad + // state then restore the backup if one existed before. + if (emailLayoutDirCtx.exists) { + await fs.emptyDir(emailLayoutDirCtx.abspath); + await fs.copy(backupDirPath, emailLayoutDirCtx.abspath); + } else { + await fs.remove(emailLayoutDirCtx.abspath); + } + + throw error; + } finally { + // Always clean up the backup directory in the temp sandbox. + await fs.remove(backupDirPath); + } +}; + +// For a given email layout payload, this function builds an "layout directoy bundle". +// This is an object which contains an `email_layout_key.json` file and all extractable fields. +// Those extractable fields are extracted out and added to the bundle as separate files. +const buildEmailLayoutDirBundle = ( + emailLayout: EmailLayoutData, +): EmailLayoutDirBundle => { + const bundle: EmailLayoutDirBundle = {}; + + // A compiled map of extraction settings of every field in the email layout + const compiledExtractionSettings = compileExtractionSettings(emailLayout); + + // Iterate through each extractable field, determine whether we need to + // extract the field content, and if so, perform the + // extraction. + for (const [objPathParts, extractionSettings] of compiledExtractionSettings) { + const { default: extractByDefault, file_ext: fileExt } = extractionSettings; + if (!extractByDefault) continue; + // Extract the field and its content + let data = get(emailLayout, objPathParts); + const relpath = formatExtractedFilePath(objPathParts, fileExt); + + data = mapValuesDeep(data, (value, key) => { + if (!FILEPATH_MARKED_RE.test(key)) return value; + + const rebaseRootDir = path.dirname(relpath); + const rebasedFilePath = path.relative(rebaseRootDir, value); + + return rebasedFilePath; + }); + + set(bundle, [relpath], data); + } + + // At this point the bundles contains all extractable fields, so we finally add the email layout JSON file. + return set( + bundle, + [`${emailLayout.key}.json`], + toEmailLayoutJson(emailLayout), + ); +}; + +const formatExtractedFilePath = ( + objPathParts: ObjKeyOrArrayIdx[], + fileExt: string, +): string => { + const fileName = objPathParts.pop(); + const paths = [`${fileName}.${fileExt}`]; -export const writeEmailLayoutFile = async ( - emailLayoutAbsPath: string, - emailLayout: EmailLayoutData, -): Promise => - fs.outputJson(emailLayoutAbsPath, emailLayout, { - spaces: DOUBLE_SPACES, - }); + return path.join(...paths).toLowerCase(); +}; From 11b6b2133f0e4aa7aa23ff5a3246fa5435ee680c Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 26 Sep 2023 16:00:58 -0300 Subject: [PATCH 04/45] feat(KNO-4367): add pull all email layout logic --- src/commands/layout/pull.ts | 67 +++++++++++++++++++++++--- src/lib/marshal/email-layout/writer.ts | 47 +++++++++++++++++- 2 files changed, 106 insertions(+), 8 deletions(-) diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts index 67e2656..47c4fea 100644 --- a/src/commands/layout/pull.ts +++ b/src/commands/layout/pull.ts @@ -6,8 +6,8 @@ import * as ApiV1 from "@/lib/api-v1"; import BaseCommand from "@/lib/base-command"; import * as CustomFlags from "@/lib/helpers/flag"; import { merge } from "@/lib/helpers/object"; -import { withSpinner } from "@/lib/helpers/request"; -import { promptToConfirm } from "@/lib/helpers/ux"; +import { formatErrorRespMessage, isSuccessResp, withSpinner } from "@/lib/helpers/request"; +import { promptToConfirm, spinner } from "@/lib/helpers/ux"; import * as EmailLayout from "@/lib/marshal/email-layout"; import { WithAnnotation } from "@/lib/marshal/shared/types"; import { @@ -15,6 +15,8 @@ import { ensureResourceDirForTarget, ResourceTarget, } from "@/lib/run-context"; +import { MAX_PAGINATION_LIMIT, PageInfo } from "@/lib/helpers/page"; +import { ApiError } from "@/lib/helpers/error"; export default class EmailLayoutPull extends BaseCommand< typeof EmailLayoutPull @@ -60,12 +62,10 @@ export default class EmailLayoutPull extends BaseCommand< ); } - return this.pullOneEmailLayout(); + return flags.all ? this.pullAllEmailLayouts() : this.pullOneEmailLayout(); } - /* - * Pull one email layout - */ + // Pull one email layout async pullOneEmailLayout(): Promise { const { flags } = this.props; @@ -90,7 +90,7 @@ export default class EmailLayoutPull extends BaseCommand< }, ); - await EmailLayout.writeWorkflowDirFromData(dirContext, resp.data); + await EmailLayout.writeEmailLayoutDirFromData(dirContext, resp.data); const action = dirContext.exists ? "updated" : "created"; this.log( @@ -98,6 +98,59 @@ export default class EmailLayoutPull extends BaseCommand< ); } + // Pull all email layout + async pullAllEmailLayouts(): Promise { + const { flags } = this.props; + + const defaultToCwd = { abspath: this.runContext.cwd, exists: true }; + const targetDirCtx = flags["layout-dir"] || defaultToCwd; + + const prompt = targetDirCtx.exists + ? `Pull latest layouts into ${targetDirCtx.abspath}?\n This will overwrite the contents of this directory.` + : `Create a new layouts directory at ${targetDirCtx.abspath}?`; + + const input = flags.force || (await promptToConfirm(prompt)); + if (!input) return; + + spinner.start(`‣ Loading`); + + const emailLayouts = await this.listAllEmailLayouts(); + + await EmailLayout.writeEmailLayoutIndexDir(targetDirCtx, emailLayouts); + spinner.stop(); + + const action = targetDirCtx.exists ? "updated" : "created"; + this.log( + `‣ Successfully ${action} the layouts directory at ${targetDirCtx.abspath}`, + ); + } + + async listAllEmailLayouts( + pageParams: Partial = {}, + emailLayoutsFetchedSoFar: EmailLayout.EmailLayoutData[] = [], + ): Promise[]> { + const props = merge(this.props, { + flags: { + ...pageParams, + annotate: true, + limit: MAX_PAGINATION_LIMIT, + }, + }); + + const resp = await this.apiV1.listEmailLayouts(props) + if (!isSuccessResp(resp)) { + const message = formatErrorRespMessage(resp); + this.error(new ApiError(message)); + } + + const { entries, page_info: pageInfo } = resp.data; + const emailLayouts = [...emailLayoutsFetchedSoFar, ...entries]; + + return pageInfo.after + ? this.listAllEmailLayouts({ after: pageInfo.after }, emailLayouts) + : emailLayouts; + } + async getEmailLayoutDirContext(): Promise { const { emailLayoutKey } = this.props.args; const { resourceDir, cwd: runCwd } = this.runContext; diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 7905406..84c11d5 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -16,6 +16,8 @@ import { EmailLayoutDirContext } from "@/lib/run-context"; import { ExtractionSettings, WithAnnotation } from "../shared/types"; import { FILEPATH_MARKED_RE } from "../workflow"; import { EmailLayoutData } from "./types"; +import { DirContext } from "@/lib/helpers/fs"; +import { isEmailLayoutDir } from "./helpers"; export type EmailLayoutDirBundle = { [relpath: string]: string; @@ -61,7 +63,7 @@ const toEmailLayoutJson = ( }; // Builds an email layout dir bundle and writes it into a layout directory on local file system. -export const writeWorkflowDirFromData = async ( +export const writeEmailLayoutDirFromData = async ( emailLayoutDirCtx: EmailLayoutDirContext, emailLayout: EmailLayoutData, ): Promise => { @@ -150,3 +152,46 @@ const formatExtractedFilePath = ( return path.join(...paths).toLowerCase(); }; + + +// This bulk write function takes the fetched email layouts from KNOCK API and writes +// them into a `layout` index directoy. + +export const writeEmailLayoutIndexDir = async ( + indexDirCtx: DirContext, + emailLayouts: EmailLayoutData[], +): Promise => { + const backupDirPath = path.resolve(sandboxDir, uniqueId("backup")); + try { + if (indexDirCtx.exists) { + await fs.copy(indexDirCtx.abspath, backupDirPath); + } + + const writeEmailLayoutDirPromises = emailLayouts.map(async (emailLayout) => { + const emailLayoutDirPath = path.resolve(indexDirCtx.abspath, emailLayout.key); + + const emailLayoutDirCtx: EmailLayoutDirContext = { + type: "email_layout", + key: emailLayout.key, + abspath: emailLayoutDirPath, + exists: indexDirCtx.exists ? await isEmailLayoutDir(emailLayoutDirPath, `${emailLayout.key}.json`) : false + }; + + return writeEmailLayoutDirFromData(emailLayoutDirCtx, emailLayout) + }); + + await Promise.all(writeEmailLayoutDirPromises) + } catch (error) { + if (indexDirCtx.exists) { + await fs.emptyDir(indexDirCtx.abspath); + await fs.copy(backupDirPath, indexDirCtx.abspath); + } else { + await fs.remove(indexDirCtx.abspath); + } + + throw error; + } finally { + // Always clean up the backup directory in the temp sandbox. + await fs.remove(backupDirPath); + } +} \ No newline at end of file From 6efb8ff515e0ce2465547afd2d5a05bfde8911ff Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 26 Sep 2023 16:33:31 -0300 Subject: [PATCH 05/45] feat(KNO-4367): add final logic for pull all email layouts --- src/commands/layout/pull.ts | 15 ++++---- src/lib/marshal/email-layout/writer.ts | 49 ++++++++++++++++---------- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts index 47c4fea..da5b7e4 100644 --- a/src/commands/layout/pull.ts +++ b/src/commands/layout/pull.ts @@ -4,9 +4,15 @@ import { Args, Flags } from "@oclif/core"; import * as ApiV1 from "@/lib/api-v1"; import BaseCommand from "@/lib/base-command"; +import { ApiError } from "@/lib/helpers/error"; import * as CustomFlags from "@/lib/helpers/flag"; import { merge } from "@/lib/helpers/object"; -import { formatErrorRespMessage, isSuccessResp, withSpinner } from "@/lib/helpers/request"; +import { MAX_PAGINATION_LIMIT, PageInfo } from "@/lib/helpers/page"; +import { + formatErrorRespMessage, + isSuccessResp, + withSpinner, +} from "@/lib/helpers/request"; import { promptToConfirm, spinner } from "@/lib/helpers/ux"; import * as EmailLayout from "@/lib/marshal/email-layout"; import { WithAnnotation } from "@/lib/marshal/shared/types"; @@ -15,8 +21,6 @@ import { ensureResourceDirForTarget, ResourceTarget, } from "@/lib/run-context"; -import { MAX_PAGINATION_LIMIT, PageInfo } from "@/lib/helpers/page"; -import { ApiError } from "@/lib/helpers/error"; export default class EmailLayoutPull extends BaseCommand< typeof EmailLayoutPull @@ -66,7 +70,6 @@ export default class EmailLayoutPull extends BaseCommand< } // Pull one email layout - async pullOneEmailLayout(): Promise { const { flags } = this.props; @@ -98,7 +101,7 @@ export default class EmailLayoutPull extends BaseCommand< ); } - // Pull all email layout + // Pull all email layouts async pullAllEmailLayouts(): Promise { const { flags } = this.props; @@ -137,7 +140,7 @@ export default class EmailLayoutPull extends BaseCommand< }, }); - const resp = await this.apiV1.listEmailLayouts(props) + const resp = await this.apiV1.listEmailLayouts(props); if (!isSuccessResp(resp)) { const message = formatErrorRespMessage(resp); this.error(new ApiError(message)); diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 84c11d5..16e41db 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -4,6 +4,7 @@ import * as fs from "fs-extra"; import { get, set, uniqueId } from "lodash"; import { sandboxDir } from "@/lib/helpers/const"; +import { DirContext } from "@/lib/helpers/fs"; import { DOUBLE_SPACES } from "@/lib/helpers/json"; import { mapValuesDeep, @@ -15,9 +16,8 @@ import { EmailLayoutDirContext } from "@/lib/run-context"; import { ExtractionSettings, WithAnnotation } from "../shared/types"; import { FILEPATH_MARKED_RE } from "../workflow"; -import { EmailLayoutData } from "./types"; -import { DirContext } from "@/lib/helpers/fs"; import { isEmailLayoutDir } from "./helpers"; +import { EmailLayoutData } from "./types"; export type EmailLayoutDirBundle = { [relpath: string]: string; @@ -153,8 +153,7 @@ const formatExtractedFilePath = ( return path.join(...paths).toLowerCase(); }; - -// This bulk write function takes the fetched email layouts from KNOCK API and writes +// This bulk write function takes the fetched email layouts from KNOCK API and writes // them into a `layout` index directoy. export const writeEmailLayoutIndexDir = async ( @@ -162,25 +161,37 @@ export const writeEmailLayoutIndexDir = async ( emailLayouts: EmailLayoutData[], ): Promise => { const backupDirPath = path.resolve(sandboxDir, uniqueId("backup")); + try { if (indexDirCtx.exists) { await fs.copy(indexDirCtx.abspath, backupDirPath); + await fs.emptyDir(indexDirCtx.abspath); } - const writeEmailLayoutDirPromises = emailLayouts.map(async (emailLayout) => { - const emailLayoutDirPath = path.resolve(indexDirCtx.abspath, emailLayout.key); - - const emailLayoutDirCtx: EmailLayoutDirContext = { - type: "email_layout", - key: emailLayout.key, - abspath: emailLayoutDirPath, - exists: indexDirCtx.exists ? await isEmailLayoutDir(emailLayoutDirPath, `${emailLayout.key}.json`) : false - }; - - return writeEmailLayoutDirFromData(emailLayoutDirCtx, emailLayout) - }); - - await Promise.all(writeEmailLayoutDirPromises) + const writeEmailLayoutDirPromises = emailLayouts.map( + async (emailLayout) => { + const emailLayoutDirPath = path.resolve( + indexDirCtx.abspath, + emailLayout.key, + ); + + const emailLayoutDirCtx: EmailLayoutDirContext = { + type: "email_layout", + key: emailLayout.key, + abspath: emailLayoutDirPath, + exists: indexDirCtx.exists + ? await isEmailLayoutDir( + emailLayoutDirPath, + `${emailLayout.key}.json`, + ) + : false, + }; + + return writeEmailLayoutDirFromData(emailLayoutDirCtx, emailLayout); + }, + ); + + await Promise.all(writeEmailLayoutDirPromises); } catch (error) { if (indexDirCtx.exists) { await fs.emptyDir(indexDirCtx.abspath); @@ -194,4 +205,4 @@ export const writeEmailLayoutIndexDir = async ( // Always clean up the backup directory in the temp sandbox. await fs.remove(backupDirPath); } -} \ No newline at end of file +}; From fa5ec3f110874111f8c62d8c8abcf0f7fbb1d2c1 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 27 Sep 2023 10:12:40 -0300 Subject: [PATCH 06/45] feat(KNO-4367): final changes --- src/commands/layout/pull.ts | 1 + src/lib/marshal/email-layout/writer.ts | 79 ++++++++++---------------- 2 files changed, 30 insertions(+), 50 deletions(-) diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts index da5b7e4..5bf1629 100644 --- a/src/commands/layout/pull.ts +++ b/src/commands/layout/pull.ts @@ -42,6 +42,7 @@ export default class EmailLayoutPull extends BaseCommand< "layout-dir": CustomFlags.dirPath({ summary: "The target directory path to pull all email layouts into.", dependsOn: ["all"], + aliases: ["email-layout-dir"], }), "hide-uncommitted-changes": Flags.boolean({ summary: "Hide any uncommitted changes.", diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 16e41db..8c0bcca 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -6,29 +6,22 @@ import { get, set, uniqueId } from "lodash"; import { sandboxDir } from "@/lib/helpers/const"; import { DirContext } from "@/lib/helpers/fs"; import { DOUBLE_SPACES } from "@/lib/helpers/json"; -import { - mapValuesDeep, - ObjKeyOrArrayIdx, - omitDeep, -} from "@/lib/helpers/object"; +import { ObjKeyOrArrayIdx, omitDeep } from "@/lib/helpers/object"; import { AnyObj, split } from "@/lib/helpers/object"; import { EmailLayoutDirContext } from "@/lib/run-context"; import { ExtractionSettings, WithAnnotation } from "../shared/types"; -import { FILEPATH_MARKED_RE } from "../workflow"; -import { isEmailLayoutDir } from "./helpers"; import { EmailLayoutData } from "./types"; export type EmailLayoutDirBundle = { [relpath: string]: string; }; -/* - * Sanitize the email latyout content into a format that's appropriate for reading - * and writing, by stripping out any annotation fields and handling readonly - * fields. - */ + type CompiledExtractionSettings = Map; +/* Traverse a given email layout data and compile extraction settings of every extractable + field into a sorted map. +*/ const compileExtractionSettings = ( emailLayout: EmailLayoutData, ): CompiledExtractionSettings => { @@ -50,6 +43,10 @@ const compileExtractionSettings = ( return map; }; +/* Sanitize the email latyout content into a format that's appropriate for reading + and writing, by stripping out any annotation fields and handling readonly + fields. +*/ const toEmailLayoutJson = ( emailLayout: EmailLayoutData, ): AnyObj => { @@ -62,14 +59,16 @@ const toEmailLayoutJson = ( return omitDeep(emailLayoutjson, ["__annotation"]); }; -// Builds an email layout dir bundle and writes it into a layout directory on local file system. +/* Builds an email layout dir bundle, which consist of the email layout JSON + the extractable files. + Then writes them into a layout directory on a local file system. +*/ export const writeEmailLayoutDirFromData = async ( emailLayoutDirCtx: EmailLayoutDirContext, emailLayout: EmailLayoutData, ): Promise => { const backupDirPath = path.resolve(sandboxDir, uniqueId("backup")); - const bundle = buildEmailLayoutDirBundle(emailLayout); + try { // We store a backup in case there's an error. if (emailLayoutDirCtx.exists) { @@ -84,6 +83,7 @@ export const writeEmailLayoutDirFromData = async ( ? fs.outputJson(filePath, fileContent, { spaces: DOUBLE_SPACES }) : fs.outputFile(filePath, fileContent); }); + await Promise.all(promises); } catch (error) { // In case of any error, wipe the target directory that is likely in a bad @@ -102,40 +102,35 @@ export const writeEmailLayoutDirFromData = async ( } }; -// For a given email layout payload, this function builds an "layout directoy bundle". -// This is an object which contains an `email_layout_key.json` file and all extractable fields. -// Those extractable fields are extracted out and added to the bundle as separate files. +/* For a given email layout payload, this function builds a "email layout directoy bundle". + This is an object which contains an `email_layout_key.json` file and all extractable fields. + Those extractable fields are extracted out and added to the bundle as separate files. +*/ const buildEmailLayoutDirBundle = ( emailLayout: EmailLayoutData, ): EmailLayoutDirBundle => { const bundle: EmailLayoutDirBundle = {}; - // A compiled map of extraction settings of every field in the email layout + // A map of extraction settings of every field in the email layout const compiledExtractionSettings = compileExtractionSettings(emailLayout); // Iterate through each extractable field, determine whether we need to // extract the field content, and if so, perform the // extraction. - for (const [objPathParts, extractionSettings] of compiledExtractionSettings) { + for (const [objPath, extractionSettings] of compiledExtractionSettings) { const { default: extractByDefault, file_ext: fileExt } = extractionSettings; - if (!extractByDefault) continue; - // Extract the field and its content - let data = get(emailLayout, objPathParts); - const relpath = formatExtractedFilePath(objPathParts, fileExt); - - data = mapValuesDeep(data, (value, key) => { - if (!FILEPATH_MARKED_RE.test(key)) return value; - const rebaseRootDir = path.dirname(relpath); - const rebasedFilePath = path.relative(rebaseRootDir, value); + if (!extractByDefault) continue; - return rebasedFilePath; - }); + // Extract the field and its content + const data = get(emailLayout, objPath); + const fileName = objPath.pop(); + const relpath = `${fileName}.${fileExt}`; set(bundle, [relpath], data); } - // At this point the bundles contains all extractable fields, so we finally add the email layout JSON file. + // At this point the bundle contains all extractable files, so we finally add the email layout JSON file. return set( bundle, [`${emailLayout.key}.json`], @@ -143,19 +138,8 @@ const buildEmailLayoutDirBundle = ( ); }; -const formatExtractedFilePath = ( - objPathParts: ObjKeyOrArrayIdx[], - fileExt: string, -): string => { - const fileName = objPathParts.pop(); - const paths = [`${fileName}.${fileExt}`]; - - return path.join(...paths).toLowerCase(); -}; - -// This bulk write function takes the fetched email layouts from KNOCK API and writes -// them into a `layout` index directoy. - +// This bulk write function takes the fetched email layouts data KNOCK API and writes +// them into a directory. export const writeEmailLayoutIndexDir = async ( indexDirCtx: DirContext, emailLayouts: EmailLayoutData[], @@ -179,12 +163,7 @@ export const writeEmailLayoutIndexDir = async ( type: "email_layout", key: emailLayout.key, abspath: emailLayoutDirPath, - exists: indexDirCtx.exists - ? await isEmailLayoutDir( - emailLayoutDirPath, - `${emailLayout.key}.json`, - ) - : false, + exists: false, }; return writeEmailLayoutDirFromData(emailLayoutDirCtx, emailLayout); From d4c5af1e46f1b2f2fdef3c6314d939519b0e1825 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 27 Sep 2023 11:48:18 -0300 Subject: [PATCH 07/45] feat(KNO-4367): add tests --- src/commands/layout/pull.ts | 6 +- test/commands/layout/pull.test.ts | 158 ++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 test/commands/layout/pull.test.ts diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts index 5bf1629..9c8a46a 100644 --- a/src/commands/layout/pull.ts +++ b/src/commands/layout/pull.ts @@ -39,10 +39,10 @@ export default class EmailLayoutPull extends BaseCommand< summary: "Whether to pull all email layouts from the specified environment.", }), - "layout-dir": CustomFlags.dirPath({ + "layouts-dir": CustomFlags.dirPath({ summary: "The target directory path to pull all email layouts into.", dependsOn: ["all"], - aliases: ["email-layout-dir"], + aliases: ["email-layouts-dir"], }), "hide-uncommitted-changes": Flags.boolean({ summary: "Hide any uncommitted changes.", @@ -107,7 +107,7 @@ export default class EmailLayoutPull extends BaseCommand< const { flags } = this.props; const defaultToCwd = { abspath: this.runContext.cwd, exists: true }; - const targetDirCtx = flags["layout-dir"] || defaultToCwd; + const targetDirCtx = flags["layouts-dir"] || defaultToCwd; const prompt = targetDirCtx.exists ? `Pull latest layouts into ${targetDirCtx.abspath}?\n This will overwrite the contents of this directory.` diff --git a/test/commands/layout/pull.test.ts b/test/commands/layout/pull.test.ts new file mode 100644 index 0000000..c4244fe --- /dev/null +++ b/test/commands/layout/pull.test.ts @@ -0,0 +1,158 @@ +import * as path from "node:path"; + +import { expect, test } from "@oclif/test"; +import enquirer from "enquirer"; +import * as fs from "fs-extra"; +import { isEqual } from "lodash"; +import * as sinon from "sinon"; + +import { factory } from "@/../test/support"; +import KnockApiV1 from "@/lib/api-v1"; +import { sandboxDir } from "@/lib/helpers/const"; +import { EmailLayoutData } from "@/lib/marshal/email-layout"; + +const currCwd = process.cwd(); + +const setupWithGetLayoutStub = (emailLayoutAttrs = {}) => + test + .env({ KNOCK_SERVICE_TOKEN: "valid-token" }) + .stub( + KnockApiV1.prototype, + "getEmailLayout", + sinon.stub().resolves( + factory.resp({ + data: factory.emailLayout(emailLayoutAttrs), + }), + ), + ) + .stub( + enquirer.prototype, + "prompt", + sinon.stub().onFirstCall().resolves({ input: "y" }), + ); + +const setupWithListLayoutsStub = ( + ...manyLayoutsAttrs: Partial[] +) => + test + .env({ KNOCK_SERVICE_TOKEN: "valid-token" }) + .stub( + KnockApiV1.prototype, + "listEmailLayouts", + sinon.stub().resolves( + factory.resp({ + data: { + entries: manyLayoutsAttrs.map((attrs) => factory.emailLayout(attrs)), + page_info: factory.pageInfo(), + }, + }), + ), + ) + .stub( + enquirer.prototype, + "prompt", + sinon.stub().onFirstCall().resolves({ input: "y" }), + ); + +describe("commands/layout/pull", () => { + beforeEach(() => { + fs.removeSync(sandboxDir); + fs.ensureDirSync(sandboxDir); + process.chdir(sandboxDir); + }); + afterEach(() => { + process.chdir(currCwd); + fs.removeSync(sandboxDir); + }); + + describe("given an email layout key arg", () => { + setupWithGetLayoutStub({ key: "default" }) + .stdout() + .command(["layout pull", "default"]) + .it("calls apiV1 getEmailLayout with an annotate param", () => { + sinon.assert.calledWith( + KnockApiV1.prototype.getEmailLayout as any, + sinon.match( + ({ args, flags }) => + isEqual(args, { + emailLayoutKey: "default", + }) && + isEqual(flags, { + "service-token": "valid-token", + environment: "development", + annotate: true, + }), + ), + ); + }); + + setupWithGetLayoutStub({ key: "messages" }) + .stdout() + .command(["layout pull", "messages"]) + .it("writes an email layout dir to the file system", () => { + const exists = fs.pathExistsSync( + path.resolve(sandboxDir, "messages", "messages.json"), + ); + + expect(exists).to.equal(true); + }); + }); + + describe("given a --all flag", () => { + setupWithListLayoutsStub({ key: "messages" }, { key: "transactional" }) + .stdout() + .command(["layout pull", "--all", "--layouts-dir", "./layouts"]) + .it("calls apiV1 listEmailLayouts with an annotate param", () => { + sinon.assert.calledWith( + KnockApiV1.prototype.listEmailLayouts as any, + sinon.match( + ({ args, flags }) => + isEqual(args, {}) && + isEqual(flags, { + all: true, + "layouts-dir": { + abspath: path.resolve(sandboxDir, "layouts"), + exists: false, + }, + "service-token": "valid-token", + environment: "development", + annotate: true, + limit: 100, + }), + ), + ); + }); + + setupWithListLayoutsStub( + { key: "default" }, + { key: "messages" }, + { key: "transactional" }, + ).stdout() + .command(["layout pull", "--all", "--layouts-dir", "./layouts"]) + .it( + "writes a layout dir to the file system (+ the layout JSON file), with individual layouts dirs inside", + () => { + const path1 = path.resolve(sandboxDir, "layouts", "default", "default.json"); + expect(fs.pathExistsSync(path1)).to.equal(true); + + const path2 = path.resolve(sandboxDir, "layouts", "messages", "messages.json"); + expect(fs.pathExistsSync(path2)).to.equal(true); + + const path3 = path.resolve(sandboxDir, "layouts", "transactional", "transactional.json"); + expect(fs.pathExistsSync(path3)).to.equal(true); + }, + ); + }); + + describe("given both an email layout key arg and a --all flag", () => { + test + .env({ KNOCK_SERVICE_TOKEN: "valid-token" }) + .stdout() + .command(["layout pull", "default", "--all"]) + .catch( + "emailLayoutKey arg `default` cannot also be provided when using --all", + ) + .it("throws an error"); + }); + +}); \ No newline at end of file From 1487b19b6d77a041cf05fc26502750024c39aeac Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 27 Sep 2023 12:52:51 -0300 Subject: [PATCH 08/45] chore(): fix lint --- test/commands/layout/pull.test.ts | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/test/commands/layout/pull.test.ts b/test/commands/layout/pull.test.ts index c4244fe..d9d50c3 100644 --- a/test/commands/layout/pull.test.ts +++ b/test/commands/layout/pull.test.ts @@ -42,7 +42,9 @@ const setupWithListLayoutsStub = ( sinon.stub().resolves( factory.resp({ data: { - entries: manyLayoutsAttrs.map((attrs) => factory.emailLayout(attrs)), + entries: manyLayoutsAttrs.map((attrs) => + factory.emailLayout(attrs), + ), page_info: factory.pageInfo(), }, }), @@ -127,18 +129,34 @@ describe("commands/layout/pull", () => { { key: "default" }, { key: "messages" }, { key: "transactional" }, - ).stdout() + ) + .stdout() .command(["layout pull", "--all", "--layouts-dir", "./layouts"]) .it( "writes a layout dir to the file system (+ the layout JSON file), with individual layouts dirs inside", () => { - const path1 = path.resolve(sandboxDir, "layouts", "default", "default.json"); + const path1 = path.resolve( + sandboxDir, + "layouts", + "default", + "default.json", + ); expect(fs.pathExistsSync(path1)).to.equal(true); - const path2 = path.resolve(sandboxDir, "layouts", "messages", "messages.json"); + const path2 = path.resolve( + sandboxDir, + "layouts", + "messages", + "messages.json", + ); expect(fs.pathExistsSync(path2)).to.equal(true); - const path3 = path.resolve(sandboxDir, "layouts", "transactional", "transactional.json"); + const path3 = path.resolve( + sandboxDir, + "layouts", + "transactional", + "transactional.json", + ); expect(fs.pathExistsSync(path3)).to.equal(true); }, ); @@ -154,5 +172,4 @@ describe("commands/layout/pull", () => { ) .it("throws an error"); }); - -}); \ No newline at end of file +}); From 28aaea546a83972c86f3be12c87b91adfe708fe4 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 27 Sep 2023 13:19:19 -0300 Subject: [PATCH 09/45] chore(): update comments --- src/lib/marshal/email-layout/writer.ts | 7 ++++--- test/commands/layout/pull.test.ts | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 8c0bcca..745d3ec 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -103,8 +103,8 @@ export const writeEmailLayoutDirFromData = async ( }; /* For a given email layout payload, this function builds a "email layout directoy bundle". - This is an object which contains an `email_layout_key.json` file and all extractable fields. - Those extractable fields are extracted out and added to the bundle as separate files. + This is an object which contains all the relative paths and its file content. + It includes the extractable fields, which are extracted out and added to the bundle as separate files. */ const buildEmailLayoutDirBundle = ( emailLayout: EmailLayoutData, @@ -130,7 +130,8 @@ const buildEmailLayoutDirBundle = ( set(bundle, [relpath], data); } - // At this point the bundle contains all extractable files, so we finally add the email layout JSON file. + // At this point the bundle contains all extractable files, so we finally add the email layout + // JSON realtive path + the file content. return set( bundle, [`${emailLayout.key}.json`], diff --git a/test/commands/layout/pull.test.ts b/test/commands/layout/pull.test.ts index d9d50c3..ccbf919 100644 --- a/test/commands/layout/pull.test.ts +++ b/test/commands/layout/pull.test.ts @@ -133,7 +133,7 @@ describe("commands/layout/pull", () => { .stdout() .command(["layout pull", "--all", "--layouts-dir", "./layouts"]) .it( - "writes a layout dir to the file system (+ the layout JSON file), with individual layouts dirs inside", + "writes a layout dir to the file system, with individual layouts dirs inside (plus a layout JSON file)", () => { const path1 = path.resolve( sandboxDir, From d9767faf9896cf8e06a9fe1d9baeebdfbfc005ab Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 27 Sep 2023 14:12:34 -0300 Subject: [PATCH 10/45] feat(KNO-4367): create layout.json when pulling email layouts --- src/commands/layout/pull.ts | 5 +---- src/lib/marshal/email-layout/helpers.ts | 16 +++++++--------- src/lib/marshal/email-layout/writer.ts | 14 ++++++-------- test/commands/layout/pull.test.ts | 8 ++++---- 4 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts index 9c8a46a..c43ed06 100644 --- a/src/commands/layout/pull.ts +++ b/src/commands/layout/pull.ts @@ -177,10 +177,7 @@ export default class EmailLayoutPull extends BaseCommand< // new email layout directory in the cwd, or update it if there is one already. if (emailLayoutKey) { const dirPath = path.resolve(runCwd, emailLayoutKey); - const exists = await EmailLayout.isEmailLayoutDir( - dirPath, - `${emailLayoutKey}.json`, - ); + const exists = await EmailLayout.isEmailLayoutDir(dirPath); return { type: "email_layout", diff --git a/src/lib/marshal/email-layout/helpers.ts b/src/lib/marshal/email-layout/helpers.ts index 1b36627..5efba60 100644 --- a/src/lib/marshal/email-layout/helpers.ts +++ b/src/lib/marshal/email-layout/helpers.ts @@ -2,6 +2,8 @@ import * as path from "node:path"; import * as fs from "fs-extra"; +export const LAYOUT_JSON = "layout.json"; + export type EmailLayoutFileContext = { key: string; abspath: string; @@ -10,23 +12,19 @@ export type EmailLayoutFileContext = { /* * Evaluates whether the given directory path is an email layout directory, by - * checking for the presence of `${email_layout_key}.json` file. + * checking for the presence of a `layout.json` file. */ -export const isEmailLayoutDir = async ( - dirPath: string, - emailLayoutJson: string, -): Promise => - Boolean(await isEmailLayoutJson(dirPath, emailLayoutJson)); +export const isEmailLayoutDir = async (dirPath: string): Promise => + Boolean(await isEmailLayoutJson(dirPath)); /* - * Check for the existance of`${email_layout_key}.json` file in the directory. + * Check for `layout.json` file and return the file path if present. */ export const isEmailLayoutJson = async ( dirPath: string, - emailLayoutJson: string, ): Promise => { - const emailLayoutJsonPath = path.resolve(dirPath, emailLayoutJson); + const emailLayoutJsonPath = path.resolve(dirPath, LAYOUT_JSON); const exists = await fs.pathExists(emailLayoutJsonPath); return exists ? emailLayoutJsonPath : undefined; diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 745d3ec..3ed0fb4 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -11,6 +11,7 @@ import { AnyObj, split } from "@/lib/helpers/object"; import { EmailLayoutDirContext } from "@/lib/run-context"; import { ExtractionSettings, WithAnnotation } from "../shared/types"; +import { LAYOUT_JSON } from "./helpers"; import { EmailLayoutData } from "./types"; export type EmailLayoutDirBundle = { @@ -43,7 +44,7 @@ const compileExtractionSettings = ( return map; }; -/* Sanitize the email latyout content into a format that's appropriate for reading +/* Sanitize the email layout content into a format that's appropriate for reading and writing, by stripping out any annotation fields and handling readonly fields. */ @@ -79,7 +80,7 @@ export const writeEmailLayoutDirFromData = async ( const promises = Object.entries(bundle).map(([relpath, fileContent]) => { const filePath = path.resolve(emailLayoutDirCtx.abspath, relpath); - return relpath === `${emailLayout.key}.json` + return relpath === LAYOUT_JSON ? fs.outputJson(filePath, fileContent, { spaces: DOUBLE_SPACES }) : fs.outputFile(filePath, fileContent); }); @@ -130,13 +131,10 @@ const buildEmailLayoutDirBundle = ( set(bundle, [relpath], data); } - // At this point the bundle contains all extractable files, so we finally add the email layout + // At this point the bundle contains all extractable files, so we finally add the layout // JSON realtive path + the file content. - return set( - bundle, - [`${emailLayout.key}.json`], - toEmailLayoutJson(emailLayout), - ); + + return set(bundle, [LAYOUT_JSON], toEmailLayoutJson(emailLayout)); }; // This bulk write function takes the fetched email layouts data KNOCK API and writes diff --git a/test/commands/layout/pull.test.ts b/test/commands/layout/pull.test.ts index ccbf919..b6913b0 100644 --- a/test/commands/layout/pull.test.ts +++ b/test/commands/layout/pull.test.ts @@ -93,7 +93,7 @@ describe("commands/layout/pull", () => { .command(["layout pull", "messages"]) .it("writes an email layout dir to the file system", () => { const exists = fs.pathExistsSync( - path.resolve(sandboxDir, "messages", "messages.json"), + path.resolve(sandboxDir, "messages", "layout.json"), ); expect(exists).to.equal(true); @@ -139,7 +139,7 @@ describe("commands/layout/pull", () => { sandboxDir, "layouts", "default", - "default.json", + "layout.json", ); expect(fs.pathExistsSync(path1)).to.equal(true); @@ -147,7 +147,7 @@ describe("commands/layout/pull", () => { sandboxDir, "layouts", "messages", - "messages.json", + "layout.json", ); expect(fs.pathExistsSync(path2)).to.equal(true); @@ -155,7 +155,7 @@ describe("commands/layout/pull", () => { sandboxDir, "layouts", "transactional", - "transactional.json", + "layout.json", ); expect(fs.pathExistsSync(path3)).to.equal(true); }, From f534639f335a12f84d4142300435b4a3de20d315 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 27 Sep 2023 15:20:26 -0300 Subject: [PATCH 11/45] chore(KNO-4367): delete email_layout resource type --- src/commands/layout/pull.ts | 2 +- src/lib/run-context/types.ts | 13 ++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts index c43ed06..1fd4080 100644 --- a/src/commands/layout/pull.ts +++ b/src/commands/layout/pull.ts @@ -163,7 +163,7 @@ export default class EmailLayoutPull extends BaseCommand< if (resourceDir) { const target: ResourceTarget = { commandId: BaseCommand.id, - type: "email_layout", + type: "layout", key: emailLayoutKey, }; diff --git a/src/lib/run-context/types.ts b/src/lib/run-context/types.ts index ec10f0d..00779bc 100644 --- a/src/lib/run-context/types.ts +++ b/src/lib/run-context/types.ts @@ -16,11 +16,7 @@ export type T = RunContext; * Resource directory context */ -export type ResourceType = - | "workflow" - | "layout" - | "translation" - | "email_layout"; +export type ResourceType = "workflow" | "layout" | "translation"; type ResourceDirContextBase = DirContext & { type: ResourceType; @@ -39,15 +35,10 @@ export type TranslationDirContext = ResourceDirContextBase & { type: "translation"; }; -export type EmailLayoutDirContext = ResourceDirContextBase & { - type: "email_layout"; -}; - export type ResourceDirContext = | WorkflowDirContext | LayoutDirContext - | TranslationDirContext - | EmailLayoutDirContext; + | TranslationDirContext; export type ResourceTarget = { commandId: string; From b2004e8d8157186a21236955fa56133e354f765e Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 27 Sep 2023 15:42:35 -0300 Subject: [PATCH 12/45] feat(KNO-4367): update run context loader --- src/commands/layout/pull.ts | 2 +- src/lib/run-context/loader.ts | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts index 1fd4080..b84b5b9 100644 --- a/src/commands/layout/pull.ts +++ b/src/commands/layout/pull.ts @@ -180,7 +180,7 @@ export default class EmailLayoutPull extends BaseCommand< const exists = await EmailLayout.isEmailLayoutDir(dirPath); return { - type: "email_layout", + type: "layout", key: emailLayoutKey, abspath: dirPath, exists, diff --git a/src/lib/run-context/loader.ts b/src/lib/run-context/loader.ts index c5ca0cb..2dc4708 100644 --- a/src/lib/run-context/loader.ts +++ b/src/lib/run-context/loader.ts @@ -7,6 +7,7 @@ import * as path from "node:path"; import * as Translation from "@/lib/marshal/translation"; import * as Workflow from "@/lib/marshal/workflow"; +import * as EmailLayout from "@/lib/marshal/email-layout"; import { RunContext } from "./types"; @@ -38,6 +39,17 @@ const evaluateRecursively = async ( }; } + // Check if we are inside a layout directory, and if so update the context. + const isLayoutDir = await EmailLayout.isEmailLayoutDir(currDir) + if (!ctx.resourceDir && isLayoutDir) { + ctx.resourceDir = { + type: "layout", + key: path.basename(currDir), + abspath: currDir, + exists: true, + }; + } + // If we've identified the resource context, no need to go further. // TODO: In the future, consider supporting a knock project config file which // we can use to (semi-)explicitly figure out the project directory structure. From 46c3bdf0db9dc29bc426e2148899d1c48e323863 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 27 Sep 2023 15:44:18 -0300 Subject: [PATCH 13/45] chore(): fix lint --- src/lib/run-context/loader.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/run-context/loader.ts b/src/lib/run-context/loader.ts index 2dc4708..19522a8 100644 --- a/src/lib/run-context/loader.ts +++ b/src/lib/run-context/loader.ts @@ -5,9 +5,9 @@ */ import * as path from "node:path"; +import * as EmailLayout from "@/lib/marshal/email-layout"; import * as Translation from "@/lib/marshal/translation"; import * as Workflow from "@/lib/marshal/workflow"; -import * as EmailLayout from "@/lib/marshal/email-layout"; import { RunContext } from "./types"; @@ -40,7 +40,7 @@ const evaluateRecursively = async ( } // Check if we are inside a layout directory, and if so update the context. - const isLayoutDir = await EmailLayout.isEmailLayoutDir(currDir) + const isLayoutDir = await EmailLayout.isEmailLayoutDir(currDir); if (!ctx.resourceDir && isLayoutDir) { ctx.resourceDir = { type: "layout", From 7f268ff815ccdc660781a489b9e6e2d1127cd8f8 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 27 Sep 2023 17:33:38 -0300 Subject: [PATCH 14/45] fix(KNO-4367): fix type check --- src/commands/layout/pull.ts | 6 +++--- src/lib/marshal/email-layout/writer.ts | 8 ++++---- src/lib/run-context/types.ts | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts index b84b5b9..f52c1fe 100644 --- a/src/commands/layout/pull.ts +++ b/src/commands/layout/pull.ts @@ -17,7 +17,7 @@ import { promptToConfirm, spinner } from "@/lib/helpers/ux"; import * as EmailLayout from "@/lib/marshal/email-layout"; import { WithAnnotation } from "@/lib/marshal/shared/types"; import { - EmailLayoutDirContext, + LayoutDirContext, ensureResourceDirForTarget, ResourceTarget, } from "@/lib/run-context"; @@ -155,7 +155,7 @@ export default class EmailLayoutPull extends BaseCommand< : emailLayouts; } - async getEmailLayoutDirContext(): Promise { + async getEmailLayoutDirContext(): Promise { const { emailLayoutKey } = this.props.args; const { resourceDir, cwd: runCwd } = this.runContext; @@ -170,7 +170,7 @@ export default class EmailLayoutPull extends BaseCommand< return ensureResourceDirForTarget( resourceDir, target, - ) as EmailLayoutDirContext; + ) as LayoutDirContext; } // Not inside any existing email layout directory, which means either create a diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 3ed0fb4..4f6fbd6 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -8,7 +8,7 @@ import { DirContext } from "@/lib/helpers/fs"; import { DOUBLE_SPACES } from "@/lib/helpers/json"; import { ObjKeyOrArrayIdx, omitDeep } from "@/lib/helpers/object"; import { AnyObj, split } from "@/lib/helpers/object"; -import { EmailLayoutDirContext } from "@/lib/run-context"; +import { LayoutDirContext } from "@/lib/run-context"; import { ExtractionSettings, WithAnnotation } from "../shared/types"; import { LAYOUT_JSON } from "./helpers"; @@ -64,7 +64,7 @@ const toEmailLayoutJson = ( Then writes them into a layout directory on a local file system. */ export const writeEmailLayoutDirFromData = async ( - emailLayoutDirCtx: EmailLayoutDirContext, + emailLayoutDirCtx: LayoutDirContext, emailLayout: EmailLayoutData, ): Promise => { const backupDirPath = path.resolve(sandboxDir, uniqueId("backup")); @@ -158,8 +158,8 @@ export const writeEmailLayoutIndexDir = async ( emailLayout.key, ); - const emailLayoutDirCtx: EmailLayoutDirContext = { - type: "email_layout", + const emailLayoutDirCtx: LayoutDirContext = { + type: "layout", key: emailLayout.key, abspath: emailLayoutDirPath, exists: false, diff --git a/src/lib/run-context/types.ts b/src/lib/run-context/types.ts index 00779bc..1e39286 100644 --- a/src/lib/run-context/types.ts +++ b/src/lib/run-context/types.ts @@ -27,7 +27,7 @@ export type WorkflowDirContext = ResourceDirContextBase & { type: "workflow"; }; -type LayoutDirContext = ResourceDirContextBase & { +export type LayoutDirContext = ResourceDirContextBase & { type: "layout"; }; From ccd7116fc3ae0694a136ebaa5eb3ed7449080123 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Thu, 28 Sep 2023 08:47:44 -0300 Subject: [PATCH 15/45] chore(KNO-4367): fix lint --- src/commands/layout/pull.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts index f52c1fe..39f90ff 100644 --- a/src/commands/layout/pull.ts +++ b/src/commands/layout/pull.ts @@ -17,8 +17,8 @@ import { promptToConfirm, spinner } from "@/lib/helpers/ux"; import * as EmailLayout from "@/lib/marshal/email-layout"; import { WithAnnotation } from "@/lib/marshal/shared/types"; import { - LayoutDirContext, ensureResourceDirForTarget, + LayoutDirContext, ResourceTarget, } from "@/lib/run-context"; From 6ae7f8a2ad914effb0d61ca0a6c27d2e569bbb24 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Thu, 28 Sep 2023 15:35:14 -0300 Subject: [PATCH 16/45] feat(KNO-4367): reference extracted files in layout.json --- src/lib/marshal/email-layout/writer.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 4f6fbd6..a726b6b 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -1,16 +1,17 @@ import path from "node:path"; import * as fs from "fs-extra"; -import { get, set, uniqueId } from "lodash"; +import { get, set, uniqueId, unset } from "lodash"; import { sandboxDir } from "@/lib/helpers/const"; import { DirContext } from "@/lib/helpers/fs"; import { DOUBLE_SPACES } from "@/lib/helpers/json"; -import { ObjKeyOrArrayIdx, omitDeep } from "@/lib/helpers/object"; +import { ObjKeyOrArrayIdx, ObjPath, omitDeep } from "@/lib/helpers/object"; import { AnyObj, split } from "@/lib/helpers/object"; import { LayoutDirContext } from "@/lib/run-context"; import { ExtractionSettings, WithAnnotation } from "../shared/types"; +import { FILEPATH_MARKER } from "../workflow"; import { LAYOUT_JSON } from "./helpers"; import { EmailLayoutData } from "./types"; @@ -119,16 +120,22 @@ const buildEmailLayoutDirBundle = ( // extract the field content, and if so, perform the // extraction. for (const [objPath, extractionSettings] of compiledExtractionSettings) { + const objPathStr = ObjPath.stringify(objPath); const { default: extractByDefault, file_ext: fileExt } = extractionSettings; if (!extractByDefault) continue; - - // Extract the field and its content + // By this point, we have a field where we need to extract its content. const data = get(emailLayout, objPath); const fileName = objPath.pop(); const relpath = `${fileName}.${fileExt}`; + // Perform the extraction by adding the content and its file path to the + // bundle for writing to the file system later. Then replace the field + // content with the extracted file path and mark the field as extracted + // with @ suffix. set(bundle, [relpath], data); + set(emailLayout, `${objPathStr}${FILEPATH_MARKER}`, relpath); + unset(emailLayout, objPathStr); } // At this point the bundle contains all extractable files, so we finally add the layout From b1fe16c7b566d115a6a4e9c6767adb45de9c8f89 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Mon, 2 Oct 2023 17:17:23 -0300 Subject: [PATCH 17/45] chore(KNO-4367): move isLayoutDir check --- src/lib/run-context/loader.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/lib/run-context/loader.ts b/src/lib/run-context/loader.ts index 19522a8..3103d74 100644 --- a/src/lib/run-context/loader.ts +++ b/src/lib/run-context/loader.ts @@ -26,24 +26,24 @@ const evaluateRecursively = async ( }; } - // NOTE: Must keep this check as last in the order of directory-type checks - // since the `isTranslationDir` only checks that the directory name is a - // valid locale name. - const isTranslationDir = Translation.isTranslationDir(currDir); - if (!ctx.resourceDir && isTranslationDir) { + // Check if we are inside a layout directory, and if so update the context. + const isLayoutDir = await EmailLayout.isEmailLayoutDir(currDir); + if (!ctx.resourceDir && isLayoutDir) { ctx.resourceDir = { - type: "translation", + type: "layout", key: path.basename(currDir), abspath: currDir, exists: true, }; } - // Check if we are inside a layout directory, and if so update the context. - const isLayoutDir = await EmailLayout.isEmailLayoutDir(currDir); - if (!ctx.resourceDir && isLayoutDir) { + // NOTE: Must keep this check as last in the order of directory-type checks + // since the `isTranslationDir` only checks that the directory name is a + // valid locale name. + const isTranslationDir = Translation.isTranslationDir(currDir); + if (!ctx.resourceDir && isTranslationDir) { ctx.resourceDir = { - type: "layout", + type: "translation", key: path.basename(currDir), abspath: currDir, exists: true, From 01dd58c27095c74545ba53c233509c3e2afa606d Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Mon, 2 Oct 2023 17:23:26 -0300 Subject: [PATCH 18/45] feat(KNO-4367): add EmailLayoutDirContext --- src/commands/layout/pull.ts | 10 +++++----- src/lib/marshal/email-layout/writer.ts | 8 ++++---- src/lib/run-context/loader.ts | 2 +- src/lib/run-context/types.ts | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts index 39f90ff..edba6a9 100644 --- a/src/commands/layout/pull.ts +++ b/src/commands/layout/pull.ts @@ -18,7 +18,7 @@ import * as EmailLayout from "@/lib/marshal/email-layout"; import { WithAnnotation } from "@/lib/marshal/shared/types"; import { ensureResourceDirForTarget, - LayoutDirContext, + EmailLayoutDirContext, ResourceTarget, } from "@/lib/run-context"; @@ -155,7 +155,7 @@ export default class EmailLayoutPull extends BaseCommand< : emailLayouts; } - async getEmailLayoutDirContext(): Promise { + async getEmailLayoutDirContext(): Promise { const { emailLayoutKey } = this.props.args; const { resourceDir, cwd: runCwd } = this.runContext; @@ -163,14 +163,14 @@ export default class EmailLayoutPull extends BaseCommand< if (resourceDir) { const target: ResourceTarget = { commandId: BaseCommand.id, - type: "layout", + type: "email_layout", key: emailLayoutKey, }; return ensureResourceDirForTarget( resourceDir, target, - ) as LayoutDirContext; + ) as EmailLayoutDirContext; } // Not inside any existing email layout directory, which means either create a @@ -180,7 +180,7 @@ export default class EmailLayoutPull extends BaseCommand< const exists = await EmailLayout.isEmailLayoutDir(dirPath); return { - type: "layout", + type: "email_layout", key: emailLayoutKey, abspath: dirPath, exists, diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index a726b6b..b440b59 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -8,7 +8,7 @@ import { DirContext } from "@/lib/helpers/fs"; import { DOUBLE_SPACES } from "@/lib/helpers/json"; import { ObjKeyOrArrayIdx, ObjPath, omitDeep } from "@/lib/helpers/object"; import { AnyObj, split } from "@/lib/helpers/object"; -import { LayoutDirContext } from "@/lib/run-context"; +import { EmailLayoutDirContext } from "@/lib/run-context"; import { ExtractionSettings, WithAnnotation } from "../shared/types"; import { FILEPATH_MARKER } from "../workflow"; @@ -65,7 +65,7 @@ const toEmailLayoutJson = ( Then writes them into a layout directory on a local file system. */ export const writeEmailLayoutDirFromData = async ( - emailLayoutDirCtx: LayoutDirContext, + emailLayoutDirCtx: EmailLayoutDirContext, emailLayout: EmailLayoutData, ): Promise => { const backupDirPath = path.resolve(sandboxDir, uniqueId("backup")); @@ -165,8 +165,8 @@ export const writeEmailLayoutIndexDir = async ( emailLayout.key, ); - const emailLayoutDirCtx: LayoutDirContext = { - type: "layout", + const emailLayoutDirCtx: EmailLayoutDirContext = { + type: "email_layout", key: emailLayout.key, abspath: emailLayoutDirPath, exists: false, diff --git a/src/lib/run-context/loader.ts b/src/lib/run-context/loader.ts index 3103d74..e929c79 100644 --- a/src/lib/run-context/loader.ts +++ b/src/lib/run-context/loader.ts @@ -30,7 +30,7 @@ const evaluateRecursively = async ( const isLayoutDir = await EmailLayout.isEmailLayoutDir(currDir); if (!ctx.resourceDir && isLayoutDir) { ctx.resourceDir = { - type: "layout", + type: "email_layout", key: path.basename(currDir), abspath: currDir, exists: true, diff --git a/src/lib/run-context/types.ts b/src/lib/run-context/types.ts index 1e39286..17bfac8 100644 --- a/src/lib/run-context/types.ts +++ b/src/lib/run-context/types.ts @@ -16,7 +16,7 @@ export type T = RunContext; * Resource directory context */ -export type ResourceType = "workflow" | "layout" | "translation"; +export type ResourceType = "workflow" | "email_layout" | "translation"; type ResourceDirContextBase = DirContext & { type: ResourceType; @@ -27,8 +27,8 @@ export type WorkflowDirContext = ResourceDirContextBase & { type: "workflow"; }; -export type LayoutDirContext = ResourceDirContextBase & { - type: "layout"; +export type EmailLayoutDirContext = ResourceDirContextBase & { + type: "email_layout"; }; export type TranslationDirContext = ResourceDirContextBase & { @@ -37,7 +37,7 @@ export type TranslationDirContext = ResourceDirContextBase & { export type ResourceDirContext = | WorkflowDirContext - | LayoutDirContext + | EmailLayoutDirContext | TranslationDirContext; export type ResourceTarget = { From 834567990ca34b561f0438cb32aedbcbb47426f9 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Mon, 2 Oct 2023 17:23:56 -0300 Subject: [PATCH 19/45] chore(): remove newline --- src/lib/marshal/email-layout/helpers.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/marshal/email-layout/helpers.ts b/src/lib/marshal/email-layout/helpers.ts index 5efba60..a8ac25f 100644 --- a/src/lib/marshal/email-layout/helpers.ts +++ b/src/lib/marshal/email-layout/helpers.ts @@ -20,7 +20,6 @@ export const isEmailLayoutDir = async (dirPath: string): Promise => /* * Check for `layout.json` file and return the file path if present. */ - export const isEmailLayoutJson = async ( dirPath: string, ): Promise => { From 3e17b714882aded39579502c2575e87e5dd2fcf1 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Mon, 2 Oct 2023 17:24:50 -0300 Subject: [PATCH 20/45] feat(KNO-4367): add lsEmailLayoutJson --- src/lib/marshal/email-layout/helpers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/marshal/email-layout/helpers.ts b/src/lib/marshal/email-layout/helpers.ts index a8ac25f..31054a4 100644 --- a/src/lib/marshal/email-layout/helpers.ts +++ b/src/lib/marshal/email-layout/helpers.ts @@ -15,12 +15,12 @@ export type EmailLayoutFileContext = { * checking for the presence of a `layout.json` file. */ export const isEmailLayoutDir = async (dirPath: string): Promise => - Boolean(await isEmailLayoutJson(dirPath)); + Boolean(await lsEmailLayoutJson(dirPath)); /* * Check for `layout.json` file and return the file path if present. */ -export const isEmailLayoutJson = async ( +export const lsEmailLayoutJson = async ( dirPath: string, ): Promise => { const emailLayoutJsonPath = path.resolve(dirPath, LAYOUT_JSON); From a2d3fb2e6bb53204f29f655a03ce064639ff9682 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Mon, 2 Oct 2023 17:25:32 -0300 Subject: [PATCH 21/45] chore(): remove unused type --- src/lib/marshal/email-layout/helpers.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/lib/marshal/email-layout/helpers.ts b/src/lib/marshal/email-layout/helpers.ts index 31054a4..071c181 100644 --- a/src/lib/marshal/email-layout/helpers.ts +++ b/src/lib/marshal/email-layout/helpers.ts @@ -4,12 +4,6 @@ import * as fs from "fs-extra"; export const LAYOUT_JSON = "layout.json"; -export type EmailLayoutFileContext = { - key: string; - abspath: string; - exists: boolean; -}; - /* * Evaluates whether the given directory path is an email layout directory, by * checking for the presence of a `layout.json` file. From d59c9835ca858603130234593e6f1c22bc273f60 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Mon, 2 Oct 2023 17:27:46 -0300 Subject: [PATCH 22/45] chore(KNO-4367): fix multi line comment style --- src/lib/marshal/email-layout/writer.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index b440b59..516a1c6 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -22,7 +22,7 @@ export type EmailLayoutDirBundle = { type CompiledExtractionSettings = Map; /* Traverse a given email layout data and compile extraction settings of every extractable - field into a sorted map. + * field into a sorted map. */ const compileExtractionSettings = ( emailLayout: EmailLayoutData, @@ -46,8 +46,8 @@ const compileExtractionSettings = ( }; /* Sanitize the email layout content into a format that's appropriate for reading - and writing, by stripping out any annotation fields and handling readonly - fields. + * and writing, by stripping out any annotation fields and handling readonly + * fields. */ const toEmailLayoutJson = ( emailLayout: EmailLayoutData, @@ -62,7 +62,7 @@ const toEmailLayoutJson = ( }; /* Builds an email layout dir bundle, which consist of the email layout JSON + the extractable files. - Then writes them into a layout directory on a local file system. + * Then writes them into a layout directory on a local file system. */ export const writeEmailLayoutDirFromData = async ( emailLayoutDirCtx: EmailLayoutDirContext, @@ -105,8 +105,8 @@ export const writeEmailLayoutDirFromData = async ( }; /* For a given email layout payload, this function builds a "email layout directoy bundle". - This is an object which contains all the relative paths and its file content. - It includes the extractable fields, which are extracted out and added to the bundle as separate files. + * This is an object which contains all the relative paths and its file content. + * It includes the extractable fields, which are extracted out and added to the bundle as separate files. */ const buildEmailLayoutDirBundle = ( emailLayout: EmailLayoutData, From 16dc22a45f050bd8e0d7f2ec2103eadb2100de93 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 3 Oct 2023 10:44:58 -0300 Subject: [PATCH 23/45] feat(KNO-4367): abstract readExtractedFileSync --- src/commands/layout/pull.ts | 2 +- src/lib/marshal/email-layout/writer.ts | 12 ++-- src/lib/marshal/shared/helpers.ts | 80 +++++++++++++++++++++++ src/lib/marshal/workflow/generator.ts | 3 +- src/lib/marshal/workflow/helpers.ts | 6 -- src/lib/marshal/workflow/reader.ts | 82 ++---------------------- src/lib/marshal/workflow/writer.ts | 11 ++-- test/lib/marshal/workflow/reader.test.ts | 2 +- 8 files changed, 101 insertions(+), 97 deletions(-) create mode 100644 src/lib/marshal/shared/helpers.ts diff --git a/src/commands/layout/pull.ts b/src/commands/layout/pull.ts index edba6a9..c43ed06 100644 --- a/src/commands/layout/pull.ts +++ b/src/commands/layout/pull.ts @@ -17,8 +17,8 @@ import { promptToConfirm, spinner } from "@/lib/helpers/ux"; import * as EmailLayout from "@/lib/marshal/email-layout"; import { WithAnnotation } from "@/lib/marshal/shared/types"; import { - ensureResourceDirForTarget, EmailLayoutDirContext, + ensureResourceDirForTarget, ResourceTarget, } from "@/lib/run-context"; diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 516a1c6..56e0c65 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -8,10 +8,10 @@ import { DirContext } from "@/lib/helpers/fs"; import { DOUBLE_SPACES } from "@/lib/helpers/json"; import { ObjKeyOrArrayIdx, ObjPath, omitDeep } from "@/lib/helpers/object"; import { AnyObj, split } from "@/lib/helpers/object"; +import { FILEPATH_MARKER } from "@/lib/marshal/shared/helpers"; +import { ExtractionSettings, WithAnnotation } from "@/lib/marshal/shared/types"; import { EmailLayoutDirContext } from "@/lib/run-context"; -import { ExtractionSettings, WithAnnotation } from "../shared/types"; -import { FILEPATH_MARKER } from "../workflow"; import { LAYOUT_JSON } from "./helpers"; import { EmailLayoutData } from "./types"; @@ -23,7 +23,7 @@ type CompiledExtractionSettings = Map; /* Traverse a given email layout data and compile extraction settings of every extractable * field into a sorted map. -*/ + */ const compileExtractionSettings = ( emailLayout: EmailLayoutData, ): CompiledExtractionSettings => { @@ -48,7 +48,7 @@ const compileExtractionSettings = ( /* Sanitize the email layout content into a format that's appropriate for reading * and writing, by stripping out any annotation fields and handling readonly * fields. -*/ + */ const toEmailLayoutJson = ( emailLayout: EmailLayoutData, ): AnyObj => { @@ -63,7 +63,7 @@ const toEmailLayoutJson = ( /* Builds an email layout dir bundle, which consist of the email layout JSON + the extractable files. * Then writes them into a layout directory on a local file system. -*/ + */ export const writeEmailLayoutDirFromData = async ( emailLayoutDirCtx: EmailLayoutDirContext, emailLayout: EmailLayoutData, @@ -107,7 +107,7 @@ export const writeEmailLayoutDirFromData = async ( /* For a given email layout payload, this function builds a "email layout directoy bundle". * This is an object which contains all the relative paths and its file content. * It includes the extractable fields, which are extracted out and added to the bundle as separate files. -*/ + */ const buildEmailLayoutDirBundle = ( emailLayout: EmailLayoutData, ): EmailLayoutDirBundle => { diff --git a/src/lib/marshal/shared/helpers.ts b/src/lib/marshal/shared/helpers.ts new file mode 100644 index 0000000..ccc5ff1 --- /dev/null +++ b/src/lib/marshal/shared/helpers.ts @@ -0,0 +1,80 @@ +import * as path from "node:path"; + +import * as fs from "fs-extra"; + +import { formatErrors, JsonDataError } from "@/lib/helpers/error"; +import { ParsedJson, parseJson } from "@/lib/helpers/json"; +import { validateLiquidSyntax } from "@/lib/helpers/liquid"; +import { VISUAL_BLOCKS_JSON } from "@/lib/marshal/workflow"; +import { EmailLayoutDirContext, WorkflowDirContext } from "@/lib/run-context"; + +// Mark any template fields we are extracting out with this suffix as a rule, +// so we can reliably interpret the field value. +export const FILEPATH_MARKER = "@"; +export const FILEPATH_MARKED_RE = new RegExp(`${FILEPATH_MARKER}$`); + +/* + * Read the file at the given path if it exists, validate the content as + * applicable, and return the content string or an error. + */ +type ExtractedFileContent = string | ParsedJson; +type ReadExtractedFileResult = + | [undefined, JsonDataError] + | [ExtractedFileContent, undefined]; + +// The following files are exepected to have valid json content, and should be +// decoded and joined into the main JSON file. +const DECODABLE_JSON_FILES = new Set([VISUAL_BLOCKS_JSON]); + +export const readExtractedFileSync = ( + relpath: string, + dirCtx: WorkflowDirContext | EmailLayoutDirContext, + objPathToFieldStr = "", +): ReadExtractedFileResult => { + // Check if the file actually exists at the given file path. + const abspath = path.resolve(dirCtx.abspath, relpath); + const exists = fs.pathExistsSync(abspath); + if (!exists) { + const error = new JsonDataError( + "must be a relative path string to a file that exists", + objPathToFieldStr, + ); + return [undefined, error]; + } + + // Read the file and check for valid liquid syntax given it is supported + // across all message templates and file extensions. + const contentStr = fs.readFileSync(abspath, "utf8"); + const liquidParseError = validateLiquidSyntax(contentStr); + + if (liquidParseError) { + const error = new JsonDataError( + `points to a file that contains invalid liquid syntax (${relpath})\n\n` + + formatErrors([liquidParseError], { indentBy: 2 }), + objPathToFieldStr, + ); + + return [undefined, error]; + } + + // If the file is expected to contain decodable json, then parse the contentStr + // as such. + const fileName = path.basename(abspath.toLowerCase()); + const decodable = DECODABLE_JSON_FILES.has(fileName); + + const [content, jsonParseErrors] = decodable + ? parseJson(contentStr) + : [contentStr, []]; + + if (jsonParseErrors.length > 0) { + const error = new JsonDataError( + `points to a file with invalid content (${relpath})\n\n` + + formatErrors(jsonParseErrors, { indentBy: 2 }), + objPathToFieldStr, + ); + + return [undefined, error]; + } + + return [content!, undefined]; +}; diff --git a/src/lib/marshal/workflow/generator.ts b/src/lib/marshal/workflow/generator.ts index d3a0b5e..3924036 100644 --- a/src/lib/marshal/workflow/generator.ts +++ b/src/lib/marshal/workflow/generator.ts @@ -2,9 +2,10 @@ import * as path from "node:path"; import { assign, get, zip } from "lodash"; +import { FILEPATH_MARKER } from "@/lib/marshal/shared/helpers"; import { WorkflowDirContext } from "@/lib/run-context"; -import { FILEPATH_MARKER, WORKFLOW_JSON } from "./helpers"; +import { WORKFLOW_JSON } from "./helpers"; import { StepType, WorkflowStepData } from "./types"; import { WorkflowDirBundle, writeWorkflowDirFromBundle } from "./writer"; diff --git a/src/lib/marshal/workflow/helpers.ts b/src/lib/marshal/workflow/helpers.ts index de88260..f4a69e2 100644 --- a/src/lib/marshal/workflow/helpers.ts +++ b/src/lib/marshal/workflow/helpers.ts @@ -16,12 +16,6 @@ export const VISUAL_BLOCKS_JSON = "visual_blocks.json"; export const workflowJsonPath = (workflowDirCtx: WorkflowDirContext): string => path.resolve(workflowDirCtx.abspath, WORKFLOW_JSON); -// Mark any template fields we are extracting out with this suffix as a rule, -// so we can reliably interpret the field value. -// TODO: Move this up to a top level directory when re-used for other resources. -export const FILEPATH_MARKER = "@"; -export const FILEPATH_MARKED_RE = new RegExp(`${FILEPATH_MARKER}$`); - /* * Validates a string input for a workflow key, and returns an error reason * if invalid. diff --git a/src/lib/marshal/workflow/reader.ts b/src/lib/marshal/workflow/reader.ts index 32e923b..f8c433a 100644 --- a/src/lib/marshal/workflow/reader.ts +++ b/src/lib/marshal/workflow/reader.ts @@ -5,13 +5,7 @@ import * as fs from "fs-extra"; import { hasIn, set } from "lodash"; import { formatErrors, JsonDataError, SourceError } from "@/lib/helpers/error"; -import { - ParsedJson, - parseJson, - ParseJsonResult, - readJson, -} from "@/lib/helpers/json"; -import { validateLiquidSyntax } from "@/lib/helpers/liquid"; +import { ParseJsonResult, readJson } from "@/lib/helpers/json"; import { AnyObj, getLastFound, @@ -19,13 +13,15 @@ import { ObjPath, omitDeep, } from "@/lib/helpers/object"; +import { + FILEPATH_MARKED_RE, + readExtractedFileSync, +} from "@/lib/marshal/shared/helpers"; import { WorkflowDirContext } from "@/lib/run-context"; import { - FILEPATH_MARKED_RE, isWorkflowDir, lsWorkflowJson, - VISUAL_BLOCKS_JSON, WORKFLOW_JSON, WorkflowCommandTarget, } from "./helpers"; @@ -39,10 +35,6 @@ export type WorkflowDirData = WorkflowDirContext & { // (e.g. workflow.json, then visual_blocks.json) const MAX_EXTRACTION_LEVEL = 2; -// The following files are exepected to have valid json content, and should be -// decoded and joined into the main workflow.json. -const DECODABLE_JSON_FILES = new Set([VISUAL_BLOCKS_JSON]); - /* * Validate the file path format of an extracted field. The file path must be: * @@ -111,68 +103,6 @@ const validateExtractedFilePath = ( return undefined; }; -/* - * Read the file at the given path if it exists, validate the content as - * applicable, and return the content string or an error. - */ -type ExtractedFileContent = string | ParsedJson; -type ReadExtractedFileResult = - | [undefined, JsonDataError] - | [ExtractedFileContent, undefined]; - -const readExtractedFileSync = ( - relpath: string, - workflowDirCtx: WorkflowDirContext, - objPathToFieldStr = "", -): ReadExtractedFileResult => { - // Check if the file actually exists at the given file path. - const abspath = path.resolve(workflowDirCtx.abspath, relpath); - const exists = fs.pathExistsSync(abspath); - if (!exists) { - const error = new JsonDataError( - "must be a relative path string to a file that exists", - objPathToFieldStr, - ); - return [undefined, error]; - } - - // Read the file and check for valid liquid syntax given it is supported - // across all message templates and file extensions. - const contentStr = fs.readFileSync(abspath, "utf8"); - const liquidParseError = validateLiquidSyntax(contentStr); - - if (liquidParseError) { - const error = new JsonDataError( - `points to a file that contains invalid liquid syntax (${relpath})\n\n` + - formatErrors([liquidParseError], { indentBy: 2 }), - objPathToFieldStr, - ); - - return [undefined, error]; - } - - // If the file is expected to contain decodable json, then parse the contentStr - // as such. - const fileName = path.basename(abspath.toLowerCase()); - const decodable = DECODABLE_JSON_FILES.has(fileName); - - const [content, jsonParseErrors] = decodable - ? parseJson(contentStr) - : [contentStr, []]; - - if (jsonParseErrors.length > 0) { - const error = new JsonDataError( - `points to a file with invalid content (${relpath})\n\n` + - formatErrors(jsonParseErrors, { indentBy: 2 }), - objPathToFieldStr, - ); - - return [undefined, error]; - } - - return [content!, undefined]; -}; - /* * Given a workflow json object, compiles all referenced extracted files from it * and returns the updated object with the extracted content joined and inlined. @@ -424,4 +354,4 @@ export const readAllForCommandTarget = async ( }; // Exported for tests. -export { checkIfValidExtractedFilePathFormat, readExtractedFileSync }; +export { checkIfValidExtractedFilePathFormat }; diff --git a/src/lib/marshal/workflow/writer.ts b/src/lib/marshal/workflow/writer.ts index 58073a5..64c1727 100644 --- a/src/lib/marshal/workflow/writer.ts +++ b/src/lib/marshal/workflow/writer.ts @@ -22,15 +22,14 @@ import { omitDeep, split, } from "@/lib/helpers/object"; -import { ExtractionSettings, WithAnnotation } from "@/lib/marshal/shared/types"; -import { WorkflowDirContext } from "@/lib/run-context"; - import { FILEPATH_MARKED_RE, FILEPATH_MARKER, - isWorkflowDir, - WORKFLOW_JSON, -} from "./helpers"; +} from "@/lib/marshal/shared/helpers"; +import { ExtractionSettings, WithAnnotation } from "@/lib/marshal/shared/types"; +import { WorkflowDirContext } from "@/lib/run-context"; + +import { isWorkflowDir, WORKFLOW_JSON } from "./helpers"; import { readWorkflowDir } from "./reader"; import { StepType, WorkflowData, WorkflowStepData } from "./types"; diff --git a/test/lib/marshal/workflow/reader.test.ts b/test/lib/marshal/workflow/reader.test.ts index 623aba2..09a1666 100644 --- a/test/lib/marshal/workflow/reader.test.ts +++ b/test/lib/marshal/workflow/reader.test.ts @@ -7,9 +7,9 @@ import { get } from "lodash"; import { xpath } from "@/../test/support"; import { sandboxDir } from "@/lib/helpers/const"; import { JsonDataError } from "@/lib/helpers/error"; +import { readExtractedFileSync } from "@/lib/marshal/shared/helpers"; import { checkIfValidExtractedFilePathFormat, - readExtractedFileSync, readWorkflowDir, VISUAL_BLOCKS_JSON, WORKFLOW_JSON, From e6a539328073e9dbcd622c80bfcd5a0e7ac7b4cc Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 3 Oct 2023 11:29:16 -0300 Subject: [PATCH 24/45] feat(KNO-4367): read local layout when pulling --- src/lib/marshal/email-layout/reader.ts | 88 ++++++++++++++++++++++++++ src/lib/marshal/email-layout/writer.ts | 30 +++++++-- 2 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 src/lib/marshal/email-layout/reader.ts diff --git a/src/lib/marshal/email-layout/reader.ts b/src/lib/marshal/email-layout/reader.ts new file mode 100644 index 0000000..a392697 --- /dev/null +++ b/src/lib/marshal/email-layout/reader.ts @@ -0,0 +1,88 @@ +import * as fs from "fs-extra"; +import { mapValues, set, unset } from "lodash"; + +import { JsonDataError } from "@/lib/helpers/error"; +import { ParseJsonResult, readJson } from "@/lib/helpers/json"; +import { AnyObj, omitDeep } from "@/lib/helpers/object"; +import { + FILEPATH_MARKED_RE, + readExtractedFileSync, +} from "@/lib/marshal/shared/helpers"; +import { EmailLayoutDirContext } from "@/lib/run-context"; + +import { lsEmailLayoutJson } from "./helpers"; + +type JoinExtractedFilesResult = [AnyObj, JsonDataError[]]; + +type ReadLayoutDirOpts = { + withExtractedFiles?: boolean; + withReadonlyField?: boolean; +}; + +/* + * The main read function that takes the layout directory context, then reads + * the layout json from the file system and returns the layout data obj. + */ +export const readEmailLayoutDir = async ( + layoutDirCtx: EmailLayoutDirContext, + opts: ReadLayoutDirOpts = {}, +): Promise => { + const { abspath } = layoutDirCtx; + const { withExtractedFiles = false, withReadonlyField = false } = opts; + + const dirExists = await fs.pathExists(abspath); + if (!dirExists) throw new Error(`${abspath} does not exist`); + + const layoutJsonPath = await lsEmailLayoutJson(abspath); + + if (!layoutJsonPath) throw new Error(`${abspath} is not a layout directory`); + + const result = await readJson(layoutJsonPath); + if (!result[0]) return result; + + let [layoutJson] = result; + + layoutJson = withReadonlyField + ? layoutJson + : omitDeep(layoutJson, ["__readonly"]); + + return withExtractedFiles + ? joinExtractedFiles(layoutDirCtx, layoutJson) + : [layoutJson, []]; +}; + +/* + * Given a layout json object, compiles all referenced extracted files from it + * and returns the updated object with the extracted content joined and inlined. + */ +const joinExtractedFiles = async ( + layoutDirCtx: EmailLayoutDirContext, + layoutJson: AnyObj, +): Promise => { + const errors: JsonDataError[] = []; + + mapValues(layoutJson, (relpath: string, key: string) => { + // If not marked with the @suffix, there's nothing to do. + if (!FILEPATH_MARKED_RE.test(key)) return; + + // By this point we have a valid extracted file path, so attempt to read the file + const [content, readExtractedFileError] = readExtractedFileSync( + relpath, + layoutDirCtx, + key, + ); + + if (readExtractedFileError) { + errors.push(readExtractedFileError); + + return; + } + + const objPathStr = key.replace(FILEPATH_MARKED_RE, ""); + // Inline the file content and remove the extracted file path + set(layoutJson, objPathStr, content); + unset(layoutJson, key); + }); + + return [layoutJson, errors]; +}; diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 56e0c65..5d04f4a 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -13,6 +13,7 @@ import { ExtractionSettings, WithAnnotation } from "@/lib/marshal/shared/types"; import { EmailLayoutDirContext } from "@/lib/run-context"; import { LAYOUT_JSON } from "./helpers"; +import { readEmailLayoutDir } from "./reader"; import { EmailLayoutData } from "./types"; export type EmailLayoutDirBundle = { @@ -68,9 +69,15 @@ export const writeEmailLayoutDirFromData = async ( emailLayoutDirCtx: EmailLayoutDirContext, emailLayout: EmailLayoutData, ): Promise => { - const backupDirPath = path.resolve(sandboxDir, uniqueId("backup")); - const bundle = buildEmailLayoutDirBundle(emailLayout); + // If the layout directory exists on the file system (i.e. previously + // pulled before), then read the layout file to use as a reference. + const [localEmailLayout] = emailLayoutDirCtx.exists + ? await readEmailLayoutDir(emailLayoutDirCtx) + : []; + + const bundle = buildEmailLayoutDirBundle(emailLayout, localEmailLayout); + const backupDirPath = path.resolve(sandboxDir, uniqueId("backup")); try { // We store a backup in case there's an error. if (emailLayoutDirCtx.exists) { @@ -110,6 +117,7 @@ export const writeEmailLayoutDirFromData = async ( */ const buildEmailLayoutDirBundle = ( emailLayout: EmailLayoutData, + localEmailLayout: AnyObj = {}, ): EmailLayoutDirBundle => { const bundle: EmailLayoutDirBundle = {}; @@ -120,14 +128,28 @@ const buildEmailLayoutDirBundle = ( // extract the field content, and if so, perform the // extraction. for (const [objPath, extractionSettings] of compiledExtractionSettings) { + // If the field at this path is extracted in the local layout, then + // always extract; otherwise extract based on the field settings default. const objPathStr = ObjPath.stringify(objPath); + + const extractedFilePath = get( + localEmailLayout, + `${objPathStr}${FILEPATH_MARKER}`, + ); const { default: extractByDefault, file_ext: fileExt } = extractionSettings; - if (!extractByDefault) continue; + if (!extractedFilePath && !extractByDefault) continue; + // By this point, we have a field where we need to extract its content. const data = get(emailLayout, objPath); const fileName = objPath.pop(); - const relpath = `${fileName}.${fileExt}`; + + // If we have an extracted file path from the local layout, we use that. In the other + // case we use the default path. + const relpath = + typeof extractedFilePath === "string" + ? extractedFilePath + : `${fileName}.${fileExt}`; // Perform the extraction by adding the content and its file path to the // bundle for writing to the file system later. Then replace the field From 960b3048dbc6adaa5ea1ac9e7a992dc0bf5e241b Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 3 Oct 2023 11:36:47 -0300 Subject: [PATCH 25/45] chore(KNO-4367): rename to remoteEmailLayout --- src/lib/marshal/email-layout/writer.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 5d04f4a..5a722bc 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -67,7 +67,7 @@ const toEmailLayoutJson = ( */ export const writeEmailLayoutDirFromData = async ( emailLayoutDirCtx: EmailLayoutDirContext, - emailLayout: EmailLayoutData, + remoteEmailLayout: EmailLayoutData, ): Promise => { // If the layout directory exists on the file system (i.e. previously // pulled before), then read the layout file to use as a reference. @@ -75,7 +75,7 @@ export const writeEmailLayoutDirFromData = async ( ? await readEmailLayoutDir(emailLayoutDirCtx) : []; - const bundle = buildEmailLayoutDirBundle(emailLayout, localEmailLayout); + const bundle = buildEmailLayoutDirBundle(remoteEmailLayout, localEmailLayout); const backupDirPath = path.resolve(sandboxDir, uniqueId("backup")); try { @@ -116,13 +116,13 @@ export const writeEmailLayoutDirFromData = async ( * It includes the extractable fields, which are extracted out and added to the bundle as separate files. */ const buildEmailLayoutDirBundle = ( - emailLayout: EmailLayoutData, + remoteEmailLayout: EmailLayoutData, localEmailLayout: AnyObj = {}, ): EmailLayoutDirBundle => { const bundle: EmailLayoutDirBundle = {}; // A map of extraction settings of every field in the email layout - const compiledExtractionSettings = compileExtractionSettings(emailLayout); + const compiledExtractionSettings = compileExtractionSettings(remoteEmailLayout); // Iterate through each extractable field, determine whether we need to // extract the field content, and if so, perform the @@ -141,7 +141,7 @@ const buildEmailLayoutDirBundle = ( if (!extractedFilePath && !extractByDefault) continue; // By this point, we have a field where we need to extract its content. - const data = get(emailLayout, objPath); + const data = get(remoteEmailLayout, objPath); const fileName = objPath.pop(); // If we have an extracted file path from the local layout, we use that. In the other @@ -156,14 +156,14 @@ const buildEmailLayoutDirBundle = ( // content with the extracted file path and mark the field as extracted // with @ suffix. set(bundle, [relpath], data); - set(emailLayout, `${objPathStr}${FILEPATH_MARKER}`, relpath); - unset(emailLayout, objPathStr); + set(remoteEmailLayout, `${objPathStr}${FILEPATH_MARKER}`, relpath); + unset(remoteEmailLayout, objPathStr); } // At this point the bundle contains all extractable files, so we finally add the layout // JSON realtive path + the file content. - return set(bundle, [LAYOUT_JSON], toEmailLayoutJson(emailLayout)); + return set(bundle, [LAYOUT_JSON], toEmailLayoutJson(remoteEmailLayout)); }; // This bulk write function takes the fetched email layouts data KNOCK API and writes From 74e65bf4e3fb3b54599afbd85c3cd824d9f952e0 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 3 Oct 2023 11:37:17 -0300 Subject: [PATCH 26/45] feat(KNO-4367): make deep copy of remote layout --- src/lib/marshal/email-layout/writer.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 5a722bc..6f031af 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -1,7 +1,7 @@ import path from "node:path"; import * as fs from "fs-extra"; -import { get, set, uniqueId, unset } from "lodash"; +import { cloneDeep, get, set, uniqueId, unset } from "lodash"; import { sandboxDir } from "@/lib/helpers/const"; import { DirContext } from "@/lib/helpers/fs"; @@ -75,7 +75,8 @@ export const writeEmailLayoutDirFromData = async ( ? await readEmailLayoutDir(emailLayoutDirCtx) : []; - const bundle = buildEmailLayoutDirBundle(remoteEmailLayout, localEmailLayout); + const mutRemoteEmailLayout = cloneDeep(remoteEmailLayout); + const bundle = buildEmailLayoutDirBundle(mutRemoteEmailLayout, localEmailLayout); const backupDirPath = path.resolve(sandboxDir, uniqueId("backup")); try { From 31de9e464d4956a5b0a816654eecdb6c3ef7401f Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 3 Oct 2023 11:39:35 -0300 Subject: [PATCH 27/45] feat(KNO-4367): continue if layout does not have path --- src/lib/marshal/email-layout/writer.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 6f031af..aa3147a 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -1,7 +1,7 @@ import path from "node:path"; import * as fs from "fs-extra"; -import { cloneDeep, get, set, uniqueId, unset } from "lodash"; +import { cloneDeep, get, has, set, uniqueId, unset } from "lodash"; import { sandboxDir } from "@/lib/helpers/const"; import { DirContext } from "@/lib/helpers/fs"; @@ -129,6 +129,9 @@ const buildEmailLayoutDirBundle = ( // extract the field content, and if so, perform the // extraction. for (const [objPath, extractionSettings] of compiledExtractionSettings) { + // If this layout doesn't have this field path, then we don't extract. + if (!has(remoteEmailLayout, objPath)) continue; + // If the field at this path is extracted in the local layout, then // always extract; otherwise extract based on the field settings default. const objPathStr = ObjPath.stringify(objPath); From 5e28104d117300c8c713eca1ec17875093821795 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 3 Oct 2023 13:07:28 -0300 Subject: [PATCH 28/45] feat(KNO-4367): add pruneLayoutsIndexDir --- src/lib/marshal/email-layout/writer.ts | 64 +++++++++++++++++++++----- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index aa3147a..73127e5 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -12,7 +12,7 @@ import { FILEPATH_MARKER } from "@/lib/marshal/shared/helpers"; import { ExtractionSettings, WithAnnotation } from "@/lib/marshal/shared/types"; import { EmailLayoutDirContext } from "@/lib/run-context"; -import { LAYOUT_JSON } from "./helpers"; +import { isEmailLayoutDir, LAYOUT_JSON } from "./helpers"; import { readEmailLayoutDir } from "./reader"; import { EmailLayoutData } from "./types"; @@ -76,7 +76,10 @@ export const writeEmailLayoutDirFromData = async ( : []; const mutRemoteEmailLayout = cloneDeep(remoteEmailLayout); - const bundle = buildEmailLayoutDirBundle(mutRemoteEmailLayout, localEmailLayout); + const bundle = buildEmailLayoutDirBundle( + mutRemoteEmailLayout, + localEmailLayout, + ); const backupDirPath = path.resolve(sandboxDir, uniqueId("backup")); try { @@ -123,7 +126,8 @@ const buildEmailLayoutDirBundle = ( const bundle: EmailLayoutDirBundle = {}; // A map of extraction settings of every field in the email layout - const compiledExtractionSettings = compileExtractionSettings(remoteEmailLayout); + const compiledExtractionSettings = + compileExtractionSettings(remoteEmailLayout); // Iterate through each extractable field, determine whether we need to // extract the field content, and if so, perform the @@ -171,34 +175,39 @@ const buildEmailLayoutDirBundle = ( }; // This bulk write function takes the fetched email layouts data KNOCK API and writes -// them into a directory. +// them into a layouts "index" directory. export const writeEmailLayoutIndexDir = async ( indexDirCtx: DirContext, - emailLayouts: EmailLayoutData[], + remoteEmailLayouts: EmailLayoutData[], ): Promise => { const backupDirPath = path.resolve(sandboxDir, uniqueId("backup")); try { if (indexDirCtx.exists) { await fs.copy(indexDirCtx.abspath, backupDirPath); - await fs.emptyDir(indexDirCtx.abspath); + await pruneLayoutsIndexDir(indexDirCtx, remoteEmailLayouts); } - const writeEmailLayoutDirPromises = emailLayouts.map( - async (emailLayout) => { + const writeEmailLayoutDirPromises = remoteEmailLayouts.map( + async (remoteEmailLayout) => { const emailLayoutDirPath = path.resolve( indexDirCtx.abspath, - emailLayout.key, + remoteEmailLayout.key, ); const emailLayoutDirCtx: EmailLayoutDirContext = { type: "email_layout", - key: emailLayout.key, + key: remoteEmailLayout.key, abspath: emailLayoutDirPath, - exists: false, + exists: indexDirCtx.exists + ? await isEmailLayoutDir(emailLayoutDirPath) + : false, }; - return writeEmailLayoutDirFromData(emailLayoutDirCtx, emailLayout); + return writeEmailLayoutDirFromData( + emailLayoutDirCtx, + remoteEmailLayout, + ); }, ); @@ -217,3 +226,34 @@ export const writeEmailLayoutIndexDir = async ( await fs.remove(backupDirPath); } }; + +/* + * Prunes the index directory by removing any files, or directories that aren't + * layout dirs found in fetched layouts. We want to preserve any layout + * dirs that are going to be updated with remote layouts, so extracted links + * can be respected. + */ +const pruneLayoutsIndexDir = async ( + indexDirCtx: DirContext, + remoteEmailLayouts: EmailLayoutData[], +): Promise => { + const emailLayoutsByKey = Object.fromEntries( + remoteEmailLayouts.map((e) => [e.key.toLowerCase(), e]), + ); + + const dirents = await fs.readdir(indexDirCtx.abspath, { + withFileTypes: true, + }); + const promises = dirents.map(async (dirent) => { + const direntName = dirent.name.toLowerCase(); + const direntPath = path.resolve(indexDirCtx.abspath, direntName); + + if ((await isEmailLayoutDir(direntPath)) && emailLayoutsByKey[direntName]) { + return; + } + + await fs.remove(direntPath); + }); + + await Promise.all(promises); +}; From be9082471e5e5f5bcf1874651cfa276276f1d7ac Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 3 Oct 2023 14:17:28 -0300 Subject: [PATCH 29/45] feat(KNO-4367): add email layout reader test --- test/lib/marshal/layout/reader.test.ts | 179 +++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 test/lib/marshal/layout/reader.test.ts diff --git a/test/lib/marshal/layout/reader.test.ts b/test/lib/marshal/layout/reader.test.ts new file mode 100644 index 0000000..62ef65f --- /dev/null +++ b/test/lib/marshal/layout/reader.test.ts @@ -0,0 +1,179 @@ +import * as path from "node:path"; + +import { expect } from "@oclif/test"; +import * as fs from "fs-extra"; +import { get } from "lodash"; + +import { xpath } from "@/../test/support"; +import { sandboxDir } from "@/lib/helpers/const"; +import { JsonDataError } from "@/lib/helpers/error"; +import { LAYOUT_JSON } from "@/lib/marshal/email-layout"; +import { readExtractedFileSync } from "@/lib/marshal/shared/helpers"; +import { EmailLayoutDirContext } from "@/lib/run-context"; +import { readEmailLayoutDir } from "@/lib/marshal/email-layout/reader"; + +const currCwd = process.cwd(); + +describe("lib/marshal/layout/reader", () => { + describe("readExtractedFileSync", () => { + const emailLayoutDirCtx: EmailLayoutDirContext = { + type: "email_layout", + key: "transactional", + abspath: path.resolve(sandboxDir, "transactional"), + exists: true, + }; + + beforeEach(() => { + fs.removeSync(sandboxDir); + fs.ensureDir(emailLayoutDirCtx.abspath); + }); + after(() => fs.removeSync(sandboxDir)); + + describe("given a valid liquid text layout", () => { + it("returns the read text layout content without errors", async () => { + const fileContent = ` + {{ vars.app_name }} ({{ vars.app_url }}) +

+ Hello {{ recipient.name }} +

+ `.trimStart(); + + const filePath = path.resolve(emailLayoutDirCtx.abspath, "sample.txt"); + fs.outputFileSync(filePath, fileContent); + + const [readContent, error] = readExtractedFileSync( + "sample.txt", + emailLayoutDirCtx, + ); + + expect(readContent).to.be.a("string"); + expect(error).to.equal(undefined); + }); + }); + + describe("given an invalid liquid text layout", () => { + it("returns the read text layout content with errors", async () => { + const fileContent = ` + {{ vars.app_name }} ({{ vars.app_url }}) +

+ Hello {{ recipient.name +

+ `.trimStart(); + + const filePath = path.resolve(emailLayoutDirCtx.abspath, "sample.txt"); + fs.outputFileSync(filePath, fileContent); + + const [readContent, error] = readExtractedFileSync( + "sample.txt", + emailLayoutDirCtx, + ); + + expect(readContent).to.equal(undefined); + expect(error).to.be.an.instanceof(JsonDataError); + }); + }); + }); + + describe("readEmailLayoutDir", () => { + const sampleEmailLayoutJson = { + name: "Transactional", + "html_layout@": "html_layout.html", + "text_layout@": "text-layout-examples/text_layout.txt", + __readonly: { + key: "transactional", + environment: "development", + created_at: "2023-09-18T18:32:18.398053Z", + updated_at: "2023-10-02T19:24:48.714630Z", + }, + }; + + const emailLayoutDirPath = path.join( + sandboxDir, + "layouts", + "transactional", + ); + + const emailLayoutDirCtx: EmailLayoutDirContext = { + type: "email_layout", + key: "transactional", + abspath: emailLayoutDirPath, + exists: true, + }; + + before(() => { + fs.removeSync(sandboxDir); + + // Set up a sample layout directory + fs.outputJsonSync( + path.join(emailLayoutDirPath, LAYOUT_JSON), + sampleEmailLayoutJson, + ); + + fs.outputJsonSync( + path.join(emailLayoutDirPath, "html_layout.html"), + "

example

", + ); + + fs.outputJsonSync( + path.join( + emailLayoutDirPath, + "text-layout-examples", + "text_layout.txt", + ), + "foo {{content}}", + ); + }); + + after(() => { + process.chdir(currCwd); + fs.removeSync(sandboxDir); + }); + + + describe("by default without any opts", () => { + it("reads layout.json without the readonly field and extracted files joined", async () => { + const [layout] = await readEmailLayoutDir(emailLayoutDirCtx); + + expect(get(layout, ["name"])).to.equal("Transactional"); + expect(get(layout, ["html_layout@"])).to.equal("html_layout.html"); + expect(get(layout, ["text_layout@"])).to.equal("text-layout-examples/text_layout.txt"); + }); + }); + + describe("with the withReadonlyField opt of true", () => { + it("reads layout.json with the readonly field", async () => { + const [layout] = await readEmailLayoutDir(emailLayoutDirCtx, { + withReadonlyField: true, + }); + + expect(get(layout, ["name"])).to.equal("Transactional"); + expect(get(layout, ["html_layout@"])).to.equal("html_layout.html"); + expect(get(layout, ["text_layout@"])).to.equal("text-layout-examples/text_layout.txt"); + + expect(get(layout, ["__readonly"])).to.eql({ + key: "transactional", + environment: "development", + created_at: "2023-09-18T18:32:18.398053Z", + updated_at: "2023-10-02T19:24:48.714630Z" + }); + }); + + describe("with the withExtractedFiles opt of true", () => { + it("reads layout.json with the extracted fields inlined", async () => { + const [layout] = await readEmailLayoutDir(emailLayoutDirCtx, { + withExtractedFiles: true, + }); + + expect(get(layout, ["name"])).to.equal("Transactional"); + + // HTML layout content should be inlined into layout data + expect(get(layout, ["html_layout"])).to.contain("

example

"); + + // Text layout content should be inlined into layout data + expect(get(layout, ["text_layout"])).to.contains("foo {{content}}"); + }); + }); + + }); + }); +}); From c6b4dc9a01e04a8befdff5c45550a6a1e82d1c27 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 3 Oct 2023 15:42:14 -0300 Subject: [PATCH 30/45] chore(KNO-4367): improve writer --- src/lib/marshal/email-layout/types.ts | 1 + src/lib/marshal/email-layout/writer.ts | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/lib/marshal/email-layout/types.ts b/src/lib/marshal/email-layout/types.ts index 5aa4ad6..60bfe50 100644 --- a/src/lib/marshal/email-layout/types.ts +++ b/src/lib/marshal/email-layout/types.ts @@ -7,6 +7,7 @@ export type EmailLayoutData = A & { html_layout: string; text_layout: string; footer_links?: Hyperlink[]; + environment: string; updated_at: string; created_at: string; }; diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 73127e5..6550797 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -75,11 +75,7 @@ export const writeEmailLayoutDirFromData = async ( ? await readEmailLayoutDir(emailLayoutDirCtx) : []; - const mutRemoteEmailLayout = cloneDeep(remoteEmailLayout); - const bundle = buildEmailLayoutDirBundle( - mutRemoteEmailLayout, - localEmailLayout, - ); + const bundle = buildEmailLayoutDirBundle(remoteEmailLayout, localEmailLayout); const backupDirPath = path.resolve(sandboxDir, uniqueId("backup")); try { @@ -124,17 +120,18 @@ const buildEmailLayoutDirBundle = ( localEmailLayout: AnyObj = {}, ): EmailLayoutDirBundle => { const bundle: EmailLayoutDirBundle = {}; + const mutRemoteEmailLayout = cloneDeep(remoteEmailLayout); // A map of extraction settings of every field in the email layout const compiledExtractionSettings = - compileExtractionSettings(remoteEmailLayout); + compileExtractionSettings(mutRemoteEmailLayout); // Iterate through each extractable field, determine whether we need to // extract the field content, and if so, perform the // extraction. for (const [objPath, extractionSettings] of compiledExtractionSettings) { // If this layout doesn't have this field path, then we don't extract. - if (!has(remoteEmailLayout, objPath)) continue; + if (!has(mutRemoteEmailLayout, objPath)) continue; // If the field at this path is extracted in the local layout, then // always extract; otherwise extract based on the field settings default. @@ -149,7 +146,7 @@ const buildEmailLayoutDirBundle = ( if (!extractedFilePath && !extractByDefault) continue; // By this point, we have a field where we need to extract its content. - const data = get(remoteEmailLayout, objPath); + const data = get(mutRemoteEmailLayout, objPath); const fileName = objPath.pop(); // If we have an extracted file path from the local layout, we use that. In the other @@ -164,14 +161,14 @@ const buildEmailLayoutDirBundle = ( // content with the extracted file path and mark the field as extracted // with @ suffix. set(bundle, [relpath], data); - set(remoteEmailLayout, `${objPathStr}${FILEPATH_MARKER}`, relpath); - unset(remoteEmailLayout, objPathStr); + set(mutRemoteEmailLayout, `${objPathStr}${FILEPATH_MARKER}`, relpath); + unset(mutRemoteEmailLayout, objPathStr); } // At this point the bundle contains all extractable files, so we finally add the layout // JSON realtive path + the file content. - return set(bundle, [LAYOUT_JSON], toEmailLayoutJson(remoteEmailLayout)); + return set(bundle, [LAYOUT_JSON], toEmailLayoutJson(mutRemoteEmailLayout)); }; // This bulk write function takes the fetched email layouts data KNOCK API and writes @@ -257,3 +254,6 @@ const pruneLayoutsIndexDir = async ( await Promise.all(promises); }; + +// Exported for tests +export { buildEmailLayoutDirBundle, pruneLayoutsIndexDir, toEmailLayoutJson }; From d22033a34a5bccb29def3ca9e8583f093ec08773 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 3 Oct 2023 15:42:38 -0300 Subject: [PATCH 31/45] feat(KNO-4367): add writer test --- test/lib/marshal/layout/reader.test.ts | 19 ++- test/lib/marshal/layout/writer.test.ts | 200 +++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 8 deletions(-) create mode 100644 test/lib/marshal/layout/writer.test.ts diff --git a/test/lib/marshal/layout/reader.test.ts b/test/lib/marshal/layout/reader.test.ts index 62ef65f..c308145 100644 --- a/test/lib/marshal/layout/reader.test.ts +++ b/test/lib/marshal/layout/reader.test.ts @@ -4,13 +4,12 @@ import { expect } from "@oclif/test"; import * as fs from "fs-extra"; import { get } from "lodash"; -import { xpath } from "@/../test/support"; import { sandboxDir } from "@/lib/helpers/const"; import { JsonDataError } from "@/lib/helpers/error"; import { LAYOUT_JSON } from "@/lib/marshal/email-layout"; +import { readEmailLayoutDir } from "@/lib/marshal/email-layout/reader"; import { readExtractedFileSync } from "@/lib/marshal/shared/helpers"; import { EmailLayoutDirContext } from "@/lib/run-context"; -import { readEmailLayoutDir } from "@/lib/marshal/email-layout/reader"; const currCwd = process.cwd(); @@ -129,14 +128,15 @@ describe("lib/marshal/layout/reader", () => { fs.removeSync(sandboxDir); }); - describe("by default without any opts", () => { it("reads layout.json without the readonly field and extracted files joined", async () => { const [layout] = await readEmailLayoutDir(emailLayoutDirCtx); expect(get(layout, ["name"])).to.equal("Transactional"); expect(get(layout, ["html_layout@"])).to.equal("html_layout.html"); - expect(get(layout, ["text_layout@"])).to.equal("text-layout-examples/text_layout.txt"); + expect(get(layout, ["text_layout@"])).to.equal( + "text-layout-examples/text_layout.txt", + ); }); }); @@ -148,13 +148,15 @@ describe("lib/marshal/layout/reader", () => { expect(get(layout, ["name"])).to.equal("Transactional"); expect(get(layout, ["html_layout@"])).to.equal("html_layout.html"); - expect(get(layout, ["text_layout@"])).to.equal("text-layout-examples/text_layout.txt"); + expect(get(layout, ["text_layout@"])).to.equal( + "text-layout-examples/text_layout.txt", + ); expect(get(layout, ["__readonly"])).to.eql({ key: "transactional", environment: "development", created_at: "2023-09-18T18:32:18.398053Z", - updated_at: "2023-10-02T19:24:48.714630Z" + updated_at: "2023-10-02T19:24:48.714630Z", }); }); @@ -167,13 +169,14 @@ describe("lib/marshal/layout/reader", () => { expect(get(layout, ["name"])).to.equal("Transactional"); // HTML layout content should be inlined into layout data - expect(get(layout, ["html_layout"])).to.contain("

example

"); + expect(get(layout, ["html_layout"])).to.contain( + "

example

", + ); // Text layout content should be inlined into layout data expect(get(layout, ["text_layout"])).to.contains("foo {{content}}"); }); }); - }); }); }); diff --git a/test/lib/marshal/layout/writer.test.ts b/test/lib/marshal/layout/writer.test.ts new file mode 100644 index 0000000..d74dad6 --- /dev/null +++ b/test/lib/marshal/layout/writer.test.ts @@ -0,0 +1,200 @@ +import path from "node:path"; + +import { expect } from "chai"; +import * as fs from "fs-extra"; +import { get } from "lodash"; + +import { sandboxDir } from "@/lib/helpers/const"; +import { + buildEmailLayoutDirBundle, + EmailLayoutData, + LAYOUT_JSON, + pruneLayoutsIndexDir, + toEmailLayoutJson, +} from "@/lib/marshal/email-layout"; +import { WithAnnotation } from "@/lib/marshal/shared/types"; + +const annotation = { + extractable_fields: { + html_layout: { default: true, file_ext: "html" }, + text_layout: { default: true, file_ext: "txt" }, + }, + readonly_fields: ["key", "environment", "created_at", "updated_at"], +}; + +const remoteEmailLayout: EmailLayoutData = { + key: "default", + name: "Default", + html_layout: "

Example content

", + text_layout: "Text {{content}}", + footer_links: [{ text: "Link 1", url: "https://example.com" }], + environment: "development", + updated_at: "2023-10-02T19:24:48.714630Z", + created_at: "2023-09-18T18:32:18.398053Z", + __annotation: annotation, +}; + +describe("lib/marshal/layout/writer", () => { + describe("toEmailLayoutJson", () => { + it("moves over layout's readonly fields under __readonly field", () => { + const layoutJson = toEmailLayoutJson(remoteEmailLayout); + + expect(layoutJson.key).to.equal(undefined); + expect(layoutJson.environment).to.equal(undefined); + expect(layoutJson.created_at).to.equal(undefined); + expect(layoutJson.updated_at).to.equal(undefined); + + expect(layoutJson.__readonly).to.eql({ + key: "default", + environment: "development", + created_at: "2023-09-18T18:32:18.398053Z", + updated_at: "2023-10-02T19:24:48.714630Z", + }); + }); + + it("removes all __annotation fields", () => { + const layoutJson = toEmailLayoutJson(remoteEmailLayout); + + expect(get(layoutJson, "__annotation")).to.equal(undefined); + }); + }); + + describe("pruneLayoutsIndexDir", () => { + const remoteEmailLayouts: EmailLayoutData[] = [ + { + key: "foo", + name: "Foo", + html_layout: "

Example content

", + text_layout: "Text {{content}}", + footer_links: [], + environment: "development", + updated_at: "2023-10-02T19:24:48.714630Z", + created_at: "2023-09-18T18:32:18.398053Z", + __annotation: annotation, + }, + ]; + + const layoutsIndexDir = path.resolve(sandboxDir, "layouts"); + + beforeEach(() => { + fs.removeSync(sandboxDir); + fs.ensureDirSync(layoutsIndexDir); + }); + + after(() => { + fs.removeSync(sandboxDir); + }); + + describe("given a file in the layouts index dir", () => { + it("removes the file", async () => { + const filePath = path.resolve(layoutsIndexDir, "foo"); + fs.ensureFileSync(filePath); + + const indexDirCtx = { abspath: layoutsIndexDir, exists: true }; + await pruneLayoutsIndexDir(indexDirCtx, remoteEmailLayouts); + + expect(fs.pathExistsSync(filePath)).to.equal(false); + }); + }); + + describe("given a non layout directory in the layouts index dir", () => { + it("removes the directory", async () => { + const dirPath = path.resolve(layoutsIndexDir, "foo"); + fs.ensureDirSync(dirPath); + + const indexDirCtx = { abspath: layoutsIndexDir, exists: true }; + await pruneLayoutsIndexDir(indexDirCtx, remoteEmailLayouts); + + expect(fs.pathExistsSync(dirPath)).to.equal(false); + }); + }); + + describe("given a layout directory not found in remote layouts", () => { + it("removes the layout directory", async () => { + const layoutJsonPath = path.resolve( + layoutsIndexDir, + "bar", + LAYOUT_JSON, + ); + fs.ensureFileSync(layoutJsonPath); + + const indexDirCtx = { abspath: layoutsIndexDir, exists: true }; + await pruneLayoutsIndexDir(indexDirCtx, remoteEmailLayouts); + + expect(fs.pathExistsSync(layoutJsonPath)).to.equal(false); + }); + }); + + describe("given a layout directory found in remote layouts", () => { + it("retains the layout directory", async () => { + const layoutJsonPath = path.resolve( + layoutsIndexDir, + "foo", + LAYOUT_JSON, + ); + fs.ensureFileSync(layoutJsonPath); + + const indexDirCtx = { abspath: layoutsIndexDir, exists: true }; + await pruneLayoutsIndexDir(indexDirCtx, remoteEmailLayouts); + + expect(fs.pathExistsSync(layoutJsonPath)).to.equal(true); + }); + }); + }); + + describe("buildEmailLayoutDirBundle", () => { + describe("given a fetched layout that has not been pulled before", () => { + const result = buildEmailLayoutDirBundle(remoteEmailLayout); + + expect(result).to.eql({ + "html_layout.html": "

Example content

", + "text_layout.txt": "Text {{content}}", + "layout.json": { + name: "Default", + footer_links: [{ text: "Link 1", url: "https://example.com" }], + "html_layout@": "html_layout.html", + "text_layout@": "text_layout.txt", + __readonly: { + key: "default", + environment: "development", + created_at: "2023-09-18T18:32:18.398053Z", + updated_at: "2023-10-02T19:24:48.714630Z", + }, + }, + }); + }); + + describe("given a fetched layout with a local version available", () => { + it("returns a dir bundle based on a local version and default extract settings", () => { + const localEmailLayout = { + name: "default", + "html_layout@": "foo/bar/layout.html", + "text_layout@": "text_layout.txt", + }; + + const result = buildEmailLayoutDirBundle( + remoteEmailLayout, + localEmailLayout, + ); + + expect(result).to.eql({ + "foo/bar/layout.html": + "

Example content

", + "text_layout.txt": "Text {{content}}", + "layout.json": { + name: "Default", + footer_links: [{ text: "Link 1", url: "https://example.com" }], + "html_layout@": "foo/bar/layout.html", + "text_layout@": "text_layout.txt", + __readonly: { + key: "default", + environment: "development", + created_at: "2023-09-18T18:32:18.398053Z", + updated_at: "2023-10-02T19:24:48.714630Z", + }, + }, + }); + }); + }); + }); +}); From ce8a5cc208cddbf70f1604f00dbf9153d5823cb6 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 3 Oct 2023 18:17:06 -0300 Subject: [PATCH 32/45] feat(KNO-4367): add email layout path validations --- src/lib/marshal/email-layout/reader.ts | 53 +++++++++++++++++++++- src/lib/marshal/email-layout/writer.ts | 2 +- src/lib/marshal/shared/helpers.ts | 47 +++++++++++++++++++ src/lib/marshal/workflow/reader.ts | 37 +-------------- test/lib/marshal/layout/reader.test.ts | 58 +++++++++++++++++++++++- test/lib/marshal/workflow/reader.test.ts | 4 +- 6 files changed, 160 insertions(+), 41 deletions(-) diff --git a/src/lib/marshal/email-layout/reader.ts b/src/lib/marshal/email-layout/reader.ts index a392697..a42b3e7 100644 --- a/src/lib/marshal/email-layout/reader.ts +++ b/src/lib/marshal/email-layout/reader.ts @@ -1,3 +1,5 @@ +import path from "node:path"; + import * as fs from "fs-extra"; import { mapValues, set, unset } from "lodash"; @@ -5,12 +7,13 @@ import { JsonDataError } from "@/lib/helpers/error"; import { ParseJsonResult, readJson } from "@/lib/helpers/json"; import { AnyObj, omitDeep } from "@/lib/helpers/object"; import { + checkIfValidExtractedFilePathFormat, FILEPATH_MARKED_RE, readExtractedFileSync, } from "@/lib/marshal/shared/helpers"; import { EmailLayoutDirContext } from "@/lib/run-context"; -import { lsEmailLayoutJson } from "./helpers"; +import { LAYOUT_JSON, lsEmailLayoutJson } from "./helpers"; type JoinExtractedFilesResult = [AnyObj, JsonDataError[]]; @@ -65,6 +68,23 @@ const joinExtractedFiles = async ( // If not marked with the @suffix, there's nothing to do. if (!FILEPATH_MARKED_RE.test(key)) return; + const objPathStr = key.replace(FILEPATH_MARKED_RE, ""); + + // Check if the extracted path found at the current field path is valid + const invalidFilePathError = validateExtractedFilePath( + relpath, + layoutDirCtx, + ); + if (invalidFilePathError) { + errors.push(invalidFilePathError); + // Wipe the invalid file path in the node so the final layout json + // object ends up with only valid file paths, this way layout writer + // can see only valid file paths and use those when pulling. + set(layoutJson, key, undefined); + + return; + } + // By this point we have a valid extracted file path, so attempt to read the file const [content, readExtractedFileError] = readExtractedFileSync( relpath, @@ -78,7 +98,6 @@ const joinExtractedFiles = async ( return; } - const objPathStr = key.replace(FILEPATH_MARKED_RE, ""); // Inline the file content and remove the extracted file path set(layoutJson, objPathStr, content); unset(layoutJson, key); @@ -86,3 +105,33 @@ const joinExtractedFiles = async ( return [layoutJson, errors]; }; + +/* + * Validate the extracted file path based on its format and uniqueness (but not + * the presence). + * + * Note, the uniqueness check is based on reading from and writing to + * uniqueFilePaths, which is MUTATED in place. + */ +const validateExtractedFilePath = ( + val: unknown, + emailLayoutDirCtx: EmailLayoutDirContext, +): JsonDataError | undefined => { + const layoutJsonPath = path.resolve(emailLayoutDirCtx.abspath, LAYOUT_JSON); + // Validate the file path format, and that it is unique per workflow. + if ( + !checkIfValidExtractedFilePathFormat(val, layoutJsonPath, { + withAbsolutePaths: true, + }) || + typeof val !== "string" + ) { + const error = new JsonDataError( + "must be a relative path string to a unique file within the directory", + String(val), + ); + + return error; + } + + return undefined; +}; diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 6550797..f5b332d 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -72,7 +72,7 @@ export const writeEmailLayoutDirFromData = async ( // If the layout directory exists on the file system (i.e. previously // pulled before), then read the layout file to use as a reference. const [localEmailLayout] = emailLayoutDirCtx.exists - ? await readEmailLayoutDir(emailLayoutDirCtx) + ? await readEmailLayoutDir(emailLayoutDirCtx, { withExtractedFiles: true }) : []; const bundle = buildEmailLayoutDirBundle(remoteEmailLayout, localEmailLayout); diff --git a/src/lib/marshal/shared/helpers.ts b/src/lib/marshal/shared/helpers.ts index ccc5ff1..bedd28b 100644 --- a/src/lib/marshal/shared/helpers.ts +++ b/src/lib/marshal/shared/helpers.ts @@ -78,3 +78,50 @@ export const readExtractedFileSync = ( return [content!, undefined]; }; + +/* + * Validate the file path format of an extracted field. The file path must be: + * + * 1) Expressed as a relative path. + * + * For exmaple: + * subject@: "email_1/subject.html" // GOOD + * subject@: "./email_1/subject.html" // GOOD + * subject@: "/workflow-x/email_1/subject.html" // BAD + * + * 2) The resolved path must be contained inside the directory + * + * For exmaple (workflow-y is a different workflow dir in this example): + * subject@: "./email_1/subject.html" // GOOD + * subject@: "../workflow-y/email_1/subject.html" // BAD + * + * Note: does not validate the presence of the file nor the uniqueness of the + * file path. + * + * Options: + * - withAbsolutePaths: it will return non absoulte paths as valid. + * For example: + * html_layout@: "html_layout.html" + * + */ +type CheckValidExtractedFilePathOptions = { + withAbsolutePaths?: boolean; +}; + +export const checkIfValidExtractedFilePathFormat = ( + relpath: unknown, + sourceFileAbspath: string, + opts: CheckValidExtractedFilePathOptions = {}, +): boolean => { + const { withAbsolutePaths = false } = opts; + if (typeof relpath !== "string") return false; + // If the option for allowing absolute paths was passed, and the path is indeed absolute, then it's valid. + if (withAbsolutePaths && !path.isAbsolute(relpath)) return true; + + if (path.isAbsolute(relpath)) return false; + + const extractedFileAbspath = path.resolve(sourceFileAbspath, relpath); + const pathDiff = path.relative(sourceFileAbspath, extractedFileAbspath); + + return !pathDiff.startsWith(".."); +}; diff --git a/src/lib/marshal/workflow/reader.ts b/src/lib/marshal/workflow/reader.ts index f8c433a..7ee553b 100644 --- a/src/lib/marshal/workflow/reader.ts +++ b/src/lib/marshal/workflow/reader.ts @@ -14,6 +14,7 @@ import { omitDeep, } from "@/lib/helpers/object"; import { + checkIfValidExtractedFilePathFormat, FILEPATH_MARKED_RE, readExtractedFileSync, } from "@/lib/marshal/shared/helpers"; @@ -35,38 +36,6 @@ export type WorkflowDirData = WorkflowDirContext & { // (e.g. workflow.json, then visual_blocks.json) const MAX_EXTRACTION_LEVEL = 2; -/* - * Validate the file path format of an extracted field. The file path must be: - * - * 1) Expressed as a relative path. - * - * For exmaple: - * subject@: "email_1/subject.html" // GOOD - * subject@: "./email_1/subject.html" // GOOD - * subject@: "/workflow-x/email_1/subject.html" // BAD - * - * 2) The resolved path must be contained inside the workflow directory - * - * For exmaple (workflow-y is a different workflow dir in this example): - * subject@: "./email_1/subject.html" // GOOD - * subject@: "../workflow-y/email_1/subject.html" // BAD - * - * Note: does not validate the presence of the file nor the uniqueness of the - * file path. - */ -const checkIfValidExtractedFilePathFormat = ( - relpath: unknown, - sourceFileAbspath: string, -): boolean => { - if (typeof relpath !== "string") return false; - if (path.isAbsolute(relpath)) return false; - - const extractedFileAbspath = path.resolve(sourceFileAbspath, relpath); - const pathDiff = path.relative(sourceFileAbspath, extractedFileAbspath); - - return !pathDiff.startsWith(".."); -}; - /* * Validate the extracted file path based on its format and uniqueness (but not * the presence). @@ -99,7 +68,6 @@ const validateExtractedFilePath = ( // Keep track of all the valid extracted file paths that have been seen, so // we can validate each file path's uniqueness as we traverse. uniqueFilePaths[val] = true; - return undefined; }; @@ -352,6 +320,3 @@ export const readAllForCommandTarget = async ( throw new Error(`Invalid workflow command target: ${target}`); } }; - -// Exported for tests. -export { checkIfValidExtractedFilePathFormat }; diff --git a/test/lib/marshal/layout/reader.test.ts b/test/lib/marshal/layout/reader.test.ts index c308145..825075a 100644 --- a/test/lib/marshal/layout/reader.test.ts +++ b/test/lib/marshal/layout/reader.test.ts @@ -8,12 +8,68 @@ import { sandboxDir } from "@/lib/helpers/const"; import { JsonDataError } from "@/lib/helpers/error"; import { LAYOUT_JSON } from "@/lib/marshal/email-layout"; import { readEmailLayoutDir } from "@/lib/marshal/email-layout/reader"; -import { readExtractedFileSync } from "@/lib/marshal/shared/helpers"; +import { + checkIfValidExtractedFilePathFormat, + readExtractedFileSync, +} from "@/lib/marshal/shared/helpers"; import { EmailLayoutDirContext } from "@/lib/run-context"; const currCwd = process.cwd(); describe("lib/marshal/layout/reader", () => { + describe("checkIfValidExtractedFilePathFormat", () => { + const emailLayoutDirCtx = { + type: "email_layout", + key: "transactional", + abspath: "/layouts/transactional", + exists: true, + } as EmailLayoutDirContext; + + describe("given an absolute path", () => { + it("returns true as it's valid", () => { + const abspath = "html_layout.html"; + + const result = checkIfValidExtractedFilePathFormat( + abspath, + emailLayoutDirCtx.abspath, + { withAbsolutePaths: true }, + ); + + expect(result).to.equal(true); + }); + }); + + describe("given an relative path that resolves outside the layouts dir", () => { + it("returns false as it is invalid", () => { + const relpath = "../foo"; + + const result = checkIfValidExtractedFilePathFormat( + relpath, + emailLayoutDirCtx.abspath, + ); + + expect(result).to.equal(false); + }); + }); + + describe("given an relative path that resolves inside the layout dir", () => { + it("returns true as it is valid", () => { + const relpath1 = "text-layouts/text_layout.txt"; + const result1 = checkIfValidExtractedFilePathFormat( + relpath1, + emailLayoutDirCtx.abspath, + ); + expect(result1).to.equal(true); + + const relpath2 = "./text-layouts/text_layout.txt"; + const result2 = checkIfValidExtractedFilePathFormat( + relpath2, + emailLayoutDirCtx.abspath, + ); + expect(result2).to.equal(true); + }); + }); + }); describe("readExtractedFileSync", () => { const emailLayoutDirCtx: EmailLayoutDirContext = { type: "email_layout", diff --git a/test/lib/marshal/workflow/reader.test.ts b/test/lib/marshal/workflow/reader.test.ts index 09a1666..bfc976c 100644 --- a/test/lib/marshal/workflow/reader.test.ts +++ b/test/lib/marshal/workflow/reader.test.ts @@ -7,9 +7,11 @@ import { get } from "lodash"; import { xpath } from "@/../test/support"; import { sandboxDir } from "@/lib/helpers/const"; import { JsonDataError } from "@/lib/helpers/error"; -import { readExtractedFileSync } from "@/lib/marshal/shared/helpers"; import { checkIfValidExtractedFilePathFormat, + readExtractedFileSync, +} from "@/lib/marshal/shared/helpers"; +import { readWorkflowDir, VISUAL_BLOCKS_JSON, WORKFLOW_JSON, From e52610cbd6915731c6dcc03a7117ca30c363b5ed Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Tue, 3 Oct 2023 18:24:35 -0300 Subject: [PATCH 33/45] chore(KNO-4367): remove extra code --- src/lib/marshal/email-layout/reader.ts | 4 +--- src/lib/marshal/shared/helpers.ts | 13 ------------- test/lib/marshal/layout/reader.test.ts | 1 - 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/src/lib/marshal/email-layout/reader.ts b/src/lib/marshal/email-layout/reader.ts index a42b3e7..c751ec6 100644 --- a/src/lib/marshal/email-layout/reader.ts +++ b/src/lib/marshal/email-layout/reader.ts @@ -120,9 +120,7 @@ const validateExtractedFilePath = ( const layoutJsonPath = path.resolve(emailLayoutDirCtx.abspath, LAYOUT_JSON); // Validate the file path format, and that it is unique per workflow. if ( - !checkIfValidExtractedFilePathFormat(val, layoutJsonPath, { - withAbsolutePaths: true, - }) || + !checkIfValidExtractedFilePathFormat(val, layoutJsonPath) || typeof val !== "string" ) { const error = new JsonDataError( diff --git a/src/lib/marshal/shared/helpers.ts b/src/lib/marshal/shared/helpers.ts index bedd28b..2a9b84a 100644 --- a/src/lib/marshal/shared/helpers.ts +++ b/src/lib/marshal/shared/helpers.ts @@ -97,26 +97,13 @@ export const readExtractedFileSync = ( * * Note: does not validate the presence of the file nor the uniqueness of the * file path. - * - * Options: - * - withAbsolutePaths: it will return non absoulte paths as valid. - * For example: - * html_layout@: "html_layout.html" - * */ -type CheckValidExtractedFilePathOptions = { - withAbsolutePaths?: boolean; -}; export const checkIfValidExtractedFilePathFormat = ( relpath: unknown, sourceFileAbspath: string, - opts: CheckValidExtractedFilePathOptions = {}, ): boolean => { - const { withAbsolutePaths = false } = opts; if (typeof relpath !== "string") return false; - // If the option for allowing absolute paths was passed, and the path is indeed absolute, then it's valid. - if (withAbsolutePaths && !path.isAbsolute(relpath)) return true; if (path.isAbsolute(relpath)) return false; diff --git a/test/lib/marshal/layout/reader.test.ts b/test/lib/marshal/layout/reader.test.ts index 825075a..6070418 100644 --- a/test/lib/marshal/layout/reader.test.ts +++ b/test/lib/marshal/layout/reader.test.ts @@ -32,7 +32,6 @@ describe("lib/marshal/layout/reader", () => { const result = checkIfValidExtractedFilePathFormat( abspath, emailLayoutDirCtx.abspath, - { withAbsolutePaths: true }, ); expect(result).to.equal(true); From 9d0f36f8ba0e7b089b20b4e2956476775d8b9584 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 4 Oct 2023 11:31:30 -0300 Subject: [PATCH 34/45] chore(KNO-4367): remove comment --- src/lib/marshal/email-layout/reader.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/lib/marshal/email-layout/reader.ts b/src/lib/marshal/email-layout/reader.ts index c751ec6..7b497dd 100644 --- a/src/lib/marshal/email-layout/reader.ts +++ b/src/lib/marshal/email-layout/reader.ts @@ -109,9 +109,6 @@ const joinExtractedFiles = async ( /* * Validate the extracted file path based on its format and uniqueness (but not * the presence). - * - * Note, the uniqueness check is based on reading from and writing to - * uniqueFilePaths, which is MUTATED in place. */ const validateExtractedFilePath = ( val: unknown, From 27d5be6a3afc777ed6c46c6d0b0871bd5d3ef987 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 4 Oct 2023 14:02:30 -0300 Subject: [PATCH 35/45] chore(): comment update --- src/lib/marshal/email-layout/reader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/marshal/email-layout/reader.ts b/src/lib/marshal/email-layout/reader.ts index 7b497dd..d6a3485 100644 --- a/src/lib/marshal/email-layout/reader.ts +++ b/src/lib/marshal/email-layout/reader.ts @@ -115,7 +115,7 @@ const validateExtractedFilePath = ( emailLayoutDirCtx: EmailLayoutDirContext, ): JsonDataError | undefined => { const layoutJsonPath = path.resolve(emailLayoutDirCtx.abspath, LAYOUT_JSON); - // Validate the file path format, and that it is unique per workflow. + // Validate the file path format, and that it is unique per layout. if ( !checkIfValidExtractedFilePathFormat(val, layoutJsonPath) || typeof val !== "string" From e47e6801143a63627ffea9df0bccd7d0a3e9f96d Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 4 Oct 2023 14:48:37 -0300 Subject: [PATCH 36/45] chore(KNO-4367): update --- src/lib/marshal/email-layout/writer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index f5b332d..089b4db 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -72,7 +72,7 @@ export const writeEmailLayoutDirFromData = async ( // If the layout directory exists on the file system (i.e. previously // pulled before), then read the layout file to use as a reference. const [localEmailLayout] = emailLayoutDirCtx.exists - ? await readEmailLayoutDir(emailLayoutDirCtx, { withExtractedFiles: true }) + ? await readEmailLayoutDir(emailLayoutDirCtx) : []; const bundle = buildEmailLayoutDirBundle(remoteEmailLayout, localEmailLayout); @@ -121,7 +121,6 @@ const buildEmailLayoutDirBundle = ( ): EmailLayoutDirBundle => { const bundle: EmailLayoutDirBundle = {}; const mutRemoteEmailLayout = cloneDeep(remoteEmailLayout); - // A map of extraction settings of every field in the email layout const compiledExtractionSettings = compileExtractionSettings(mutRemoteEmailLayout); @@ -141,6 +140,7 @@ const buildEmailLayoutDirBundle = ( localEmailLayout, `${objPathStr}${FILEPATH_MARKER}`, ); + const { default: extractByDefault, file_ext: fileExt } = extractionSettings; if (!extractedFilePath && !extractByDefault) continue; From 29019c885b9eccc00c50f21860c2692145bfb270 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 4 Oct 2023 15:27:35 -0300 Subject: [PATCH 37/45] chore(KNO-4367): rename isLayoutDir --- src/lib/run-context/loader.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/run-context/loader.ts b/src/lib/run-context/loader.ts index e929c79..c0eccf3 100644 --- a/src/lib/run-context/loader.ts +++ b/src/lib/run-context/loader.ts @@ -27,8 +27,8 @@ const evaluateRecursively = async ( } // Check if we are inside a layout directory, and if so update the context. - const isLayoutDir = await EmailLayout.isEmailLayoutDir(currDir); - if (!ctx.resourceDir && isLayoutDir) { + const isEmailLayoutDir = await EmailLayout.isEmailLayoutDir(currDir); + if (!ctx.resourceDir && isEmailLayoutDir) { ctx.resourceDir = { type: "email_layout", key: path.basename(currDir), From 67a50381e881ecc1eb1e7a46f17bfc4060099efb Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 4 Oct 2023 15:29:35 -0300 Subject: [PATCH 38/45] chore(KNO-4367): remove newlines --- src/lib/marshal/email-layout/reader.ts | 1 - src/lib/marshal/shared/helpers.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/src/lib/marshal/email-layout/reader.ts b/src/lib/marshal/email-layout/reader.ts index d6a3485..10563db 100644 --- a/src/lib/marshal/email-layout/reader.ts +++ b/src/lib/marshal/email-layout/reader.ts @@ -37,7 +37,6 @@ export const readEmailLayoutDir = async ( if (!dirExists) throw new Error(`${abspath} does not exist`); const layoutJsonPath = await lsEmailLayoutJson(abspath); - if (!layoutJsonPath) throw new Error(`${abspath} is not a layout directory`); const result = await readJson(layoutJsonPath); diff --git a/src/lib/marshal/shared/helpers.ts b/src/lib/marshal/shared/helpers.ts index 2a9b84a..55c16d5 100644 --- a/src/lib/marshal/shared/helpers.ts +++ b/src/lib/marshal/shared/helpers.ts @@ -98,7 +98,6 @@ export const readExtractedFileSync = ( * Note: does not validate the presence of the file nor the uniqueness of the * file path. */ - export const checkIfValidExtractedFilePathFormat = ( relpath: unknown, sourceFileAbspath: string, From 447e001aaa1f30dd1bcf22afda816740f4025807 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 4 Oct 2023 15:49:57 -0300 Subject: [PATCH 39/45] feat(KNO-4367): improve joinExtractedFiles --- src/lib/marshal/email-layout/reader.ts | 24 +++++++++++++++--------- src/lib/marshal/email-layout/writer.ts | 2 +- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/lib/marshal/email-layout/reader.ts b/src/lib/marshal/email-layout/reader.ts index 10563db..16b77e2 100644 --- a/src/lib/marshal/email-layout/reader.ts +++ b/src/lib/marshal/email-layout/reader.ts @@ -1,11 +1,11 @@ import path from "node:path"; import * as fs from "fs-extra"; -import { mapValues, set, unset } from "lodash"; +import { set } from "lodash"; import { JsonDataError } from "@/lib/helpers/error"; import { ParseJsonResult, readJson } from "@/lib/helpers/json"; -import { AnyObj, omitDeep } from "@/lib/helpers/object"; +import { AnyObj, mapValuesDeep, ObjPath, omitDeep } from "@/lib/helpers/object"; import { checkIfValidExtractedFilePathFormat, FILEPATH_MARKED_RE, @@ -63,11 +63,12 @@ const joinExtractedFiles = async ( ): Promise => { const errors: JsonDataError[] = []; - mapValues(layoutJson, (relpath: string, key: string) => { + mapValuesDeep(layoutJson, (relpath: string, key: string, parts) => { // If not marked with the @suffix, there's nothing to do. if (!FILEPATH_MARKED_RE.test(key)) return; - const objPathStr = key.replace(FILEPATH_MARKED_RE, ""); + const objPathToFieldStr = ObjPath.stringify(parts); + const inlinObjPathStr = objPathToFieldStr.replace(FILEPATH_MARKED_RE, ""); // Check if the extracted path found at the current field path is valid const invalidFilePathError = validateExtractedFilePath( @@ -79,12 +80,12 @@ const joinExtractedFiles = async ( // Wipe the invalid file path in the node so the final layout json // object ends up with only valid file paths, this way layout writer // can see only valid file paths and use those when pulling. - set(layoutJson, key, undefined); - + set(layoutJson, inlinObjPathStr, undefined); + set(layoutJson, objPathToFieldStr, undefined); return; } - // By this point we have a valid extracted file path, so attempt to read the file + // By this point we have a valid extracted file path, so attempt to read the file. const [content, readExtractedFileError] = readExtractedFileSync( relpath, layoutDirCtx, @@ -94,12 +95,17 @@ const joinExtractedFiles = async ( if (readExtractedFileError) { errors.push(readExtractedFileError); + // If there's an error, replace the extracted file path with the original one, and set the + // inlined field path in layout object with empty content, so we know + // we do not need to try inlining again. + set(layoutJson, objPathToFieldStr, relpath); + set(layoutJson, inlinObjPathStr, undefined); return; } // Inline the file content and remove the extracted file path - set(layoutJson, objPathStr, content); - unset(layoutJson, key); + set(layoutJson, objPathToFieldStr, relpath); + set(layoutJson, inlinObjPathStr, content); }); return [layoutJson, errors]; diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 089b4db..be54949 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -72,7 +72,7 @@ export const writeEmailLayoutDirFromData = async ( // If the layout directory exists on the file system (i.e. previously // pulled before), then read the layout file to use as a reference. const [localEmailLayout] = emailLayoutDirCtx.exists - ? await readEmailLayoutDir(emailLayoutDirCtx) + ? await readEmailLayoutDir(emailLayoutDirCtx, { withExtractedFiles: true }) : []; const bundle = buildEmailLayoutDirBundle(remoteEmailLayout, localEmailLayout); From d9c1595668e145cae470aa14385e1947bebe47c7 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 4 Oct 2023 15:51:35 -0300 Subject: [PATCH 40/45] chore(KNO-4367): update comment --- src/lib/marshal/email-layout/writer.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index be54949..eb7515e 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -24,6 +24,8 @@ type CompiledExtractionSettings = Map; /* Traverse a given email layout data and compile extraction settings of every extractable * field into a sorted map. + * + * NOTE: Currently we do NOT support content extraction at nested levels for email layouts. */ const compileExtractionSettings = ( emailLayout: EmailLayoutData, From 77caaa039a8bf7b0c6efb30e044d77d4d7674734 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 4 Oct 2023 15:52:26 -0300 Subject: [PATCH 41/45] chore(KNO-4367): update objPathParts --- src/lib/marshal/email-layout/writer.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index eb7515e..8f709ee 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -130,13 +130,13 @@ const buildEmailLayoutDirBundle = ( // Iterate through each extractable field, determine whether we need to // extract the field content, and if so, perform the // extraction. - for (const [objPath, extractionSettings] of compiledExtractionSettings) { + for (const [objPathParts, extractionSettings] of compiledExtractionSettings) { // If this layout doesn't have this field path, then we don't extract. - if (!has(mutRemoteEmailLayout, objPath)) continue; + if (!has(mutRemoteEmailLayout, objPathParts)) continue; // If the field at this path is extracted in the local layout, then // always extract; otherwise extract based on the field settings default. - const objPathStr = ObjPath.stringify(objPath); + const objPathStr = ObjPath.stringify(objPathParts); const extractedFilePath = get( localEmailLayout, @@ -148,8 +148,8 @@ const buildEmailLayoutDirBundle = ( if (!extractedFilePath && !extractByDefault) continue; // By this point, we have a field where we need to extract its content. - const data = get(mutRemoteEmailLayout, objPath); - const fileName = objPath.pop(); + const data = get(mutRemoteEmailLayout, objPathParts); + const fileName = objPathParts.pop(); // If we have an extracted file path from the local layout, we use that. In the other // case we use the default path. From ce503278ddf47adc9ce3c6f72825009939604317 Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 4 Oct 2023 16:19:07 -0300 Subject: [PATCH 42/45] feat(KNO-4367): abstract validateExtractedFilePath --- src/lib/marshal/email-layout/reader.ts | 39 +++++++------------------ src/lib/marshal/email-layout/writer.ts | 4 +-- src/lib/marshal/shared/helpers.ts | 37 ++++++++++++++++++++++++ src/lib/marshal/workflow/reader.ts | 40 ++------------------------ 4 files changed, 52 insertions(+), 68 deletions(-) diff --git a/src/lib/marshal/email-layout/reader.ts b/src/lib/marshal/email-layout/reader.ts index 16b77e2..54338cd 100644 --- a/src/lib/marshal/email-layout/reader.ts +++ b/src/lib/marshal/email-layout/reader.ts @@ -1,5 +1,3 @@ -import path from "node:path"; - import * as fs from "fs-extra"; import { set } from "lodash"; @@ -7,9 +5,9 @@ import { JsonDataError } from "@/lib/helpers/error"; import { ParseJsonResult, readJson } from "@/lib/helpers/json"; import { AnyObj, mapValuesDeep, ObjPath, omitDeep } from "@/lib/helpers/object"; import { - checkIfValidExtractedFilePathFormat, FILEPATH_MARKED_RE, readExtractedFileSync, + validateExtractedFilePath, } from "@/lib/marshal/shared/helpers"; import { EmailLayoutDirContext } from "@/lib/run-context"; @@ -63,6 +61,11 @@ const joinExtractedFiles = async ( ): Promise => { const errors: JsonDataError[] = []; + // Tracks each new valid extracted file path seen (rebased to be relative to + // layout.json) in the layout json node. Mutated in place, and used + // to validate the uniqueness of an extracted path encountered. + const uniqueFilePaths = {}; + mapValuesDeep(layoutJson, (relpath: string, key: string, parts) => { // If not marked with the @suffix, there's nothing to do. if (!FILEPATH_MARKED_RE.test(key)) return; @@ -73,7 +76,10 @@ const joinExtractedFiles = async ( // Check if the extracted path found at the current field path is valid const invalidFilePathError = validateExtractedFilePath( relpath, - layoutDirCtx, + layoutDirCtx.abspath, + LAYOUT_JSON, + uniqueFilePaths, + objPathToFieldStr, ); if (invalidFilePathError) { errors.push(invalidFilePathError); @@ -110,28 +116,3 @@ const joinExtractedFiles = async ( return [layoutJson, errors]; }; - -/* - * Validate the extracted file path based on its format and uniqueness (but not - * the presence). - */ -const validateExtractedFilePath = ( - val: unknown, - emailLayoutDirCtx: EmailLayoutDirContext, -): JsonDataError | undefined => { - const layoutJsonPath = path.resolve(emailLayoutDirCtx.abspath, LAYOUT_JSON); - // Validate the file path format, and that it is unique per layout. - if ( - !checkIfValidExtractedFilePathFormat(val, layoutJsonPath) || - typeof val !== "string" - ) { - const error = new JsonDataError( - "must be a relative path string to a unique file within the directory", - String(val), - ); - - return error; - } - - return undefined; -}; diff --git a/src/lib/marshal/email-layout/writer.ts b/src/lib/marshal/email-layout/writer.ts index 8f709ee..c77e042 100644 --- a/src/lib/marshal/email-layout/writer.ts +++ b/src/lib/marshal/email-layout/writer.ts @@ -24,8 +24,8 @@ type CompiledExtractionSettings = Map; /* Traverse a given email layout data and compile extraction settings of every extractable * field into a sorted map. - * - * NOTE: Currently we do NOT support content extraction at nested levels for email layouts. + * + * NOTE: Currently we do NOT support content extraction at nested levels for email layouts. */ const compileExtractionSettings = ( emailLayout: EmailLayoutData, diff --git a/src/lib/marshal/shared/helpers.ts b/src/lib/marshal/shared/helpers.ts index 55c16d5..909d3c5 100644 --- a/src/lib/marshal/shared/helpers.ts +++ b/src/lib/marshal/shared/helpers.ts @@ -79,6 +79,43 @@ export const readExtractedFileSync = ( return [content!, undefined]; }; +/* + * Validate the extracted file path based on its format and uniqueness (but not + * the presence). + * + * Note, the uniqueness check is based on reading from and writing to + * uniqueFilePaths, which is MUTATED in place. + */ + +/* eslint-disable max-params */ +export const validateExtractedFilePath = ( + val: unknown, + sourceFileAbspath: string, + sourceJson: string, + uniqueFilePaths: Record, + objPathToFieldStr: string, +): JsonDataError | undefined => { + const jsonPath = path.resolve(sourceFileAbspath, sourceJson); + // Validate the file path format, and that it is unique per entity. + if ( + !checkIfValidExtractedFilePathFormat(val, jsonPath) || + typeof val !== "string" || + val in uniqueFilePaths + ) { + const error = new JsonDataError( + "must be a relative path string to a unique file within the directory", + objPathToFieldStr, + ); + + return error; + } + + // Keep track of all the valid extracted file paths that have been seen, so + // we can validate each file path's uniqueness as we traverse. + uniqueFilePaths[val] = true; + return undefined; +}; + /* * Validate the file path format of an extracted field. The file path must be: * diff --git a/src/lib/marshal/workflow/reader.ts b/src/lib/marshal/workflow/reader.ts index 7ee553b..d0c0786 100644 --- a/src/lib/marshal/workflow/reader.ts +++ b/src/lib/marshal/workflow/reader.ts @@ -14,9 +14,9 @@ import { omitDeep, } from "@/lib/helpers/object"; import { - checkIfValidExtractedFilePathFormat, FILEPATH_MARKED_RE, readExtractedFileSync, + validateExtractedFilePath, } from "@/lib/marshal/shared/helpers"; import { WorkflowDirContext } from "@/lib/run-context"; @@ -36,41 +36,6 @@ export type WorkflowDirData = WorkflowDirContext & { // (e.g. workflow.json, then visual_blocks.json) const MAX_EXTRACTION_LEVEL = 2; -/* - * Validate the extracted file path based on its format and uniqueness (but not - * the presence). - * - * Note, the uniqueness check is based on reading from and writing to - * uniqueFilePaths, which is MUTATED in place. - */ -const validateExtractedFilePath = ( - val: unknown, - workflowDirCtx: WorkflowDirContext, - uniqueFilePaths: Record, - objPathToFieldStr: string, -): JsonDataError | undefined => { - const workflowJsonPath = path.resolve(workflowDirCtx.abspath, WORKFLOW_JSON); - - // Validate the file path format, and that it is unique per workflow. - if ( - !checkIfValidExtractedFilePathFormat(val, workflowJsonPath) || - typeof val !== "string" || - val in uniqueFilePaths - ) { - const error = new JsonDataError( - "must be a relative path string to a unique file within the directory", - objPathToFieldStr, - ); - - return error; - } - - // Keep track of all the valid extracted file paths that have been seen, so - // we can validate each file path's uniqueness as we traverse. - uniqueFilePaths[val] = true; - return undefined; -}; - /* * Given a workflow json object, compiles all referenced extracted files from it * and returns the updated object with the extracted content joined and inlined. @@ -148,7 +113,8 @@ const joinExtractedFiles = async ( const invalidFilePathError = validateExtractedFilePath( rebasedFilePath, - workflowDirCtx, + workflowDirCtx.abspath, + WORKFLOW_JSON, uniqueFilePaths, objPathToFieldStr, ); From 00862760b70c5bf028f2f2a4e3131dc8847720cb Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 4 Oct 2023 16:26:26 -0300 Subject: [PATCH 43/45] chore(KNO-4367): add dot --- src/lib/marshal/email-layout/reader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/marshal/email-layout/reader.ts b/src/lib/marshal/email-layout/reader.ts index 54338cd..8787956 100644 --- a/src/lib/marshal/email-layout/reader.ts +++ b/src/lib/marshal/email-layout/reader.ts @@ -109,7 +109,7 @@ const joinExtractedFiles = async ( return; } - // Inline the file content and remove the extracted file path + // Inline the file content and remove the extracted file path. set(layoutJson, objPathToFieldStr, relpath); set(layoutJson, inlinObjPathStr, content); }); From 7fe20bdb0e9a18bae89fd70e25bb6f74060002ca Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 4 Oct 2023 16:34:45 -0300 Subject: [PATCH 44/45] chore(KNO-4367): update --- src/lib/marshal/email-layout/reader.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/marshal/email-layout/reader.ts b/src/lib/marshal/email-layout/reader.ts index 8787956..61de0ac 100644 --- a/src/lib/marshal/email-layout/reader.ts +++ b/src/lib/marshal/email-layout/reader.ts @@ -1,5 +1,5 @@ import * as fs from "fs-extra"; -import { set } from "lodash"; +import { hasIn, set } from "lodash"; import { JsonDataError } from "@/lib/helpers/error"; import { ParseJsonResult, readJson } from "@/lib/helpers/json"; @@ -59,6 +59,7 @@ const joinExtractedFiles = async ( layoutDirCtx: EmailLayoutDirContext, layoutJson: AnyObj, ): Promise => { + // Tracks any errors encountered during traversal. Mutated in place. const errors: JsonDataError[] = []; // Tracks each new valid extracted file path seen (rebased to be relative to @@ -73,6 +74,9 @@ const joinExtractedFiles = async ( const objPathToFieldStr = ObjPath.stringify(parts); const inlinObjPathStr = objPathToFieldStr.replace(FILEPATH_MARKED_RE, ""); + // If there is inlined content present already, then nothing more to do. + if (hasIn(layoutJson, inlinObjPathStr)) return; + // Check if the extracted path found at the current field path is valid const invalidFilePathError = validateExtractedFilePath( relpath, From 48743fa2778f8b378b568a38d3c6d7173d1e5b6b Mon Sep 17 00:00:00 2001 From: Franco Borrelli Date: Wed, 4 Oct 2023 17:23:14 -0300 Subject: [PATCH 45/45] feat(KNO-4367): improve validateExtractedFilePath --- src/lib/marshal/email-layout/reader.ts | 5 +++-- src/lib/marshal/shared/helpers.ts | 6 +----- src/lib/marshal/workflow/reader.ts | 3 +-- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/lib/marshal/email-layout/reader.ts b/src/lib/marshal/email-layout/reader.ts index 61de0ac..7b6c560 100644 --- a/src/lib/marshal/email-layout/reader.ts +++ b/src/lib/marshal/email-layout/reader.ts @@ -1,3 +1,5 @@ +import path from "node:path"; + import * as fs from "fs-extra"; import { hasIn, set } from "lodash"; @@ -80,8 +82,7 @@ const joinExtractedFiles = async ( // Check if the extracted path found at the current field path is valid const invalidFilePathError = validateExtractedFilePath( relpath, - layoutDirCtx.abspath, - LAYOUT_JSON, + path.resolve(layoutDirCtx.abspath, LAYOUT_JSON), uniqueFilePaths, objPathToFieldStr, ); diff --git a/src/lib/marshal/shared/helpers.ts b/src/lib/marshal/shared/helpers.ts index 909d3c5..d0023ce 100644 --- a/src/lib/marshal/shared/helpers.ts +++ b/src/lib/marshal/shared/helpers.ts @@ -86,19 +86,15 @@ export const readExtractedFileSync = ( * Note, the uniqueness check is based on reading from and writing to * uniqueFilePaths, which is MUTATED in place. */ - -/* eslint-disable max-params */ export const validateExtractedFilePath = ( val: unknown, sourceFileAbspath: string, - sourceJson: string, uniqueFilePaths: Record, objPathToFieldStr: string, ): JsonDataError | undefined => { - const jsonPath = path.resolve(sourceFileAbspath, sourceJson); // Validate the file path format, and that it is unique per entity. if ( - !checkIfValidExtractedFilePathFormat(val, jsonPath) || + !checkIfValidExtractedFilePathFormat(val, sourceFileAbspath) || typeof val !== "string" || val in uniqueFilePaths ) { diff --git a/src/lib/marshal/workflow/reader.ts b/src/lib/marshal/workflow/reader.ts index d0c0786..0c50767 100644 --- a/src/lib/marshal/workflow/reader.ts +++ b/src/lib/marshal/workflow/reader.ts @@ -113,8 +113,7 @@ const joinExtractedFiles = async ( const invalidFilePathError = validateExtractedFilePath( rebasedFilePath, - workflowDirCtx.abspath, - WORKFLOW_JSON, + path.resolve(workflowDirCtx.abspath, WORKFLOW_JSON), uniqueFilePaths, objPathToFieldStr, );