From cc02290551539b1584fb88b90d887e56ba6dd1d9 Mon Sep 17 00:00:00 2001 From: Finlay Birnie Date: Thu, 21 Nov 2024 13:41:23 +0000 Subject: [PATCH 01/10] remove dotenv expand; it is not needed to mimic serverless --- api/jest-setup.ts | 6 ++---- api/package-lock.json | 16 ---------------- api/package.json | 1 - api/scripts/fullAriImport.ts | 3 +-- 4 files changed, 3 insertions(+), 23 deletions(-) diff --git a/api/jest-setup.ts b/api/jest-setup.ts index 6f93c2a1a..815e13dd5 100644 --- a/api/jest-setup.ts +++ b/api/jest-setup.ts @@ -1,7 +1,5 @@ -// Ensure that process.env values are defined when jest runs code, to match the behaviour -// configured by serverless's "useDotenv" option (which also uses dotenv-expand). +// Ensure that process.env values are defined when jest runs code, to match serverless's behaviour. import * as dotenv from 'dotenv'; -import { expand } from 'dotenv-expand'; -expand(dotenv.config()); +dotenv.config(); diff --git a/api/package-lock.json b/api/package-lock.json index 416f6ae70..87d7ca3e0 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -22,7 +22,6 @@ "axios": "^1.7.4", "cheerio": "^1.0.0-rc.12", "dotenv": "16.4.5", - "dotenv-expand": "11.0.7", "html-to-text": "^9.0.5", "isomorphic-dompurify": "^2.12.0", "jsonwebtoken": "^9.0.2", @@ -14841,21 +14840,6 @@ "url": "https://dotenvx.com" } }, - "node_modules/dotenv-expand": { - "version": "11.0.7", - "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-11.0.7.tgz", - "integrity": "sha512-zIHwmZPRshsCdpMDyVsqGmgyP0yT8GAgXUnkdAoJisxvf33k7yO6OuoKmcTGuXPWSsm8Oh88nZicRLA9Y0rUeA==", - "license": "BSD-2-Clause", - "dependencies": { - "dotenv": "^16.4.5" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", diff --git a/api/package.json b/api/package.json index a287646ac..07f2698c7 100644 --- a/api/package.json +++ b/api/package.json @@ -54,7 +54,6 @@ "aws-lambda": "^1.0.7", "cheerio": "^1.0.0-rc.12", "dotenv": "16.4.5", - "dotenv-expand": "11.0.7", "html-to-text": "^9.0.5", "isomorphic-dompurify": "^2.12.0", "jsonwebtoken": "^9.0.2", diff --git a/api/scripts/fullAriImport.ts b/api/scripts/fullAriImport.ts index e458b2dc9..082b6b693 100644 --- a/api/scripts/fullAriImport.ts +++ b/api/scripts/fullAriImport.ts @@ -1,10 +1,9 @@ import axios from 'axios'; import * as fs from 'fs/promises'; import * as dotenv from 'dotenv'; -import { expand } from 'dotenv-expand'; // Important to do this so that environment variables are treated the same as in deployed code. -expand(dotenv.config()); +dotenv.config(); import * as ariUtils from 'integration/ariUtils'; import * as Helpers from 'lib/helpers'; From 90dd3d207b697dbc65caee89e0481a1fd0d8399d Mon Sep 17 00:00:00 2001 From: Finlay Birnie Date: Thu, 21 Nov 2024 13:54:56 +0000 Subject: [PATCH 02/10] add full arg to ari script --- api/scripts/fullAriImport.ts | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/api/scripts/fullAriImport.ts b/api/scripts/fullAriImport.ts index 082b6b693..858e75e67 100644 --- a/api/scripts/fullAriImport.ts +++ b/api/scripts/fullAriImport.ts @@ -9,26 +9,38 @@ import * as ariUtils from 'integration/ariUtils'; import * as Helpers from 'lib/helpers'; import * as I from 'interface'; -// Can take an argument to run for all departments, rather than just the ones specified in the -// PARTICIPATING_ARI_USER_IDS environment variable. -// npm run fullAriImport -- allDepartments=true -const parseArguments = (): { importAllDepartments: boolean } => { +/** + * Can take the following arguments: + * - allDepartments: If "true", the script will run for all departments, + * rather than just the ones specified in the PARTICIPATING_ARI_USER_IDS environment variable. + * - full: If "true", the script will import all ARIs from the ARI DB, instead of stopping when it + * thinks it has found all the new ones (the incremental way). + * + * e.g.: + * npm run ariImport -- allDepartments=true full=true + */ +const parseArguments = (): { importAllDepartments: boolean; full: boolean } => { const args = Helpers.parseNpmScriptArgs(); for (const arg of Object.keys(args)) { - if (!['allDepartments'].includes(arg)) { + if (!['allDepartments', 'full'].includes(arg)) { throw new Error(`Unexpected argument: ${arg}`); } } - const allDepartmentsArg = args.allDepartments; + const { allDepartments: allDepartmentsArg, full: fullArg } = args; if (allDepartmentsArg && !(allDepartmentsArg === 'true' || allDepartmentsArg === 'false')) { - throw new Error('allDepartments must be "true" or "false"'); + throw new Error('"allDepartments" must be "true" or "false"'); + } + + if (fullArg && !(fullArg === 'true' || fullArg === 'false')) { + throw new Error('"full" must be "true" or "false"'); } return { - importAllDepartments: !!allDepartmentsArg + importAllDepartments: !!allDepartmentsArg, + full: !!fullArg }; }; From b079a574ee55b9001a01fadf9fb6fcce43a16d33 Mon Sep 17 00:00:00 2001 From: Finlay Birnie Date: Mon, 25 Nov 2024 10:39:45 +0000 Subject: [PATCH 03/10] rename script --- api/package.json | 4 ++-- api/scripts/{fullAriImport.ts => ariImport.ts} | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) rename api/scripts/{fullAriImport.ts => ariImport.ts} (97%) diff --git a/api/package.json b/api/package.json index 07f2698c7..ec217d9e6 100644 --- a/api/package.json +++ b/api/package.json @@ -32,12 +32,12 @@ "test:local": "STAGE=local jest --runInBand --testTimeout=60000", "test:watch": "npm run test:local -- --watch", "type": "tsc --noEmit", - "reindex": "ts-node scripts/reindex.ts", + "ariImport": "ts-node -r tsconfig-paths/register scripts/ariImport.ts", "createOrganisationalAccounts": "ts-node -r tsconfig-paths/register scripts/createOrganisationalAccounts.ts", "createTopicMappings": "ts-node -r tsconfig-paths/register scripts/createTopicMappings.ts", "createTopics": "ts-node -r tsconfig-paths/register scripts/createTopics.ts", "createUserMappings": "ts-node -r tsconfig-paths/register scripts/createUserMappings.ts", - "fullAriImport": "ts-node -r tsconfig-paths/register scripts/fullAriImport.ts", + "reindex": "ts-node scripts/reindex.ts", "updateOrganisationalAccounts": "ts-node -r tsconfig-paths/register scripts/updateOrganisationalAccounts.ts" }, "dependencies": { diff --git a/api/scripts/fullAriImport.ts b/api/scripts/ariImport.ts similarity index 97% rename from api/scripts/fullAriImport.ts rename to api/scripts/ariImport.ts index 858e75e67..7e4d3d69b 100644 --- a/api/scripts/fullAriImport.ts +++ b/api/scripts/ariImport.ts @@ -44,7 +44,8 @@ const parseArguments = (): { importAllDepartments: boolean; full: boolean } => { }; }; -const fullAriImport = async (allDepartments?: boolean): Promise => { +const ariImport = async (allDepartments?: boolean, full?: boolean): Promise => { + console.log(full); const startTime = performance.now(); // Collect all ARIs in a variable. @@ -175,8 +176,8 @@ ${unrecognisedTopicsArray.length ? '\nUnrecognised topics: "' + unrecognisedTopi } ARIs in ${duration} seconds.`; }; -const { importAllDepartments } = parseArguments(); +const { importAllDepartments, full } = parseArguments(); -fullAriImport(importAllDepartments) +ariImport(importAllDepartments, full) .then((message) => console.log(message)) .catch((err) => console.log(err)); From 81f010262bec475b36fb74475c70bb58dfad7aa1 Mon Sep 17 00:00:00 2001 From: Finlay Birnie Date: Mon, 25 Nov 2024 14:37:45 +0000 Subject: [PATCH 04/10] allow incremental ari import to be called from script --- api/.gitignore | 2 +- api/scripts/ariImport.ts | 96 ++++++++++++-------- api/src/components/integration/ariUtils.ts | 83 +++++++++++++++++ api/src/components/integration/controller.ts | 2 +- api/src/components/integration/service.ts | 8 +- api/src/lib/email.ts | 64 +------------ 6 files changed, 152 insertions(+), 103 deletions(-) diff --git a/api/.gitignore b/api/.gitignore index 2820c8359..8183916fa 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -14,4 +14,4 @@ prod.env coverage Octopus.postman_collection.json dist -full-ari-import-report.txt \ No newline at end of file +ari-import-report.txt \ No newline at end of file diff --git a/api/scripts/ariImport.ts b/api/scripts/ariImport.ts index 7e4d3d69b..23df91ffa 100644 --- a/api/scripts/ariImport.ts +++ b/api/scripts/ariImport.ts @@ -1,5 +1,4 @@ import axios from 'axios'; -import * as fs from 'fs/promises'; import * as dotenv from 'dotenv'; // Important to do this so that environment variables are treated the same as in deployed code. @@ -8,44 +7,58 @@ dotenv.config(); import * as ariUtils from 'integration/ariUtils'; import * as Helpers from 'lib/helpers'; import * as I from 'interface'; +import * as integrationService from 'integration/service'; + +const checkBooleanArgValue = (arg: string): void => { + if (arg && !(arg === 'true' || arg === 'false')) { + throw new Error(`"${arg}" must be "true" or "false"`); + } +}; /** * Can take the following arguments: * - allDepartments: If "true", the script will run for all departments, * rather than just the ones specified in the PARTICIPATING_ARI_USER_IDS environment variable. + * - Has no effect unless "full" is "true". + * - Default: false + * - dryRun: If "true", the script will not actually create or update any publications, + * and instead report on what it would have done. + * - Default: false * - full: If "true", the script will import all ARIs from the ARI DB, instead of stopping when it * thinks it has found all the new ones (the incremental way). + * - Default: false * * e.g.: * npm run ariImport -- allDepartments=true full=true */ -const parseArguments = (): { importAllDepartments: boolean; full: boolean } => { +const parseArguments = (): { importAllDepartments: boolean; dryRun: boolean; full: boolean } => { const args = Helpers.parseNpmScriptArgs(); for (const arg of Object.keys(args)) { - if (!['allDepartments', 'full'].includes(arg)) { + if (!['allDepartments', 'dryRun', 'full'].includes(arg)) { throw new Error(`Unexpected argument: ${arg}`); } } - const { allDepartments: allDepartmentsArg, full: fullArg } = args; + const { allDepartments: allDepartmentsArg, dryRun: dryRunArg, full: fullArg } = args; - if (allDepartmentsArg && !(allDepartmentsArg === 'true' || allDepartmentsArg === 'false')) { - throw new Error('"allDepartments" must be "true" or "false"'); - } - - if (fullArg && !(fullArg === 'true' || fullArg === 'false')) { - throw new Error('"full" must be "true" or "false"'); + for (const arg of [allDepartmentsArg, dryRunArg, fullArg]) { + checkBooleanArgValue(arg); } return { importAllDepartments: !!allDepartmentsArg, + dryRun: !!dryRunArg, full: !!fullArg }; }; -const ariImport = async (allDepartments?: boolean, full?: boolean): Promise => { - console.log(full); +/** + * Full ARI ingest. + * Differs from incremental ingest by fetching all ARIs before processing them. + * It will not stop until all ARIs have been processed. + */ +export const fullAriIngest = async (allDepartments: boolean, dryRun: boolean): Promise => { const startTime = performance.now(); // Collect all ARIs in a variable. @@ -99,7 +112,7 @@ const ariImport = async (allDepartments?: boolean, full?: boolean): Promise(); for (const ari of aris) { - const handleAri = await ariUtils.handleIncomingARI(ari); + const handleAri = await ariUtils.handleIncomingARI(ari, dryRun); if (handleAri.unrecognisedDepartment) { unrecognisedDepartments.add(handleAri.unrecognisedDepartment); @@ -113,16 +126,24 @@ const ariImport = async (allDepartments?: boolean, full?: boolean): Promise setTimeout(resolve, 1000)); + if (!dryRun) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + break; case 'update': updatedCount++; + // We hit datacite once when updating an ARI in place, so wait half a second. - await new Promise((resolve) => setTimeout(resolve, 500)); + if (!dryRun) { + await new Promise((resolve) => setTimeout(resolve, 500)); + } + break; case 'none': skippedCount++; @@ -152,32 +173,35 @@ const ariImport = async (allDepartments?: boolean, full?: boolean): Promise => { + if (!full) { + return await integrationService.incrementalAriIngest(dryRun, 'file'); + } else { + return await fullAriIngest(allDepartments, dryRun); + } }; -const { importAllDepartments, full } = parseArguments(); +const { importAllDepartments, dryRun, full } = parseArguments(); -ariImport(importAllDepartments, full) +ariImport(importAllDepartments, dryRun, full) .then((message) => console.log(message)) .catch((err) => console.log(err)); diff --git a/api/src/components/integration/ariUtils.ts b/api/src/components/integration/ariUtils.ts index 51e2f50fb..8b8028a59 100644 --- a/api/src/components/integration/ariUtils.ts +++ b/api/src/components/integration/ariUtils.ts @@ -1,6 +1,7 @@ import * as client from 'lib/client'; import * as coAuthorService from 'coAuthor/service'; import * as config from 'config'; +import * as email from 'email'; import * as Helpers from 'lib/helpers'; import * as I from 'interface'; import * as publicationService from 'publication/service'; @@ -9,6 +10,7 @@ import * as topicMappingService from 'topicMapping/service'; import * as userMappingService from 'userMapping/service'; import * as userService from 'user/service'; +import * as fs from 'fs/promises'; import { Prisma } from '@prisma/client'; const parseAriTextField = (value: string): string => { @@ -408,3 +410,84 @@ export const getParticipatingDepartmentNames = async (): Promise => { return queryResults.flatMap((userMappings) => userMappings.map((userMapping) => userMapping.value)); }; + +export const ingestReport = async ( + format: 'email' | 'file', + ingestDetails: { + checkedCount: number; + durationSeconds: number; + createdCount: number; + updatedCount: number; + unrecognisedDepartments: string[]; + unrecognisedTopics: string[]; + dryRun: boolean; + full: boolean; + } +): Promise => { + const { + checkedCount, + durationSeconds, + createdCount, + updatedCount, + unrecognisedDepartments, + unrecognisedTopics, + dryRun, + full + } = ingestDetails; + const intro = `${full ? 'Full' : 'Incremental'} ARI import ${dryRun ? 'dry ' : ''}run completed.`; + const timingInfo = + `Duration: ${durationSeconds} seconds.` + + (dryRun && (createdCount || updatedCount) + ? ` A real run would have taken a minimum of ${ + createdCount + updatedCount / 2 + } additional seconds due to datacite API rate limits while creating/updating publications.` + : ''); + const detailsPrefix = `The ${dryRun ? 'simulated ' : ''}results of this run are as follows.`; + const text = ` +${intro} +${timingInfo} +${detailsPrefix} +ARIs checked: ${checkedCount}. +Publications created: ${createdCount}. +Publications updated: ${updatedCount}. +${unrecognisedDepartments.length ? 'Unrecognised departments: "' + unrecognisedDepartments.join('", "') + '".' : ''} +${unrecognisedTopics.length ? 'Unrecognised topics: "' + unrecognisedTopics.join('", "') + '".' : ''}`; + + if (format === 'file') { + const fileName = 'ari-import-report.txt'; + await fs.writeFile(fileName, text); + console.log(`Report file written to ${fileName}.`); + } + + if (format === 'email') { + const cleanDepartments = unrecognisedDepartments.map((department) => Helpers.getSafeHTML(department)); + const cleanTopics = unrecognisedTopics.map((topic) => Helpers.getSafeHTML(topic)); + const html = ` + + +

${intro}

+

${timingInfo}

+

${detailsPrefix}

+
    +
  • ARIs checked: ${checkedCount}
    • +
  • Publications created: ${createdCount}
    • +
  • Publications updated: ${updatedCount}
    • + ${ + cleanDepartments.length + ? '
  • Unrecognised departments:
    • ' + + cleanDepartments.join('
    • ') + + '
    • ' + : '' + } + ${ + cleanTopics.length + ? '
  • Unrecognised topics:
    • ' + cleanTopics.join('
    • ') + '
    • ' + : '' + } +
+ + + `; + await email.ariIngestReport(html, text); + } +}; diff --git a/api/src/components/integration/controller.ts b/api/src/components/integration/controller.ts index 19ad75938..3c8de9254 100644 --- a/api/src/components/integration/controller.ts +++ b/api/src/components/integration/controller.ts @@ -36,7 +36,7 @@ export const incrementalAriIngest = async ( } try { - const ingestResult = await integrationService.incrementalAriIngest(dryRun); + const ingestResult = await integrationService.incrementalAriIngest(dryRun, 'email'); return response.json( 200, diff --git a/api/src/components/integration/service.ts b/api/src/components/integration/service.ts index b42fad31c..6e88de4f4 100644 --- a/api/src/components/integration/service.ts +++ b/api/src/components/integration/service.ts @@ -1,6 +1,5 @@ import axios from 'axios'; import * as ariUtils from 'integration/ariUtils'; -import * as email from 'lib/email'; import * as ingestLogService from 'ingestLog/service'; /** @@ -11,7 +10,7 @@ import * as ingestLogService from 'ingestLog/service'; * - It encounters an ARI with dateUpdated before the start time of the most * recent successful ingest (if this start time is available). */ -export const incrementalAriIngest = async (dryRun: boolean): Promise => { +export const incrementalAriIngest = async (dryRun: boolean, reportFormat: 'email' | 'file'): Promise => { const start = new Date(); const MAX_UNCHANGED_STREAK = 5; // Get most start time of last successful run to help us know when to stop. @@ -135,14 +134,15 @@ export const incrementalAriIngest = async (dryRun: boolean): Promise => await ingestLogService.setEndTime(logId, end); } - await email.incrementalAriIngestReport({ + await ariUtils.ingestReport(reportFormat, { checkedCount, durationSeconds, createdCount, updatedCount, unrecognisedDepartments: Array.from(unrecognisedDepartments).sort(), unrecognisedTopics: Array.from(unrecognisedTopics).sort(), - dryRun + dryRun, + full: false }); const writeCount = createdCount + updatedCount; diff --git a/api/src/lib/email.ts b/api/src/lib/email.ts index b47054501..605b4bc13 100644 --- a/api/src/lib/email.ts +++ b/api/src/lib/email.ts @@ -921,70 +921,12 @@ export const newAriChildPublication = async (options: { }); }; -export const incrementalAriIngestReport = async (options: { - checkedCount: number; - durationSeconds: number; - createdCount: number; - updatedCount: number; - unrecognisedDepartments: string[]; - unrecognisedTopics: string[]; - dryRun: boolean; -}): Promise => { - const { createdCount, dryRun, updatedCount } = options; - const cleanDepartments = options.unrecognisedDepartments.map((department) => Helpers.getSafeHTML(department)); - const cleanTopics = options.unrecognisedTopics.map((topic) => Helpers.getSafeHTML(topic)); - const intro = `Incremental ARI import ${dryRun ? 'dry ' : ''}run completed.`; - const timingInfo = - `Duration: ${options.durationSeconds} seconds.` + - (dryRun && (createdCount || updatedCount) - ? ` A real run would have taken a minimum of ${ - createdCount + updatedCount / 2 - } additional seconds due to datacite API rate limits while creating/updating publications.` - : ''); - const detailsPrefix = `The ${dryRun ? 'simulated ' : ''}results of this run are as follows.`; - const html = ` - - -

${intro}

-

${timingInfo}

-

${detailsPrefix}

-
    -
  • ARIs checked: ${options.checkedCount}
    • -
  • Publications created: ${createdCount}
    • -
  • Publications updated: ${updatedCount}
    • - ${ - cleanDepartments.length - ? '
  • Unrecognised departments:
    • ' + - cleanDepartments.join('
    • ') + - '
    • ' - : '' - } - ${ - cleanTopics.length - ? '
  • Unrecognised topics:
    • ' + cleanTopics.join('
    • ') + '
    • ' - : '' - } -
- - - `; - const text = ` -${intro} -${timingInfo} -${detailsPrefix} -ARIs checked: ${options.checkedCount}. -Publications created: ${options.createdCount}. -Publications updated: ${options.updatedCount}. -${ - options.unrecognisedDepartments.length - ? 'Unrecognised departments: "' + options.unrecognisedDepartments.join('", "') + '".' - : '' -} -${options.unrecognisedTopics.length ? 'Unrecognised topics: "' + options.unrecognisedTopics.join('", "') + '".' : ''}`; +// See ariUtils file for body construction. +export const ariIngestReport = async (html: string, text: string): Promise => { await send({ html, text, to: process.env.INGEST_REPORT_RECIPIENTS ? process.env.INGEST_REPORT_RECIPIENTS.split(',') : '', - subject: 'Incremental ARI ingest report' + subject: 'ARI ingest report' }); }; From e244c79bed4c09b6fcc00fcc1d22f0760f176f85 Mon Sep 17 00:00:00 2001 From: Finlay Birnie Date: Mon, 25 Nov 2024 14:57:34 +0000 Subject: [PATCH 05/10] add returns --- api/src/components/integration/ariUtils.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/src/components/integration/ariUtils.ts b/api/src/components/integration/ariUtils.ts index 8b8028a59..7c332bbdd 100644 --- a/api/src/components/integration/ariUtils.ts +++ b/api/src/components/integration/ariUtils.ts @@ -457,6 +457,8 @@ ${unrecognisedTopics.length ? 'Unrecognised topics: "' + unrecognisedTopics.join const fileName = 'ari-import-report.txt'; await fs.writeFile(fileName, text); console.log(`Report file written to ${fileName}.`); + + return; } if (format === 'email') { @@ -489,5 +491,7 @@ ${unrecognisedTopics.length ? 'Unrecognised topics: "' + unrecognisedTopics.join `; await email.ariIngestReport(html, text); + + return; } }; From 189803f559728ecb800f33240b01ac767d0b6afe Mon Sep 17 00:00:00 2001 From: Finlay Birnie Date: Tue, 26 Nov 2024 09:51:33 +0000 Subject: [PATCH 06/10] use tsx rather than ts-node --- api/package-lock.json | 1 + api/package.json | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/api/package-lock.json b/api/package-lock.json index 87d7ca3e0..d84ffd8be 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -65,6 +65,7 @@ "serverless-prune-plugin": "^2.1.0", "ts-jest": "^29.1.1", "ts-loader": "^9.4.4", + "tsx": "^4.19.2", "typescript": "^5.2.2" }, "engines": { diff --git a/api/package.json b/api/package.json index ec217d9e6..2fbdf73ab 100644 --- a/api/package.json +++ b/api/package.json @@ -13,7 +13,7 @@ "node": "20.x" }, "prisma": { - "seed": "ts-node -r tsconfig-paths/register prisma/seed.ts" + "seed": "tsx prisma/seed.ts" }, "scripts": { "build": "swc src -d dist --copy-files -s false", @@ -32,13 +32,13 @@ "test:local": "STAGE=local jest --runInBand --testTimeout=60000", "test:watch": "npm run test:local -- --watch", "type": "tsc --noEmit", - "ariImport": "ts-node -r tsconfig-paths/register scripts/ariImport.ts", - "createOrganisationalAccounts": "ts-node -r tsconfig-paths/register scripts/createOrganisationalAccounts.ts", - "createTopicMappings": "ts-node -r tsconfig-paths/register scripts/createTopicMappings.ts", - "createTopics": "ts-node -r tsconfig-paths/register scripts/createTopics.ts", - "createUserMappings": "ts-node -r tsconfig-paths/register scripts/createUserMappings.ts", - "reindex": "ts-node scripts/reindex.ts", - "updateOrganisationalAccounts": "ts-node -r tsconfig-paths/register scripts/updateOrganisationalAccounts.ts" + "ariImport": "tsx scripts/ariImport.ts", + "createOrganisationalAccounts": "tsx scripts/createOrganisationalAccounts.ts", + "createTopicMappings": "tsx scripts/createTopicMappings.ts", + "createTopics": "tsx scripts/createTopics.ts", + "createUserMappings": "tsx scripts/createUserMappings.ts", + "reindex": "tsx scripts/reindex.ts", + "updateOrganisationalAccounts": "tsx scripts/updateOrganisationalAccounts.ts" }, "dependencies": { "@middy/core": "^4.7.0", @@ -50,8 +50,8 @@ "@sparticuz/chromium": "^119.0.2", "ajv": "^8.12.0", "ajv-formats": "^2.1.1", - "axios": "^1.7.4", "aws-lambda": "^1.0.7", + "axios": "^1.7.4", "cheerio": "^1.0.0-rc.12", "dotenv": "16.4.5", "html-to-text": "^9.0.5", @@ -97,6 +97,7 @@ "serverless-prune-plugin": "^2.1.0", "ts-jest": "^29.1.1", "ts-loader": "^9.4.4", + "tsx": "^4.19.2", "typescript": "^5.2.2" }, "optionalDependencies": { From 5edd97b2a1e8ba185b0998a350934c54f6a2ed6c Mon Sep 17 00:00:00 2001 From: Finlay Birnie Date: Tue, 26 Nov 2024 10:25:51 +0000 Subject: [PATCH 07/10] try to fix typing error --- ui/src/components/Delay/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/components/Delay/index.tsx b/ui/src/components/Delay/index.tsx index ea643504b..6a23f612b 100644 --- a/ui/src/components/Delay/index.tsx +++ b/ui/src/components/Delay/index.tsx @@ -1,7 +1,7 @@ import React from 'react'; type Props = { - children: React.ReactChildren | React.ReactChild | React.ReactElement; + children: React.ReactNode; delay: number; }; From 1603f93315931cec261d6beabe37143de7cff7de Mon Sep 17 00:00:00 2001 From: Finlay Birnie Date: Tue, 26 Nov 2024 14:44:02 +0000 Subject: [PATCH 08/10] don't care about array order --- .../__tests__/createPublication.test.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/api/src/components/publication/__tests__/createPublication.test.ts b/api/src/components/publication/__tests__/createPublication.test.ts index 5677bf302..3feb17a1f 100644 --- a/api/src/components/publication/__tests__/createPublication.test.ts +++ b/api/src/components/publication/__tests__/createPublication.test.ts @@ -500,13 +500,14 @@ describe('Create live publication', () => { }); expect(createPublicationRequest.status).toEqual(201); - expect(createPublicationRequest.body.linkedTo).toHaveLength(2); - expect(createPublicationRequest.body.linkedTo[0].publicationToId).toEqual('publication-problem-live'); - expect(createPublicationRequest.body.linkedTo[0].versionToId).toEqual('publication-problem-live-v2'); - expect(createPublicationRequest.body.linkedTo[0].draft).toEqual(false); - expect(createPublicationRequest.body.linkedTo[1].publicationToId).toEqual('publication-problem-live-2'); - expect(createPublicationRequest.body.linkedTo[1].versionToId).toEqual('publication-problem-live-2-v1'); - expect(createPublicationRequest.body.linkedTo[1].draft).toEqual(false); + expect(createPublicationRequest.body.linkedTo).toMatchObject([ + { publicationToId: 'publication-problem-live', versionToId: 'publication-problem-live-v2', draft: false }, + { + publicationToId: 'publication-problem-live-2', + versionToId: 'publication-problem-live-2-v1', + draft: false + } + ]); }); test('Invalid links cannot be created via direct publishing', async () => { From 6ac2fe7a63b5cc01457678c7c72cac2bfe2a5838 Mon Sep 17 00:00:00 2001 From: Finlay Birnie Date: Tue, 26 Nov 2024 15:16:58 +0000 Subject: [PATCH 09/10] ensure order before matching --- .../publication/__tests__/createPublication.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/src/components/publication/__tests__/createPublication.test.ts b/api/src/components/publication/__tests__/createPublication.test.ts index 3feb17a1f..0d9c13097 100644 --- a/api/src/components/publication/__tests__/createPublication.test.ts +++ b/api/src/components/publication/__tests__/createPublication.test.ts @@ -500,7 +500,9 @@ describe('Create live publication', () => { }); expect(createPublicationRequest.status).toEqual(201); - expect(createPublicationRequest.body.linkedTo).toMatchObject([ + expect( + createPublicationRequest.body.linkedTo.sort((a, b) => a.publicationToId - b.publicationToId) + ).toMatchObject([ { publicationToId: 'publication-problem-live', versionToId: 'publication-problem-live-v2', draft: false }, { publicationToId: 'publication-problem-live-2', From cac8f45727d2904f28d176bed7f428d4f9e236e1 Mon Sep 17 00:00:00 2001 From: Finlay Birnie Date: Tue, 26 Nov 2024 15:32:22 +0000 Subject: [PATCH 10/10] use a reliable string sorting function --- .../components/publication/__tests__/createPublication.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/components/publication/__tests__/createPublication.test.ts b/api/src/components/publication/__tests__/createPublication.test.ts index 0d9c13097..b1450ceab 100644 --- a/api/src/components/publication/__tests__/createPublication.test.ts +++ b/api/src/components/publication/__tests__/createPublication.test.ts @@ -501,7 +501,7 @@ describe('Create live publication', () => { expect(createPublicationRequest.status).toEqual(201); expect( - createPublicationRequest.body.linkedTo.sort((a, b) => a.publicationToId - b.publicationToId) + createPublicationRequest.body.linkedTo.sort((a, b) => a.publicationToId.localeCompare(b.publicationToId)) ).toMatchObject([ { publicationToId: 'publication-problem-live', versionToId: 'publication-problem-live-v2', draft: false }, {