OC-924: Allow ECS task to be triggerable on demand

api/package.json

@@ -63,6 +63,7 @@
         "supertest": "^6.3.3"
     },
     "devDependencies": {
+        "@aws-sdk/client-ecs": "^3.709.0",
         "@aws-sdk/client-s3": "^3.413.0",
         "@aws-sdk/client-ses": "^3.413.0",
         "@aws-sdk/client-sqs": "^3.413.0",

api/serverless-config-default.yml

@@ -540,29 +540,6 @@ package:
         - '!serverless*'
         - '!src/**'
         - '!tsconfig*'
-    # From devdependencies - due to an issue serverless's automatic devDependency exclusion doesn't work.
-    # https://github.com/serverless/serverless/issues/12911
-    # Please remove these rules when it's fixed.
-        - '!node_modules/.bin/esbuild'
-        - '!node_modules/@aws-crypto/**'
-        - '!node_modules/@babel/**'
-        - '!node_modules/@esbuild/**'
-        - '!node_modules/@eslint/**'
-        - '!node_modules/@hapi/**'
-        - '!node_modules/@types/**'
-        - '!node_modules/@typescript-eslint/**'
-        - '!node_modules/devtools-protocol/**'
-        - '!node_modules/es-abstract/**'
-        - '!node_modules/esbuild/**'
-        - '!node_modules/eslint/**'
-        - '!node_modules/eslint-plugin-import/**'
-        - '!node_modules/java-invoke-local/**'
-        - '!node_modules/prettier/**'
-        - '!node_modules/rxjs/**'
-        - '!node_modules/serverless/**'
-        - '!node_modules/terser/**'
-        - '!node_modules/web-streams-polyfill/**'
-        - '!node_modules/webpack/**'
     # Inclusions.
         - 'node_modules/.prisma/client/libquery_engine-linux-arm64-*'
 
@@ -600,29 +577,6 @@ package:
         - '!serverless*'
         - '!src/**'
         - '!tsconfig*'
-        # From devdependencies - due to an issue serverless's automatic devDependency exclusion doesn't work.
-        # https://github.com/serverless/serverless/issues/12911
-        # Please remove these rules when it's fixed.
-        - '!node_modules/.bin/esbuild'
-        - '!node_modules/@aws-crypto/**'
-        - '!node_modules/@babel/**'
-        - '!node_modules/@esbuild/**'
-        - '!node_modules/@eslint/**'
-        - '!node_modules/@hapi/**'
-        - '!node_modules/@types/**'
-        - '!node_modules/@typescript-eslint/**'
-        - '!node_modules/devtools-protocol/**'
-        - '!node_modules/es-abstract/**'
-        - '!node_modules/esbuild/**'
-        - '!node_modules/eslint/**'
-        - '!node_modules/eslint-plugin-import/**'
-        - '!node_modules/java-invoke-local/**'
-        - '!node_modules/prettier/**'
-        - '!node_modules/rxjs/**'
-        - '!node_modules/serverless/**'
-        - '!node_modules/terser/**'
-        - '!node_modules/web-streams-polyfill/**'
-        - '!node_modules/webpack/**'
         # Inclusions.
         - 'node_modules/@sparticuz/chromium/**'
         - 'node_modules/.prisma/client/libquery_engine-rhel-*'

api/serverless-config-deploy.yml

@@ -3,4 +3,11 @@ functions:
     generatePDFsFromQueue:
         handler: dist/src/components/sqs/handler.generatePDFs
         events:
-            - sqs: 'arn:aws:sqs:${aws:region}:${aws:accountId}:science-octopus-pdf-queue-${self:provider.stage}'
+            - sqs: 'arn:aws:sqs:${aws:region}:${aws:accountId}:science-octopus-pdf-queue-${self:provider.stage}'
+    triggerECSTask:
+        handler: dist/src/components/integration/routes.triggerECSTask
+        events:
+            - http:
+                path: ${self:custom.versions.v1}/integrations/simple-ecs-task
+                method: POST
+                cors: true

api/serverless.yml

@@ -50,6 +50,10 @@ provider:
         TRIGGER_ARI_INGEST_API_KEY: ${ssm:/trigger_ari_ingest_api_key_${self:provider.stage}_octopus}
         INGEST_REPORT_RECIPIENTS: ${ssm:/ingest_report_recipients_${self:provider.stage}_octopus}
         PARTICIPATING_ARI_USER_IDS: ${ssm:/participating_ari_user_ids_${self:provider.stage}_octopus}
+        ECS_CLUSTER_ARN: ${ssm:/ecs_cluster_arn_${self:provider.stage}_octopus}
+        ECS_TASK_DEFINITION_ID: ${ssm:/ecs_task_definition_id_${self:provider.stage}_octopus}
+        ECS_TASK_SECURITY_GROUP_ID: ${ssm:/ecs_task_security_group_id_${self:provider.stage}_octopus}
+        PRIVATE_SUBNET_IDS: ${ssm:/${self:provider.stage}_octopus_private_subnet_az1},${ssm:/${self:provider.stage}_octopus_private_subnet_az2},${ssm:/${self:provider.stage}_octopus_private_subnet_az3}
     deploymentBucket:
         tags:
             Project: Octopus
@@ -84,6 +88,15 @@ provider:
               - 'sqs:GetQueueAttributes'
               - 'sqs:ReceiveMessage'
               - 'sqs:SendMessage'
+        # Required to trigger ECS tasks
+        - Effect: 'Allow'
+          Resource: "arn:aws:ecs:${aws:region}:${aws:accountId}:task-definition/${ssm:/ecs_task_definition_id_${self:provider.stage}_octopus}"
+          Action: 'ecs:RunTask'
+        - Effect: 'Allow'
+          Resource:
+              - 'arn:aws:iam::${aws:accountId}:role/octopus-ecs-task-role-${self:provider.stage}'
+              - 'arn:aws:iam::${aws:accountId}:role/octopus-ecs-task-exec-role-${self:provider.stage}'
+          Action: 'iam:PassRole'
 custom:
     splitStacks:
         perFunction: true

api/src/components/integration/__tests__/ari.test.ts

@@ -415,16 +415,9 @@ describe('ARI import processes', () => {
     test('Incremental import endpoint requires API key', async () => {
         const triggerImport = await testUtils.agent.post('/integrations/ari/incremental');
 
-        expect(triggerImport.status).toEqual(400);
+        expect(triggerImport.status).toEqual(401);
         expect(triggerImport.body).toMatchObject({
-            message: [
-                {
-                    keyword: 'required',
-                    params: {
-                        missingProperty: 'apiKey'
-                    }
-                }
-            ]
+            message: "Please provide a valid 'apiKey'."
         });
     });
 

api/src/components/integration/controller.ts

@@ -6,20 +6,9 @@ import * as response from 'lib/response';
 export const incrementalAriIngest = async (
     event: I.APIRequest | I.EventBridgeEvent<'Scheduled Event', string>
 ): Promise<I.JSONResponse> => {
-    const triggeredByHttp = event && 'headers' in event;
-
-    // This can also be triggered on a schedule, in which case we don't need to check for an API key,
-    // so only check for the API key if the event is an API request.
-    if (triggeredByHttp) {
-        const apiKey = event.queryStringParameters?.apiKey;
-
-        if (apiKey !== process.env.TRIGGER_ARI_INGEST_API_KEY) {
-            return response.json(401, { message: "Please provide a valid 'apiKey'." });
-        }
-    }
-
     // Check if a process is currently running.
     const lastLog = await ingestLogService.getMostRecentLog('ARI', true);
+    const triggeredByHttp = event && 'headers' in event;
     const dryRun = triggeredByHttp ? !!event.queryStringParameters?.dryRun : false;
     const dryRunMessages: string[] = [];
 
@@ -48,3 +37,9 @@ export const incrementalAriIngest = async (
         return response.json(500, { message: 'Unknown server error.' });
     }
 };
+
+export const triggerECSTask = async (): Promise<I.JSONResponse> => {
+    const triggerTaskOutput = await integrationService.triggerECSTask();
+
+    return response.json(200, { message: triggerTaskOutput });
+};

api/src/components/integration/routes.ts

@@ -1,9 +1,17 @@
 import middy from '@middy/core';
 
+import * as Helpers from 'lib/helpers';
 import * as integrationController from 'integration/controller';
 import * as integrationSchema from 'integration/schema';
 import * as middleware from 'middleware';
 
+const triggerAriIngestApiKey = Helpers.checkEnvVariable('TRIGGER_ARI_INGEST_API_KEY');
+
 export const incrementalAriIngest = middy(integrationController.incrementalAriIngest)
     .use(middleware.doNotWaitForEmptyEventLoop({ runOnError: true, runOnBefore: true, runOnAfter: true }))
+    .use(middleware.authentication(false, false, triggerAriIngestApiKey))
     .use(middleware.validator(integrationSchema.incrementalAriIngestHttp, 'queryStringParameters'));
+
+export const triggerECSTask = middy(integrationController.triggerECSTask)
+    .use(middleware.doNotWaitForEmptyEventLoop({ runOnError: true, runOnBefore: true, runOnAfter: true }))
+    .use(middleware.authentication(false, false, triggerAriIngestApiKey));

api/src/components/integration/service.ts

@@ -1,6 +1,8 @@
 import axios from 'axios';
 import * as ariUtils from 'integration/ariUtils';
+import * as ecs from 'lib/ecs';
 import * as ingestLogService from 'ingestLog/service';
+import * as Helpers from 'lib/helpers';
 
 /**
  * Incremental ARI ingest.
@@ -151,3 +153,14 @@ export const incrementalAriIngest = async (dryRun: boolean, reportFormat: 'email
 
     return `${preamble} ${writeCount} publication${writeCount !== 1 ? 's' : ''}.`;
 };
+
+export const triggerECSTask = async (): Promise<string> => {
+    await ecs.runFargateTask({
+        clusterArn: Helpers.checkEnvVariable('ECS_CLUSTER_ARN'),
+        securityGroups: [Helpers.checkEnvVariable('ECS_TASK_SECURITY_GROUP_ID')],
+        subnetIds: Helpers.checkEnvVariable('PRIVATE_SUBNET_IDS').split(','),
+        taskDefinitionId: Helpers.checkEnvVariable('ECS_TASK_DEFINITION_ID')
+    });
+
+    return 'Done';
+};

api/src/components/user/controller.ts

@@ -77,13 +77,7 @@ export const getPublications = async (
     }
 };
 
-export const getUserList = async (event: I.APIRequest): Promise<I.JSONResponse> => {
-    const apiKey = event.queryStringParameters?.apiKey;
-
-    if (apiKey !== process.env.LIST_USERS_API_KEY) {
-        return response.json(401, { message: "Please provide a valid 'apiKey'." });
-    }
-
+export const getUserList = async (): Promise<I.JSONResponse> => {
     try {
         const userList = await userService.getUserList();
 

api/src/components/user/routes.ts

@@ -1,5 +1,6 @@
 import middy from '@middy/core';
 
+import * as Helpers from 'lib/helpers';
 import * as middleware from 'middleware';
 import * as userController from 'user/controller';
 import * as userSchema from 'user/schema';
@@ -20,7 +21,9 @@ export const getPublications = middy(userController.getPublications)
     .use(middleware.authentication(true))
     .use(middleware.validator(userSchema.getPublications, 'queryStringParameters'));
 
-export const getUserList = userController.getUserList;
+export const getUserList = middy(userController.getUserList)
+    .use(middleware.doNotWaitForEmptyEventLoop({ runOnError: true, runOnBefore: true, runOnAfter: true }))
+    .use(middleware.authentication(false, false, Helpers.checkEnvVariable('LIST_USERS_API_KEY')));
 
 export const getUserControlRequests = middy(userController.getUserControlRequests)
     .use(middleware.doNotWaitForEmptyEventLoop({ runOnError: true, runOnBefore: true, runOnAfter: true }))

api/src/lib/ecs.ts

@@ -0,0 +1,25 @@
+import { ECSClient, RunTaskCommand, RunTaskCommandInput } from '@aws-sdk/client-ecs';
+
+const client = new ECSClient();
+
+export const runFargateTask = async (config: {
+    clusterArn: string;
+    securityGroups: string[];
+    subnetIds: string[];
+    taskDefinitionId: string;
+}): Promise<void> => {
+    const input: RunTaskCommandInput = {
+        cluster: config.clusterArn,
+        launchType: 'FARGATE',
+        networkConfiguration: {
+            awsvpcConfiguration: {
+                securityGroups: config.securityGroups,
+                subnets: config.subnetIds
+            }
+        },
+        taskDefinition: config.taskDefinitionId
+    };
+    const command = new RunTaskCommand(input);
+    const response = await client.send(command);
+    console.log(response);
+};

api/src/middleware/authentication.ts

@@ -5,36 +5,47 @@ import * as response from 'lib/response';
 import * as userService from 'user/service';
 import * as authorizationService from 'authorization/service';
 
-const authentication = (optional = false, requiresName = true): middy.MiddlewareObj => {
+const authentication = (optional = false, requiresName = true, endpointSpecificKey?: string): middy.MiddlewareObj => {
     const before: middy.MiddlewareFn<I.APIGatewayProxyEventV2> = async (request): Promise<I.JSONResponse | void> => {
         try {
             let user: null | I.User = null;
 
             const apiKey = request.event.queryStringParameters?.apiKey;
-            const bearerToken = request.event.headers.Authorization;
 
-            if (apiKey) {
-                user = await userService.getByApiKey(apiKey);
-            } else if (bearerToken) {
-                user = authorizationService.validateJWT(bearerToken.split(' ')[1]);
-            }
-
-            if (!optional) {
-                // If there's no user account, and authentication is *not* optional, then the request is blocked.
-                if (!user) {
-                    return response.json(401, { message: 'Please enter either a valid apiKey or bearer token.' });
+            if (endpointSpecificKey) {
+                // The function only accepts one specific api key.)
+                if (apiKey !== endpointSpecificKey) {
+                    return response.json(401, { message: "Please provide a valid 'apiKey'." });
+                }
+            } else {
+                // The function attempts to authenticate an existing user,
+                // looking them up either by their api key or JWT,
+                // and then adds the user details to the request event.
+                const bearerToken = request.event.headers.Authorization;
+
+                if (apiKey) {
+                    user = await userService.getByApiKey(apiKey);
+                } else if (bearerToken) {
+                    user = authorizationService.validateJWT(bearerToken.split(' ')[1]);
                 }
 
-                // If the user hasn't made their name visible in ORCiD, we want to disallow them from doing most actions.
-                if (!user?.firstName && !user?.lastName && requiresName) {
-                    return response.json(403, {
-                        message:
-                            'No name detected. Please ensure your name visibility is set to "Everyone" or "Trusted parties" on your ORCiD account, then re-authorize at /authorization.'
-                    });
+                if (!optional) {
+                    // If there's no user account, and authentication is *not* optional, then the request is blocked.
+                    if (!user) {
+                        return response.json(401, { message: 'Please enter either a valid apiKey or bearer token.' });
+                    }
+
+                    // If the user hasn't made their name visible in ORCiD, we want to disallow them from doing most actions.
+                    if (!user?.firstName && !user?.lastName && requiresName) {
+                        return response.json(403, {
+                            message:
+                                'No name detected. Please ensure your name visibility is set to "Everyone" or "Trusted parties" on your ORCiD account, then re-authorize at /authorization.'
+                        });
+                    }
                 }
-            }
 
-            Object.assign(request.event, { user });
+                Object.assign(request.event, { user });
+            }
         } catch (err) {
             console.log(err);
 

infra/modules/ecs/ecs.tf

@@ -48,3 +48,9 @@ resource "aws_ecs_cluster" "ecs" {
     namespace = aws_service_discovery_private_dns_namespace.namespace.arn
   }
 }
+
+resource "aws_ssm_parameter" "ecs-cluster-arn" {
+  name  = "ecs_cluster_arn_${var.environment}_${var.project_name}"
+  type  = "String"
+  value = aws_ecs_cluster.ecs.arn
+}