From b98a4b2f051f5c151e0b400c134f46a1e51cb355 Mon Sep 17 00:00:00 2001 From: Kostiantyn Vorobiov Date: Wed, 24 Feb 2021 21:07:35 +0200 Subject: [PATCH 1/3] Secondary instance type (#2) * Secondary instance type for spot * log error for runInstances call --- .../runners/src/scale-runners/runners.ts | 101 ++++++++++++------ modules/runners/scale-up.tf | 1 + modules/runners/variables.tf | 6 ++ 3 files changed, 76 insertions(+), 32 deletions(-) diff --git a/modules/runners/lambdas/runners/src/scale-runners/runners.ts b/modules/runners/lambdas/runners/src/scale-runners/runners.ts index d2a5c4b92d..4f0e6a35b9 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/runners.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/runners.ts @@ -13,6 +13,26 @@ export interface ListRunnerFilters { environment?: string; } +export interface instanceParams { + MaxCount: number, + MinCount: number, + LaunchTemplate: { + LaunchTemplateName: string, + Version: string, + }, + SubnetId: string, + TagSpecifications: [ + { + ResourceType: string, + Tags: [ + { Key: string, Value?: string }, + { Key: string, Value?: string }, + ], + }, + ], + InstanceType?: string +} + export async function listRunners(filters: ListRunnerFilters | undefined = undefined): Promise { const ec2 = new EC2(); const ec2Filters = [ @@ -73,40 +93,57 @@ export async function createRunner(runnerParameters: RunnerInputParameters): Pro const subnets = (process.env.SUBNET_IDS as string).split(','); const randomSubnet = subnets[Math.floor(Math.random() * subnets.length)]; console.debug('Runner configuration: ' + JSON.stringify(runnerParameters)); - const ec2 = new EC2(); - const runInstancesResponse = await ec2 - .runInstances({ - MaxCount: 1, - MinCount: 1, - LaunchTemplate: { - LaunchTemplateName: launchTemplateName, - Version: launchTemplateVersion, + + const instanceParams : instanceParams = { + MaxCount: 1, + MinCount: 1, + LaunchTemplate: { + LaunchTemplateName: launchTemplateName, + Version: launchTemplateVersion, + }, + SubnetId: randomSubnet, + TagSpecifications: [ + { + ResourceType: 'instance', + Tags: [ + { Key: 'Application', Value: 'github-action-runner' }, + { + Key: runnerParameters.orgName ? 'Org' : 'Repo', + Value: runnerParameters.orgName ? runnerParameters.orgName : runnerParameters.repoName, + }, + ], }, - SubnetId: randomSubnet, - TagSpecifications: [ - { - ResourceType: 'instance', - Tags: [ - { Key: 'Application', Value: 'github-action-runner' }, - { - Key: runnerParameters.orgName ? 'Org' : 'Repo', - Value: runnerParameters.orgName ? runnerParameters.orgName : runnerParameters.repoName, - }, - ], - }, - ], - }) - .promise(); - console.info('Created instance(s): ', runInstancesResponse.Instances?.map((i) => i.InstanceId).join(',')); + ], + } - const ssm = new SSM(); - runInstancesResponse.Instances?.forEach(async (i: EC2.Instance) => { - await ssm - .putParameter({ - Name: runnerParameters.environment + '-' + (i.InstanceId as string), - Value: runnerParameters.runnerConfig, - Type: 'SecureString', + const ec2 = new EC2(); + try { + const runInstancesResponse = await ec2 + .runInstances(instanceParams, + async function(err, data) { + if (err) { + console.info(err) + instanceParams.InstanceType = process.env.SECONDARY_INSTANCE_TYPE as string; + return await ec2.runInstances(instanceParams) + } + return data; }) .promise(); - }); + + console.info('Created instance(s): ', runInstancesResponse.Instances?.map((i) => i.InstanceId).join(',')); + const ssm = new SSM(); + runInstancesResponse.Instances?.forEach(async (i: EC2.Instance) => { + await ssm + .putParameter({ + Name: runnerParameters.environment + '-' + (i.InstanceId as string), + Value: runnerParameters.runnerConfig, + Type: 'SecureString', + }) + .promise(); + }); + } + catch (e) { + console.error(e); + } + } diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf index 5418880662..611789d513 100644 --- a/modules/runners/scale-up.tf +++ b/modules/runners/scale-up.tf @@ -42,6 +42,7 @@ resource "aws_lambda_function" "scale_up" { LAUNCH_TEMPLATE_NAME = aws_launch_template.runner.name LAUNCH_TEMPLATE_VERSION = aws_launch_template.runner.latest_version SUBNET_IDS = join(",", var.subnet_ids) + SECONDARY_INSTANCE_TYPE = var.secondary_instance_type } } diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index 0abd518684..9ee35fc69e 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -63,6 +63,12 @@ variable "instance_type" { default = "m5.large" } +variable "secondary_instance_type" { + description = "(optional) Secondary instance type to avoid no Spot capacity available error" + type = string + default = "m5.xlarge" +} + variable "ami_filter" { description = "List of maps used to create the AMI filter for the action runner AMI." type = map(list(string)) From abe146213f4932470d79d28e8ebd0bd208ee0e36 Mon Sep 17 00:00:00 2001 From: Kostiantyn-Vorobiov Date: Wed, 24 Feb 2021 21:21:18 +0200 Subject: [PATCH 2/3] linter fixes --- .../runners/lambdas/runners/src/scale-runners/runners.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/runners/lambdas/runners/src/scale-runners/runners.ts b/modules/runners/lambdas/runners/src/scale-runners/runners.ts index 4f0e6a35b9..b00d284d3b 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/runners.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/runners.ts @@ -114,7 +114,7 @@ export async function createRunner(runnerParameters: RunnerInputParameters): Pro ], }, ], - } + }; const ec2 = new EC2(); try { @@ -122,9 +122,9 @@ export async function createRunner(runnerParameters: RunnerInputParameters): Pro .runInstances(instanceParams, async function(err, data) { if (err) { - console.info(err) + console.info(err); instanceParams.InstanceType = process.env.SECONDARY_INSTANCE_TYPE as string; - return await ec2.runInstances(instanceParams) + return await ec2.runInstances(instanceParams); } return data; }) From 0ef34e665f38af7bf66f06cb5332418671d33874 Mon Sep 17 00:00:00 2001 From: Kostiantyn-Vorobiov Date: Fri, 26 Feb 2021 12:33:36 +0200 Subject: [PATCH 3/3] try with promises for runInstances call --- .../runners/src/scale-runners/runners.ts | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/modules/runners/lambdas/runners/src/scale-runners/runners.ts b/modules/runners/lambdas/runners/src/scale-runners/runners.ts index b00d284d3b..06f07f4f39 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/runners.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/runners.ts @@ -86,6 +86,25 @@ export async function terminateRunner(runner: RunnerInfo): Promise { console.debug('Runner terminated.' + runner.instanceId); } +function launchInstance(params: instanceParams): Promise { + const ec2 = new EC2(); + return ec2.runInstances(params).promise(); +} + +async function putSSMparameter(runInstancesResponse:EC2.Reservation, runnerParameters:RunnerInputParameters) { + console.info('Created instance(s): ', runInstancesResponse.Instances?.map((i) => i.InstanceId).join(',')); + const ssm = new SSM(); + runInstancesResponse.Instances?.forEach(async (i: EC2.Instance) => { + await ssm + .putParameter({ + Name: runnerParameters.environment + '-' + (i.InstanceId as string), + Value: runnerParameters.runnerConfig, + Type: 'SecureString', + }) + .promise(); + }); +} + export async function createRunner(runnerParameters: RunnerInputParameters): Promise { const launchTemplateName = process.env.LAUNCH_TEMPLATE_NAME as string; const launchTemplateVersion = process.env.LAUNCH_TEMPLATE_VERSION as string; @@ -116,34 +135,17 @@ export async function createRunner(runnerParameters: RunnerInputParameters): Pro ], }; - const ec2 = new EC2(); - try { - const runInstancesResponse = await ec2 - .runInstances(instanceParams, - async function(err, data) { - if (err) { - console.info(err); - instanceParams.InstanceType = process.env.SECONDARY_INSTANCE_TYPE as string; - return await ec2.runInstances(instanceParams); - } - return data; - }) - .promise(); - + launchInstance(instanceParams) + .then(async function (runInstancesResponse) { + putSSMparameter(runInstancesResponse, runnerParameters); + }) + .catch( async function (err) { + console.info(err); + instanceParams.InstanceType = process.env.SECONDARY_INSTANCE_TYPE as string; + return launchInstance(instanceParams).then(runInstancesResponse => { console.info('Created instance(s): ', runInstancesResponse.Instances?.map((i) => i.InstanceId).join(',')); - const ssm = new SSM(); - runInstancesResponse.Instances?.forEach(async (i: EC2.Instance) => { - await ssm - .putParameter({ - Name: runnerParameters.environment + '-' + (i.InstanceId as string), - Value: runnerParameters.runnerConfig, - Type: 'SecureString', - }) - .promise(); - }); - } - catch (e) { - console.error(e); - } - + putSSMparameter(runInstancesResponse, runnerParameters); + }) + }) + .catch(e => console.error(e)); }