feat: Added publishing to workflow_job event queue for multi runner m…

…odule. (#2570) * feat: Remove support check_run (#2521) * chore: Remove support check_run * format, lint * feat: Remove old scale down mechanism (< 0.19.0) (#2519) fix: Remove old cleanup mechanism (< 0.19.0) * chore: Enable pre releases for next branch * feat: Add multi-runner capability (#2472) * feat: Remove support check_run (#2521) * chore: Remove support check_run * format, lint * feat: Remove old scale down mechanism (< 0.19.0) (#2519) fix: Remove old cleanup mechanism (< 0.19.0) * feat: added changes for multi runner. * fix: region. * fix: more fixes. * tuple to list. * fixes. * fixes. * fixes. * fixes. * fixes. * fixes. * fix: formatting. * fix: formatting. * fix: formatting. * fix: moved some blocks outside runner config. * fix: few more updates * fix: liniting. * fix: updated example output * changed runner group name. * fix: updated the tests. * fix: addressed review comments. * fix: linting issues. * fix: formatting. * fix: updated tf version. * fix: Remove removed prerelease option * Add ubuntu runner to example * refactor: use each instead of count * fix: few small issues. * refactor: syncer to count for multi runner * fix: comments. * fix: added Readme. * fix: errors. * move variable to runner config * fix: updated the readme. * Add todos * feat: added windows runner configuration, completed todos and added the weight for runner config matchers. * chore: Update docs * fix: reverted tf versions. * fix: addressed comments. * fix: missed. * fix: formatting. * Update terraform versions in CI * Update terraform versions in CI * Update docs * fix: coverage. * Update docs * improve test coverage webhook * Apply suggestions from code review * fix: formatting. * fix: fixed merge issues. * fix: syntax. Co-authored-by: Niek Palm <npalm@users.noreply.github.com> Co-authored-by: Niek Palm <niek.palm@philips.com> Co-authored-by: navdeepg2021 <navdeepg2021@gmail.com> * chore(ci): Enable ci for next branch BREAKING CHANGE: - Add option to deploy multiple runner at once, see for details: #2472 #2517 - Drop support check_run event, see for details: #2521 #2517 - Remove pre 0.19.0 scale down mechanism, see details: #2519 #2517 * chore(release): 2.0.0-next.1 [skip ci] * **ci:** * Add multi-runner capability ([#2472](#2472)) ([c08b335](c08b335)), closes [#2521](#2521) [#2519](#2519) * Experimental feature - Duplicate workflow job event to extra queue ([#2268](#2268)) ([ac046b8](ac046b8)) * Remove old scale down mechanism (< 0.19.0) ([#2519](#2519)) ([721d7c3](721d7c3)) * Remove support check_run ([#2521](#2521)) ([272a293](272a293)) * **ci:** Enable ci for next branch ([48769ca](48769ca)), closes [#2472](#2472) [#2517](#2517) [#2521](#2521) [#2517](#2517) [#2519](#2519) [#2517](#2517) * chore: fix release workflow * fix: added changes for publishing events to secondary queue. Co-authored-by: Niek Palm <niek.palm@philips.com> Co-authored-by: Niek Palm <npalm@users.noreply.github.com> Co-authored-by: navdeepg2021 <navdeepg2021@gmail.com> Co-authored-by: semantic-release-bot <semantic-release-bot@martynus.net>
philips-labs · Dec 13, 2022 · 7ef28b4 · 7ef28b4
1 parent cf8408d
commit 7ef28b4
Show file tree

Hide file tree

Showing 6 changed files with 46 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -31,7 +31,6 @@
 
 * Experimental feature - Duplicate workflow job event to extra queue ([#2268](https://github.com/philips-labs/terraform-aws-github-runner/issues/2268)) ([985e722](https://github.com/philips-labs/terraform-aws-github-runner/commit/985e722229ce464235d206484df3d989db03e143))
 
-
 ## [1.12.0](https://github.com/philips-labs/terraform-aws-github-runner/compare/v1.11.0...v1.12.0) (2022-10-12)
 
 

diff --git a/examples/multi-runner/main.tf b/examples/multi-runner/main.tf
@@ -159,7 +159,9 @@ module "multi-runner" {
  # runner_binaries_syncer_lambda_zip = "lambdas-download/runner-binaries-syncer.zip"
  # runners_lambda_zip = "lambdas-download/runners.zip"
 
+ # enable_workflow_job_events_queue = true
  # override delay of events in seconds
 
  # log_level = "debug"
+
 }
diff --git a/modules/multi-runner/queues.tf b/modules/multi-runner/queues.tf
@@ -70,3 +70,22 @@ resource "aws_sqs_queue_policy" "build_queue_dlq_policy" {
  queue_url = aws_sqs_queue.queued_builds_dlq[each.key].id
  policy = data.aws_iam_policy_document.deny_unsecure_transport.json
 }
+
+resource "aws_sqs_queue" "webhook_events_workflow_job_queue" {
+ count = var.enable_workflow_job_events_queue ? 1 : 0
+ name = "${var.prefix}-webhook_events_workflow_job_queue"
+ delay_seconds = var.workflow_job_queue_configuration.delay_seconds
+ visibility_timeout_seconds = var.workflow_job_queue_configuration.visibility_timeout_seconds
+ message_retention_seconds = var.workflow_job_queue_configuration.message_retention_seconds
+ fifo_queue = false
+ receive_wait_time_seconds = 0
+ content_based_deduplication = false
+ redrive_policy = null
+
+ sqs_managed_sse_enabled = var.queue_encryption.sqs_managed_sse_enabled
+ kms_master_key_id = var.queue_encryption.kms_master_key_id
+ kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds
+
+ tags = var.tags
+}
+
diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf
@@ -489,3 +489,24 @@ variable "pool_lambda_reserved_concurrent_executions" {
  type = number
  default = 1
 }
+
+variable "enable_workflow_job_events_queue" {
+ description = "Enabling this experimental feature will create a secondory sqs queue to wich a copy of the workflow_job event will be delivered."
+ type = bool
+ default = false
+}
+
+variable "workflow_job_queue_configuration" {
+ description = "Configuration options for workflow job queue which is only applicable if the flag enable_workflow_job_events_queue is set to true."
+ type = object({
+ delay_seconds = number
+ visibility_timeout_seconds = number
+ message_retention_seconds = number
+ })
+ default = {
+ "delay_seconds" : null,
+ "visibility_timeout_seconds" : null,
+ "message_retention_seconds" : null
+ }
+}
+
diff --git a/modules/multi-runner/webhook.tf b/modules/multi-runner/webhook.tf
@@ -6,6 +6,8 @@ module "webhook" {
 
  runner_config = local.runner_config
  github_app_webhook_secret_arn = module.ssm.parameters.github_app_webhook_secret.arn
+ sqs_workflow_job_queue = length(aws_sqs_queue.webhook_events_workflow_job_queue) > 0 ? aws_sqs_queue.webhook_events_workflow_job_queue[0] : null
+
 
  lambda_s3_bucket = var.lambda_s3_bucket
  webhook_lambda_s3_key = var.webhook_lambda_s3_key

diff --git a/modules/webhook/lambdas/webhook/src/webhook/handler.ts b/modules/webhook/lambdas/webhook/src/webhook/handler.ts
@@ -63,10 +63,10 @@ export async function handle(headers: IncomingHttpHeaders, body: string): Promis
 
  const workflowJobEvent = payload as WorkflowJobEvent;
  response = await handleWorkflowJob(workflowJobEvent, githubEvent, queuesConfig);
- await sendWorkflowJobEvents(githubEvent, workflowJobEvent);
+ await sendWorkflowJobEvents(workflowJobEvent);
  return response;
 }
-async function sendWorkflowJobEvents(githubEvent: string, workflowEventPayload: WorkflowJobEvent) {
+async function sendWorkflowJobEvents(workflowEventPayload: WorkflowJobEvent) {
  await sendWebhookEventToWorkflowJobQueue({
  workflowJobEvent: workflowEventPayload,
  });