Add prettier (format check) for lambda to CI

.ci/Dockerfile

@@ -1,13 +1,26 @@
-FROM node:12
-
+#syntax=docker/dockerfile:1.2
+FROM node:12 as build
+WORKDIR /lambda
 RUN apt-get update \
-    && apt-get install -y zip \
-    && rm -rf /var/lib/apt/lists/*
+        && apt-get install -y zip \
+        && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /lambda
+FROM build as runner-binaries-syncer
+COPY modules/runner-binaries-syncer/lambdas/runner-binaries-syncer /lambda
+RUN --mount=type=cache,target=/lambda/node_modules,id=runner-binaries-syncer \
+        yarn install && yarn dist
 
-COPY . /lambda
+FROM build as runners
+COPY modules/runners/lambdas/runners /lambda
+RUN --mount=type=cache,target=/lambda/node_modules,id=runners \
+        yarn install && yarn dist
 
-RUN yarn install \
-    && yarn run dist
+FROM build as webhook
+COPY modules/webhook/lambdas/webhook /lambda
+RUN --mount=type=cache,target=/lambda/node_modules,id=webhook \
+        yarn install && yarn dist
 
+FROM scratch as final
+COPY --from=runner-binaries-syncer /lambda/runner-binaries-syncer.zip /runner-binaries-syncer.zip
+COPY --from=runners                /lambda/runners.zip                /runners.zip
+COPY --from=webhook                /lambda/webhook.zip                /webhook.zip

.ci/build.sh

@@ -1,14 +1,22 @@
 #!/usr/bin/env bash
 set -e
 
-lambdaSrcDirs=("modules/runner-binaries-syncer/lambdas/runner-binaries-syncer" "modules/runners/lambdas/runners" "modules/webhook/lambdas/webhook")
-repoRoot=$(dirname $(dirname $(realpath ${BASH_SOURCE[0]})))
-
-for lambdaDir in ${lambdaSrcDirs[@]}; do
-    cd "$repoRoot/${lambdaDir}"
-    docker build -t lambda -f ../../../../.ci/Dockerfile .
-    docker create --name lambda lambda
-    zipName=$(basename "$PWD")
-    docker cp lambda:/lambda/${zipName}.zip ${zipName}.zip
-    docker rm lambda
-done
+# NOTE: This build requires docker buildkit integration which was introduced
+#       in Docker v19.03+ and at least 4GB of memory available to the 
+#       docker daemon
+
+set -eou pipefail
+
+TOP_DIR=$(git rev-parse --show-toplevel)
+OUTPUT_DIR=${OUTPUT_DIR:-${TOP_DIR}/lambda_output}
+
+mkdir -p "${OUTPUT_DIR}"
+
+(
+    set -x
+    DOCKER_BUILDKIT=1 docker build \
+        --target=final \
+        --output=type=local,dest="${OUTPUT_DIR}" \
+        -f "${TOP_DIR}/.ci/Dockerfile" \
+        "${TOP_DIR}"
+)

.github/workflows/auto-approve-dependabot.yml

@@ -14,6 +14,6 @@ jobs:
     if: github.actor == 'dependabot[bot]' || github.actor == 'dependabot-preview[bot]'
     runs-on: ubuntu-latest
     steps:
-      - uses: hmarr/auto-approve-action@v2.0.0
+      - uses: hmarr/auto-approve-action@v2.1.0
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/lambda-runner-binaries-syncer.yml

@@ -24,6 +24,8 @@ jobs:
         run: yarn install
       - name: Run linter
         run: yarn lint
+      - name: Run format check
+        run : yarn run format-check
       - name: Run tests
         run: yarn test
       - name: Build distribution

.github/workflows/lambda-runners.yml

@@ -22,6 +22,8 @@ jobs:
         run: yarn install
       - name: Run linter
         run: yarn lint
+      - name: Run format check
+        run : yarn run format-check
       - name: Run tests
         run: yarn test
       - name: Build distribution

.github/workflows/lambda-webhook.yml

@@ -22,6 +22,8 @@ jobs:
         run: yarn install
       - name: Run linter
         run: yarn lint
+      - name: Run format check
+        run : yarn run format-check
       - name: Run tests
         run: yarn test
       - name: Build distribution

.gitignore

@@ -11,12 +11,12 @@
 .idea
 .DS_Store
 *.out
-example/*.secrets*.tfvars
+secrets.auto.tfvars
 .envrc
 *.zip
 *.gz
 *.tgz
 *.env
 .vscode
 
-**/coverage/*
+**/coverage/*

.release/yarn.lock

@@ -1022,7 +1022,7 @@ debug@^3.1.0:
   dependencies:
     ms "^2.1.1"
 
-debuglog@*, debuglog@^1.0.1:
+debuglog@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
   integrity sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=
@@ -1865,7 +1865,7 @@ import-lazy@^2.1.0:
   resolved "https://registry.yarnpkg.com/import-lazy/-/import-lazy-2.1.0.tgz#05698e3d45c88e8d7e9d92cb0584e77f096f3e43"
   integrity sha1-BWmOPUXIjo1+nZLLBYTnfwlvPkM=
 
-imurmurhash@*, imurmurhash@^0.1.4:
+imurmurhash@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea"
   integrity sha1-khi5srkoojixPcT7a21XbyMUU+o=
@@ -2424,11 +2424,6 @@ lockfile@^1.0.4:
   dependencies:
     signal-exit "^3.0.2"
 
-lodash._baseindexof@*:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/lodash._baseindexof/-/lodash._baseindexof-3.1.0.tgz#fe52b53a1c6761e42618d654e4a25789ed61822c"
-  integrity sha1-/lK1OhxnYeQmGNZU5KJXie1hgiw=
-
 lodash._baseuniq@~4.6.0:
   version "4.6.0"
   resolved "https://registry.yarnpkg.com/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz#0ebb44e456814af7905c6212fa2c9b2d51b841e8"
@@ -2437,33 +2432,11 @@ lodash._baseuniq@~4.6.0:
     lodash._createset "~4.0.0"
     lodash._root "~3.0.0"
 
-lodash._bindcallback@*:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/lodash._bindcallback/-/lodash._bindcallback-3.0.1.tgz#e531c27644cf8b57a99e17ed95b35c748789392e"
-  integrity sha1-5THCdkTPi1epnhftlbNcdIeJOS4=
-
-lodash._cacheindexof@*:
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/lodash._cacheindexof/-/lodash._cacheindexof-3.0.2.tgz#3dc69ac82498d2ee5e3ce56091bafd2adc7bde92"
-  integrity sha1-PcaayCSY0u5ePOVgkbr9Ktx73pI=
-
-lodash._createcache@*:
-  version "3.1.2"
-  resolved "https://registry.yarnpkg.com/lodash._createcache/-/lodash._createcache-3.1.2.tgz#56d6a064017625e79ebca6b8018e17440bdcf093"
-  integrity sha1-VtagZAF2JeeevKa4AY4XRAvc8JM=
-  dependencies:
-    lodash._getnative "^3.0.0"
-
 lodash._createset@~4.0.0:
   version "4.0.3"
   resolved "https://registry.yarnpkg.com/lodash._createset/-/lodash._createset-4.0.3.tgz#0f4659fbb09d75194fa9e2b88a6644d363c9fe26"
   integrity sha1-D0ZZ+7CddRlPqeK4imZE02PJ/iY=
 
-lodash._getnative@*, lodash._getnative@^3.0.0:
-  version "3.9.1"
-  resolved "https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5"
-  integrity sha1-VwvH3t5G1hzc3mh9ZdPuy6o6r/U=
-
 lodash._root@~3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/lodash._root/-/lodash._root-3.0.1.tgz#fba1c4524c19ee9a5f8136b4609f017cf4ded692"
@@ -2499,11 +2472,6 @@ lodash.isstring@^4.0.1:
   resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451"
   integrity sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=
 
-lodash.restparam@*:
-  version "3.6.1"
-  resolved "https://registry.yarnpkg.com/lodash.restparam/-/lodash.restparam-3.6.1.tgz#936a4e309ef330a7645ed4145986c85ae5b20805"
-  integrity sha1-k2pOMJ7zMKdkXtQUWYbIWuWyCAU=
-
 lodash.toarray@^4.4.0:
   version "4.4.0"
   resolved "https://registry.yarnpkg.com/lodash.toarray/-/lodash.toarray-4.4.0.tgz#24c4bfcd6b2fba38bfd0594db1179d8e9b656561"
@@ -4121,9 +4089,9 @@ sshpk@^1.7.0:
     tweetnacl "~0.14.0"
 
 ssri@^6.0.0, ssri@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/ssri/-/ssri-6.0.1.tgz#2a3c41b28dd45b62b63676ecb74001265ae9edd8"
-  integrity sha512-3Wge10hNcT1Kur4PDFwEieXSCMCJs/7WvSACcrMYrNp+b8kDL1/0wJch5Ni2WrtwEa2IO8OsVfeKIciKCDx/QA==
+  version "6.0.2"
+  resolved "https://registry.yarnpkg.com/ssri/-/ssri-6.0.2.tgz#157939134f20464e7301ddba3e90ffa8f7728ac5"
+  integrity sha512-cepbSq/neFK7xB6A50KHN0xHDotYzq58wWCa5LeWqnPrHG8GzfEjO/4O8kpmcGW+oaxkvhEJCWgbgNk4/ZV93Q==
   dependencies:
     figgy-pudding "^3.5.1"
 

README.md

@@ -1,8 +1,7 @@
-[![awesome-runners](https://img.shields.io/badge/listed%20on-awesome--runners-blue.svg)](https://github.com/jonico/awesome-runners)[![Terraform registry](https://img.shields.io/github/v/release/philips-labs/terraform-aws-github-runner?label=Terraform%20Registry)](https://registry.terraform.io/modules/philips-labs/github-runner/aws/) ![Terraform checks](https://github.com/philips-labs/terraform-aws-github-runner/workflows/Terraform%20root%20module%20checks/badge.svg) ![Lambda Webhook](https://github.com/philips-labs/terraform-aws-github-runner/workflows/Lambda%20Agent%20Webhook/badge.svg) ![Lambda Runners](https://github.com/philips-labs/terraform-aws-github-runner/workflows/Lambda%20Runners/badge.svg) ![Lambda Syncer](https://github.com/philips-labs/terraform-aws-github-runner/workflows/Lambda%20Runner%20Binaries%20Syncer/badge.svg)
-
-
 # Terraform module for scalable self hosted GitHub action runners <!-- omit in toc -->
 
+[![awesome-runners](https://img.shields.io/badge/listed%20on-awesome--runners-blue.svg)](https://github.com/jonico/awesome-runners)[![Terraform registry](https://img.shields.io/github/v/release/philips-labs/terraform-aws-github-runner?label=Terraform%20Registry)](https://registry.terraform.io/modules/philips-labs/github-runner/aws/) ![Terraform checks](https://github.com/philips-labs/terraform-aws-github-runner/workflows/Terraform%20root%20module%20checks/badge.svg) ![Lambda Webhook](https://github.com/philips-labs/terraform-aws-github-runner/workflows/Lambda%20Agent%20Webhook/badge.svg) ![Lambda Runners](https://github.com/philips-labs/terraform-aws-github-runner/workflows/Lambda%20Runners/badge.svg) ![Lambda Syncer](https://github.com/philips-labs/terraform-aws-github-runner/workflows/Lambda%20Runner%20Binaries%20Syncer/badge.svg)
+
 This [Terraform](https://www.terraform.io/) module creates the required infrastructure needed to host [GitHub Actions](https://github.com/features/actions) self hosted, auto scaling runners on [AWS spot instances](https://aws.amazon.com/ec2/spot/). It provides the required logic to handle the life cycle for scaling up and down using a set of AWS Lambda functions. Runners are scaled down to zero to avoid costs when no workflows are active.
 
 - [Motivation](#motivation)
@@ -75,32 +74,50 @@ Examples are provided in [the example directory](examples/). Please ensure you h
 - AWS cli (optional)
 - Node and yarn (for lambda development).
 
-The module supports two main scenarios for creating runners. On repository level a runner will be dedicated to only one repository, no other repository can use the runner. On organization level you can use the runner(s) for all the repositories within the organization. See https://help.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners for more information. Before starting the deployment you have to choose one option.
+The module supports two main scenarios for creating runners. On repository level a runner will be dedicated to only one repository, no other repository can use the runner. On organization level you can use the runner(s) for all the repositories within the organization. See [GitHub instructions](https://help.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners) for more information. Before starting the deployment you have to choose one option.
+
+GitHub workflows fail immediately if there is no action runner available for your builds. Since this module supports scaling down to zero, builds will fail in case there is no active runner available. We recommend to create an offline runner with matching labels to the configuration. Create this runner manually by following the [GitHub instructions](https://help.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners) for adding a new runner on your local machine. If you stop the process after the step of running the `config.sh` script the runner will remain offline. This offline runner ensures that builds will not fail immediately and stay queued until there is an EC2 runner to pick it up.
+
+Another convenient way of deploying this temporary required runner is using following approach. This automates all the manual labor.
+
+<details>
+  <summary>Temporary runner using Docker</summary>
+
+  ```bash
+  docker run -it --name my-runner \
+      -e RUNNER_LABELS=selfhosted,Linux,Ubuntu -e RUNNER_NAME=my-repo-docker-runner \
+      -e GITHUB_ACCESS_TOKEN=$GH_PERSONAL_ACCESS_TOKEN \
+      -e RUNNER_REPOSITORY_URL=https://github.com/my-org/my-repo \
+      -v /var/run/docker.sock:/var/run/docker.sock \
+      tcardonne/github-runner:ubuntu-20.04
+  ```
+
+</details>
 
-GitHub workflows fail immediately if there is no action runner available for your builds. Since this module supports scaling down to zero, builds will fail in case there is no active runner available. We recommend to create an offline runner with matching labels to the configuration. Create this runner manually by following the GitHub instructions for adding a new runner on your local machine. If you stop the process after the step of running the `config.sh` script the runner will remain offline. This offline runner ensures that builds will not fail immediately and stay queued until there is an EC2 runner to pick it up.
+You should stop and remove the container once the runner is registered as the builds would otherwise go to your local Docker container.
 
 The setup consists of running Terraform to create all AWS resources and manually configuring the GitHub App. The Terraform module requires configuration from the GitHub App and the GitHub app requires output from Terraform. Therefore you first create the GitHub App and configure the basics, then run Terraform, and afterwards finalize the configuration of the GitHub App.
 
 ### Setup GitHub App (part 1)
 
-Go to GitHub and create a new app. Beware you can create apps your organization or for a user. For now we support only organization level apps.
+Go to GitHub and [create a new app](https://docs.github.com/en/developers/apps/creating-a-github-app). Beware you can create apps your organization or for a user. For now we support only organization level apps.
 
 1. Create app in Github
 2. Choose a name
 3. Choose a website (mandatory, not required for the module).
 4. Disable the webhook for now (we will configure this later).
 5. Permissions for all runners:
-  - Repository:
-    - `Actions`: Read-only (check for queued jobs)
-    - `Checks`: Read-only (receive events for new builds)
-    - `Metadata`: Read-only (default/required)
+    - Repository:
+      - `Actions`: Read-only (check for queued jobs)
+      - `Checks`: Read-only (receive events for new builds)
+      - `Metadata`: Read-only (default/required)
 6. _Permissions for repo level runners only_:
-  - Repository:
-    - `Administration`: Read & write (to register runner)
+   - Repository:
+     - `Administration`: Read & write (to register runner)
 7. _Permissions for organization level runners only_:
-  - Organization
-    - `Administration`: Read & write (to register runner)
-    - `Self-hosted runners`: Read & write (to register runner)
+   - Organization
+     - `Administration`: Read & write (to register runner)
+     - `Self-hosted runners`: Read & write (to register runner)
 8. Save the new app.
 9. On the General page, make a note of the "App ID" and "Client ID" parameters.
 10. Create a new client secret and also write it down.
@@ -120,7 +137,7 @@ For local development you can build all the lambdas at once using `.ci/build.sh`
 
 #### Service-linked role <!-- omit in toc -->
 
-To create spot instances the `AWSServiceRoleForEC2Spot` role needs to be added to your account. You can do that manually by following the [AWS docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-requests.html#service-linked-roles-spot-instance-requests). To use terraform for creating the role, either add the following resource or let the module manage the the service linked role by setting `create_service_linked_role` to `true`. Be aware this is an account global role, so maybe you don't want to mange it via a specific deployment.
+To create spot instances the `AWSServiceRoleForEC2Spot` role needs to be added to your account. You can do that manually by following the [AWS docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-requests.html#service-linked-roles-spot-instance-requests). To use terraform for creating the role, either add the following resource or let the module manage the the service linked role by setting `create_service_linked_role_spot` to `true`. Be aware this is an account global role, so maybe you don't want to manage it via a specific deployment.
 
 ```hcl
 resource "aws_iam_service_linked_role" "spot" {
@@ -180,7 +197,7 @@ Go back to the GitHub App and update the following settings.
 1. Enable the webhook.
 2. Provide the webhook url, should be part of the output of terraform.
 3. Provide the webhook secret.
-4. Enable the `check_run` event for the webhook.
+4. In the "Permissions & Events" section and then "Subscribe to Events" subsection, check "Check Run".
 5. In the "Install App" section, install the App in your organization, either in all or in selected repositories.
 
 You are now ready to run action workloads on self hosted runner. Remember that builds will fail if there is no (offline) runner available with matching labels.
@@ -197,7 +214,7 @@ This is the default, no additional configuration is required.
 
 You have to create an configure you KMS key. The module will use the context with key: `Environment` and value `var.environment` as encryption context.
 
-```HCL
+```hcl
 resource "aws_kms_key" "github" {
   is_enabled = true
 }
@@ -231,7 +248,7 @@ idle_config = [{
 
 Cron expressions are parsed by [cron-parser](https://github.com/harrisiirak/cron-parser#readme). The supported syntax.
 
-```
+```bash
 *    *    *    *    *    *
 ┬    ┬    ┬    ┬    ┬    ┬
 │    │    │    │    │    |
@@ -328,7 +345,7 @@ No requirements.
 | lambda\_subnet\_ids | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
 | logging\_retention\_in\_days | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no |
 | manage\_kms\_key | Let the module manage the KMS key. | `bool` | `true` | no |
-| market\_options | Set it to null to use on demand runners. | `string` | `"spot"` | no |
+| market\_options | Market options for the action runner instances. Setting the value to `null` let the scaler create on-demand instances instead of spot instances. | `string` | `"spot"` | no |
 | minimum\_running\_time\_in\_minutes | The time an ec2 action runner should be running at minimum before terminated if non busy. | `number` | `5` | no |
 | role\_path | The path that will be added to role path for created roles, if not set the environment name will be used. | `string` | `null` | no |
 | role\_permissions\_boundary | Permissions boundary that will be added to the created roles. | `string` | `null` | no |
@@ -379,7 +396,7 @@ We welcome contribution, please checkout the [contribution guide](CONTRIBUTING.m
 
 This module is part of the Philips Forest.
 
-```
+```bash
 
                                                      ___                   _
                                                     / __\__  _ __ ___  ___| |_
@@ -394,7 +411,3 @@ This module is part of the Philips Forest.
 Talk to the forestkeepers in the `forest`-channel on Slack.
 
 [![Slack](https://philips-software-slackin.now.sh/badge.svg)](https://philips-software-slackin.now.sh)
-
-```
-
-```

examples/ubuntu/main.tf

@@ -51,10 +51,20 @@ module "runners" {
 
   runner_log_files = [
     {
+      "log_group_name" : "syslog",
+      "prefix_log_group" : true,
+      "file_path" : "/var/log/syslog",
+      "log_stream_name" : "{instance_id}"
+    },
+    {
+      "log_group_name" : "user_data",
+      "prefix_log_group" : true,
       "file_path" : "/var/log/user-data.log",
       "log_stream_name" : "{instance_id}/user_data"
     },
     {
+      "log_group_name" : "runner",
+      "prefix_log_group" : true,
       "file_path" : "/home/runners/actions-runner/_diag/Runner_**.log",
       "log_stream_name" : "{instance_id}/runner"
     }

examples/ubuntu/providers.tf

@@ -12,6 +12,11 @@ terraform {
 
 provider "aws" {
   region = local.aws_region
+
+  // If you use roles with specific permissons please add your role
+  // assume_role {
+  //   role_arn = "arn:aws:iam::123456789012:role/MyAdminRole"
+  // }
 }
 
 provider "random" {