Skip to content

Commit

Permalink
feat: add option to format logging in JSON for lambdas (philips-labs#…
Browse files Browse the repository at this point in the history
…1228)

* feat(log): Adding Support for JSON logging

* Passing in log_type to other lambdas

* Casting e as Error

* Add log_level variable, fix formatting

* Correcting failed resolution

* Using latest releases for testing

* Passing in log_level to sub-modules

* Update terraform.yml

* Manually removing `node-fetch`

* Moving logger object to new file

* Resolving merge from develop
Wouldn't be a conflict resolve without a miss

* Logging when job is not queued

* Updating handler names
  • Loading branch information
mcaulifn authored Nov 3, 2021
1 parent 04c102c commit a250b96
Show file tree
Hide file tree
Showing 40 changed files with 532 additions and 228 deletions.
9 changes: 3 additions & 6 deletions .github/workflows/terraform.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,11 @@ env:
tf_working_dir: "."
AWS_REGION: eu-west-1
jobs:

verify_module:
name: Verify module
strategy:
matrix:
terraform: [1.0.8]
terraform: [1.0.8]
runs-on: ubuntu-latest
container:
image: hashicorp/terraform:${{ matrix.terraform }}
Expand All @@ -43,15 +42,15 @@ jobs:
strategy:
fail-fast: false
matrix:
terraform: [0.14.1, 0.15.0, 1.0.8]
terraform: [0.14.3, 0.15.5, 1.0.8]
example: ["default", "ubuntu"]
defaults:
run:
working-directory: examples/${{ matrix.example }}
runs-on: ubuntu-latest
container:
image: hashicorp/terraform:${{ matrix.terraform }}
steps:
steps:
- uses: actions/checkout@v2
- name: terraform init
run: terraform init -get -backend=false -input=false
Expand All @@ -61,5 +60,3 @@ jobs:
continue-on-error: true
- name: validate terraform
run: terraform validate


160 changes: 82 additions & 78 deletions README.md

Large diffs are not rendered by default.

9 changes: 9 additions & 0 deletions main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ module "webhook" {
role_path = var.role_path
role_permissions_boundary = var.role_permissions_boundary
repository_white_list = var.repository_white_list

log_type = var.log_type
log_level = var.log_level
}

module "runners" {
Expand Down Expand Up @@ -133,6 +136,9 @@ module "runners" {
ghes_ssl_verify = var.ghes_ssl_verify

kms_key_arn = var.kms_key_arn

log_type = var.log_type
log_level = var.log_level
}

module "runner_binaries" {
Expand All @@ -156,6 +162,9 @@ module "runner_binaries" {

role_path = var.role_path
role_permissions_boundary = var.role_permissions_boundary

log_type = var.log_type
log_level = var.log_level
}

resource "aws_resourcegroups_group" "resourcegroups_group" {
Expand Down
36 changes: 19 additions & 17 deletions modules/runner-binaries-syncer/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,23 +75,25 @@ No modules.

| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| <a name="input_aws_region"></a> [aws\_region](#input\_aws\_region) | AWS region. | `string` | n/a | yes |
| <a name="input_distribution_bucket_name"></a> [distribution\_bucket\_name](#input\_distribution\_bucket\_name) | Bucket for storing the action runner distribution. | `string` | n/a | yes |
| <a name="input_environment"></a> [environment](#input\_environment) | A name that identifies the environment, used as prefix and for tagging. | `string` | n/a | yes |
| <a name="input_lambda_s3_bucket"></a> [lambda\_s3\_bucket](#input\_lambda\_s3\_bucket) | S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly. | `any` | `null` | no |
| <a name="input_lambda_schedule_expression"></a> [lambda\_schedule\_expression](#input\_lambda\_schedule\_expression) | Scheduler expression for action runner binary syncer. | `string` | `"cron(27 * * * ? *)"` | no |
| <a name="input_lambda_security_group_ids"></a> [lambda\_security\_group\_ids](#input\_lambda\_security\_group\_ids) | List of security group IDs associated with the Lambda function. | `list(string)` | `[]` | no |
| <a name="input_lambda_subnet_ids"></a> [lambda\_subnet\_ids](#input\_lambda\_subnet\_ids) | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
| <a name="input_lambda_timeout"></a> [lambda\_timeout](#input\_lambda\_timeout) | Time out of the lambda in seconds. | `number` | `300` | no |
| <a name="input_lambda_zip"></a> [lambda\_zip](#input\_lambda\_zip) | File location of the lambda zip file. | `string` | `null` | no |
| <a name="input_logging_retention_in_days"></a> [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `7` | no |
| <a name="input_role_path"></a> [role\_path](#input\_role\_path) | The path that will be added to the role, if not set the environment name will be used. | `string` | `null` | no |
| <a name="input_role_permissions_boundary"></a> [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no |
| <a name="input_runner_allow_prerelease_binaries"></a> [runner\_allow\_prerelease\_binaries](#input\_runner\_allow\_prerelease\_binaries) | Allow the runners to update to prerelease binaries. | `bool` | `false` | no |
| <a name="input_runner_architecture"></a> [runner\_architecture](#input\_runner\_architecture) | The platform architecture for the runner instance (x64, arm64), defaults to 'x64' | `string` | `"x64"` | no |
| <a name="input_syncer_lambda_s3_key"></a> [syncer\_lambda\_s3\_key](#input\_syncer\_lambda\_s3\_key) | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no |
| <a name="input_syncer_lambda_s3_object_version"></a> [syncer\_lambda\_s3\_object\_version](#input\_syncer\_lambda\_s3\_object\_version) | S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no |
| <a name="input_tags"></a> [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no |
| aws\_region | AWS region. | `string` | n/a | yes |
| distribution\_bucket\_name | Bucket for storing the action runner distribution. | `string` | n/a | yes |
| environment | A name that identifies the environment, used as prefix and for tagging. | `string` | n/a | yes |
| lambda\_s3\_bucket | S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly. | `any` | `null` | no |
| lambda\_schedule\_expression | Scheduler expression for action runner binary syncer. | `string` | `"cron(27 * * * ? *)"` | no |
| lambda\_security\_group\_ids | List of security group IDs associated with the Lambda function. | `list(string)` | `[]` | no |
| lambda\_subnet\_ids | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
| lambda\_timeout | Time out of the lambda in seconds. | `number` | `300` | no |
| lambda\_zip | File location of the lambda zip file. | `string` | `null` | no |
| log\_level | Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no |
| log\_type | Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. | `string` | `"pretty"` | no |
| logging\_retention\_in\_days | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `7` | no |
| role\_path | The path that will be added to the role, if not set the environment name will be used. | `string` | `null` | no |
| role\_permissions\_boundary | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no |
| runner\_allow\_prerelease\_binaries | Allow the runners to update to prerelease binaries. | `bool` | `false` | no |
| runner\_architecture | The platform architecture for the runner instance (x64, arm64), defaults to 'x64' | `string` | `"x64"` | no |
| syncer\_lambda\_s3\_key | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no |
| syncer\_lambda\_s3\_object\_version | S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no |
| tags | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no |

## Outputs

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
"typescript": "^4.4.4"
},
"dependencies": {
"tslog": "^3.2.2",
"axios": "^0.24.0"
}
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
import { handle } from './syncer/handler';
import { logger } from './syncer/logger';

// eslint-disable-next-line
export const handler = async (event: any, context: any, callback: any): Promise<void> => {
logger.setSettings({ requestId: context.awsRequestId });
logger.debug(JSON.stringify(event));
try {
await handle();
callback(null);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ import { PassThrough } from 'stream';
import { S3 } from 'aws-sdk';
import AWS from 'aws-sdk';
import axios from 'axios';
import { logger as rootLogger } from './logger';

const logger = rootLogger.getChildLogger();

const versionKey = 'name';

Expand All @@ -22,7 +25,7 @@ async function getCachedVersion(s3: S3, cacheObject: CacheObject): Promise<strin
const versions = objectTagging.TagSet?.filter((t: S3.Tag) => t.Key === versionKey);
return versions.length === 1 ? versions[0].Value : undefined;
} catch (e) {
console.debug('No tags found');
logger.debug('No tags found');
return undefined;
}
}
Expand Down Expand Up @@ -73,7 +76,7 @@ async function uploadToS3(s3: S3, cacheObject: CacheObject, actionRunnerReleaseA
})
.promise();

console.debug('Start downloading %s and uploading to S3.', actionRunnerReleaseAsset.name);
logger.debug('Start downloading %s and uploading to S3.', actionRunnerReleaseAsset.name);

const readPromise = new Promise<void>((resolve, reject) => {
axios
Expand All @@ -93,9 +96,9 @@ async function uploadToS3(s3: S3, cacheObject: CacheObject, actionRunnerReleaseA
});

await Promise.all([readPromise, writePromise])
.then(() => console.info(`The new distribution is uploaded to S3.`))
.then(() => logger.info(`The new distribution is uploaded to S3.`))
.catch((error) => {
console.error(`Uploading of the new distribution to S3 failed: ${error}`);
logger.error(`Uploading of the new distribution to S3 failed: ${error}`);
throw error;
});
}
Expand All @@ -120,10 +123,10 @@ export const handle = async (): Promise<void> => {
}

const currentVersion = await getCachedVersion(s3, cacheObject);
console.debug('latest: ' + currentVersion);
logger.debug('latest: ' + currentVersion);
if (currentVersion === undefined || currentVersion != actionRunnerReleaseAsset.name) {
uploadToS3(s3, cacheObject, actionRunnerReleaseAsset);
} else {
console.debug('Distribution is up-to-date, no action.');
logger.debug('Distribution is up-to-date, no action.');
}
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import { Logger } from 'tslog';

export const logger = new Logger({
colorizePrettyLogs: false,
displayInstanceName: false,
minLevel: process.env.LOG_LEVEL || 'info',
name: 'runner-binaries-syncer',
overwriteConsole: true,
type: process.env.LOG_TYPE || 'pretty',
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
declare namespace NodeJS {
export interface ProcessEnv {
LOG_LEVEL: 'silly' | 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal';
LOG_TYPE: 'json' | 'pretty' | 'hidden';
}
}
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
{
"compilerOptions": {
/* Basic Options */
"target": "es6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */,
"target": "es2019" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */,
"module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */,
"outDir": "build",
"lib": ["es2020", "DOM"] /* Specify library files to be included in the compilation. */,
"lib": [
"es2020",
"DOM"
] /* Specify library files to be included in the compilation. */,
"allowJs": true /* Allow javascript files to be compiled. */,
// "checkJs": true, /* Report errors in .js files. */
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
Expand Down Expand Up @@ -54,7 +57,10 @@
/* Experimental Options */
"experimentalDecorators": true /* Enables experimental support for ES7 decorators. */,
"emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */,
"resolveJsonModule": true
"resolveJsonModule": true,
"sourceMap": true
},
"include": ["src/**/*"]
"include": [
"src/**/*"
]
}
Original file line number Diff line number Diff line change
Expand Up @@ -3148,10 +3148,10 @@ source-map-support@^0.5.12, source-map-support@^0.5.17:
buffer-from "^1.0.0"
source-map "^0.6.0"

source-map-support@^0.5.6:
source-map-support@^0.5.19, source-map-support@^0.5.6:
version "0.5.20"
resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.20.tgz#12166089f8f5e5e8c56926b377633392dd2cb6c9"
integrity sha512-n1lZZ8Ve4ksRqizaBQgxXDgKwttHDhyfQjA6YZZn8+AroHbsIz+JjwxQDxbp+7y5OYCI8t1Yk7etjD9CRd2hIw==
integrity sha1-EhZgifj15ejFaSazd2Mzkt0stsk=
dependencies:
buffer-from "^1.0.0"
source-map "^0.6.0"
Expand Down Expand Up @@ -3421,6 +3421,13 @@ tslib@^1.8.1:
resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==

tslog@^3.2.2:
version "3.2.2"
resolved "https://registry.yarnpkg.com/tslog/-/tslog-3.2.2.tgz#5bbaa1fab685c4273e59b38064227321a69a0694"
integrity sha1-W7qh+raFxCc+WbOAZCJzIaaaBpQ=
dependencies:
source-map-support "^0.5.19"

tsutils@^3.21.0:
version "3.21.0"
resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623"
Expand Down
6 changes: 4 additions & 2 deletions modules/runner-binaries-syncer/runner-binaries-syncer.tf
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@ resource "aws_lambda_function" "syncer" {

environment {
variables = {
S3_BUCKET_NAME = aws_s3_bucket.action_dist.id
S3_OBJECT_KEY = local.action_runner_distribution_object_key
GITHUB_RUNNER_ARCHITECTURE = var.runner_architecture
GITHUB_RUNNER_ALLOW_PRERELEASE_BINARIES = var.runner_allow_prerelease_binaries
LOG_LEVEL = var.log_level
LOG_TYPE = var.log_type
S3_BUCKET_NAME = aws_s3_bucket.action_dist.id
S3_OBJECT_KEY = local.action_runner_distribution_object_key
}
}
dynamic "vpc_config" {
Expand Down
32 changes: 32 additions & 0 deletions modules/runner-binaries-syncer/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -98,3 +98,35 @@ variable "lambda_security_group_ids" {
type = list(string)
default = []
}

variable "log_type" {
description = "Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. "
type = string
default = "pretty"
validation {
condition = anytrue([
var.log_type == "json",
var.log_type == "pretty",
var.log_type == "hidden",
])
error_message = "`log_type` value not valid. Valid values are 'json', 'pretty', 'hidden'."
}
}

variable "log_level" {
description = "Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'."
type = string
default = "info"
validation {
condition = anytrue([
var.log_level == "silly",
var.log_level == "trace",
var.log_level == "debug",
var.log_level == "info",
var.log_level == "warn",
var.log_level == "error",
var.log_level == "fatal",
])
error_message = "`log_level` value not valid. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'."
}
}
2 changes: 2 additions & 0 deletions modules/runners/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,8 @@ No modules.
| <a name="input_lambda_timeout_scale_down"></a> [lambda\_timeout\_scale\_down](#input\_lambda\_timeout\_scale\_down) | Time out for the scale down lambda in seconds. | `number` | `60` | no |
| <a name="input_lambda_timeout_scale_up"></a> [lambda\_timeout\_scale\_up](#input\_lambda\_timeout\_scale\_up) | Time out for the scale up lambda in seconds. | `number` | `60` | no |
| <a name="input_lambda_zip"></a> [lambda\_zip](#input\_lambda\_zip) | File location of the lambda zip file. | `string` | `null` | no |
| <a name="input_log_level"></a> [log\_level](#input\_log\_level) | Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no |
| <a name="input_log_type"></a> [log\_type](#input\_log\_type) | Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. | `string` | `"pretty"` | no |
| <a name="input_logging_retention_in_days"></a> [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no |
| <a name="input_market_options"></a> [market\_options](#input\_market\_options) | Market options for the action runner instances. | `string` | `"spot"` | no |
| <a name="input_minimum_running_time_in_minutes"></a> [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated if non busy. | `number` | `5` | no |
Expand Down
1 change: 1 addition & 0 deletions modules/runners/lambdas/runners/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
"@types/node": "^16.11.6",
"aws-sdk": "^2.1019.0",
"cron-parser": "^4.1.0",
"tslog": "^3.2.2",
"typescript": "^4.4.4"
}
}
28 changes: 16 additions & 12 deletions modules/runners/lambdas/runners/src/lambda.ts
Original file line number Diff line number Diff line change
@@ -1,27 +1,31 @@
import { scaleUp as scaleUpAction } from './scale-runners/scale-up';
import { scaleDown as scaleDownAction } from './scale-runners/scale-down';
import { SQSEvent, ScheduledEvent, Context } from 'aws-lambda';
import { scaleUp } from './scale-runners/scale-up';
import { scaleDown } from './scale-runners/scale-down';
import { SQSEvent, ScheduledEvent, Context, Callback } from 'aws-lambda';
import { logger } from './scale-runners/logger';
import 'source-map-support/register';

export const scaleUp = async (event: SQSEvent, context: Context, callback: any): Promise<void> => {
console.debug(JSON.stringify(event));
export async function scaleUpHandler(event: SQSEvent, context: Context, callback: Callback): Promise<void> {
logger.setSettings({ requestId: context.awsRequestId });
logger.debug(JSON.stringify(event));
try {
for (const e of event.Records) {
await scaleUpAction(e.eventSource, JSON.parse(e.body));
await scaleUp(e.eventSource, JSON.parse(e.body));
}

callback(null);
} catch (e) {
console.error(e);
logger.error(e);
callback('Failed handling SQS event');
}
};
}

export const scaleDown = async (event: ScheduledEvent, context: Context, callback: any): Promise<void> => {
export async function scaleDownHandler(event: ScheduledEvent, context: Context, callback: Callback): Promise<void> {
logger.setSettings({ requestId: context.awsRequestId });
try {
await scaleDownAction();
await scaleDown();
callback(null);
} catch (e) {
console.error(e);
logger.error(e);
callback('Failed');
}
};
}
4 changes: 4 additions & 0 deletions modules/runners/lambdas/runners/src/scale-runners/gh-auth.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ import {
} from '@octokit/auth-app/dist-types/types';
import { OctokitOptions } from '@octokit/core/dist-types/types';
import { getParameterValue } from './ssm';
import { logger as rootLogger } from './logger';

const logger = rootLogger.getChildLogger();

export async function createOctoClient(token: string, ghesApiUrl = ''): Promise<Octokit> {
const ocktokitOptions: OctokitOptions = {
Expand Down Expand Up @@ -52,6 +55,7 @@ async function createAuth(installationId: number | undefined, ghesApiUrl: string
};
if (installationId) authOptions = { ...authOptions, installationId };

logger.debug(`GHES API URL: ${ghesApiUrl}`);
if (ghesApiUrl) {
authOptions.request = request.defaults({
baseUrl: ghesApiUrl,
Expand Down
Loading

0 comments on commit a250b96

Please sign in to comment.