From 2f836bfed72f83230c3989269c7ae8344ada37be Mon Sep 17 00:00:00 2001 From: Conor Eby Date: Wed, 19 Apr 2023 11:50:20 -0500 Subject: [PATCH 1/4] feat: Add option to read Gitlab Runner Registration from SSM by default --- locals.tf | 7 ++++--- main.tf | 2 +- template/gitlab-runner.tftpl | 9 ++++++++- variables.tf | 6 ++++++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/locals.tf b/locals.tf index d7126f554..58071bc0c 100644 --- a/locals.tf +++ b/locals.tf @@ -55,9 +55,10 @@ locals { # Ensure max builds is optional runners_max_builds_string = var.runners_max_builds == 0 ? "" : format("MaxBuilds = %d", var.runners_max_builds) - # Define key for runner token for SSM - secure_parameter_store_runner_token_key = "${var.environment}-${var.secure_parameter_store_runner_token_key}" - secure_parameter_store_runner_sentry_dsn = "${var.environment}-${var.secure_parameter_store_runner_sentry_dsn}" + # Define key for runner and registration token for SSM + secure_parameter_store_registration_token_key = "${var.environment}-${var.secure_parameter_store_registration_token_key}" + secure_parameter_store_runner_token_key = "${var.environment}-${var.secure_parameter_store_runner_token_key}" + secure_parameter_store_runner_sentry_dsn = "${var.environment}-${var.secure_parameter_store_runner_sentry_dsn}" # Custom names for runner agent instance, security groups, and IAM objects name_runner_agent_instance = var.overrides["name_runner_agent_instance"] == "" ? local.tags["Name"] : var.overrides["name_runner_agent_instance"] diff --git a/main.tf b/main.tf index 609352f46..a92f3a654 100644 --- a/main.tf +++ b/main.tf @@ -69,7 +69,7 @@ locals { secure_parameter_store_runner_token_key = local.secure_parameter_store_runner_token_key secure_parameter_store_runner_sentry_dsn = local.secure_parameter_store_runner_sentry_dsn secure_parameter_store_region = var.aws_region - gitlab_runner_registration_token = var.gitlab_runner_registration_config["registration_token"] + gitlab_runner_registration_token = lookup(var.gitlab_runner_registration_config, "registration_token", "__GITLAB_REGISTRATION_TOKEN_FROM_SSM__") gitlab_runner_description = var.gitlab_runner_registration_config["description"] gitlab_runner_tag_list = var.gitlab_runner_registration_config["tag_list"] gitlab_runner_locked_to_project = var.gitlab_runner_registration_config["locked_to_project"] diff --git a/template/gitlab-runner.tftpl b/template/gitlab-runner.tftpl index 5ab812986..8ea3a4e11 100644 --- a/template/gitlab-runner.tftpl +++ b/template/gitlab-runner.tftpl @@ -32,10 +32,17 @@ then [[ "$valid_token_response" != "200" ]] && valid_token=false fi +gitlab_runner_registration_token=${gitlab_runner_registration_token} +# fetch registration token from SSM +if [[ "$gitlab_runner_registration_token" == "__GITLAB_REGISTRATION_TOKEN_FROM_SSM__" ]] +then + gitlab_runner_registration_token=$(aws ssm get-parameters --names "${secure_parameter_store_registration_token_key}" --with-decryption --region "${secure_parameter_store_region}" | jq -r ".Parameters | .[0] | .Value") +fi + if [[ "${runners_token}" == "__REPLACED_BY_USER_DATA__" && "$token" == "null" ]] || [[ "$valid_token" == "false" ]] then token=$(curl ${curl_cacert} --request POST -L "${runners_gitlab_url}/api/v4/runners" \ - --form "token=${gitlab_runner_registration_token}" \ + --form "token=$gitlab_runner_registration_token" \ --form "tag_list=${gitlab_runner_tag_list}" \ --form "description=${gitlab_runner_description}" \ --form "locked=${gitlab_runner_locked_to_project}" \ diff --git a/variables.tf b/variables.tf index 42176f76c..1dc04da17 100644 --- a/variables.tf +++ b/variables.tf @@ -615,6 +615,12 @@ variable "gitlab_runner_registration_config" { } } +variable "secure_parameter_store_registration_token_key" { + description = "The key name used store the Gitlab runner registration token in Secure Parameter Store" + type = string + default = "registration-token" +} + variable "secure_parameter_store_runner_token_key" { description = "The key name used store the Gitlab runner token in Secure Parameter Store" type = string From 2c7fa7387aa144871c028be2f468426671ced1ea Mon Sep 17 00:00:00 2001 From: Conor Eby Date: Mon, 1 May 2023 11:26:32 -0500 Subject: [PATCH 2/4] Implement CR feedback --- locals.tf | 10 ++++--- main.tf | 55 ++++++++++++++++++------------------ template/gitlab-runner.tftpl | 2 +- variables.tf | 6 ++-- 4 files changed, 38 insertions(+), 35 deletions(-) diff --git a/locals.tf b/locals.tf index 58071bc0c..539e55d5a 100644 --- a/locals.tf +++ b/locals.tf @@ -55,10 +55,12 @@ locals { # Ensure max builds is optional runners_max_builds_string = var.runners_max_builds == 0 ? "" : format("MaxBuilds = %d", var.runners_max_builds) - # Define key for runner and registration token for SSM - secure_parameter_store_registration_token_key = "${var.environment}-${var.secure_parameter_store_registration_token_key}" - secure_parameter_store_runner_token_key = "${var.environment}-${var.secure_parameter_store_runner_token_key}" - secure_parameter_store_runner_sentry_dsn = "${var.environment}-${var.secure_parameter_store_runner_sentry_dsn}" + # + secure_parameter_store_gitlab_runner_registration_token_name = var.secure_parameter_store_gitlab_runner_registration_token_name + + # Define key for runner token for SSM + secure_parameter_store_runner_token_key = "${var.environment}-${var.secure_parameter_store_runner_token_key}" + secure_parameter_store_runner_sentry_dsn = "${var.environment}-${var.secure_parameter_store_runner_sentry_dsn}" # Custom names for runner agent instance, security groups, and IAM objects name_runner_agent_instance = var.overrides["name_runner_agent_instance"] == "" ? local.tags["Name"] : var.overrides["name_runner_agent_instance"] diff --git a/main.tf b/main.tf index a92f3a654..94d2cc9c3 100644 --- a/main.tf +++ b/main.tf @@ -53,33 +53,34 @@ locals { template_gitlab_runner = templatefile("${path.module}/template/gitlab-runner.tftpl", { - gitlab_runner_version = var.gitlab_runner_version - docker_machine_version = var.docker_machine_version - docker_machine_download_url = var.docker_machine_download_url - runners_config = local.template_runner_config - runners_userdata = var.runners_userdata - runners_executor = var.runners_executor - runners_install_amazon_ecr_credential_helper = var.runners_install_amazon_ecr_credential_helper - curl_cacert = length(var.runners_gitlab_certificate) > 0 ? "--cacert /etc/gitlab-runner/certs/gitlab.crt" : "" - pre_install_certificates = local.pre_install_certificates - pre_install = var.userdata_pre_install - post_install = var.userdata_post_install - runners_gitlab_url = var.runners_gitlab_url - runners_token = var.runners_token - secure_parameter_store_runner_token_key = local.secure_parameter_store_runner_token_key - secure_parameter_store_runner_sentry_dsn = local.secure_parameter_store_runner_sentry_dsn - secure_parameter_store_region = var.aws_region - gitlab_runner_registration_token = lookup(var.gitlab_runner_registration_config, "registration_token", "__GITLAB_REGISTRATION_TOKEN_FROM_SSM__") - gitlab_runner_description = var.gitlab_runner_registration_config["description"] - gitlab_runner_tag_list = var.gitlab_runner_registration_config["tag_list"] - gitlab_runner_locked_to_project = var.gitlab_runner_registration_config["locked_to_project"] - gitlab_runner_run_untagged = var.gitlab_runner_registration_config["run_untagged"] - gitlab_runner_maximum_timeout = var.gitlab_runner_registration_config["maximum_timeout"] - gitlab_runner_access_level = lookup(var.gitlab_runner_registration_config, "access_level", "not_protected") - sentry_dsn = var.sentry_dsn - public_key = var.use_fleet == true ? tls_private_key.fleet[0].public_key_openssh : "" - use_fleet = var.use_fleet - private_key = var.use_fleet == true ? tls_private_key.fleet[0].private_key_pem : "" + gitlab_runner_version = var.gitlab_runner_version + docker_machine_version = var.docker_machine_version + docker_machine_download_url = var.docker_machine_download_url + runners_config = local.template_runner_config + runners_userdata = var.runners_userdata + runners_executor = var.runners_executor + runners_install_amazon_ecr_credential_helper = var.runners_install_amazon_ecr_credential_helper + curl_cacert = length(var.runners_gitlab_certificate) > 0 ? "--cacert /etc/gitlab-runner/certs/gitlab.crt" : "" + pre_install_certificates = local.pre_install_certificates + pre_install = var.userdata_pre_install + post_install = var.userdata_post_install + runners_gitlab_url = var.runners_gitlab_url + runners_token = var.runners_token + secure_parameter_store_runner_token_key = local.secure_parameter_store_runner_token_key + secure_parameter_store_runner_sentry_dsn = local.secure_parameter_store_runner_sentry_dsn + secure_parameter_store_gitlab_runner_registration_token_name = local.secure_parameter_store_gitlab_runner_registration_token_name + secure_parameter_store_region = var.aws_region + gitlab_runner_registration_token = lookup(var.gitlab_runner_registration_config, "registration_token", "__GITLAB_REGISTRATION_TOKEN_FROM_SSM__") + gitlab_runner_description = var.gitlab_runner_registration_config["description"] + gitlab_runner_tag_list = var.gitlab_runner_registration_config["tag_list"] + gitlab_runner_locked_to_project = var.gitlab_runner_registration_config["locked_to_project"] + gitlab_runner_run_untagged = var.gitlab_runner_registration_config["run_untagged"] + gitlab_runner_maximum_timeout = var.gitlab_runner_registration_config["maximum_timeout"] + gitlab_runner_access_level = lookup(var.gitlab_runner_registration_config, "access_level", "not_protected") + sentry_dsn = var.sentry_dsn + public_key = var.use_fleet == true ? tls_private_key.fleet[0].public_key_openssh : "" + use_fleet = var.use_fleet + private_key = var.use_fleet == true ? tls_private_key.fleet[0].private_key_pem : "" }) template_runner_config = templatefile("${path.module}/template/runner-config.tftpl", diff --git a/template/gitlab-runner.tftpl b/template/gitlab-runner.tftpl index 8ea3a4e11..390680d4a 100644 --- a/template/gitlab-runner.tftpl +++ b/template/gitlab-runner.tftpl @@ -36,7 +36,7 @@ gitlab_runner_registration_token=${gitlab_runner_registration_token} # fetch registration token from SSM if [[ "$gitlab_runner_registration_token" == "__GITLAB_REGISTRATION_TOKEN_FROM_SSM__" ]] then - gitlab_runner_registration_token=$(aws ssm get-parameters --names "${secure_parameter_store_registration_token_key}" --with-decryption --region "${secure_parameter_store_region}" | jq -r ".Parameters | .[0] | .Value") + gitlab_runner_registration_token=$(aws ssm get-parameter --name "${secure_parameter_store_gitlab_runner_registration_token_name}" --with-decryption --region "${secure_parameter_store_region}" | jq -r ".Parameter | .Value") fi if [[ "${runners_token}" == "__REPLACED_BY_USER_DATA__" && "$token" == "null" ]] || [[ "$valid_token" == "false" ]] diff --git a/variables.tf b/variables.tf index 1dc04da17..5564bfeb0 100644 --- a/variables.tf +++ b/variables.tf @@ -615,10 +615,10 @@ variable "gitlab_runner_registration_config" { } } -variable "secure_parameter_store_registration_token_key" { - description = "The key name used store the Gitlab runner registration token in Secure Parameter Store" +variable "secure_parameter_store_gitlab_runner_registration_token_name" { + description = "The name of the SSM parameter to read the GitLab Runner registration token from." type = string - default = "registration-token" + default = "gitlab-runner-registration-token" } variable "secure_parameter_store_runner_token_key" { From 37496fb44dd93cac7201b29190c0bfb24a880b4a Mon Sep 17 00:00:00 2001 From: Conor Eby Date: Mon, 1 May 2023 14:28:54 -0500 Subject: [PATCH 3/4] Fix comment, update readme --- README.md | 2 ++ locals.tf | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5bb01c1cb..9a1e847ed 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,8 @@ gitlab_runner_registration_config = { access_level = "" } ``` +The registration token can also be read in via SSM parameter store. If no registration token is passed in, the module +will look up the token in the SSM parameter store at the location specified by `secure_parameter_store_gitlab_runner_registration_token_name`. For migration to the new setup simply add the runner token to the parameter store. Once the runner is started it will lookup the required values via the parameter store. If the value is `null` a new runner will be registered and a new token created/stored. diff --git a/locals.tf b/locals.tf index 539e55d5a..11ed70aed 100644 --- a/locals.tf +++ b/locals.tf @@ -55,7 +55,7 @@ locals { # Ensure max builds is optional runners_max_builds_string = var.runners_max_builds == 0 ? "" : format("MaxBuilds = %d", var.runners_max_builds) - # + # Define name for registration token in SSM secure_parameter_store_gitlab_runner_registration_token_name = var.secure_parameter_store_gitlab_runner_registration_token_name # Define key for runner token for SSM From 6ff18fc92db0934b3f378619078b39d6054900c6 Mon Sep 17 00:00:00 2001 From: Matthias Kay Date: Wed, 3 May 2023 08:04:28 +0200 Subject: [PATCH 4/4] remove local variable --- README.md | 13 ++++++++++--- locals.tf | 3 --- main.tf | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index df1bd7ff2..6d55af232 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,7 @@ gitlab_runner_registration_config = { access_level = "" } ``` + The registration token can also be read in via SSM parameter store. If no registration token is passed in, the module will look up the token in the SSM parameter store at the location specified by `secure_parameter_store_gitlab_runner_registration_token_name`. @@ -382,12 +383,18 @@ module "runner" { ### Scenario: Use of Spot Fleet -Since spot instances can be taken over by AWS depending on the instance type and AZ you are using, you may want multiple instances types in multiple AZs. This is where spot fleets come in, when there is no capacity on one instance type and one AZ, AWS will take the next instance type and so on. This update has been possible since the [fork](https://gitlab.com/cki-project/docker-machine/-/tree/v0.16.2-gitlab.19-cki.2) of docker-machine supports spot fleets. +Since spot instances can be taken over by AWS depending on the instance type and AZ you are using, you may want multiple instances +types in multiple AZs. This is where spot fleets come in, when there is no capacity on one instance type and one AZ, AWS will take +the next instance type and so on. This update has been possible since the [fork](https://gitlab.com/cki-project/docker-machine/-/tree/v0.16.2-gitlab.19-cki.2) +of docker-machine supports spot fleets. -We have seen that the [fork](https://gitlab.com/cki-project/docker-machine/-/tree/v0.16.2-gitlab.19-cki.2) of docker-machine this module is using consume more RAM using spot fleets. -For comparison, if you launch 50 machines in the same time, it consumes ~1.2GB of RAM. In our case, we had to change the `instance_type` of the runner from `t3.micro` to `t3.small`. +We have seen that the [fork](https://gitlab.com/cki-project/docker-machine/-/tree/v0.16.2-gitlab.19-cki.2) of docker-machine this +module is using consume more RAM using spot fleets. +For comparison, if you launch 50 machines in the same time, it consumes ~1.2GB of RAM. In our case, we had to change the +`instance_type` of the runner from `t3.micro` to `t3.small`. #### Configuration example + ```hcl module "runner" { # https://registry.terraform.io/modules/npalm/gitlab-runner/aws/ diff --git a/locals.tf b/locals.tf index 11ed70aed..d7126f554 100644 --- a/locals.tf +++ b/locals.tf @@ -55,9 +55,6 @@ locals { # Ensure max builds is optional runners_max_builds_string = var.runners_max_builds == 0 ? "" : format("MaxBuilds = %d", var.runners_max_builds) - # Define name for registration token in SSM - secure_parameter_store_gitlab_runner_registration_token_name = var.secure_parameter_store_gitlab_runner_registration_token_name - # Define key for runner token for SSM secure_parameter_store_runner_token_key = "${var.environment}-${var.secure_parameter_store_runner_token_key}" secure_parameter_store_runner_sentry_dsn = "${var.environment}-${var.secure_parameter_store_runner_sentry_dsn}" diff --git a/main.tf b/main.tf index 94d2cc9c3..5f9ed1bae 100644 --- a/main.tf +++ b/main.tf @@ -68,7 +68,7 @@ locals { runners_token = var.runners_token secure_parameter_store_runner_token_key = local.secure_parameter_store_runner_token_key secure_parameter_store_runner_sentry_dsn = local.secure_parameter_store_runner_sentry_dsn - secure_parameter_store_gitlab_runner_registration_token_name = local.secure_parameter_store_gitlab_runner_registration_token_name + secure_parameter_store_gitlab_runner_registration_token_name = var.secure_parameter_store_gitlab_runner_registration_token_name secure_parameter_store_region = var.aws_region gitlab_runner_registration_token = lookup(var.gitlab_runner_registration_config, "registration_token", "__GITLAB_REGISTRATION_TOKEN_FROM_SSM__") gitlab_runner_description = var.gitlab_runner_registration_config["description"]