diff --git a/modules/cloud-config-container/simple-nva/README.md b/modules/cloud-config-container/simple-nva/README.md index f70842e8c4..e0800d17e1 100644 --- a/modules/cloud-config-container/simple-nva/README.md +++ b/modules/cloud-config-container/simple-nva/README.md @@ -62,16 +62,88 @@ module "vm" { } # tftest modules=1 resources=1 ``` + +### Example with advanced routing capabilities + +Find below a sample terraform example for bootstrapping a simple NVA powered by [COS](https://cloud.google.com/container-optimized-os/docs) and running [FRRouting](https://frrouting.org/) container. +Please find below a sample frr.conf file based on the documentation available [here](https://docs.frrouting.org/en/latest/basic.html) for hosting a BGP service with ASN 65001 on FRR container establishing a BGP session with a remote neighbor with IP address 10.128.0.2 and ASN 65002. + +``` +# tftest-file id=frr_conf path=./frr.conf +# Example frr.conmf file + +log syslog informational +no ipv6 forwarding +router bgp 65001 + neighbor 10.128.0.2 remote-as 65002 +line vty +``` + +Following code assumes a file in the same folder named frr.conf exists. + +```hcl +locals { + network_interfaces = [ + { + addresses = null + name = "dev" + nat = false + network = "dev_vpc_self_link" + routes = ["10.128.0.0/9"] + subnetwork = "dev_vpc_nva_subnet_self_link" + enable_masquerading = true + non_masq_cidrs = ["10.0.0.0/8"] + }, + { + addresses = null + name = "prod" + nat = false + network = "prod_vpc_self_link" + routes = ["10.0.0.0/9"] + subnetwork = "prod_vpc_nva_subnet_self_link" + } + ] +} + +module "cos-nva" { + source = "./fabric/modules/cloud-config-container/simple-nva" + enable_health_checks = true + network_interfaces = local.network_interfaces + frr_config = { config_file = "./frr.conf", daemons_enabled = ["bgpd"] } + optional_run_cmds = ["ls -l"] +} + +module "vm" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west8-b" + name = "cos-nva" + network_interfaces = local.network_interfaces + metadata = { + user-data = module.cos-nva.cloud_config + google-logging-enabled = true + } + boot_disk = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + tags = ["nva", "ssh"] +} +# tftest modules=1 resources=1 files=frr_conf +``` ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [network_interfaces](variables.tf#L39) | Network interfaces configuration. | list(object({…})) | ✓ | | +| [network_interfaces](variables.tf#L75) | Network interfaces configuration. | list(object({…})) | ✓ | | | [cloud_config](variables.tf#L17) | Cloud config template path. If null default will be used. | string | | null | | [enable_health_checks](variables.tf#L23) | Configures routing to enable responses to health check probes. | bool | | false | | [files](variables.tf#L29) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | +| [frr_config](variables.tf#L39) | FRR configuration for container running on the NVA. | object({…}) | | null | +| [optional_run_cmds](variables.tf#L84) | Optional Cloud Init run commands to execute. | list(string) | | [] | ## Outputs diff --git a/modules/cloud-config-container/simple-nva/cloud-config.yaml b/modules/cloud-config-container/simple-nva/cloud-config.yaml index f1d71e8262..f44cd08e6b 100644 --- a/modules/cloud-config-container/simple-nva/cloud-config.yaml +++ b/modules/cloud-config-container/simple-nva/cloud-config.yaml @@ -1,6 +1,6 @@ #cloud-config -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ write_files: content: | ${indent(6, data.content)} %{ endfor } + - path: /etc/systemd/system/routing.service permissions: 0644 owner: root @@ -34,6 +35,7 @@ write_files: Wants=network-online.target [Service] ExecStart=/bin/sh -c "/var/run/nva/start-routing.sh" + - path: /var/run/nva/start-routing.sh permissions: 0744 owner: root @@ -43,6 +45,12 @@ write_files: %{ if enable_health_checks ~} /var/run/nva/policy_based_routing.sh ${interface.name} %{ endif ~} +%{ if interface.enable_masquerading ~} +%{ for cidr in interface.non_masq_cidrs ~} + iptables -t nat -A POSTROUTING -o ${interface.name} -d ${cidr} -j ACCEPT +%{ endfor ~} + iptables -t nat -A POSTROUTING -o ${interface.name} -j MASQUERADE +%{ endif ~} %{ for route in interface.routes ~} ip route add ${route} via `curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/${interface.number}/gateway -H "Metadata-Flavor:Google"` dev ${interface.name} %{ endfor ~} @@ -55,4 +63,6 @@ runcmd: - systemctl daemon-reload - systemctl enable routing - systemctl start routing - +%{ for cmd in optional_run_cmds ~} + - ${cmd} +%{ endfor ~} diff --git a/modules/cloud-config-container/simple-nva/files/frr/daemons b/modules/cloud-config-container/simple-nva/files/frr/daemons new file mode 100644 index 0000000000..0a388df012 --- /dev/null +++ b/modules/cloud-config-container/simple-nva/files/frr/daemons @@ -0,0 +1,65 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +zebra=${zebra_enabled} +bgpd=${bgpd_enabled} +ospfd=${ospfd_enabled} +ospf6d=${ospf6d_enabled} +ripd=${ripd_enabled} +ripngd=${ripngd_enabled} +isisd=${isisd_enabled} +pimd=${pimd_enabled} +ldpd=${ldpd_enabled} +nhrpd=${nhrpd_enabled} +eigrpd=${eigrpd_enabled} +babeld=${babeld_enabled} +sharpd=${sharpd_enabled} +staticd=${staticd_enabled} +pbrd=${pbrd_enabled} +bfdd=${bfdd_enabled} +fabricd=${fabricd_enabled} + +# If this option is set the /etc/init.d/frr script automatically loads +# the config via "vtysh -b" when the servers are started. +# Check /etc/pam.d/frr if you intend to use "vtysh"! + +vtysh_enable=yes +zebra_options=" -A 127.0.0.1 -s 90000000" +bgpd_options=" -A 127.0.0.1" +ospfd_options=" --daemon -A 127.0.0.1" +ospf6d_options=" --daemon -A ::1" +ripd_options=" --daemon -A 127.0.0.1" +ripngd_options=" --daemon -A ::1" +isisd_options=" --daemon -A 127.0.0.1" +pimd_options=" --daemon -A 127.0.0.1" +ldpd_options=" --daemon -A 127.0.0.1" +nhrpd_options=" --daemon -A 127.0.0.1" +eigrpd_options=" --daemon -A 127.0.0.1" +babeld_options=" --daemon -A 127.0.0.1" +sharpd_options=" --daemon -A 127.0.0.1" +staticd_options=" --daemon -A 127.0.0.1" +pbrd_options=" --daemon -A 127.0.0.1" +bfdd_options=" --daemon -A 127.0.0.1" +fabricd_options=" --daemon -A 127.0.0.1" + +#MAX_FDS=1024 +# The list of daemons to watch is automatically generated by the init script. +#watchfrr_options="" + +# for debugging purposes, you can specify a "wrap" command to start instead +# of starting the daemon directly, e.g. to use valgrind on ospfd: +# ospfd_wrap="/usr/bin/valgrind" +# or you can use "all_wrap" for all daemons, e.g. to use perf record: +# all_wrap="/usr/bin/perf record --call-graph -" +# the normal daemon command is added to this at the end. diff --git a/modules/cloud-config-container/simple-nva/files/frr/frr.service b/modules/cloud-config-container/simple-nva/files/frr/frr.service new file mode 100644 index 0000000000..a560602eb9 --- /dev/null +++ b/modules/cloud-config-container/simple-nva/files/frr/frr.service @@ -0,0 +1,27 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=Start FRR container +After=gcr-online.target docker.socket +Wants=gcr-online.target docker.socket docker-events-collector.service +[Service] +Environment="HOME=/home/frr" +ExecStart=/usr/bin/docker run --rm --name=frr \ +--privileged \ +--network host \ +-v /etc/frr:/etc/frr \ +frrouting/frr +ExecStop=/usr/bin/docker stop frr +ExecStopPost=/usr/bin/docker rm frr diff --git a/modules/cloud-config-container/simple-nva/files/ipprefix_by_netmask.sh b/modules/cloud-config-container/simple-nva/files/ipprefix_by_netmask.sh index a1c69822df..1694382535 100644 --- a/modules/cloud-config-container/simple-nva/files/ipprefix_by_netmask.sh +++ b/modules/cloud-config-container/simple-nva/files/ipprefix_by_netmask.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh index 951396d356..49f3828837 100644 --- a/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh +++ b/modules/cloud-config-container/simple-nva/files/policy_based_routing.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/modules/cloud-config-container/simple-nva/main.tf b/modules/cloud-config-container/simple-nva/main.tf index 4ff0afe29b..110983bf93 100644 --- a/modules/cloud-config-container/simple-nva/main.tf +++ b/modules/cloud-config-container/simple-nva/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,42 +15,109 @@ */ locals { - cloud_config = templatefile(local.template, merge({ - files = local.files - enable_health_checks = var.enable_health_checks - network_interfaces = local.network_interfaces - })) + _files = merge( + { + "/var/run/nva/ipprefix_by_netmask.sh" = { + content = file("${path.module}/files/ipprefix_by_netmask.sh") + owner = "root" + permissions = "0744" + } + "/var/run/nva/policy_based_routing.sh" = { + content = file("${path.module}/files/policy_based_routing.sh") + owner = "root" + permissions = "0744" + } + }, { + for path, attrs in var.files : path => { + content = attrs.content, + owner = attrs.owner, + permissions = attrs.permissions + } + }, + try(var.frr_config != null, false) ? { + "/etc/frr/daemons" = { + content = templatefile("${path.module}/files/frr/daemons", local._frr_daemons_enabled) + owner = "root" + permissions = "0744" + } + "/etc/frr/frr.conf" = { + content = file(var.frr_config.config_file) + owner = "root" + permissions = "0744" + } + "/etc/systemd/system/frr.service" = { + content = file("${path.module}/files/frr/frr.service") + owner = "root" + permissions = "0644" + } + "/var/lib/docker/daemon.json" = { + content = < { - content = attrs.content, - owner = attrs.owner, - permissions = attrs.permissions - } - }) + _frr_daemons = [ + "zebra", + "bgpd", + "ospfd", + "ospf6d", + "ripd", + "ripngd", + "isisd", + "pimd", + "ldpd", + "nhrpd", + "eigrpd", + "babeld", + "sharpd", + "staticd", + "pbrd", + "bfdd", + "fabricd" + ] - network_interfaces = [ + _frr_daemons_enabled = try( + { + for daemon in local._frr_daemons : + "${daemon}_enabled" => contains(var.frr_config.daemons_enabled, daemon) ? "yes" : "no" + }, {}) + + _network_interfaces = [ for index, interface in var.network_interfaces : { - name = "eth${index}" - number = index - routes = interface.routes + name = "eth${index}" + number = index + routes = interface.routes + enable_masquerading = interface.enable_masquerading != null ? interface.enable_masquerading : false + non_masq_cidrs = interface.non_masq_cidrs != null ? interface.non_masq_cidrs : [] } ] - template = ( + _optional_run_cmds = ( + try(var.frr_config != null, false) + ? concat(["systemctl start frr"], var.optional_run_cmds) + : var.optional_run_cmds + ) + + _template = ( var.cloud_config == null ? "${path.module}/cloud-config.yaml" : var.cloud_config ) + + cloud_config = templatefile(local._template, { + enable_health_checks = var.enable_health_checks + files = local._files + network_interfaces = local._network_interfaces + optional_run_cmds = local._optional_run_cmds + }) } diff --git a/modules/cloud-config-container/simple-nva/outputs.tf b/modules/cloud-config-container/simple-nva/outputs.tf index 7d8d41656b..54942c1ad5 100644 --- a/modules/cloud-config-container/simple-nva/outputs.tf +++ b/modules/cloud-config-container/simple-nva/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/cloud-config-container/simple-nva/variables.tf b/modules/cloud-config-container/simple-nva/variables.tf index 39d96d913f..84f62f6989 100644 --- a/modules/cloud-config-container/simple-nva/variables.tf +++ b/modules/cloud-config-container/simple-nva/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,9 +36,53 @@ variable "files" { default = {} } +variable "frr_config" { + description = "FRR configuration for container running on the NVA." + type = object({ + daemons_enabled = optional(list(string)) + config_file = string + }) + default = null + validation { + condition = try(alltrue([ + for daemon in var.frr_config.daemons_enabled : contains([ + "zebra", + "bgpd", + "ospfd", + "ospf6d", + "ripd", + "ripngd", + "isisd", + "pimd", + "ldpd", + "nhrpd", + "eigrpd", + "babeld", + "sharpd", + "staticd", + "pbrd", + "bfdd", + "fabricd" + ], daemon) + ]), true) + error_message = <