diff --git a/gen/k8s/v6test/configmap.yml b/gen/k8s/v6test/configmap.yml new file mode 100644 index 0000000..c946a88 --- /dev/null +++ b/gen/k8s/v6test/configmap.yml @@ -0,0 +1,11 @@ +{ + "apiVersion": "v1", + "data": { + "frr.conf": "!\nfrr version 9.1\nfrr defaults traditional\nservice integrated-vtysh-config\n!\nrouter bgp 64567\n no bgp default ipv4-unicast\n bgp disable-ebgp-connected-route-check\n neighbor v6gw peer-group\n neighbor v6gw remote-as 64566\n neighbor v6gw ebgp-multihop\n neighbor v6gw timers 10 30\n neighbor 10.33.136.182 peer-group v6gw\n neighbor 10.33.159.57 peer-group v6gw\n !\n address-family l2vpn evpn\n neighbor v6gw activate\n neighbor v6gw route-map evpn in\n neighbor v6gw route-map evpn out\n advertise-all-vni\n exit-address-family\nexit\n!\nroute-map evpn permit 10\n match evpn vni 66\nexit\n!\nend\n" + }, + "kind": "ConfigMap", + "metadata": { + "name": "v6test-frr-config" + } +} +--- diff --git a/gen/k8s/v6test/deployment.yml b/gen/k8s/v6test/deployment.yml new file mode 100644 index 0000000..f45ba3d --- /dev/null +++ b/gen/k8s/v6test/deployment.yml @@ -0,0 +1,88 @@ +{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "labels": { + "rubykaigi.org/app": "v6test" + }, + "name": "v6test" + }, + "spec": { + "replicas": 1, + "selector": { + "matchLabels": { + "rubykaigi.org/app": "v6test" + } + }, + "template": { + "metadata": { + "labels": { + "rubykaigi.org/app": "v6test" + } + }, + "spec": { + "containers": [ + { + "command": [ + "/bin/bash", + "-xe", + "-c", + "if ! ip link show dev vxlan-66; then\n underlay_iface=eth0\n local_addr=$(ip -br addr show \"$underlay_iface\" | awk 'match($0, /10\\.33\\.\\d+\\.\\d+/) { print substr($0, RSTART, RLENGTH) }')\n ip link add vxlan-66 type vxlan id 66 local \"$local_addr\" dstport 4789 nolearning dev \"$underlay_iface\"\n ip link add br-66 type bridge stp_state 0\n ip link set up br-66\n ip link set up vxlan-66 master br-66\n ip addr add 2001:df0:8500:ca60::\"$local_addr\"/64 dev br-66\n ip route add default via 2001:df0:8500:ca60::a21:88b6\nfi\n\nln -sf /config/frr/frr.conf /etc/frr/\nsed -i 's|bgpd=no|bgpd=yes|' /etc/frr/daemons\nexec /usr/lib/frr/docker-start\n" + ], + "image": "quay.io/frrouting/frr:9.1.0", + "name": "frr", + "securityContext": { + "capabilities": { + "add": [ + "SYS_ADMIN", + "NET_ADMIN" + ] + } + }, + "volumeMounts": [ + { + "mountPath": "/config/frr", + "name": "frr-config", + "readOnly": false + } + ] + }, + { + "command": [ + "/bin/sleep", + "1d" + ], + "image": "public.ecr.aws/docker/library/debian:stable", + "name": "v6test", + "securityContext": { + "capabilities": { + "add": [ + "NET_ADMIN" + ] + } + } + } + ], + "nodeSelector": { + "rubykaigi.org/node-group": "onpremises" + }, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "dedicated", + "value": "onpremises" + } + ], + "volumes": [ + { + "configMap": { + "name": "v6test-frr-config" + }, + "name": "frr-config" + } + ] + } + } + } +} +--- diff --git a/gen/k8s/v6test/kustomization.yml b/gen/k8s/v6test/kustomization.yml new file mode 100644 index 0000000..215e9bf --- /dev/null +++ b/gen/k8s/v6test/kustomization.yml @@ -0,0 +1,9 @@ +{ + "apiVersion": "kustomize.config.k8s.io/v1beta1", + "kind": "Kustomization", + "namespace": "default", + "resources": [ + "./configmap.yml", + "./deployment.yml" + ] +} diff --git a/k8s/v6test/config/frr.conf b/k8s/v6test/config/frr.conf new file mode 100644 index 0000000..cdbd866 --- /dev/null +++ b/k8s/v6test/config/frr.conf @@ -0,0 +1,28 @@ +! +frr version 9.1 +frr defaults traditional +service integrated-vtysh-config +! +router bgp 64567 + no bgp default ipv4-unicast + bgp disable-ebgp-connected-route-check + neighbor v6gw peer-group + neighbor v6gw remote-as 64566 + neighbor v6gw ebgp-multihop + neighbor v6gw timers 10 30 + neighbor 10.33.136.182 peer-group v6gw + neighbor 10.33.159.57 peer-group v6gw + ! + address-family l2vpn evpn + neighbor v6gw activate + neighbor v6gw route-map evpn in + neighbor v6gw route-map evpn out + advertise-all-vni + exit-address-family +exit +! +route-map evpn permit 10 + match evpn vni 66 +exit +! +end diff --git a/k8s/v6test/configmap.jsonnet b/k8s/v6test/configmap.jsonnet new file mode 100644 index 0000000..f2f09e1 --- /dev/null +++ b/k8s/v6test/configmap.jsonnet @@ -0,0 +1,12 @@ +[ + { + apiVersion: 'v1', + kind: 'ConfigMap', + metadata: { + name: 'v6test-frr-config', + }, + data: { + 'frr.conf': importstr './config/frr.conf', + }, + }, +] diff --git a/k8s/v6test/deployment.jsonnet b/k8s/v6test/deployment.jsonnet new file mode 100644 index 0000000..0a4ffb8 --- /dev/null +++ b/k8s/v6test/deployment.jsonnet @@ -0,0 +1,80 @@ +[ + { + apiVersion: 'apps/v1', + kind: 'Deployment', + metadata: { + name: 'v6test', + labels: { + 'rubykaigi.org/app': 'v6test', + }, + }, + spec: { + replicas: 1, + selector: { + matchLabels: { 'rubykaigi.org/app': 'v6test' }, + }, + template: { + metadata: { + labels: { 'rubykaigi.org/app': 'v6test' }, + }, + spec: { + nodeSelector: { + 'rubykaigi.org/node-group': 'onpremises', + }, + tolerations: [ + { key: 'dedicated', value: 'onpremises', effect: 'NoSchedule' }, + ], + containers: [ + { + name: 'frr', + image: 'quay.io/frrouting/frr:9.1.0', + command: [ + '/bin/bash', + '-xe', + '-c', + ||| + if ! ip link show dev vxlan-66; then + underlay_iface=eth0 + local_addr=$(ip -br addr show "$underlay_iface" | awk 'match($0, /10\.33\.\d+\.\d+/) { print substr($0, RSTART, RLENGTH) }') + ip link add vxlan-66 type vxlan id 66 local "$local_addr" dstport 4789 nolearning dev "$underlay_iface" + ip link add br-66 type bridge stp_state 0 + ip link set up br-66 + ip link set up vxlan-66 master br-66 addrgenmode none + ip addr add 2001:df0:8500:ca60::"$local_addr"/64 dev br-66 + ip route add default via 2001:df0:8500:ca60::a21:88b6 + fi + + ln -sf /config/frr/frr.conf /etc/frr/ + sed -i 's|bgpd=no|bgpd=yes|' /etc/frr/daemons + exec /usr/lib/frr/docker-start + |||, + ], + securityContext: { + capabilities: { add: ['SYS_ADMIN', 'NET_ADMIN'] }, + }, + volumeMounts: [ + { name: 'frr-config', mountPath: '/config/frr', readOnly: false }, + ], + }, + { + name: 'v6test', + image: 'public.ecr.aws/docker/library/debian:stable', + command: ['/bin/sleep', '1d'], + securityContext: { + capabilities: { add: ['NET_ADMIN'] }, + }, + }, + ], + volumes: [ + { + name: 'frr-config', + configMap: { + name: 'v6test-frr-config', + }, + }, + ], + }, + }, + }, + }, +] diff --git a/k8s/v6test/kustomization.jsonnet b/k8s/v6test/kustomization.jsonnet new file mode 100644 index 0000000..4957682 --- /dev/null +++ b/k8s/v6test/kustomization.jsonnet @@ -0,0 +1,9 @@ +{ + apiVersion: 'kustomize.config.k8s.io/v1beta1', + kind: 'Kustomization', + namespace: 'default', + resources: [ + './configmap.yml', + './deployment.yml', + ], +} diff --git a/route53/rubykaigi.net.route b/route53/rubykaigi.net.route index 887fda0..75b403c 100644 --- a/route53/rubykaigi.net.route +++ b/route53/rubykaigi.net.route @@ -46,6 +46,8 @@ template 'rubykaigi.net-common' do ignore 'captioner.apne1.rubykaigi.net' + ignore_under 'v6gw.apne1.rubykaigi.net' + # external-dns ni sitai rrset 'tftp.rubykaigi.net', 'CNAME' do ttl 60 diff --git a/tf/v6gw/.gitignore b/tf/v6gw/.gitignore new file mode 100644 index 0000000..6f66c74 --- /dev/null +++ b/tf/v6gw/.gitignore @@ -0,0 +1 @@ +*.zip \ No newline at end of file diff --git a/tf/v6gw/.terraform.lock.hcl b/tf/v6gw/.terraform.lock.hcl new file mode 100644 index 0000000..67d0d70 --- /dev/null +++ b/tf/v6gw/.terraform.lock.hcl @@ -0,0 +1,99 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.4.2" + hashes = [ + "h1:G4v6F6Lhqlo3EKGBKEK/kJRhNcQiRrhEdUiVpBHKHOA=", + "zh:08faed7c9f42d82bc3d406d0d9d4971e2d1c2d34eae268ad211b8aca57b7f758", + "zh:3564112ed2d097d7e0672378044a69b06642c326f6f1584d81c7cdd32ebf3a08", + "zh:53cd9afd223c15828c1916e68cb728d2be1cbccb9545568d6c2b122d0bac5102", + "zh:5ae4e41e3a1ce9d40b6458218a85bbde44f21723943982bca4a3b8bb7c103670", + "zh:5b65499218b315b96e95c5d3463ea6d7c66245b59461217c99eaa1611891cd2c", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7f45b35a8330bebd184c2545a41782ff58240ed6ba947274d9881dd5da44b02e", + "zh:87e67891033214e55cfead1391d68e6a3bf37993b7607753237e82aa3250bb71", + "zh:de3590d14037ad81fc5cedf7cfa44614a92452d7b39676289b704a962050bc5e", + "zh:e7e6f2ea567f2dbb3baa81c6203be69f9cd6aeeb01204fd93e3cf181e099b610", + "zh:fd24d03c89a7702628c2e5a3c732c0dede56fa75a08da4a1efe17b5f881c88e2", + "zh:febf4b7b5f3ff2adff0573ef6361f09b6638105111644bdebc0e4f575373935f", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.46.0" + constraints = ">= 5.0.0" + hashes = [ + "h1:GK1y1qAGcPHPZxz01Ko6v+815T7kZPXt6POBsLg9c/k=", + "zh:05ae6180a7f23071435f6e5e59c19af0b6c5da42ee600c6c1568c8660214d548", + "zh:0d878d1565d5e57ce6b34ec5f04b28662044a50c999ec5770c374aa1f1020de2", + "zh:25ef1467af2514d8011c44759307445f7057836ff87dfe4503c3e1c9776d5c1a", + "zh:26c006df6200f0063b827aab05bec94f9f3f77848e82ed72e48a51d1170d1961", + "zh:37cdf4292649a10f12858622826925e18ad4eca354c31f61d02c66895eb91274", + "zh:4315b0433c2fc512666c74e989e2d95240934ef370bea1c690d36cb02d30c4ce", + "zh:75df0b3f631b78aeff1832cc77d99b527c2a5e79d40f7aac40bdc4a66124dac2", + "zh:90693d936c9a556d2bf945de4920ff82052002eb73139bd7164fafd02920f0ef", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:c9177ad09804c60fd2ed25950570407b6bdcdf0fcc309e1673b584f06a827fae", + "zh:ca8e8db24a4d62d92afd8d3d383b81a08693acac191a2e0a110fb46deeff56a3", + "zh:d5fa3a36e13957d63bfe9bbd6df0426a2422214403aac9f20b60c36f8d9ebec6", + "zh:e4ede44a112296c9cc77b15e439e41ee15c0e8b3a0dec94ae34df5ebba840e8b", + "zh:f2d4de8d8cde69caffede1544ebea74e69fcc4552e1b79ae053519a05c060706", + "zh:fc19e9266b1841d4a3aeefa8a5b5ad6988baed6540f85a373b6c2d0dc1ca5830", + ] +} + +provider "registry.terraform.io/hashicorp/external" { + version = "2.3.3" + hashes = [ + "h1:H+3QlVPs/7CDa3I4KU/a23wYeGeJxeBlgvR7bfK1t1w=", + "zh:03d81462f9578ec91ce8e26f887e34151eda0e100f57e9772dbea86363588239", + "zh:37ec2a20f6a3ec3a0fd95d3f3de26da6cb9534b30488bc45723e118a0911c0d8", + "zh:4eb5b119179539f2749ce9de0e1b9629d025990f062f4f4dddc161562bb89d37", + "zh:5a31bb58414f41bee5e09b939012df5b88654120b0238a89dfd6691ba197619a", + "zh:6221a05e52a6a2d4f520ffe7cbc741f4f6080e0855061b0ed54e8be4a84eb9b7", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8bb068496b4679bef625e4710d9f3432e301c3a56602271f04e60eadf7f8a94c", + "zh:94742aa5378bab626ce34f79bcef6a373e4f86ea7a8b762e9f71270a899e0d00", + "zh:a485831b5a525cd8f40e8982fa37da40ff70b1ae092c8b755fcde123f0b1238d", + "zh:a647ff16d071eabcabd87ea8183eb90a775a0294ddd735d742075d62fff09193", + "zh:b74710c5954aaa3faf262c18d36a8c2407862d9f842c63e7fa92fa4de3d29df6", + "zh:fa73d83edc92af2e551857594c2232ba6a9e3603ad34b0a5940865202c08d8d7", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.2" + hashes = [ + "h1:zT1ZbegaAYHwQa+QwIFugArWikRJI9dqohj8xb0GY88=", + "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", + "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", + "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", + "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", + "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", + "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", + "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", + "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", + "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", + "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", + ] +} + +provider "registry.terraform.io/hashicorp/template" { + version = "2.2.0" + hashes = [ + "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", + "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", + "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", + "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", + "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", + "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", + "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", + "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", + "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", + "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", + "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", + ] +} diff --git a/tf/v6gw/aws.tf b/tf/v6gw/aws.tf new file mode 100644 index 0000000..c3d3983 --- /dev/null +++ b/tf/v6gw/aws.tf @@ -0,0 +1,12 @@ +provider "aws" { + region = "ap-northeast-1" + allowed_account_ids = ["005216166247"] + default_tags { + tags = { + Project = "rk24net" + Component = "v6gw" + } + } +} + +data "aws_caller_identity" "current" {} diff --git a/tf/v6gw/backend.tf b/tf/v6gw/backend.tf new file mode 100644 index 0000000..2154c79 --- /dev/null +++ b/tf/v6gw/backend.tf @@ -0,0 +1,8 @@ +terraform { + backend "s3" { + bucket = "rk-infra" + region = "ap-northeast-1" + key = "terraform/nw-v6gw.tfstate" + dynamodb_table = "rk-terraform" + } +} diff --git a/tf/v6gw/cloudconfig.jsonnet b/tf/v6gw/cloudconfig.jsonnet new file mode 100644 index 0000000..b270278 --- /dev/null +++ b/tf/v6gw/cloudconfig.jsonnet @@ -0,0 +1,19 @@ +(import '../cloudconfig.base.libsonnet') { + apt: { + sources: { + nekomit: { + source: 'deb https://deb.nekom.it/ jammy main', + key: importstr 'nekomit.key', + }, + }, + }, + + packages: [ + { + apt: [ + 'mitamae', + 'zip', + ], + }, + ], +} diff --git a/tf/v6gw/ec2.tf b/tf/v6gw/ec2.tf new file mode 100644 index 0000000..61154f1 --- /dev/null +++ b/tf/v6gw/ec2.tf @@ -0,0 +1,139 @@ +locals { + v6gws = tomap({ for subnet in data.aws_subnet.main-private : + "v6gw-${substr(subnet.availability_zone, -1, -1)}001" => { + az = subnet.availability_zone, + seq = 1 + } + }) +} + +data "aws_ami" "ubuntu" { + most_recent = true + owners = ["099720109477"] # Canonical + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-server-*"] + } +} + +data "external" "v6gw-cloudconfig" { + for_each = local.v6gws + + program = ["../jsonnet.rb"] + + query = { + path = "./cloudconfig.jsonnet" + } +} + +data "template_cloudinit_config" "v6gw" { + for_each = local.v6gws + + gzip = false + base64_encode = false + + part { + content_type = "text/cloud-config" + content = data.external.v6gw-cloudconfig[each.key].result.json + } +} + +resource "terraform_data" "v6gw-userdata" { + for_each = local.v6gws + + input = data.template_cloudinit_config.v6gw[each.key].rendered +} + +resource "aws_instance" "v6gw" { + for_each = local.v6gws + + ami = data.aws_ami.ubuntu.id + instance_type = "t4g.nano" + subnet_id = data.aws_subnet.main-private[each.value.az].id + + vpc_security_group_ids = [ + data.aws_security_group.default.id, + aws_security_group.v6gw.id, + ] + + user_data = terraform_data.v6gw-userdata[each.key].output + + key_name = "hanazuki-1p-main" # TODO: remove + + ipv6_addresses = [ + replace(data.aws_subnet.main-private[each.value.az].ipv6_cidr_block, "::/64", format("::66:%x", each.value.seq)), + ] + + tags = { + Name = each.key + } + + lifecycle { + ignore_changes = [ami] + replace_triggered_by = [terraform_data.v6gw-userdata[each.key]] + } +} + +resource "terraform_data" "nodedata" { + for_each = local.v6gws + + input = jsonencode({ + overlay_prefix = "2001:df0:8500:ca60::/64" + ipv6token = format("::66:%x", each.value.seq), + }) +} + +resource "null_resource" "v6gw-provision" { + for_each = local.v6gws + + connection { + host = aws_instance.v6gw[each.key].private_ip + user = "rk" + + bastion_host = "bastion.rubykaigi.net" + bastion_port = 9922 + bastion_user = "rk" + } + + provisioner "file" { + destination = "/tmp/itamae-recipes.zip" + source = data.archive_file.recipes.output_path + } + + provisioner "file" { + destination = "/tmp/itamae-node.json" + content = terraform_data.nodedata[each.key].output + } + + provisioner "remote-exec" { + inline = [ + <<-EOF + #!/bin/bash + set -ex + cloud-init status --wait >/dev/null + workdir=$(mktemp -d) + pushd "$workdir" + unzip /tmp/itamae-recipes.zip + sudo mitamae local -j /tmp/itamae-node.json ./default.rb + popd + rm -rf "$workdir" + EOF + ] + } + + triggers = { + instance = aws_instance.v6gw[each.key].private_ip + recipes = filesha1(data.archive_file.recipes.output_path) + node = terraform_data.nodedata[each.key].output + } +} + +data "archive_file" "recipes" { + type = "zip" + output_path = "./itamae-recipes.zip" + source_dir = "./itamae" +} + +output "private_ips" { + value = [for i in aws_instance.v6gw : i.private_ip] +} diff --git a/tf/v6gw/itamae/default.rb b/tf/v6gw/itamae/default.rb new file mode 100644 index 0000000..e4468f2 --- /dev/null +++ b/tf/v6gw/itamae/default.rb @@ -0,0 +1,10 @@ +node[:ec2] = JSON.parse(File.read('/run/cloud-init/instance-data.json')).dig('ds', 'meta-data') + +template '/etc/iproute2/rt_tables.d/v6gw.conf' do + owner 'root' + group 'root' + mode '0644' +end + +include_recipe './systemd-networkd.rb' +include_recipe './frr.rb' diff --git a/tf/v6gw/itamae/frr.rb b/tf/v6gw/itamae/frr.rb new file mode 100644 index 0000000..c9adadd --- /dev/null +++ b/tf/v6gw/itamae/frr.rb @@ -0,0 +1,21 @@ +package 'frr' + +file '/etc/frr/daemons' do + action :edit + block do |content| + content.sub!(/^bgpd=.*$/, 'bgpd=yes') + content.sub!(/^bfdd=.*$/, 'bfdd=yes') + end + notifies :restart, 'service[frr]', :immediately +end + +template '/etc/frr/frr.conf' do + owner 'frr' + group 'frr' + mode '0640' + notifies :reload, 'service[frr]', :immediately +end + +service 'frr' do + action [:enable, :start] +end diff --git a/tf/v6gw/itamae/systemd-networkd.rb b/tf/v6gw/itamae/systemd-networkd.rb new file mode 100644 index 0000000..2fbe57d --- /dev/null +++ b/tf/v6gw/itamae/systemd-networkd.rb @@ -0,0 +1,37 @@ +package 'linux-modules-extra-aws' + +%w[ + tun-hnd.netdev tun-nrt.netdev tun.network + vxlan-66.netdev vxlan-66.network + br-66.netdev br-66.network + vrf-v6gw.netdev vrf-v6gw.network +].each do |_| + template "/etc/systemd/network/#{_}" do + owner 'root' + group 'root' + mode '0644' + notifies :run, 'execute[networkctl reload]' + end +end + +directory '/etc/systemd/network/10-netplan-ens5.network.d' do + owner 'root' + group 'root' + mode '0755' +end + +template '/etc/systemd/network/10-netplan-ens5.network.d/ens5.override.conf' do + owner 'root' + group 'root' + mode '0644' + notifies :run, 'execute[networkctl reload]' + notifies :run, 'execute[networkctl reconfigure ens5]' +end + +execute 'networkctl reload' do + action :nothing +end + +execute 'networkctl reconfigure ens5' do + action :nothing +end diff --git a/tf/v6gw/itamae/templates/br-66.netdev b/tf/v6gw/itamae/templates/br-66.netdev new file mode 100644 index 0000000..eaece63 --- /dev/null +++ b/tf/v6gw/itamae/templates/br-66.netdev @@ -0,0 +1,6 @@ +[NetDev] +Name=br-66 +Kind=bridge + +[Bridge] +STP=no diff --git a/tf/v6gw/itamae/templates/br-66.network b/tf/v6gw/itamae/templates/br-66.network new file mode 100644 index 0000000..192ca02 --- /dev/null +++ b/tf/v6gw/itamae/templates/br-66.network @@ -0,0 +1,7 @@ +[Match] +Name=br-66 + +[Network] +VRF=vrf-v6gw +Address=<%= node.fetch('overlay_prefix').sub('::', "::#{node['ec2'].fetch('local-ipv4')}") %> +IPForward=ipv6 diff --git a/tf/v6gw/itamae/templates/ens5.override.conf b/tf/v6gw/itamae/templates/ens5.override.conf new file mode 100644 index 0000000..6d8bb1f --- /dev/null +++ b/tf/v6gw/itamae/templates/ens5.override.conf @@ -0,0 +1,6 @@ +[Network] +Tunnel=tun-hnd +Tunnel=tun-nrt +VXLAN=vxlan-66 + +KeepConfiguration=dhcp diff --git a/tf/v6gw/itamae/templates/frr.conf b/tf/v6gw/itamae/templates/frr.conf new file mode 100644 index 0000000..b876435 --- /dev/null +++ b/tf/v6gw/itamae/templates/frr.conf @@ -0,0 +1,147 @@ +<%- + + subnets = { + 'ap-northeast-1c' => %w[10.33.160.0/23], + 'ap-northeast-1d' => %w[10.33.162.0/23], + } + + overlay_prefix = '2001:df0:8500:ca60::/64' + + az = node[:ec2][:placement].fetch('availability-zone') + vpc_ipv4 = node[:ec2].fetch('local-ipv4') + vpc_ipv6 = node[:ec2].fetch(:ipv6) + + v6gw_peers = %w[2406:da14:dfe:c0c1:0:0:66:1 2406:da14:dfe:c0d1:0:0:66:1] + v6gw_peers.delete(vpc_ipv6) + + az_nibble = {'ap-northeast-1c' => 1, 'ap-northeast-1d' => 2}.fetch(az) + br_peers = { + hnd: {ipv6: "2001:df0:8500:ca22:66#{az_nibble}1::a", asn: 65001}, + nrt: {ipv6: "2001:df0:8500:ca22:66#{az_nibble}2::a", asn: 65002}, + } +-%> +frr version 8.4.4 +frr defaults traditional +hostname ip-10-33-136-182 +log syslog +service integrated-vtysh-config +! +vrf vrf-v6gw +<%- subnets.fetch(az).each do |subnet| -%> + ipv6 route 2001:df0:8500:ca60::<%= subnet.sub(%r{/(\d+)}) { ?/ + ($1.to_i + 96).to_s } %> br-66 +<%- end -%> +exit-vrf +! +interface tun-hnd + ipv6 address 2001:df0:8500:ca22:66<%= az_nibble %>1::b/124 +exit +! +interface tun-nrt + ipv6 address 2001:df0:8500:ca22:66<%= az_nibble %>2::b/124 +exit +! +router bgp 64566 + bgp router-id <%= vpc_ipv4 %> + bgp log-neighbor-changes + no bgp default ipv4-unicast + ! + neighbor client-azc peer-group + neighbor client-azc remote-as 64567 + neighbor client-azc timers 10 30 + neighbor client-azc capability extended-nexthop +<%- subnets.fetch('ap-northeast-1c').each do |subnet| -%> + bgp listen range <%= subnet %> peer-group client-azc +<%- end -%> + ! + neighbor client-azd peer-group + neighbor client-azd remote-as 64567 + neighbor client-azd timers 10 30 + neighbor client-azd capability extended-nexthop +<%- subnets.fetch('ap-northeast-1d').each do |subnet| -%> + bgp listen range <%= subnet %> peer-group client-azd +<%- end -%> + ! + neighbor v6gw peer-group + neighbor v6gw remote-as 64566 + neighbor v6gw timers 10 30 + neighbor v6gw capability extended-nexthop +<%- v6gw_peers.each do |peer| -%> + neighbor <%= peer %> peer-group v6gw +<%- end -%> + ! + address-family ipv6 unicast + neighbor v6gw activate + neighbor client-azc activate + neighbor client-azc prefix-list default out + neighbor client-azd activate + neighbor client-azd prefix-list default out + exit-address-family + ! + address-family l2vpn evpn + neighbor v6gw activate + neighbor client-azc activate + neighbor client-azc route-map evpn in + neighbor client-azc route-map evpn out + neighbor client-azd activate + neighbor client-azd route-map evpn in + neighbor client-azd route-map evpn out + advertise-all-vni + advertise-svi-ip + exit-address-family +exit +! +router bgp 64566 vrf vrf-v6gw + bgp router-id <%= vpc_ipv4 %> + bgp log-neighbor-changes + no bgp default ipv4-unicast + ! + neighbor br peer-group + neighbor br timers 10 30 +<%- br_peers.each do |name, peer| -%> + neighbor <%= peer.fetch(:ipv6) %> peer-group br + neighbor <%= peer.fetch(:ipv6) %> remote-as <%= peer.fetch(:asn) %> +<%- end -%> + ! + address-family ipv6 unicast + network 2001:df0:8500:ca60::/64 + redistribute static route-map overlay-subnet + neighbor br activate + neighbor br prefix-list default in + neighbor br prefix-list overlay out +<%- br_peers.each do |name, peer| -%> + neighbor <%= peer.fetch(:ipv6) %> route-map br-<%= name %>-in in + neighbor <%= peer.fetch(:ipv6) %> route-map br-<%= name %>-out out +<%- end -%> + exit-address-family +exit +! +ipv6 prefix-list default seq 5 permit ::/0 ge 0 le 0 +ipv6 prefix-list overlay seq 5 permit 2001:df0:8500:ca60::/64 ge 64 +! +route-map br-hnd-in permit 10 + set local-preference 200 +exit +! +route-map br-hnd-out permit 10 + set metric 0 +exit +! +route-map br-nrt-in permit 10 + set local-preference 100 +exit +! +route-map br-nrt-out permit 10 + set metric 100 +exit +! +route-map overlay-subnet permit 10 + match ipv6 address prefix-list overlay +exit +! +route-map evpn permit 10 +exit +! +ip nht resolve-via-default +! +ipv6 nht resolve-via-default +! diff --git a/tf/v6gw/itamae/templates/rt_tables.d/v6gw.conf b/tf/v6gw/itamae/templates/rt_tables.d/v6gw.conf new file mode 100644 index 0000000..1eb5efa --- /dev/null +++ b/tf/v6gw/itamae/templates/rt_tables.d/v6gw.conf @@ -0,0 +1 @@ +66 v6gw diff --git a/tf/v6gw/itamae/templates/tun-hnd.netdev b/tf/v6gw/itamae/templates/tun-hnd.netdev new file mode 100644 index 0000000..c91bdfa --- /dev/null +++ b/tf/v6gw/itamae/templates/tun-hnd.netdev @@ -0,0 +1,9 @@ +[NetDev] +Name=tun-hnd +Kind=ip6tnl + +[Tunnel] +Mode=ip6ip6 +Local=dhcp6 +Remote=2001:df0:8500:ca00::1 +EncapsulationLimit=none diff --git a/tf/v6gw/itamae/templates/tun-nrt.netdev b/tf/v6gw/itamae/templates/tun-nrt.netdev new file mode 100644 index 0000000..57e4b8d --- /dev/null +++ b/tf/v6gw/itamae/templates/tun-nrt.netdev @@ -0,0 +1,9 @@ +[NetDev] +Name=tun-nrt +Kind=ip6tnl + +[Tunnel] +Mode=ip6ip6 +Local=dhcp6 +Remote=2001:df0:8500:ca00::2 +EncapsulationLimit=none diff --git a/tf/v6gw/itamae/templates/tun.network b/tf/v6gw/itamae/templates/tun.network new file mode 100644 index 0000000..2a7f305 --- /dev/null +++ b/tf/v6gw/itamae/templates/tun.network @@ -0,0 +1,7 @@ +[Match] +Name=tun-* + +[Network] +VRF=vrf-v6gw +IPForward=ipv6 +KeepConfiguration=static diff --git a/tf/v6gw/itamae/templates/vrf-v6gw.netdev b/tf/v6gw/itamae/templates/vrf-v6gw.netdev new file mode 100644 index 0000000..c3a5eee --- /dev/null +++ b/tf/v6gw/itamae/templates/vrf-v6gw.netdev @@ -0,0 +1,6 @@ +[NetDev] +Name=vrf-v6gw +Kind=vrf + +[VRF] +Table=66 diff --git a/tf/v6gw/itamae/templates/vrf-v6gw.network b/tf/v6gw/itamae/templates/vrf-v6gw.network new file mode 100644 index 0000000..6ec54c6 --- /dev/null +++ b/tf/v6gw/itamae/templates/vrf-v6gw.network @@ -0,0 +1,2 @@ +[Match] +Name=vrf-v6gw diff --git a/tf/v6gw/itamae/templates/vxlan-66.netdev b/tf/v6gw/itamae/templates/vxlan-66.netdev new file mode 100644 index 0000000..8ea99ad --- /dev/null +++ b/tf/v6gw/itamae/templates/vxlan-66.netdev @@ -0,0 +1,9 @@ +[NetDev] +Name=vxlan-66 +Kind=vxlan + +[VXLAN] +VNI=66 +Local=dhcp4 +DestinationPort=4789 +MacLearning=no diff --git a/tf/v6gw/itamae/templates/vxlan-66.network b/tf/v6gw/itamae/templates/vxlan-66.network new file mode 100644 index 0000000..92e2de8 --- /dev/null +++ b/tf/v6gw/itamae/templates/vxlan-66.network @@ -0,0 +1,5 @@ +[Match] +Name=vxlan-66 + +[Network] +Bridge=br-66 diff --git a/tf/v6gw/nekomit.key b/tf/v6gw/nekomit.key new file mode 100644 index 0000000..29232d5 --- /dev/null +++ b/tf/v6gw/nekomit.key @@ -0,0 +1,50 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBFdkWwcBCADZ2Ltd88dVko6aZ1TjFj/Ls5vXNaCQF4Y6+CNJUj7UXxSY1AIb +KPhmclt7ILMaDfUhZeWc0ENVvZSXDmXs+xkTrCwkKpy4VY3HSV869If89TlJzGbK +cLiptIdyuKGUCICNwkjMmnUCEbygIUmud/CH8rLiXPy4oYAP0V4PK1UZQyotnd7l +yJNIu/wnDoin8UcBqJ46bo12wwSlYjhQDC+sIs0IciL9XMO9uSpiZK5wBPK2OHTI +Z9xZdgzgAJSrKVfB1MtczMk5imZC80qgHjeCLlU02wO9ydcLuBDQq/5iYPfVKZ41 +05pI6dMfmbpdJv3sAqeKHxkR0VBa0d3ZppnPABEBAAG0Mm5la29taXQtYmludHJh +eSA8bmVrb21pdC1iaW50cmF5LWdwZy1rZXlAc29yYWguanA+iQE3BBMBCAAhBQJX +ZFsHAhsDBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEKNZkiXD/zMFYRYIANj7 +U7NzB1xRg7qLb2fAqLNOqPp5ERn7r2j/3b70hSdG9YSKxovWeXU294AWCkoEUwmv +TzG06NWK4rtNbjjqkiYaaObP29mqRBDB9mgZ9nd1S6iQi1cxC508kYEJSXzR7JEu +xeYnu4KAtbUuU5qERd1fPTTgbtDJf/4lAru+/L6on13WQy2PPD+u9rfbU3d0aDNA +FSAs9GpXEPWrc4zAvOtKrYn6I4/fCJ6NAD9RNX28pQRrC4IvJpSV2EWqfCCFLSI5 +4yONNlzHxLKnPGZIG5Cy50knOdIhBWISdtqfFFyCTxsMyn45rm+uf/lvy1vhfHP2 +XjMwLWPAqTdHz0YuSEmJAhwEEAEIAAYFAldkXKsACgkQNPenwvTAiVzHyA//Ukvq +f9g2899Oz3frvB/nF4my881GDLnPMgEMBA8icznm2+NgbkPhbTexD/5tF94nGNx0 +iG4PaTeyV0qDW5R4Uzz7GmKwcVziuVU3l2JvrOSaHflaSRrwiiR0SfoDEkAg8rRa +8db0Z1n4CDcG/2oXYVueIL2yoyPt8bvpgggyAAH9d9YGpv8NJDZaaO8rzfkDPM/b +lNdT5OBwP2IVnvctWlqahRVX3JLGp9qm2eFa5XCjLDl94B4e0RpKbboH65Eo9Oa/ +k0nRvnrcMKy7+szmknBRquNIjPekgv8SYfK1u4xu5pUDJR89OZyYWw0CvRQC8Pd9 +jJbzMnYVdMp377hes5cUYA75ol6awzO9xdzCgBJm7FhlcWAusfyyk5Fs7mDRCKo4 +ISxy485NKVD3B7YTw7zKgQ4HoeffUQi5TSVPPW/6fe98+dT6+FeWjQSSR79PO3wi +bo9OCNIUKtLWPAYbuj3ItWkxyUyaYgu2b2WS7o7to4JbbuNugLvLFpsrexhxWPTv +EzvtWV2Y8SsHEI/gL+2e92mruaxCb1DhaYxz+Q6mSMbOMMqmWD6w3o9cWrkDenGa +siH1YOfb7rogzEP3DkerU7Zu/AowJwKeKbjkuTfjiPHnWRmFOY4D6gir3IX7SuJd +PWo3uBllPw7uzgI47q2qx2NZstl92dtQgz6lEOCJAbMEEAEIAB0WIQQRg+++mSZ8 +P7cA9yhvSet2PajyTAUCXGdhJwAKCRBvSet2PajyTPkODACkbenG6ePm3FHCmUGx +1dcewBijzQ935ycSzTuwj2vSSbrHUuH5E/x/SDpFHnenKPf90+FI64iKGoQ7gPdH +nMQ3Jkr7piC68pjD5Hc/6qehG8EYQR9m+4bnI+JFhpF5FYyzFsEVxYl6HmoMafm/ +XABmxxXZgFtR0tJVyg3bDlzgJ+i4crs4yN+UVSFq+Ho5HeJuKX1WIHf5XATlKc8H ++sLWybGEagBJkz73544J2D3dnQgLAZenQp+UGkBYivy2mtsgw4m4kXRZ4Q2VVVEl +NkJ+t67xOZwEf3QQrcRnK8dT4VTreoQiXQ7mRicOAT2fFmhLQOgc0jzGT251sez1 +LPWnZ2oUpByom4tRlFWkTqlOxNd0aJzx+LjkuJ4MuPqRdvCZe9aDHXdez7yi6vTu +isXhgFeoMeoed0r44iiwNWjZR0HgFzrhJzwpbhW67MVKiH2KSE8eeNEtb6TEayk8 +OhopHnsLZYW3L6kbUE1tizAenk7mQURMnZibV26bINAZV5q5AQ0EV2RbBwEIANeb +9nVsdx5X10KCYgkDpwE6/awuOkq1PIvGh6iE6Qbc5uzEV3GirXq98gIcvq5C0Wyj +Z6eGDO6EzYDZRfxCNBDabuaVJhlOFAyQLyu9LJjE493zytEZ3e9RseZ8EfI3uHfM +xfaEvu39zZUXqdGOmg3NxcxRCpY+InVTmn7GTTijL+RZoDyNgpTpKXuqw9QIB+ve +1aONurYNLlYA4zfH34jbMccQEVndTcA7nxEJKpXBmUyeTI/exQSR9IAwrlVN69z2 +zBj6gkXPcJ0yDa6RTOZr416qcM5onapdGISyj3Mz1JtBla+GHag6FrtBIOQArSj+ +st+fwrSaXc4nKdR7/gMAEQEAAYkBHwQYAQgACQUCV2RbBwIbDAAKCRCjWZIlw/8z +BSw8B/9NqRrNh3erkCx1/I8YwGM91C2g3dFCMEZD7bqFe4lINHHxtKxcU6KlkhiF +LS7b8LA2fGKWPXrotE6sDcUVFYQWMNnvfwpkU8TBZHEWaoKX2WrtMPkwhV49o4Xk +qUDKTEZI/4koCAW3A3Z/XY2eS7cxCKRz9DzQ0WV7dYUrquR9r9sT5cTBsAIgYd77 +dQWSF6fhJHi1YZvR54DQFC1BOaUh9/QSKDp93Xj4FCbjixuHgqNx3WZ+qiBqsbMV +wZXMXISDxXsO0Iu+yckOuKQ/LmMJblOys5gBrqklVUSVs8UZwJdiRjLsNIBnhfX7 +BfxXVVCyIqtl5cRqg7MnY4rgUgpf +=/QoT +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tf/v6gw/route53.tf b/tf/v6gw/route53.tf new file mode 100644 index 0000000..7bc571a --- /dev/null +++ b/tf/v6gw/route53.tf @@ -0,0 +1,12 @@ +data "aws_route53_zone" "rubykaigi_net-private" { + name = "rubykaigi.net." + private_zone = true +} + +resource "aws_route53_record" "origin" { + zone_id = data.aws_route53_zone.rubykaigi_net-private.zone_id + name = "_bgp._tcp.v6gw.apne1.rubykaigi.net" + type = "SRV" + ttl = 30 + records = [for i in aws_instance.v6gw : "0 1 179 ${i.private_dns}."] +} diff --git a/tf/v6gw/sg.tf b/tf/v6gw/sg.tf new file mode 100644 index 0000000..bdf4b1d --- /dev/null +++ b/tf/v6gw/sg.tf @@ -0,0 +1,66 @@ +data "aws_security_group" "default" { + name = "default" + vpc_id = data.aws_vpc.main.id +} + +data "aws_security_group" "eks-node" { + vpc_id = data.aws_vpc.main.id + tags = { + "aws:eks:cluster-name" = "rknet", + "kubernetes.io/cluster/rknet" = "owned" + } +} + +resource "aws_security_group" "v6gw" { + name = "v6gw" + vpc_id = data.aws_vpc.main.id +} + +resource "aws_vpc_security_group_ingress_rule" "ip6tnl" { + for_each = tomap({ + ipv6 = 41, + }) + + security_group_id = aws_security_group.v6gw.id + + ip_protocol = each.value + cidr_ipv6 = "2001:df0:8500:ca00::/120" + from_port = -1 + to_port = -1 +} + +resource "aws_vpc_security_group_ingress_rule" "bgp-eks-node" { + security_group_id = aws_security_group.v6gw.id + referenced_security_group_id = data.aws_security_group.eks-node.id + + ip_protocol = "tcp" + from_port = 179 + to_port = 179 +} + +resource "aws_vpc_security_group_ingress_rule" "bgp-peer-v6gw" { + security_group_id = aws_security_group.v6gw.id + referenced_security_group_id = aws_security_group.v6gw.id + + ip_protocol = "tcp" + from_port = 179 + to_port = 179 +} + +resource "aws_vpc_security_group_ingress_rule" "vxlan-eks-node" { + security_group_id = aws_security_group.v6gw.id + referenced_security_group_id = data.aws_security_group.eks-node.id + + ip_protocol = "udp" + from_port = 4789 + to_port = 4789 +} + +resource "aws_vpc_security_group_ingress_rule" "vxlan-peer-v6gw" { + security_group_id = aws_security_group.v6gw.id + referenced_security_group_id = aws_security_group.v6gw.id + + ip_protocol = "udp" + from_port = 4789 + to_port = 4789 +} diff --git a/tf/v6gw/versions.tf b/tf/v6gw/versions.tf new file mode 100644 index 0000000..746cf57 --- /dev/null +++ b/tf/v6gw/versions.tf @@ -0,0 +1,21 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5" + } + archive = { + source = "hashicorp/archive" + } + null = { + source = "hashicorp/null" + } + external = { + source = "hashicorp/external" + } + template = { + source = "hashicorp/template" + } + } + required_version = ">= 1.8" +} diff --git a/tf/v6gw/vpc.tf b/tf/v6gw/vpc.tf new file mode 100644 index 0000000..7edc303 --- /dev/null +++ b/tf/v6gw/vpc.tf @@ -0,0 +1,24 @@ +locals { + azs = toset(["ap-northeast-1c", "ap-northeast-1d"]) +} + +data "aws_vpc" "main" { + id = "vpc-004eca6fe0bf3494d" +} + +data "aws_subnet" "main-private" { + for_each = local.azs + + filter { + name = "vpc-id" + values = [data.aws_vpc.main.id] + } + filter { + name = "availability-zone" + values = [each.key] + } + filter { + name = "tag:Tier" + values = ["private"] + } +}