Creating private subnets without direct external internet access and …

…updating proxy e2e to use this instead

Using 1b21187 for reference

Populated by running:

for REGION in us-east-1 us-east-2 us-west-1 us-west-2
do
  COUNT=3
  if test us-west-1 = "${REGION}"
  then
    COUNT=2
  fi
  for INDEX in 1
  do
    NAME="do-not-delete-shared-vpc-blackhole-${INDEX}"
    aws --region "${REGION}" cloudformation create-stack --stack-name "${NAME}" --template-body "$(cat ci-operator/step-registry/ipi/conf/aws/blackholenetwork/blackhole_vpc.yaml)" --parameters "ParameterKey=AvailabilityZoneCount,ParameterValue=${COUNT}" >/dev/null
    aws --region "${REGION}" cloudformation wait stack-create-complete --stack-name "${NAME}"
    SUBNETS="$(aws --region "${REGION}" cloudformation describe-stacks --stack-name "${NAME}" | jq -c '[.Stacks[].Outputs[] | select(.OutputKey | endswith("SubnetIds")).OutputValue | split(",")[]]' | sed "s/\"/'/g")"
    echo "${REGION}_$((INDEX - 1))) subnets=\"${SUBNETS}\";;"
  done
done

ci-operator/step-registry/ipi/conf/aws/blackholenetwork/OWNERS

@@ -0,0 +1,3 @@
+approvers:
+- wking
+- ewolinetz

ci-operator/step-registry/ipi/conf/aws/blackholenetwork/blackhole_vpc_yaml.md

@@ -0,0 +1,217 @@
+# This is the template file used to generate blackhole VPC and subnet entries.
+AWSTemplateFormatVersion: 2010-09-09
+Description: Template for Best Practice VPC with 1-3 AZs
+
+Parameters:
+ VpcCidr:
+ AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(1[6-9]|2[0-4]))$
+ ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-24.
+ Default: 10.0.0.0/16
+ Description: CIDR block for VPC.
+ Type: String
+ AvailabilityZoneCount:
+ ConstraintDescription: "The number of availability zones. (Min: 1, Max: 3)"
+ MinValue: 1
+ MaxValue: 3
+ Default: 1
+ Description: "How many AZs to create VPC subnets for. (Min: 1, Max: 3)"
+ Type: Number
+ SubnetBits:
+ ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/19-27.
+ MinValue: 5
+ MaxValue: 13
+ Default: 12
+ Description: "Size of each subnet to create within the availability zones. (Min: 5 = /27, Max: 13 = /19)"
+ Type: Number
+
+Metadata:
+ AWS::CloudFormation::Interface:
+ ParameterGroups:
+ - Label:
+ default: "Network Configuration"
+ Parameters:
+ - VpcCidr
+ - SubnetBits
+ - Label:
+ default: "Availability Zones"
+ Parameters:
+ - AvailabilityZoneCount
+ ParameterLabels:
+ AvailabilityZoneCount:
+ default: "Availability Zone Count"
+ VpcCidr:
+ default: "VPC CIDR"
+ SubnetBits:
+ default: "Bits Per Subnet"
+
+Conditions:
+ DoAz3: !Equals [3, !Ref AvailabilityZoneCount]
+ DoAz2: !Or [!Equals [2, !Ref AvailabilityZoneCount], Condition: DoAz3]
+
+Resources:
+ VPC:
+ Type: "AWS::EC2::VPC"
+ Properties:
+ EnableDnsSupport: "true"
+ EnableDnsHostnames: "true"
+ CidrBlock: !Ref VpcCidr
+ PublicSubnet:
+ Type: "AWS::EC2::Subnet"
+ Properties:
+ VpcId: !Ref VPC
+ CidrBlock: !Select [0, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]]
+ AvailabilityZone: !Select
+ - 0
+ - Fn::GetAZs: !Ref "AWS::Region"
+ PublicSubnet2:
+ Type: "AWS::EC2::Subnet"
+ Condition: DoAz2
+ Properties:
+ VpcId: !Ref VPC
+ CidrBlock: !Select [1, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]]
+ AvailabilityZone: !Select
+ - 1
+ - Fn::GetAZs: !Ref "AWS::Region"
+ PublicSubnet3:
+ Type: "AWS::EC2::Subnet"
+ Condition: DoAz3
+ Properties:
+ VpcId: !Ref VPC
+ CidrBlock: !Select [2, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]]
+ AvailabilityZone: !Select
+ - 2
+ - Fn::GetAZs: !Ref "AWS::Region"
+ InternetGateway:
+ Type: "AWS::EC2::InternetGateway"
+ GatewayToInternet:
+ Type: "AWS::EC2::VPCGatewayAttachment"
+ Properties:
+ VpcId: !Ref VPC
+ InternetGatewayId: !Ref InternetGateway
+ PublicRouteTable:
+ Type: "AWS::EC2::RouteTable"
+ Properties:
+ VpcId: !Ref VPC
+ PublicRoute:
+ Type: "AWS::EC2::Route"
+ DependsOn: GatewayToInternet
+ Properties:
+ RouteTableId: !Ref PublicRouteTable
+ DestinationCidrBlock: 0.0.0.0/0
+ GatewayId: !Ref InternetGateway
+ PublicSubnetRouteTableAssociation:
+ Type: "AWS::EC2::SubnetRouteTableAssociation"
+ Properties:
+ SubnetId: !Ref PublicSubnet
+ RouteTableId: !Ref PublicRouteTable
+ PublicSubnetRouteTableAssociation2:
+ Type: "AWS::EC2::SubnetRouteTableAssociation"
+ Condition: DoAz2
+ Properties:
+ SubnetId: !Ref PublicSubnet2
+ RouteTableId: !Ref PublicRouteTable
+ PublicSubnetRouteTableAssociation3:
+ Condition: DoAz3
+ Type: "AWS::EC2::SubnetRouteTableAssociation"
+ Properties:
+ SubnetId: !Ref PublicSubnet3
+ RouteTableId: !Ref PublicRouteTable
+ PrivateSubnet:
+ Type: "AWS::EC2::Subnet"
+ Properties:
+ VpcId: !Ref VPC
+ CidrBlock: !Select [3, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]]
+ AvailabilityZone: !Select
+ - 0
+ - Fn::GetAZs: !Ref "AWS::Region"
+ PrivateRouteTable:
+ Type: "AWS::EC2::RouteTable"
+ Properties:
+ VpcId: !Ref VPC
+ PrivateSubnetRouteTableAssociation:
+ Type: "AWS::EC2::SubnetRouteTableAssociation"
+ Properties:
+ SubnetId: !Ref PrivateSubnet
+ RouteTableId: !Ref PrivateRouteTable
+ PrivateSubnet2:
+ Type: "AWS::EC2::Subnet"
+ Condition: DoAz2
+ Properties:
+ VpcId: !Ref VPC
+ CidrBlock: !Select [4, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]]
+ AvailabilityZone: !Select
+ - 1
+ - Fn::GetAZs: !Ref "AWS::Region"
+ PrivateRouteTable2:
+ Type: "AWS::EC2::RouteTable"
+ Condition: DoAz2
+ Properties:
+ VpcId: !Ref VPC
+ PrivateSubnetRouteTableAssociation2:
+ Type: "AWS::EC2::SubnetRouteTableAssociation"
+ Condition: DoAz2
+ Properties:
+ SubnetId: !Ref PrivateSubnet2
+ RouteTableId: !Ref PrivateRouteTable2
+ PrivateSubnet3:
+ Type: "AWS::EC2::Subnet"
+ Condition: DoAz3
+ Properties:
+ VpcId: !Ref VPC
+ CidrBlock: !Select [5, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]]
+ AvailabilityZone: !Select
+ - 2
+ - Fn::GetAZs: !Ref "AWS::Region"
+ PrivateRouteTable3:
+ Type: "AWS::EC2::RouteTable"
+ Condition: DoAz3
+ Properties:
+ VpcId: !Ref VPC
+ PrivateSubnetRouteTableAssociation3:
+ Type: "AWS::EC2::SubnetRouteTableAssociation"
+ Condition: DoAz3
+ Properties:
+ SubnetId: !Ref PrivateSubnet3
+ RouteTableId: !Ref PrivateRouteTable3
+ S3Endpoint:
+ Type: AWS::EC2::VPCEndpoint
+ Properties:
+ PolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Principal: '*'
+ Action:
+ - '*'
+ Resource:
+ - '*'
+ RouteTableIds:
+ - !Ref PublicRouteTable
+ - !Ref PrivateRouteTable
+ - !If [DoAz2, !Ref PrivateRouteTable2, !Ref "AWS::NoValue"]
+ - !If [DoAz3, !Ref PrivateRouteTable3, !Ref "AWS::NoValue"]
+ ServiceName: !Join
+ - ''
+ - - com.amazonaws.
+ - !Ref 'AWS::Region'
+ - .s3
+ VpcId: !Ref VPC
+
+Outputs:
+ VpcId:
+ Description: ID of the new VPC.
+ Value: !Ref VPC
+ PublicSubnetIds:
+ Description: Subnet IDs of the public subnets.
+ Value:
+ !Join [
+ ",",
+ [!Ref PublicSubnet, !If [DoAz2, !Ref PublicSubnet2, !Ref "AWS::NoValue"], !If [DoAz3, !Ref PublicSubnet3, !Ref "AWS::NoValue"]]
+ ]
+ PrivateSubnetIds:
+ Description: Subnet IDs of the private subnets.
+ Value:
+ !Join [
+ ",",
+ [!Ref PrivateSubnet, !If [DoAz2, !Ref PrivateSubnet2, !Ref "AWS::NoValue"], !If [DoAz3, !Ref PrivateSubnet3, !Ref "AWS::NoValue"]]
+ ]

ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-chain.yaml

@@ -0,0 +1,8 @@
+chain:
+ as: ipi-conf-aws-blackholenetwork
+ steps:
+ - ref: ipi-conf
+ - ref: ipi-conf-aws
+ - ref: ipi-conf-aws-blackholenetwork
+ documentation: |-
+ The IPI configure step chain generates the install-config.yaml file based on the cluster profile and optional input files.

ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-commands.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+# TODO: move to image
+curl -L https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_amd64 -o /tmp/yq && chmod +x /tmp/yq
+
+CONFIG="${SHARED_DIR}/install-config.yaml"
+PATCH="${SHARED_DIR}/install-config-blackholenetwork.yaml.patch"
+
+aws_region=$(/tmp/yq r "${CONFIG}" 'platform.aws.region')
+
+subnets="[]"
+case "${aws_region}" in
+us-east-1) subnets="['subnet-0a7491aa76f9b88d7','subnet-0f0b2dcccdcbc7c1d','subnet-0680badf68cbf198c','subnet-02b25dd65f806e41b','subnet-010235a3bff34cf6f','subnet-085c78d8c562b5a51']";;
+us-east-2) subnets="['subnet-0ea117d9499ef624f','subnet-00adc83d4719d4176','subnet-0b9399990fa424d7f','subnet-060d997b25f5bb922','subnet-015f4e65b0ef1b0e1','subnet-02296b47817923bfb']";;
+us-west-1) subnets="['subnet-0d003f08a541855a2','subnet-04007c47f50891b1d','subnet-02cdb70a3a4beb754','subnet-0d813eca318034290']";;
+us-west-2) subnets="['subnet-05d8f8ae35e720611','subnet-0f3f254b13d40e352','subnet-0e23da17ea081d614','subnet-0f380906f83c55df7','subnet-0a2c5167d94c1a5f8','subnet-01375df3b11699b77']";;
+*) echo >&2 "invalid subnets index"; exit 1;;
+esac
+echo "Subnets : ${subnets}"
+
+cat >> "${PATCH}" << EOF
+platform:
+ aws:
+ subnets: ${subnets}
+EOF
+
+/tmp/yq m -x -i "${CONFIG}" "${PATCH}"

ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-ref.yaml

@@ -0,0 +1,10 @@
+ref:
+ as: ipi-conf-aws-blackholenetwork
+ from: base
+ commands: ipi-conf-aws-blackholenetwork-commands.sh
+ resources:
+ requests:
+ cpu: 10m
+ memory: 100Mi
+ documentation: |-
+ The IPI AWS blackholenetwork configure step generates the AWS-specific install-config.yaml contents based on the cluster profile and optional input files using subnets where the private ones do not have direct egress access.

ci-operator/step-registry/ipi/conf/aws/proxy/ipi-conf-aws-proxy-chain.yaml

@@ -1,7 +1,7 @@
 chain:
  as: ipi-conf-aws-proxy
  steps:
- - chain: ipi-conf-aws-sharednetwork
+ - chain: ipi-conf-aws-blackholenetwork
  - ref: ipi-conf-aws-proxy
  documentation: |-
  The IPI configure aws proxy step chain spins up a squid proxy in a separate ec2 instance and appends the proxy info to the install-config.yaml file.