From 9b39dd2ddedbf2334e5cc1709def3b710ecb3baa Mon Sep 17 00:00:00 2001 From: Eric Wolinetz Date: Mon, 20 Jul 2020 11:48:43 -0500 Subject: [PATCH] Creating private subnets without direct external internet access and updating proxy e2e to use this instead Using https://github.com/openshift/release/pull/6949/commits/1b21187950b7d1d83f87774e9c52e74616e1b6c4 for reference Populated by running: for REGION in us-east-1 us-east-2 us-west-1 us-west-2 do COUNT=3 if test us-west-1 = "${REGION}" then COUNT=2 fi for INDEX in 1 do NAME="do-not-delete-shared-vpc-blackhole-${INDEX}" aws --region "${REGION}" cloudformation create-stack --stack-name "${NAME}" --template-body "$(cat ci-operator/step-registry/ipi/conf/aws/blackholenetwork/blackhole_vpc.yaml)" --parameters "ParameterKey=AvailabilityZoneCount,ParameterValue=${COUNT}" >/dev/null aws --region "${REGION}" cloudformation wait stack-create-complete --stack-name "${NAME}" SUBNETS="$(aws --region "${REGION}" cloudformation describe-stacks --stack-name "${NAME}" | jq -c '[.Stacks[].Outputs[] | select(.OutputKey | endswith("SubnetIds")).OutputValue | split(",")[]]' | sed "s/\"/'/g")" echo "${REGION}_$((INDEX - 1))) subnets=\"${SUBNETS}\";;" done done --- .../ipi/conf/aws/blackholenetwork/OWNERS | 3 + .../blackholenetwork/blackhole_vpc_yaml.md | 217 ++++++++++++++++++ .../ipi-conf-aws-blackholenetwork-chain.yaml | 8 + .../ipi-conf-aws-blackholenetwork-commands.sh | 31 +++ .../ipi-conf-aws-blackholenetwork-ref.yaml | 10 + .../aws/proxy/ipi-conf-aws-proxy-chain.yaml | 2 +- 6 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 ci-operator/step-registry/ipi/conf/aws/blackholenetwork/OWNERS create mode 100644 ci-operator/step-registry/ipi/conf/aws/blackholenetwork/blackhole_vpc_yaml.md create mode 100644 ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-chain.yaml create mode 100755 ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-commands.sh create mode 100644 ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-ref.yaml diff --git a/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/OWNERS b/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/OWNERS new file mode 100644 index 000000000000..c001880e10a5 --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/OWNERS @@ -0,0 +1,3 @@ +approvers: +- wking +- ewolinetz diff --git a/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/blackhole_vpc_yaml.md b/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/blackhole_vpc_yaml.md new file mode 100644 index 000000000000..1820cccb5fb3 --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/blackhole_vpc_yaml.md @@ -0,0 +1,217 @@ +# This is the template file used to generate blackhole VPC and subnet entries. +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for Best Practice VPC with 1-3 AZs + +Parameters: + VpcCidr: + AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(1[6-9]|2[0-4]))$ + ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-24. + Default: 10.0.0.0/16 + Description: CIDR block for VPC. + Type: String + AvailabilityZoneCount: + ConstraintDescription: "The number of availability zones. (Min: 1, Max: 3)" + MinValue: 1 + MaxValue: 3 + Default: 1 + Description: "How many AZs to create VPC subnets for. (Min: 1, Max: 3)" + Type: Number + SubnetBits: + ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/19-27. + MinValue: 5 + MaxValue: 13 + Default: 12 + Description: "Size of each subnet to create within the availability zones. (Min: 5 = /27, Max: 13 = /19)" + Type: Number + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Network Configuration" + Parameters: + - VpcCidr + - SubnetBits + - Label: + default: "Availability Zones" + Parameters: + - AvailabilityZoneCount + ParameterLabels: + AvailabilityZoneCount: + default: "Availability Zone Count" + VpcCidr: + default: "VPC CIDR" + SubnetBits: + default: "Bits Per Subnet" + +Conditions: + DoAz3: !Equals [3, !Ref AvailabilityZoneCount] + DoAz2: !Or [!Equals [2, !Ref AvailabilityZoneCount], Condition: DoAz3] + +Resources: + VPC: + Type: "AWS::EC2::VPC" + Properties: + EnableDnsSupport: "true" + EnableDnsHostnames: "true" + CidrBlock: !Ref VpcCidr + PublicSubnet: + Type: "AWS::EC2::Subnet" + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [0, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 0 + - Fn::GetAZs: !Ref "AWS::Region" + PublicSubnet2: + Type: "AWS::EC2::Subnet" + Condition: DoAz2 + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [1, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 1 + - Fn::GetAZs: !Ref "AWS::Region" + PublicSubnet3: + Type: "AWS::EC2::Subnet" + Condition: DoAz3 + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [2, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 2 + - Fn::GetAZs: !Ref "AWS::Region" + InternetGateway: + Type: "AWS::EC2::InternetGateway" + GatewayToInternet: + Type: "AWS::EC2::VPCGatewayAttachment" + Properties: + VpcId: !Ref VPC + InternetGatewayId: !Ref InternetGateway + PublicRouteTable: + Type: "AWS::EC2::RouteTable" + Properties: + VpcId: !Ref VPC + PublicRoute: + Type: "AWS::EC2::Route" + DependsOn: GatewayToInternet + Properties: + RouteTableId: !Ref PublicRouteTable + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + PublicSubnetRouteTableAssociation: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Properties: + SubnetId: !Ref PublicSubnet + RouteTableId: !Ref PublicRouteTable + PublicSubnetRouteTableAssociation2: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Condition: DoAz2 + Properties: + SubnetId: !Ref PublicSubnet2 + RouteTableId: !Ref PublicRouteTable + PublicSubnetRouteTableAssociation3: + Condition: DoAz3 + Type: "AWS::EC2::SubnetRouteTableAssociation" + Properties: + SubnetId: !Ref PublicSubnet3 + RouteTableId: !Ref PublicRouteTable + PrivateSubnet: + Type: "AWS::EC2::Subnet" + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [3, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 0 + - Fn::GetAZs: !Ref "AWS::Region" + PrivateRouteTable: + Type: "AWS::EC2::RouteTable" + Properties: + VpcId: !Ref VPC + PrivateSubnetRouteTableAssociation: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Properties: + SubnetId: !Ref PrivateSubnet + RouteTableId: !Ref PrivateRouteTable + PrivateSubnet2: + Type: "AWS::EC2::Subnet" + Condition: DoAz2 + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [4, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 1 + - Fn::GetAZs: !Ref "AWS::Region" + PrivateRouteTable2: + Type: "AWS::EC2::RouteTable" + Condition: DoAz2 + Properties: + VpcId: !Ref VPC + PrivateSubnetRouteTableAssociation2: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Condition: DoAz2 + Properties: + SubnetId: !Ref PrivateSubnet2 + RouteTableId: !Ref PrivateRouteTable2 + PrivateSubnet3: + Type: "AWS::EC2::Subnet" + Condition: DoAz3 + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [5, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 2 + - Fn::GetAZs: !Ref "AWS::Region" + PrivateRouteTable3: + Type: "AWS::EC2::RouteTable" + Condition: DoAz3 + Properties: + VpcId: !Ref VPC + PrivateSubnetRouteTableAssociation3: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Condition: DoAz3 + Properties: + SubnetId: !Ref PrivateSubnet3 + RouteTableId: !Ref PrivateRouteTable3 + S3Endpoint: + Type: AWS::EC2::VPCEndpoint + Properties: + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: '*' + Action: + - '*' + Resource: + - '*' + RouteTableIds: + - !Ref PublicRouteTable + - !Ref PrivateRouteTable + - !If [DoAz2, !Ref PrivateRouteTable2, !Ref "AWS::NoValue"] + - !If [DoAz3, !Ref PrivateRouteTable3, !Ref "AWS::NoValue"] + ServiceName: !Join + - '' + - - com.amazonaws. + - !Ref 'AWS::Region' + - .s3 + VpcId: !Ref VPC + +Outputs: + VpcId: + Description: ID of the new VPC. + Value: !Ref VPC + PublicSubnetIds: + Description: Subnet IDs of the public subnets. + Value: + !Join [ + ",", + [!Ref PublicSubnet, !If [DoAz2, !Ref PublicSubnet2, !Ref "AWS::NoValue"], !If [DoAz3, !Ref PublicSubnet3, !Ref "AWS::NoValue"]] + ] + PrivateSubnetIds: + Description: Subnet IDs of the private subnets. + Value: + !Join [ + ",", + [!Ref PrivateSubnet, !If [DoAz2, !Ref PrivateSubnet2, !Ref "AWS::NoValue"], !If [DoAz3, !Ref PrivateSubnet3, !Ref "AWS::NoValue"]] + ] \ No newline at end of file diff --git a/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-chain.yaml b/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-chain.yaml new file mode 100644 index 000000000000..1707d36ad97c --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-chain.yaml @@ -0,0 +1,8 @@ +chain: + as: ipi-conf-aws-blackholenetwork + steps: + - ref: ipi-conf + - ref: ipi-conf-aws + - ref: ipi-conf-aws-blackholenetwork + documentation: |- + The IPI configure step chain generates the install-config.yaml file based on the cluster profile and optional input files. diff --git a/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-commands.sh b/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-commands.sh new file mode 100755 index 000000000000..6deb335973fd --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-commands.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +# TODO: move to image +curl -L https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_amd64 -o /tmp/yq && chmod +x /tmp/yq + +CONFIG="${SHARED_DIR}/install-config.yaml" +PATCH="${SHARED_DIR}/install-config-blackholenetwork.yaml.patch" + +aws_region=$(/tmp/yq r "${CONFIG}" 'platform.aws.region') + +subnets="[]" +case "${aws_region}" in +us-east-1) subnets="['subnet-0a7491aa76f9b88d7','subnet-0f0b2dcccdcbc7c1d','subnet-0680badf68cbf198c','subnet-02b25dd65f806e41b','subnet-010235a3bff34cf6f','subnet-085c78d8c562b5a51']";; +us-east-2) subnets="['subnet-0ea117d9499ef624f','subnet-00adc83d4719d4176','subnet-0b9399990fa424d7f','subnet-060d997b25f5bb922','subnet-015f4e65b0ef1b0e1','subnet-02296b47817923bfb']";; +us-west-1) subnets="['subnet-0d003f08a541855a2','subnet-04007c47f50891b1d','subnet-02cdb70a3a4beb754','subnet-0d813eca318034290']";; +us-west-2) subnets="['subnet-05d8f8ae35e720611','subnet-0f3f254b13d40e352','subnet-0e23da17ea081d614','subnet-0f380906f83c55df7','subnet-0a2c5167d94c1a5f8','subnet-01375df3b11699b77']";; +*) echo >&2 "invalid subnets index"; exit 1;; +esac +echo "Subnets : ${subnets}" + +cat >> "${PATCH}" << EOF +platform: + aws: + subnets: ${subnets} +EOF + +/tmp/yq m -x -i "${CONFIG}" "${PATCH}" diff --git a/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-ref.yaml b/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-ref.yaml new file mode 100644 index 000000000000..d3977318051e --- /dev/null +++ b/ci-operator/step-registry/ipi/conf/aws/blackholenetwork/ipi-conf-aws-blackholenetwork-ref.yaml @@ -0,0 +1,10 @@ +ref: + as: ipi-conf-aws-blackholenetwork + from: base + commands: ipi-conf-aws-blackholenetwork-commands.sh + resources: + requests: + cpu: 10m + memory: 100Mi + documentation: |- + The IPI AWS blackholenetwork configure step generates the AWS-specific install-config.yaml contents based on the cluster profile and optional input files using subnets where the private ones do not have direct egress access. diff --git a/ci-operator/step-registry/ipi/conf/aws/proxy/ipi-conf-aws-proxy-chain.yaml b/ci-operator/step-registry/ipi/conf/aws/proxy/ipi-conf-aws-proxy-chain.yaml index a582dc2f3769..0f402ba1edbd 100644 --- a/ci-operator/step-registry/ipi/conf/aws/proxy/ipi-conf-aws-proxy-chain.yaml +++ b/ci-operator/step-registry/ipi/conf/aws/proxy/ipi-conf-aws-proxy-chain.yaml @@ -1,7 +1,7 @@ chain: as: ipi-conf-aws-proxy steps: - - chain: ipi-conf-aws-sharednetwork + - chain: ipi-conf-aws-blackholenetwork - ref: ipi-conf-aws-proxy documentation: |- The IPI configure aws proxy step chain spins up a squid proxy in a separate ec2 instance and appends the proxy info to the install-config.yaml file. \ No newline at end of file