Skip to content

Commit

Permalink
Refactor bastion
Browse files Browse the repository at this point in the history
  • Loading branch information
@hebelsan
hebelsan committed Oct 31, 2024
1 parent 6210999 commit 1d2e58a
Show file tree
Hide file tree
Showing 9 changed files with 751 additions and 244 deletions.
4 changes: 4 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ EFFECTIVE_VERSION := $(VERSION)-$(shell git rev-parse HEAD)
LD_FLAGS := "-w $(shell bash $(GARDENER_HACK_DIR)/get-build-ld-flags.sh k8s.io/component-base $(REPO_ROOT)/VERSION $(EXTENSION_PREFIX))"
LEADER_ELECTION := false
IGNORE_OPERATION_ANNOTATION := true
LOG_LEVEL := info

WEBHOOK_CONFIG_PORT := 8443
WEBHOOK_CONFIG_MODE := url
Expand Down Expand Up @@ -169,6 +170,7 @@ integration-test-infra:
--secret-access-key='$(shell cat $(SECRET_ACCESS_KEY_FILE))' \
--region=$(REGION) \
--reconciler=$(RECONCILER)
--love-level=$(LOG_LEVEL)

.PHONY: integration-test-bastion
integration-test-bastion:
Expand All @@ -178,6 +180,7 @@ integration-test-bastion:
--access-key-id='$(shell cat $(ACCESS_KEY_ID_FILE))' \
--secret-access-key='$(shell cat $(SECRET_ACCESS_KEY_FILE))' \
--region=$(REGION)
--love-level=$(LOG_LEVEL)

.PHONY: integration-test-dnsrecord
integration-test-dnsrecord:
Expand All @@ -186,3 +189,4 @@ integration-test-dnsrecord:
--kubeconfig=${KUBECONFIG} \
--access-key-id='$(shell cat $(ACCESS_KEY_ID_FILE))' \
--secret-access-key='$(shell cat $(SECRET_ACCESS_KEY_FILE))'
--love-level=$(LOG_LEVEL)
157 changes: 37 additions & 120 deletions pkg/controller/bastion/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,17 @@ package bastion
import (
"context"
"fmt"
"regexp"
"slices"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
awsv1alpha1 "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws/v1alpha1"
awsclient "github.com/gardener/gardener-extension-provider-aws/pkg/aws/client"
"github.com/gardener/gardener/extensions/pkg/controller"
gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
"github.com/gardener/gardener/pkg/client/kubernetes"
"github.com/gardener/gardener/pkg/extensions"
"k8s.io/apimachinery/pkg/util/sets"

awsv1alpha1 "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws/v1alpha1"
awsclient "github.com/gardener/gardener-extension-provider-aws/pkg/aws/client"
)

// Options contains provider-related information required for setting up
Expand All @@ -46,6 +44,7 @@ type Options struct {
// function does not create any IaaS resources.
func DetermineOptions(ctx context.Context, bastion *extensionsv1alpha1.Bastion, cluster *controller.Cluster, awsClient *awsclient.Client) (*Options, error) {
name := cluster.ObjectMeta.Name
region := cluster.Shoot.Spec.Region
subnetName := name + "-public-utility-z0"
instanceName := fmt.Sprintf("%s-%s-bastion", name, bastion.Name)

Expand All @@ -67,21 +66,23 @@ func DetermineOptions(ctx context.Context, bastion *extensionsv1alpha1.Bastion,
return nil, fmt.Errorf("security group for worker node does not exist yet")
}

cloudProfileConfig, err := getCloudProfileConfig(cluster)
vmDetails, err := DetermineVmDetails(cluster.CloudProfile.Spec)
if err != nil {
return nil, fmt.Errorf("failed to extract cloud provider config from cluster: %w", err)
return nil, fmt.Errorf("failed to determine VM details for bastion host: %w", err)
}

imageID, err := determineImageID(cluster.Shoot, cloudProfileConfig)
cloudProfileConfig, err := getCloudProfileConfig(cluster)
if err != nil {
return nil, fmt.Errorf("failed to determine OS image for bastion host: %w", err)
return nil, fmt.Errorf("failed to extract cloud provider config from cluster: %w", err)
}

instanceType, err := determineInstanceType(ctx, imageID, awsClient)
machineImageVersion, err := getProviderSpecificImage(cloudProfileConfig.MachineImages, vmDetails)
if err != nil {
return nil, fmt.Errorf("failed to determine instance type: %w", err)
return nil, fmt.Errorf("failed to extract image from provider config: %w", err)
}

ami, err := findImageAMIByRegion(machineImageVersion, vmDetails, region)

return &Options{
Shoot: cluster.Shoot,
SubnetID: subnetID,
Expand All @@ -90,8 +91,8 @@ func DetermineOptions(ctx context.Context, bastion *extensionsv1alpha1.Bastion,
WorkerSecurityGroupName: workerSecurityGroupName,
WorkerSecurityGroupID: *workerSecurityGroup.GroupId,
InstanceName: instanceName,
InstanceType: instanceType,
ImageID: imageID,
InstanceType: vmDetails.MachineName,
ImageID: ami,
}, nil
}

Expand Down Expand Up @@ -138,124 +139,40 @@ func getCloudProfileConfig(cluster *extensions.Cluster) (*awsv1alpha1.CloudProfi
return cloudProfileConfig, nil
}

// determineImageID finds the first AMI that is configured for the same region as the shoot cluster.
// If no image is found, an error is returned.
func determineImageID(shoot *gardencorev1beta1.Shoot, providerConfig *awsv1alpha1.CloudProfileConfig) (string, error) {
// TODO(hebelsan): remove version hack after bastion image is well defined, e.g. in cloudProfile
// only allow garden linux versions 1312.x.x because they have ssh enabled by default
re := regexp.MustCompile(`^1312\.\d+\.\d+$`)
for _, image := range providerConfig.MachineImages {
for _, version := range image.Versions {
if image.Name == "gardenlinux" && !re.MatchString(version.Version) {
continue
}
for _, region := range version.Regions {
if region.Name == shoot.Spec.Region {
return region.AMI, nil
}
}
}
}

return "", fmt.Errorf("found no suitable AMI for machines in region %q", shoot.Spec.Region)
}

func determineInstanceType(ctx context.Context, imageID string, awsClient *awsclient.Client) (string, error) {
var preferredType string
imageInfo, err := getImages(ctx, imageID, awsClient)
if err != nil {
return "", err
}

if imageInfo.Architecture == nil {
return "", fmt.Errorf("image architecture is empty")
}

imageArchitecture := imageInfo.Architecture

// default instance type
switch *imageArchitecture {
case "x86_64":
preferredType = "t2.nano"
case "arm64":
preferredType = "t4g.nano"
default:
return "", fmt.Errorf("image architecture not supported")
}

exist, err := getInstanceTypeOfferings(ctx, preferredType, awsClient)
if err != nil {
return "", err
}

if len(exist.InstanceTypeOfferings) != 0 {
return preferredType, nil
}

// filter t type instance
tTypes, err := getInstanceTypeOfferings(ctx, "t*", awsClient)
if err != nil {
return "", err
}

if len(tTypes.InstanceTypeOfferings) == 0 {
return "", fmt.Errorf("no t* instance type offerings available")
}

tTypeSet := sets.NewString()
for _, t := range tTypes.InstanceTypeOfferings {
tTypeSet.Insert(*t.InstanceType)
}

result, err := awsClient.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{
InstanceTypes: aws.StringSlice(tTypeSet.UnsortedList()),
Filters: []*ec2.Filter{
{
Name: aws.String("processor-info.supported-architecture"),
Values: []*string{imageArchitecture},
},
},
// getProviderSpecificImage returns the provider specific MachineImageVersion that matches with the given VmDetails
func getProviderSpecificImage(images []awsv1alpha1.MachineImages, vm VmDetails) (awsv1alpha1.MachineImageVersion, error) {
imageIndex := slices.IndexFunc(images, func(image awsv1alpha1.MachineImages) bool {
return image.Name == vm.ImageBaseName
})

if err != nil {
return "", err
if imageIndex == -1 {
return awsv1alpha1.MachineImageVersion{},
fmt.Errorf("machine image with name %s not found in cloudProfileConfig", vm.ImageBaseName)
}

if len(result.InstanceTypes) == 0 {
return "", fmt.Errorf("no instance types returned for architecture %s and instance types list %v", *imageArchitecture, tTypeSet.UnsortedList())
}
versions := images[imageIndex].Versions
versionIndex := slices.IndexFunc(versions, func(version awsv1alpha1.MachineImageVersion) bool {
return version.Version == vm.ImageVersion
})

if result.InstanceTypes[0].InstanceType == nil {
return "", fmt.Errorf("instanceType is empty")
if versionIndex == -1 {
return awsv1alpha1.MachineImageVersion{},
fmt.Errorf("version %s for arch %s of image %s not found in cloudProfileConfig",
vm.ImageVersion, vm.Architecture, vm.ImageBaseName)
}

return *result.InstanceTypes[0].InstanceType, nil
return versions[versionIndex], nil
}

func getImages(ctx context.Context, ami string, awsClient *awsclient.Client) (*ec2.Image, error) {
imageInfo, err := awsClient.EC2.DescribeImagesWithContext(ctx, &ec2.DescribeImagesInput{
ImageIds: []*string{
aws.String(ami),
},
func findImageAMIByRegion(image awsv1alpha1.MachineImageVersion, vmDetails VmDetails, region string) (string, error) {
regionIndex := slices.IndexFunc(image.Regions, func(RegionAMIMapping awsv1alpha1.RegionAMIMapping) bool {
return RegionAMIMapping.Name == region && RegionAMIMapping.Architecture != nil && *RegionAMIMapping.Architecture == vmDetails.Architecture
})

if err != nil {
return nil, fmt.Errorf("failed to get Images Info: %w", err)
}

if len(imageInfo.Images) == 0 {
return nil, fmt.Errorf("images info not found: %w", err)
if regionIndex == -1 {
return "", fmt.Errorf("image '%s' with version '%s' and architecture '%s' not found in region '%s'",
vmDetails.ImageBaseName, image.Version, vmDetails.Architecture, region)
}
return imageInfo.Images[0], nil
}

func getInstanceTypeOfferings(ctx context.Context, filter string, awsClient *awsclient.Client) (*ec2.DescribeInstanceTypeOfferingsOutput, error) {
return awsClient.EC2.DescribeInstanceTypeOfferingsWithContext(ctx, &ec2.DescribeInstanceTypeOfferingsInput{
Filters: []*ec2.Filter{
{
Name: aws.String("instance-type"),
Values: []*string{aws.String(filter)},
},
},
})
return image.Regions[regionIndex].AMI, nil
}
96 changes: 57 additions & 39 deletions pkg/controller/bastion/options_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,68 +5,86 @@
package bastion

import (
"github.com/gardener/gardener/pkg/apis/core/v1beta1"
apisaws "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws/v1alpha1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

apisaws "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws/v1alpha1"
"k8s.io/utils/ptr"
)

var _ = Describe("Bastion Options", func() {
var region = "test-region"
var image = "gardenlinux"
var version = "1.0.0"
var ami = "test-ami"
var cloudProfileConfig *apisaws.CloudProfileConfig
var shoot *v1beta1.Shoot
var machineName = "test-machine"
var architecture = "amd64"
var amiMapping []apisaws.RegionAMIMapping
var machineImageVersion apisaws.MachineImageVersion
var machineImages []apisaws.MachineImages
var vmDetails VmDetails

BeforeEach(func() {
cloudProfileConfig = &apisaws.CloudProfileConfig{
MachineImages: []apisaws.MachineImages{
{
Name: image,
Versions: []apisaws.MachineImageVersion{
{
Regions: []apisaws.RegionAMIMapping{
{
Name: region,
AMI: ami,
},
},
},
},
},
amiMapping = []apisaws.RegionAMIMapping{
{
Name: region,
AMI: ami,
Architecture: ptr.To(architecture),
},
}
shoot = &v1beta1.Shoot{
Spec: v1beta1.ShootSpec{
Region: region,
machineImageVersion = apisaws.MachineImageVersion{
Version: version,
Regions: amiMapping,
}
machineImages = []apisaws.MachineImages{
{
Name: image,
Versions: []apisaws.MachineImageVersion{machineImageVersion},
},
}
vmDetails = VmDetails{
MachineName: machineName,
Architecture: architecture,
ImageBaseName: image,
ImageVersion: version,
}
})

Context("determineImageID", func() {
var supportedGardenLinuxVersion = "1312.2.0"
var unsupportedGardenLinuxVersion = "1443.1.0"

It("should find imageID", func() {
cloudProfileConfig.MachineImages[0].Versions[0].Version = supportedGardenLinuxVersion
foundAmi, err := determineImageID(shoot, cloudProfileConfig)
Context("getProviderSpecificImage", func() {
It("should succeed for existing image and version", func() {
machineImageVersion, err := getProviderSpecificImage(machineImages, vmDetails)
Expect(err).NotTo(HaveOccurred())
Expect(foundAmi).To(Equal(ami))
Expect(machineImageVersion).To(Equal(machineImageVersion))
})

It("fail if image name does not exist", func() {
vmDetails.ImageBaseName = "unknown"
_, err := getProviderSpecificImage(machineImages, vmDetails)
Expect(err).To(HaveOccurred())
})

It("should fail for unsupported image version", func() {
cloudProfileConfig.MachineImages[0].Versions[0].Version = unsupportedGardenLinuxVersion
_, err := determineImageID(shoot, cloudProfileConfig)
It("fail if image version does not exist", func() {
vmDetails.ImageVersion = "6.6.6"
_, err := getProviderSpecificImage(machineImages, vmDetails)
Expect(err).To(HaveOccurred())
})
})

It("unsupported image version should pass for none gardenlinux images", func() {
cloudProfileConfig.MachineImages[0].Versions[0].Version = unsupportedGardenLinuxVersion
cloudProfileConfig.MachineImages[0].Name = "ubuntu"
foundAmi, err := determineImageID(shoot, cloudProfileConfig)
Context("findImageAMIByRegion", func() {
It("should find image AMI by region", func() {
imageAmi, err := findImageAMIByRegion(machineImageVersion, vmDetails, region)
Expect(err).NotTo(HaveOccurred())
Expect(foundAmi).To(Equal(ami))
Expect(imageAmi).To(Equal(ami))
})

It("fail if region does not match", func() {
_, err := findImageAMIByRegion(machineImageVersion, vmDetails, "unknown")
Expect(err).To(HaveOccurred())
})

It("fail if architecture does not match", func() {
vmDetails.Architecture = "x86"
_, err := findImageAMIByRegion(machineImageVersion, vmDetails, region)
Expect(err).To(HaveOccurred())
})
})
})
Loading

0 comments on commit 1d2e58a

Please sign in to comment.