Add e2e tests for ProviderConfigs

Signed-off-by: Erhan Cagirici <erhan@upbound.io>

.gitignore

@@ -10,6 +10,9 @@ cover.out
 /.vendor-new
 .DS_Store
 kubeconfig
+e2e/*/.cache
+e2e/*/.work
+e2e/*/_output
 
 # ignore IDE folders
 .vscode/

Makefile

@@ -209,6 +209,38 @@ uptest: $(UPTEST) $(KUBECTL) $(KUTTL)
 	@KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) CROSSPLANE_NAMESPACE=$(CROSSPLANE_NAMESPACE) $(UPTEST) e2e "${UPTEST_EXAMPLE_LIST}" --data-source="${UPTEST_DATASOURCE_PATH}" --setup-script=cluster/test/setup.sh --default-conditions="Test" || $(FAIL)
 	@$(OK) running automated tests
 
+# This target triggers an e2e test for testing provider configs.
+# It first builds and publishes the
+# provider-family-aws, provider-aws-ec2 and provider-aws-rds.
+# Then triggers the e2e provider config tests via `make`,
+# which resides in the `e2e/providerconfig-aws-e2e-test` directory
+#
+# For the e2e test, an EKS cluster is created and some demo resources are
+# created with different provider configs. The demo resources are from
+# AWS EC2 and RDS providers.
+# Therefore, the provider packages needs to be published to a registry
+# that the EKS cluster has access to. This defaults to "xpkg.upbound.io"
+# If another registry needs to be used, `XPKG_REG_ORGS` needs to be overridden
+# with a registry that the EKS cluster has access, while invoking this make target.
+#
+# This target also requires the `UPTEST_CLOUD_CREDENTIALS` environment variable
+# to be set. This is used for provisioning the target E2E test environment,
+# including the EKS cluster and necessary environments.
+providerconfig-e2e:
+	$(MAKE) SUBPACKAGES="ec2 rds config" build.all publish
+	AWS_FAMILY_PACKAGE_IMAGE="$(XPKG_REG_ORGS)/provider-family-aws:$(VERSION)" \
+	AWS_EC2_PACKAGE_IMAGE="$(XPKG_REG_ORGS)/provider-aws-ec2:$(VERSION)" \
+	AWS_RDS_PACKAGE_IMAGE="$(XPKG_REG_ORGS)/provider-aws-rds:$(VERSION)" \
+	TARGET_CROSSPLANE_VERSION="1.15.2" \
+		$(MAKE) -C e2e/providerconfig-aws-e2e-test e2e
+
+providerconfig-e2e-nopublish:
+	AWS_FAMILY_PACKAGE_IMAGE="$(XPKG_REG_ORGS)/provider-family-aws:$(VERSION)" \
+	AWS_EC2_PACKAGE_IMAGE="$(XPKG_REG_ORGS)/provider-aws-ec2:$(VERSION)" \
+	AWS_RDS_PACKAGE_IMAGE="$(XPKG_REG_ORGS)/provider-aws-rds:$(VERSION)" \
+	TARGET_CROSSPLANE_VERSION="1.15.2" \
+		$(MAKE) -C e2e/providerconfig-aws-e2e-test e2e
+
 uptest-local:
 	@$(WARN) "this target is deprecated, please use 'make uptest' instead"
 

e2e/providerconfig-aws-e2e-test/Makefile

@@ -0,0 +1,106 @@
+# Project Setup
+PROJECT_NAME := providerconfig-aws-e2e-test
+PROJECT_REPO := github.com/upbound/provider-upjet-aws
+
+# NOTE(hasheddan): the platform is insignificant here as Configuration package
+# images are not architecture-specific. We constrain to one platform to avoid
+# needlessly pushing a multi-arch image.
+PLATFORMS ?= linux_amd64
+-include ../../build/makelib/common.mk
+
+# ====================================================================================
+# Setup Kubernetes tools
+
+KIND_VERSION = v0.22.0
+UP_VERSION = v0.28.0
+UP_CHANNEL = stable
+UPTEST_VERSION = v0.11.1
+YQ_VERSION = v4.40.5
+
+-include ../../build/makelib/k8s_tools.mk
+# ====================================================================================
+# Setup XPKG
+XPKG_DIR = $(shell pwd)/package
+XPKG_IGNORE = .github/workflows/*.yaml,.github/workflows/*.yml,examples/*.yaml,.work/uptest-datasource.yaml,.cache/**,_output/**
+XPKG_REG_ORGS ?= xpkg.upbound.io/upbound
+# NOTE(hasheddan): skip promoting on xpkg.upbound.io as channel tags are
+# inferred.
+XPKG_REG_ORGS_NO_PROMOTE ?= xpkg.upbound.io/upbound
+XPKGS = $(PROJECT_NAME)
+-include ../../build/makelib/xpkg.mk
+
+CROSSPLANE_NAMESPACE = upbound-system
+CROSSPLANE_ARGS = "--enable-usages,--debug"
+-include ../../build/makelib/local.xpkg.mk
+-include ../../build/makelib/controlplane.mk
+
+
+# ====================================================================================
+# Targets
+
+# run `make help` to see the targets and options
+
+# We want submodules to be set up the first time `make` is run.
+# We manage the build/ folder and its Makefiles as a submodule.
+# The first time `make` is run, the includes of build/*.mk files will
+# all fail, and this target will be run. The next time, the default as defined
+# by the includes will be run instead.
+fallthrough: submodules
+	@echo Initial setup complete. Running make again . . .
+	@make
+
+# Update the submodules, such as the common build scripts.
+submodules:
+	@git submodule sync
+	@git submodule update --init --recursive
+
+# We must ensure up is installed in tool cache prior to build as including the k8s_tools machinery prior to the xpkg
+# machinery sets UP to point to tool cache.
+build.init: $(UP)
+
+# ====================================================================================
+# End to End Testing
+
+# This target requires the following environment variables to be set:
+# - UPTEST_CLOUD_CREDENTIALS, cloud credentials for the provider being tested, e.g. export UPTEST_CLOUD_CREDENTIALS=$(cat ~/.aws/credentials)
+# - To ensure the proper functioning of the end-to-end test resource pre-deletion hook, it is crucial to arrange your resources appropriately.
+#   You can check the basic implementation here: https://github.com/upbound/uptest/blob/main/internal/templates/01-delete.yaml.tmpl.
+# - UPTEST_DATASOURCE_PATH (optional), see https://github.com/upbound/uptest#injecting-dynamic-values-and-datasource
+uptest: $(UPTEST) $(KUBECTL) $(KUTTL) $(YQ)
+	@$(INFO) running automated tests
+	@KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) CROSSPLANE_NAMESPACE=$(CROSSPLANE_NAMESPACE) $(UPTEST) e2e package/examples/e2etestcluster-claim.yaml --data-source="${UPTEST_DATASOURCE_PATH}" --setup-script=test/setup.sh --default-timeout=5400 || $(FAIL)
+	@$(OK) running automated tests
+
+# This target requires the following environment variables to be set:
+# - UPTEST_CLOUD_CREDENTIALS, cloud credentials for the provider being tested, e.g. export UPTEST_CLOUD_CREDENTIALS=$(cat ~/.aws/credentials)
+e2e: build controlplane.up local.xpkg.deploy.configuration.$(PROJECT_NAME) uptest-e2e
+
+e2e-lite: build controlplane.up local.xpkg.deploy.configuration.$(PROJECT_NAME)
+
+uptest-e2e: $(UPTEST) $(KUBECTL) $(KUTTL) $(YQ)
+	@$(INFO) dump e2e claim:
+	@mkdir -p "_output"
+	@$(YQ) '(.spec.parameters.targetClusterParameters.provider.familyPackage = env(AWS_FAMILY_PACKAGE_IMAGE)) | \
+		   (.spec.parameters.targetClusterParameters.provider.ec2Package = env(AWS_EC2_PACKAGE_IMAGE)) | \
+		   (.spec.parameters.targetClusterParameters.provider.rdsPackage = env(AWS_RDS_PACKAGE_IMAGE)) | \
+		   (.spec.parameters.targetClusterParameters.crossplaneVersion = env(TARGET_CROSSPLANE_VERSION)) ' \
+				package/examples/e2etestcluster-claim.yaml > '_output/e2etestcluster-claim.yaml'
+	if [ -n "${AWS_EKS_IAM_DEFAULT_ADMIN_ROLE}" ]; \
+	then \
+  		echo "overriding EKS cluster default IAM role from environment";\
+  		$(YQ) -i '(.spec.parameters.iam.roleArn = env(AWS_EKS_IAM_DEFAULT_ADMIN_ROLE)) ' _output/e2etestcluster-claim.yaml;\
+  	fi
+	@cat _output/e2etestcluster-claim.yaml
+	@$(INFO) running automated tests
+	@KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) CROSSPLANE_NAMESPACE=$(CROSSPLANE_NAMESPACE) $(UPTEST) e2e _output/e2etestcluster-claim.yaml --data-source="${UPTEST_DATASOURCE_PATH}" --setup-script=test/setup.sh --default-timeout=5400 || $(FAIL)
+	@$(OK) running automated tests
+
+render:
+	crossplane beta render package/examples/e2etestcluster-claim.yaml package/apis/e2etestcluster/composition.yaml package/examples/functions.yaml -r
+
+yamllint:
+	@$(INFO) running yamllint
+	@yamllint ./apis || $(FAIL)
+	@$(OK) running yamllint
+
+.PHONY: uptest e2e render yamllint

e2e/providerconfig-aws-e2e-test/README.md

@@ -0,0 +1,190 @@
+# AWS ProviderConfig E2E testing
+
+## Introduction
+
+This Crossplane configuration package aims to provide a base environment for testing 
+various `ProviderConfig` scenarios of `provider-upjet-aws`.
+
+It provisions:
+- An AWS EKS cluster
+- AWS IRSA-related IAM resources for the EKS cluster for testing IRSA authentication
+- AWS WebIdentity authentication related IAM resources for the EKS cluster, to test WebIdentity authentication
+After creating the EKS cluster, it deploys:
+- `Crossplane` into the EKS cluster
+- `DeploymentRuntimeConfig`s for XP providers to enable IRSA
+- `provider-family-aws` and 2 example providers for testing
+  - `provider-aws-ec2` and `provider-aws-rds`
+- various AWS `ProviderConfig` manifests for testing scenarios
+- example Managed Resources (MRs) from AWS EC2 and RDS groups, referencing the `ProviderConfig`s in test 
+
+## Package Structure
+
+The package consists of the Composite Resource (XR) 
+`xe2etestclusters.aws.platformref.upbound.io`
+
+This composite resource makes use of the existing configuration 
+packages `configuration-aws-eks` and `configuration-aws-eks-irsa`.
+
+It is structured in a way that starting from a local crossplane control plane, 
+it sets up another control plane in an EKS cluster with Crossplane. 
+Via the `provider-kubernetes` and `provider-helm` at the local control plane, 
+the remote EKS control plane is bootstrapped and relevant test resources are deployed.
+This can be considered as a "A crossplane control plane is Managed by another Crossplane control plane".
+This setup allows conducting tests from a local control plane.
+
+![pc-e2e-diagram.png](docs%2Fimg%2Fpc-e2e-diagram.png)
+
+Explicit deletion ordering inside composition is implemented via `Usages`. For some resources, 
+Crossplane runtime already handles the implicit dependencies such as MR <-> ProviderConfig, ProviderConfig <-> Providers.
+The dependencies are depicted in the diagram above.
+
+### `e2etestcluster.platformref.upbound.io` XRC
+
+You can find an example test cluster claim at [package/examples/e2etestcluster-claim.yaml](package%2Fexamples%2Fe2etestcluster-claim.yaml)
+When this claim is is created and ready, it means that the tests are passing.
+
+```yaml
+apiVersion: aws.platformref.upbound.io/v1alpha1
+kind: E2ETestCluster
+metadata:
+  name: aws-pc-e2e-test
+  namespace: default
+spec:
+  compositeDeletePolicy: Foreground
+  parameters:
+    id: aws-pc-e2e-test
+    region: us-west-2 # EKS cluster region
+    version: "1.28" # EKS cluster k8s version
+    iam:
+      # replace with your custom roleArn that will administer the EKS cluster:
+      roleArn: "arn:aws:iam::123456789012:role/mydefaulteksadminrole"
+    nodes: # eks nodes configuration
+      count: 1  
+      instanceType: t3.medium
+    irsa: # IRSA configuration for the AWS role that will be used by XP providers
+      condition: StringEquals
+      # The policy of the IRSA role
+      policyDocument: |
+        {
+            "Version": "2012-10-17",
+            "Statement": [
+                {
+                    "Action": [
+                        "ec2:*"
+                    ],
+                    "Effect": "Allow",
+                    "Resource": "*"
+                }
+            ]
+        } 
+      serviceAccount: # name of the k8s service account to be created for provider pods 
+        name: my-xpsa
+        namespace: upbound-system
+    targetClusterParameters: # the parameters for the target EKS control plane cluster 
+      provider: # provider package urls to be used in testing
+        familyPackage: "xpkg.upbound.io/upbound/provider-family-aws:v1.3.0"
+        ec2Package: "xpkg.upbound.io/upbound/provider-aws-ec2:v1.3.0"
+        rdsPackage: "xpkg.upbound.io/upbound/provider-aws-rds:v1.3.0"
+      crossplaneVersion: 1.15.2 # the crossplane version to be installed in the testing control plane
+  writeConnectionSecretToRef:
+    name: aws-pc-e2e-test-kubeconfig
+status:
+  irsa:
+    roleArn: irsa-role-arn
+    chainedRoleARNs:
+      - "chained-role-arn"
+  webIdentity:
+    roleArn: webid-role-arn
+    chainedRoleARNs:
+      - "chained-role-arn"
+
+```
+
+## Usage
+
+### Prerequisites
+
+- An AWS account and relevant credentials capable of creating and managing EC2, EKS and IAM resources
+- An OCI image registry from which the EKS cluster can pull images (e.g. Dockerhub, xpkg.upbound.io)
+
+### Utilizing Uptest
+
+In order to conduct an e2e test using uptest, the `make` targets can be used.
+
+### option 1. inside configuration package: make target `e2e`
+
+This make target:
+- builds the `providerconfig-aws-e2e-test` configuration package
+- spins up a local `kind` cluster
+- deploys the `providerconfig-aws-e2e-test` configuration package to the `kind` cluster
+- runs the e2e tests
+
+This make target expects the target AWS provider packages in test to be already built and pushed to
+a registry that the target EKS cluster can reach.  
+
+The make target expects the following environment variables to be set:
+
+- `AWS_FAMILY_PACKAGE_IMAGE`: The package URL for `provider-family-aws` 
+- `AWS_EC2_PACKAGE_IMAGE`: The package URL for `provider-aws-ec2`
+- `AWS_RDS_PACKAGE_IMAGE`: The package URL for `provider-aws-rds`
+- `AWS_EKS_IAM_DEFAULT_ADMIN_ROLE`: the ARN of an existing IAM role. This will be assigned as the E2E test EKS cluster default admin
+- `TARGET_CROSSPLANE_VERSION`: The target crossplane version to be deployed into the testing cluster
+- `UPTEST_CLOUD_CREDENTIALS`: The AWS credentials for the AWS account that the e2e tests will run on. Should be in the format of AWS CLI INI config. 
+
+An example usage:
+
+my-aws-creds.txt
+```ini
+[default]
+aws_access_key_id = YOUR-AWS-ACCESS-KEY
+aws_secret_access_key = your-aws-secret-access-key
+```
+
+```shell
+export AWS_FAMILY_PACKAGE_IMAGE="xpkg.upbound.io/upbound/provider-family-aws:1.4.0"
+export AWS_EC2_PACKAGE_IMAGE="xpkg.upbound.io/upbound/provider-aws-ec2:1.4.0"
+export AWS_RDS_PACKAGE_IMAGE="xpkg.upbound.io/upbound/provider-aws-rds:1.4.0"
+export AWS_EKS_IAM_DEFAULT_ADMIN_ROLE="arn:aws:iam::123456789012:role/mydefaulteksadminrole"
+export TARGET_CROSSPLANE_VERSION="1.15.2"
+export UPTEST_CLOUD_CREDENTIALS="$(cat my-aws-creds.txt)"
+# from repo root
+make -C e2e/providerconfig-aws-e2e-test e2e
+```
+
+### option 2. with provider image publish: target `providerconfig-e2e`
+
+This make target:
+- builds and publishes the providers
+- builds the `providerconfig-aws-e2e-test` configuration package
+- spins up a local `kind` cluster
+- deploys the `providerconfig-aws-e2e-test` configuration package to the `kind` cluster
+- runs the e2e tests using `uptest` with the published provider images
+
+The make target expects 
+- `XPKG_REG_ORGS`: the target OCI repository URL for provider images to be published
+- `VERSION`: the version tag of the published provider images
+- `UPTEST_CLOUD_CREDENTIALS`: The AWS credentials for the AWS account that the e2e tests will run on. Should be in the format of AWS CLI INI config.
+- `AWS_EKS_IAM_DEFAULT_ADMIN_ROLE`: the ARN of an existing IAM role. This will be assigned as the E2E test EKS cluster default admin
+
+example usage:
+```shell
+export UPTEST_CLOUD_CREDENTIALS="$(cat my-aws-creds.txt)"
+export AWS_EKS_IAM_DEFAULT_ADMIN_ROLE="arn:aws:iam::123456789012:role/mydefaulteksadminrole"
+# from repo root
+make VERSION=v1.4.0-testversion XPKG_REG_ORGS=index.docker.io/erhancag providerconfig-e2e
+```
+
+### Via Github Actions from PR 
+TBD
+
+### Manual
+In your desired k8s environment:
+
+- Deploy Crossplane
+- build and publish the e2e testing configuration package
+- install configuration package
+- create the example claim in the `package/examples/e2etestcluster-claim.yaml`. Before creation, modify the claim accordingly if needed.
+- wait for the claim to be ready.
+
+## Extending Test Scenarios
+TBD