diff --git a/examples/terraform-aws-endpoint-example/README.md b/examples/terraform-aws-endpoint-example/README.md new file mode 100644 index 0000000000..807ab78c8c --- /dev/null +++ b/examples/terraform-aws-endpoint-example/README.md @@ -0,0 +1,28 @@ +# Terraform AWS S3 Example + +This folder contains a simple Terraform module to demonstrate using custom +endpoints. It's deploying some AWS resources to `http://localhost:5000`, which +is the default port for [moto running in server +mode](http://docs.getmoto.org/en/latest/docs/server_mode.html). This allows for +testing terraform modules locally with no connection to AWS. + +Check out +[test/terraform_aws_endpoint_example_test.go](/test/terraform_aws_endpoint_example_test.go) +to see how you can write automated tests for this module. + +## Running this module manually + +1. Run [Moto locally in server mode](http://docs.getmoto.org/en/latest/docs/server_mode.html) +1. Install [Terraform](https://www.terraform.io/) and make sure it's on your `PATH`. +1. Run `terraform init`. +1. Run `terraform apply`. +1. When you're done, run `terraform destroy`. + +## Running automated tests against this module + +1. Run [Moto locally in server mode](http://docs.getmoto.org/en/latest/docs/server_mode.html) +1. Install [Terraform](https://www.terraform.io/) and make sure it's on your `PATH`. +1. Install [Golang](https://golang.org/) and make sure this code is checked out into your `GOPATH`. +1. `cd test` +1. `dep ensure` +1. `go test -v -run TestTerraformAwsEndpointExample` diff --git a/examples/terraform-aws-endpoint-example/main.tf b/examples/terraform-aws-endpoint-example/main.tf new file mode 100644 index 0000000000..3a34b6a5a8 --- /dev/null +++ b/examples/terraform-aws-endpoint-example/main.tf @@ -0,0 +1,131 @@ +# --------------------------------------------------------------------------------------------------------------------- +# PIN TERRAFORM VERSION TO >= 0.12 +# The examples have been upgraded to 0.12 syntax +# --------------------------------------------------------------------------------------------------------------------- +provider "aws" { + region = var.region + access_key = "dummy" + secret_key = "dummy" + + endpoints { + sts = "http://localhost:5000" + s3 = "http://localhost:5000" + } +} + +terraform { + # This module is now only being tested with Terraform 0.13.x. However, to make upgrading easier, we are setting + # 0.12.26 as the minimum version, as that version added support for required_providers with source URLs, making it + # forwards compatible with 0.13.x code. + required_version = ">= 0.12.26" +} + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A S3 BUCKET WITH VERSIONING ENABLED INCLUDING TAGS TO A LOCAL ENDPOINT +# See test/terraform_aws_endpoint_example_test.go for how to write automated tests for this code. +# --------------------------------------------------------------------------------------------------------------------- + +# Deploy and configure test S3 bucket with versioning and access log +resource "aws_s3_bucket" "test_bucket" { + bucket = "${local.aws_account_id}-${var.tag_bucket_name}" + + tags = { + Name = var.tag_bucket_name + Environment = var.tag_bucket_environment + } +} + +resource "aws_s3_bucket_logging" "test_bucket" { + bucket = aws_s3_bucket.test_bucket.id + target_bucket = aws_s3_bucket.test_bucket_logs.id + target_prefix = "TFStateLogs/" +} + +resource "aws_s3_bucket_versioning" "test_bucket" { + bucket = aws_s3_bucket.test_bucket.id + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_acl" "test_bucket" { + bucket = aws_s3_bucket.test_bucket.id + acl = "private" +} + + +# Deploy S3 bucket to collect access logs for test bucket +resource "aws_s3_bucket" "test_bucket_logs" { + bucket = "${local.aws_account_id}-${var.tag_bucket_name}-logs" + + tags = { + Name = "${local.aws_account_id}-${var.tag_bucket_name}-logs" + Environment = var.tag_bucket_environment + } + + force_destroy = true +} + +resource "aws_s3_bucket_acl" "test_bucket_logs" { + bucket = aws_s3_bucket.test_bucket_logs.id + acl = "log-delivery-write" +} + +# Configure bucket access policies + +resource "aws_s3_bucket_policy" "bucket_access_policy" { + count = var.with_policy ? 1 : 0 + bucket = aws_s3_bucket.test_bucket.id + policy = data.aws_iam_policy_document.s3_bucket_policy.json +} + +data "aws_iam_policy_document" "s3_bucket_policy" { + statement { + effect = "Allow" + principals { + # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to + # force an interpolation expression to be interpreted as a list by wrapping it + # in an extra set of list brackets. That form was supported for compatibility in + # v0.11, but is no longer supported in Terraform v0.12. + # + # If the expression in the following list itself returns a list, remove the + # brackets to avoid interpretation as a list of lists. If the expression + # returns a single list item then leave it as-is and remove this TODO comment. + identifiers = [local.aws_account_id] + type = "AWS" + } + actions = ["*"] + resources = ["${aws_s3_bucket.test_bucket.arn}/*"] + } + + statement { + effect = "Deny" + principals { + identifiers = ["*"] + type = "AWS" + } + actions = ["*"] + resources = ["${aws_s3_bucket.test_bucket.arn}/*"] + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = [ + "false", + ] + } + } +} + +# --------------------------------------------------------------------------------------------------------------------- +# LOCALS +# Used to represent any data that requires complex expressions/interpolations +# --------------------------------------------------------------------------------------------------------------------- + +data "aws_caller_identity" "current" { +} + +locals { + aws_account_id = data.aws_caller_identity.current.account_id +} + diff --git a/examples/terraform-aws-endpoint-example/outputs.tf b/examples/terraform-aws-endpoint-example/outputs.tf new file mode 100644 index 0000000000..8a9719a4f2 --- /dev/null +++ b/examples/terraform-aws-endpoint-example/outputs.tf @@ -0,0 +1,15 @@ +output "bucket_id" { + value = aws_s3_bucket.test_bucket.id +} + +output "bucket_arn" { + value = aws_s3_bucket.test_bucket.arn +} + +output "logging_target_bucket" { + value = aws_s3_bucket_logging.test_bucket.target_bucket +} + +output "logging_target_prefix" { + value = aws_s3_bucket_logging.test_bucket.target_prefix +} diff --git a/examples/terraform-aws-endpoint-example/variables.tf b/examples/terraform-aws-endpoint-example/variables.tf new file mode 100644 index 0000000000..55ba48f2f3 --- /dev/null +++ b/examples/terraform-aws-endpoint-example/variables.tf @@ -0,0 +1,40 @@ +# --------------------------------------------------------------------------------------------------------------------- +# ENVIRONMENT VARIABLES +# Define these secrets as environment variables +# --------------------------------------------------------------------------------------------------------------------- + +# AWS_ACCESS_KEY_ID +# AWS_SECRET_ACCESS_KEY + +# --------------------------------------------------------------------------------------------------------------------- +# REQUIRED PARAMETERS +# You must provide a value for each of these parameters. +# --------------------------------------------------------------------------------------------------------------------- +variable "region" { + description = "The AWS region to deploy to" + type = string +} + +# --------------------------------------------------------------------------------------------------------------------- +# OPTIONAL PARAMETERS +# These parameters have reasonable defaults. +# --------------------------------------------------------------------------------------------------------------------- + +variable "with_policy" { + description = "If set to `true`, the bucket will be created with a bucket policy." + type = bool + default = false +} + +variable "tag_bucket_name" { + description = "The Name tag to set for the S3 Bucket." + type = string + default = "Test Bucket" +} + +variable "tag_bucket_environment" { + description = "The Environment tag to set for the S3 Bucket." + type = string + default = "Test" +} + diff --git a/modules/aws/auth.go b/modules/aws/auth.go index f2aa6f78c5..55c0358748 100644 --- a/modules/aws/auth.go +++ b/modules/aws/auth.go @@ -16,6 +16,7 @@ import ( const ( AuthAssumeRoleEnvVar = "TERRATEST_IAM_ROLE" // OS environment variable name through which Assume Role ARN may be passed for authentication + CustomEndpointEnvVar = "TERRATEST_CUSTOM_ENDPOINT" // Custom endpoint to use as aws service ) // NewAuthenticatedSession creates an AWS session following to standard AWS authentication workflow. @@ -32,6 +33,11 @@ func NewAuthenticatedSession(region string) (*session.Session, error) { func NewAuthenticatedSessionFromDefaultCredentials(region string) (*session.Session, error) { awsConfig := aws.NewConfig().WithRegion(region) + if customEndpoint, ok := os.LookupEnv(CustomEndpointEnvVar); ok { + awsConfig.WithEndpoint(customEndpoint) + } + awsConfig.WithEndpoint(os.Getenv(CustomEndpointEnvVar)) + sessionOptions := session.Options{ Config: *awsConfig, SharedConfigState: session.SharedConfigEnable, @@ -68,7 +74,13 @@ func NewAuthenticatedSessionFromRole(region string, roleARN string) (*session.Se // CreateAwsSessionFromRole returns a new AWS session after assuming the role // whose ARN is provided in roleARN. func CreateAwsSessionFromRole(region string, roleARN string) (*session.Session, error) { - sess, err := session.NewSession(aws.NewConfig().WithRegion(region)) + awsConfig := aws.NewConfig().WithRegion(region) + + if customEndpoint, ok := os.LookupEnv(CustomEndpointEnvVar); ok { + awsConfig.WithEndpoint(customEndpoint) + } + + sess, err := session.NewSession(awsConfig) if err != nil { return nil, err } @@ -86,8 +98,15 @@ func AssumeRole(sess *session.Session, roleARN string) *session.Session { // CreateAwsSessionWithCreds creates a new AWS session using explicit credentials. This is useful if you want to create an IAM User dynamically and // create an AWS session authenticated as the new IAM User. func CreateAwsSessionWithCreds(region string, accessKeyID string, secretAccessKey string) (*session.Session, error) { - creds := CreateAwsCredentials(accessKeyID, secretAccessKey) - return session.NewSession(aws.NewConfig().WithRegion(region).WithCredentials(creds)) + awsConfig := aws.NewConfig().WithRegion(region) + + if customEndpoint, ok := os.LookupEnv(CustomEndpointEnvVar); ok { + awsConfig.WithEndpoint(customEndpoint) + } + + awsConfig.WithCredentials(CreateAwsCredentials(accessKeyID, secretAccessKey)) + + return session.NewSession(awsConfig) } // CreateAwsSessionWithMfa creates a new AWS session authenticated using an MFA token retrieved using the given STS client and MFA Device. @@ -109,8 +128,15 @@ func CreateAwsSessionWithMfa(region string, stsClient *sts.STS, mfaDevice *iam.V secretAccessKey := *output.Credentials.SecretAccessKey sessionToken := *output.Credentials.SessionToken - creds := CreateAwsCredentialsWithSessionToken(accessKeyID, secretAccessKey, sessionToken) - return session.NewSession(aws.NewConfig().WithRegion(region).WithCredentials(creds)) + awsConfig := aws.NewConfig().WithRegion(region) + + if customEndpoint, ok := os.LookupEnv(CustomEndpointEnvVar); ok { + awsConfig.WithEndpoint(customEndpoint) + } + + awsConfig.WithCredentials(CreateAwsCredentialsWithSessionToken(accessKeyID, secretAccessKey, sessionToken)) + + return session.NewSession(awsConfig) } // CreateAwsCredentials creates an AWS Credentials configuration with specific AWS credentials. diff --git a/test/terraform_aws_endpoint_example_test.go b/test/terraform_aws_endpoint_example_test.go new file mode 100644 index 0000000000..eb1ce1fe26 --- /dev/null +++ b/test/terraform_aws_endpoint_example_test.go @@ -0,0 +1,75 @@ +package test + +import ( + "os" + "fmt" + "strings" + "testing" + + "github.com/gruntwork-io/terratest/modules/aws" + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/assert" +) + +// An example of how to test the Terraform module in examples/terraform-aws-endpoint-example using Terratest. +func TestTerraformAwsEndpointExample(t *testing.T) { + t.Parallel() + + // Set a custom endpoint for AWS, and set the keys to dummy keys to + // pass that check + os.Setenv("TERRATEST_CUSTOM_ENDPOINT", "http://localhost:5000") + os.Setenv("AWS_ACCESS_KEY_ID", "dummy") + os.Setenv("AWS_SECRET_ACCESS_KEY", "dummy") + + // Give this S3 Bucket a unique ID for a name tag so we can distinguish it from any other Buckets provisioned + // in your AWS account + expectedName := fmt.Sprintf("terratest-aws-endpoint-example-%s", strings.ToLower(random.UniqueId())) + + // Give this S3 Bucket an environment to operate as a part of for the purposes of resource tagging + expectedEnvironment := "Automated Testing" + + // Pick a random AWS region to test in. This helps ensure your code works in all regions. + awsRegion := aws.GetRandomStableRegion(t, nil, nil) + + // Construct the terraform options with default retryable errors to handle the most common retryable errors in + // terraform testing. + terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ + // The path to where our Terraform code is located + TerraformDir: "../examples/terraform-aws-endpoint-example", + + // Variables to pass to our Terraform code using -var options + Vars: map[string]interface{}{ + "tag_bucket_name": expectedName, + "tag_bucket_environment": expectedEnvironment, + "with_policy": "true", + "region": awsRegion, + }, + }) + + // At the end of the test, run `terraform destroy` to clean up any resources that were created + defer terraform.Destroy(t, terraformOptions) + + // This will run `terraform init` and `terraform apply` and fail the test if there are any errors + terraform.InitAndApply(t, terraformOptions) + + // Run `terraform output` to get the value of an output variable + bucketID := terraform.Output(t, terraformOptions, "bucket_id") + + // Verify that our Bucket has versioning enabled + actualStatus := aws.GetS3BucketVersioning(t, awsRegion, bucketID) + expectedStatus := "Enabled" + assert.Equal(t, expectedStatus, actualStatus) + + // Verify that our Bucket has a policy attached + aws.AssertS3BucketPolicyExists(t, awsRegion, bucketID) + + // Verify that our bucket has server access logging TargetBucket set to what's expected + loggingTargetBucket := aws.GetS3BucketLoggingTarget(t, awsRegion, bucketID) + expectedLogsTargetBucket := fmt.Sprintf("%s-logs", bucketID) + loggingObjectTargetPrefix := aws.GetS3BucketLoggingTargetPrefix(t, awsRegion, bucketID) + expectedLogsTargetPrefix := "TFStateLogs/" + + assert.Equal(t, expectedLogsTargetBucket, loggingTargetBucket) + assert.Equal(t, expectedLogsTargetPrefix, loggingObjectTargetPrefix) +}