From 5f5459a1fbc5efc0fde9f8edea47f7549eff9375 Mon Sep 17 00:00:00 2001 From: clint shryock Date: Thu, 10 Dec 2015 15:43:13 -0600 Subject: [PATCH 1/2] provider/aws: Refactor AWS Authentication chain - update auth checking to check metadata header - refactor tests to not export os env vars --- builtin/providers/aws/config.go | 66 +++++- builtin/providers/aws/config_test.go | 299 +++++++++++++++++++++++++++ builtin/providers/aws/provider.go | 95 +-------- 3 files changed, 369 insertions(+), 91 deletions(-) create mode 100644 builtin/providers/aws/config_test.go diff --git a/builtin/providers/aws/config.go b/builtin/providers/aws/config.go index d8a9ff862d86..e7c7628dc412 100644 --- a/builtin/providers/aws/config.go +++ b/builtin/providers/aws/config.go @@ -3,14 +3,19 @@ package aws import ( "fmt" "log" + "net/http" + "os" "strings" + "time" "github.com/hashicorp/go-cleanhttp" "github.com/hashicorp/go-multierror" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" + awsCredentials "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" + "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/cloudformation" @@ -104,9 +109,14 @@ func (c *Config) Client() (interface{}, error) { client.region = c.Region log.Println("[INFO] Building AWS auth structure") - // We fetched all credential sources in Provider. If they are - // available, they'll already be in c. See Provider definition. - creds := credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.Token) + creds := getCreds(c.AccessKey, c.SecretKey, c.Token) + // Call Get to check for credential provider. If nothing found, we'll get an + // error, and we can present it nicely to the user + _, err = creds.Get() + if err != nil { + errs = append(errs, fmt.Errorf("Error loading credentials for AWS Provider: %s", err)) + return nil, &multierror.Error{Errors: errs} + } awsConfig := &aws.Config{ Credentials: creds, Region: aws.String(c.Region), @@ -118,7 +128,7 @@ func (c *Config) Client() (interface{}, error) { sess := session.New(awsConfig) client.iamconn = iam.New(sess) - err := c.ValidateCredentials(client.iamconn) + err = c.ValidateCredentials(client.iamconn) if err != nil { errs = append(errs, err) } @@ -316,3 +326,49 @@ func (c *Config) ValidateAccountId(iamconn *iam.IAM) error { return nil } + +// This function is responsible for reading credentials from the +// environment in the case that they're not explicitly specified +// in the Terraform configuration. +func getCreds(key, secret, token string) *awsCredentials.Credentials { + // build a chain provider, lazy-evaulated by aws-sdk + providers := []awsCredentials.Provider{ + &awsCredentials.StaticProvider{Value: awsCredentials.Value{ + AccessKeyID: key, + SecretAccessKey: secret, + SessionToken: token, + }}, + &awsCredentials.EnvProvider{}, + &awsCredentials.SharedCredentialsProvider{}, + } + + // We only look in the EC2 metadata API if we can connect + // to the metadata service within a reasonable amount of time + metadataURL := os.Getenv("AWS_METADATA_URL") + if metadataURL == "" { + metadataURL = "http://169.254.169.254:80/latest" + } + c := http.Client{ + Timeout: 100 * time.Millisecond, + } + + r, err := c.Get(metadataURL) + var useIAM bool + if err == nil { + if r.Header["Server"] != nil && strings.Contains(r.Header["Server"][0], "EC2") { + useIAM = true + } + } + + if useIAM { + log.Printf("[DEBUG] EC2 Metadata service found, adding EC2 Role Credential Provider") + providers = append(providers, &ec2rolecreds.EC2RoleProvider{ + Client: ec2metadata.New(session.New(&aws.Config{ + Endpoint: aws.String(metadataURL), + })), + }) + } else { + log.Printf("[DEBUG] EC2 Metadata service not found, not adding EC2 Role Credential Provider") + } + return awsCredentials.NewChainCredentials(providers) +} diff --git a/builtin/providers/aws/config_test.go b/builtin/providers/aws/config_test.go new file mode 100644 index 000000000000..316bf1893975 --- /dev/null +++ b/builtin/providers/aws/config_test.go @@ -0,0 +1,299 @@ +package aws + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/aws/aws-sdk-go/aws/awserr" +) + +func TestAWSConfig_shouldError(t *testing.T) { + resetEnv := unsetEnv(t) + defer resetEnv() + cfg := Config{} + + c := getCreds(cfg.AccessKey, cfg.SecretKey, cfg.Token) + _, err := c.Get() + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() != "NoCredentialProviders" { + t.Fatalf("Expected NoCredentialProviders error") + } + } + if err == nil { + t.Fatalf("Expected an error with empty env, keys, and IAM in AWS Config") + } +} + +func TestAWSConfig_shouldBeStatic(t *testing.T) { + simple := []struct { + Key, Secret, Token string + }{ + { + Key: "test", + Secret: "secret", + }, { + Key: "test", + Secret: "test", + Token: "test", + }, + } + + for _, c := range simple { + cfg := Config{ + AccessKey: c.Key, + SecretKey: c.Secret, + Token: c.Token, + } + + creds := getCreds(cfg.AccessKey, cfg.SecretKey, cfg.Token) + if creds == nil { + t.Fatalf("Expected a static creds provider to be returned") + } + v, err := creds.Get() + if err != nil { + t.Fatalf("Error gettings creds: %s", err) + } + if v.AccessKeyID != c.Key { + t.Fatalf("AccessKeyID mismatch, expected: (%s), got (%s)", c.Key, v.AccessKeyID) + } + if v.SecretAccessKey != c.Secret { + t.Fatalf("SecretAccessKey mismatch, expected: (%s), got (%s)", c.Secret, v.SecretAccessKey) + } + if v.SessionToken != c.Token { + t.Fatalf("SessionToken mismatch, expected: (%s), got (%s)", c.Token, v.SessionToken) + } + } +} + +// TestAWSConfig_shouldIAM is designed to test the scenario of running Terraform +// from an EC2 instance, without environment variables or manually supplied +// credentials. +func TestAWSConfig_shouldIAM(t *testing.T) { + // clear AWS_* environment variables + resetEnv := unsetEnv(t) + defer resetEnv() + + // capture the test server's close method, to call after the test returns + ts := awsEnv(t) + defer ts() + + // An empty config, no key supplied + cfg := Config{} + + creds := getCreds(cfg.AccessKey, cfg.SecretKey, cfg.Token) + if creds == nil { + t.Fatalf("Expected a static creds provider to be returned") + } + + v, err := creds.Get() + if err != nil { + t.Fatalf("Error gettings creds: %s", err) + } + if v.AccessKeyID != "somekey" { + t.Fatalf("AccessKeyID mismatch, expected: (somekey), got (%s)", v.AccessKeyID) + } + if v.SecretAccessKey != "somesecret" { + t.Fatalf("SecretAccessKey mismatch, expected: (somesecret), got (%s)", v.SecretAccessKey) + } + if v.SessionToken != "sometoken" { + t.Fatalf("SessionToken mismatch, expected: (sometoken), got (%s)", v.SessionToken) + } +} + +// TestAWSConfig_shouldIAM is designed to test the scenario of running Terraform +// from an EC2 instance, without environment variables or manually supplied +// credentials. +func TestAWSConfig_shouldIgnoreIAM(t *testing.T) { + resetEnv := unsetEnv(t) + defer resetEnv() + // capture the test server's close method, to call after the test returns + ts := awsEnv(t) + defer ts() + simple := []struct { + Key, Secret, Token string + }{ + { + Key: "test", + Secret: "secret", + }, { + Key: "test", + Secret: "test", + Token: "test", + }, + } + + for _, c := range simple { + cfg := Config{ + AccessKey: c.Key, + SecretKey: c.Secret, + Token: c.Token, + } + + creds := getCreds(cfg.AccessKey, cfg.SecretKey, cfg.Token) + if creds == nil { + t.Fatalf("Expected a static creds provider to be returned") + } + v, err := creds.Get() + if err != nil { + t.Fatalf("Error gettings creds: %s", err) + } + if v.AccessKeyID != c.Key { + t.Fatalf("AccessKeyID mismatch, expected: (%s), got (%s)", c.Key, v.AccessKeyID) + } + if v.SecretAccessKey != c.Secret { + t.Fatalf("SecretAccessKey mismatch, expected: (%s), got (%s)", c.Secret, v.SecretAccessKey) + } + if v.SessionToken != c.Token { + t.Fatalf("SessionToken mismatch, expected: (%s), got (%s)", c.Token, v.SessionToken) + } + } +} + +func TestAWSConfig_shouldBeENV(t *testing.T) { + // need to set the environment variables to a dummy string, as we don't know + // what they may be at runtime without hardcoding here + s := "some_env" + resetEnv := setEnv(s, t) + defer resetEnv() + + cfg := Config{} + creds := getCreds(cfg.AccessKey, cfg.SecretKey, cfg.Token) + if creds == nil { + t.Fatalf("Expected a static creds provider to be returned") + } + v, err := creds.Get() + if err != nil { + t.Fatalf("Error gettings creds: %s", err) + } + if v.AccessKeyID != s { + t.Fatalf("AccessKeyID mismatch, expected: (%s), got (%s)", s, v.AccessKeyID) + } + if v.SecretAccessKey != s { + t.Fatalf("SecretAccessKey mismatch, expected: (%s), got (%s)", s, v.SecretAccessKey) + } + if v.SessionToken != s { + t.Fatalf("SessionToken mismatch, expected: (%s), got (%s)", s, v.SessionToken) + } +} + +// unsetEnv unsets enviornment variables for testing a "clean slate" with no +// credentials in the environment +func unsetEnv(t *testing.T) func() { + // Grab any existing AWS keys and preserve. In some tests we'll unset these, so + // we need to have them and restore them after + e := getEnv() + if err := os.Unsetenv("AWS_ACCESS_KEY_ID"); err != nil { + t.Fatalf("Error unsetting env var AWS_ACCESS_KEY_ID: %s", err) + } + if err := os.Unsetenv("AWS_SECRET_ACCESS_KEY"); err != nil { + t.Fatalf("Error unsetting env var AWS_SECRET_ACCESS_KEY: %s", err) + } + if err := os.Unsetenv("AWS_SESSION_TOKEN"); err != nil { + t.Fatalf("Error unsetting env var AWS_SESSION_TOKEN: %s", err) + } + + return func() { + // re-set all the envs we unset above + if err := os.Setenv("AWS_ACCESS_KEY_ID", e.Key); err != nil { + t.Fatalf("Error resetting env var AWS_ACCESS_KEY_ID: %s", err) + } + if err := os.Setenv("AWS_SECRET_ACCESS_KEY", e.Secret); err != nil { + t.Fatalf("Error resetting env var AWS_SECRET_ACCESS_KEY: %s", err) + } + if err := os.Setenv("AWS_SESSION_TOKEN", e.Token); err != nil { + t.Fatalf("Error resetting env var AWS_SESSION_TOKEN: %s", err) + } + } +} + +func setEnv(s string, t *testing.T) func() { + e := getEnv() + // Set all the envs to a dummy value + if err := os.Setenv("AWS_ACCESS_KEY_ID", s); err != nil { + t.Fatalf("Error setting env var AWS_ACCESS_KEY_ID: %s", err) + } + if err := os.Setenv("AWS_SECRET_ACCESS_KEY", s); err != nil { + t.Fatalf("Error setting env var AWS_SECRET_ACCESS_KEY: %s", err) + } + if err := os.Setenv("AWS_SESSION_TOKEN", s); err != nil { + t.Fatalf("Error setting env var AWS_SESSION_TOKEN: %s", err) + } + + return func() { + // re-set all the envs we unset above + if err := os.Setenv("AWS_ACCESS_KEY_ID", e.Key); err != nil { + t.Fatalf("Error resetting env var AWS_ACCESS_KEY_ID: %s", err) + } + if err := os.Setenv("AWS_SECRET_ACCESS_KEY", e.Secret); err != nil { + t.Fatalf("Error resetting env var AWS_SECRET_ACCESS_KEY: %s", err) + } + if err := os.Setenv("AWS_SESSION_TOKEN", e.Token); err != nil { + t.Fatalf("Error resetting env var AWS_SESSION_TOKEN: %s", err) + } + } +} + +// awsEnv establishes a httptest server to mock out the internal AWS Metadata +// service. IAM Credentials are retrieved by the EC2RoleProvider, which makes +// API calls to this internal URL. By replacing the server with a test server, +// we can simulate an AWS environment +func awsEnv(t *testing.T) func() { + routes := routes{} + if err := json.Unmarshal([]byte(aws_routes), &routes); err != nil { + t.Fatalf("Failed to unmarshal JSON in AWS ENV test: %s", err) + } + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.Header().Add("Server", "MockEC2") + for _, e := range routes.Endpoints { + if r.RequestURI == e.Uri { + fmt.Fprintln(w, e.Body) + } + } + })) + + os.Setenv("AWS_METADATA_URL", ts.URL+"/latest") + return ts.Close +} + +func getEnv() *currentEnv { + // Grab any existing AWS keys and preserve. In some tests we'll unset these, so + // we need to have them and restore them after + return ¤tEnv{ + Key: os.Getenv("AWS_ACCESS_KEY_ID"), + Secret: os.Getenv("AWS_SECRET_ACCESS_KEY"), + Token: os.Getenv("AWS_SESSION_TOKEN"), + } +} + +// struct to preserve the current environment +type currentEnv struct { + Key, Secret, Token string +} + +type routes struct { + Endpoints []*endpoint `json:"endpoints"` +} +type endpoint struct { + Uri string `json:"uri"` + Body string `json:"body"` +} + +const aws_routes = ` +{ + "endpoints": [ + { + "uri": "/latest/meta-data/iam/security-credentials", + "body": "test_role" + }, + { + "uri": "/latest/meta-data/iam/security-credentials/test_role", + "body": "{\"Code\":\"Success\",\"LastUpdated\":\"2015-12-11T17:17:25Z\",\"Type\":\"AWS-HMAC\",\"AccessKeyId\":\"somekey\",\"SecretAccessKey\":\"somesecret\",\"Token\":\"sometoken\"}" + } + ] +} +` diff --git a/builtin/providers/aws/provider.go b/builtin/providers/aws/provider.go index 313f74b18a73..2edb94b06608 100644 --- a/builtin/providers/aws/provider.go +++ b/builtin/providers/aws/provider.go @@ -1,19 +1,10 @@ package aws import ( - "net" - "sync" - "time" - "github.com/hashicorp/terraform/helper/hashcode" "github.com/hashicorp/terraform/helper/mutexkv" "github.com/hashicorp/terraform/helper/schema" "github.com/hashicorp/terraform/terraform" - - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" - "github.com/aws/aws-sdk-go/aws/ec2metadata" - "github.com/aws/aws-sdk-go/aws/session" ) // Provider returns a terraform.ResourceProvider. @@ -21,95 +12,27 @@ func Provider() terraform.ResourceProvider { // TODO: Move the validation to this, requires conditional schemas // TODO: Move the configuration to this, requires validation - // These variables are closed within the `getCreds` function below. - // This function is responsible for reading credentials from the - // environment in the case that they're not explicitly specified - // in the Terraform configuration. - // - // By using the getCreds function here instead of making the default - // empty, we avoid asking for input on credentials if they're available - // in the environment. - var credVal credentials.Value - var credErr error - var once sync.Once - getCreds := func() { - // Build the list of providers to look for creds in - providers := []credentials.Provider{ - &credentials.EnvProvider{}, - &credentials.SharedCredentialsProvider{}, - } - - // We only look in the EC2 metadata API if we can connect - // to the metadata service within a reasonable amount of time - conn, err := net.DialTimeout("tcp", "169.254.169.254:80", 100*time.Millisecond) - if err == nil { - conn.Close() - providers = append(providers, &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(session.New())}) - } - - credVal, credErr = credentials.NewChainCredentials(providers).Get() - - // If we didn't successfully find any credentials, just - // set the error to nil. - if credErr == credentials.ErrNoValidProvidersFoundInChain { - credErr = nil - } - } - - // getCredDefault is a function used by DefaultFunc below to - // get the default value for various parts of the credentials. - // This function properly handles loading the credentials, checking - // for errors, etc. - getCredDefault := func(def interface{}, f func() string) (interface{}, error) { - once.Do(getCreds) - - // If there was an error, that is always first - if credErr != nil { - return nil, credErr - } - - // If the value is empty string, return nil (not set) - val := f() - if val == "" { - return def, nil - } - - return val, nil - } - // The actual provider return &schema.Provider{ Schema: map[string]*schema.Schema{ "access_key": &schema.Schema{ - Type: schema.TypeString, - Required: true, - DefaultFunc: func() (interface{}, error) { - return getCredDefault(nil, func() string { - return credVal.AccessKeyID - }) - }, + Type: schema.TypeString, + Optional: true, + Default: "", Description: descriptions["access_key"], }, "secret_key": &schema.Schema{ - Type: schema.TypeString, - Required: true, - DefaultFunc: func() (interface{}, error) { - return getCredDefault(nil, func() string { - return credVal.SecretAccessKey - }) - }, + Type: schema.TypeString, + Optional: true, + Default: "", Description: descriptions["secret_key"], }, "token": &schema.Schema{ - Type: schema.TypeString, - Optional: true, - DefaultFunc: func() (interface{}, error) { - return getCredDefault("", func() string { - return credVal.SessionToken - }) - }, + Type: schema.TypeString, + Optional: true, + Default: "", Description: descriptions["token"], }, From adf417809a7fdd44da0241efc76164b36e6e24be Mon Sep 17 00:00:00 2001 From: clint shryock Date: Tue, 15 Dec 2015 10:49:23 -0600 Subject: [PATCH 2/2] add some comments on auth refactoring --- builtin/providers/aws/config.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builtin/providers/aws/config.go b/builtin/providers/aws/config.go index e7c7628dc412..e3e2243f1ca1 100644 --- a/builtin/providers/aws/config.go +++ b/builtin/providers/aws/config.go @@ -353,8 +353,12 @@ func getCreds(key, secret, token string) *awsCredentials.Credentials { } r, err := c.Get(metadataURL) + // Flag to determine if we should add the EC2Meta data provider. Default false var useIAM bool if err == nil { + // AWS will add a "Server: EC2ws" header value for the metadata request. We + // check the headers for this value to ensure something else didn't just + // happent to be listening on that IP:Port if r.Header["Server"] != nil && strings.Contains(r.Header["Server"][0], "EC2") { useIAM = true }