From 98d138ece9dc27acf20266994e25bef4d43c3d7b Mon Sep 17 00:00:00 2001 From: Eugene Babichev <10692625+BEvgeniyS@users.noreply.github.com> Date: Wed, 7 Sep 2022 21:30:04 +1000 Subject: [PATCH] feat: cross account caching with role (#336) This allows to have a single account for caching in multi-account environments. Changes: - Added support to assume role - Added the ability to specify access policy - Added the ability to specify lifecycle policy Initially, I had two approaches to make this work: 1) let other accounts create repos and fill them with images 2) use the role in the target account (this PR) While 1) would be preferable, unfortunately, it's doesn't look like it's possible: you can allow other accounts to create repos, but not put any policies. --- .gitignore | 2 + .k8s-image-swapper.yml | 44 ++++++++++++ cmd/root.go | 2 +- docs/getting-started.md | 85 +++++++++++++++++++++++ pkg/config/config.go | 7 +- pkg/registry/ecr.go | 111 +++++++++++++++++++++++------- pkg/webhook/image_swapper_test.go | 8 ++- 7 files changed, 228 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index ee770a66..cead1d23 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ # vendor/ .idea/ +coverage.txt +k8s-image-swapper diff --git a/.k8s-image-swapper.yml b/.k8s-image-swapper.yml index bdad7ab8..8c8a4e44 100644 --- a/.k8s-image-swapper.yml +++ b/.k8s-image-swapper.yml @@ -41,6 +41,7 @@ target: aws: accountId: 123456789 region: ap-southeast-2 + role: arn:aws:iam::123456789012:role/roleName ecrOptions: tags: - key: CreatedBy @@ -51,5 +52,48 @@ target: encryptionConfiguration: encryptionType: AES256 kmsKey: string + accessPolicy: | + { + "Version": "2008-10-17", + "Statement": [ + { + "Sid": "AllowCrossAccountPull", + "Effect": "Allow", + "Principal": { + "AWS": "*" + }, + "Action": [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ], + "Condition": { + "StringEquals": { + "aws:PrincipalOrgID": [ + "o-xxxxxxxx" + ] + } + } + } + ] + } + + lifecyclePolicy: | + { + "rules": [ + { + "rulePriority": 1, + "description": "Rule 1", + "selection": { + "tagStatus": "any", + "countType": "imageCountMoreThan", + "countNumber": 1 + }, + "action": { + "type": "expire" + } + } + ] + } # dockerio: # quayio: diff --git a/cmd/root.go b/cmd/root.go index 8005a970..7147b53e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -64,7 +64,7 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`, //metricsRec := metrics.NewPrometheus(promReg) log.Trace().Interface("config", cfg).Msg("config") - rClient, err := registry.NewECRClient(cfg.Target.AWS.Region, cfg.Target.AWS.EcrDomain()) + rClient, err := registry.NewECRClient(cfg.Target.AWS.Region, cfg.Target.AWS.EcrDomain(), cfg.Target.AWS.AccountID, cfg.Target.AWS.Role, cfg.Target.AWS.AccessPolicy, cfg.Target.AWS.LifecyclePolicy) if err != nil { log.Err(err).Msg("error connecting to registry client") os.Exit(1) diff --git a/docs/getting-started.md b/docs/getting-started.md index 52afe9e8..527a0428 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -29,6 +29,91 @@ Choose from one of the strategies below or an alternative if needed. --from-literal=aws_secret_access_key=<...> ``` +#### Using ECR registries cross-account + +Although ECR allows creating registry policy that allows reposistories creation from different account, there's no way to push anything to these repositories. +ECR resource-level policy can not be applied during creation, and to apply it afterwards we need ecr:SetRepositoryPolicy permission, which foreign account doesn't have. + +One way out of this conundrum is to assume the role in target account + +```yaml +target: + type: aws + aws: + accountId: 123456789 + region: ap-southeast-2 + role: arn:aws:iam::123456789012:role/roleName +``` +!!! note +Make sure that target role has proper trust permissions that allow to assume it cross-account + +!!! note +In order te be able to pull images from outside accounts, you will have to apply proper access policy + + +#### Access policy + +You can specify the access policy that will be applied to the created repos in config. Policy should be raw json string. +For example: +```yaml +target: + aws: + accountId: 123456789 + region: ap-southeast-2 + role: arn:aws:iam::123456789012:role/roleName + accessPolicy: '{ + "Statement": [ + { + "Sid": "AllowCrossAccountPull", + "Effect": "Allow", + "Principal": { + "AWS": "*" + }, + "Action": [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ], + "Condition": { + "StringEquals": { + "aws:PrincipalOrgID": "o-xxxxxxxxxx" + } + } + } + ], + "Version": "2008-10-17" +}' +``` + +#### Lifecycle policy + +Similarly to access policy, lifecycle policy can be specified, for example: + +```yaml +target: + aws: + accountId: 123456789 + region: ap-southeast-2 + role: arn:aws:iam::123456789012:role/roleName + accessPolicy: '{ + "rules": [ + { + "rulePriority": 1, + "description": "Rule 1", + "selection": { + "tagStatus": "any", + "countType": "imageCountMoreThan", + "countNumber": 1000 + }, + "action": { + "type": "expire" + } + } + ] +} +' +``` + #### Service Account 1. Create an Webidentity IAM role (e.g. `k8s-image-swapper`) with the following trust policy, e.g diff --git a/pkg/config/config.go b/pkg/config/config.go index 6ff3540a..eb2e0bc5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -54,8 +54,11 @@ type Target struct { } type AWS struct { - AccountID string `yaml:"accountId"` - Region string `yaml:"region"` + AccountID string `yaml:"accountId"` + Region string `yaml:"region"` + Role string `yaml:"role"` + AccessPolicy string `yaml:"accessPolicy"` + LifecyclePolicy string `yaml:"lifecyclePolicy"` } func (a *AWS) EcrDomain() string { diff --git a/pkg/registry/ecr.go b/pkg/registry/ecr.go index d9340cfd..c8417a5f 100644 --- a/pkg/registry/ecr.go +++ b/pkg/registry/ecr.go @@ -8,6 +8,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ecr" "github.com/aws/aws-sdk-go/service/ecr/ecriface" @@ -19,11 +20,14 @@ import ( var execCommand = exec.Command type ECRClient struct { - client ecriface.ECRAPI - ecrDomain string - authToken []byte - cache *ristretto.Cache - scheduler *gocron.Scheduler + client ecriface.ECRAPI + ecrDomain string + authToken []byte + cache *ristretto.Cache + scheduler *gocron.Scheduler + targetAccount string + accessPolicy string + lifecyclePolicy string } func (e *ECRClient) Credentials() string { @@ -41,6 +45,7 @@ func (e *ECRClient) CreateRepository(name string) error { ScanOnPush: aws.Bool(true), }, ImageTagMutability: aws.String(ecr.ImageTagMutabilityMutable), + RegistryId: &e.targetAccount, Tags: []*ecr.Tag{ { Key: aws.String("CreatedBy"), @@ -48,6 +53,7 @@ func (e *ECRClient) CreateRepository(name string) error { }, }, }) + if err != nil { if aerr, ok := err.(awserr.Error); ok { switch aerr.Code() { @@ -63,6 +69,37 @@ func (e *ECRClient) CreateRepository(name string) error { } } + if len(e.accessPolicy) > 0 { + log.Info().Msg("Setting access policy on" + name) + log.Debug().Msg("Access policy: \n" + e.accessPolicy) + _, err := e.client.SetRepositoryPolicy(&ecr.SetRepositoryPolicyInput{ + PolicyText: &e.accessPolicy, + RegistryId: &e.targetAccount, + RepositoryName: aws.String(name), + }) + + if err != nil { + log.Err(err).Msg(err.Error()) + return err + } + } + + if len(e.lifecyclePolicy) > 0 { + log.Info().Msg("Setting lifecycle policy on" + name) + log.Debug().Msg("Lifecycle policy: \n" + e.lifecyclePolicy) + + _, err := e.client.PutLifecyclePolicy(&ecr.PutLifecyclePolicyInput{ + LifecyclePolicyText: &e.lifecyclePolicy, + RegistryId: &e.targetAccount, + RepositoryName: aws.String(name), + }) + + if err != nil { + log.Err(err).Msg(err.Error()) + return err + } + } + e.cache.Set(name, "", 1) return nil @@ -115,7 +152,10 @@ func (e *ECRClient) Endpoint() string { // requestAuthToken requests and returns an authentication token from ECR with its expiration date func (e *ECRClient) requestAuthToken() ([]byte, time.Time, error) { - getAuthTokenOutput, err := e.client.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{}) + getAuthTokenOutput, err := e.client.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{ + RegistryIds: []*string{&e.targetAccount}, + }) + if err != nil { return []byte(""), time.Time{}, err } @@ -146,18 +186,33 @@ func (e *ECRClient) scheduleTokenRenewal() error { return nil } -func NewECRClient(region string, ecrDomain string) (*ECRClient, error) { - sess := session.Must(session.NewSessionWithOptions(session.Options{ +func NewECRClient(region string, ecrDomain string, targetAccount string, role string, accessPolicy string, lifecyclePolicy string) (*ECRClient, error) { + var sess *session.Session + var config *aws.Config + if role != "" { + log.Debug().Msg("Role is specified. Assuming " + role) + stsSession, _ := session.NewSession(config) + creds := stscreds.NewCredentials(stsSession, role) + config = aws.NewConfig(). + WithRegion(region). + WithCredentialsChainVerboseErrors(true). + WithHTTPClient(&http.Client{ + Timeout: 3 * time.Second, + }). + WithCredentials(creds) + } else { + config = aws.NewConfig(). + WithRegion(region). + WithCredentialsChainVerboseErrors(true). + WithHTTPClient(&http.Client{ + Timeout: 3 * time.Second, + }) + } + + sess = session.Must(session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, + Config: (*config), })) - - config := aws.NewConfig(). - WithRegion(region). - WithCredentialsChainVerboseErrors(true). - WithHTTPClient(&http.Client{ - Timeout: 3 * time.Second, - }) - ecrClient := ecr.New(sess, config) cache, err := ristretto.NewCache(&ristretto.Config{ @@ -173,10 +228,13 @@ func NewECRClient(region string, ecrDomain string) (*ECRClient, error) { scheduler.StartAsync() client := &ECRClient{ - client: ecrClient, - ecrDomain: ecrDomain, - cache: cache, - scheduler: scheduler, + client: ecrClient, + ecrDomain: ecrDomain, + cache: cache, + scheduler: scheduler, + targetAccount: targetAccount, + accessPolicy: accessPolicy, + lifecyclePolicy: lifecyclePolicy, } if err := client.scheduleTokenRenewal(); err != nil { @@ -186,13 +244,14 @@ func NewECRClient(region string, ecrDomain string) (*ECRClient, error) { return client, nil } -func NewMockECRClient(ecrClient ecriface.ECRAPI, region string, ecrDomain string) (*ECRClient, error) { +func NewMockECRClient(ecrClient ecriface.ECRAPI, region string, ecrDomain string, targetAccount, role string) (*ECRClient, error) { client := &ECRClient{ - client: ecrClient, - ecrDomain: ecrDomain, - cache: nil, - scheduler: nil, - authToken: []byte("mock-ecr-client-fake-auth-token"), + client: ecrClient, + ecrDomain: ecrDomain, + cache: nil, + scheduler: nil, + targetAccount: targetAccount, + authToken: []byte("mock-ecr-client-fake-auth-token"), } return client, nil diff --git a/pkg/webhook/image_swapper_test.go b/pkg/webhook/image_swapper_test.go index 7e288a29..88b800c3 100644 --- a/pkg/webhook/image_swapper_test.go +++ b/pkg/webhook/image_swapper_test.go @@ -245,6 +245,7 @@ func TestImageSwapper_Mutate(t *testing.T) { }, ImageTagMutability: aws.String("MUTABLE"), RepositoryName: aws.String("docker.io/library/init-container"), + RegistryId: aws.String("123456789"), Tags: []*ecr.Tag{{ Key: aws.String("CreatedBy"), Value: aws.String("k8s-image-swapper"), @@ -258,6 +259,7 @@ func TestImageSwapper_Mutate(t *testing.T) { }, ImageTagMutability: aws.String("MUTABLE"), RepositoryName: aws.String("docker.io/library/nginx"), + RegistryId: aws.String("123456789"), Tags: []*ecr.Tag{{ Key: aws.String("CreatedBy"), Value: aws.String("k8s-image-swapper"), @@ -271,13 +273,14 @@ func TestImageSwapper_Mutate(t *testing.T) { }, ImageTagMutability: aws.String("MUTABLE"), RepositoryName: aws.String("k8s.gcr.io/ingress-nginx/controller"), + RegistryId: aws.String("123456789"), Tags: []*ecr.Tag{{ Key: aws.String("CreatedBy"), Value: aws.String("k8s-image-swapper"), }}, }).Return(mock.Anything) - registryClient, _ := registry.NewMockECRClient(ecrClient, "ap-southeast-2", "123456789.dkr.ecr.ap-southeast-2.amazonaws.com") + registryClient, _ := registry.NewMockECRClient(ecrClient, "ap-southeast-2", "123456789.dkr.ecr.ap-southeast-2.amazonaws.com", "123456789", "arn:aws:iam::123456789:role/fakerole") admissionReview, _ := readAdmissionReviewFromFile("admissionreview-simple.json") admissionReviewModel := model.NewAdmissionReviewV1(admissionReview) @@ -323,6 +326,7 @@ func TestImageSwapper_MutateWithImagePullSecrets(t *testing.T) { ScanOnPush: aws.Bool(true), }, ImageTagMutability: aws.String("MUTABLE"), + RegistryId: aws.String("123456789"), RepositoryName: aws.String("docker.io/library/nginx"), Tags: []*ecr.Tag{{ Key: aws.String("CreatedBy"), @@ -330,7 +334,7 @@ func TestImageSwapper_MutateWithImagePullSecrets(t *testing.T) { }}, }).Return(mock.Anything) - registryClient, _ := registry.NewMockECRClient(ecrClient, "ap-southeast-2", "123456789.dkr.ecr.ap-southeast-2.amazonaws.com") + registryClient, _ := registry.NewMockECRClient(ecrClient, "ap-southeast-2", "123456789.dkr.ecr.ap-southeast-2.amazonaws.com", "123456789", "arn:aws:iam::123456789:role/fakerole") admissionReview, _ := readAdmissionReviewFromFile("admissionreview-imagepullsecrets.json") admissionReviewModel := model.NewAdmissionReviewV1(admissionReview)