Skip to content

Commit

Permalink
Update broker tagging (#355)
Browse files Browse the repository at this point in the history 
* add new cloud-gov/go-broker-tags module

* update IAM code to implement new TagManager struct from broker-tags module

* update code to set broker tags on IAM users

* update go-broker-tags module version

* refactor code for setting tags on ES instances and IAM users on creation

* remove commented code

* update brokertags module version

* update code for compatibility with new version of brokertags module

* add test for expected tag generation

* add test for IAM user tags

* update dev deployment to include necessary env vars

* update go modules

* fix README formatting

* add CF API settings to settings struct

* add mocking for unit tests

* update README

* fix backwards code logic

* add tag manager to redis broker

* update go-broker-tags module and code for compatbility

* update code for compatbility with go-broker-tags

* add code to set tags on redis instances

* update code to set tags when creating redis instance

* refactor code

* refactor code for testing

* scaffold code to generate tags for RDS databases

* update unit test

* update unit test
  • Loading branch information
@markdboyd
markdboyd authored Jan 16, 2024
1 parent a6186b2 commit 7e3172d
Show file tree
Hide file tree
Showing 14 changed files with 538 additions and 225 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.20
require (
code.cloudfoundry.org/lager v2.0.0+incompatible
github.com/aws/aws-sdk-go v1.44.10
github.com/cloud-gov/go-broker-tags v0.0.0-20240110222823-ade415005c99
github.com/cloud-gov/go-broker-tags v0.0.0-20240112192542-8f1bb5859679
github.com/go-co-op/gocron v1.13.0
github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab
github.com/go-sql-driver/mysql v1.6.0
Expand Down
31 changes: 2 additions & 29 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,28 +4,8 @@ github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBK
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/aws/aws-sdk-go v1.44.10 h1:ohCdgQpJ9ojzm0fOk7ykrMTgTpHJBk5nnA7X+HzmnOA=
github.com/aws/aws-sdk-go v1.44.10/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/cloud-gov/go-broker-tags v0.0.0-20240105211141-b80d15d9ce0f h1:KP0snh758deBncNqw7T/XPXrxYmh95N8Vqv/MpKZCOY=
github.com/cloud-gov/go-broker-tags v0.0.0-20240105211141-b80d15d9ce0f/go.mod h1:vQsBqs6ze14lotTKOZbubtKsM49gK5cwFPUbQE+ZU30=
github.com/cloud-gov/go-broker-tags v0.0.0-20240108151510-b2d89626d8de h1:kHHHoIhdaESciq0g1D2c8Epiierb5JwGu7CSy7ECVME=
github.com/cloud-gov/go-broker-tags v0.0.0-20240108151510-b2d89626d8de/go.mod h1:vQsBqs6ze14lotTKOZbubtKsM49gK5cwFPUbQE+ZU30=
github.com/cloud-gov/go-broker-tags v0.0.0-20240108151858-26bcae3652cb h1:GRDhwcJjhBV3UWoq+1s6gSEv//onTNpn6TdhydT/9Xo=
github.com/cloud-gov/go-broker-tags v0.0.0-20240108151858-26bcae3652cb/go.mod h1:vQsBqs6ze14lotTKOZbubtKsM49gK5cwFPUbQE+ZU30=
github.com/cloud-gov/go-broker-tags v0.0.0-20240108201238-2a4d047860b1 h1:z3rccmuuLvlm2EyCHez40uHp+m2OaJOkHqOWO3fvdxI=
github.com/cloud-gov/go-broker-tags v0.0.0-20240108201238-2a4d047860b1/go.mod h1:vQsBqs6ze14lotTKOZbubtKsM49gK5cwFPUbQE+ZU30=
github.com/cloud-gov/go-broker-tags v0.0.0-20240109161858-c6e376a24a19 h1:ZGwAYk3bBfHVyqDO7y99XSOHuD4drvs2+kVy49eoVWY=
github.com/cloud-gov/go-broker-tags v0.0.0-20240109161858-c6e376a24a19/go.mod h1:vQsBqs6ze14lotTKOZbubtKsM49gK5cwFPUbQE+ZU30=
github.com/cloud-gov/go-broker-tags v0.0.0-20240109164923-9e38d875341e h1:0sJn6bcLnCC+rd80fN5DARGZTHVLHukBMFsWyjN93xg=
github.com/cloud-gov/go-broker-tags v0.0.0-20240109164923-9e38d875341e/go.mod h1:vQsBqs6ze14lotTKOZbubtKsM49gK5cwFPUbQE+ZU30=
github.com/cloud-gov/go-broker-tags v0.0.0-20240110162616-5bbb90329543 h1:mPLeN3b+ESP/Y8PMs7oxh7CoIQdnVE3TpoLBA0RPSDA=
github.com/cloud-gov/go-broker-tags v0.0.0-20240110162616-5bbb90329543/go.mod h1:vQsBqs6ze14lotTKOZbubtKsM49gK5cwFPUbQE+ZU30=
github.com/cloud-gov/go-broker-tags v0.0.0-20240110162755-d239cdae50ab h1:P/RFIj+LLuIZslBpSWQOO7xaxDw4c1aUZyriAAzGdeg=
github.com/cloud-gov/go-broker-tags v0.0.0-20240110162755-d239cdae50ab/go.mod h1:vQsBqs6ze14lotTKOZbubtKsM49gK5cwFPUbQE+ZU30=
github.com/cloud-gov/go-broker-tags v0.0.0-20240110164341-ab30d7ae3e42 h1:EftSBdjrHhAIZVJd5mm5aJibwu2/+6xsVyr/evCmDkY=
github.com/cloud-gov/go-broker-tags v0.0.0-20240110164341-ab30d7ae3e42/go.mod h1:vQsBqs6ze14lotTKOZbubtKsM49gK5cwFPUbQE+ZU30=
github.com/cloud-gov/go-broker-tags v0.0.0-20240110200930-76754207769b h1:8edy0USZHwhVNnEF8Uwk8315VSB/hB1F5wjcYnqqa8U=
github.com/cloud-gov/go-broker-tags v0.0.0-20240110200930-76754207769b/go.mod h1:vQsBqs6ze14lotTKOZbubtKsM49gK5cwFPUbQE+ZU30=
github.com/cloud-gov/go-broker-tags v0.0.0-20240110222823-ade415005c99 h1:ca/zkC5tKUNhCwnp7kP1n/Qa0UeOh03MzFZ2wrPGiZE=
github.com/cloud-gov/go-broker-tags v0.0.0-20240110222823-ade415005c99/go.mod h1:vQsBqs6ze14lotTKOZbubtKsM49gK5cwFPUbQE+ZU30=
github.com/cloud-gov/go-broker-tags v0.0.0-20240112192542-8f1bb5859679 h1:fPU5aeiTFYpB6w/54uOzFzWkbAwX7ie0u3pZ+C2cVUE=
github.com/cloud-gov/go-broker-tags v0.0.0-20240112192542-8f1bb5859679/go.mod h1:vQsBqs6ze14lotTKOZbubtKsM49gK5cwFPUbQE+ZU30=
github.com/cloudfoundry-community/go-cfclient/v3 v3.0.0-alpha.6 h1:mF8LXapcJsG+zqNFSlfWssERIuK0Nf0UEAyAR/s0TAI=
github.com/cloudfoundry-community/go-cfclient/v3 v3.0.0-alpha.6/go.mod h1:3tjqtK8cGhfhGNhDVKLQ7AaTDzP9K7fyfeNtYqmNWWM=
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
Expand Down Expand Up @@ -129,7 +109,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
Expand All @@ -144,12 +123,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ=
golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand All @@ -172,8 +147,6 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand Down
38 changes: 35 additions & 3 deletions manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,50 @@ import (
"github.com/18F/aws-broker/services/rds"
"github.com/18F/aws-broker/services/redis"
"github.com/18F/aws-broker/taskqueue"
brokertags "github.com/cloud-gov/go-broker-tags"
"github.com/jinzhu/gorm"
)

type mockTagGenerator struct {
tags map[string]string
}

func (mt *mockTagGenerator) GenerateTags(
action brokertags.Action,
serviceName string,
servicePlanName string,
resourceGUIDs brokertags.ResourceGUIDs,
getMissingResources bool,
) (map[string]string, error) {
return mt.tags, nil
}

func findBroker(serviceID string, c *catalog.Catalog, brokerDb *gorm.DB, settings *config.Settings, taskqueue *taskqueue.QueueManager) (base.Broker, response.Response) {
var tagManager brokertags.TagManager
if settings.Environment == "test" {
tagManager = &mockTagGenerator{}
} else {
var err error
tagManager, err = brokertags.NewCFTagManager(
"AWS broker",
settings.Environment,
settings.CfApiUrl,
settings.CfApiClientId,
settings.CfApiClientSecret,
)
if err != nil {
return nil, response.NewErrorResponse(http.StatusInternalServerError, err.Error())
}
}

switch serviceID {
// RDS Service
case c.RdsService.ID:
return rds.InitRDSBroker(brokerDb, settings), nil
return rds.InitRDSBroker(brokerDb, settings, tagManager), nil
case c.RedisService.ID:
return redis.InitRedisBroker(brokerDb, settings), nil
return redis.InitRedisBroker(brokerDb, settings, tagManager), nil
case c.ElasticsearchService.ID:
broker, err := elasticsearch.InitElasticsearchBroker(brokerDb, settings, taskqueue)
broker, err := elasticsearch.InitElasticsearchBroker(brokerDb, settings, taskqueue, tagManager)
if err != nil {
return nil, response.NewErrorResponse(http.StatusInternalServerError, err.Error())
}
Expand Down
53 changes: 12 additions & 41 deletions services/elasticsearch/broker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,47 +51,16 @@ type elasticsearchBroker struct {
tagManager brokertags.TagManager
}

type mockTagGenerator struct {
tags map[string]string
}

func (mt *mockTagGenerator) GenerateTags(
action brokertags.Action,
environment string,
serviceGUID string,
servicePlanGUID string,
organizationGUID string,
spaceGUID string,
instanceGUID string,
) (map[string]string, error) {
return mt.tags, nil
}

// InitelasticsearchBroker is the constructor for the elasticsearchBroker.
func InitElasticsearchBroker(brokerDB *gorm.DB, settings *config.Settings, taskqueue *taskqueue.QueueManager) (base.Broker, error) {
func InitElasticsearchBroker(
brokerDB *gorm.DB,
settings *config.Settings,
taskqueue *taskqueue.QueueManager,
tagManager brokertags.TagManager,
) (base.Broker, error) {
logger := lager.NewLogger("aws-es-broker")
logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))

if settings.Environment == "test" {
return &elasticsearchBroker{
brokerDB: brokerDB,
settings: settings,
taskqueue: taskqueue,
logger: logger,
tagManager: &mockTagGenerator{},
}, nil
}

tagManager, err := brokertags.NewCFTagManager(
"AWS broker",
settings.CfApiUrl,
settings.CfApiClientId,
settings.CfApiClientSecret,
)
if err != nil {
return nil, err
}

return &elasticsearchBroker{
brokerDB,
settings,
Expand Down Expand Up @@ -176,12 +145,14 @@ func (broker *elasticsearchBroker) CreateInstance(c *catalog.Catalog, id string,

tags, err := broker.tagManager.GenerateTags(
brokertags.Create,
broker.settings.Environment,
c.ElasticsearchService.Name,
plan.Name,
createRequest.OrganizationGUID,
createRequest.SpaceGUID,
id,
brokertags.ResourceGUIDs{
InstanceGUID: id,
SpaceGUID: createRequest.SpaceGUID,
OrganizationGUID: createRequest.OrganizationGUID,
},
false,
)
if err != nil {
return response.NewErrorResponse(http.StatusInternalServerError, "There was an error generating the tags. Error: "+err.Error())
Expand Down
9 changes: 9 additions & 0 deletions services/elasticsearch/elasticsearch_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ func TestPrepareCreateDomainInput(t *testing.T) {
InstanceType: "db.m5.xlarge",
NodeToNodeEncryption: true,
AutomatedSnapshotStartHour: 0,
Tags: map[string]string{
"foo": "bar",
},
},
accessPolicy: "fake-access-policy",
expectedParams: &opensearchservice.CreateDomainInput{
Expand Down Expand Up @@ -63,6 +66,12 @@ func TestPrepareCreateDomainInput(t *testing.T) {
EncryptionAtRestOptions: &opensearchservice.EncryptionAtRestOptions{
Enabled: aws.Bool(false),
},
TagList: []*opensearchservice.Tag{
{
Key: aws.String("foo"),
Value: aws.String("bar"),
},
},
},
},
"data count is greater than 1": {
Expand Down
32 changes: 26 additions & 6 deletions services/rds/broker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/rds"
brokertags "github.com/cloud-gov/go-broker-tags"
"github.com/jinzhu/gorm"

"github.com/18F/aws-broker/base"
Expand Down Expand Up @@ -62,8 +63,9 @@ func (o Options) Validate(settings *config.Settings) error {
}

type rdsBroker struct {
brokerDB *gorm.DB
settings *config.Settings
brokerDB *gorm.DB
settings *config.Settings
tagManager brokertags.TagManager
}

// initializeAdapter is the main function to create database instances
Expand Down Expand Up @@ -94,8 +96,8 @@ func initializeAdapter(plan catalog.RDSPlan, s *config.Settings, c *catalog.Cata
}

// InitRDSBroker is the constructor for the rdsBroker.
func InitRDSBroker(brokerDB *gorm.DB, settings *config.Settings) base.Broker {
return &rdsBroker{brokerDB, settings}
func InitRDSBroker(brokerDB *gorm.DB, settings *config.Settings, tagManager brokertags.TagManager) base.Broker {
return &rdsBroker{brokerDB, settings, tagManager}
}

// this helps the manager to respond appropriately depending on whether a service/plan needs an operation to be async
Expand Down Expand Up @@ -149,14 +151,32 @@ func (broker *rdsBroker) CreateInstance(c *catalog.Catalog, id string, createReq
)
}
}
err := newInstance.init(

tags, err := broker.tagManager.GenerateTags(
brokertags.Create,
c.RdsService.Name,
plan.Name,
brokertags.ResourceGUIDs{
InstanceGUID: id,
SpaceGUID: createRequest.SpaceGUID,
OrganizationGUID: createRequest.OrganizationGUID,
},
false,
)
if err != nil {
return response.NewErrorResponse(http.StatusInternalServerError, "There was an error generating the tags. Error: "+err.Error())
}

err = newInstance.init(
id,
createRequest.OrganizationGUID,
createRequest.SpaceGUID,
createRequest.ServiceID,
plan,
options,
broker.settings)
broker.settings,
tags,
)

if err != nil {
return response.NewErrorResponse(http.StatusBadRequest, "There was an error initializing the instance. Error: "+err.Error())
Expand Down
Loading

0 comments on commit 7e3172d

Please sign in to comment.