Skip to content

Commit

Permalink
Merge pull request #5588 from terraform-providers/b-aws_waf_ipset-100…
Browse files Browse the repository at this point in the history 
…0-batches

resource/aws_waf(regional)_ipset: Properly handle updates and deletions over 1000 IP set descriptors
  • Loading branch information
@bflad
bflad authored Aug 20, 2018
2 parents 066db6f + 0a3549b commit ede83e8
Show file tree
Hide file tree
Showing 4 changed files with 243 additions and 115 deletions.
47 changes: 32 additions & 15 deletions aws/resource_aws_waf_ipset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ import (
"github.com/hashicorp/terraform/helper/schema"
)

// WAF requires UpdateIPSet operations be split into batches of 1000 Updates
const wafUpdateIPSetUpdatesLimit = 1000

func resourceAwsWafIPSet() *schema.Resource {
return &schema.Resource{
Create: resourceAwsWafIPSetCreate,
Expand Down Expand Up @@ -138,7 +141,7 @@ func resourceAwsWafIPSetDelete(d *schema.ResourceData, meta interface{}) error {
noDescriptors := []interface{}{}
err := updateWafIpSetDescriptors(d.Id(), oldDescriptors, noDescriptors, conn)
if err != nil {
return fmt.Errorf("Error updating IPSetDescriptors: %s", err)
return fmt.Errorf("Error Deleting IPSetDescriptors: %s", err)
}
}

Expand All @@ -159,25 +162,28 @@ func resourceAwsWafIPSetDelete(d *schema.ResourceData, meta interface{}) error {
}

func updateWafIpSetDescriptors(id string, oldD, newD []interface{}, conn *waf.WAF) error {
wr := newWafRetryer(conn, "global")
_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
req := &waf.UpdateIPSetInput{
ChangeToken: token,
IPSetId: aws.String(id),
Updates: diffWafIpSetDescriptors(oldD, newD),
for _, ipSetUpdates := range diffWafIpSetDescriptors(oldD, newD) {
wr := newWafRetryer(conn, "global")
_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
req := &waf.UpdateIPSetInput{
ChangeToken: token,
IPSetId: aws.String(id),
Updates: ipSetUpdates,
}
log.Printf("[INFO] Updating IPSet descriptors: %s", req)
return conn.UpdateIPSet(req)
})
if err != nil {
return fmt.Errorf("Error Updating WAF IPSet: %s", err)
}
log.Printf("[INFO] Updating IPSet descriptors: %s", req)
return conn.UpdateIPSet(req)
})
if err != nil {
return fmt.Errorf("Error Updating WAF IPSet: %s", err)
}

return nil
}

func diffWafIpSetDescriptors(oldD, newD []interface{}) []*waf.IPSetUpdate {
updates := make([]*waf.IPSetUpdate, 0)
func diffWafIpSetDescriptors(oldD, newD []interface{}) [][]*waf.IPSetUpdate {
updates := make([]*waf.IPSetUpdate, 0, wafUpdateIPSetUpdatesLimit)
updatesBatches := make([][]*waf.IPSetUpdate, 0)

for _, od := range oldD {
descriptor := od.(map[string]interface{})
Expand All @@ -187,6 +193,11 @@ func diffWafIpSetDescriptors(oldD, newD []interface{}) []*waf.IPSetUpdate {
continue
}

if len(updates) == wafUpdateIPSetUpdatesLimit {
updatesBatches = append(updatesBatches, updates)
updates = make([]*waf.IPSetUpdate, 0, wafUpdateIPSetUpdatesLimit)
}

updates = append(updates, &waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionDelete),
IPSetDescriptor: &waf.IPSetDescriptor{
Expand All @@ -199,6 +210,11 @@ func diffWafIpSetDescriptors(oldD, newD []interface{}) []*waf.IPSetUpdate {
for _, nd := range newD {
descriptor := nd.(map[string]interface{})

if len(updates) == wafUpdateIPSetUpdatesLimit {
updatesBatches = append(updatesBatches, updates)
updates = make([]*waf.IPSetUpdate, 0, wafUpdateIPSetUpdatesLimit)
}

updates = append(updates, &waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionInsert),
IPSetDescriptor: &waf.IPSetDescriptor{
Expand All @@ -207,5 +223,6 @@ func diffWafIpSetDescriptors(oldD, newD []interface{}) []*waf.IPSetUpdate {
},
})
}
return updates
updatesBatches = append(updatesBatches, updates)
return updatesBatches
}
141 changes: 98 additions & 43 deletions aws/resource_aws_waf_ipset_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ package aws

import (
"fmt"
"net"
"reflect"
"regexp"
"strings"
"testing"

"github.com/hashicorp/terraform/helper/resource"
Expand Down Expand Up @@ -169,11 +171,51 @@ func TestAccAWSWafIPSet_noDescriptors(t *testing.T) {
})
}

func TestAccAWSWafIPSet_IpSetDescriptors_1000UpdateLimit(t *testing.T) {
var ipset waf.IPSet
ipsetName := fmt.Sprintf("ip-set-%s", acctest.RandString(5))
resourceName := "aws_waf_ipset.ipset"

incrementIP := func(ip net.IP) {
for j := len(ip) - 1; j >= 0; j-- {
ip[j]++
if ip[j] > 0 {
break
}
}
}

// Generate 2048 IPs
ip, ipnet, err := net.ParseCIDR("10.0.0.0/21")
if err != nil {
t.Fatal(err)
}
ipSetDescriptors := make([]string, 0, 2048)
for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); incrementIP(ip) {
ipSetDescriptors = append(ipSetDescriptors, fmt.Sprintf("ip_set_descriptors {\ntype=\"IPV4\"\nvalue=\"%s/32\"\n}", ip))
}

resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckAWSWafIPSetDestroy,
Steps: []resource.TestStep{
{
Config: testAccAWSWafIPSetConfig_IpSetDescriptors(ipsetName, strings.Join(ipSetDescriptors, "\n")),
Check: resource.ComposeAggregateTestCheckFunc(
testAccCheckAWSWafIPSetExists(resourceName, &ipset),
resource.TestCheckResourceAttr(resourceName, "ip_set_descriptors.#", "2048"),
),
},
},
})
}

func TestDiffWafIpSetDescriptors(t *testing.T) {
testCases := []struct {
Old []interface{}
New []interface{}
ExpectedUpdates []*waf.IPSetUpdate
ExpectedUpdates [][]*waf.IPSetUpdate
}{
{
// Change
Expand All @@ -183,19 +225,21 @@ func TestDiffWafIpSetDescriptors(t *testing.T) {
New: []interface{}{
map[string]interface{}{"type": "IPV4", "value": "192.0.8.0/24"},
},
ExpectedUpdates: []*waf.IPSetUpdate{
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionDelete),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("192.0.7.0/24"),
ExpectedUpdates: [][]*waf.IPSetUpdate{
[]*waf.IPSetUpdate{
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionDelete),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("192.0.7.0/24"),
},
},
},
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionInsert),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("192.0.8.0/24"),
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionInsert),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("192.0.8.0/24"),
},
},
},
},
Expand All @@ -208,26 +252,28 @@ func TestDiffWafIpSetDescriptors(t *testing.T) {
map[string]interface{}{"type": "IPV4", "value": "10.0.2.0/24"},
map[string]interface{}{"type": "IPV4", "value": "10.0.3.0/24"},
},
ExpectedUpdates: []*waf.IPSetUpdate{
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionInsert),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("10.0.1.0/24"),
ExpectedUpdates: [][]*waf.IPSetUpdate{
[]*waf.IPSetUpdate{
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionInsert),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("10.0.1.0/24"),
},
},
},
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionInsert),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("10.0.2.0/24"),
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionInsert),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("10.0.2.0/24"),
},
},
},
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionInsert),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("10.0.3.0/24"),
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionInsert),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("10.0.3.0/24"),
},
},
},
},
Expand All @@ -239,19 +285,21 @@ func TestDiffWafIpSetDescriptors(t *testing.T) {
map[string]interface{}{"type": "IPV4", "value": "192.0.8.0/24"},
},
New: []interface{}{},
ExpectedUpdates: []*waf.IPSetUpdate{
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionDelete),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("192.0.7.0/24"),
ExpectedUpdates: [][]*waf.IPSetUpdate{
[]*waf.IPSetUpdate{
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionDelete),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("192.0.7.0/24"),
},
},
},
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionDelete),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("192.0.8.0/24"),
&waf.IPSetUpdate{
Action: aws.String(waf.ChangeActionDelete),
IPSetDescriptor: &waf.IPSetDescriptor{
Type: aws.String("IPV4"),
Value: aws.String("192.0.8.0/24"),
},
},
},
},
Expand Down Expand Up @@ -401,6 +449,13 @@ func testAccAWSWafIPSetConfigChangeIPSetDescriptors(name string) string {
}`, name)
}

func testAccAWSWafIPSetConfig_IpSetDescriptors(name, ipSetDescriptors string) string {
return fmt.Sprintf(`resource "aws_waf_ipset" "ipset" {
name = "%s"
%s
}`, name, ipSetDescriptors)
}

func testAccAWSWafIPSetConfig_noDescriptors(name string) string {
return fmt.Sprintf(`resource "aws_waf_ipset" "ipset" {
name = "%s"
Expand Down
29 changes: 15 additions & 14 deletions aws/resource_aws_wafregional_ipset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ func resourceAwsWafRegionalIPSetDelete(d *schema.ResourceData, meta interface{})
err := updateIPSetResourceWR(d.Id(), oldD, noD, conn, region)

if err != nil {
return fmt.Errorf("Error Removing IPSetDescriptors: %s", err)
return fmt.Errorf("Error Deleting IPSetDescriptors: %s", err)
}
}

Expand All @@ -164,20 +164,21 @@ func resourceAwsWafRegionalIPSetDelete(d *schema.ResourceData, meta interface{})
}

func updateIPSetResourceWR(id string, oldD, newD []interface{}, conn *wafregional.WAFRegional, region string) error {

wr := newWafRegionalRetryer(conn, region)
_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
req := &waf.UpdateIPSetInput{
ChangeToken: token,
IPSetId: aws.String(id),
Updates: diffWafIpSetDescriptors(oldD, newD),
for _, ipSetUpdates := range diffWafIpSetDescriptors(oldD, newD) {
wr := newWafRegionalRetryer(conn, region)
_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
req := &waf.UpdateIPSetInput{
ChangeToken: token,
IPSetId: aws.String(id),
Updates: ipSetUpdates,
}
log.Printf("[INFO] Updating IPSet descriptor: %s", req)

return conn.UpdateIPSet(req)
})
if err != nil {
return fmt.Errorf("Error Updating WAF IPSet: %s", err)
}
log.Printf("[INFO] Updating IPSet descriptor: %s", req)

return conn.UpdateIPSet(req)
})
if err != nil {
return fmt.Errorf("Error Updating WAF IPSet: %s", err)
}

return nil
Expand Down
Loading

0 comments on commit ede83e8

Please sign in to comment.