Skip to content

Commit

Permalink
add AWS IP matching with real data
Browse files Browse the repository at this point in the history
  • Loading branch information
BenTheElder committed Apr 13, 2022
1 parent 552a203 commit 2982e29
Show file tree
Hide file tree
Showing 10 changed files with 54,337 additions and 0 deletions.
20 changes: 20 additions & 0 deletions pkg/net/cidrs/aws/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// aws contains AWS CIDR matching
package aws

//go:generate ./internal/ranges2go/run.sh
94 changes: 94 additions & 0 deletions pkg/net/cidrs/aws/internal/ranges2go/gen.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"fmt"
"io"
)

const fileHeader = `/*
Copyright The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// File generated by ranges2go DO NOT EDIT
package aws
import (
"net/netip"
)
// regionToRanges contains a preparsed map of AWS regions to netip.Prefix
var regionToRanges = map[string][]netip.Prefix{
`

// TODO: ipv6
func generateRangesGo(w io.Writer, rtp regionsToPrefixes) error {
// generate source file
if _, err := io.WriteString(w, fileHeader); err != nil {
return err
}

for region, prefixes := range rtp {
if _, err := fmt.Fprintf(w, "\t%q: {\n", region); err != nil {
return err
}
for _, prefix := range prefixes {
addr := prefix.Addr()
bits := prefix.Bits()
// using netip.*From avoids additional runtime allocation
if addr.Is4() {
b := addr.As4()
if _, err := fmt.Fprintf(w,
"\t\tnetip.PrefixFrom(netip.AddrFrom4([4]byte{%d, %d, %d, %d}), %d),\n",
b[0], b[1], b[2], b[3], bits,
); err != nil {
return err
}
} else {
b := addr.As16()
if _, err := fmt.Fprintf(w,
"\t\tnetip.PrefixFrom(netip.AddrFrom16([16]byte{%d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d}), %d),\n",
b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15], bits,
); err != nil {
return err
}
}
}
if _, err := io.WriteString(w, "\t},\n"); err != nil {
return err
}
}
if _, err := io.WriteString(w, "}\n"); err != nil {
return err
}

return nil
}
54 changes: 54 additions & 0 deletions pkg/net/cidrs/aws/internal/ranges2go/genrawdata.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/bin/bash

# Copyright 2022 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit -o nounset -o pipefail

# cd to self
cd "$(dirname "${BASH_SOURCE[0]}")"

DATA_URL='https://ip-ranges.amazonaws.com/ip-ranges.json'

# emit ip ranges data into a go source file with the string contents
# this data changes infrequently and this simplifies generating the runtime
# data
{
cat <<EOF
/*
Copyright $(date +%Y) The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// File generated by genrawdata.sh DO NOT EDIT
package main
// ipRangesRaw contains the contents of ${DATA_URL}
var ipRangesRaw = \`
EOF
curl "${DATA_URL}"
echo '`'
}>zz_generated_rawdata.go
40 changes: 40 additions & 0 deletions pkg/net/cidrs/aws/internal/ranges2go/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// ranges2go generates a go source file with pre-parsed AWS IP ranges data.
// See also genrawdata.sh for downloading the raw data to this binary.
package main

import "os"

func main() {
parsed, err := parseIPRangesJSON([]byte(ipRangesRaw))
if err != nil {
panic(err)
}
rtp, err := regionsToPrefixesFromData(parsed)
if err != nil {
panic(err)
}
// emit file
f, err := os.Create("./../../zz_generated_range_data.go")
if err != nil {
panic(err)
}
if err := generateRangesGo(f, rtp); err != nil {
panic(err)
}
}
114 changes: 114 additions & 0 deletions pkg/net/cidrs/aws/internal/ranges2go/parse.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"encoding/json"
"net/netip"
"sort"
)

/*
For more on these datatypes see:
https://docs.aws.amazon.com/general/latest/gr/aws-ip-ranges.html
*/

type IPRangesJSON struct {
Prefixes []Prefix `json:"prefixes"`
IPv6Prefixes []IPv6Prefix `json:"ipv6_prefixes"`
// syncToken and createDate omitted
}

type Prefix struct {
IPPrefix string `json:"ip_prefix"`
Region string `json:"region"`
Service string `json:"service"`
// network_border_group omitted
}

type IPv6Prefix struct {
IPv6Prefix string `json:"ipv6_prefix"`
Region string `json:"region"`
Service string `json:"service"`
// network_border_group omitted
}

// parseIPRangesJSON parse AWS IP ranges JSON data
// https://docs.aws.amazon.com/general/latest/gr/aws-ip-ranges.html
func parseIPRangesJSON(rawJSON []byte) (*IPRangesJSON, error) {
r := &IPRangesJSON{}
if err := json.Unmarshal(rawJSON, r); err != nil {
return nil, err
}
return r, nil
}

// regionsToPrefixes is the structure we process the JSON into
type regionsToPrefixes map[string][]netip.Prefix

// regionsToPrefixesFromData processes the raw unmarshalled JSON into regionsToPrefixes map
func regionsToPrefixesFromData(data *IPRangesJSON) (regionsToPrefixes, error) {
// convert from AWS published structure to a map by region, parse Prefixes
rtp := regionsToPrefixes{}
for _, prefix := range data.Prefixes {
region := prefix.Region
ipPrefix, err := netip.ParsePrefix(prefix.IPPrefix)
if err != nil {
return nil, err
}
rtp[region] = append(rtp[region], ipPrefix)
}
for _, prefix := range data.IPv6Prefixes {
region := prefix.Region
ipPrefix, err := netip.ParsePrefix(prefix.IPv6Prefix)
if err != nil {
return nil, err
}
rtp[region] = append(rtp[region], ipPrefix)
}

// flatten
numPrefixes := 0
for region := range rtp {
// this approach allows us to produce consistent generated results
// since the ip ranges will be ordered
sort.Slice(rtp[region], func(i, j int) bool {
return rtp[region][i].String() < rtp[region][j].String()
})
rtp[region] = dedupeSortedPrefixes(rtp[region])
numPrefixes += len(rtp[region])
}

return rtp, nil
}

func dedupeSortedPrefixes(s []netip.Prefix) []netip.Prefix {
l := len(s)
// nothing to do for <= 1
if l <= 1 {
return s
}
// for 1..len(s) if previous entry does not match, keep current
j := 0
for i := 1; i < l; i++ {
if s[i].String() != s[i-1].String() {
s[j] = s[i]
j++
}
}
return s[0:j]
}
22 changes: 22 additions & 0 deletions pkg/net/cidrs/aws/internal/ranges2go/run.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/bin/bash

# Copyright 2022 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit -o nounset -o pipefail

# cd to self
cd "$(dirname "${BASH_SOURCE[0]}")"

go run ./*.go
Loading

0 comments on commit 2982e29

Please sign in to comment.