Merge pull request #42 from BenTheElder/cidr-mapping

implement CIDR matching

hack/tools/.golangci.yml

@@ -12,7 +12,9 @@ linters:
     - ineffassign
     - staticcheck
     - typecheck
-    - unused
+    # TODO: reenable, seems to be broken in go 1.18
+    # keep an eye on https://github.com/golangci/golangci-lint/issues/2649
+    #- unused
 
     # additional lints
     - gochecknoinits

pkg/net/cidrs/aws/doc.go

@@ -0,0 +1,20 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// aws contains AWS CIDR matching
+package aws
+
+//go:generate ./internal/ranges2go/run.sh

pkg/net/cidrs/aws/internal/ranges2go/gen.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io"
+)
+
+const fileHeader = `/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// File generated by ranges2go DO NOT EDIT
+
+package aws
+
+import (
+	"net/netip"
+)
+
+// regionToRanges contains a preparsed map of AWS regions to netip.Prefix
+var regionToRanges = map[string][]netip.Prefix{
+`
+
+func generateRangesGo(w io.Writer, rtp regionsToPrefixes) error {
+	// generate source file
+	if _, err := io.WriteString(w, fileHeader); err != nil {
+		return err
+	}
+
+	for region, prefixes := range rtp {
+		if _, err := fmt.Fprintf(w, "\t%q: {\n", region); err != nil {
+			return err
+		}
+		for _, prefix := range prefixes {
+			addr := prefix.Addr()
+			bits := prefix.Bits()
+			// Using netip.*From avoids additional runtime allocation.
+			//
+			// It also means we don't need error checking / parsing cannot fail
+			// at runtime, we've already parsed these and re-emitted them
+			// as pre-computed IP address / bit mask values.
+			if addr.Is4() {
+				b := addr.As4()
+				if _, err := fmt.Fprintf(w,
+					"\t\tnetip.PrefixFrom(netip.AddrFrom4([4]byte{%d, %d, %d, %d}), %d),\n",
+					b[0], b[1], b[2], b[3], bits,
+				); err != nil {
+					return err
+				}
+			} else {
+				b := addr.As16()
+				if _, err := fmt.Fprintf(w,
+					"\t\tnetip.PrefixFrom(netip.AddrFrom16([16]byte{%d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d}), %d),\n",
+					b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15], bits,
+				); err != nil {
+					return err
+				}
+			}
+		}
+		if _, err := io.WriteString(w, "\t},\n"); err != nil {
+			return err
+		}
+	}
+	if _, err := io.WriteString(w, "}\n"); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/net/cidrs/aws/internal/ranges2go/genrawdata.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit -o nounset -o pipefail
+
+# cd to self
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+DATA_URL='https://ip-ranges.amazonaws.com/ip-ranges.json'
+
+# emit ip ranges data into a go source file with the string contents
+# this data changes infrequently and this simplifies generating the runtime
+# data
+{
+    cat <<EOF
+/*
+Copyright $(date +%Y) The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// File generated by genrawdata.sh DO NOT EDIT
+
+package main
+
+// ipRangesRaw contains the contents of ${DATA_URL}
+var ipRangesRaw = \`
+EOF
+    curl "${DATA_URL}"
+    echo '`'
+}>zz_generated_rawdata.go

pkg/net/cidrs/aws/internal/ranges2go/main.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// ranges2go generates a go source file with pre-parsed AWS IP ranges data.
+// See also genrawdata.sh for downloading the raw data to this binary.
+package main
+
+import "os"
+
+func main() {
+	parsed, err := parseIPRangesJSON([]byte(ipRangesRaw))
+	if err != nil {
+		panic(err)
+	}
+	rtp, err := regionsToPrefixesFromData(parsed)
+	if err != nil {
+		panic(err)
+	}
+	// emit file
+	f, err := os.Create("./../../zz_generated_range_data.go")
+	if err != nil {
+		panic(err)
+	}
+	if err := generateRangesGo(f, rtp); err != nil {
+		panic(err)
+	}
+}

pkg/net/cidrs/aws/internal/ranges2go/parse.go

@@ -0,0 +1,114 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"encoding/json"
+	"net/netip"
+	"sort"
+)
+
+/*
+	For more on these datatypes see:
+	https://docs.aws.amazon.com/general/latest/gr/aws-ip-ranges.html
+*/
+
+type IPRangesJSON struct {
+	Prefixes     []Prefix     `json:"prefixes"`
+	IPv6Prefixes []IPv6Prefix `json:"ipv6_prefixes"`
+	// syncToken and createDate omitted
+}
+
+type Prefix struct {
+	IPPrefix string `json:"ip_prefix"`
+	Region   string `json:"region"`
+	Service  string `json:"service"`
+	// network_border_group omitted
+}
+
+type IPv6Prefix struct {
+	IPv6Prefix string `json:"ipv6_prefix"`
+	Region     string `json:"region"`
+	Service    string `json:"service"`
+	// network_border_group omitted
+}
+
+// parseIPRangesJSON parse AWS IP ranges JSON data
+// https://docs.aws.amazon.com/general/latest/gr/aws-ip-ranges.html
+func parseIPRangesJSON(rawJSON []byte) (*IPRangesJSON, error) {
+	r := &IPRangesJSON{}
+	if err := json.Unmarshal(rawJSON, r); err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+// regionsToPrefixes is the structure we process the JSON into
+type regionsToPrefixes map[string][]netip.Prefix
+
+// regionsToPrefixesFromData processes the raw unmarshalled JSON into regionsToPrefixes map
+func regionsToPrefixesFromData(data *IPRangesJSON) (regionsToPrefixes, error) {
+	// convert from AWS published structure to a map by region, parse Prefixes
+	rtp := regionsToPrefixes{}
+	for _, prefix := range data.Prefixes {
+		region := prefix.Region
+		ipPrefix, err := netip.ParsePrefix(prefix.IPPrefix)
+		if err != nil {
+			return nil, err
+		}
+		rtp[region] = append(rtp[region], ipPrefix)
+	}
+	for _, prefix := range data.IPv6Prefixes {
+		region := prefix.Region
+		ipPrefix, err := netip.ParsePrefix(prefix.IPv6Prefix)
+		if err != nil {
+			return nil, err
+		}
+		rtp[region] = append(rtp[region], ipPrefix)
+	}
+
+	// flatten
+	numPrefixes := 0
+	for region := range rtp {
+		// this approach allows us to produce consistent generated results
+		// since the ip ranges will be ordered
+		sort.Slice(rtp[region], func(i, j int) bool {
+			return rtp[region][i].String() < rtp[region][j].String()
+		})
+		rtp[region] = dedupeSortedPrefixes(rtp[region])
+		numPrefixes += len(rtp[region])
+	}
+
+	return rtp, nil
+}
+
+func dedupeSortedPrefixes(s []netip.Prefix) []netip.Prefix {
+	l := len(s)
+	// nothing to do for <= 1
+	if l <= 1 {
+		return s
+	}
+	// for 1..len(s) if previous entry does not match, keep current
+	j := 0
+	for i := 1; i < l; i++ {
+		if s[i].String() != s[i-1].String() {
+			s[j] = s[i]
+			j++
+		}
+	}
+	return s[0:j]
+}

pkg/net/cidrs/aws/internal/ranges2go/run.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit -o nounset -o pipefail
+
+# cd to self
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+go run ./*.go