Add AliasIP range support to flannel

backend/gce/api.go

@@ -1,3 +1,5 @@
+// Copyright 2021 Splunk Inc.
+//
 // Copyright 2015 flannel authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,8 +19,10 @@ package gce
 
 import (
  "context"
+ "errors"
  "fmt"
  "os"
+ "path"
  "time"
 
  log "github.com/golang/glog"
@@ -31,9 +35,24 @@ import (
 // When set, network routes will be created within a network project instead of the project running the instances
 const EnvGCENetworkProjectID = "GCE_NETWORK_PROJECT_ID"
 
+// EnvKubeClusterID is an environment variable that contains the cluster name
+// This variable is used as the subnetwork secondary range name
+const EnvKubeClusterID = "KUBE_CLUSTER_ID"
+
+// Constants used for operation polling
+const (
+ zoneScope = "zone"
+ globalScope = "global"
+)
+
 type gceAPI struct {
- project string
- useIPNextHop bool
+ networkProject string
+ useIPNextHop bool
+ instanceProject string
+ instanceName string
+ instanceZone string
+ instanceRegion string
+
  computeService *compute.Service
  gceNetwork *compute.Network
  gceInstance *compute.Instance
@@ -61,7 +80,7 @@ func newAPI() (*gceAPI, error) {
  return nil, fmt.Errorf("error getting network metadata: %v", err)
  }
 
- prj, err := projectFromMetadata()
+ instanceProject, err := projectFromMetadata()
  if err != nil {
  return nil, fmt.Errorf("error getting project: %v", err)
  }
@@ -76,46 +95,59 @@ func newAPI() (*gceAPI, error) {
  return nil, fmt.Errorf("error getting instance zone: %v", err)
  }
 
- // netPrj refers to the project which owns the network being used
+ instanceRegion, err := instanceRegionFromMetadata()
+ if err != nil {
+ return nil, fmt.Errorf("error getting instance region: %v", err)
+ }
+
+ // networkProject refers to the project which owns the network being used
  // defaults to what is read by the metadata
- netPrj := prj
+ networkProject := instanceProject
  // has the network project been provided?
  if v := os.Getenv(EnvGCENetworkProjectID); v != "" {
- netPrj = v
+ networkProject = v
  }
 
- gn, err := cs.Networks.Get(netPrj, networkName).Do()
+ gn, err := cs.Networks.Get(networkProject, networkName).Do()
  if err != nil {
  return nil, fmt.Errorf("error getting network from compute service: %v", err)
  }
 
- gi, err := cs.Instances.Get(prj, instanceZone, instanceName).Do()
+ gi, err := cs.Instances.Get(instanceProject, instanceZone, instanceName).Do()
  if err != nil {
  return nil, fmt.Errorf("error getting instance from compute service: %v", err)
  }
 
+ if len(gi.NetworkInterfaces) == 0 {
+ return nil, errors.New("expected at least 1 network interface but found zero")
+ }
+
  // if the instance project is different from the network project
  // we need to use the ip as the next hop when creating routes
  // cross project referencing is not allowed for instances
- useIPNextHop := prj != netPrj
+ useIPNextHop := instanceProject != networkProject
 
  return &gceAPI{
- project: netPrj,
- useIPNextHop: useIPNextHop,
- computeService: cs,
- gceNetwork: gn,
- gceInstance: gi,
+ networkProject: networkProject,
+ instanceProject: instanceProject,
+ instanceZone: instanceZone,
+ instanceRegion: instanceRegion,
+ instanceName: instanceName,
+ useIPNextHop: useIPNextHop,
+ computeService: cs,
+ gceNetwork: gn,
+ gceInstance: gi,
  }, nil
 }
 
 func (api *gceAPI) getRoute(subnet string) (*compute.Route, error) {
  routeName := formatRouteName(subnet)
- return api.computeService.Routes.Get(api.project, routeName).Do()
+ return api.computeService.Routes.Get(api.networkProject, routeName).Do()
 }
 
 func (api *gceAPI) deleteRoute(subnet string) (*compute.Operation, error) {
  routeName := formatRouteName(subnet)
- return api.computeService.Routes.Delete(api.project, routeName).Do()
+ return api.computeService.Routes.Delete(api.networkProject, routeName).Do()
 }
 
 func (api *gceAPI) insertRoute(subnet string) (*compute.Operation, error) {
@@ -139,12 +171,128 @@ func (api *gceAPI) insertRoute(subnet string) (*compute.Operation, error) {
  route.NextHopInstance = api.gceInstance.SelfLink
  }
 
- return api.computeService.Routes.Insert(api.project, route).Do()
+ return api.computeService.Routes.Insert(api.networkProject, route).Do()
+}
+
+// refresh the held instance with the most recent information
+func (api *gceAPI) refreshInstance() error {
+ instance, err := api.computeService.Instances.Get(api.instanceProject, api.instanceZone, api.instanceName).Do()
+ if err != nil {
+ return err
+ }
+ api.gceInstance = instance
+ return nil
+}
+
+// combine ranges by name, updating any existing entries
+func combineSecondaryRanges(ranges []*compute.SubnetworkSecondaryRange, newRange *compute.SubnetworkSecondaryRange) []*compute.SubnetworkSecondaryRange {
+ m := make(map[string]*compute.SubnetworkSecondaryRange)
+
+ for i, secondaryRange := range ranges {
+ m[secondaryRange.RangeName] = ranges[i]
+ }
+
+ m[newRange.RangeName] = newRange
+
+ combined := make([]*compute.SubnetworkSecondaryRange, 0, len(m))
+ for key := range m {
+ combined = append(combined, m[key])
+ }
+ return combined
+}
+
+func (api *gceAPI) addSubnetSecondaryRange(networkCidr string, rangeName string) (*compute.Operation, error) {
+ subnetworkName := path.Base(api.gceInstance.NetworkInterfaces[0].Subnetwork)
+ subnetwork, err := api.computeService.Subnetworks.Get(api.networkProject, api.instanceRegion, subnetworkName).Do()
+ if err != nil {
+ return nil, err
+ }
+
+ for _, secondaryIPRange := range subnetwork.SecondaryIpRanges {
+ if secondaryIPRange.RangeName == rangeName && secondaryIPRange.IpCidrRange == networkCidr {
+ log.Infof("Found existing secondary IP range '%s' with cidr '%s'", secondaryIPRange.RangeName, secondaryIPRange.IpCidrRange)
+ return nil, nil
+ }
+ }
+
+ newRange := &compute.SubnetworkSecondaryRange{
+ IpCidrRange: networkCidr,
+ RangeName: rangeName,
+ }
+
+ subnetworkUpdate := &compute.Subnetwork{
+ Fingerprint: subnetwork.Fingerprint,
+ SecondaryIpRanges: combineSecondaryRanges(subnetwork.SecondaryIpRanges, newRange),
+ }
+
+ log.Infof("Adding secondary range '%s' with network '%s' to subnet '%s'", rangeName, networkCidr, subnetwork.Name)
+ return api.computeService.Subnetworks.Patch(api.networkProject, api.instanceRegion, subnetwork.Name, subnetworkUpdate).Do()
+}
+
+// combine ranges by name, updating any existing entries
+func combineAliasRanges(ranges []*compute.AliasIpRange, newRange *compute.AliasIpRange) []*compute.AliasIpRange {
+ m := make(map[string]*compute.AliasIpRange)
+
+ for i, aliasRange := range ranges {
+ m[aliasRange.SubnetworkRangeName] = ranges[i]
+ }
+
+ m[newRange.SubnetworkRangeName] = newRange
+
+ combined := make([]*compute.AliasIpRange, 0, len(m))
+ for key := range m {
+ combined = append(combined, m[key])
+ }
+ return combined
+}
+
+func (api *gceAPI) addAliasIPRange(subnetCidr string, rangeName string) (*compute.Operation, error) {
+ err := api.refreshInstance()
+ if err != nil {
+ return nil, err
+ }
+
+ newRange := &compute.AliasIpRange{
+ IpCidrRange: subnetCidr,
+ SubnetworkRangeName: rangeName,
+ }
+
+ networkInterface := &compute.NetworkInterface{
+ Fingerprint: api.gceInstance.NetworkInterfaces[0].Fingerprint,
+ AliasIpRanges: combineAliasRanges(api.gceInstance.NetworkInterfaces[0].AliasIpRanges, newRange),
+ }
+
+ log.Infof("Adding alias cidr '%s' as part of range '%s' to instance '%s'", subnetCidr, rangeName, api.instanceName)
+ operation, err := api.computeService.Instances.UpdateNetworkInterface(api.instanceProject,
+ api.instanceZone,
+ api.instanceName,
+ api.gceInstance.NetworkInterfaces[0].Name,
+ networkInterface).Do()
+
+ if err != nil {
+ return nil, err
+ }
+ return operation, nil
 }
 
-func (api *gceAPI) pollOperationStatus(operationName string) error {
+func (api *gceAPI) pollOperationStatus(project string, scope string, o *compute.Operation) error {
+ if o == nil || o.Status == "DONE" {
+ return nil
+ }
+
+ operationName := o.Name
  for i := 0; i < 100; i++ {
- operation, err := api.computeService.GlobalOperations.Get(api.project, operationName).Do()
+ var operation *compute.Operation
+ var err error
+ switch scope {
+ case globalScope:
+ operation, err = api.computeService.GlobalOperations.Get(project, operationName).Do()
+ case zoneScope:
+ operation, err = api.computeService.ZoneOperations.Get(project, api.instanceZone, operationName).Do()
+ default:
+ return fmt.Errorf("Unsupported operation scope: %s", scope)
+ }
+
  if err != nil {
  return fmt.Errorf("error fetching operation status: %v", err)
  }