Skip to content

SDN-3708: Support configurable masquerade subnet in ovn-k #1807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-bot merged 1 commit into from
Jan 22, 2024
Merged

SDN-3708: Support configurable masquerade subnet in ovn-k #1807

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion bindata/network/ovn-kubernetes/managed/004-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,14 +144,21 @@ data:
{{- if (index . "V6JoinSubnet") }}
v6-join-subnet="{{.V6JoinSubnet}}"
{{- end }}
{{- if (index . "V4InternalMasqueradeSubnet")}}
v4-internal-masquerade-subnet="{{.V4InternalMasqueradeSubnet}}"
{{- end }}
{{- if (index . "V6InternalMasqueradeSubnet")}}
v6-internal-masquerade-subnet="{{.V6InternalMasqueradeSubnet}}"
{{- end }}


[logging]
libovsdblogfile=/var/log/ovnkube/libovsdb.log
logfile-maxsize=100
logfile-maxbackups=5
logfile-maxage=0
{{- if .OVNHybridOverlayEnable }}

{{- if .OVNHybridOverlayEnable }}
[hybridoverlay]
enabled=true
{{- if .OVNHybridOverlayNetCIDR }}
Expand Down
7 changes: 7 additions & 0 deletions bindata/network/ovn-kubernetes/self-hosted/004-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,13 @@ data:
v6-join-subnet="{{.V6JoinSubnet}}"
{{- end }}

{{- if (index . "V4InternalMasqueradeSubnet")}}
v4-internal-masquerade-subnet="{{.V4InternalMasqueradeSubnet}}"
{{- end }}
{{- if (index . "V6InternalMasqueradeSubnet")}}
v6-internal-masquerade-subnet="{{.V6InternalMasqueradeSubnet}}"
{{- end }}

Comment on lines +69 to +75
Copy link
Contributor

@jcaamano jcaamano Jan 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think that this will cause a rollout/restart of ovn-k, making it a day-1 only configuration parameter. Is that ok?

Copy link
Contributor Author

@bpickard22 bpickard22 Jan 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes that is the current intended behavior

[logging]
libovsdblogfile=/var/log/ovnkube/libovsdb.log
logfile-maxsize=100
Expand Down
143 changes: 110 additions & 33 deletions pkg/network/ovn_kubernetes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,17 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo
data.Data["TokenAudience"] = os.Getenv("TOKEN_AUDIENCE")
data.Data["MTU"] = c.MTU
data.Data["RoutableMTU"] = nil
// v4 and v6 join subnet are used when the user wants to use the addresses that we reserve for the join subnet in ovn-k
// TODO: this field is being deprecated and will turn into c.GatewayConfig.IPv4/6.InternalJoinSubnet when we introduce the transit switch config into the api
data.Data["V4JoinSubnet"] = c.V4InternalSubnet
data.Data["V6JoinSubnet"] = c.V6InternalSubnet
// v4 and v6InternalMasqueradeSubnet are used when the user wants to use the addresses that we reserve in ovn-k for ip masquerading
if c.GatewayConfig != nil && c.GatewayConfig.IPv4.InternalMasqueradeSubnet != "" {
data.Data["V4InternalMasqueradeSubnet"] = c.GatewayConfig.IPv4.InternalMasqueradeSubnet
}
if c.GatewayConfig != nil && c.GatewayConfig.IPv6.InternalMasqueradeSubnet != "" {
data.Data["V6InternalMasqueradeSubnet"] = c.GatewayConfig.IPv6.InternalMasqueradeSubnet
}
data.Data["EnableUDPAggregation"] = !bootstrapResult.OVN.OVNKubernetesConfig.DisableUDPAggregation
data.Data["NETWORK_NODE_IDENTITY_ENABLE"] = bootstrapResult.Infra.NetworkNodeIdentityEnabled
data.Data["NodeIdentityCertDuration"] = OVN_NODE_IDENTITY_CERT_DURATION
Expand Down Expand Up @@ -813,62 +822,130 @@ func validateOVNKubernetes(conf *operv1.NetworkSpec) []error {
if oc.GenevePort != nil && (*oc.GenevePort < 1 || *oc.GenevePort > 65535) {
out = append(out, errors.Errorf("invalid GenevePort %d", *oc.GenevePort))
}
var v4Net, v6Net *net.IPNet
var v4JoinNet, v6JoinNet, v4MasqNet, v6MasqNet *net.IPNet
var err error
if oc.V4InternalSubnet != "" {
if !cnHasIPv4 {
out = append(out, errors.Errorf("v4InternalSubnet and ClusterNetwork must have matching IP families"))
out = append(out, errors.Errorf("v4InternalSubnet %s and ClusterNetwork must have matching IP families", oc.V4InternalSubnet))
}
_, v4Net, err = net.ParseCIDR(oc.V4InternalSubnet)
_, v4JoinNet, err = net.ParseCIDR(oc.V4InternalSubnet)
if err != nil {
out = append(out, errors.Errorf("v4InternalSubnet is invalid: %s", err))
} else if !isV4InternalSubnetLargeEnough(conf) {
out = append(out, errors.Errorf("v4InternalSubnet %s is not large enough for the maximum number of nodes which can be supported by ClusterNetwork", oc.V4InternalSubnet))
}
if !isV4InternalSubnetLargeEnough(conf) {
out = append(out, errors.Errorf("v4InternalSubnet is no large enough for the maximum number of nodes which can be supported by ClusterNetwork"))
if v4JoinNet != nil {
for _, cn := range conf.ClusterNetwork {
if utilnet.IsIPv4CIDRString(cn.CIDR) {
_, v4ClusterNet, _ := net.ParseCIDR(cn.CIDR)
if iputil.NetsOverlap(*v4JoinNet, *v4ClusterNet) {
out = append(out, errors.Errorf("v4InternalSubnet %s overlaps with ClusterNetwork %s", oc.V4InternalSubnet, cn.CIDR))
}
}
}
for _, sn := range conf.ServiceNetwork {
if utilnet.IsIPv4CIDRString(sn) {
_, v4ServiceNet, _ := net.ParseCIDR(sn)
if iputil.NetsOverlap(*v4JoinNet, *v4ServiceNet) {
out = append(out, errors.Errorf("v4InternalSubnet %s overlaps with ServiceNetwork %s", oc.V4InternalSubnet, sn))
}
}
}
}
}
if oc.V6InternalSubnet != "" {
if !cnHasIPv6 {
out = append(out, errors.Errorf("v6InternalSubnet and ClusterNetwork must have matching IP families"))
out = append(out, errors.Errorf("v6InternalSubnet %s and ClusterNetwork must have matching IP families", oc.V6InternalSubnet))
}
_, v6Net, err = net.ParseCIDR(oc.V6InternalSubnet)
_, v6JoinNet, err = net.ParseCIDR(oc.V6InternalSubnet)
if err != nil {
out = append(out, errors.Errorf("v6InternalSubnet is invalid: %s", err))
} else if !isV6InternalSubnetLargeEnough(conf) {
out = append(out, errors.Errorf("v6InternalSubnet %s is not large enough for the maximum number of nodes which can be supported by ClusterNetwork", oc.V6InternalSubnet))
}
if !isV6InternalSubnetLargeEnough(conf) {
out = append(out, errors.Errorf("v6InternalSubnet is no large enough for the maximum number of nodes which can be supported by ClusterNetwork"))
}
}
for _, cn := range conf.ClusterNetwork {
if utilnet.IsIPv6CIDRString(cn.CIDR) {
if oc.V6InternalSubnet != "" {
_, v6ClusterNet, _ := net.ParseCIDR(cn.CIDR)
if iputil.NetsOverlap(*v6Net, *v6ClusterNet) {
out = append(out, errors.Errorf("v6InternalSubnet overlaps with ClusterNetwork %s", cn.CIDR))
if v6JoinNet != nil {
for _, cn := range conf.ClusterNetwork {
if utilnet.IsIPv6CIDRString(cn.CIDR) {
_, v6ClusterNet, _ := net.ParseCIDR(cn.CIDR)
if iputil.NetsOverlap(*v6JoinNet, *v6ClusterNet) {
out = append(out, errors.Errorf("v6InternalSubnet %s overlaps with ClusterNetwork %s", oc.V6InternalSubnet, cn.CIDR))
}
}
}
} else {
if oc.V4InternalSubnet != "" {
_, v4ClusterNet, _ := net.ParseCIDR(cn.CIDR)
if iputil.NetsOverlap(*v4Net, *v4ClusterNet) {
out = append(out, errors.Errorf("v4InternalSubnet overlaps with ClusterNetwork %s", cn.CIDR))
for _, sn := range conf.ServiceNetwork {
if utilnet.IsIPv6CIDRString(sn) {
_, v6ServiceNet, _ := net.ParseCIDR(sn)
if iputil.NetsOverlap(*v6JoinNet, *v6ServiceNet) {
out = append(out, errors.Errorf("v6InternalSubnet %s overlaps with ServiceNetwork %s", oc.V6InternalSubnet, sn))
}
}
}
}
}
for _, sn := range conf.ServiceNetwork {
if utilnet.IsIPv6CIDRString(sn) {
if oc.V6InternalSubnet != "" {
_, v6ServiceNet, _ := net.ParseCIDR(sn)
if iputil.NetsOverlap(*v6Net, *v6ServiceNet) {
out = append(out, errors.Errorf("v6InternalSubnet overlaps with ServiceNetwork %s", sn))

// Gateway Configurable Subnet Checks
if oc.GatewayConfig != nil {
if oc.GatewayConfig.IPv4.InternalMasqueradeSubnet != "" {
if !cnHasIPv4 {
out = append(out, errors.Errorf("v4InternalMasqueradeSubnet %s and ClusterNetwork must have matching IP families", oc.GatewayConfig.IPv4.InternalMasqueradeSubnet))
}
_, v4MasqNet, err = net.ParseCIDR(oc.GatewayConfig.IPv4.InternalMasqueradeSubnet)
if err != nil {
out = append(out, errors.Errorf("v4InternalMasqueradeSubnet is invalid: %s", err))
}
if v4MasqNet != nil {
for _, cn := range conf.ClusterNetwork {
if utilnet.IsIPv4CIDRString(cn.CIDR) {
_, v4ClusterNet, _ := net.ParseCIDR(cn.CIDR)
if iputil.NetsOverlap(*v4MasqNet, *v4ClusterNet) {
out = append(out, errors.Errorf("v4InternalMasqueradeSubnet %s overlaps with ClusterNetwork %s", oc.GatewayConfig.IPv4.InternalMasqueradeSubnet, cn.CIDR))
}
}
}
for _, sn := range conf.ServiceNetwork {
if utilnet.IsIPv4CIDRString(sn) {
_, v4ServiceNet, _ := net.ParseCIDR(sn)
if iputil.NetsOverlap(*v4MasqNet, *v4ServiceNet) {
out = append(out, errors.Errorf("v4InternalMasqueradeSubnet %s overlaps with ServiceNetwork %s", oc.GatewayConfig.IPv4.InternalMasqueradeSubnet, sn))
}
}
}
if oc.V4InternalSubnet != "" && v4JoinNet != nil {
if iputil.NetsOverlap(*v4JoinNet, *v4MasqNet) {
out = append(out, errors.Errorf("v4InternalMasqueradeSubnet %s overlaps with v4InternalSubnet %s", oc.GatewayConfig.IPv4.InternalMasqueradeSubnet, oc.V4InternalSubnet))
}
}
} //TODO: ADD a utility to check for overlap between ovn-k configured subnets i.e. masq subnet, join subnet, and upcoming transit switch subnet
}
if oc.GatewayConfig.IPv6.InternalMasqueradeSubnet != "" {
if !cnHasIPv6 {
out = append(out, errors.Errorf("v6InternalMasqueradeSubnet %s and ClusterNetwork must have matching IP families", oc.GatewayConfig.IPv6.InternalMasqueradeSubnet))
}
} else {
if oc.V4InternalSubnet != "" {
_, v4ServiceNet, _ := net.ParseCIDR(sn)
if iputil.NetsOverlap(*v4Net, *v4ServiceNet) {
out = append(out, errors.Errorf("v4InternalSubnet overlaps with ServiceNetwork %s", sn))
_, v6MasqNet, err = net.ParseCIDR(oc.GatewayConfig.IPv6.InternalMasqueradeSubnet)
if err != nil {
out = append(out, errors.Errorf("v6InternalMasqueradeSubnet is invalid: %s", err))
}
if v6MasqNet != nil {
for _, cn := range conf.ClusterNetwork {
if utilnet.IsIPv6CIDRString(cn.CIDR) {
_, v6ClusterNet, _ := net.ParseCIDR(cn.CIDR)
if iputil.NetsOverlap(*v6MasqNet, *v6ClusterNet) {
out = append(out, errors.Errorf("v6InternalMasqueradeSubnet %s overlaps with ClusterNetwork %s", oc.GatewayConfig.IPv6.InternalMasqueradeSubnet, cn.CIDR))
}
}
}
for _, sn := range conf.ServiceNetwork {
if utilnet.IsIPv6CIDRString(sn) {
_, v6ServiceNet, _ := net.ParseCIDR(sn)
if iputil.NetsOverlap(*v6MasqNet, *v6ServiceNet) {
out = append(out, errors.Errorf("v6InternalMasqueradeSubnet %s overlaps with ServiceNetwork %s", oc.GatewayConfig.IPv6.InternalMasqueradeSubnet, sn))
}
}
}
if v6JoinNet != nil {
if iputil.NetsOverlap(*v6JoinNet, *v6MasqNet) {
out = append(out, errors.Errorf("v6InternalMasqueradeSubnet %s overlaps with v6InternalSubnet %s", oc.GatewayConfig.IPv6.InternalMasqueradeSubnet, oc.V6InternalSubnet))
}
}
}
}
Expand Down
90 changes: 79 additions & 11 deletions pkg/network/ovn_kubernetes_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,53 @@ logfile-maxage=0`,
v4InternalSubnet: "100.99.0.0/16",
controlPlaneReplicaCount: 2,
},
{
desc: "custom masquerade subnet",
expected: `
[default]
mtu="1500"
cluster-subnets="10.128.0.0/15/23,10.0.0.0/14/24"
encap-port="8061"
enable-lflow-cache=true
lflow-cache-limit-kb=1048576
enable-udp-aggregation=true

[kubernetes]
service-cidrs="172.30.0.0/16"
ovn-config-namespace="openshift-ovn-kubernetes"
apiserver="https://testing.test:8443"
host-network-namespace="openshift-host-network"
platform-type="GCP"
healthz-bind-address="0.0.0.0:10256"
dns-service-namespace="openshift-dns"
dns-service-name="dns-default"

[ovnkubernetesfeature]
enable-egress-ip=true
enable-egress-firewall=true
enable-egress-qos=true
enable-egress-service=true
egressip-node-healthcheck-port=9107
enable-multi-network=true
enable-multi-external-gateway=true

[gateway]
mode=shared
nodeport=true
v4-internal-masquerade-subnet="100.98.0.0/16"

[logging]
libovsdblogfile=/var/log/ovnkube/libovsdb.log
logfile-maxsize=100
logfile-maxbackups=5
logfile-maxage=0`,
controlPlaneReplicaCount: 2,
gatewayConfig: &operv1.GatewayConfig{
IPv4: operv1.IPv4GatewayConfig{
InternalMasqueradeSubnet: "100.98.0.0/16",
},
},
},
{
desc: "HybridOverlay",
expected: `
Expand Down Expand Up @@ -783,8 +830,11 @@ logfile-maxage=0`,
if tc.hybridOverlayConfig != nil {
OVNKubeConfig.Spec.DefaultNetwork.OVNKubernetesConfig.HybridOverlayConfig = tc.hybridOverlayConfig
}
if tc.hybridOverlayConfig != nil {
if tc.gatewayConfig != nil {
OVNKubeConfig.Spec.DefaultNetwork.OVNKubernetesConfig.GatewayConfig = tc.gatewayConfig
if tc.gatewayConfig.IPv4.InternalMasqueradeSubnet != "" {
OVNKubeConfig.Spec.DefaultNetwork.OVNKubernetesConfig.GatewayConfig.IPv4.InternalMasqueradeSubnet = tc.gatewayConfig.IPv4.InternalMasqueradeSubnet
}
}
if tc.egressIPConfig != nil {
OVNKubeConfig.Spec.DefaultNetwork.OVNKubernetesConfig.EgressIPConfig = *tc.egressIPConfig
Expand Down Expand Up @@ -988,6 +1038,7 @@ func TestValidateOVNKubernetes(t *testing.T) {
crd := OVNKubernetesConfig.DeepCopy()
config := &crd.Spec
ovnConfig := config.DefaultNetwork.OVNKubernetesConfig
ovnConfig.GatewayConfig = &operv1.GatewayConfig{}

err := validateOVNKubernetes(config)
g.Expect(err).To(BeEmpty())
Expand All @@ -1009,15 +1060,21 @@ func TestValidateOVNKubernetes(t *testing.T) {
}

ovnConfig.V4InternalSubnet = "100.64.0.0/22"
errExpect("v4InternalSubnet is no large enough for the maximum number of nodes which can be supported by ClusterNetwork")
errExpect("v4InternalSubnet 100.64.0.0/22 is not large enough for the maximum number of nodes which can be supported by ClusterNetwork")
ovnConfig.V4InternalSubnet = "100.64.0.0/21"
errNotExpect("v4InternalSubnet is no large enough for the maximum number of nodes which can be supported by ClusterNetwork")
errNotExpect("v4InternalSubnet 100.64.0.0/21 is not large enough for the maximum number of nodes which can be supported by ClusterNetwork")
ovnConfig.V6InternalSubnet = "fd01::/48"
errExpect("v6InternalSubnet and ClusterNetwork must have matching IP families")
errExpect("v6InternalSubnet fd01::/48 and ClusterNetwork must have matching IP families")
ovnConfig.V4InternalSubnet = "10.128.0.0/16"
errExpect("v4InternalSubnet overlaps with ClusterNetwork 10.128.0.0/15")
errExpect("v4InternalSubnet 10.128.0.0/16 overlaps with ClusterNetwork 10.128.0.0/15")
ovnConfig.V4InternalSubnet = "172.30.0.0/18"
errExpect("v4InternalSubnet overlaps with ServiceNetwork 172.30.0.0/16")
errExpect("v4InternalSubnet 172.30.0.0/18 overlaps with ServiceNetwork 172.30.0.0/16")
ovnConfig.GatewayConfig.IPv4.InternalMasqueradeSubnet = "10.128.0.0/16"
errExpect("v4InternalMasqueradeSubnet 10.128.0.0/16 overlaps with ClusterNetwork 10.128.0.0/15")
ovnConfig.GatewayConfig.IPv6.InternalMasqueradeSubnet = "fd01::/48"
errExpect("v6InternalMasqueradeSubnet fd01::/48 and ClusterNetwork must have matching IP families")
ovnConfig.GatewayConfig.IPv4.InternalMasqueradeSubnet = "172.30.0.0/18"
errExpect("v4InternalMasqueradeSubnet 172.30.0.0/18 overlaps with ServiceNetwork 172.30.0.0/16")

// set mtu to insanity
ovnConfig.MTU = ptrToUint32(70000)
Expand All @@ -1031,15 +1088,26 @@ func TestValidateOVNKubernetes(t *testing.T) {
config.ClusterNetwork = []operv1.ClusterNetworkEntry{{
CIDR: "fd01::/48", HostPrefix: 64,
}}
errExpect("v4InternalSubnet and ClusterNetwork must have matching IP families")
errExpect("v4InternalSubnet 172.30.0.0/18 and ClusterNetwork must have matching IP families")
errExpect("v4InternalMasqueradeSubnet 172.30.0.0/18 and ClusterNetwork must have matching IP families")
ovnConfig.V6InternalSubnet = "fd01::/64"
errExpect("v6InternalSubnet overlaps with ClusterNetwork fd01::/48")
errExpect("v6InternalSubnet fd01::/64 overlaps with ClusterNetwork fd01::/48")
ovnConfig.V6InternalSubnet = "fd03::/112"
errExpect("v6InternalSubnet is no large enough for the maximum number of nodes which can be supported by ClusterNetwork")
errExpect("v6InternalSubnet fd03::/112 is not large enough for the maximum number of nodes which can be supported by ClusterNetwork")
ovnConfig.V6InternalSubnet = "fd03::/111"
errNotExpect("v6InternalSubnet is no large enough for the maximum number of nodes which can be supported by ClusterNetwork")
errNotExpect("v6InternalSubnet fd03::/111 is not large enough for the maximum number of nodes which can be supported by ClusterNetwork")
ovnConfig.V6InternalSubnet = "fd02::/64"
errExpect("v6InternalSubnet overlaps with ServiceNetwork fd02::/112")
errExpect("v6InternalSubnet fd02::/64 overlaps with ServiceNetwork fd02::/112")
ovnConfig.GatewayConfig.IPv6.InternalMasqueradeSubnet = "fd01::/64"
errExpect("v6InternalMasqueradeSubnet fd01::/64 overlaps with ClusterNetwork fd01::/48")
ovnConfig.GatewayConfig.IPv6.InternalMasqueradeSubnet = "fd02::/64"
errExpect("v6InternalMasqueradeSubnet fd02::/64 overlaps with ServiceNetwork fd02::/112")
ovnConfig.V4InternalSubnet = "100.99.0.0/16"
ovnConfig.GatewayConfig.IPv4.InternalMasqueradeSubnet = "100.99.0.0/16"
errExpect("v4InternalMasqueradeSubnet 100.99.0.0/16 overlaps with v4InternalSubnet 100.99.0.0/16")
ovnConfig.V6InternalSubnet = "fd69::/125"
ovnConfig.GatewayConfig.IPv6.InternalMasqueradeSubnet = "fd69::/125"
errExpect("v6InternalMasqueradeSubnet fd69::/125 overlaps with v6InternalSubnet fd69::/125")

// invalid ipv6 mtu
ovnConfig.MTU = ptrToUint32(576)
Expand Down