From a293e291a9feec28a6192e380670d2a6b385d769 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 11 Sep 2024 17:21:22 -0600 Subject: [PATCH 01/26] Adds support for MultiNetworkPolicy objects We want to be able to apply NetworkPolicy definitions to experiment pods, but your CNI plugin needs to support applying NetworkPolicy definitions. We use both flannel and multus, either of which support this. However, there is another effort from the k8s Network Plumbing Working Group (same people who maintain multus) to implement a system that allows NetworkPolicy definitions to pods with multiple interfaces. https://github.com/k8snetworkplumbingwg/multi-networkpolicy https://github.com/k8snetworkplumbingwg/multi-networkpolicy-iptables --- config/multi-networkpolicy.jsonnet | 32 + .../multi-networkpolicy.jsonnet | 775 ++++++++++++++++++ .../core/multi-networkpolicy.jsonnet | 126 +++ k8s/roles/multi-networkpolicy.jsonnet | 90 ++ system.jsonnet | 4 + 5 files changed, 1027 insertions(+) create mode 100644 config/multi-networkpolicy.jsonnet create mode 100644 k8s/custom-resource-definitions/multi-networkpolicy.jsonnet create mode 100644 k8s/daemonsets/core/multi-networkpolicy.jsonnet create mode 100644 k8s/roles/multi-networkpolicy.jsonnet diff --git a/config/multi-networkpolicy.jsonnet b/config/multi-networkpolicy.jsonnet new file mode 100644 index 00000000..06dde594 --- /dev/null +++ b/config/multi-networkpolicy.jsonnet @@ -0,0 +1,32 @@ +[ + { + kind: 'ConfigMap', + apiVersion: 'v1', + metadata: { + name: 'multi-networkpolicy-custom-v4-rules', + namespace: 'kube-system', + labels: { + tier: 'node', + app: 'multi-networkpolicy', + }, + }, + data: { + 'custom-v4-rules.txt': '# accept redirect\n-p icmp --icmp-type redirect -j ACCEPT\n# accept fragmentation-needed (for MTU discovery)\n-p icmp --icmp-type fragmentation-needed -j ACCEPT\n', + }, + }, + { + kind: 'ConfigMap', + apiVersion: 'v1', + metadata: { + name: 'multi-networkpolicy-custom-v6-rules', + namespace: 'kube-system', + labels: { + tier: 'node', + app: 'multi-networkpolicy', + }, + }, + data: { + 'custom-v6-rules.txt': '# accept NDP\n-p icmpv6 --icmpv6-type neighbor-solicitation -j ACCEPT\n-p icmpv6 --icmpv6-type neighbor-advertisement -j ACCEPT\n# accept RA/RS\n-p icmpv6 --icmpv6-type router-solicitation -j ACCEPT\n-p icmpv6 --icmpv6-type router-advertisement -j ACCEPT\n# accept redirect\n-p icmpv6 --icmpv6-type redirect -j ACCEPT\n# accept packet-too-big (for MTU discovery)\n-p icmpv6 --icmpv6-type packet-too-big -j ACCEPT\n', + }, + }, +] diff --git a/k8s/custom-resource-definitions/multi-networkpolicy.jsonnet b/k8s/custom-resource-definitions/multi-networkpolicy.jsonnet new file mode 100644 index 00000000..ef8c1d8a --- /dev/null +++ b/k8s/custom-resource-definitions/multi-networkpolicy.jsonnet @@ -0,0 +1,775 @@ +{ + apiVersion: 'apiextensions.k8s.io/v1', + kind: 'CustomResourceDefinition', + metadata: { + name: 'multi-networkpolicies.k8s.cni.cncf.io', + }, + spec: { + group: 'k8s.cni.cncf.io', + scope: 'Namespaced', + names: { + plural: 'multi-networkpolicies', + singular: 'multi-networkpolicy', + kind: 'MultiNetworkPolicy', + shortNames: [ + 'multi-policy', + ], + }, + versions: [ + { + name: 'v1beta1', + served: true, + storage: true, + schema: { + openAPIV3Schema: { + description: 'MultiNetworkPolicy is a CRD schema to provide NetworkPolicy mechanism for net-attach-def which is specified by the Network Plumbing Working Group. MultiNetworkPolicy is identical to Kubernetes NetworkPolicy, See: https://kubernetes.io/docs/concepts/services-networking/network-policies/ .', + properties: { + spec: { + description: 'Specification of the desired behavior for this MultiNetworkPolicy.', + properties: { + egress: { + description: 'List of egress rules to be applied to the selected pods. Outgoing traffic is allowed if there are no NetworkPolicies selecting the pod (and cluster policy otherwise allows the traffic), OR if the traffic matches at least one egress rule across all of the NetworkPolicy objects whose podSelector matches the pod. If this field is empty then this NetworkPolicy limits all outgoing traffic (and serves solely to ensure that the pods it selects are isolated by default). This field is beta-level in 1.8', + items: { + description: "NetworkPolicyEgressRule describes a particular set of traffic that is allowed out of pods matched by a NetworkPolicySpec's podSelector. The traffic must match both ports and to. This type is beta-level in 1.8", + properties: { + ports: { + description: 'List of destination ports for outgoing traffic. Each item in this list is combined using a logical OR. If this field is empty or missing, this rule matches all ports (traffic not restricted by port). If this field is present and contains at least one item, then this rule allows traffic only if the traffic matches at least one port in the list.', + items: { + description: 'NetworkPolicyPort describes a port to allow traffic on', + properties: { + port: { + anyOf: [ + { + type: 'integer', + }, + { + type: 'string', + }, + ], + description: 'The port on the given protocol. This can either be a numerical or named port on a pod. If this field is not provided, this matches all port names and numbers.', + 'x-kubernetes-int-or-string': true, + }, + protocol: { + description: 'The protocol (TCP, UDP, or SCTP) which traffic must match. If not specified, this field defaults to TCP.', + type: 'string', + }, + }, + type: 'object', + }, + type: 'array', + }, + to: { + description: 'List of destinations for outgoing traffic of pods selected for this rule. Items in this list are combined using a logical OR operation. If this field is empty or missing, this rule matches all destinations (traffic not restricted by destination). If this field is present and contains at least one item, this rule allows traffic only if the traffic matches at least one item in the to list.', + items: { + description: 'NetworkPolicyPeer describes a peer to allow traffic from. Only certain combinations of fields are allowed', + properties: { + ipBlock: { + description: 'IPBlock defines policy on a particular IPBlock. If this field is set then neither of the other fields can be.', + properties: { + cidr: { + description: "CIDR is a string representing the IP Block Valid examples are '192.168.1.1/24'", + type: 'string', + }, + except: { + description: "Except is a slice of CIDRs that should not be included within an IP Block Valid examples are '192.168.1.1/24' Except values will be rejected if they are outside the CIDR range", + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'cidr', + ], + type: 'object', + }, + namespaceSelector: { + description: 'Selects Namespaces using cluster-scoped labels. This field follows standard label selector semantics; if present but empty, it selects all namespaces. \n If PodSelector is also set, then the NetworkPolicyPeer as a whole selects the Pods matching PodSelector in the Namespaces selected by NamespaceSelector. Otherwise it selects all Pods in the Namespaces selected by NamespaceSelector.', + properties: { + matchExpressions: { + description: 'matchExpressions is a list of label selector requirements. The requirements are ANDed.', + items: { + description: 'A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.', + properties: { + key: { + description: 'key is the label key that the selector applies to.', + type: 'string', + }, + operator: { + description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + type: 'string', + }, + values: { + description: 'values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.', + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'key', + 'operator', + ], + type: 'object', + }, + type: 'array', + }, + matchLabels: { + additionalProperties: { + type: 'string', + }, + description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is 'key', the operator is 'In', and the values array contains only 'value'. The requirements are ANDed.", + type: 'object', + }, + }, + type: 'object', + }, + podSelector: { + description: "This is a label selector which selects Pods. This field follows standard label selector semantics; if present but empty, it selects all pods. \n If NamespaceSelector is also set, then the NetworkPolicyPeer as a whole selects the Pods matching PodSelector in the Namespaces selected by NamespaceSelector. Otherwise it selects the Pods matching PodSelector in the policy's own Namespace.", + properties: { + matchExpressions: { + description: 'matchExpressions is a list of label selector requirements. The requirements are ANDed.', + items: { + description: 'A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.', + properties: { + key: { + description: 'key is the label key that the selector applies to.', + type: 'string', + }, + operator: { + description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + type: 'string', + }, + values: { + description: 'values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.', + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'key', + 'operator', + ], + type: 'object', + }, + type: 'array', + }, + matchLabels: { + additionalProperties: { + type: 'string', + }, + description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is 'key', the operator is 'In', and the values array contains only 'value'. The requirements are ANDed.", + type: 'object', + }, + }, + type: 'object', + }, + }, + type: 'object', + }, + type: 'array', + }, + }, + type: 'object', + }, + type: 'array', + }, + ingress: { + description: "List of ingress rules to be applied to the selected pods. Traffic is allowed to a pod if there are no NetworkPolicies selecting the pod (and cluster policy otherwise allows the traffic), OR if the traffic source is the pod's local node, OR if the traffic matches at least one ingress rule across all of the NetworkPolicy objects whose podSelector matches the pod. If this field is empty then this NetworkPolicy does not allow any traffic (and serves solely to ensure that the pods it selects are isolated by default)", + items: { + description: "NetworkPolicyIngressRule describes a particular set of traffic that is allowed to the pods matched by a NetworkPolicySpec's podSelector. The traffic must match both ports and from.", + properties: { + from: { + description: 'List of sources which should be able to access the pods selected for this rule. Items in this list are combined using a logical OR operation. If this field is empty or missing, this rule matches all sources (traffic not restricted by source). If this field is present and contains at least one item, this rule allows traffic only if the traffic matches at least one item in the from list.', + items: { + description: 'NetworkPolicyPeer describes a peer to allow traffic from. Only certain combinations of fields are allowed', + properties: { + ipBlock: { + description: 'IPBlock defines policy on a particular IPBlock. If this field is set then neither of the other fields can be.', + properties: { + cidr: { + description: "CIDR is a string representing the IP Block Valid examples are '192.168.1.1/24'", + type: 'string', + }, + except: { + description: "Except is a slice of CIDRs that should not be included within an IP Block Valid examples are '192.168.1.1/24' Except values will be rejected if they are outside the CIDR range", + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'cidr', + ], + type: 'object', + }, + namespaceSelector: { + description: 'Selects Namespaces using cluster-scoped labels. This field follows standard label selector semantics; if present but empty, it selects all namespaces. \n If PodSelector is also set, then the NetworkPolicyPeer as a whole selects the Pods matching PodSelector in the Namespaces selected by NamespaceSelector. Otherwise it selects all Pods in the Namespaces selected by NamespaceSelector.', + properties: { + matchExpressions: { + description: 'matchExpressions is a list of label selector requirements. The requirements are ANDed.', + items: { + description: 'A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.', + properties: { + key: { + description: 'key is the label key that the selector applies to.', + type: 'string', + }, + operator: { + description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + type: 'string', + }, + values: { + description: 'values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.', + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'key', + 'operator', + ], + type: 'object', + }, + type: 'array', + }, + matchLabels: { + additionalProperties: { + type: 'string', + }, + description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is 'key', the operator is 'In', and the values array contains only 'value'. The requirements are ANDed.", + type: 'object', + }, + }, + type: 'object', + }, + podSelector: { + description: "This is a label selector which selects Pods. This field follows standard label selector semantics; if present but empty, it selects all pods. \n If NamespaceSelector is also set, then the NetworkPolicyPeer as a whole selects the Pods matching PodSelector in the Namespaces selected by NamespaceSelector. Otherwise it selects the Pods matching PodSelector in the policy's own Namespace.", + properties: { + matchExpressions: { + description: 'matchExpressions is a list of label selector requirements. The requirements are ANDed.', + items: { + description: 'A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.', + properties: { + key: { + description: 'key is the label key that the selector applies to.', + type: 'string', + }, + operator: { + description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + type: 'string', + }, + values: { + description: 'values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.', + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'key', + 'operator', + ], + type: 'object', + }, + type: 'array', + }, + matchLabels: { + additionalProperties: { + type: 'string', + }, + description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is 'key', the operator is 'In', and the values array contains only 'value'. The requirements are ANDed.", + type: 'object', + }, + }, + type: 'object', + }, + }, + type: 'object', + }, + type: 'array', + }, + ports: { + description: 'List of ports which should be made accessible on the pods selected for this rule. Each item in this list is combined using a logical OR. If this field is empty or missing, this rule matches all ports (traffic not restricted by port). If this field is present and contains at least one item, then this rule allows traffic only if the traffic matches at least one port in the list.', + items: { + description: 'NetworkPolicyPort describes a port to allow traffic on', + properties: { + port: { + anyOf: [ + { + type: 'integer', + }, + { + type: 'string', + }, + ], + description: 'The port on the given protocol. This can either be a numerical or named port on a pod. If this field is not provided, this matches all port names and numbers.', + 'x-kubernetes-int-or-string': true, + }, + protocol: { + description: 'The protocol (TCP, UDP, or SCTP) which traffic must match. If not specified, this field defaults to TCP.', + type: 'string', + }, + }, + type: 'object', + }, + type: 'array', + }, + }, + type: 'object', + }, + type: 'array', + }, + podSelector: { + description: "This is a label selector which selects Pods. This field follows standard label selector semantics; if present but empty, it selects all pods. \n If NamespaceSelector is also set, then the NetworkPolicyPeer as a whole selects the Pods matching PodSelector in the Namespaces selected by NamespaceSelector. Otherwise it selects the Pods matching PodSelector in the policy's own Namespace.", + properties: { + matchExpressions: { + description: 'matchExpressions is a list of label selector requirements. The requirements are ANDed.', + items: { + description: 'A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.', + properties: { + key: { + description: 'key is the label key that the selector applies to.', + type: 'string', + }, + operator: { + description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + type: 'string', + }, + values: { + description: 'values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.', + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'key', + 'operator', + ], + type: 'object', + }, + type: 'array', + }, + matchLabels: { + additionalProperties: { + type: 'string', + description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is 'key', the operator is 'In', and the values array contains only 'value'. The requirements are ANDed.", + }, + type: 'object', + }, + }, + type: 'object', + }, + policyTypes: { + description: "List of rule types that the NetworkPolicy relates to. Valid options are 'Ingress', 'Egress', or 'Ingress,Egress'. If this field is not specified, it will default based on the existence of Ingress or Egress rules; policies that contain an Egress section are assumed to affect Egress, and all policies (whether or not they contain an Ingress section) are assumed to affect Ingress. If you want to write an egress-only policy, you must explicitly specify policyTypes [ 'Egress' ]. Likewise, if you want to write a policy that specifies that no egress is allowed, you must specify a policyTypes value that include 'Egress' (since such a policy would not include an Egress section and would otherwise default to just [ 'Ingress' ]). This field is beta-level in 1.8", + items: { + description: 'Policy Type string describes the NetworkPolicy type This type is beta-level in 1.8', + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'podSelector', + ], + type: 'object', + }, + }, + type: 'object', + }, + }, + }, + { + name: 'v1beta2', + served: false, + storage: false, + schema: { + openAPIV3Schema: { + description: 'MultiNetworkPolicy is a CRD schema to provide NetworkPolicy mechanism for net-attach-def which is specified by the Network Plumbing Working Group. MultiNetworkPolicy is identical to Kubernetes NetworkPolicy, See: https://kubernetes.io/docs/concepts/services-networking/network-policies/ .', + properties: { + spec: { + description: 'Specification of the desired behavior for this MultiNetworkPolicy.', + properties: { + egress: { + description: 'List of egress rules to be applied to the selected pods. Outgoing traffic is allowed if there are no NetworkPolicies selecting the pod (and cluster policy otherwise allows the traffic), OR if the traffic matches at least one egress rule across all of the NetworkPolicy objects whose podSelector matches the pod. If this field is empty then this NetworkPolicy limits all outgoing traffic (and serves solely to ensure that the pods it selects are isolated by default). This field is beta-level in 1.8', + items: { + description: "NetworkPolicyEgressRule describes a particular set of traffic that is allowed out of pods matched by a NetworkPolicySpec's podSelector. The traffic must match both ports and to. This type is beta-level in 1.8", + properties: { + ports: { + description: 'List of destination ports for outgoing traffic. Each item in this list is combined using a logical OR. If this field is empty or missing, this rule matches all ports (traffic not restricted by port). If this field is present and contains at least one item, then this rule allows traffic only if the traffic matches at least one port in the list.', + items: { + description: 'NetworkPolicyPort describes a port to allow traffic on', + properties: { + port: { + anyOf: [ + { + type: 'integer', + }, + { + type: 'string', + }, + ], + description: 'The port on the given protocol. This can either be a numerical or named port on a pod. If this field is not provided, this matches all port names and numbers.', + 'x-kubernetes-int-or-string': true, + }, + endPort: { + type: 'integer', + format: 'int32', + description: 'If set, indicates that the range of ports from port to endPort, inclusive, should be allowed by the policy. This field cannot be defined if the port field is not defined or if the port field is defined as a named (string) port. The endPort must be equal or greater than port.', + }, + protocol: { + description: 'The protocol (TCP, UDP, or SCTP) which traffic must match. If not specified, this field defaults to TCP.', + type: 'string', + }, + }, + type: 'object', + }, + type: 'array', + }, + to: { + description: 'List of destinations for outgoing traffic of pods selected for this rule. Items in this list are combined using a logical OR operation. If this field is empty or missing, this rule matches all destinations (traffic not restricted by destination). If this field is present and contains at least one item, this rule allows traffic only if the traffic matches at least one item in the to list.', + items: { + description: 'NetworkPolicyPeer describes a peer to allow traffic from. Only certain combinations of fields are allowed', + properties: { + ipBlock: { + description: 'IPBlock defines policy on a particular IPBlock. If this field is set then neither of the other fields can be.', + properties: { + cidr: { + description: "CIDR is a string representing the IP Block Valid examples are '192.168.1.1/24'", + type: 'string', + }, + except: { + description: "Except is a slice of CIDRs that should not be included within an IP Block Valid examples are '192.168.1.1/24' Except values will be rejected if they are outside the CIDR range", + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'cidr', + ], + type: 'object', + }, + namespaceSelector: { + description: 'Selects Namespaces using cluster-scoped labels. This field follows standard label selector semantics; if present but empty, it selects all namespaces. \n If PodSelector is also set, then the NetworkPolicyPeer as a whole selects the Pods matching PodSelector in the Namespaces selected by NamespaceSelector. Otherwise it selects all Pods in the Namespaces selected by NamespaceSelector.', + properties: { + matchExpressions: { + description: 'matchExpressions is a list of label selector requirements. The requirements are ANDed.', + items: { + description: 'A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.', + properties: { + key: { + description: 'key is the label key that the selector applies to.', + type: 'string', + }, + operator: { + description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + type: 'string', + }, + values: { + description: 'values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.', + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'key', + 'operator', + ], + type: 'object', + }, + type: 'array', + }, + matchLabels: { + additionalProperties: { + type: 'string', + }, + description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is 'key', the operator is 'In', and the values array contains only 'value'. The requirements are ANDed.", + type: 'object', + }, + }, + type: 'object', + }, + podSelector: { + description: "This is a label selector which selects Pods. This field follows standard label selector semantics; if present but empty, it selects all pods. \n If NamespaceSelector is also set, then the NetworkPolicyPeer as a whole selects the Pods matching PodSelector in the Namespaces selected by NamespaceSelector. Otherwise it selects the Pods matching PodSelector in the policy's own Namespace.", + properties: { + matchExpressions: { + description: 'matchExpressions is a list of label selector requirements. The requirements are ANDed.', + items: { + description: 'A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.', + properties: { + key: { + description: 'key is the label key that the selector applies to.', + type: 'string', + }, + operator: { + description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + type: 'string', + }, + values: { + description: 'values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.', + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'key', + 'operator', + ], + type: 'object', + }, + type: 'array', + }, + matchLabels: { + additionalProperties: { + type: 'string', + }, + description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is 'key', the operator is 'In', and the values array contains only 'value'. The requirements are ANDed.", + type: 'object', + }, + }, + type: 'object', + }, + }, + type: 'object', + }, + type: 'array', + }, + }, + type: 'object', + }, + type: 'array', + }, + ingress: { + description: "List of ingress rules to be applied to the selected pods. Traffic is allowed to a pod if there are no NetworkPolicies selecting the pod (and cluster policy otherwise allows the traffic), OR if the traffic source is the pod's local node, OR if the traffic matches at least one ingress rule across all of the NetworkPolicy objects whose podSelector matches the pod. If this field is empty then this NetworkPolicy does not allow any traffic (and serves solely to ensure that the pods it selects are isolated by default)", + items: { + description: "NetworkPolicyIngressRule describes a particular set of traffic that is allowed to the pods matched by a NetworkPolicySpec's podSelector. The traffic must match both ports and from.", + properties: { + from: { + description: 'List of sources which should be able to access the pods selected for this rule. Items in this list are combined using a logical OR operation. If this field is empty or missing, this rule matches all sources (traffic not restricted by source). If this field is present and contains at least one item, this rule allows traffic only if the traffic matches at least one item in the from list.', + items: { + description: 'NetworkPolicyPeer describes a peer to allow traffic from. Only certain combinations of fields are allowed', + properties: { + ipBlock: { + description: 'IPBlock defines policy on a particular IPBlock. If this field is set then neither of the other fields can be.', + properties: { + cidr: { + description: "CIDR is a string representing the IP Block Valid examples are '192.168.1.1/24'", + type: 'string', + }, + except: { + description: "Except is a slice of CIDRs that should not be included within an IP Block Valid examples are '192.168.1.1/24' Except values will be rejected if they are outside the CIDR range", + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'cidr', + ], + type: 'object', + }, + namespaceSelector: { + description: 'Selects Namespaces using cluster-scoped labels. This field follows standard label selector semantics; if present but empty, it selects all namespaces. \n If PodSelector is also set, then the NetworkPolicyPeer as a whole selects the Pods matching PodSelector in the Namespaces selected by NamespaceSelector. Otherwise it selects all Pods in the Namespaces selected by NamespaceSelector.', + properties: { + matchExpressions: { + description: 'matchExpressions is a list of label selector requirements. The requirements are ANDed.', + items: { + description: 'A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.', + properties: { + key: { + description: 'key is the label key that the selector applies to.', + type: 'string', + }, + operator: { + description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + type: 'string', + }, + values: { + description: 'values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.', + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'key', + 'operator', + ], + type: 'object', + }, + type: 'array', + }, + matchLabels: { + additionalProperties: { + type: 'string', + }, + description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is 'key', the operator is 'In', and the values array contains only 'value'. The requirements are ANDed.", + type: 'object', + }, + }, + type: 'object', + }, + podSelector: { + description: "This is a label selector which selects Pods. This field follows standard label selector semantics; if present but empty, it selects all pods. \n If NamespaceSelector is also set, then the NetworkPolicyPeer as a whole selects the Pods matching PodSelector in the Namespaces selected by NamespaceSelector. Otherwise it selects the Pods matching PodSelector in the policy's own Namespace.", + properties: { + matchExpressions: { + description: 'matchExpressions is a list of label selector requirements. The requirements are ANDed.', + items: { + description: 'A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.', + properties: { + key: { + description: 'key is the label key that the selector applies to.', + type: 'string', + }, + operator: { + description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + type: 'string', + }, + values: { + description: 'values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.', + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'key', + 'operator', + ], + type: 'object', + }, + type: 'array', + }, + matchLabels: { + additionalProperties: { + type: 'string', + }, + description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is 'key', the operator is 'In', and the values array contains only 'value'. The requirements are ANDed.", + type: 'object', + }, + }, + type: 'object', + }, + }, + type: 'object', + }, + type: 'array', + }, + ports: { + description: 'List of ports which should be made accessible on the pods selected for this rule. Each item in this list is combined using a logical OR. If this field is empty or missing, this rule matches all ports (traffic not restricted by port). If this field is present and contains at least one item, then this rule allows traffic only if the traffic matches at least one port in the list.', + items: { + description: 'NetworkPolicyPort describes a port to allow traffic on', + properties: { + port: { + anyOf: [ + { + type: 'integer', + }, + { + type: 'string', + }, + ], + description: 'The port on the given protocol. This can either be a numerical or named port on a pod. If this field is not provided, this matches all port names and numbers.', + 'x-kubernetes-int-or-string': true, + }, + endPort: { + type: 'integer', + format: 'int32', + description: 'If set, indicates that the range of ports from port to endPort, inclusive, should be allowed by the policy. This field cannot be defined if the port field is not defined or if the port field is defined as a named (string) port. The endPort must be equal or greater than port.', + }, + protocol: { + description: 'The protocol (TCP, UDP, or SCTP) which traffic must match. If not specified, this field defaults to TCP.', + type: 'string', + }, + }, + type: 'object', + }, + type: 'array', + }, + }, + type: 'object', + }, + type: 'array', + }, + podSelector: { + description: "This is a label selector which selects Pods. This field follows standard label selector semantics; if present but empty, it selects all pods. \n If NamespaceSelector is also set, then the NetworkPolicyPeer as a whole selects the Pods matching PodSelector in the Namespaces selected by NamespaceSelector. Otherwise it selects the Pods matching PodSelector in the policy's own Namespace.", + properties: { + matchExpressions: { + description: 'matchExpressions is a list of label selector requirements. The requirements are ANDed.', + items: { + description: 'A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.', + properties: { + key: { + description: 'key is the label key that the selector applies to.', + type: 'string', + }, + operator: { + description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + type: 'string', + }, + values: { + description: 'values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.', + items: { + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'key', + 'operator', + ], + type: 'object', + }, + type: 'array', + }, + matchLabels: { + additionalProperties: { + type: 'string', + description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is 'key', the operator is 'In', and the values array contains only 'value'. The requirements are ANDed.", + }, + type: 'object', + }, + }, + type: 'object', + }, + policyTypes: { + description: "List of rule types that the NetworkPolicy relates to. Valid options are 'Ingress', 'Egress', or 'Ingress,Egress'. If this field is not specified, it will default based on the existence of Ingress or Egress rules; policies that contain an Egress section are assumed to affect Egress, and all policies (whether or not they contain an Ingress section) are assumed to affect Ingress. If you want to write an egress-only policy, you must explicitly specify policyTypes [ 'Egress' ]. Likewise, if you want to write a policy that specifies that no egress is allowed, you must specify a policyTypes value that include 'Egress' (since such a policy would not include an Egress section and would otherwise default to just [ 'Ingress' ]). This field is beta-level in 1.8", + items: { + description: 'Policy Type string describes the NetworkPolicy type This type is beta-level in 1.8', + type: 'string', + }, + type: 'array', + }, + }, + required: [ + 'podSelector', + ], + type: 'object', + }, + }, + type: 'object', + }, + }, + }, + ], + }, +} diff --git a/k8s/daemonsets/core/multi-networkpolicy.jsonnet b/k8s/daemonsets/core/multi-networkpolicy.jsonnet new file mode 100644 index 00000000..794ac6da --- /dev/null +++ b/k8s/daemonsets/core/multi-networkpolicy.jsonnet @@ -0,0 +1,126 @@ +{ + apiVersion: 'apps/v1', + kind: 'DaemonSet', + metadata: { + name: 'multi-networkpolicy-ds-amd64', + namespace: 'kube-system', + labels: { + tier: 'node', + app: 'multi-networkpolicy', + name: 'multi-networkpolicy', + }, + }, + spec: { + selector: { + matchLabels: { + name: 'multi-networkpolicy', + }, + }, + updateStrategy: { + type: 'RollingUpdate', + }, + template: { + metadata: { + labels: { + tier: 'node', + app: 'multi-networkpolicy', + name: 'multi-networkpolicy', + }, + }, + spec: { + hostNetwork: true, + nodeSelector: { + 'kubernetes.io/arch': 'amd64', + }, + tolerations: [ + { + operator: 'Exists', + effect: 'NoSchedule', + }, + ], + serviceAccountName: 'multi-networkpolicy', + containers: [ + { + name: 'multi-networkpolicy', + image: 'ghcr.io/k8snetworkplumbingwg/multi-networkpolicy-iptables:snapshot', + imagePullPolicy: 'Always', + command: [ + '/usr/bin/multi-networkpolicy-iptables', + ], + args: [ + '--host-prefix=/host', + '--container-runtime-endpoint=/run/crio/crio.sock', + '--pod-iptables=/var/lib/multi-networkpolicy/iptables', + ], + resources: { + requests: { + cpu: '100m', + memory: '80Mi', + }, + limits: { + cpu: '100m', + memory: '150Mi', + }, + }, + securityContext: { + privileged: true, + capabilities: { + add: [ + 'SYS_ADMIN', + 'NET_ADMIN', + ], + }, + }, + volumeMounts: [ + { + name: 'host', + mountPath: '/host', + }, + { + name: 'var-lib-multinetworkpolicy', + mountPath: '/var/lib/multi-networkpolicy', + }, + { + name: 'multi-networkpolicy-custom-rules', + mountPath: '/etc/multi-networkpolicy/rules', + readOnly: true, + }, + ], + }, + ], + volumes: [ + { + name: 'host', + hostPath: { + path: '/', + }, + }, + { + name: 'var-lib-multinetworkpolicy', + hostPath: { + path: '/var/lib/multi-networkpolicy', + }, + }, + { + name: 'multi-networkpolicy-custom-rules', + projected: { + sources: [ + { + configMap: { + name: 'multi-networkpolicy-custom-v4-rules', + }, + }, + { + configMap: { + name: 'multi-networkpolicy-custom-v6-rules', + }, + }, + ], + }, + }, + ], + }, + }, + }, +} + diff --git a/k8s/roles/multi-networkpolicy.jsonnet b/k8s/roles/multi-networkpolicy.jsonnet new file mode 100644 index 00000000..6c2ebfae --- /dev/null +++ b/k8s/roles/multi-networkpolicy.jsonnet @@ -0,0 +1,90 @@ +[ + { + kind: 'ClusterRole', + apiVersion: 'rbac.authorization.k8s.io/v1', + metadata: { + name: 'multi-networkpolicy', + }, + rules: [ + { + apiGroups: [ + 'k8s.cni.cncf.io', + ], + resources: [ + '*', + ], + verbs: [ + '*', + ], + }, + { + apiGroups: [ + '', + ], + resources: [ + 'pods', + 'namespaces', + ], + verbs: [ + 'list', + 'watch', + 'get', + ], + }, + { + apiGroups: [ + 'networking.k8s.io', + ], + resources: [ + 'networkpolicies', + ], + verbs: [ + 'watch', + 'list', + ], + }, + { + apiGroups: [ + '', + 'events.k8s.io', + ], + resources: [ + 'events', + ], + verbs: [ + 'create', + 'patch', + 'update', + ], + }, + ], + }, + { + kind: 'ClusterRoleBinding', + apiVersion: 'rbac.authorization.k8s.io/v1', + metadata: { + name: 'multi-networkpolicy', + }, + roleRef: { + apiGroup: 'rbac.authorization.k8s.io', + kind: 'ClusterRole', + name: 'multi-networkpolicy', + }, + subjects: [ + { + kind: 'ServiceAccount', + name: 'multi-networkpolicy', + namespace: 'kube-system', + }, + ], + }, + { + kind: 'ServiceAccount', + apiVersion: 'v1', + metadata: { + name: 'multi-networkpolicy', + namespace: 'kube-system', + }, + }, +] + diff --git a/system.jsonnet b/system.jsonnet index c61ac89f..b781e72f 100644 --- a/system.jsonnet +++ b/system.jsonnet @@ -5,15 +5,18 @@ // Configmaps import 'config/disco.jsonnet', import 'config/flannel.jsonnet', + import 'config/multi-networkpolicy.jsonnet', import 'config/nodeinfo.jsonnet', import 'config/prometheus.jsonnet', // Custom resource definitions + import 'k8s/custom-resource-definitions/multi-networkpolicy.jsonnet', import 'k8s/custom-resource-definitions/network-attachment-definition.jsonnet', // Daemonsets import 'k8s/daemonsets/core/cadvisor.jsonnet', import 'k8s/daemonsets/core/disco.jsonnet', import 'k8s/daemonsets/core/flannel.jsonnet', import 'k8s/daemonsets/core/host.jsonnet', + import 'k8s/daemonsets/core/multi-networkpolicy.jsonnet', import 'k8s/daemonsets/core/node-exporter.jsonnet', import 'k8s/daemonsets/experiments/ndt.jsonnet', import 'k8s/daemonsets/experiments/ndt-virtual.jsonnet', @@ -61,6 +64,7 @@ import 'k8s/roles/heartbeat-experiment.jsonnet', import 'k8s/roles/kube-rbac-proxy.jsonnet', import 'k8s/roles/kube-state-metrics.jsonnet', + import 'k8s/roles/multi-networkpolicy.jsonnet', import 'k8s/roles/rbac-prometheus.jsonnet', import 'k8s/roles/reloader.jsonnet', ]), From 183d8ff974590992d8b4bf5562b7b247ed116f2a Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 11 Sep 2024 17:55:44 -0600 Subject: [PATCH 02/26] Condenses multi-networkpolicy configmaps into a single one --- config/multi-networkpolicy.jsonnet | 43 ++++++------------- .../core/multi-networkpolicy.jsonnet | 15 +------ 2 files changed, 16 insertions(+), 42 deletions(-) diff --git a/config/multi-networkpolicy.jsonnet b/config/multi-networkpolicy.jsonnet index 06dde594..0648b28c 100644 --- a/config/multi-networkpolicy.jsonnet +++ b/config/multi-networkpolicy.jsonnet @@ -1,32 +1,17 @@ -[ - { - kind: 'ConfigMap', - apiVersion: 'v1', - metadata: { - name: 'multi-networkpolicy-custom-v4-rules', - namespace: 'kube-system', - labels: { - tier: 'node', - app: 'multi-networkpolicy', - }, - }, - data: { - 'custom-v4-rules.txt': '# accept redirect\n-p icmp --icmp-type redirect -j ACCEPT\n# accept fragmentation-needed (for MTU discovery)\n-p icmp --icmp-type fragmentation-needed -j ACCEPT\n', +{ + kind: 'ConfigMap', + apiVersion: 'v1', + metadata: { + name: 'multi-networkpolicy-custom-rules', + namespace: 'kube-system', + labels: { + tier: 'node', + app: 'multi-networkpolicy', }, }, - { - kind: 'ConfigMap', - apiVersion: 'v1', - metadata: { - name: 'multi-networkpolicy-custom-v6-rules', - namespace: 'kube-system', - labels: { - tier: 'node', - app: 'multi-networkpolicy', - }, - }, - data: { - 'custom-v6-rules.txt': '# accept NDP\n-p icmpv6 --icmpv6-type neighbor-solicitation -j ACCEPT\n-p icmpv6 --icmpv6-type neighbor-advertisement -j ACCEPT\n# accept RA/RS\n-p icmpv6 --icmpv6-type router-solicitation -j ACCEPT\n-p icmpv6 --icmpv6-type router-advertisement -j ACCEPT\n# accept redirect\n-p icmpv6 --icmpv6-type redirect -j ACCEPT\n# accept packet-too-big (for MTU discovery)\n-p icmpv6 --icmpv6-type packet-too-big -j ACCEPT\n', - }, + data: { + 'custom-v4-rules.txt': '# accept redirect\n-p icmp --icmp-type redirect -j ACCEPT\n# accept fragmentation-needed (for MTU discovery)\n-p icmp --icmp-type fragmentation-needed -j ACCEPT\n', }, -] + 'custom-v6-rules.txt': '# accept NDP\n-p icmpv6 --icmpv6-type neighbor-solicitation -j ACCEPT\n-p icmpv6 --icmpv6-type neighbor-advertisement -j ACCEPT\n# accept RA/RS\n-p icmpv6 --icmpv6-type router-solicitation -j ACCEPT\n-p icmpv6 --icmpv6-type router-advertisement -j ACCEPT\n# accept redirect\n-p icmpv6 --icmpv6-type redirect -j ACCEPT\n# accept packet-too-big (for MTU discovery)\n-p icmpv6 --icmpv6-type packet-too-big -j ACCEPT\n', +} + diff --git a/k8s/daemonsets/core/multi-networkpolicy.jsonnet b/k8s/daemonsets/core/multi-networkpolicy.jsonnet index 794ac6da..a387d37f 100644 --- a/k8s/daemonsets/core/multi-networkpolicy.jsonnet +++ b/k8s/daemonsets/core/multi-networkpolicy.jsonnet @@ -103,19 +103,8 @@ }, { name: 'multi-networkpolicy-custom-rules', - projected: { - sources: [ - { - configMap: { - name: 'multi-networkpolicy-custom-v4-rules', - }, - }, - { - configMap: { - name: 'multi-networkpolicy-custom-v6-rules', - }, - }, - ], + configMap: { + name: 'multi-networkpolicy-custom-rules', }, }, ], From 3ade9cb053843e9712c08279d6e47af8097e6838 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 11 Sep 2024 19:18:55 -0600 Subject: [PATCH 03/26] Fixes location of closing bracket in multi-networkpolicy configmap --- config/multi-networkpolicy.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/multi-networkpolicy.jsonnet b/config/multi-networkpolicy.jsonnet index 0648b28c..3980b4b5 100644 --- a/config/multi-networkpolicy.jsonnet +++ b/config/multi-networkpolicy.jsonnet @@ -11,7 +11,7 @@ }, data: { 'custom-v4-rules.txt': '# accept redirect\n-p icmp --icmp-type redirect -j ACCEPT\n# accept fragmentation-needed (for MTU discovery)\n-p icmp --icmp-type fragmentation-needed -j ACCEPT\n', - }, 'custom-v6-rules.txt': '# accept NDP\n-p icmpv6 --icmpv6-type neighbor-solicitation -j ACCEPT\n-p icmpv6 --icmpv6-type neighbor-advertisement -j ACCEPT\n# accept RA/RS\n-p icmpv6 --icmpv6-type router-solicitation -j ACCEPT\n-p icmpv6 --icmpv6-type router-advertisement -j ACCEPT\n# accept redirect\n-p icmpv6 --icmpv6-type redirect -j ACCEPT\n# accept packet-too-big (for MTU discovery)\n-p icmpv6 --icmpv6-type packet-too-big -j ACCEPT\n', + }, } From bf865c53cb037dd2f87a4919685de779a908a123 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Thu, 12 Sep 2024 14:30:35 -0600 Subject: [PATCH 04/26] Fixes multi-networkplicy DaemonSet and add test policy for NDT --- .../core/multi-networkpolicy.jsonnet | 5 +-- k8s/networkpolicies/ndt.jsonnet | 31 +++++++++++++++++++ system.jsonnet | 2 ++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 k8s/networkpolicies/ndt.jsonnet diff --git a/k8s/daemonsets/core/multi-networkpolicy.jsonnet b/k8s/daemonsets/core/multi-networkpolicy.jsonnet index a387d37f..4121d1a9 100644 --- a/k8s/daemonsets/core/multi-networkpolicy.jsonnet +++ b/k8s/daemonsets/core/multi-networkpolicy.jsonnet @@ -2,7 +2,7 @@ apiVersion: 'apps/v1', kind: 'DaemonSet', metadata: { - name: 'multi-networkpolicy-ds-amd64', + name: 'multi-networkpolicy', namespace: 'kube-system', labels: { tier: 'node', @@ -49,7 +49,8 @@ ], args: [ '--host-prefix=/host', - '--container-runtime-endpoint=/run/crio/crio.sock', + '--container-runtime-endpoint=/run/containerd/containerd.sock', + '--network-plugins=ipvlan', '--pod-iptables=/var/lib/multi-networkpolicy/iptables', ], resources: { diff --git a/k8s/networkpolicies/ndt.jsonnet b/k8s/networkpolicies/ndt.jsonnet new file mode 100644 index 00000000..80d84b7e --- /dev/null +++ b/k8s/networkpolicies/ndt.jsonnet @@ -0,0 +1,31 @@ +{ + apiVersion: 'k8s.cni.cncf.io/v1beta1', + kind: 'MultiNetworkPolicy', + metadata: { + name: 'ndt-network-policy', + namespace: 'default', + annotations: { + 'k8s.v1.cni.cncf.io/policy-for': 'index2ip-index-2-conf', + }, + }, + spec: { + podSelector: {}, + policyTypes: [ + 'Ingress', + ], + ingress: [ + { + ports: [ + { + protocol: 'TCP', + port: 80, + }, + { + protocol: 'TCP', + port: 443, + }, + ], + }, + ], + }, +} diff --git a/system.jsonnet b/system.jsonnet index b781e72f..46ae15bf 100644 --- a/system.jsonnet +++ b/system.jsonnet @@ -24,6 +24,8 @@ import 'k8s/daemonsets/experiments/neubot.jsonnet', import 'k8s/daemonsets/experiments/revtr.jsonnet', import 'k8s/daemonsets/experiments/packet-test.jsonnet', + // NetworkPolicies + import 'k8s/networkpolicies/ndt.jsonnet', ] + ( if std.extVar('PROJECT_ID') == 'mlab-sandbox' then [ // responsiveness commented out by Kinkade. It's stuck in sandbox and not From 4dd2750e000c21d05b23037749b3c6792cb50d41 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Tue, 17 Sep 2024 15:36:10 -0600 Subject: [PATCH 05/26] Changes experiment net-attach-defs to CNI v0.3.1 It was previously v0.2.0, which is what index2ip was designed around, but I discovered that using that version was impacting multus' functionality with regard to annotating pods correctly, and possibly in other ways we weren't even noticing. --- k8s/networks/networks.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/networks/networks.jsonnet b/k8s/networks/networks.jsonnet index 8a119696..cfdadaa8 100644 --- a/k8s/networks/networks.jsonnet +++ b/k8s/networks/networks.jsonnet @@ -55,7 +55,7 @@ }, spec: { local cniConfig = { - cniVersion: '0.2.0', + cniVersion: '0.3.1', name: 'ipvlan-index-' + index, plugins: [ { From d662e731b264f1c73c39d5f004ebed4298046d18 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Thu, 19 Sep 2024 17:18:54 -0600 Subject: [PATCH 06/26] Use v1beta2 of MultiNetworkPolicy This version introduces the "endPort" field, allowing you to specify a port range, which we need to do for ndt5. Also, allow additional ports for ndt: 3001, 3010, and ephemeral port range. The range reflects the value of net.ipv4.ip_local_port_range on our Ubuntu systems. --- .../multi-networkpolicy.jsonnet | 6 ++--- .../core/multi-networkpolicy.jsonnet | 6 +++-- k8s/networkpolicies/ndt.jsonnet | 22 +++++++++++++++---- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/k8s/custom-resource-definitions/multi-networkpolicy.jsonnet b/k8s/custom-resource-definitions/multi-networkpolicy.jsonnet index ef8c1d8a..fa575a87 100644 --- a/k8s/custom-resource-definitions/multi-networkpolicy.jsonnet +++ b/k8s/custom-resource-definitions/multi-networkpolicy.jsonnet @@ -19,7 +19,7 @@ { name: 'v1beta1', served: true, - storage: true, + storage: false, schema: { openAPIV3Schema: { description: 'MultiNetworkPolicy is a CRD schema to provide NetworkPolicy mechanism for net-attach-def which is specified by the Network Plumbing Working Group. MultiNetworkPolicy is identical to Kubernetes NetworkPolicy, See: https://kubernetes.io/docs/concepts/services-networking/network-policies/ .', @@ -390,8 +390,8 @@ }, { name: 'v1beta2', - served: false, - storage: false, + served: true, + storage: true, schema: { openAPIV3Schema: { description: 'MultiNetworkPolicy is a CRD schema to provide NetworkPolicy mechanism for net-attach-def which is specified by the Network Plumbing Working Group. MultiNetworkPolicy is identical to Kubernetes NetworkPolicy, See: https://kubernetes.io/docs/concepts/services-networking/network-policies/ .', diff --git a/k8s/daemonsets/core/multi-networkpolicy.jsonnet b/k8s/daemonsets/core/multi-networkpolicy.jsonnet index 4121d1a9..0aca3a45 100644 --- a/k8s/daemonsets/core/multi-networkpolicy.jsonnet +++ b/k8s/daemonsets/core/multi-networkpolicy.jsonnet @@ -48,9 +48,11 @@ '/usr/bin/multi-networkpolicy-iptables', ], args: [ - '--host-prefix=/host', + '--allow-icmp', + '--allow-icmpv6', '--container-runtime-endpoint=/run/containerd/containerd.sock', - '--network-plugins=ipvlan', + '--host-prefix=/host', + '--network-plugins=netctl,ipvlan', '--pod-iptables=/var/lib/multi-networkpolicy/iptables', ], resources: { diff --git a/k8s/networkpolicies/ndt.jsonnet b/k8s/networkpolicies/ndt.jsonnet index 80d84b7e..823e1c82 100644 --- a/k8s/networkpolicies/ndt.jsonnet +++ b/k8s/networkpolicies/ndt.jsonnet @@ -1,14 +1,15 @@ { - apiVersion: 'k8s.cni.cncf.io/v1beta1', + apiVersion: 'k8s.cni.cncf.io/v1beta2', kind: 'MultiNetworkPolicy', metadata: { - name: 'ndt-network-policy', + name: 'ndt', namespace: 'default', annotations: { 'k8s.v1.cni.cncf.io/policy-for': 'index2ip-index-2-conf', }, }, spec: { + namespaceSelector: {}, podSelector: {}, policyTypes: [ 'Ingress', @@ -17,12 +18,25 @@ { ports: [ { - protocol: 'TCP', port: 80, + protocol: 'TCP', }, { - protocol: 'TCP', port: 443, + protocol: 'TCP', + }, + { + port: 3001, + protocol: 'TCP', + }, + { + port: 3010, + protocol: 'TCP', + }, + { + port: 32768, + endPort: 60999, + protocol: 'TCP', }, ], }, From f898d0e2fff464e7640fcdef6fea6b7e90e53792 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Thu, 19 Sep 2024 17:30:20 -0600 Subject: [PATCH 07/26] Removes namespaceSelector from MultiNetworkPolicy It was not valid in that location. --- k8s/networkpolicies/ndt.jsonnet | 1 - 1 file changed, 1 deletion(-) diff --git a/k8s/networkpolicies/ndt.jsonnet b/k8s/networkpolicies/ndt.jsonnet index 823e1c82..7870e96f 100644 --- a/k8s/networkpolicies/ndt.jsonnet +++ b/k8s/networkpolicies/ndt.jsonnet @@ -9,7 +9,6 @@ }, }, spec: { - namespaceSelector: {}, podSelector: {}, policyTypes: [ 'Ingress', From 6e6afa3627598d853d1dc4e891e3c22fed7252f5 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Tue, 24 Sep 2024 11:44:48 -0600 Subject: [PATCH 08/26] Uses image from m-lab fork of multi-networkpolicy-iptables Also updates ICMP flags to use "accept" instead of "allow" --- k8s/daemonsets/core/multi-networkpolicy.jsonnet | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/daemonsets/core/multi-networkpolicy.jsonnet b/k8s/daemonsets/core/multi-networkpolicy.jsonnet index 0aca3a45..21caa5de 100644 --- a/k8s/daemonsets/core/multi-networkpolicy.jsonnet +++ b/k8s/daemonsets/core/multi-networkpolicy.jsonnet @@ -42,14 +42,14 @@ containers: [ { name: 'multi-networkpolicy', - image: 'ghcr.io/k8snetworkplumbingwg/multi-networkpolicy-iptables:snapshot', + image: 'measurementlab/multi-networkpolicy-iptables:latest', imagePullPolicy: 'Always', command: [ '/usr/bin/multi-networkpolicy-iptables', ], args: [ - '--allow-icmp', - '--allow-icmpv6', + '--accept-icmp', + '--accept-icmpv6', '--container-runtime-endpoint=/run/containerd/containerd.sock', '--host-prefix=/host', '--network-plugins=netctl,ipvlan', From fa43ae0af07b7a6afee30854c6ce70f55c42c324 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Tue, 24 Sep 2024 17:14:53 -0600 Subject: [PATCH 09/26] Adds MultiNetworkPolicy defs for msak, revt, wehe and neubot --- k8s/networkpolicies/msak.jsonnet | 36 +++++++ k8s/networkpolicies/neubot.jsonnet | 32 ++++++ k8s/networkpolicies/revtr.jsonnet | 36 +++++++ k8s/networkpolicies/wehe.jsonnet | 152 +++++++++++++++++++++++++++++ 4 files changed, 256 insertions(+) create mode 100644 k8s/networkpolicies/msak.jsonnet create mode 100644 k8s/networkpolicies/neubot.jsonnet create mode 100644 k8s/networkpolicies/revtr.jsonnet create mode 100644 k8s/networkpolicies/wehe.jsonnet diff --git a/k8s/networkpolicies/msak.jsonnet b/k8s/networkpolicies/msak.jsonnet new file mode 100644 index 00000000..edc75c6a --- /dev/null +++ b/k8s/networkpolicies/msak.jsonnet @@ -0,0 +1,36 @@ +{ + apiVersion: 'k8s.cni.cncf.io/v1beta2', + kind: 'MultiNetworkPolicy', + metadata: { + name: 'msak', + namespace: 'default', + annotations: { + 'k8s.v1.cni.cncf.io/policy-for': 'index2ip-index-1-conf', + }, + }, + spec: { + podSelector: {}, + policyTypes: [ + 'Ingress', + ], + ingress: [ + { + ports: [ + { + port: 80, + protocol: 'TCP', + }, + { + port: 443, + protocol: 'TCP', + }, + { + port: 1053, + protocol: 'UDP', + }, + ], + }, + ], + }, +} + diff --git a/k8s/networkpolicies/neubot.jsonnet b/k8s/networkpolicies/neubot.jsonnet new file mode 100644 index 00000000..3cf4ed8e --- /dev/null +++ b/k8s/networkpolicies/neubot.jsonnet @@ -0,0 +1,32 @@ +{ + apiVersion: 'k8s.cni.cncf.io/v1beta2', + kind: 'MultiNetworkPolicy', + metadata: { + name: 'neubot', + namespace: 'default', + annotations: { + 'k8s.v1.cni.cncf.io/policy-for': 'index2ip-index-10-conf', + }, + }, + spec: { + podSelector: {}, + policyTypes: [ + 'Ingress', + ], + ingress: [ + { + ports: [ + { + port: 80, + protocol: 'TCP', + }, + { + port: 443, + protocol: 'TCP', + }, + ], + }, + ], + }, +} + diff --git a/k8s/networkpolicies/revtr.jsonnet b/k8s/networkpolicies/revtr.jsonnet new file mode 100644 index 00000000..cf95d87f --- /dev/null +++ b/k8s/networkpolicies/revtr.jsonnet @@ -0,0 +1,36 @@ +{ + apiVersion: 'k8s.cni.cncf.io/v1beta2', + kind: 'MultiNetworkPolicy', + metadata: { + name: 'revtr', + namespace: 'default', + annotations: { + 'k8s.v1.cni.cncf.io/policy-for': 'index2ip-index-3-conf', + }, + }, + spec: { + podSelector: {}, + policyTypes: [ + 'Ingress', + ], + ingress: [ + { + ports: [ + { + port: 55000, + protocol: 'TCP', + }, + { + port: 55557, + protocol: 'TCP', + }, + { + port: 65000, + protocol: 'TCP', + }, + ], + }, + ], + }, +} + diff --git a/k8s/networkpolicies/wehe.jsonnet b/k8s/networkpolicies/wehe.jsonnet new file mode 100644 index 00000000..dd04f6ba --- /dev/null +++ b/k8s/networkpolicies/wehe.jsonnet @@ -0,0 +1,152 @@ +{ + apiVersion: 'k8s.cni.cncf.io/v1beta2', + kind: 'MultiNetworkPolicy', + metadata: { + name: 'wehe', + namespace: 'default', + annotations: { + 'k8s.v1.cni.cncf.io/policy-for': 'index2ip-index-5-conf', + }, + }, + spec: { + podSelector: {}, + policyTypes: [ + 'Ingress', + ], + ingress: [ + { + ports: [ + { + port: 80, + protocol: 'TCP', + }, + { + port: 81, + protocol: 'TCP', + }, + { + port: 443, + protocol: 'TCP', + }, + { + port: 443, + protocol: 'UDP', + }, + { + port: 465, + protocol: 'TCP', + }, + { + port: 853, + protocol: 'TCP', + }, + { + port: 993, + protocol: 'TCP', + }, + { + port: 995, + protocol: 'TCP', + }, + { + port: 1194, + protocol: 'TCP', + }, + { + port: 1701, + protocol: 'TCP', + }, + { + port: 3478, + protocol: 'UDP', + }, + { + port: 3480, + protocol: 'UDP', + }, + { + port: 4443, + protocol: 'TCP', + }, + { + port: 5004, + protocol: 'UDP', + }, + { + port: 5061, + protocol: 'TCP', + }, + { + port: 6881, + protocol: 'TCP', + }, + { + port: 8080, + protocol: 'TCP', + }, + { + port: 8443, + protocol: 'TCP', + }, + { + port: 8801, + protocol: 'UDP', + }, + { + port: 9000, + protocol: 'UDP', + }, + { + port: 9989, + protocol: 'TCP', + }, + { + port: 19305, + protocol: 'UDP', + }, + { + port: 35253, + protocol: 'TCP', + }, + { + port: 49882, + protocol: 'UDP', + }, + { + port: 50002, + protocol: 'UDP', + }, + { + port: 55555, + protocol: 'TCP', + }, + { + port: 55556, + protocol: 'TCP', + }, + { + port: 55557, + protocol: 'TCP', + }, + { + port: 56565, + protocol: 'TCP', + }, + { + port: 56566, + protocol: 'TCP', + }, + { + port: 62065, + protocol: 'UDP', + }, + { + port: 63308, + protocol: 'UDP', + }, + ], + }, + ], + }, +} + From f259e1169ff579fc7e3ff0900ae6f752649beda9 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 25 Sep 2024 11:34:16 -0600 Subject: [PATCH 10/26] Modifies the allowed ports for revtr in its MultiNetworkPolicy --- k8s/networkpolicies/revtr.jsonnet | 8 -------- 1 file changed, 8 deletions(-) diff --git a/k8s/networkpolicies/revtr.jsonnet b/k8s/networkpolicies/revtr.jsonnet index cf95d87f..bc556c80 100644 --- a/k8s/networkpolicies/revtr.jsonnet +++ b/k8s/networkpolicies/revtr.jsonnet @@ -16,18 +16,10 @@ ingress: [ { ports: [ - { - port: 55000, - protocol: 'TCP', - }, { port: 55557, protocol: 'TCP', }, - { - port: 65000, - protocol: 'TCP', - }, ], }, ], From f3774ab5f797ccf14babff8525460800b57416f1 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 25 Sep 2024 11:48:32 -0600 Subject: [PATCH 11/26] Loads MultiNetworkPolicies for mask, revtr, wehe and neubot --- system.jsonnet | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/system.jsonnet b/system.jsonnet index 46ae15bf..479ba1a9 100644 --- a/system.jsonnet +++ b/system.jsonnet @@ -25,7 +25,11 @@ import 'k8s/daemonsets/experiments/revtr.jsonnet', import 'k8s/daemonsets/experiments/packet-test.jsonnet', // NetworkPolicies + import 'k8s/networkpolicies/msak.jsonnet', import 'k8s/networkpolicies/ndt.jsonnet', + import 'k8s/networkpolicies/revtr.jsonnet', + import 'k8s/networkpolicies/wehe.jsonnet', + import 'k8s/networkpolicies/neubot.jsonnet', ] + ( if std.extVar('PROJECT_ID') == 'mlab-sandbox' then [ // responsiveness commented out by Kinkade. It's stuck in sandbox and not From 38468ff592efbfb49bef8b1f043fdd849937bfbb Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 25 Sep 2024 14:43:26 -0600 Subject: [PATCH 12/26] Adds podSelectors for MultiNetworkPolicies After adding multiple MultiNetworkPolicies the multi-networkpolicy DaemonSets seems to not be applying the rules properly. I had figured that the NetworkAttachmentDefinition would be sufficient. This may or may not change/fix anything. --- k8s/networkpolicies/msak.jsonnet | 4 +++- k8s/networkpolicies/ndt.jsonnet | 4 +++- k8s/networkpolicies/neubot.jsonnet | 4 +++- k8s/networkpolicies/revtr.jsonnet | 4 +++- k8s/networkpolicies/wehe.jsonnet | 4 +++- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/k8s/networkpolicies/msak.jsonnet b/k8s/networkpolicies/msak.jsonnet index edc75c6a..a47eeb08 100644 --- a/k8s/networkpolicies/msak.jsonnet +++ b/k8s/networkpolicies/msak.jsonnet @@ -9,7 +9,9 @@ }, }, spec: { - podSelector: {}, + podSelector: + matchLabels: + workload: msak policyTypes: [ 'Ingress', ], diff --git a/k8s/networkpolicies/ndt.jsonnet b/k8s/networkpolicies/ndt.jsonnet index 7870e96f..8be9b00a 100644 --- a/k8s/networkpolicies/ndt.jsonnet +++ b/k8s/networkpolicies/ndt.jsonnet @@ -9,7 +9,9 @@ }, }, spec: { - podSelector: {}, + podSelector: + matchLabels: + workload: ndt policyTypes: [ 'Ingress', ], diff --git a/k8s/networkpolicies/neubot.jsonnet b/k8s/networkpolicies/neubot.jsonnet index 3cf4ed8e..3f88be99 100644 --- a/k8s/networkpolicies/neubot.jsonnet +++ b/k8s/networkpolicies/neubot.jsonnet @@ -9,7 +9,9 @@ }, }, spec: { - podSelector: {}, + podSelector: + matchLabels: + workload: neubot policyTypes: [ 'Ingress', ], diff --git a/k8s/networkpolicies/revtr.jsonnet b/k8s/networkpolicies/revtr.jsonnet index bc556c80..7253bd85 100644 --- a/k8s/networkpolicies/revtr.jsonnet +++ b/k8s/networkpolicies/revtr.jsonnet @@ -9,7 +9,9 @@ }, }, spec: { - podSelector: {}, + podSelector: + matchLabels: + workload: revtr policyTypes: [ 'Ingress', ], diff --git a/k8s/networkpolicies/wehe.jsonnet b/k8s/networkpolicies/wehe.jsonnet index dd04f6ba..c94d5273 100644 --- a/k8s/networkpolicies/wehe.jsonnet +++ b/k8s/networkpolicies/wehe.jsonnet @@ -9,7 +9,9 @@ }, }, spec: { - podSelector: {}, + podSelector: + matchLabels: + workload: wehe policyTypes: [ 'Ingress', ], From 4c10942eada1b33147cbc89effb7eda4a7b86d80 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 25 Sep 2024 14:49:40 -0600 Subject: [PATCH 13/26] Formats MultiNetworkPolicies as jsonnet I had accidentally inserted some plain JSON. --- k8s/networkpolicies/msak.jsonnet | 8 +++++--- k8s/networkpolicies/ndt.jsonnet | 8 +++++--- k8s/networkpolicies/neubot.jsonnet | 8 +++++--- k8s/networkpolicies/revtr.jsonnet | 8 +++++--- k8s/networkpolicies/wehe.jsonnet | 8 +++++--- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/k8s/networkpolicies/msak.jsonnet b/k8s/networkpolicies/msak.jsonnet index a47eeb08..d7b5fac6 100644 --- a/k8s/networkpolicies/msak.jsonnet +++ b/k8s/networkpolicies/msak.jsonnet @@ -9,9 +9,11 @@ }, }, spec: { - podSelector: - matchLabels: - workload: msak + podSelector: { + matchLabels: { + workload: 'msak', + }, + }, policyTypes: [ 'Ingress', ], diff --git a/k8s/networkpolicies/ndt.jsonnet b/k8s/networkpolicies/ndt.jsonnet index 8be9b00a..87fe5cd6 100644 --- a/k8s/networkpolicies/ndt.jsonnet +++ b/k8s/networkpolicies/ndt.jsonnet @@ -9,9 +9,11 @@ }, }, spec: { - podSelector: - matchLabels: - workload: ndt + podSelector: { + matchLabels: { + workload: 'ndt', + }, + }, policyTypes: [ 'Ingress', ], diff --git a/k8s/networkpolicies/neubot.jsonnet b/k8s/networkpolicies/neubot.jsonnet index 3f88be99..4c548215 100644 --- a/k8s/networkpolicies/neubot.jsonnet +++ b/k8s/networkpolicies/neubot.jsonnet @@ -9,9 +9,11 @@ }, }, spec: { - podSelector: - matchLabels: - workload: neubot + podSelector: { + matchLabels: { + workload: 'neubot', + }, + }, policyTypes: [ 'Ingress', ], diff --git a/k8s/networkpolicies/revtr.jsonnet b/k8s/networkpolicies/revtr.jsonnet index 7253bd85..e51b3458 100644 --- a/k8s/networkpolicies/revtr.jsonnet +++ b/k8s/networkpolicies/revtr.jsonnet @@ -9,9 +9,11 @@ }, }, spec: { - podSelector: - matchLabels: - workload: revtr + podSelector: { + matchLabels: { + workload: 'revtr', + }, + }, policyTypes: [ 'Ingress', ], diff --git a/k8s/networkpolicies/wehe.jsonnet b/k8s/networkpolicies/wehe.jsonnet index c94d5273..588139c3 100644 --- a/k8s/networkpolicies/wehe.jsonnet +++ b/k8s/networkpolicies/wehe.jsonnet @@ -9,9 +9,11 @@ }, }, spec: { - podSelector: - matchLabels: - workload: wehe + podSelector: { + matchLabels: { + workload: 'wehe', + }, + }, policyTypes: [ 'Ingress', ], From 64b4d5f5791288c3313375513740508894ade895 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Thu, 26 Sep 2024 11:20:20 -0600 Subject: [PATCH 14/26] Removes podSelector from MultiNetworkPolicies It turns out the issue was apparently just me pushing changes to k8s-support in sandbox with the multi-networkpolicy changes, and then another branch without, and then with, and then without, which got pod annotations all messed up. --- k8s/networkpolicies/msak.jsonnet | 6 +----- k8s/networkpolicies/ndt.jsonnet | 6 +----- k8s/networkpolicies/neubot.jsonnet | 6 +----- k8s/networkpolicies/revtr.jsonnet | 6 +----- k8s/networkpolicies/wehe.jsonnet | 6 +----- 5 files changed, 5 insertions(+), 25 deletions(-) diff --git a/k8s/networkpolicies/msak.jsonnet b/k8s/networkpolicies/msak.jsonnet index d7b5fac6..edc75c6a 100644 --- a/k8s/networkpolicies/msak.jsonnet +++ b/k8s/networkpolicies/msak.jsonnet @@ -9,11 +9,7 @@ }, }, spec: { - podSelector: { - matchLabels: { - workload: 'msak', - }, - }, + podSelector: {}, policyTypes: [ 'Ingress', ], diff --git a/k8s/networkpolicies/ndt.jsonnet b/k8s/networkpolicies/ndt.jsonnet index 87fe5cd6..7870e96f 100644 --- a/k8s/networkpolicies/ndt.jsonnet +++ b/k8s/networkpolicies/ndt.jsonnet @@ -9,11 +9,7 @@ }, }, spec: { - podSelector: { - matchLabels: { - workload: 'ndt', - }, - }, + podSelector: {}, policyTypes: [ 'Ingress', ], diff --git a/k8s/networkpolicies/neubot.jsonnet b/k8s/networkpolicies/neubot.jsonnet index 4c548215..3cf4ed8e 100644 --- a/k8s/networkpolicies/neubot.jsonnet +++ b/k8s/networkpolicies/neubot.jsonnet @@ -9,11 +9,7 @@ }, }, spec: { - podSelector: { - matchLabels: { - workload: 'neubot', - }, - }, + podSelector: {}, policyTypes: [ 'Ingress', ], diff --git a/k8s/networkpolicies/revtr.jsonnet b/k8s/networkpolicies/revtr.jsonnet index e51b3458..bc556c80 100644 --- a/k8s/networkpolicies/revtr.jsonnet +++ b/k8s/networkpolicies/revtr.jsonnet @@ -9,11 +9,7 @@ }, }, spec: { - podSelector: { - matchLabels: { - workload: 'revtr', - }, - }, + podSelector: {}, policyTypes: [ 'Ingress', ], diff --git a/k8s/networkpolicies/wehe.jsonnet b/k8s/networkpolicies/wehe.jsonnet index 588139c3..dd04f6ba 100644 --- a/k8s/networkpolicies/wehe.jsonnet +++ b/k8s/networkpolicies/wehe.jsonnet @@ -9,11 +9,7 @@ }, }, spec: { - podSelector: { - matchLabels: { - workload: 'wehe', - }, - }, + podSelector: {}, policyTypes: [ 'Ingress', ], From 6c6697b8372fde76a9014f43f98bfca45f9ac9e6 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Mon, 7 Oct 2024 14:44:16 -0600 Subject: [PATCH 15/26] Removes sample iptables custom rules from configmap The rules are not utilized by multi-networkpolicy unless you pass special flags, which are outlined in a comment in the file. --- config/multi-networkpolicy.jsonnet | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/config/multi-networkpolicy.jsonnet b/config/multi-networkpolicy.jsonnet index 3980b4b5..2623d7b0 100644 --- a/config/multi-networkpolicy.jsonnet +++ b/config/multi-networkpolicy.jsonnet @@ -9,9 +9,29 @@ app: 'multi-networkpolicy', }, }, + // Add custom iptables rules below. The rules will not be applied unless you + // pass at least one of the following flags to multi-networkpolicy: + // + // --custom-v4-igress-rule-file + // --custom-v4-egress-rule-file + // --custom-v6-igress-rule-file + // --custom-v4-egress-rule-file + // + // Add iptables rules one per line in the appropriate sections below, minus + // "iptables -A " as that is added for you by multi-networkpolicy. data: { - 'custom-v4-rules.txt': '# accept redirect\n-p icmp --icmp-type redirect -j ACCEPT\n# accept fragmentation-needed (for MTU discovery)\n-p icmp --icmp-type fragmentation-needed -j ACCEPT\n', - 'custom-v6-rules.txt': '# accept NDP\n-p icmpv6 --icmpv6-type neighbor-solicitation -j ACCEPT\n-p icmpv6 --icmpv6-type neighbor-advertisement -j ACCEPT\n# accept RA/RS\n-p icmpv6 --icmpv6-type router-solicitation -j ACCEPT\n-p icmpv6 --icmpv6-type router-advertisement -j ACCEPT\n# accept redirect\n-p icmpv6 --icmpv6-type redirect -j ACCEPT\n# accept packet-too-big (for MTU discovery)\n-p icmpv6 --icmpv6-type packet-too-big -j ACCEPT\n', + 'custom-v4-ingress-rules.txt': ||| + # No custom rules, yet. + |||, + 'custom-v4-egress-rules.txt': ||| + # No custom rules, yet. + |||, + 'custom-v6-ingress-rules.txt': ||| + # No custom rules, yet. + |||, + 'custom-v6-egress-rules.txt': ||| + # No custom rules, yet. + |||, }, } From f5e0d85ab87381fefd86fe8c9b0c1f242b7d4441 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 9 Oct 2024 14:21:23 -0600 Subject: [PATCH 16/26] Adds new MultiNetworkPolicy template to templates.jsonnet --- k8s/daemonsets/templates.jsonnet | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/k8s/daemonsets/templates.jsonnet b/k8s/daemonsets/templates.jsonnet index cc320297..345fed64 100644 --- a/k8s/daemonsets/templates.jsonnet +++ b/k8s/daemonsets/templates.jsonnet @@ -738,6 +738,36 @@ local Heartbeat(expName, tcpPort, hostNetwork, services) = [ else [] ; +local MultiNetworkPolicy(expName, index, ports) = { + apiVersion: 'k8s.cni.cncf.io/v1beta2', + kind: 'MultiNetworkPolicy', + metadata: { + name: expName, + namespace: 'default', + annotations: { + 'k8s.v1.cni.cncf.io/policy-for': 'index2ip-index-' + index + '-conf', + }, + }, + spec: { + podSelector: {}, + policyTypes: [ + 'Ingress', + ], + ingress: [ + { + ports: [ + { + port: std.split(std.split(p, '/')[0], ':')[0], + [if std.length(std.split(p, ':')) == 2 then 'endPort']: std.split(std.split(p, '/')[0], ':')[1], + protocol: std.split(p, '/')[1], + }, + for p in ports + ], + }, + ], + }, +}; + local ExperimentNoIndex(name, bucket, anonMode, datatypesArchived, datatypesAutoloaded, hostNetwork, siteType='physical') = { local allDatatypes = ['tcpinfo', 'pcap', 'annotation2', 'scamper1', 'hopannotation2'] + datatypesArchived, local allVolumes = datatypesArchived + datatypesAutoloaded, @@ -894,6 +924,9 @@ local Experiment(name, index, bucket, anonMode, datatypes=[], datatypesAutoloade // Returns a "container" configuration for the heartbeat service. Heartbeat(expName, hostNetwork, services):: Heartbeat(expName, 9996, hostNetwork, services), + // Returns a manifest for a MultiNetworkPolicy CRD object. + MultiNetworkPolicy(expName, index, ports):: MultiNetworkPolicy(expName, index, ports), + // Volumes, volumemounts and other data and configs for experiment metadata. Metadata:: Metadata, From 515ed6ceb5f93ff7a87377aa9596a46db61c1757 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 9 Oct 2024 14:22:49 -0600 Subject: [PATCH 17/26] Adds a MultiNetworkPolicy definition to the ndt.jsonnet manifest --- k8s/daemonsets/experiments/ndt.jsonnet | 221 +++++++++++++------------ 1 file changed, 114 insertions(+), 107 deletions(-) diff --git a/k8s/daemonsets/experiments/ndt.jsonnet b/k8s/daemonsets/experiments/ndt.jsonnet index c23edbd7..bf8f5bc8 100644 --- a/k8s/daemonsets/experiments/ndt.jsonnet +++ b/k8s/daemonsets/experiments/ndt.jsonnet @@ -7,124 +7,131 @@ local services = [ ]; local PROJECT_ID = std.extVar('PROJECT_ID'); -exp.Experiment(expName, 2, 'pusher-' + std.extVar('PROJECT_ID'), "none", datatypes, []) + { - spec+: { - template+: { - metadata+: { - annotations+: { - "secret.reloader.stakater.com/reload": "measurement-lab-org-tls", - }, - }, - spec+: { - nodeSelector+: { - 'mlab/ndt-version': 'production', +// List of ports that need to be opened in the pod network namespace. +local ports = ['80/TCP', '443/TCP', '3001/TCP', '3010/TCP', '32768:60999/TCP']; + +[ + exp.Experiment(expName, 2, 'pusher-' + std.extVar('PROJECT_ID'), "none", datatypes, []) + { + spec+: { + template+: { + metadata+: { + annotations+: { + "secret.reloader.stakater.com/reload": "measurement-lab-org-tls", + }, }, - serviceAccountName: 'heartbeat-experiment', - containers+: [ - { - name: 'ndt-server', - image: 'measurementlab/ndt-server:' + exp.ndtVersion, - // The max-rate flag value is stored in a file on the host - // filesystem, created by the systemd max-rate.service. - command: [ - '/bin/sh', - '-c', - '/ndt-server -txcontroller.max-rate=$(cat /metadata/iface-max-rate) $@', - '--', - ], - args: [ - '-uuid-prefix-file=' + exp.uuid.prefixfile, - '-prometheusx.listen-address=$(PRIVATE_IP):9990', - '-datadir=/var/spool/' + expName, - '-txcontroller.device=net1', - '-htmldir=html/mlab', - '-key=/certs/tls.key', - '-cert=/certs/tls.crt', - '-token.machine=$(NODE_NAME)', - '-token.verify-key=/verify/jwk_sig_EdDSA_locate_20200409.pub', - '-ndt7.token.required=true', - '-label=type=physical', - '-label=deployment=stable', - '-label=managed=@' + exp.Metadata.path + '/managed', - '-label=loadbalanced=@' + exp.Metadata.path + '/loadbalanced', - ], - env: [ - { - name: 'PRIVATE_IP', - valueFrom: { - fieldRef: { - fieldPath: 'status.podIP', + spec+: { + nodeSelector+: { + 'mlab/ndt-version': 'production', + }, + serviceAccountName: 'heartbeat-experiment', + containers+: [ + { + name: 'ndt-server', + image: 'measurementlab/ndt-server:' + exp.ndtVersion, + // The max-rate flag value is stored in a file on the host + // filesystem, created by the systemd max-rate.service. + command: [ + '/bin/sh', + '-c', + '/ndt-server -txcontroller.max-rate=$(cat /metadata/iface-max-rate) $@', + '--', + ], + args: [ + '-uuid-prefix-file=' + exp.uuid.prefixfile, + '-prometheusx.listen-address=$(PRIVATE_IP):9990', + '-datadir=/var/spool/' + expName, + '-txcontroller.device=net1', + '-htmldir=html/mlab', + '-key=/certs/tls.key', + '-cert=/certs/tls.crt', + '-token.machine=$(NODE_NAME)', + '-token.verify-key=/verify/jwk_sig_EdDSA_locate_20200409.pub', + '-ndt7.token.required=true', + '-label=type=physical', + '-label=deployment=stable', + '-label=managed=@' + exp.Metadata.path + '/managed', + '-label=loadbalanced=@' + exp.Metadata.path + '/loadbalanced', + ], + env: [ + { + name: 'PRIVATE_IP', + valueFrom: { + fieldRef: { + fieldPath: 'status.podIP', + }, }, }, - }, - { - name: 'NODE_NAME', - valueFrom: { - fieldRef: { - fieldPath: 'spec.nodeName', + { + name: 'NODE_NAME', + valueFrom: { + fieldRef: { + fieldPath: 'spec.nodeName', + }, }, }, + ], + securityContext: { + capabilities: { + drop: [ + 'all', + ], + }, }, - ], - securityContext: { - capabilities: { - drop: [ - 'all', - ], + volumeMounts: [ + { + mountPath: '/certs', + name: 'measurement-lab-org-tls', + readOnly: true, + }, + { + mountPath: '/verify', + name: 'locate-verify-keys', + readOnly: true, + }, + exp.uuid.volumemount, + exp.Metadata.volumemount, + ] + [ + exp.VolumeMount(expName + '/' + d) for d in datatypes + ], + ports: [ + { + containerPort: 9990, + }, + ], + + } + ] + std.flattenArrays([ + exp.Heartbeat(expName, false, services), + ]) + std.flattenArrays([ + // NOTE: exclude from production until design doc is approved, service + // is monitored and scales to millions of requests/day. + exp.Revtr(expName) + ]), + volumes+: [ + { + name: 'measurement-lab-org-tls', + secret: { + secretName: 'measurement-lab-org-tls', }, }, - volumeMounts: [ - { - mountPath: '/certs', - name: 'measurement-lab-org-tls', - readOnly: true, + { + name: 'locate-verify-keys', + secret: { + secretName: 'locate-verify-keys', }, - { - mountPath: '/verify', - name: 'locate-verify-keys', - readOnly: true, - }, - exp.uuid.volumemount, - exp.Metadata.volumemount, - ] + [ - exp.VolumeMount(expName + '/' + d) for d in datatypes - ], - ports: [ - { - containerPort: 9990, - }, - ], - - } - ] + std.flattenArrays([ - exp.Heartbeat(expName, false, services), - ]) + std.flattenArrays([ - // NOTE: exclude from production until design doc is approved, service - // is monitored and scales to millions of requests/day. - exp.Revtr(expName) - ]), - volumes+: [ - { - name: 'measurement-lab-org-tls', - secret: { - secretName: 'measurement-lab-org-tls', - }, - }, - { - name: 'locate-verify-keys', - secret: { - secretName: 'locate-verify-keys', }, - }, - exp.Metadata.volume, - { - name: 'revtr-apikey', - secret: { - secretName: 'revtr-apikey', + exp.Metadata.volume, + { + name: 'revtr-apikey', + secret: { + secretName: 'revtr-apikey', + }, }, - }, - ], + ], + }, }, }, }, -} + exp.MultiNetworkPolicy(expName, 2, ports), +] + From 4b6ad50922483d71d3e40bc7f6ede1f14eeaeed5 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 9 Oct 2024 14:26:51 -0600 Subject: [PATCH 18/26] Adds MultiNetworkPolicy definition to revtr manifest --- k8s/daemonsets/experiments/revtr.jsonnet | 91 +++++++++++++----------- 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/k8s/daemonsets/experiments/revtr.jsonnet b/k8s/daemonsets/experiments/revtr.jsonnet index d6d1037d..a971b1b5 100644 --- a/k8s/daemonsets/experiments/revtr.jsonnet +++ b/k8s/daemonsets/experiments/revtr.jsonnet @@ -1,47 +1,54 @@ local exp = import '../templates.jsonnet'; -exp.Experiment('revtr', 3, 'pusher-' + std.extVar('PROJECT_ID'), 'none', [], []) + { - spec+: { - template+: { - spec+: { - containers+: [ - { - name: 'revtrvp', - image: 'measurementlab/revtrvp:v0.3.2', - args: [ - '/server.crt', - '/plvp.config', - ], - securityContext: { - capabilities: { - // The container processes run as nobody:nogroup, but the - // scamper binary has/needs these capabilities. - // scamper in traceroute-caller also has most of these - // capabilities, except CHOWN. scamper in this container is - // version 20211212a, while traceroute-caller uses scamper - // version 20230302. Perhaps the need for CHOWN was removed in - // the newer version? - // TODO(kinkade): if revtr updates scamper, check to see - // whether we can remove the CHOWN capability. - add: [ - 'CHOWN', - 'DAC_OVERRIDE', - 'NET_RAW', - 'SETGID', - 'SETUID', - 'SYS_CHROOT', - ], - drop: [ - 'all', - ], +// List of ports that need to be opened in the pod network namespace. +local ports = ['55557/TCP']; + +[ + exp.Experiment('revtr', 3, 'pusher-' + std.extVar('PROJECT_ID'), 'none', [], []) + { + spec+: { + template+: { + spec+: { + containers+: [ + { + name: 'revtrvp', + image: 'measurementlab/revtrvp:v0.3.2', + args: [ + '/server.crt', + '/plvp.config', + ], + securityContext: { + capabilities: { + // The container processes run as nobody:nogroup, but the + // scamper binary has/needs these capabilities. + // scamper in traceroute-caller also has most of these + // capabilities, except CHOWN. scamper in this container is + // version 20211212a, while traceroute-caller uses scamper + // version 20230302. Perhaps the need for CHOWN was removed in + // the newer version? + // TODO(kinkade): if revtr updates scamper, check to see + // whether we can remove the CHOWN capability. + add: [ + 'CHOWN', + 'DAC_OVERRIDE', + 'NET_RAW', + 'SETGID', + 'SETUID', + 'SYS_CHROOT', + ], + drop: [ + 'all', + ], + }, }, - }, - } - ], - volumes+: [ - exp.Metadata.volume, - ], + } + ], + volumes+: [ + exp.Metadata.volume, + ], + } } } - } -} + }, + exp.MultiNetworkPolicy('revtr', 3, ports), +] + From d44b36cbab6b04b71987018698f93d354aa7213b Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 9 Oct 2024 14:36:32 -0600 Subject: [PATCH 19/26] Adds MultiNetworkPolicy definition to the wehe manifest --- k8s/daemonsets/experiments/wehe.jsonnet | 396 ++++++++++++------------ 1 file changed, 205 insertions(+), 191 deletions(-) diff --git a/k8s/daemonsets/experiments/wehe.jsonnet b/k8s/daemonsets/experiments/wehe.jsonnet index 453a532b..784ff585 100644 --- a/k8s/daemonsets/experiments/wehe.jsonnet +++ b/k8s/daemonsets/experiments/wehe.jsonnet @@ -5,214 +5,228 @@ local services = [ 'wehe/replay=wss://:4443/v0/envelope/access', ]; -exp.Experiment(expName, 5, 'pusher-' + std.extVar('PROJECT_ID'), 'netblock', ['replay'], autoloadedDatatypes) + { - spec+: { - template+: { - metadata+: { - annotations+: { - "secret.reloader.stakater.com/reload": "measurement-lab-org-tls", - }, - }, - spec+: { - serviceAccountName: 'heartbeat-experiment', - initContainers+: [ - { - args: [ - 'cp', '/wehe-ca/ca.key', '/wehe-ca/ca.crt', '/wehe/ssl/', - ], - image: 'busybox:1.34', - // Wehe expects the ca.key and ca.crt to be in a - // directory to which it can write the resulting keys - // produced. Secrets can't be mounted read/write, so - // before we start we copy those files from the mounted - // secret (read-only) to a cache directory (read-write). - name: 'ca-copy', - volumeMounts: [ - { - mountPath: '/wehe/ssl/', - name: 'wehe-ca-cache', - }, - { - mountPath: '/wehe-ca/', - name: 'wehe-ca', - }, - ], +// List of ports that need to be opened in the pod network namespace. +local ports = [ + '80/TCP', '81/TCP', '443/TCP', '443/UDP', '465/TCP', '853/TCP', '993/TCP', + '995/TCP', '1194/TCP', '1701/TCP', '3478/UDP', '3480/UDP', '4443/TCP', + '5004/UDP', '5061/TCP', '6881/TCP', '8080/TCP', '8443/TCP', '8801/UDP', + '9000/UDP', '9989/TCP', '19305/UDP', '35253/TCP', '49882/UDP', '50002/UDP', + '55555/TCP', '55556/TCP', '55557/TCP', '56565/TCP', '56566/TCP', '62065/UDP', + '63308/UDP' +]; + +[ + exp.Experiment(expName, 5, 'pusher-' + std.extVar('PROJECT_ID'), 'netblock', ['replay'], autoloadedDatatypes) + { + spec+: { + template+: { + metadata+: { + annotations+: { + "secret.reloader.stakater.com/reload": "measurement-lab-org-tls", }, - ], - containers+: [ - { - args: [ - '-envelope.key=/certs/tls.key', - '-envelope.cert=/certs/tls.crt', - '-envelope.listen-address=:4443', - '-envelope.device=net1', - '-envelope.max-clients=5', - '-envelope.subject=wehe', - '-envelope.machine=$(MLAB_NODE_NAME)', - '-envelope.verify-key=/verify/jwk_sig_EdDSA_locate_20200409.pub', - '-envelope.token-required=true', - '-prometheusx.listen-address=:9989', - // Maximum timeout for a client to hold the envelope open. - '-timeout=10m', - ], - env: [ - { - name: 'MLAB_NODE_NAME', - valueFrom: { - fieldRef: { - fieldPath: 'spec.nodeName', + }, + spec+: { + serviceAccountName: 'heartbeat-experiment', + initContainers+: [ + { + args: [ + 'cp', '/wehe-ca/ca.key', '/wehe-ca/ca.crt', '/wehe/ssl/', + ], + image: 'busybox:1.34', + // Wehe expects the ca.key and ca.crt to be in a + // directory to which it can write the resulting keys + // produced. Secrets can't be mounted read/write, so + // before we start we copy those files from the mounted + // secret (read-only) to a cache directory (read-write). + name: 'ca-copy', + volumeMounts: [ + { + mountPath: '/wehe/ssl/', + name: 'wehe-ca-cache', + }, + { + mountPath: '/wehe-ca/', + name: 'wehe-ca', + }, + ], + }, + ], + containers+: [ + { + args: [ + '-envelope.key=/certs/tls.key', + '-envelope.cert=/certs/tls.crt', + '-envelope.listen-address=:4443', + '-envelope.device=net1', + '-envelope.max-clients=5', + '-envelope.subject=wehe', + '-envelope.machine=$(MLAB_NODE_NAME)', + '-envelope.verify-key=/verify/jwk_sig_EdDSA_locate_20200409.pub', + '-envelope.token-required=true', + '-prometheusx.listen-address=:9989', + // Maximum timeout for a client to hold the envelope open. + '-timeout=10m', + ], + env: [ + { + name: 'MLAB_NODE_NAME', + valueFrom: { + fieldRef: { + fieldPath: 'spec.nodeName', + }, }, }, + ], + image: 'measurementlab/access:v0.0.10', + name: 'access', + // The access envelope needs to be able to manipulate firewall + // rules. + securityContext: { + capabilities: { + add: [ + 'NET_ADMIN', + 'NET_RAW', + ], + drop: [ + 'all', + ], + }, + runAsUser: 0, }, - ], - image: 'measurementlab/access:v0.0.10', - name: 'access', - // The access envelope needs to be able to manipulate firewall - // rules. - securityContext: { - capabilities: { - add: [ - 'NET_ADMIN', - 'NET_RAW', - ], - drop: [ - 'all', - ], - }, - runAsUser: 0, + volumeMounts: [ + { + mountPath: '/certs', + name: 'measurement-lab-org-tls', + readOnly: true, + }, + { + mountPath: '/verify', + name: 'locate-verify-keys', + readOnly: true, + }, + ], + // Advertise the prometheus port so it can be discovered by Prometheus. + ports: [ + { + containerPort: 9989, + }, + ], }, - volumeMounts: [ - { - mountPath: '/certs', - name: 'measurement-lab-org-tls', - readOnly: true, - }, - { - mountPath: '/verify', - name: 'locate-verify-keys', - readOnly: true, - }, - ], - // Advertise the prometheus port so it can be discovered by Prometheus. - ports: [ - { - containerPort: 9989, - }, - ], - }, - { - args: [ - 'wehe.$(MLAB_NODE_NAME)', - 'net1', - ], - env: [ - { - name: 'MLAB_NODE_NAME', - valueFrom: { - fieldRef: { - fieldPath: 'spec.nodeName', + { + args: [ + 'wehe.$(MLAB_NODE_NAME)', + 'net1', + ], + env: [ + { + name: 'MLAB_NODE_NAME', + valueFrom: { + fieldRef: { + fieldPath: 'spec.nodeName', + }, }, }, + { + name: 'UUID_PREFIX', + value: '/var/local/uuid/prefix', + }, + ], + image: 'measurementlab/wehe-py3:v0.3.12', + livenessProbe+: { + httpGet: { + path: '/metrics', + port: 9990, + }, + // After startup, liveness should never fail. + initialDelaySeconds: 300, // TODO: eliminate with k8s v1.18+. + failureThreshold: 1, + timeoutSeconds: 10, + periodSeconds: 30, + }, + name: expName, + // Advertise the prometheus port so it can be discovered by Prometheus. + ports: [ + { + // Replay server + containerPort: 9990, + }, + { + // Analyzer server + containerPort: 9091, + }, + ], + resources+: { + limits: { + memory: "5Gi", + }, + requests: { + memory: "1Gi", + }, }, - { - name: 'UUID_PREFIX', - value: '/var/local/uuid/prefix', - }, - ], - image: 'measurementlab/wehe-py3:v0.3.12', - livenessProbe+: { - httpGet: { - path: '/metrics', - port: 9990, + // Wehe runs packet captures, which requires being root. Run as + // root, but with only the NET_RAW capability. + securityContext: { + capabilities: { + add: [ + 'NET_RAW', + ], + drop: [ + 'all' + ], + }, }, - // After startup, liveness should never fail. - initialDelaySeconds: 300, // TODO: eliminate with k8s v1.18+. - failureThreshold: 1, - timeoutSeconds: 10, - periodSeconds: 30, + startupProbe+: { + httpGet: { + path: '/metrics', + port: 9990, + }, + // Allow up to 5min for the service to startup: 30*10. + failureThreshold: 30, + periodSeconds: 10, + }, + volumeMounts: [ + exp.VolumeMount('wehe/replay'), + { + mountPath: '/wehe/ssl/', + name: 'wehe-ca-cache', + }, + exp.uuid.volumemount, + exp.VolumeMountDatatypes(expName), + ] + [ + exp.VolumeMount(expName + '/' + d) for d in autoloadedDatatypes + ], }, - name: expName, - // Advertise the prometheus port so it can be discovered by Prometheus. - ports: [ - { - // Replay server - containerPort: 9990, - }, - { - // Analyzer server - containerPort: 9091, - }, - ], - resources+: { - limits: { - memory: "5Gi", - }, - requests: { - memory: "1Gi", + ] + std.flattenArrays([ + exp.Heartbeat(expName, false, services), + ]), + volumes+: [ + { + name: 'measurement-lab-org-tls', + secret: { + secretName: 'measurement-lab-org-tls', }, }, - // Wehe runs packet captures, which requires being root. Run as - // root, but with only the NET_RAW capability. - securityContext: { - capabilities: { - add: [ - 'NET_RAW', - ], - drop: [ - 'all' - ], - }, + { + emptyDir: {}, + name: 'wehe-ca-cache', }, - startupProbe+: { - httpGet: { - path: '/metrics', - port: 9990, + { + name: 'wehe-ca', + secret: { + secretName: 'wehe-ca', }, - // Allow up to 5min for the service to startup: 30*10. - failureThreshold: 30, - periodSeconds: 10, }, - volumeMounts: [ - exp.VolumeMount('wehe/replay'), - { - mountPath: '/wehe/ssl/', - name: 'wehe-ca-cache', + { + name: 'locate-verify-keys', + secret: { + secretName: 'locate-verify-keys', }, - exp.uuid.volumemount, - exp.VolumeMountDatatypes(expName), - ] + [ - exp.VolumeMount(expName + '/' + d) for d in autoloadedDatatypes - ], - }, - ] + std.flattenArrays([ - exp.Heartbeat(expName, false, services), - ]), - volumes+: [ - { - name: 'measurement-lab-org-tls', - secret: { - secretName: 'measurement-lab-org-tls', - }, - }, - { - emptyDir: {}, - name: 'wehe-ca-cache', - }, - { - name: 'wehe-ca', - secret: { - secretName: 'wehe-ca', }, - }, - { - name: 'locate-verify-keys', - secret: { - secretName: 'locate-verify-keys', - }, - }, - exp.Metadata.volume, - ], + exp.Metadata.volume, + ], + }, }, }, }, -} + exp.MultiNetworkPolicy(expName, 5, ports), +] + From f55707a2cd5b35bcc19d4afe34ca3cebaca9e634 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 9 Oct 2024 14:38:56 -0600 Subject: [PATCH 20/26] Adds MultiNetworkPolicy definition to msak manifest --- k8s/daemonsets/experiments/msak.jsonnet | 207 ++++++++++++------------ 1 file changed, 107 insertions(+), 100 deletions(-) diff --git a/k8s/daemonsets/experiments/msak.jsonnet b/k8s/daemonsets/experiments/msak.jsonnet index 54cbd189..6d5183ce 100644 --- a/k8s/daemonsets/experiments/msak.jsonnet +++ b/k8s/daemonsets/experiments/msak.jsonnet @@ -7,116 +7,123 @@ local services = [ 'msak/latency1=http:///latency/v1/authorize,https:///latency/v1/authorize,http:///latency/v1/result,https:///latency/v1/result', ]; -exp.Experiment(expName, 1, 'pusher-' + std.extVar('PROJECT_ID'), "none", [], datatypes) + { - spec+: { - template+: { - metadata+: { - annotations+: { - 'secret.reloader.stakater.com/reload': 'measurement-lab-org-tls', - }, - }, - spec+: { - serviceAccountName: 'heartbeat-experiment', - initContainers+: [ - { - // Copy the JSON schema where jostler expects it to be. - name: 'copy-schema', - image: 'measurementlab/msak:' + expVersion, - command: [ - '/bin/sh', - '-c', - 'cp /msak/throughput1.json /var/spool/datatypes/throughput1.json && ' + - 'cp /msak/latency1.json /var/spool/datatypes/latency1.json', - ], - volumeMounts: [ - exp.VolumeMountDatatypes(expName), - ], +// List of ports that need to be opened in the pod network namespace. +local ports = ['80/TCP', '443/TCP', '1053/UDP']; + +[ + exp.Experiment(expName, 1, 'pusher-' + std.extVar('PROJECT_ID'), "none", [], datatypes) + { + spec+: { + template+: { + metadata+: { + annotations+: { + 'secret.reloader.stakater.com/reload': 'measurement-lab-org-tls', }, - ], - containers+: [ - { - args: [ - '-ws_addr=:80', - '-wss_addr=:443', - '-cert=/certs/tls.crt', - '-key=/certs/tls.key', - '-datadir=/var/spool/' + expName, - '-token.machine=$(NODE_NAME)', - '-token.verify-key=/verify/jwk_sig_EdDSA_locate_20200409.pub', - '-token.verify=true', - '-uuid-prefix-file=' + exp.uuid.prefixfile, - '-prometheusx.listen-address=$(PRIVATE_IP):9990', - ], - env: [ - { - name: 'NODE_NAME', - valueFrom: { - fieldRef: { - fieldPath: 'spec.nodeName', + }, + spec+: { + serviceAccountName: 'heartbeat-experiment', + initContainers+: [ + { + // Copy the JSON schema where jostler expects it to be. + name: 'copy-schema', + image: 'measurementlab/msak:' + expVersion, + command: [ + '/bin/sh', + '-c', + 'cp /msak/throughput1.json /var/spool/datatypes/throughput1.json && ' + + 'cp /msak/latency1.json /var/spool/datatypes/latency1.json', + ], + volumeMounts: [ + exp.VolumeMountDatatypes(expName), + ], + }, + ], + containers+: [ + { + args: [ + '-ws_addr=:80', + '-wss_addr=:443', + '-cert=/certs/tls.crt', + '-key=/certs/tls.key', + '-datadir=/var/spool/' + expName, + '-token.machine=$(NODE_NAME)', + '-token.verify-key=/verify/jwk_sig_EdDSA_locate_20200409.pub', + '-token.verify=true', + '-uuid-prefix-file=' + exp.uuid.prefixfile, + '-prometheusx.listen-address=$(PRIVATE_IP):9990', + ], + env: [ + { + name: 'NODE_NAME', + valueFrom: { + fieldRef: { + fieldPath: 'spec.nodeName', + }, }, }, - }, - { - name: 'PRIVATE_IP', - valueFrom: { - fieldRef: { - fieldPath: 'status.podIP', + { + name: 'PRIVATE_IP', + valueFrom: { + fieldRef: { + fieldPath: 'status.podIP', + }, }, }, + ], + image: 'measurementlab/msak:' + expVersion, + name: 'msak', + command: [ + '/msak/msak-server', + ], + securityContext: { + capabilities: { + drop: [ + 'all', + ], + }, }, - ], - image: 'measurementlab/msak:' + expVersion, - name: 'msak', - command: [ - '/msak/msak-server', - ], - securityContext: { - capabilities: { - drop: [ - 'all', - ], - }, + volumeMounts: [ + { + mountPath: '/certs', + name: 'measurement-lab-org-tls', + readOnly: true, + }, + { + mountPath: '/verify', + name: 'locate-verify-keys', + readOnly: true, + }, + exp.uuid.volumemount, + ] + [ + exp.VolumeMount(expName + '/' + d) for d in datatypes + ], + ports: [ + { + containerPort: 9990, + }, + ], }, - volumeMounts: [ - { - mountPath: '/certs', - name: 'measurement-lab-org-tls', - readOnly: true, - }, - { - mountPath: '/verify', - name: 'locate-verify-keys', - readOnly: true, - }, - exp.uuid.volumemount, - ] + [ - exp.VolumeMount(expName + '/' + d) for d in datatypes - ], - ports: [ - { - containerPort: 9990, + ] + std.flattenArrays([ + exp.Heartbeat(expName, false, services), + ]), + volumes+: [ + { + name: 'measurement-lab-org-tls', + secret: { + secretName: 'measurement-lab-org-tls', }, - ], - }, - ] + std.flattenArrays([ - exp.Heartbeat(expName, false, services), - ]), - volumes+: [ - { - name: 'measurement-lab-org-tls', - secret: { - secretName: 'measurement-lab-org-tls', }, - }, - { - name: 'locate-verify-keys', - secret: { - secretName: 'locate-verify-keys', + { + name: 'locate-verify-keys', + secret: { + secretName: 'locate-verify-keys', + }, }, - }, - exp.Metadata.volume, - ], + exp.Metadata.volume, + ], + }, }, }, }, -} + exp.MultiNetworkPolicy(expName, 1, ports), +] + From 3033844eaff419d45126aae01d03504a235967a7 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 9 Oct 2024 14:41:02 -0600 Subject: [PATCH 21/26] Adds MultiNetworkPolicy definition to neubot manifest --- k8s/daemonsets/experiments/neubot.jsonnet | 139 ++++++++++++---------- 1 file changed, 73 insertions(+), 66 deletions(-) diff --git a/k8s/daemonsets/experiments/neubot.jsonnet b/k8s/daemonsets/experiments/neubot.jsonnet index b27c78fc..54f5fb8f 100644 --- a/k8s/daemonsets/experiments/neubot.jsonnet +++ b/k8s/daemonsets/experiments/neubot.jsonnet @@ -5,81 +5,88 @@ local services = [ 'neubot/dash=https:///negotiate/dash', ]; -exp.Experiment(expName, 10, 'pusher-' + std.extVar('PROJECT_ID'), "none", datatypes, []) + { - spec+: { - template+: { - metadata+: { - annotations+: { - "secret.reloader.stakater.com/reload": "measurement-lab-org-tls", +// List of ports that need to be opened in the pod network namespace. +local ports = ['80/TCP', '443/TCP']; + +[ + exp.Experiment(expName, 10, 'pusher-' + std.extVar('PROJECT_ID'), "none", datatypes, []) + { + spec+: { + template+: { + metadata+: { + annotations+: { + "secret.reloader.stakater.com/reload": "measurement-lab-org-tls", + }, }, - }, - spec+: { - serviceAccountName: 'heartbeat-experiment', - containers+: [ - { - name: 'dash', - image: 'measurementlab/dash:v0.4.3', - args: [ - '-datadir=/var/spool/' + expName, - '-prometheusx.listen-address=$(PRIVATE_IP):9990', - '-http-listen-address=:80', - '-https-listen-address=:443', - '-tls-cert=/certs/tls.crt', - '-tls-key=/certs/tls.key', - ], - env: [ + spec+: { + serviceAccountName: 'heartbeat-experiment', + containers+: [ { - name: 'PRIVATE_IP', - valueFrom: { - fieldRef: { - fieldPath: 'status.podIP', + name: 'dash', + image: 'measurementlab/dash:v0.4.3', + args: [ + '-datadir=/var/spool/' + expName, + '-prometheusx.listen-address=$(PRIVATE_IP):9990', + '-http-listen-address=:80', + '-https-listen-address=:443', + '-tls-cert=/certs/tls.crt', + '-tls-key=/certs/tls.key', + ], + env: [ + { + name: 'PRIVATE_IP', + valueFrom: { + fieldRef: { + fieldPath: 'status.podIP', + }, }, }, + ], + securityContext: { + capabilities: { + drop: [ + 'all', + ], + }, }, - ], - securityContext: { - capabilities: { - drop: [ - 'all', - ], + volumeMounts: [ + { + mountPath: '/certs', + name: 'measurement-lab-org-tls', + readOnly: true, + }, + ] + [ + exp.VolumeMount(expName + '/' + d) for d in datatypes + ], + ports: [ + { + containerPort: 9990, + }, + ], + livenessProbe+: { + httpGet: { + path: '/metrics', + port: 9990, + }, + initialDelaySeconds: 5, + periodSeconds: 30, }, }, - volumeMounts: [ - { - mountPath: '/certs', - name: 'measurement-lab-org-tls', - readOnly: true, - }, - ] + [ - exp.VolumeMount(expName + '/' + d) for d in datatypes - ], - ports: [ - { - containerPort: 9990, - }, - ], - livenessProbe+: { - httpGet: { - path: '/metrics', - port: 9990, + ] + std.flattenArrays([ + exp.Heartbeat(expName, false, services), + ]), + volumes+: [ + { + name: 'measurement-lab-org-tls', + secret: { + secretName: 'measurement-lab-org-tls', }, - initialDelaySeconds: 5, - periodSeconds: 30, }, - }, - ] + std.flattenArrays([ - exp.Heartbeat(expName, false, services), - ]), - volumes+: [ - { - name: 'measurement-lab-org-tls', - secret: { - secretName: 'measurement-lab-org-tls', - }, - }, - exp.Metadata.volume, - ], + exp.Metadata.volume, + ], + }, }, }, }, -} + exp.MultiNetworkPolicy(expName, 10, ports), +] + From f3d0462dffbd952c74bb0ac4147f2b23575ec311 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 9 Oct 2024 14:45:06 -0600 Subject: [PATCH 22/26] Removes static MultiNetworkPolicy definitions These are now handled by a new template, and called from each experiment manifest. --- k8s/networkpolicies/msak.jsonnet | 36 ------- k8s/networkpolicies/ndt.jsonnet | 44 --------- k8s/networkpolicies/neubot.jsonnet | 32 ------ k8s/networkpolicies/revtr.jsonnet | 28 ------ k8s/networkpolicies/wehe.jsonnet | 152 ----------------------------- 5 files changed, 292 deletions(-) delete mode 100644 k8s/networkpolicies/msak.jsonnet delete mode 100644 k8s/networkpolicies/ndt.jsonnet delete mode 100644 k8s/networkpolicies/neubot.jsonnet delete mode 100644 k8s/networkpolicies/revtr.jsonnet delete mode 100644 k8s/networkpolicies/wehe.jsonnet diff --git a/k8s/networkpolicies/msak.jsonnet b/k8s/networkpolicies/msak.jsonnet deleted file mode 100644 index edc75c6a..00000000 --- a/k8s/networkpolicies/msak.jsonnet +++ /dev/null @@ -1,36 +0,0 @@ -{ - apiVersion: 'k8s.cni.cncf.io/v1beta2', - kind: 'MultiNetworkPolicy', - metadata: { - name: 'msak', - namespace: 'default', - annotations: { - 'k8s.v1.cni.cncf.io/policy-for': 'index2ip-index-1-conf', - }, - }, - spec: { - podSelector: {}, - policyTypes: [ - 'Ingress', - ], - ingress: [ - { - ports: [ - { - port: 80, - protocol: 'TCP', - }, - { - port: 443, - protocol: 'TCP', - }, - { - port: 1053, - protocol: 'UDP', - }, - ], - }, - ], - }, -} - diff --git a/k8s/networkpolicies/ndt.jsonnet b/k8s/networkpolicies/ndt.jsonnet deleted file mode 100644 index 7870e96f..00000000 --- a/k8s/networkpolicies/ndt.jsonnet +++ /dev/null @@ -1,44 +0,0 @@ -{ - apiVersion: 'k8s.cni.cncf.io/v1beta2', - kind: 'MultiNetworkPolicy', - metadata: { - name: 'ndt', - namespace: 'default', - annotations: { - 'k8s.v1.cni.cncf.io/policy-for': 'index2ip-index-2-conf', - }, - }, - spec: { - podSelector: {}, - policyTypes: [ - 'Ingress', - ], - ingress: [ - { - ports: [ - { - port: 80, - protocol: 'TCP', - }, - { - port: 443, - protocol: 'TCP', - }, - { - port: 3001, - protocol: 'TCP', - }, - { - port: 3010, - protocol: 'TCP', - }, - { - port: 32768, - endPort: 60999, - protocol: 'TCP', - }, - ], - }, - ], - }, -} diff --git a/k8s/networkpolicies/neubot.jsonnet b/k8s/networkpolicies/neubot.jsonnet deleted file mode 100644 index 3cf4ed8e..00000000 --- a/k8s/networkpolicies/neubot.jsonnet +++ /dev/null @@ -1,32 +0,0 @@ -{ - apiVersion: 'k8s.cni.cncf.io/v1beta2', - kind: 'MultiNetworkPolicy', - metadata: { - name: 'neubot', - namespace: 'default', - annotations: { - 'k8s.v1.cni.cncf.io/policy-for': 'index2ip-index-10-conf', - }, - }, - spec: { - podSelector: {}, - policyTypes: [ - 'Ingress', - ], - ingress: [ - { - ports: [ - { - port: 80, - protocol: 'TCP', - }, - { - port: 443, - protocol: 'TCP', - }, - ], - }, - ], - }, -} - diff --git a/k8s/networkpolicies/revtr.jsonnet b/k8s/networkpolicies/revtr.jsonnet deleted file mode 100644 index bc556c80..00000000 --- a/k8s/networkpolicies/revtr.jsonnet +++ /dev/null @@ -1,28 +0,0 @@ -{ - apiVersion: 'k8s.cni.cncf.io/v1beta2', - kind: 'MultiNetworkPolicy', - metadata: { - name: 'revtr', - namespace: 'default', - annotations: { - 'k8s.v1.cni.cncf.io/policy-for': 'index2ip-index-3-conf', - }, - }, - spec: { - podSelector: {}, - policyTypes: [ - 'Ingress', - ], - ingress: [ - { - ports: [ - { - port: 55557, - protocol: 'TCP', - }, - ], - }, - ], - }, -} - diff --git a/k8s/networkpolicies/wehe.jsonnet b/k8s/networkpolicies/wehe.jsonnet deleted file mode 100644 index dd04f6ba..00000000 --- a/k8s/networkpolicies/wehe.jsonnet +++ /dev/null @@ -1,152 +0,0 @@ -{ - apiVersion: 'k8s.cni.cncf.io/v1beta2', - kind: 'MultiNetworkPolicy', - metadata: { - name: 'wehe', - namespace: 'default', - annotations: { - 'k8s.v1.cni.cncf.io/policy-for': 'index2ip-index-5-conf', - }, - }, - spec: { - podSelector: {}, - policyTypes: [ - 'Ingress', - ], - ingress: [ - { - ports: [ - { - port: 80, - protocol: 'TCP', - }, - { - port: 81, - protocol: 'TCP', - }, - { - port: 443, - protocol: 'TCP', - }, - { - port: 443, - protocol: 'UDP', - }, - { - port: 465, - protocol: 'TCP', - }, - { - port: 853, - protocol: 'TCP', - }, - { - port: 993, - protocol: 'TCP', - }, - { - port: 995, - protocol: 'TCP', - }, - { - port: 1194, - protocol: 'TCP', - }, - { - port: 1701, - protocol: 'TCP', - }, - { - port: 3478, - protocol: 'UDP', - }, - { - port: 3480, - protocol: 'UDP', - }, - { - port: 4443, - protocol: 'TCP', - }, - { - port: 5004, - protocol: 'UDP', - }, - { - port: 5061, - protocol: 'TCP', - }, - { - port: 6881, - protocol: 'TCP', - }, - { - port: 8080, - protocol: 'TCP', - }, - { - port: 8443, - protocol: 'TCP', - }, - { - port: 8801, - protocol: 'UDP', - }, - { - port: 9000, - protocol: 'UDP', - }, - { - port: 9989, - protocol: 'TCP', - }, - { - port: 19305, - protocol: 'UDP', - }, - { - port: 35253, - protocol: 'TCP', - }, - { - port: 49882, - protocol: 'UDP', - }, - { - port: 50002, - protocol: 'UDP', - }, - { - port: 55555, - protocol: 'TCP', - }, - { - port: 55556, - protocol: 'TCP', - }, - { - port: 55557, - protocol: 'TCP', - }, - { - port: 56565, - protocol: 'TCP', - }, - { - port: 56566, - protocol: 'TCP', - }, - { - port: 62065, - protocol: 'UDP', - }, - { - port: 63308, - protocol: 'UDP', - }, - ], - }, - ], - }, -} - From 3d94eb8b6d07a5c57be607993f7ad5bde99c7848 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 9 Oct 2024 14:45:55 -0600 Subject: [PATCH 23/26] Flattens arrays for experiment imports in system.jsonnet The imports for experiments are now an array consisting of the experiment DaemonSet definition and it's corresponsing MultiNetworkPolicy definition. --- system.jsonnet | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/system.jsonnet b/system.jsonnet index 479ba1a9..892b0073 100644 --- a/system.jsonnet +++ b/system.jsonnet @@ -18,11 +18,15 @@ import 'k8s/daemonsets/core/host.jsonnet', import 'k8s/daemonsets/core/multi-networkpolicy.jsonnet', import 'k8s/daemonsets/core/node-exporter.jsonnet', + ] + std.flattenArrays([ + import 'k8s/daemonsets/experiments/msak.jsonnet', import 'k8s/daemonsets/experiments/ndt.jsonnet', + import 'k8s/daemonsets/experiments/revtr.jsonnet', + import 'k8s/daemonsets/experiments/wehe.jsonnet', + import 'k8s/daemonsets/experiments/neubot.jsonnet', + ]) + [ import 'k8s/daemonsets/experiments/ndt-virtual.jsonnet', import 'k8s/daemonsets/experiments/ndt-canary.jsonnet', - import 'k8s/daemonsets/experiments/neubot.jsonnet', - import 'k8s/daemonsets/experiments/revtr.jsonnet', import 'k8s/daemonsets/experiments/packet-test.jsonnet', // NetworkPolicies import 'k8s/networkpolicies/msak.jsonnet', @@ -38,10 +42,7 @@ // uncomment the following line. //import 'k8s/daemonsets/experiments/responsiveness.jsonnet', ] else [] - ) + [ - import 'k8s/daemonsets/experiments/msak.jsonnet', - import 'k8s/daemonsets/experiments/wehe.jsonnet', - ] + ( + ) + ( if std.extVar('PROJECT_ID') != 'mlab-oti' then [ // A internal Google service we are experimenting with only in sandbox // and staging. From 3793317d2c2b83e2322d4dbac28a99a1fcd7e87d Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 9 Oct 2024 14:51:48 -0600 Subject: [PATCH 24/26] Removes import of now nonexistent static MultiNetworkPolicies --- system.jsonnet | 6 ------ 1 file changed, 6 deletions(-) diff --git a/system.jsonnet b/system.jsonnet index 892b0073..6901ae81 100644 --- a/system.jsonnet +++ b/system.jsonnet @@ -28,12 +28,6 @@ import 'k8s/daemonsets/experiments/ndt-virtual.jsonnet', import 'k8s/daemonsets/experiments/ndt-canary.jsonnet', import 'k8s/daemonsets/experiments/packet-test.jsonnet', - // NetworkPolicies - import 'k8s/networkpolicies/msak.jsonnet', - import 'k8s/networkpolicies/ndt.jsonnet', - import 'k8s/networkpolicies/revtr.jsonnet', - import 'k8s/networkpolicies/wehe.jsonnet', - import 'k8s/networkpolicies/neubot.jsonnet', ] + ( if std.extVar('PROJECT_ID') == 'mlab-sandbox' then [ // responsiveness commented out by Kinkade. It's stuck in sandbox and not From bdf364cc5ce701511bb94c4fdb3006edb82a3bb1 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 9 Oct 2024 15:01:45 -0600 Subject: [PATCH 25/26] Casts endPort string to an int The MultiNetworkPolicy CRD schema doesn't allow for that field to be a string, unlike "port", which is odd, but whatever. --- k8s/daemonsets/templates.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/daemonsets/templates.jsonnet b/k8s/daemonsets/templates.jsonnet index 345fed64..14c33766 100644 --- a/k8s/daemonsets/templates.jsonnet +++ b/k8s/daemonsets/templates.jsonnet @@ -758,7 +758,7 @@ local MultiNetworkPolicy(expName, index, ports) = { ports: [ { port: std.split(std.split(p, '/')[0], ':')[0], - [if std.length(std.split(p, ':')) == 2 then 'endPort']: std.split(std.split(p, '/')[0], ':')[1], + [if std.length(std.split(p, ':')) == 2 then 'endPort']: std.parseInt(std.split(std.split(p, '/')[0], ':')[1]), protocol: std.split(p, '/')[1], }, for p in ports From 4e48d2df6c3f8a2d9ccc1069e90e81fffd43a1b5 Mon Sep 17 00:00:00 2001 From: Nathan Kinkade Date: Wed, 9 Oct 2024 16:07:03 -0600 Subject: [PATCH 26/26] Adds additional comments clarify custom iptables rules --- config/multi-networkpolicy.jsonnet | 7 +++++-- k8s/daemonsets/core/multi-networkpolicy.jsonnet | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/config/multi-networkpolicy.jsonnet b/config/multi-networkpolicy.jsonnet index 2623d7b0..b25d2120 100644 --- a/config/multi-networkpolicy.jsonnet +++ b/config/multi-networkpolicy.jsonnet @@ -10,7 +10,9 @@ }, }, // Add custom iptables rules below. The rules will not be applied unless you - // pass at least one of the following flags to multi-networkpolicy: + // pass at least one of the following flags to multi-networkpolicy-iptables + // in the multi-networkpolicy DaemonSet in + // k8s/daemonsets/core/multi-networkpolicy.jsonnet: // // --custom-v4-igress-rule-file // --custom-v4-egress-rule-file @@ -18,7 +20,8 @@ // --custom-v4-egress-rule-file // // Add iptables rules one per line in the appropriate sections below, minus - // "iptables -A " as that is added for you by multi-networkpolicy. + // "iptables -A " as that is added for you by + // multi-networkpolicy-iptables. data: { 'custom-v4-ingress-rules.txt': ||| # No custom rules, yet. diff --git a/k8s/daemonsets/core/multi-networkpolicy.jsonnet b/k8s/daemonsets/core/multi-networkpolicy.jsonnet index 21caa5de..8be08074 100644 --- a/k8s/daemonsets/core/multi-networkpolicy.jsonnet +++ b/k8s/daemonsets/core/multi-networkpolicy.jsonnet @@ -54,6 +54,14 @@ '--host-prefix=/host', '--network-plugins=netctl,ipvlan', '--pod-iptables=/var/lib/multi-networkpolicy/iptables', + // If any custom iptables rules are needed that cannot be + // provisioned by MultiNetworkPolicy definitions, then you can + // add them to the file configs/multi-networkpolicy.jsonnet in + // this repo, and uncomment the following flags as necessary: + // '--custom-v4-ingress-rule-file=/etc/multi-networkpolicy/rules/custom-v4-ingress-rules.txt', + // '--custom-v6-ingress-rule-file=/etc/multi-networkpolicy/rules/custom-v6-ingress-rules.txt', + // '--custom-v4-egress-rule-file=/etc/multi-networkpolicy/rules/custom-v4-egress-rules.txt', + // '--custom-v6-egress-rule-file=/etc/multi-networkpolicy/rules/custom-v6-egress-rules.txt', ], resources: { requests: {