diff --git a/go.mod b/go.mod index cfa76cf878e..354c19dba1a 100644 --- a/go.mod +++ b/go.mod @@ -147,7 +147,3 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect sigs.k8s.io/yaml v1.2.0 // indirect ) - -// hcshim repo is modifed to add "AdditionalParams" field to HNSEndpoint struct. -// We will use this replace before pushing the change to hcshim upstream repo. -replace github.com/Microsoft/hcsshim v0.8.9 => github.com/ruicao93/hcsshim v0.8.10-0.20210114035434-63fe00c1b9aa diff --git a/go.sum b/go.sum index fe3d355e78e..8b238d5690a 100644 --- a/go.sum +++ b/go.sum @@ -42,9 +42,12 @@ github.com/Mellanox/sriovnet v1.0.2 h1:VTQHD7OHU6QejTtclt5a2obDfsW1ATRxTCgZmsiKm github.com/Mellanox/sriovnet v1.0.2/go.mod h1:pXdSZwahlvP0Xn8nuXcVthBE38Nqf2czo449p5ALLXY= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331 h1:3YnB7Hpmh1lPecPE8doMOtYCrMdrpedZOvxfuNES/Vk= github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.9 h1:VrfodqvztU8YSOvygU+DN1BGaSGxmrNfqOv5oOuX2Bk= +github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= @@ -111,6 +114,7 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/confluentinc/bincover v0.1.0 h1:M4Gfj4rCXuUQVe8TqT/VXcAMjLyvN81oDRy79fjSv3o= github.com/confluentinc/bincover v0.1.0/go.mod h1:qeI1wx0RxdGTZtrJY0HVlgJ4NqC/X2Z+fHbvy87tgHE= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59 h1:qWj4qVYZ95vLWwqyNJCQg7rDsG5wPdze0UaPolH7DUk= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= @@ -258,6 +262,7 @@ github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85n github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -516,6 +521,7 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -574,8 +580,6 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/ruicao93/hcsshim v0.8.10-0.20210114035434-63fe00c1b9aa h1:hHmWLNqmjp+23XdtiBZOvDf5iXze75ypta1xe/eHaW4= -github.com/ruicao93/hcsshim v0.8.10-0.20210114035434-63fe00c1b9aa/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -642,6 +646,7 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= @@ -817,6 +822,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -855,7 +861,6 @@ golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201017003518-b09fb700fbb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/agent/cniserver/interface_configuration_linux.go b/pkg/agent/cniserver/interface_configuration_linux.go index c0a4623b1aa..2d86a771a3c 100644 --- a/pkg/agent/cniserver/interface_configuration_linux.go +++ b/pkg/agent/cniserver/interface_configuration_linux.go @@ -590,6 +590,6 @@ func isVeth(link netlink.Link) bool { return isVeth } -func (ic *ifConfigurator) getOVSInterfaceType() int { +func (ic *ifConfigurator) getOVSInterfaceType(ovsPortName string) int { return defaultOVSInterfaceType } diff --git a/pkg/agent/cniserver/interface_configuration_windows.go b/pkg/agent/cniserver/interface_configuration_windows.go index 81538da8243..a400e7eb673 100644 --- a/pkg/agent/cniserver/interface_configuration_windows.go +++ b/pkg/agent/cniserver/interface_configuration_windows.go @@ -29,15 +29,12 @@ import ( "github.com/Microsoft/hcsshim/hcn" cnitypes "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/cni/pkg/types/current" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" - "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/util" cnipb "antrea.io/antrea/pkg/apis/cni/v1beta1" "antrea.io/antrea/pkg/ovs/ovsconfig" - "antrea.io/antrea/pkg/util/k8s" ) const ( @@ -179,44 +176,13 @@ func (ic *ifConfigurator) createContainerLink(endpointName string, result *curre if err != nil { return nil, err } - containerIPStr, err := parseContainerIPs(result.IPs) - if err != nil { - klog.Errorf("Failed to find container %s IP", containerID) - } - // Save interface config to HNSEndpoint. It's used for creating missing OVS - // ports during antrea-agent boot stage. The change is introduced mainly for - // Containerd support. When working with Containerd runtime, antrea-agent creates - // OVS ports in an asynchronous way. So the OVS ports can be lost if antrea-agent - // gets stopped/restarted before port creation completes. - // - // The interface config will be rebuilt based on the params saved in the "AdditionalParams" - // field of HNSEndpoint. - // - endpointName: the name of the host interface without Hyper-V prefix(vEthernet). - // The name is same as the OVS port name and HNSEndpoint name. - // - containerID: used as the goroutine lock to avoid concurrency issue. - // - podName and PodNamespace: Used to identify the owner of the HNSEndpoint. - // - Other params will be passed to OVS port. - ifaceConfig := interfacestore.NewContainerInterface( - endpointName, - containerID, - podName, - podNamespace, - nil, - containerIPStr) - ovsAttachInfoData := BuildOVSPortExternalIDs(ifaceConfig) - ovsAttachInfo := make(map[string]string) - for k, v := range ovsAttachInfoData { - valueStr, _ := v.(string) - ovsAttachInfo[k] = valueStr - } epRequest := &hcsshim.HNSEndpoint{ - Name: endpointName, - VirtualNetwork: ic.hnsNetwork.Id, - DNSServerList: strings.Join(result.DNS.Nameservers, ","), - DNSSuffix: strings.Join(result.DNS.Search, ","), - GatewayAddress: containerIP.Gateway.String(), - IPAddress: containerIP.Address.IP, - AdditionalParams: ovsAttachInfo, + Name: endpointName, + VirtualNetwork: ic.hnsNetwork.Id, + DNSServerList: strings.Join(result.DNS.Nameservers, ","), + DNSSuffix: strings.Join(result.DNS.Search, ","), + GatewayAddress: containerIP.Gateway.String(), + IPAddress: containerIP.Address.IP, } hnsEP, err := epRequest.Create() if err != nil { @@ -227,40 +193,6 @@ func (ic *ifConfigurator) createContainerLink(endpointName string, result *curre return hnsEP, nil } -func (ic *ifConfigurator) getInterfaceConfigForPods(pods sets.String) map[string]*interfacestore.InterfaceConfig { - interfaces := make(map[string]*interfacestore.InterfaceConfig) - ic.epCache.Range(func(key, value interface{}) bool { - ep, _ := value.(*hcsshim.HNSEndpoint) - ifConfig := parseOVSPortInterfaceConfigFromHNSEndpoint(ep) - if ifConfig == nil { - return true - } - namespacedName := k8s.NamespacedName(ifConfig.PodNamespace, ifConfig.PodName) - if pods.Has(namespacedName) { - interfaces[namespacedName] = ifConfig - } - return true - }) - return interfaces -} - -func parseOVSPortInterfaceConfigFromHNSEndpoint(ep *hcsshim.HNSEndpoint) *interfacestore.InterfaceConfig { - portData := &ovsconfig.OVSPortData{ - Name: ep.Name, - ExternalIDs: ep.AdditionalParams, - } - ifaceConfig := ParseOVSPortInterfaceConfig(portData, nil, false) - if ifaceConfig != nil { - var err error - ifaceConfig.MAC, err = net.ParseMAC(ep.MacAddress) - if err != nil { - klog.Errorf("Failed to parse MAC address from HNSEndpoint %s: %v", ep.MacAddress, err) - return nil - } - } - return ifaceConfig -} - // attachContainerLink takes the result of the IPAM plugin, and adds the appropriate IP // addresses and routes to the interface. // For different CRI runtimes we need to use the appropriate Windows container API: @@ -515,7 +447,11 @@ func (ic *ifConfigurator) getInterceptedInterfaces( } // getOVSInterfaceType returns "internal". Windows uses internal OVS interface for container vNIC. -func (ic *ifConfigurator) getOVSInterfaceType() int { +func (ic *ifConfigurator) getOVSInterfaceType(ovsPortName string) int { + ifaceName := fmt.Sprintf("vEthernet (%s)", ovsPortName) + if !util.HostInterfaceExists(ifaceName) { + return defaultOVSInterfaceType + } return internalOVSInterfaceType } diff --git a/pkg/agent/cniserver/pod_configuration.go b/pkg/agent/cniserver/pod_configuration.go index 3ba7d6349d7..9d4896615e0 100644 --- a/pkg/agent/cniserver/pod_configuration.go +++ b/pkg/agent/cniserver/pod_configuration.go @@ -261,7 +261,7 @@ func (pc *podConfigurator) configureInterfaces( func (pc *podConfigurator) createOVSPort(ovsPortName string, ovsAttachInfo map[string]interface{}) (string, error) { var portUUID string var err error - switch pc.ifConfigurator.getOVSInterfaceType() { + switch pc.ifConfigurator.getOVSInterfaceType(ovsPortName) { case internalOVSInterfaceType: portUUID, err = pc.ovsBridgeClient.CreateInternalPort(ovsPortName, 0, ovsAttachInfo) default: @@ -416,7 +416,14 @@ func (pc *podConfigurator) reconcile(pods []corev1.Pod, containerAccess *contain desiredPods.Insert(k8s.NamespacedName(pod.Namespace, pod.Name)) } + missingIfConfigs := make([]*interfacestore.InterfaceConfig, 0) for _, containerConfig := range knownInterfaces { + // Find the OVS ports which are not connected to host interfaces, this is useful on Windows if the runtime is + // containerd, because the host interface is created async from the OVS port. + if containerConfig.OFPort == -1 { + missingIfConfigs = append(missingIfConfigs, containerConfig) + continue + } namespacedName := k8s.NamespacedName(containerConfig.PodNamespace, containerConfig.PodName) actualPods.Insert(namespacedName) if desiredPods.Has(namespacedName) { @@ -442,9 +449,9 @@ func (pc *podConfigurator) reconcile(pods []corev1.Pod, containerAccess *contain // interface should no longer be in store after the call to removeInterfaces } } - - missingPods := desiredPods.Difference(actualPods) - pc.reconcileMissingPods(missingPods, containerAccess) + if len(missingIfConfigs) > 0 { + pc.reconcileMissingPods(missingIfConfigs, containerAccess) + } return nil } @@ -465,15 +472,9 @@ func (pc *podConfigurator) connectInterfaceToOVSCommon(ovsPortName string, conta }() // GetOFPort will wait for up to 1 second for OVSDB to report the OFPort number. - ofPort, err := pc.ovsBridgeClient.GetOFPort(ovsPortName) + ofPort, err := pc.installPodFlows(ovsPortName, containerConfig) if err != nil { - return fmt.Errorf("failed to get of_port of OVS port %s: %v", ovsPortName, err) - } - - klog.V(2).Infof("Setting up Openflow entries for container %s", containerID) - err = pc.ofClient.InstallPodFlows(ovsPortName, containerConfig.IPs, containerConfig.MAC, uint32(ofPort)) - if err != nil { - return fmt.Errorf("failed to add Openflow entries for container %s: %v", containerID, err) + return err } containerConfig.OVSPortConfig = &interfacestore.OVSPortConfig{PortUUID: portUUID, OFPort: ofPort} // Add containerConfig into local cache @@ -485,6 +486,20 @@ func (pc *podConfigurator) connectInterfaceToOVSCommon(ovsPortName string, conta return nil } +func (pc *podConfigurator) installPodFlows(ovsPortName string, containerConfig *interfacestore.InterfaceConfig) (int32, error) { + // GetOFPort will wait for up to 1 second for OVSDB to report the OFPort number. + ofPort, err := pc.ovsBridgeClient.GetOFPort(ovsPortName) + if err != nil { + return 0, fmt.Errorf("failed to get of_port of OVS port %s: %v", ovsPortName, err) + } + containerID := containerConfig.ContainerID + klog.V(2).Infof("Setting up Openflow entries for container %s", containerID) + if err := pc.ofClient.InstallPodFlows(ovsPortName, containerConfig.IPs, containerConfig.MAC, uint32(ofPort)); err != nil { + return 0, fmt.Errorf("failed to add Openflow entries for container %s: %v", containerID, err) + } + return ofPort, nil +} + // disconnectInterfaceFromOVS disconnects an existing interface from ovs br-int. func (pc *podConfigurator) disconnectInterfaceFromOVS(containerConfig *interfacestore.InterfaceConfig) error { containerID := containerConfig.ContainerID diff --git a/pkg/agent/cniserver/pod_configuration_linux.go b/pkg/agent/cniserver/pod_configuration_linux.go index 961e153a5e1..28ca3b544da 100644 --- a/pkg/agent/cniserver/pod_configuration_linux.go +++ b/pkg/agent/cniserver/pod_configuration_linux.go @@ -19,7 +19,6 @@ package cniserver import ( "github.com/containernetworking/cni/pkg/types/current" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" "antrea.io/antrea/pkg/agent/interfacestore" @@ -41,8 +40,8 @@ func (pc *podConfigurator) connectInterfaceToOVS( return containerConfig, pc.connectInterfaceToOVSCommon(ovsPortName, containerConfig) } -func (pc *podConfigurator) reconcileMissingPods(pods sets.String, containerAccess *containerAccessArbitrator) { - for pod := range pods { +func (pc *podConfigurator) reconcileMissingPods(ifConfigs []*interfacestore.InterfaceConfig, containerAccess *containerAccessArbitrator) { + for i := range ifConfigs { // This should not happen since OVSDB is persisted on the Node. // TODO: is there anything else we should be doing? Assuming that the Pod's // interface still exists, we can repair the interface store since we can @@ -52,6 +51,7 @@ func (pc *podConfigurator) reconcileMissingPods(pods sets.String, containerAcces // we store in the cache, but this ID is not used for anything at the // moment. However, if the interface does not exist, there is nothing we can // do since we do not have the original CNI parameters. - klog.Warningf("Interface for Pod %s not found in the interface store", pod) + ifaceConfig := ifConfigs[i] + klog.Warningf("Interface for Pod %s/%s not found in the interface store", ifaceConfig.PodNamespace, ifaceConfig.PodName) } } diff --git a/pkg/agent/cniserver/pod_configuration_windows.go b/pkg/agent/cniserver/pod_configuration_windows.go index 9e58e84197d..a4ba8a3573d 100644 --- a/pkg/agent/cniserver/pod_configuration_windows.go +++ b/pkg/agent/cniserver/pod_configuration_windows.go @@ -21,11 +21,13 @@ import ( "fmt" "github.com/containernetworking/cni/pkg/types/current" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" "antrea.io/antrea/pkg/agent/interfacestore" + "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/agent/util" + "antrea.io/antrea/pkg/apis/controlplane/v1beta2" + "antrea.io/antrea/pkg/util/k8s" ) // connectInterfaceToOVSAsync waits for an interface to be created and connects it to OVS br-int asynchronously @@ -34,7 +36,21 @@ import ( func (pc *podConfigurator) connectInterfaceToOVSAsync(ifConfig *interfacestore.InterfaceConfig, containerAccess *containerAccessArbitrator) error { ovsPortName := ifConfig.InterfaceName return pc.ifConfigurator.addPostInterfaceCreateHook(ifConfig.ContainerID, ovsPortName, containerAccess, func() error { - return pc.connectInterfaceToOVSCommon(ovsPortName, ifConfig) + err := pc.ovsBridgeClient.SetInterfaceTypeInternal(ovsPortName) + if err != nil { + return err + } + ofPort, ofErr := pc.installPodFlows(ovsPortName, ifConfig) + if ofErr != nil { + return ofErr + } + // Update interface config with the ofPort. + ifConfig.OVSPortConfig.OFPort = ofPort + // Notify the Pod update event to required components. + pc.entityUpdates <- types.EntityReference{ + Pod: &v1beta2.PodReference{Name: ifConfig.PodName, Namespace: ifConfig.PodNamespace}, + } + return nil }) } @@ -62,19 +78,24 @@ func (pc *podConfigurator) connectInterfaceToOVS( if util.HostInterfaceExists(hostIfAlias) { return containerConfig, pc.connectInterfaceToOVSCommon(ovsPortName, containerConfig) } + klog.V(2).Infof("Adding OVS port %s for container %s", ovsPortName, containerID) + ovsAttachInfo := BuildOVSPortExternalIDs(containerConfig) + portUUID, err := pc.createOVSPort(ovsPortName, ovsAttachInfo) + if err != nil { + return nil, err + } + containerConfig.OVSPortConfig = &interfacestore.OVSPortConfig{PortUUID: portUUID} + // Add containerConfig into local cache + pc.ifaceStore.AddInterface(containerConfig) return containerConfig, pc.connectInterfaceToOVSAsync(containerConfig, containerAccess) } -func (pc *podConfigurator) reconcileMissingPods(pods sets.String, containerAccess *containerAccessArbitrator) { - interfacesConfig := pc.ifConfigurator.getInterfaceConfigForPods(pods) - for pod := range pods { - ifaceConfig, ok := interfacesConfig[pod] - if !ok { - klog.Errorf("Failed to reconcile Pod %s: interface config not found", pod) - continue - } +func (pc *podConfigurator) reconcileMissingPods(ifConfigs []*interfacestore.InterfaceConfig, containerAccess *containerAccessArbitrator) { + for i := range ifConfigs { + ifaceConfig := ifConfigs[i] + portName := k8s.NamespacedName(ifaceConfig.PodNamespace, ifaceConfig.PodName) if err := pc.connectInterfaceToOVSAsync(ifaceConfig, containerAccess); err != nil { - klog.Errorf("Failed to reconcile Pod %s: %v", pod, err) + klog.Errorf("Failed to reconcile Pod %s: %v", portName, err) } } } diff --git a/pkg/ovs/ovsconfig/interfaces.go b/pkg/ovs/ovsconfig/interfaces.go index 941f6fa8e63..e869a7aac74 100644 --- a/pkg/ovs/ovsconfig/interfaces.go +++ b/pkg/ovs/ovsconfig/interfaces.go @@ -54,4 +54,5 @@ type OVSBridgeClient interface { GetBridgeName() string IsHardwareOffloadEnabled() bool GetOVSDatapathType() OVSDatapathType + SetInterfaceTypeInternal(name string) Error } diff --git a/pkg/ovs/ovsconfig/ovs_client.go b/pkg/ovs/ovsconfig/ovs_client.go index 8659cd807aa..b7bda7115aa 100644 --- a/pkg/ovs/ovsconfig/ovs_client.go +++ b/pkg/ovs/ovsconfig/ovs_client.go @@ -879,3 +879,41 @@ func (br *OVSBridge) getHardwareOffload() (bool, Error) { func (br *OVSBridge) GetOVSDatapathType() OVSDatapathType { return br.datapathType } + +// SetInterfaceTypeInternal modifies the OVS Interface type as "internal", and return until the ofport is allocated by +// OVS. This function is used on Windows when the Pod interface is created after the OVS port creation. +func (br *OVSBridge) SetInterfaceTypeInternal(name string) Error { + // Update Interface type, and the caller ensures the host Interface exists. + tx1 := br.ovsdb.Transaction(openvSwitchSchema) + tx1.Update(dbtransaction.Update{ + Table: "Interface", + Where: [][]interface{}{{"name", "==", name}}, + Row: map[string]interface{}{ + "type": "internal", + }, + }) + _, err, temporary := tx1.Commit() + if err != nil { + klog.Error("Transaction failed: ", err) + return NewTransactionError(err, temporary) + } + + // Wait until OVS allocated the ofport to the Interface. + tx2 := br.ovsdb.Transaction(openvSwitchSchema) + tx2.Wait(dbtransaction.Wait{ + Table: "Interface", + Timeout: uint64(defaultGetPortTimeout / time.Millisecond), // The unit of timeout is millisecond + Columns: []string{"ofport"}, + Until: "!=", + Rows: []interface{}{map[string]interface{}{ + "ofport": []interface{}{"set", []int32{-1}}, + }}, + Where: [][]interface{}{{"name", "==", name}}, + }) + _, err, temporary = tx2.Commit() + if err != nil { + klog.Error("Failed to allocate ofport after changing Interface type: ", err) + return NewTransactionError(err, temporary) + } + return nil +} diff --git a/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go b/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go index a519d0b6492..be9d327df56 100644 --- a/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go +++ b/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go @@ -409,3 +409,17 @@ func (mr *MockOVSBridgeClientMockRecorder) SetInterfaceOptions(arg0, arg1 interf mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetInterfaceOptions", reflect.TypeOf((*MockOVSBridgeClient)(nil).SetInterfaceOptions), arg0, arg1) } + +// SetInterfaceTypeInternal mocks base method +func (m *MockOVSBridgeClient) SetInterfaceTypeInternal(arg0 string) ovsconfig.Error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetInterfaceTypeInternal", arg0) + ret0, _ := ret[0].(ovsconfig.Error) + return ret0 +} + +// SetInterfaceTypeInternal indicates an expected call of SetInterfaceTypeInternal +func (mr *MockOVSBridgeClientMockRecorder) SetInterfaceTypeInternal(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetInterfaceTypeInternal", reflect.TypeOf((*MockOVSBridgeClient)(nil).SetInterfaceTypeInternal), arg0) +}