diff --git a/virtcontainers/bridgedmacvlan_endpoint.go b/virtcontainers/bridgedmacvlan_endpoint.go index d2efd56f53..3856ea40dc 100644 --- a/virtcontainers/bridgedmacvlan_endpoint.go +++ b/virtcontainers/bridgedmacvlan_endpoint.go @@ -9,6 +9,7 @@ import ( "fmt" "github.com/containernetworking/plugins/pkg/ns" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" ) // BridgedMacvlanEndpoint represents a macvlan endpoint that is bridged to the VM @@ -115,3 +116,52 @@ func (endpoint *BridgedMacvlanEndpoint) HotAttach(h hypervisor) error { func (endpoint *BridgedMacvlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot detach") } + +func (endpoint *BridgedMacvlanEndpoint) save() (s persistapi.NetworkEndpoint) { + s.Type = string(endpoint.Type()) + s.BridgedMacvlan = &persistapi.BridgedMacvlanEndpoint{ + NetPair: persistapi.NetworkInterfacePair{ + TapInterface: persistapi.TapInterface{ + ID: endpoint.NetPair.TapInterface.ID, + Name: endpoint.NetPair.TapInterface.Name, + TAPIface: persistapi.NetworkInterface{ + Name: endpoint.NetPair.TapInterface.TAPIface.Name, + HardAddr: endpoint.NetPair.TapInterface.TAPIface.HardAddr, + Addrs: endpoint.NetPair.TapInterface.TAPIface.Addrs, + }, + }, + VirtIface: persistapi.NetworkInterface{ + Name: endpoint.NetPair.VirtIface.Name, + HardAddr: endpoint.NetPair.VirtIface.HardAddr, + Addrs: endpoint.NetPair.VirtIface.Addrs, + }, + NetInterworkingModel: int(endpoint.NetPair.NetInterworkingModel), + }, + } + return +} + +func (endpoint *BridgedMacvlanEndpoint) load(s persistapi.NetworkEndpoint) { + endpoint.EndpointType = BridgedMacvlanEndpointType + + if s.BridgedMacvlan != nil { + iface := s.BridgedMacvlan + endpoint.NetPair = NetworkInterfacePair{ + TapInterface: TapInterface{ + ID: iface.NetPair.TapInterface.ID, + Name: iface.NetPair.TapInterface.Name, + TAPIface: NetworkInterface{ + Name: iface.NetPair.TapInterface.TAPIface.Name, + HardAddr: iface.NetPair.TapInterface.TAPIface.HardAddr, + Addrs: iface.NetPair.TapInterface.TAPIface.Addrs, + }, + }, + VirtIface: NetworkInterface{ + Name: iface.NetPair.VirtIface.Name, + HardAddr: iface.NetPair.VirtIface.HardAddr, + Addrs: iface.NetPair.VirtIface.Addrs, + }, + NetInterworkingModel: NetInterworkingModel(iface.NetPair.NetInterworkingModel), + } + } +} diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go index 978da2dc10..da9443e983 100644 --- a/virtcontainers/endpoint.go +++ b/virtcontainers/endpoint.go @@ -7,6 +7,8 @@ package virtcontainers import ( "fmt" + + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" ) // Endpoint represents a physical or virtual network interface. @@ -24,6 +26,9 @@ type Endpoint interface { Detach(netNsCreated bool, netNsPath string) error HotAttach(h hypervisor) error HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error + + save() persistapi.NetworkEndpoint + load(persistapi.NetworkEndpoint) } // EndpointType identifies the type of the network endpoint. diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index f5e8b8a705..2ef5ffc594 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -324,11 +324,13 @@ func (fc *firecracker) fcInit(timeout int) error { // Fetch sandbox network to be able to access it from the sandbox structure. var networkNS NetworkNamespace - if err := fc.store.Load(store.Network, &networkNS); err == nil { - if networkNS.NetNsPath == "" { - fc.Logger().WithField("NETWORK NAMESPACE NULL", networkNS).Warn() + if fc.store != nil { + if err := fc.store.Load(store.Network, &networkNS); err == nil { + if networkNS.NetNsPath == "" { + fc.Logger().WithField("NETWORK NAMESPACE NULL", networkNS).Warn() + } + fc.netNSPath = networkNS.NetNsPath } - fc.netNSPath = networkNS.NetNsPath } err := os.MkdirAll(fc.jailerRoot, store.DirMode) diff --git a/virtcontainers/ipvlan_endpoint.go b/virtcontainers/ipvlan_endpoint.go index b71bbd0fbe..7e84f42d8c 100644 --- a/virtcontainers/ipvlan_endpoint.go +++ b/virtcontainers/ipvlan_endpoint.go @@ -9,6 +9,7 @@ import ( "fmt" "github.com/containernetworking/plugins/pkg/ns" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" ) // IPVlanEndpoint represents a ipvlan endpoint that is bridged to the VM @@ -118,3 +119,52 @@ func (endpoint *IPVlanEndpoint) HotAttach(h hypervisor) error { func (endpoint *IPVlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { return fmt.Errorf("IPVlanEndpoint does not support Hot detach") } + +func (endpoint *IPVlanEndpoint) save() (s persistapi.NetworkEndpoint) { + s.Type = string(endpoint.Type()) + s.IPVlan = &persistapi.IPVlanEndpoint{ + NetPair: persistapi.NetworkInterfacePair{ + TapInterface: persistapi.TapInterface{ + ID: endpoint.NetPair.TapInterface.ID, + Name: endpoint.NetPair.TapInterface.Name, + TAPIface: persistapi.NetworkInterface{ + Name: endpoint.NetPair.TapInterface.TAPIface.Name, + HardAddr: endpoint.NetPair.TapInterface.TAPIface.HardAddr, + Addrs: endpoint.NetPair.TapInterface.TAPIface.Addrs, + }, + }, + VirtIface: persistapi.NetworkInterface{ + Name: endpoint.NetPair.VirtIface.Name, + HardAddr: endpoint.NetPair.VirtIface.HardAddr, + Addrs: endpoint.NetPair.VirtIface.Addrs, + }, + NetInterworkingModel: int(endpoint.NetPair.NetInterworkingModel), + }, + } + return +} + +func (endpoint *IPVlanEndpoint) load(s persistapi.NetworkEndpoint) { + endpoint.EndpointType = IPVlanEndpointType + + if s.IPVlan != nil { + iface := s.IPVlan + endpoint.NetPair = NetworkInterfacePair{ + TapInterface: TapInterface{ + ID: iface.NetPair.TapInterface.ID, + Name: iface.NetPair.TapInterface.Name, + TAPIface: NetworkInterface{ + Name: iface.NetPair.TapInterface.TAPIface.Name, + HardAddr: iface.NetPair.TapInterface.TAPIface.HardAddr, + Addrs: iface.NetPair.TapInterface.TAPIface.Addrs, + }, + }, + VirtIface: NetworkInterface{ + Name: iface.NetPair.VirtIface.Name, + HardAddr: iface.NetPair.VirtIface.HardAddr, + Addrs: iface.NetPair.VirtIface.Addrs, + }, + NetInterworkingModel: NetInterworkingModel(iface.NetPair.NetInterworkingModel), + } + } +} diff --git a/virtcontainers/macvtap_endpoint.go b/virtcontainers/macvtap_endpoint.go index d35d9c70cb..2eb708b205 100644 --- a/virtcontainers/macvtap_endpoint.go +++ b/virtcontainers/macvtap_endpoint.go @@ -8,6 +8,8 @@ package virtcontainers import ( "fmt" "os" + + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" ) // MacvtapEndpoint represents a macvtap endpoint @@ -102,3 +104,17 @@ func (endpoint *MacvtapEndpoint) SetPciAddr(pciAddr string) { func (endpoint *MacvtapEndpoint) NetworkPair() *NetworkInterfacePair { return nil } + +func (endpoint *MacvtapEndpoint) save() (s persistapi.NetworkEndpoint) { + s.Type = string(endpoint.Type()) + s.Macvtap = &persistapi.MacvtapEndpoint{ + PCIAddr: endpoint.PCIAddr, + } + return +} +func (endpoint *MacvtapEndpoint) load(s persistapi.NetworkEndpoint) { + endpoint.EndpointType = MacvtapEndpointType + if s.Macvtap != nil { + endpoint.PCIAddr = s.Macvtap.PCIAddr + } +} diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go index ccf9f00557..3569c52d5c 100644 --- a/virtcontainers/persist.go +++ b/virtcontainers/persist.go @@ -160,6 +160,17 @@ func (s *Sandbox) dumpAgent(ss *persistapi.SandboxState) { } } +func (s *Sandbox) dumpNetwork(ss *persistapi.SandboxState) { + ss.Network = persistapi.NetworkInfo{ + NetNsPath: s.networkNS.NetNsPath, + NetmonPID: s.networkNS.NetmonPID, + NetNsCreated: s.networkNS.NetNsCreated, + } + for _, e := range s.networkNS.Endpoints { + ss.Network.Endpoints = append(ss.Network.Endpoints, e.save()) + } +} + func (s *Sandbox) Save() error { var ( ss = persistapi.SandboxState{} @@ -173,6 +184,7 @@ func (s *Sandbox) Save() error { s.dumpProcess(cs) s.dumpMounts(cs) s.dumpAgent(&ss) + s.dumpNetwork(&ss) if err := s.newStore.ToDisk(ss, cs); err != nil { return err @@ -248,6 +260,38 @@ func (c *Container) loadContProcess(cs persistapi.ContainerState) { } } +func (s *Sandbox) loadNetwork(netInfo persistapi.NetworkInfo) { + s.networkNS = NetworkNamespace{ + NetNsPath: netInfo.NetNsPath, + NetmonPID: netInfo.NetmonPID, + NetNsCreated: netInfo.NetNsCreated, + } + + for _, e := range netInfo.Endpoints { + var ep Endpoint + switch EndpointType(e.Type) { + case PhysicalEndpointType: + ep = &PhysicalEndpoint{} + case VethEndpointType: + ep = &VethEndpoint{} + case VhostUserEndpointType: + ep = &VhostUserEndpoint{} + case BridgedMacvlanEndpointType: + ep = &BridgedMacvlanEndpoint{} + case MacvtapEndpointType: + ep = &MacvtapEndpoint{} + case TapEndpointType: + ep = &TapEndpoint{} + case IPVlanEndpointType: + ep = &IPVlanEndpoint{} + default: + continue + } + ep.load(e) + s.networkNS.Endpoints = append(s.networkNS.Endpoints, ep) + } +} + // Restore will restore sandbox data from persist file on disk func (s *Sandbox) Restore() error { ss, _, err := s.newStore.FromDisk(s.id) @@ -259,6 +303,7 @@ func (s *Sandbox) Restore() error { s.loadHypervisor(ss.HypervisorState) s.loadDevices(ss.Devices) s.loadAgent(ss.AgentState) + s.loadNetwork(ss.Network) return nil } diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go index c9744682db..2efebf978b 100644 --- a/virtcontainers/persist/api/network.go +++ b/virtcontainers/persist/api/network.go @@ -6,27 +6,86 @@ package persistapi +import ( + "github.com/vishvananda/netlink" +) + // ============= sandbox level resources ============= +type NetworkInterface struct { + Name string + HardAddr string + Addrs []netlink.Addr +} + +// TapInterface defines a tap interface +type TapInterface struct { + ID string + Name string + TAPIface NetworkInterface + // remove VMFds and VhostFds +} + +// NetworkInterfacePair defines a pair between VM and virtual network interfaces. +type NetworkInterfacePair struct { + TapInterface + VirtIface NetworkInterface + NetInterworkingModel int +} + +type PhysicalEndpoint struct { + BDF string + Driver string + VendorDeviceID string +} + +type MacvtapEndpoint struct { + // This is for showing information. + // Remove this field won't impact anything. + PCIAddr string +} + +type TapEndpoint struct { + TapInterface TapInterface +} + +type BridgedMacvlanEndpoint struct { + NetPair NetworkInterfacePair +} + +type VethEndpoint struct { + NetPair NetworkInterfacePair +} + +type IPVlanEndpoint struct { + NetPair NetworkInterfacePair +} + +type VhostUserEndpoint struct { + // This is for showing information. + // Remove these fields won't impact anything. + IfaceName string + PCIAddr string +} + // NetworkEndpoint contains network interface information type NetworkEndpoint struct { Type string - // ID used to pass the netdev option to qemu - ID string - - // Name of the interface - Name string - - // Index of interface - Index int + // One and only one of these below are not nil according to Type. + Physical *PhysicalEndpoint `json:",omitempty"` + Veth *VethEndpoint `json:",omitempty"` + VhostUser *VhostUserEndpoint `json:",omitempty"` + BridgedMacvlan *BridgedMacvlanEndpoint `json:",omitempty"` + Macvtap *MacvtapEndpoint `json:",omitempty"` + Tap *TapEndpoint `json:",omitempty"` + IPVlan *IPVlanEndpoint `json:",omitempty"` } // NetworkInfo contains network information of sandbox type NetworkInfo struct { - NetNsPath string - NetmonPID int - NetNsCreated bool - InterworkingModel string - Endpoints []NetworkEndpoint + NetNsPath string + NetmonPID int + NetNsCreated bool + Endpoints []NetworkEndpoint } diff --git a/virtcontainers/physical_endpoint.go b/virtcontainers/physical_endpoint.go index 989233f158..90c4593b3a 100644 --- a/virtcontainers/physical_endpoint.go +++ b/virtcontainers/physical_endpoint.go @@ -14,6 +14,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/safchain/ethtool" ) @@ -200,3 +201,22 @@ func bindNICToVFIO(endpoint *PhysicalEndpoint) error { func bindNICToHost(endpoint *PhysicalEndpoint) error { return drivers.BindDevicetoHost(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID) } + +func (endpoint *PhysicalEndpoint) save() (s persistapi.NetworkEndpoint) { + s.Type = string(endpoint.Type()) + s.Physical = &persistapi.PhysicalEndpoint{ + BDF: endpoint.BDF, + Driver: endpoint.Driver, + VendorDeviceID: endpoint.VendorDeviceID, + } + return +} + +func (endpoint *PhysicalEndpoint) load(s persistapi.NetworkEndpoint) { + endpoint.EndpointType = PhysicalEndpointType + if s.Physical != nil { + endpoint.BDF = s.Physical.BDF + endpoint.Driver = s.Physical.Driver + endpoint.VendorDeviceID = s.Physical.VendorDeviceID + } +} diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 8a1d174c37..82f5157b79 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -467,18 +467,18 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac s.Logger().WithField("features", s.config.Experimental).Infof("Enable experimental features") } - // Fetch sandbox network to be able to access it from the sandbox structure. - var networkNS NetworkNamespace - if err := s.store.Load(store.Network, &networkNS); err == nil { - s.networkNS = networkNS - } - if s.supportNewStore() { // Restored successfully from newstore before. if s.state.State != "" { return s, nil } } else { + // Fetch sandbox network to be able to access it from the sandbox structure. + var networkNS NetworkNamespace + if err := s.store.Load(store.Network, &networkNS); err == nil { + s.networkNS = networkNS + } + devices, err := s.store.LoadDevices() if err != nil { s.Logger().WithError(err).WithField("sandboxid", s.id).Warning("load sandbox devices failed") @@ -563,11 +563,6 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor if err != nil { s.Logger().WithError(err).WithField("sandboxid", s.id).Error("Create new sandbox failed") globalSandboxList.removeSandbox(s.id) - } - }() - - defer func() { - if err != nil { s.store.Delete() } }() @@ -846,7 +841,10 @@ func (s *Sandbox) createNetwork() error { } // Store the network - return s.store.Store(store.Network, s.networkNS) + if !s.supportNewStore() { + return s.store.Store(store.Network, s.networkNS) + } + return nil } func (s *Sandbox) postCreatedNetwork() error { @@ -919,14 +917,14 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro // Update the sandbox storage s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint) - if err := s.store.Store(store.Network, s.networkNS); err != nil { - return nil, err - } - if s.supportNewStore() { if err := s.Save(); err != nil { return nil, err } + } else { + if err := s.store.Store(store.Network, s.networkNS); err != nil { + return nil, err + } } // Add network for vm @@ -943,14 +941,15 @@ func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, e return inf, err } s.networkNS.Endpoints = append(s.networkNS.Endpoints[:i], s.networkNS.Endpoints[i+1:]...) - if err := s.store.Store(store.Network, s.networkNS); err != nil { - return inf, err - } if s.supportNewStore() { if err := s.Save(); err != nil { return inf, err } + } else { + if err := s.store.Store(store.Network, s.networkNS); err != nil { + return inf, err + } } break @@ -1024,8 +1023,11 @@ func (s *Sandbox) startVM() (err error) { return err } } - if err := s.store.Store(store.Network, s.networkNS); err != nil { - return err + + if !s.supportNewStore() { + if err := s.store.Store(store.Network, s.networkNS); err != nil { + return err + } } } diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go index c3cd3c34cc..2a59a27615 100644 --- a/virtcontainers/tap_endpoint.go +++ b/virtcontainers/tap_endpoint.go @@ -11,6 +11,7 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/vishvananda/netlink" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" ) @@ -187,3 +188,35 @@ func unTapNetwork(name string) error { } return nil } + +func (endpoint *TapEndpoint) save() (s persistapi.NetworkEndpoint) { + s.Type = string(endpoint.Type()) + s.Tap = &persistapi.TapEndpoint{ + TapInterface: persistapi.TapInterface{ + ID: endpoint.TapInterface.ID, + Name: endpoint.TapInterface.Name, + TAPIface: persistapi.NetworkInterface{ + Name: endpoint.TapInterface.TAPIface.Name, + HardAddr: endpoint.TapInterface.TAPIface.HardAddr, + Addrs: endpoint.TapInterface.TAPIface.Addrs, + }, + }, + } + return +} +func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) { + endpoint.EndpointType = TapEndpointType + + if s.Tap != nil { + iface := s.Tap + endpoint.TapInterface = TapInterface{ + ID: iface.TapInterface.ID, + Name: iface.TapInterface.Name, + TAPIface: NetworkInterface{ + Name: iface.TapInterface.TAPIface.Name, + HardAddr: iface.TapInterface.TAPIface.HardAddr, + Addrs: iface.TapInterface.TAPIface.Addrs, + }, + } + } +} diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go index cafb454c10..d57110ea44 100644 --- a/virtcontainers/veth_endpoint.go +++ b/virtcontainers/veth_endpoint.go @@ -9,6 +9,7 @@ import ( "fmt" "github.com/containernetworking/plugins/pkg/ns" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" ) // VethEndpoint gathers a network pair and its properties. @@ -141,3 +142,52 @@ func (endpoint *VethEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPa } return nil } + +func (endpoint *VethEndpoint) save() (s persistapi.NetworkEndpoint) { + s.Type = string(endpoint.Type()) + s.Veth = &persistapi.VethEndpoint{ + NetPair: persistapi.NetworkInterfacePair{ + TapInterface: persistapi.TapInterface{ + ID: endpoint.NetPair.TapInterface.ID, + Name: endpoint.NetPair.TapInterface.Name, + TAPIface: persistapi.NetworkInterface{ + Name: endpoint.NetPair.TapInterface.TAPIface.Name, + HardAddr: endpoint.NetPair.TapInterface.TAPIface.HardAddr, + Addrs: endpoint.NetPair.TapInterface.TAPIface.Addrs, + }, + }, + VirtIface: persistapi.NetworkInterface{ + Name: endpoint.NetPair.VirtIface.Name, + HardAddr: endpoint.NetPair.VirtIface.HardAddr, + Addrs: endpoint.NetPair.VirtIface.Addrs, + }, + NetInterworkingModel: int(endpoint.NetPair.NetInterworkingModel), + }, + } + return +} + +func (endpoint *VethEndpoint) load(s persistapi.NetworkEndpoint) { + endpoint.EndpointType = VethEndpointType + + if s.Veth != nil { + iface := s.Veth + endpoint.NetPair = NetworkInterfacePair{ + TapInterface: TapInterface{ + ID: iface.NetPair.TapInterface.ID, + Name: iface.NetPair.TapInterface.Name, + TAPIface: NetworkInterface{ + Name: iface.NetPair.TapInterface.TAPIface.Name, + HardAddr: iface.NetPair.TapInterface.TAPIface.HardAddr, + Addrs: iface.NetPair.TapInterface.TAPIface.Addrs, + }, + }, + VirtIface: NetworkInterface{ + Name: iface.NetPair.VirtIface.Name, + HardAddr: iface.NetPair.VirtIface.HardAddr, + Addrs: iface.NetPair.VirtIface.Addrs, + }, + NetInterworkingModel: NetInterworkingModel(iface.NetPair.NetInterworkingModel), + } + } +} diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/vhostuser_endpoint.go index 4960dba1be..9dc6967845 100644 --- a/virtcontainers/vhostuser_endpoint.go +++ b/virtcontainers/vhostuser_endpoint.go @@ -11,6 +11,7 @@ import ( "os" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -149,3 +150,20 @@ func vhostUserSocketPath(info interface{}) (string, error) { } } + +func (endpoint *VhostUserEndpoint) save() (s persistapi.NetworkEndpoint) { + s.Type = string(endpoint.Type()) + s.VhostUser = &persistapi.VhostUserEndpoint{ + IfaceName: endpoint.IfaceName, + PCIAddr: endpoint.PCIAddr, + } + return +} + +func (endpoint *VhostUserEndpoint) load(s persistapi.NetworkEndpoint) { + endpoint.EndpointType = VhostUserEndpointType + if s.VhostUser != nil { + endpoint.IfaceName = s.VhostUser.IfaceName + endpoint.PCIAddr = s.VhostUser.PCIAddr + } +}