diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go index 855571ca97..7fdb831bb1 100644 --- a/virtcontainers/acrn.go +++ b/virtcontainers/acrn.go @@ -17,6 +17,7 @@ import ( "time" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" @@ -230,7 +231,18 @@ func (a *acrn) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto a.store = vcStore a.config = *hypervisorConfig a.arch = newAcrnArch(a.config) - if err = a.store.Load(store.Hypervisor, &a.state); err != nil { + + var create bool + + if a.store != nil { //use old store + if err = a.store.Load(store.Hypervisor, &a.info); err != nil { + create = true + } + } else if a.info.PID == 0 { // new store + create = true + } + + if create { // acrn currently supports only known UUIDs for security // reasons (FuSa). When launching VM, only these pre-defined // UUID should be used else VM launch will fail. acrn team is @@ -246,15 +258,11 @@ func (a *acrn) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto return err } - if err = a.store.Store(store.Hypervisor, a.state); err != nil { + if err = a.storeInfo(); err != nil { return err } } - if err = a.store.Load(store.Hypervisor, &a.info); err != nil { - a.Logger().WithField("function", "setup").WithError(err).Info("No info could be fetched") - } - return nil } @@ -619,3 +627,24 @@ func (a *acrn) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, func (a *acrn) toGrpc() ([]byte, error) { return nil, errors.New("acrn is not supported by VM cache") } + +func (a *acrn) storeInfo() error { + if a.store != nil { + if err := a.store.Store(store.Hypervisor, a.info); err != nil { + return err + } + } + return nil +} + +func (a *acrn) save() (s persistapi.HypervisorState) { + s.Pid = a.pid() + s.Type = string(AcrnHypervisor) + s.UUID = a.state.UUID + return +} + +func (a *acrn) load(s persistapi.HypervisorState) { + a.info.PID = s.Pid + a.state.UUID = s.UUID +} diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index 09b83d4fc7..f5e8b8a705 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -22,6 +22,7 @@ import ( httptransport "github.com/go-openapi/runtime/client" "github.com/go-openapi/strfmt" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client" models "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/models" ops "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/operations" @@ -234,8 +235,10 @@ func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS N // No need to return an error from there since there might be nothing // to fetch if this is the first time the hypervisor is created. - if err := fc.store.Load(store.Hypervisor, &fc.info); err != nil { - fc.Logger().WithField("function", "init").WithError(err).Info("No info could be fetched") + if fc.store != nil { + if err := fc.store.Load(store.Hypervisor, &fc.info); err != nil { + fc.Logger().WithField("function", "init").WithError(err).Info("No info could be fetched") + } } return nil @@ -388,7 +391,10 @@ func (fc *firecracker) fcInit(timeout int) error { fc.state.set(apiReady) // Store VMM information - return fc.store.Store(store.Hypervisor, fc.info) + if fc.store != nil { + return fc.store.Store(store.Hypervisor, fc.info) + } + return nil } func (fc *firecracker) fcEnd() (err error) { @@ -988,3 +994,13 @@ func (fc *firecracker) fromGrpc(ctx context.Context, hypervisorConfig *Hyperviso func (fc *firecracker) toGrpc() ([]byte, error) { return nil, errors.New("firecracker is not supported by VM cache") } + +func (fc *firecracker) save() (s persistapi.HypervisorState) { + s.Pid = fc.pid() + s.Type = string(FirecrackerHypervisor) + return +} + +func (fc *firecracker) load(s persistapi.HypervisorState) { + fc.info.PID = s.Pid +} diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 4439d1d611..f12595328b 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -15,6 +15,7 @@ import ( "strings" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" ) @@ -670,4 +671,7 @@ type hypervisor interface { pid() int fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, store *store.VCStore, j []byte) error toGrpc() ([]byte, error) + + save() persistapi.HypervisorState + load(persistapi.HypervisorState) } diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go index a429d62578..1c198f47bd 100644 --- a/virtcontainers/mock_hypervisor.go +++ b/virtcontainers/mock_hypervisor.go @@ -10,6 +10,7 @@ import ( "errors" "os" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" ) @@ -114,3 +115,9 @@ func (m *mockHypervisor) fromGrpc(ctx context.Context, hypervisorConfig *Hypervi func (m *mockHypervisor) toGrpc() ([]byte, error) { return nil, errors.New("firecracker is not supported by VM cache") } + +func (m *mockHypervisor) save() (s persistapi.HypervisorState) { + return +} + +func (m *mockHypervisor) load(s persistapi.HypervisorState) {} diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go index eaa1ac489e..394388f45d 100644 --- a/virtcontainers/persist.go +++ b/virtcontainers/persist.go @@ -57,7 +57,9 @@ func (s *Sandbox) dumpState(ss *persistapi.SandboxState, cs map[string]persistap } } -func (s *Sandbox) dumpHypervisor(ss *persistapi.SandboxState, cs map[string]persistapi.ContainerState) { +func (s *Sandbox) dumpHypervisor(ss *persistapi.SandboxState) { + ss.HypervisorState = s.hypervisor.save() + // BlockIndex will be moved from sandbox state to hypervisor state later ss.HypervisorState.BlockIndex = s.state.BlockIndex } @@ -160,7 +162,7 @@ func (s *Sandbox) Save() error { s.dumpVersion(&ss) s.dumpState(&ss, cs) - s.dumpHypervisor(&ss, cs) + s.dumpHypervisor(&ss) s.dumpDevices(&ss, cs) s.dumpProcess(cs) s.dumpMounts(cs) @@ -190,6 +192,10 @@ func (c *Container) loadContState(cs persistapi.ContainerState) { } } +func (s *Sandbox) loadHypervisor(hs persistapi.HypervisorState) { + s.hypervisor.load(hs) +} + func (s *Sandbox) loadDevices(devStates []persistapi.DeviceState) { s.devManager.LoadDevices(devStates) } @@ -237,6 +243,7 @@ func (s *Sandbox) Restore() error { } s.loadState(ss) + s.loadHypervisor(ss.HypervisorState) s.loadDevices(ss.Devices) return nil } diff --git a/virtcontainers/persist/api/hypervisor.go b/virtcontainers/persist/api/hypervisor.go new file mode 100644 index 0000000000..d61917da90 --- /dev/null +++ b/virtcontainers/persist/api/hypervisor.go @@ -0,0 +1,43 @@ +// Copyright (c) 2016 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package persistapi + +// Bridge is a bridge where devices can be hot plugged +type Bridge struct { + // DeviceAddr contains information about devices plugged and its address in the bridge + DeviceAddr map[uint32]string + + // Type is the type of the bridge (pci, pcie, etc) + Type string + + //ID is used to identify the bridge in the hypervisor + ID string + + // Addr is the PCI/e slot of the bridge + Addr int +} + +// CPUDevice represents a CPU device which was hot-added in a running VM +type CPUDevice struct { + // ID is used to identify this CPU in the hypervisor options. + ID string +} + +type HypervisorState struct { + Pid int + // Type of hypervisor, E.g. qemu/firecracker/acrn. + Type string + BlockIndex int + UUID string + + // Belows are qemu specific + // Refs: virtcontainers/qemu.go:QemuState + Bridges []Bridge + // HotpluggedCPUs is the list of CPUs that were hot-added + HotpluggedVCPUs []CPUDevice + HotpluggedMemory int + HotplugVFIOOnRootBus bool +} diff --git a/virtcontainers/persist/api/sandbox.go b/virtcontainers/persist/api/sandbox.go index b78a5a447d..96828df51e 100644 --- a/virtcontainers/persist/api/sandbox.go +++ b/virtcontainers/persist/api/sandbox.go @@ -8,44 +8,6 @@ package persistapi // ============= sandbox level resources ============= -// SetFunc is function hook used for setting sandbox/container state -// It can be registered to dynamically set state files when dump -type SetFunc (func(*SandboxState, map[string]ContainerState) error) - -// Bridge is a bridge where devices can be hot plugged -type Bridge struct { - // DeviceAddr contains information about devices plugged and its address in the bridge - DeviceAddr map[uint32]string - - // Type is the type of the bridge (pci, pcie, etc) - Type string - - //ID is used to identify the bridge in the hypervisor - ID string - - // Addr is the PCI/e slot of the bridge - Addr int -} - -// CPUDevice represents a CPU device which was hot-added in a running VM -type CPUDevice struct { - // ID is used to identify this CPU in the hypervisor options. - ID string -} - -// HypervisorState saves state of hypervisor -// Refs: virtcontainers/qemu.go:QemuState -type HypervisorState struct { - Pid int - Bridges []Bridge - // HotpluggedCPUs is the list of CPUs that were hot-added - HotpluggedVCPUs []CPUDevice - HotpluggedMemory int - UUID string - HotplugVFIOOnRootBus bool - BlockIndex int -} - // ProxyState save proxy state data type ProxyState struct { // Pid of proxy process diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index d395ec0112..e3da96ffcb 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -29,9 +29,11 @@ import ( "github.com/sirupsen/logrus" "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" + "golang.org/x/sys/unix" ) @@ -252,7 +254,17 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto q.nvdimmCount = 0 } - if err = q.store.Load(store.Hypervisor, &q.state); err != nil { + var create bool + if q.store != nil { //use old store + if err := q.store.Load(store.Hypervisor, &q.state); err != nil { + // hypervisor doesn't exist, create new one + create = true + } + } else if q.state.UUID == "" { // new store + create = true + } + + if create { q.Logger().Debug("Creating bridges") q.state.Bridges = q.arch.bridges(q.config.DefaultBridges) @@ -267,7 +279,7 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto return err } - if err = q.store.Store(store.Hypervisor, q.state); err != nil { + if err = q.storeState(); err != nil { return err } } @@ -1172,7 +1184,7 @@ func (q *qemu) hotplugAddDevice(devInfo interface{}, devType deviceType) (interf return data, err } - return data, q.store.Store(store.Hypervisor, q.state) + return data, q.storeState() } func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { @@ -1184,7 +1196,7 @@ func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (int return data, err } - return data, q.store.Store(store.Hypervisor, q.state) + return data, q.storeState() } func (q *qemu) hotplugCPUs(vcpus uint32, op operation) (uint32, error) { @@ -1264,12 +1276,12 @@ func (q *qemu) hotplugAddCPUs(amount uint32) (uint32, error) { hotpluggedVCPUs++ if hotpluggedVCPUs == amount { // All vCPUs were hotplugged - return amount, q.store.Store(store.Hypervisor, q.state) + return amount, q.storeState() } } // All vCPUs were NOT hotplugged - if err := q.store.Store(store.Hypervisor, q.state); err != nil { + if err := q.storeState(); err != nil { q.Logger().Errorf("failed to save hypervisor state after hotplug %d vCPUs: %v", hotpluggedVCPUs, err) } @@ -1289,7 +1301,7 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) { // get the last vCPUs and try to remove it cpu := q.state.HotpluggedVCPUs[len(q.state.HotpluggedVCPUs)-1] if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, cpu.ID); err != nil { - _ = q.store.Store(store.Hypervisor, q.state) + q.storeState() return i, fmt.Errorf("failed to hotunplug CPUs, only %d CPUs were hotunplugged: %v", i, err) } @@ -1297,7 +1309,7 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) { q.state.HotpluggedVCPUs = q.state.HotpluggedVCPUs[:len(q.state.HotpluggedVCPUs)-1] } - return amount, q.store.Store(store.Hypervisor, q.state) + return amount, q.storeState() } func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) (int, error) { @@ -1387,7 +1399,7 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) { } } q.state.HotpluggedMemory += memDev.sizeMB - return memDev.sizeMB, q.store.Store(store.Hypervisor, q.state) + return memDev.sizeMB, q.storeState() } func (q *qemu) pauseSandbox() error { @@ -1837,3 +1849,57 @@ func (q *qemu) toGrpc() ([]byte, error) { return json.Marshal(&qp) } + +func (q *qemu) storeState() error { + if q.store != nil { + if err := q.store.Store(store.Hypervisor, q.state); err != nil { + return err + } + } + return nil +} + +func (q *qemu) save() (s persistapi.HypervisorState) { + s.Pid = q.pid() + s.Type = string(QemuHypervisor) + s.UUID = q.state.UUID + s.HotpluggedMemory = q.state.HotpluggedMemory + s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus + + for _, bridge := range q.state.Bridges { + s.Bridges = append(s.Bridges, persistapi.Bridge{ + DeviceAddr: bridge.Address, + Type: string(bridge.Type), + ID: bridge.ID, + Addr: bridge.Addr, + }) + } + + for _, cpu := range q.state.HotpluggedVCPUs { + s.HotpluggedVCPUs = append(s.HotpluggedVCPUs, persistapi.CPUDevice{ + ID: cpu.ID, + }) + } + return +} + +func (q *qemu) load(s persistapi.HypervisorState) { + q.state.UUID = s.UUID + q.state.HotpluggedMemory = s.HotpluggedMemory + q.state.HotplugVFIOOnRootBus = s.HotplugVFIOOnRootBus + + for _, bridge := range s.Bridges { + q.state.Bridges = append(q.state.Bridges, types.PCIBridge{ + Address: bridge.DeviceAddr, + Type: types.PCIType(bridge.Type), + ID: bridge.ID, + Addr: bridge.Addr, + }) + } + + for _, cpu := range q.state.HotpluggedVCPUs { + q.state.HotpluggedVCPUs = append(q.state.HotpluggedVCPUs, CPUDevice{ + ID: cpu.ID, + }) + } +} diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 0d43a279d7..047357db70 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -574,8 +574,15 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor } }() - if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.store); err != nil { - return nil, err + if s.supportNewStore() { + // new store doesn't require hypervisor to be stored immediately + if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, nil); err != nil { + return nil, err + } + } else { + if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.store); err != nil { + return nil, err + } } agentConfig, err := newAgentConfig(sandboxConfig.AgentType, sandboxConfig.AgentConfig)