From 290339da6b9d094a8c73e1252ccafa42ca75c6b1 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Wed, 8 Jan 2020 22:40:21 +0800 Subject: [PATCH] compatibility: keep oldstore for compatibility Keep old store restore functions for keeping backward compatibility, if old store files are found from disk, restore them with old store first. Signed-off-by: Wei Zhang --- virtcontainers/container.go | 96 +++++++++++++++++++++++---- virtcontainers/kata_agent.go | 7 ++ virtcontainers/persist.go | 24 +++++++ virtcontainers/sandbox.go | 70 ++++++++++++++++--- virtcontainers/virtcontainers_test.go | 5 ++ 5 files changed, 177 insertions(+), 25 deletions(-) diff --git a/virtcontainers/container.go b/virtcontainers/container.go index eedfe225b5..6c3d759edc 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -29,6 +29,7 @@ import ( "github.com/kata-containers/runtime/pkg/rootless" "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/manager" + "github.com/kata-containers/runtime/virtcontainers/store" ) // https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h @@ -335,6 +336,8 @@ type Container struct { systemMountsInfo SystemMountsInfo ctx context.Context + + store *store.VCStore } // ID returns the container identifier string. @@ -426,9 +429,17 @@ func (c *Container) setContainerState(state types.StateString) error { // update in-memory state c.state.State = state - // flush data to storage - if err := c.sandbox.Save(); err != nil { - return err + if useOldStore(c.sandbox.ctx) { + // experimental runtime use "persist.json" which doesn't need "state.json" anymore + // update on-disk state + if err := c.store.Store(store.State, c.state); err != nil { + return err + } + } else { + // flush data to storage + if err := c.sandbox.Save(); err != nil { + return err + } } return nil @@ -678,31 +689,76 @@ func newContainer(sandbox *Sandbox, contConfig *ContainerConfig) (*Container, er ctx: sandbox.ctx, } - // experimental runtime use "persist.json" instead of legacy "state.json" as storage - err := c.Restore() - if err == nil { - //container restored - return c, nil - } + if useOldStore(sandbox.ctx) { + ctrStore, err := store.NewVCContainerStore(sandbox.ctx, c.sandboxID, c.id) + if err != nil { + return nil, err + } + c.store = ctrStore + state, err := c.store.LoadContainerState() + if err == nil { + c.state = state + } - // Unexpected error - if !os.IsNotExist(err) && err != errContainerPersistNotExist { - return nil, err + var process Process + if err := c.store.Load(store.Process, &process); err == nil { + c.process = process + } + } else { + // experimental runtime use "persist.json" instead of legacy "state.json" as storage + err := c.Restore() + if err == nil { + //container restored + return c, nil + } + + // Unexpected error + if !os.IsNotExist(err) && err != errContainerPersistNotExist { + return nil, err + } } // Go to next step for first created container - if err = c.createMounts(); err != nil { + if err := c.createMounts(); err != nil { return nil, err } - if err = c.createDevices(contConfig); err != nil { + if err := c.createDevices(contConfig); err != nil { return nil, err } return c, nil } +func (c *Container) loadMounts() ([]Mount, error) { + var mounts []Mount + if err := c.store.Load(store.Mounts, &mounts); err != nil { + return []Mount{}, err + } + + return mounts, nil +} + +func (c *Container) loadDevices() ([]ContainerDevice, error) { + var devices []ContainerDevice + + if err := c.store.Load(store.DeviceIDs, &devices); err != nil { + return []ContainerDevice{}, err + } + + return devices, nil +} + func (c *Container) createMounts() error { + if useOldStore(c.sandbox.ctx) { + mounts, err := c.loadMounts() + if err == nil { + // restore mounts from disk + c.mounts = mounts + return nil + } + } + // Create block devices for newly created container if err := c.createBlockDevices(); err != nil { return err @@ -712,6 +768,18 @@ func (c *Container) createMounts() error { } func (c *Container) createDevices(contConfig *ContainerConfig) error { + // If sandbox supports "newstore", only newly created container can reach this function, + // so we don't call restore when `supportNewStore` is true + if useOldStore(c.sandbox.ctx) { + // Devices will be found in storage after create stage has completed. + // We load devices from storage at all other stages. + storedDevices, err := c.loadDevices() + if err == nil { + c.devices = storedDevices + return nil + } + } + // If devices were not found in storage, create Device implementations // from the configuration. This should happen at create. var storedDevices []ContainerDevice diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index a3fa65fe24..1c72b5b521 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -31,6 +31,7 @@ import ( ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" + "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/opencontainers/runtime-spec/specs-go" opentracing "github.com/opentracing/opentracing-go" @@ -317,6 +318,12 @@ func (k *kataAgent) init(ctx context.Context, sandbox *Sandbox, config interface k.proxyBuiltIn = isProxyBuiltIn(sandbox.config.ProxyType) + // Fetch agent runtime info. + if useOldStore(sandbox.ctx) { + if err := sandbox.store.Load(store.Agent, &k.state); err != nil { + k.Logger().Debug("Could not retrieve anything from storage") + } + } return disableVMShutdown, nil } diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go index 95cdb5df65..e9475a33ae 100644 --- a/virtcontainers/persist.go +++ b/virtcontainers/persist.go @@ -6,12 +6,14 @@ package virtcontainers import ( + "context" "errors" "github.com/kata-containers/runtime/virtcontainers/device/api" exp "github.com/kata-containers/runtime/virtcontainers/experimental" "github.com/kata-containers/runtime/virtcontainers/persist" persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" + "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/mitchellh/mapstructure" ) @@ -559,3 +561,25 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { } return sconfig, nil } + +var oldstoreKey = struct{}{} + +func loadSandboxConfigFromOldStore(ctx context.Context, sid string) (*SandboxConfig, context.Context, error) { + var config SandboxConfig + // We're bootstrapping + vcStore, err := store.NewVCSandboxStore(ctx, sid) + if err != nil { + return nil, ctx, err + } + + if err := vcStore.Load(store.Configuration, &config); err != nil { + return nil, ctx, err + } + + return &config, context.WithValue(ctx, oldstoreKey, true), nil +} + +func useOldStore(ctx context.Context) bool { + v := ctx.Value(oldstoreKey) + return v != nil +} diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 18363fc375..46b9ea9c0d 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -37,6 +37,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/runtime/virtcontainers/pkg/compatoci" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" + "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -177,7 +178,9 @@ type Sandbox struct { factory Factory hypervisor hypervisor agent agent - newStore persistapi.PersistDriver + store *store.VCStore + // store is used to replace VCStore step by step + newStore persistapi.PersistDriver network Network monitor *monitor @@ -544,16 +547,47 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor } }() - s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, nil) + if useOldStore(ctx) { + vcStore, err := store.NewVCSandboxStore(ctx, s.id) + if err != nil { + return nil, err + } - // Ignore the error. Restore can fail for a new sandbox - if err := s.Restore(); err != nil { - s.Logger().WithError(err).Debug("restore sandbox failed") - } + s.store = vcStore - // new store doesn't require hypervisor to be stored immediately - if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.stateful); err != nil { - return nil, err + // Fetch sandbox network to be able to access it from the sandbox structure. + var networkNS NetworkNamespace + if err = s.store.Load(store.Network, &networkNS); err == nil { + s.networkNS = networkNS + } + + devices, err := s.store.LoadDevices() + if err != nil { + s.Logger().WithError(err).WithField("sandboxid", s.id).Warning("load sandbox devices failed") + } + s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, devices) + + // Load sandbox state. The hypervisor.createSandbox call, may need to access statei. + state, err := s.store.LoadState() + if err == nil { + s.state = state + } + + if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.stateful); err != nil { + return nil, err + } + } else { + s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, nil) + + // Ignore the error. Restore can fail for a new sandbox + if err := s.Restore(); err != nil { + s.Logger().WithError(err).Debug("restore sandbox failed") + } + + // new store doesn't require hypervisor to be stored immediately + if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.stateful); err != nil { + return nil, err + } } agentConfig, err := newAgentConfig(sandboxConfig.AgentType, sandboxConfig.AgentConfig) @@ -612,12 +646,22 @@ func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err var config SandboxConfig - c, err := loadSandboxConfig(sandboxID) + // Try to load sandbox config from old store at first. + c, ctx, err := loadSandboxConfigFromOldStore(ctx, sandboxID) if err != nil { - return nil, err + virtLog.Warningf("failed to get sandbox config from old store: %v", err) + // If we failed to load sandbox config from old store, try again with new store. + c, err = loadSandboxConfig(sandboxID) + if err != nil { + virtLog.Warningf("failed to get sandbox config from new store: %v", err) + return nil, err + } } config = *c + if useOldStore(ctx) { + virtLog.Infof("Warning: old store has been deprecated.") + } // fetchSandbox is not suppose to create new sandbox VM. sandbox, err = createSandbox(ctx, config, nil) if err != nil { @@ -1478,6 +1522,10 @@ func (s *Sandbox) setSandboxState(state types.StateString) error { // update in-memory state s.state.State = state + + if useOldStore(s.ctx) { + return s.store.Store(store.State, s.state) + } return nil } diff --git a/virtcontainers/virtcontainers_test.go b/virtcontainers/virtcontainers_test.go index a690e438c3..f7a17bdf34 100644 --- a/virtcontainers/virtcontainers_test.go +++ b/virtcontainers/virtcontainers_test.go @@ -16,6 +16,7 @@ import ( "testing" "github.com/kata-containers/runtime/virtcontainers/persist/fs" + "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/utils" "github.com/sirupsen/logrus" ) @@ -60,10 +61,14 @@ func cleanUp() { os.RemoveAll(testDir) os.MkdirAll(testDir, DirMode) + store.DeleteAll() + store.VCStorePrefix = "" + setup() } func setup() { + store.VCStorePrefix = testDir os.Mkdir(filepath.Join(testDir, testBundle), DirMode) for _, filename := range []string{testQemuKernelPath, testQemuInitrdPath, testQemuImagePath, testQemuPath} {