diff --git a/cli/main.go b/cli/main.go index 8a624a4c8c..9e63da089d 100644 --- a/cli/main.go +++ b/cli/main.go @@ -21,6 +21,7 @@ import ( "github.com/kata-containers/runtime/pkg/rootless" "github.com/kata-containers/runtime/pkg/signals" vc "github.com/kata-containers/runtime/virtcontainers" + exp "github.com/kata-containers/runtime/virtcontainers/experimental" vf "github.com/kata-containers/runtime/virtcontainers/factory" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -346,6 +347,11 @@ func beforeSubcommands(c *cli.Context) error { "arguments": `"` + args + `"`, } + err = addExpFeatures(c, runtimeConfig) + if err != nil { + return err + } + kataLog.WithFields(fields).Info() // make the data accessible to the sub-commands. @@ -401,6 +407,24 @@ func setupTracing(context *cli.Context, rootSpanName string) error { return nil } +// add supported experimental features in context +func addExpFeatures(clictx *cli.Context, runtimeConfig oci.RuntimeConfig) error { + ctx, err := cliContextToContext(clictx) + if err != nil { + return err + } + + var exps []string + for _, e := range runtimeConfig.Experimental { + exps = append(exps, e.Name) + } + + ctx = exp.ContextWithExp(ctx, exps) + // Add tracer to metadata and update the context + clictx.App.Metadata["context"] = ctx + return nil +} + func afterSubcommands(c *cli.Context) error { ctx, err := cliContextToContext(c) if err != nil { diff --git a/virtcontainers/container.go b/virtcontainers/container.go index aff4e85696..d26569834b 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -417,6 +417,7 @@ func (c *Container) storeContainer() error { if err := c.sandbox.Save(); err != nil { return err } + return nil } return c.store.Store(store.Configuration, *(c.config)) } diff --git a/virtcontainers/experimental/experimental.go b/virtcontainers/experimental/experimental.go index 4204e963e5..8eb3eddd3c 100644 --- a/virtcontainers/experimental/experimental.go +++ b/virtcontainers/experimental/experimental.go @@ -6,6 +6,7 @@ package experimental import ( + "context" "fmt" "regexp" ) @@ -22,8 +23,11 @@ type Feature struct { ExpRelease string } +type contextKey struct{} + var ( supportedFeatures = make(map[string]Feature) + expContextKey = contextKey{} ) // Register register a new experimental feature @@ -61,3 +65,16 @@ func validateFeature(feature Feature) error { return nil } + +func ContextWithExp(ctx context.Context, names []string) context.Context { + return context.WithValue(ctx, expContextKey, names) +} + +func ExpFromContext(ctx context.Context) []string { + value := ctx.Value(expContextKey) + if value == nil { + return nil + } + names := value.([]string) + return names +} diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go index c8ec42a484..6c4c6b6831 100644 --- a/virtcontainers/persist.go +++ b/virtcontainers/persist.go @@ -13,6 +13,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/persist" persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/mitchellh/mapstructure" ) var ( @@ -171,6 +172,86 @@ func (s *Sandbox) dumpNetwork(ss *persistapi.SandboxState) { } } +func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { + sconfig := s.config + ss.Config = persistapi.SandboxConfig{ + HypervisorType: string(sconfig.HypervisorType), + AgentType: string(sconfig.AgentType), + ProxyType: string(sconfig.ProxyType), + ProxyConfig: persistapi.ProxyConfig{ + Path: sconfig.ProxyConfig.Path, + Debug: sconfig.ProxyConfig.Debug, + }, + ShimType: string(sconfig.ShimType), + NetworkConfig: persistapi.NetworkConfig{ + NetNSPath: sconfig.NetworkConfig.NetNSPath, + NetNsCreated: sconfig.NetworkConfig.NetNsCreated, + DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs, + InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel), + }, + + ShmSize: sconfig.ShmSize, + SharePidNs: sconfig.SharePidNs, + Stateful: sconfig.Stateful, + SystemdCgroup: sconfig.SystemdCgroup, + SandboxCgroupOnly: sconfig.SandboxCgroupOnly, + DisableGuestSeccomp: sconfig.DisableGuestSeccomp, + } + + for _, e := range sconfig.Experimental { + ss.Config.Experimental = append(ss.Config.Experimental, e.Name) + } + + ss.Config.HypervisorConfig = persistapi.HypervisorConfig{ + NumVCPUs: sconfig.HypervisorConfig.NumVCPUs, + DefaultMaxVCPUs: sconfig.HypervisorConfig.DefaultMaxVCPUs, + MemorySize: sconfig.HypervisorConfig.MemorySize, + MemSlots: sconfig.HypervisorConfig.MemSlots, + KernelPath: sconfig.HypervisorConfig.KernelPath, + ImagePath: sconfig.HypervisorConfig.ImagePath, + InitrdPath: sconfig.HypervisorConfig.InitrdPath, + SharedFS: sconfig.HypervisorConfig.SharedFS, + VirtioFSDaemon: sconfig.HypervisorConfig.VirtioFSDaemon, + DisableBlockDeviceUse: sconfig.HypervisorConfig.DisableBlockDeviceUse, + UseVSock: sconfig.HypervisorConfig.UseVSock, + DisableVhostNet: sconfig.HypervisorConfig.DisableVhostNet, + } + + if sconfig.AgentType == "kata" { + var sagent KataAgentConfig + err := mapstructure.Decode(sconfig.AgentConfig, &sagent) + if err != nil { + s.Logger().WithError(err).Error("internal error: KataAgentConfig failed to decode") + } else { + ss.Config.KataAgentConfig = &persistapi.KataAgentConfig{ + LongLiveConn: sagent.LongLiveConn, + UseVSock: sagent.UseVSock, + } + } + } + + if sconfig.ShimType == "kataShim" { + var shim ShimConfig + err := mapstructure.Decode(sconfig.ShimConfig, &shim) + if err != nil { + s.Logger().WithError(err).Error("internal error: ShimConfig failed to decode") + } else { + ss.Config.KataShimConfig = &persistapi.ShimConfig{ + Path: shim.Path, + Debug: shim.Debug, + } + } + } + + for _, contConf := range sconfig.Containers { + ss.Config.ContainerConfigs = append(ss.Config.ContainerConfigs, persistapi.ContainerConfig{ + ID: contConf.ID, + Annotations: contConf.Annotations, + Resources: contConf.Resources, + }) + } +} + func (s *Sandbox) Save() error { var ( ss = persistapi.SandboxState{} @@ -185,6 +266,7 @@ func (s *Sandbox) Save() error { s.dumpMounts(cs) s.dumpAgent(&ss) s.dumpNetwork(&ss) + s.dumpConfig(&ss) if err := s.newStore.ToDisk(ss, cs); err != nil { return err @@ -335,3 +417,85 @@ func (s *Sandbox) supportNewStore() bool { } return false } + +func loadSandboxConfig(id string) (*SandboxConfig, error) { + store, err := persist.GetDriver("fs") + if err != nil || store == nil { + return nil, errors.New("failed to get fs persist driver") + } + + ss, _, err := store.FromDisk(id) + if err != nil { + return nil, err + } + + savedConf := ss.Config + sconfig := &SandboxConfig{ + ID: id, + HypervisorType: HypervisorType(savedConf.HypervisorType), + AgentType: AgentType(savedConf.AgentType), + ProxyType: ProxyType(savedConf.ProxyType), + ProxyConfig: ProxyConfig{ + Path: savedConf.ProxyConfig.Path, + Debug: savedConf.ProxyConfig.Debug, + }, + ShimType: ShimType(savedConf.ShimType), + NetworkConfig: NetworkConfig{ + NetNSPath: savedConf.NetworkConfig.NetNSPath, + NetNsCreated: savedConf.NetworkConfig.NetNsCreated, + DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs, + InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel), + }, + + ShmSize: savedConf.ShmSize, + SharePidNs: savedConf.SharePidNs, + Stateful: savedConf.Stateful, + SystemdCgroup: savedConf.SystemdCgroup, + SandboxCgroupOnly: savedConf.SandboxCgroupOnly, + DisableGuestSeccomp: savedConf.DisableGuestSeccomp, + } + + for _, name := range savedConf.Experimental { + sconfig.Experimental = append(sconfig.Experimental, *exp.Get(name)) + } + + hconf := savedConf.HypervisorConfig + sconfig.HypervisorConfig = HypervisorConfig{ + NumVCPUs: hconf.NumVCPUs, + DefaultMaxVCPUs: hconf.DefaultMaxVCPUs, + MemorySize: hconf.MemorySize, + MemSlots: hconf.MemSlots, + KernelPath: hconf.KernelPath, + ImagePath: hconf.ImagePath, + InitrdPath: hconf.InitrdPath, + SharedFS: hconf.SharedFS, + VirtioFSDaemon: hconf.VirtioFSDaemon, + DisableBlockDeviceUse: hconf.DisableBlockDeviceUse, + UseVSock: hconf.UseVSock, + DisableVhostNet: hconf.DisableVhostNet, + } + + if savedConf.AgentType == "kata" { + + sconfig.AgentConfig = KataAgentConfig{ + LongLiveConn: savedConf.KataAgentConfig.LongLiveConn, + UseVSock: savedConf.KataAgentConfig.UseVSock, + } + } + + if savedConf.ShimType == "kataShim" { + sconfig.ShimConfig = ShimConfig{ + Path: savedConf.KataShimConfig.Path, + Debug: savedConf.KataShimConfig.Debug, + } + } + + for _, contConf := range savedConf.ContainerConfigs { + sconfig.Containers = append(sconfig.Containers, ContainerConfig{ + ID: contConf.ID, + Annotations: contConf.Annotations, + Resources: contConf.Resources, + }) + } + return sconfig, nil +} diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go index 1d10ed483c..f7b9719d51 100644 --- a/virtcontainers/persist/api/config.go +++ b/virtcontainers/persist/api/config.go @@ -6,6 +6,10 @@ package persistapi +import ( + specs "github.com/opencontainers/runtime-spec/specs-go" +) + // Param is a key/value representation for hypervisor and kernel parameters. type Param struct { Key string @@ -42,6 +46,9 @@ type HypervisorConfig struct { // MemOffset specifies memory space for nvdimm device MemOffset uint32 + // VirtioFSCacheSize is the DAX cache size in MiB + VirtioFSCacheSize uint32 + // KernelParams are additional guest kernel parameters. KernelParams []Param @@ -67,6 +74,12 @@ type HypervisorConfig struct { // HypervisorPath is the hypervisor executable host path. HypervisorPath string + // HypervisorCtlPath is the hypervisor ctl executable host path. + HypervisorCtlPath string + + // JailerPath is the jailer executable host path. + JailerPath string + // BlockDeviceDriver specifies the driver to be used for block device // either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver BlockDeviceDriver string @@ -87,6 +100,19 @@ type HypervisorConfig struct { // entropy (/dev/random, /dev/urandom or real hardware RNG device) EntropySource string + // Shared file system type: + // - virtio-9p (default) + // - virtio-fs + SharedFS string + + // VirtioFSDaemon is the virtio-fs vhost-user daemon path + VirtioFSDaemon string + + // VirtioFSCache cache mode for fs version cache or "none" + VirtioFSCache string + + // VirtioFSExtraArgs passes options to virtiofsd daemon + VirtioFSExtraArgs []string // customAssets is a map of assets. // Each value in that map takes precedence over the configured assets. // For example, if there is a value for the "kernel" key in this map, @@ -161,6 +187,11 @@ type HypervisorConfig struct { type KataAgentConfig struct { LongLiveConn bool UseVSock bool + // Debug bool + // Trace bool + // TraceMode string + // TraceType string + // KernelModules []string } // HyperstartConfig is a structure storing information needed for @@ -182,10 +213,23 @@ type ProxyConfig struct { type ShimConfig struct { Path string Debug bool + // Trace bool } // NetworkConfig is the network configuration related to a network. type NetworkConfig struct { + NetNSPath string + NetNsCreated bool + DisableNewNetNs bool + // NetmonConfig NetmonConfig + InterworkingModel int +} + +type ContainerConfig struct { + ID string + Annotations map[string]string + // Resources for recoding update + Resources specs.LinuxResources } // SandboxConfig is a sandbox configuration. @@ -195,17 +239,15 @@ type SandboxConfig struct { HypervisorConfig HypervisorConfig // only one agent config can be non-nil according to agent type - AgentType string - KataAgentConfig *KataAgentConfig `json:",omitempty"` - HyperstartConfig *HyperstartConfig `json:",omitempty"` + AgentType string + KataAgentConfig *KataAgentConfig `json:",omitempty"` ProxyType string ProxyConfig ProxyConfig ShimType string - KataShimConfig ShimConfig + KataShimConfig *ShimConfig - NetworkModel string NetworkConfig NetworkConfig ShmSize uint64 @@ -220,11 +262,18 @@ type SandboxConfig struct { // SystemdCgroup enables systemd cgroup support SystemdCgroup bool + // SandboxCgroupOnly enables cgroup only at podlevel in the host + SandboxCgroupOnly bool + + DisableGuestSeccomp bool + // Experimental enables experimental features - Experimental bool + Experimental []string // Information for fields not saved: // * Annotation: this is kind of casual data, we don't need casual data in persist file, // if you know this data needs to persist, please gives it // a specific field + + ContainerConfigs []ContainerConfig } diff --git a/virtcontainers/persist/fs/fs.go b/virtcontainers/persist/fs/fs.go index 1e7552785f..158141b566 100644 --- a/virtcontainers/persist/fs/fs.go +++ b/virtcontainers/persist/fs/fs.go @@ -14,6 +14,7 @@ import ( "path/filepath" "syscall" + "github.com/kata-containers/runtime/pkg/rootless" persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/sirupsen/logrus" ) @@ -38,7 +39,13 @@ const sandboxPathSuffix = "sbs" // runStoragePath is the sandbox runtime directory. // It will contain one state.json and one lock file for each created sandbox. -var runStoragePath = filepath.Join("/run", storagePathSuffix, sandboxPathSuffix) +var runStoragePath = func() string { + path := filepath.Join("/run", storagePathSuffix, sandboxPathSuffix) + if rootless.IsRootless() { + return filepath.Join(rootless.GetRootlessDir(), path) + } + return path +} // FS storage driver implementation type FS struct { @@ -76,7 +83,7 @@ func (fs *FS) sandboxDir() (string, error) { return "", fmt.Errorf("sandbox container id required") } - return filepath.Join(runStoragePath, id), nil + return filepath.Join(runStoragePath(), id), nil } // ToDisk sandboxState and containerState to disk @@ -257,5 +264,7 @@ func (fs *FS) unlock() error { // TestSetRunStoragePath set runStoragePath to path // this function is only used for testing purpose func TestSetRunStoragePath(path string) { - runStoragePath = path + runStoragePath = func() string { + return path + } } diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 9771f0d241..848e105e0d 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -242,7 +242,10 @@ func (s *Sandbox) SetAnnotations(annotations map[string]string) error { s.config.Annotations[k] = v } - return s.store.Store(store.Configuration, *(s.config)) + if !s.supportNewStore() { + return s.store.Store(store.Configuration, *(s.config)) + } + return nil } // GetAnnotations returns sandbox's annotations @@ -617,25 +620,24 @@ func (s *Sandbox) storeSandbox() error { span, _ := s.trace("storeSandbox") defer span.Finish() - err := s.store.Store(store.Configuration, *(s.config)) - if err != nil { - return err - } - - for _, container := range s.containers { - err = container.store.Store(store.Configuration, *(container.config)) - if err != nil { - return err - } - } - if s.supportNewStore() { // flush data to storage if err := s.Save(); err != nil { return err } - } + } else { + err := s.store.Store(store.Configuration, *(s.config)) + if err != nil { + return err + } + for _, container := range s.containers { + err = container.store.Store(store.Configuration, *(container.config)) + if err != nil { + return err + } + } + } return nil } @@ -673,6 +675,22 @@ func unlockSandbox(ctx context.Context, sandboxID, token string) error { return store.Unlock(token) } +func supportNewStore(ctx context.Context) bool { + if exp.Get("newstore") == nil { + return false + } + + // check if client context enabled "newstore" feature + exps := exp.ExpFromContext(ctx) + for _, v := range exps { + if v == "newstore" { + return true + } + } + + return false +} + // fetchSandbox fetches a sandbox config from a sandbox ID and returns a sandbox. func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err error) { virtLog.Info("fetch sandbox") @@ -685,15 +703,24 @@ func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err return sandbox, err } - // We're bootstrapping - vcStore, err := store.NewVCSandboxStore(ctx, sandboxID) - if err != nil { - return nil, err - } - var config SandboxConfig - if err := vcStore.Load(store.Configuration, &config); err != nil { - return nil, err + + if supportNewStore(ctx) { + c, err := loadSandboxConfig(sandboxID) + if err != nil { + return nil, err + } + config = *c + } else { + // We're bootstrapping + vcStore, err := store.NewVCSandboxStore(ctx, sandboxID) + if err != nil { + return nil, err + } + + if err := vcStore.Load(store.Configuration, &config); err != nil { + return nil, err + } } // fetchSandbox is not suppose to create new sandbox VM.