diff --git a/.cirrus.yml b/.cirrus.yml index 1a4c0e7750..d688001540 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -72,6 +72,8 @@ fedora_testing_task: &fedora_testing TEST_DRIVER: "vfs" - env: TEST_DRIVER: "overlay" + - env: + TEST_DRIVER: "overlay-transient" - env: TEST_DRIVER: "fuse-overlay" - env: diff --git a/cmd/containers-storage/main.go b/cmd/containers-storage/main.go index c00d014ff6..dccd5b5462 100644 --- a/cmd/containers-storage/main.go +++ b/cmd/containers-storage/main.go @@ -41,6 +41,7 @@ func main() { flags := mflag.NewFlagSet(command, eh) flags.StringVar(&options.RunRoot, []string{"-run", "R"}, options.RunRoot, "Root of the runtime state tree") flags.StringVar(&options.GraphRoot, []string{"-graph", "g"}, options.GraphRoot, "Root of the storage tree") + flags.BoolVar(&options.TransientStore, []string{"-transient-store"}, options.TransientStore, "Transient store") flags.StringVar(&options.GraphDriverName, []string{"-storage-driver", "s"}, options.GraphDriverName, "Storage driver to use ($STORAGE_DRIVER)") flags.Var(opts.NewListOptsRef(&options.GraphDriverOptions, nil), []string{"-storage-opt"}, "Set storage driver options ($STORAGE_OPTS)") flags.BoolVar(&debug, []string{"-debug", "D"}, debug, "Print debugging information") diff --git a/containers.go b/containers.go index 7ec7e8acbf..7db03d37c3 100644 --- a/containers.go +++ b/containers.go @@ -15,6 +15,22 @@ import ( digest "github.com/opencontainers/go-digest" ) +type containerLocations uint8 + +// The backing store is split in two json files, one (the volatile) +// that is written without fsync() meaning it isn't as robust to +// unclean shutdown +const ( + stableContainerLocation containerLocations = 1 << iota + volatileContainerLocation + + numContainerLocationIndex = iota +) + +func containerLocationFromIndex(index int) containerLocations { + return 1 << index +} + // A Container is a reference to a read-write layer with metadata. type Container struct { // ID is either one which was specified at create-time, or a random @@ -64,6 +80,9 @@ type Container struct { GIDMap []idtools.IDMap `json:"gidmap,omitempty"` Flags map[string]interface{} `json:"flags,omitempty"` + + // volatileStore is true if the container is from the volatile json file + volatileStore bool `json:"-"` } // rwContainerStore provides bookkeeping for information about Containers. @@ -115,11 +134,15 @@ type rwContainerStore interface { // Containers returns a slice enumerating the known containers. Containers() ([]Container, error) + + // Clean up unreferenced datadirs + GarbageCollect() error } type containerStore struct { lockfile Locker dir string + jsonPath [numContainerLocationIndex]string containers []*Container idindex *truncindex.TruncIndex byid map[string]*Container @@ -142,6 +165,7 @@ func copyContainer(c *Container) *Container { UIDMap: copyIDMap(c.UIDMap), GIDMap: copyIDMap(c.GIDMap), Flags: copyStringInterfaceMap(c.Flags), + volatileStore: c.volatileStore, } } @@ -176,6 +200,13 @@ func (c *Container) MountOpts() []string { } } +func containerLocation(c *Container) containerLocations { + if c.volatileStore { + return volatileContainerLocation + } + return stableContainerLocation +} + // startWritingWithReload makes sure the store is fresh if canReload, and locks it for writing. // If this succeeds, the caller MUST call stopWriting(). // @@ -289,8 +320,37 @@ func (r *containerStore) Containers() ([]Container, error) { return containers, nil } -func (r *containerStore) containerspath() string { - return filepath.Join(r.dir, "containers.json") +// This looks for datadirs in the store directory that are not referenced +// by the json file and removes it. These can happen in the case of unclean +// shutdowns or regular restarts in transient store mode. +func (r *containerStore) GarbageCollect() error { + entries, err := os.ReadDir(r.dir) + if err != nil { + // Unexpected, don't try any GC + return err + } + + for _, entry := range entries { + id := entry.Name() + // Does it look like a datadir directory? + if !entry.IsDir() || !nameLooksLikeID(id) { + continue + } + + // Should the id be there? + if r.byid[id] != nil { + continue + } + + // Otherwise remove datadir + moreErr := os.RemoveAll(filepath.Join(r.dir, id)) + // Propagate first error + if moreErr != nil && err == nil { + err = moreErr + } + } + + return err } func (r *containerStore) datadir(id string) string { @@ -309,31 +369,53 @@ func (r *containerStore) datapath(id, key string) string { // If !lockedForWriting and this function fails, the return value indicates whether // retrying with lockedForWriting could succeed. func (r *containerStore) load(lockedForWriting bool) (bool, error) { - rpath := r.containerspath() - data, err := os.ReadFile(rpath) - if err != nil && !os.IsNotExist(err) { - return false, err - } - + var modifiedLocations containerLocations containers := []*Container{} - if len(data) != 0 { - if err := json.Unmarshal(data, &containers); err != nil { - return false, fmt.Errorf("loading %q: %w", rpath, err) + + ids := make(map[string]*Container) + + for locationIndex := 0; locationIndex < numContainerLocationIndex; locationIndex++ { + location := containerLocationFromIndex(locationIndex) + rpath := r.jsonPath[locationIndex] + + data, err := os.ReadFile(rpath) + if err != nil && !os.IsNotExist(err) { + return false, err + } + + locationContainers := []*Container{} + if len(data) != 0 { + if err := json.Unmarshal(data, &locationContainers); err != nil { + return false, fmt.Errorf("loading %q: %w", rpath, err) + } + } + + for _, container := range locationContainers { + // There should be no duplicated ids between json files, but lets check to be sure + if ids[container.ID] != nil { + continue // skip invalid duplicated container + } + // Remember where the container came from + if location == volatileContainerLocation { + container.volatileStore = true + } + containers = append(containers, container) + ids[container.ID] = container } } + idlist := make([]string, 0, len(containers)) layers := make(map[string]*Container) - ids := make(map[string]*Container) names := make(map[string]*Container) var errorToResolveBySaving error // == nil for n, container := range containers { idlist = append(idlist, container.ID) - ids[container.ID] = containers[n] layers[container.LayerID] = containers[n] for _, name := range container.Names { if conflict, ok := names[name]; ok { r.removeName(conflict, name) errorToResolveBySaving = errors.New("container store is inconsistent and the current caller does not hold a write lock") + modifiedLocations |= containerLocation(container) } names[name] = containers[n] } @@ -348,34 +430,64 @@ func (r *containerStore) load(lockedForWriting bool) (bool, error) { if !lockedForWriting { return true, errorToResolveBySaving } - return false, r.Save() + return false, r.save(modifiedLocations) } return false, nil } // Save saves the contents of the store to disk. It should be called with // the lock held, locked for writing. -func (r *containerStore) Save() error { +func (r *containerStore) save(saveLocations containerLocations) error { r.lockfile.AssertLockedForWriting() - rpath := r.containerspath() - if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil { - return err - } - jdata, err := json.Marshal(&r.containers) - if err != nil { - return err - } - if err := ioutils.AtomicWriteFile(rpath, jdata, 0600); err != nil { - return err + for locationIndex := 0; locationIndex < numContainerLocationIndex; locationIndex++ { + location := containerLocationFromIndex(locationIndex) + if location&saveLocations == 0 { + continue + } + rpath := r.jsonPath[locationIndex] + if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil { + return err + } + subsetContainers := make([]*Container, 0, len(r.containers)) + for _, container := range r.containers { + if containerLocation(container) == location { + subsetContainers = append(subsetContainers, container) + } + } + + jdata, err := json.Marshal(&subsetContainers) + if err != nil { + return err + } + var opts *ioutils.AtomicFileWriterOptions + if location == volatileContainerLocation { + opts = &ioutils.AtomicFileWriterOptions{ + NoSync: true, + } + } + if err := ioutils.AtomicWriteFileWithOpts(rpath, jdata, 0600, opts); err != nil { + return err + } } return r.lockfile.Touch() } -func newContainerStore(dir string) (rwContainerStore, error) { +func (r *containerStore) saveFor(modifiedContainer *Container) error { + return r.save(containerLocation(modifiedContainer)) +} + +func newContainerStore(dir string, runDir string, transient bool) (rwContainerStore, error) { if err := os.MkdirAll(dir, 0700); err != nil { return nil, err } - lockfile, err := GetLockfile(filepath.Join(dir, "containers.lock")) + volatileDir := dir + if transient { + if err := os.MkdirAll(runDir, 0700); err != nil { + return nil, err + } + volatileDir = runDir + } + lockfile, err := GetLockfile(filepath.Join(volatileDir, "containers.lock")) if err != nil { return nil, err } @@ -386,7 +498,12 @@ func newContainerStore(dir string) (rwContainerStore, error) { byid: make(map[string]*Container), bylayer: make(map[string]*Container), byname: make(map[string]*Container), + jsonPath: [numContainerLocationIndex]string{ + filepath.Join(dir, "containers.json"), + filepath.Join(volatileDir, "volatile-containers.json"), + }, } + if err := cstore.startWritingWithReload(false); err != nil { return nil, err } @@ -418,7 +535,7 @@ func (r *containerStore) ClearFlag(id string, flag string) error { return ErrContainerUnknown } delete(container.Flags, flag) - return r.Save() + return r.saveFor(container) } func (r *containerStore) SetFlag(id string, flag string, value interface{}) error { @@ -430,7 +547,7 @@ func (r *containerStore) SetFlag(id string, flag string, value interface{}) erro container.Flags = make(map[string]interface{}) } container.Flags[flag] = value - return r.Save() + return r.saveFor(container) } func (r *containerStore) Create(id string, names []string, image, layer, metadata string, options *ContainerOptions) (container *Container, err error) { @@ -476,6 +593,7 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat Flags: copyStringInterfaceMap(options.Flags), UIDMap: copyIDMap(options.UIDMap), GIDMap: copyIDMap(options.GIDMap), + volatileStore: options.Volatile, } r.containers = append(r.containers, container) r.byid[id] = container @@ -486,7 +604,7 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat for _, name := range names { r.byname[name] = container } - err = r.Save() + err = r.saveFor(container) container = copyContainer(container) return container, err } @@ -501,7 +619,7 @@ func (r *containerStore) Metadata(id string) (string, error) { func (r *containerStore) SetMetadata(id, metadata string) error { if container, ok := r.lookup(id); ok { container.Metadata = metadata - return r.Save() + return r.saveFor(container) } return ErrContainerUnknown } @@ -530,7 +648,7 @@ func (r *containerStore) updateNames(id string, names []string, op updateNameOpe r.byname[name] = container } container.Names = names - return r.Save() + return r.saveFor(container) } func (r *containerStore) Delete(id string) error { @@ -562,7 +680,7 @@ func (r *containerStore) Delete(id string) error { r.containers = append(r.containers[:toDeleteIndex], r.containers[toDeleteIndex+1:]...) } } - if err := r.Save(); err != nil { + if err := r.saveFor(container); err != nil { return err } if err := os.RemoveAll(r.datadir(id)); err != nil { @@ -705,7 +823,7 @@ func (r *containerStore) SetBigData(id, key string, data []byte) error { save = true } if save { - err = r.Save() + err = r.saveFor(c) } } return err diff --git a/contrib/cirrus/build_and_test.sh b/contrib/cirrus/build_and_test.sh index cfb3a189a6..01e06d6126 100755 --- a/contrib/cirrus/build_and_test.sh +++ b/contrib/cirrus/build_and_test.sh @@ -13,6 +13,9 @@ case $TEST_DRIVER in overlay) showrun make STORAGE_DRIVER=overlay local-test-integration local-test-unit ;; + overlay-transient) + showrun make STORAGE_DRIVER=overlay STORAGE_TRANSIENT=1 local-test-integration local-test-unit + ;; fuse-overlay) showrun make STORAGE_DRIVER=overlay STORAGE_OPTION=overlay.mount_program=/usr/bin/fuse-overlayfs local-test-integration local-test-unit ;; diff --git a/drivers/aufs/aufs.go b/drivers/aufs/aufs.go index e384750237..10341d41ac 100644 --- a/drivers/aufs/aufs.go +++ b/drivers/aufs/aufs.go @@ -251,6 +251,11 @@ func (a *Driver) Exists(id string) bool { return true } +// List layers (not including additional image stores) +func (a *Driver) ListLayers() ([]string, error) { + return nil, graphdriver.ErrNotSupported +} + // AdditionalImageStores returns additional image stores supported by the driver func (a *Driver) AdditionalImageStores() []string { return nil diff --git a/drivers/btrfs/btrfs.go b/drivers/btrfs/btrfs.go index 1d6d466628..e8b16b858e 100644 --- a/drivers/btrfs/btrfs.go +++ b/drivers/btrfs/btrfs.go @@ -676,6 +676,11 @@ func (d *Driver) Exists(id string) bool { return err == nil } +// List layers (not including additional image stores) +func (d *Driver) ListLayers() ([]string, error) { + return nil, graphdriver.ErrNotSupported +} + // AdditionalImageStores returns additional image stores supported by the driver func (d *Driver) AdditionalImageStores() []string { return nil diff --git a/drivers/devmapper/driver.go b/drivers/devmapper/driver.go index 27a58a9eab..8b3ee51df7 100644 --- a/drivers/devmapper/driver.go +++ b/drivers/devmapper/driver.go @@ -267,6 +267,11 @@ func (d *Driver) Exists(id string) bool { return d.DeviceSet.HasDevice(id) } +// List layers (not including additional image stores) +func (d *Driver) ListLayers() ([]string, error) { + return nil, graphdriver.ErrNotSupported +} + // AdditionalImageStores returns additional image stores supported by the driver func (d *Driver) AdditionalImageStores() []string { return nil diff --git a/drivers/driver.go b/drivers/driver.go index e47d50a71c..1c1f017432 100644 --- a/drivers/driver.go +++ b/drivers/driver.go @@ -109,6 +109,9 @@ type ProtoDriver interface { // Exists returns whether a filesystem layer with the specified // ID exists on this driver. Exists(id string) bool + // Returns a list of layer ids that exist on this driver (does not include + // additional storage layers). Not supported by all backends. + ListLayers() ([]string, error) // Status returns a set of key-value pairs which give low // level diagnostic status about this driver. Status() [][2]string diff --git a/drivers/overlay/overlay.go b/drivers/overlay/overlay.go index d08e82dff0..49da0679a2 100644 --- a/drivers/overlay/overlay.go +++ b/drivers/overlay/overlay.go @@ -17,6 +17,7 @@ import ( "strings" "sync" "syscall" + "unicode" graphdriver "github.com/containers/storage/drivers" "github.com/containers/storage/drivers/overlayutils" @@ -1697,6 +1698,40 @@ func (d *Driver) Exists(id string) bool { return err == nil } +func nameLooksLikeID(name string) bool { + if len(name) != 64 { + return false + } + for _, c := range name { + if !unicode.Is(unicode.ASCII_Hex_Digit, c) { + return false + } + } + return true +} + +// List layers (not including additional image stores) +func (d *Driver) ListLayers() ([]string, error) { + entries, err := os.ReadDir(d.home) + if err != nil { + return nil, err + } + + layers := make([]string, 0) + + for _, entry := range entries { + id := entry.Name() + // Does it look like a datadir directory? + if !entry.IsDir() || !nameLooksLikeID(id) { + continue + } + + layers = append(layers, id) + } + + return layers, err +} + // isParent returns if the passed in parent is the direct parent of the passed in layer func (d *Driver) isParent(id, parent string) bool { lowers, err := d.getLowerDirs(id) diff --git a/drivers/vfs/driver.go b/drivers/vfs/driver.go index 50b4b28178..9deaa7c3a8 100644 --- a/drivers/vfs/driver.go +++ b/drivers/vfs/driver.go @@ -8,6 +8,7 @@ import ( "runtime" "strconv" "strings" + "unicode" graphdriver "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/archive" @@ -265,6 +266,40 @@ func (d *Driver) Exists(id string) bool { return err == nil } +func nameLooksLikeID(name string) bool { + if len(name) != 64 { + return false + } + for _, c := range name { + if !unicode.Is(unicode.ASCII_Hex_Digit, c) { + return false + } + } + return true +} + +// List layers (not including additional image stores) +func (d *Driver) ListLayers() ([]string, error) { + entries, err := os.ReadDir(d.homes[0]) + if err != nil { + return nil, err + } + + layers := make([]string, 0) + + for _, entry := range entries { + id := entry.Name() + // Does it look like a datadir directory? + if !entry.IsDir() || !nameLooksLikeID(id) { + continue + } + + layers = append(layers, id) + } + + return layers, err +} + // AdditionalImageStores returns additional image stores supported by the driver func (d *Driver) AdditionalImageStores() []string { if len(d.homes) > 1 { diff --git a/drivers/windows/windows.go b/drivers/windows/windows.go index 4cd9fa7244..66aa460cf8 100644 --- a/drivers/windows/windows.go +++ b/drivers/windows/windows.go @@ -185,6 +185,11 @@ func (d *Driver) Exists(id string) bool { return result } +// List layers (not including additional image stores) +func (d *Driver) ListLayers() ([]string, error) { + return nil, graphdriver.ErrNotSupported +} + // CreateFromTemplate creates a layer with the same contents and parent as another layer. func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { return graphdriver.NaiveCreateFromTemplate(d, id, template, templateIDMappings, parent, parentIDMappings, opts, readWrite) diff --git a/drivers/zfs/zfs.go b/drivers/zfs/zfs.go index d146782404..66da588b09 100644 --- a/drivers/zfs/zfs.go +++ b/drivers/zfs/zfs.go @@ -506,6 +506,11 @@ func (d *Driver) Exists(id string) bool { return d.filesystemsCache[d.zfsPath(id)] } +// List layers (not including additional image stores) +func (d *Driver) ListLayers() ([]string, error) { + return nil, graphdriver.ErrNotSupported +} + // AdditionalImageStores returns additional image stores supported by the driver func (d *Driver) AdditionalImageStores() []string { return nil diff --git a/layers.go b/layers.go index a0b9da4e93..3b580e2009 100644 --- a/layers.go +++ b/layers.go @@ -42,6 +42,22 @@ const ( maxLayerStoreCleanupIterations = 3 ) +type layerLocations uint8 + +// The backing store is split in two json files, one (the volatile) +// that is written without fsync() meaning it isn't as robust to +// unclean shutdown +const ( + stableLayerLocation layerLocations = 1 << iota + volatileLayerLocation + + numLayerLocationIndex = iota +) + +func layerLocationFromIndex(index int) layerLocations { + return 1 << index +} + // A Layer is a record of a copy-on-write layer that's stored by the lower // level graph driver. type Layer struct { @@ -123,6 +139,9 @@ type Layer struct { // ReadOnly is true if this layer resides in a read-only layer store. ReadOnly bool `json:"-"` + // volatileStore is true if the container is from the volatile json file + volatileStore bool `json:"-"` + // BigDataNames is a list of names of data items that we keep for the // convenience of the caller. They can be large, and are only in // memory when being read from or written to disk. @@ -276,23 +295,34 @@ type rwLayerStore interface { // store. // This API is experimental and can be changed without bumping the major version number. PutAdditionalLayer(id string, parentLayer *Layer, names []string, aLayer drivers.AdditionalLayer) (layer *Layer, err error) + + // Clean up unreferenced layers + GarbageCollect() error } type layerStore struct { - lockfile Locker - mountsLockfile Locker - rundir string - driver drivers.Driver - layerdir string - layers []*Layer - idindex *truncindex.TruncIndex - byid map[string]*Layer - byname map[string]*Layer - bymount map[string]*Layer - bycompressedsum map[digest.Digest][]string - byuncompressedsum map[digest.Digest][]string - loadMut sync.Mutex - layerspathModified time.Time + lockfile Locker + mountsLockfile Locker + rundir string + jsonPath [numLayerLocationIndex]string + driver drivers.Driver + layerdir string + layers []*Layer + idindex *truncindex.TruncIndex + byid map[string]*Layer + byname map[string]*Layer + bymount map[string]*Layer + bycompressedsum map[digest.Digest][]string + byuncompressedsum map[digest.Digest][]string + loadMut sync.Mutex + layerspathsModified [numLayerLocationIndex]time.Time +} + +func layerLocation(l *Layer) layerLocations { + if l.volatileStore { + return volatileLayerLocation + } + return stableLayerLocation } func copyLayer(l *Layer) *Layer { @@ -311,6 +341,7 @@ func copyLayer(l *Layer) *Layer { UncompressedSize: l.UncompressedSize, CompressionType: l.CompressionType, ReadOnly: l.ReadOnly, + volatileStore: l.volatileStore, BigDataNames: copyStringSlice(l.BigDataNames), Flags: copyStringInterfaceMap(l.Flags), UIDMap: copyIDMap(l.UIDMap), @@ -419,7 +450,7 @@ func (r *layerStore) stopReading() { // Modified() checks if the most recent writer was a party other than the // last recorded writer. It should only be called with the lock held. func (r *layerStore) Modified() (bool, error) { - var mmodified, tmodified bool + var mmodified bool lmodified, err := r.lockfile.Modified() if err != nil { return lmodified, err @@ -437,17 +468,20 @@ func (r *layerStore) Modified() (bool, error) { return true, nil } - // If the layers.json file has been modified manually, then we have to - // reload the storage in any case. - info, err := os.Stat(r.layerspath()) - if err != nil && !os.IsNotExist(err) { - return false, fmt.Errorf("stat layers file: %w", err) - } - if info != nil { - tmodified = info.ModTime() != r.layerspathModified + // If the layers.json file or container-layers.json has been + // modified manually, then we have to reload the storage in + // any case. + for locationIndex := 0; locationIndex < numLayerLocationIndex; locationIndex++ { + info, err := os.Stat(r.jsonPath[locationIndex]) + if err != nil && !os.IsNotExist(err) { + return false, fmt.Errorf("stat layers file: %w", err) + } + if info != nil && info.ModTime() != r.layerspathsModified[locationIndex] { + return true, nil + } } - return tmodified, nil + return false, nil } // reloadIfChanged reloads the contents of the store from disk if it is changed. @@ -481,12 +515,35 @@ func (r *layerStore) Layers() ([]Layer, error) { return layers, nil } -func (r *layerStore) mountspath() string { - return filepath.Join(r.rundir, "mountpoints.json") +func (r *layerStore) GarbageCollect() error { + layers, err := r.driver.ListLayers() + + if err != nil { + if errors.Is(err, drivers.ErrNotSupported) { + return nil + } + return err + } + + for _, id := range layers { + // Is the id still referenced + if r.byid[id] != nil { + continue + } + + // Remove layer and any related data of unreferenced id + if err := r.driver.Remove(id); err != nil { + return err + } + + os.Remove(r.tspath(id)) + os.RemoveAll(r.datadir(id)) + } + return nil } -func (r *layerStore) layerspath() string { - return filepath.Join(r.layerdir, "layers.json") +func (r *layerStore) mountspath() string { + return filepath.Join(r.rundir, "mountpoints.json") } // load reloads the contents of the store from disk. @@ -497,28 +554,49 @@ func (r *layerStore) layerspath() string { // If !lockedForWriting and this function fails, the return value indicates whether // retrying with lockedForWriting could succeed. func (r *layerStore) load(lockedForWriting bool) (bool, error) { - rpath := r.layerspath() - info, err := os.Stat(rpath) - if err != nil { - if !os.IsNotExist(err) { + var modifiedLocations layerLocations + + layers := []*Layer{} + ids := make(map[string]*Layer) + + for locationIndex := 0; locationIndex < numLayerLocationIndex; locationIndex++ { + location := layerLocationFromIndex(locationIndex) + rpath := r.jsonPath[locationIndex] + info, err := os.Stat(rpath) + if err != nil { + if !os.IsNotExist(err) { + return false, err + } + } else { + r.layerspathsModified[locationIndex] = info.ModTime() + } + data, err := os.ReadFile(rpath) + if err != nil && !os.IsNotExist(err) { return false, err } - } else { - r.layerspathModified = info.ModTime() - } - data, err := os.ReadFile(rpath) - if err != nil && !os.IsNotExist(err) { - return false, err - } - layers := []*Layer{} - if len(data) != 0 { - if err := json.Unmarshal(data, &layers); err != nil { - return false, fmt.Errorf("loading %q: %w", rpath, err) + locationLayers := []*Layer{} + if len(data) != 0 { + if err := json.Unmarshal(data, &locationLayers); err != nil { + return false, fmt.Errorf("loading %q: %w", rpath, err) + } + } + + for _, layer := range locationLayers { + // There should be no duplicated ids between json files, but lets check to be sure + if ids[layer.ID] != nil { + continue // skip invalid duplicated layer + } + // Remember where the layer came from + if location == volatileLayerLocation { + layer.volatileStore = true + } + layers = append(layers, layer) + ids[layer.ID] = layer } } + idlist := make([]string, 0, len(layers)) - ids := make(map[string]*Layer) names := make(map[string]*Layer) compressedsums := make(map[digest.Digest][]string) uncompressedsums := make(map[digest.Digest][]string) @@ -527,12 +605,12 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) { selinux.ClearLabels() } for n, layer := range layers { - ids[layer.ID] = layers[n] idlist = append(idlist, layer.ID) for _, name := range layer.Names { if conflict, ok := names[name]; ok { r.removeName(conflict, name) errorToResolveBySaving = ErrDuplicateLayerNames + modifiedLocations |= layerLocation(conflict) } names[name] = layers[n] } @@ -593,7 +671,7 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) { } if layerHasIncompleteFlag(layer) { logrus.Warnf("Found incomplete layer %#v, deleting it", layer.ID) - err = r.deleteInternal(layer.ID) + err := r.deleteInternal(layer.ID) if err != nil { // Don't return the error immediately, because deleteInternal does not saveLayers(); // Even if deleting one incomplete layer fails, call saveLayers() so that other possible successfully @@ -601,9 +679,10 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) { incompleteDeletionErrors = multierror.Append(incompleteDeletionErrors, fmt.Errorf("deleting layer %#v: %w", layer.ID, err)) } + modifiedLocations |= layerLocation(layer) } } - if err := r.saveLayers(); err != nil { + if err := r.saveLayers(modifiedLocations); err != nil { return false, err } if incompleteDeletionErrors != nil { @@ -652,37 +731,62 @@ func (r *layerStore) loadMounts() error { // Save saves the contents of the store to disk. It should be called with // the lock held, locked for writing. -func (r *layerStore) Save() error { +func (r *layerStore) save(saveLocations layerLocations) error { r.mountsLockfile.Lock() defer r.mountsLockfile.Unlock() - if err := r.saveLayers(); err != nil { + if err := r.saveLayers(saveLocations); err != nil { return err } return r.saveMounts() } -func (r *layerStore) saveLayers() error { +func (r *layerStore) saveFor(modifiedLayer *Layer) error { + return r.save(layerLocation(modifiedLayer)) +} + +func (r *layerStore) saveLayers(saveLocations layerLocations) error { if !r.lockfile.IsReadWrite() { - return fmt.Errorf("not allowed to modify the layer store at %q: %w", r.layerspath(), ErrStoreIsReadOnly) + return fmt.Errorf("not allowed to modify the layer store at %q: %w", r.layerdir, ErrStoreIsReadOnly) } r.lockfile.AssertLockedForWriting() - rpath := r.layerspath() - if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil { - return err - } - jldata, err := json.Marshal(&r.layers) - if err != nil { - return err - } - if err := ioutils.AtomicWriteFile(rpath, jldata, 0600); err != nil { - return err + + for locationIndex := 0; locationIndex < numLayerLocationIndex; locationIndex++ { + location := layerLocationFromIndex(locationIndex) + if location&saveLocations == 0 { + continue + } + rpath := r.jsonPath[locationIndex] + if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil { + return err + } + subsetLayers := make([]*Layer, 0, len(r.layers)) + for _, layer := range r.layers { + if layerLocation(layer) == location { + subsetLayers = append(subsetLayers, layer) + } + } + + jldata, err := json.Marshal(&subsetLayers) + if err != nil { + return err + } + var opts *ioutils.AtomicFileWriterOptions + if location == volatileLayerLocation { + opts = &ioutils.AtomicFileWriterOptions{ + NoSync: true, + } + } + if err := ioutils.AtomicWriteFileWithOpts(rpath, jldata, 0600, opts); err != nil { + return err + } + return r.lockfile.Touch() } - return r.lockfile.Touch() + return nil } func (r *layerStore) saveMounts() error { if !r.lockfile.IsReadWrite() { - return fmt.Errorf("not allowed to modify the layer store at %q: %w", r.layerspath(), ErrStoreIsReadOnly) + return fmt.Errorf("not allowed to modify the layer store at %q: %w", r.layerdir, ErrStoreIsReadOnly) } r.mountsLockfile.AssertLockedForWriting() mpath := r.mountspath() @@ -712,13 +816,18 @@ func (r *layerStore) saveMounts() error { return r.loadMounts() } -func (s *store) newLayerStore(rundir string, layerdir string, driver drivers.Driver) (rwLayerStore, error) { +func (s *store) newLayerStore(rundir string, layerdir string, driver drivers.Driver, transient bool) (rwLayerStore, error) { if err := os.MkdirAll(rundir, 0700); err != nil { return nil, err } if err := os.MkdirAll(layerdir, 0700); err != nil { return nil, err } + // Note: While the containers.lock file is in rundir for transient stores + // we don't want to do this here, because the non-transient layers in + // layers.json might be used externally as a read-only layer (using e.g. + // additionalimagestores), and that would look for the lockfile in the + // same directory lockfile, err := GetLockfile(filepath.Join(layerdir, "layers.lock")) if err != nil { return nil, err @@ -727,6 +836,10 @@ func (s *store) newLayerStore(rundir string, layerdir string, driver drivers.Dri if err != nil { return nil, err } + volatileDir := layerdir + if transient { + volatileDir = rundir + } rlstore := layerStore{ lockfile: lockfile, mountsLockfile: mountsLockfile, @@ -736,6 +849,10 @@ func (s *store) newLayerStore(rundir string, layerdir string, driver drivers.Dri byid: make(map[string]*Layer), bymount: make(map[string]*Layer), byname: make(map[string]*Layer), + jsonPath: [numLayerLocationIndex]string{ + filepath.Join(layerdir, "layers.json"), + filepath.Join(volatileDir, "volatile-layers.json"), + }, } if err := rlstore.startWritingWithReload(false); err != nil { return nil, err @@ -761,6 +878,10 @@ func newROLayerStore(rundir string, layerdir string, driver drivers.Driver) (roL byid: make(map[string]*Layer), bymount: make(map[string]*Layer), byname: make(map[string]*Layer), + jsonPath: [numLayerLocationIndex]string{ + filepath.Join(layerdir, "layers.json"), + filepath.Join(layerdir, "volatile-layers.json"), + }, } if err := rlstore.startReadingWithReload(false); err != nil { return nil, err @@ -800,19 +921,19 @@ func (r *layerStore) Size(name string) (int64, error) { func (r *layerStore) ClearFlag(id string, flag string) error { if !r.lockfile.IsReadWrite() { - return fmt.Errorf("not allowed to clear flags on layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly) + return fmt.Errorf("not allowed to clear flags on layers at %q: %w", r.layerdir, ErrStoreIsReadOnly) } layer, ok := r.lookup(id) if !ok { return ErrLayerUnknown } delete(layer.Flags, flag) - return r.Save() + return r.saveFor(layer) } func (r *layerStore) SetFlag(id string, flag string, value interface{}) error { if !r.lockfile.IsReadWrite() { - return fmt.Errorf("not allowed to set flags on layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly) + return fmt.Errorf("not allowed to set flags on layers at %q: %w", r.layerdir, ErrStoreIsReadOnly) } layer, ok := r.lookup(id) if !ok { @@ -822,7 +943,7 @@ func (r *layerStore) SetFlag(id string, flag string, value interface{}) error { layer.Flags = make(map[string]interface{}) } layer.Flags[flag] = value - return r.Save() + return r.saveFor(layer) } func (r *layerStore) Status() ([][2]string, error) { @@ -876,7 +997,7 @@ func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []s if layer.UncompressedDigest != "" { r.byuncompressedsum[layer.UncompressedDigest] = append(r.byuncompressedsum[layer.UncompressedDigest], layer.ID) } - if err := r.Save(); err != nil { + if err := r.saveFor(layer); err != nil { if err2 := r.driver.Remove(id); err2 != nil { logrus.Errorf("While recovering from a failure to save layers, error deleting layer %#v: %v", id, err2) } @@ -887,7 +1008,7 @@ func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []s func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}, diff io.Reader) (*Layer, int64, error) { if !r.lockfile.IsReadWrite() { - return nil, -1, fmt.Errorf("not allowed to create new layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly) + return nil, -1, fmt.Errorf("not allowed to create new layers at %q: %w", r.layerdir, ErrStoreIsReadOnly) } if err := os.MkdirAll(r.rundir, 0700); err != nil { return nil, -1, err @@ -975,6 +1096,7 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab UIDMap: copyIDMap(moreOptions.UIDMap), GIDMap: copyIDMap(moreOptions.GIDMap), BigDataNames: []string{}, + volatileStore: moreOptions.Volatile, } r.layers = append(r.layers, layer) // This can only fail if the ID is already missing, which shouldn’t happen — and in that case the index is already in the desired state anyway. @@ -1004,7 +1126,7 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab } }() - err := r.Save() + err := r.saveFor(layer) if err != nil { cleanupFailureContext = "saving incomplete layer metadata" return nil, -1, err @@ -1070,7 +1192,7 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab } } delete(layer.Flags, incompleteFlag) - err = r.Save() + err = r.saveFor(layer) if err != nil { cleanupFailureContext = "saving finished layer metadata" return nil, -1, err @@ -1285,7 +1407,7 @@ func (r *layerStore) removeName(layer *Layer, name string) { func (r *layerStore) updateNames(id string, names []string, op updateNameOperation) error { if !r.lockfile.IsReadWrite() { - return fmt.Errorf("not allowed to change layer name assignments at %q: %w", r.layerspath(), ErrStoreIsReadOnly) + return fmt.Errorf("not allowed to change layer name assignments at %q: %w", r.layerdir, ErrStoreIsReadOnly) } layer, ok := r.lookup(id) if !ok { @@ -1306,7 +1428,7 @@ func (r *layerStore) updateNames(id string, names []string, op updateNameOperati r.byname[name] = layer } layer.Names = names - return r.Save() + return r.saveFor(layer) } func (r *layerStore) datadir(id string) string { @@ -1333,7 +1455,7 @@ func (r *layerStore) SetBigData(id, key string, data io.Reader) error { return fmt.Errorf("can't set empty name for layer big data item: %w", ErrInvalidBigDataName) } if !r.lockfile.IsReadWrite() { - return fmt.Errorf("not allowed to save data items associated with layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly) + return fmt.Errorf("not allowed to save data items associated with layers at %q: %w", r.layerdir, ErrStoreIsReadOnly) } layer, ok := r.lookup(id) if !ok { @@ -1370,7 +1492,7 @@ func (r *layerStore) SetBigData(id, key string, data io.Reader) error { } if addName { layer.BigDataNames = append(layer.BigDataNames, key) - return r.Save() + return r.saveFor(layer) } return nil } @@ -1392,11 +1514,11 @@ func (r *layerStore) Metadata(id string) (string, error) { func (r *layerStore) SetMetadata(id, metadata string) error { if !r.lockfile.IsReadWrite() { - return fmt.Errorf("not allowed to modify layer metadata at %q: %w", r.layerspath(), ErrStoreIsReadOnly) + return fmt.Errorf("not allowed to modify layer metadata at %q: %w", r.layerdir, ErrStoreIsReadOnly) } if layer, ok := r.lookup(id); ok { layer.Metadata = metadata - return r.Save() + return r.saveFor(layer) } return ErrLayerUnknown } @@ -1418,7 +1540,7 @@ func layerHasIncompleteFlag(layer *Layer) bool { func (r *layerStore) deleteInternal(id string) error { if !r.lockfile.IsReadWrite() { - return fmt.Errorf("not allowed to delete layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly) + return fmt.Errorf("not allowed to delete layers at %q: %w", r.layerdir, ErrStoreIsReadOnly) } layer, ok := r.lookup(id) if !ok { @@ -1430,7 +1552,7 @@ func (r *layerStore) deleteInternal(id string) error { layer.Flags = make(map[string]interface{}) } layer.Flags[incompleteFlag] = true - if err := r.Save(); err != nil { + if err := r.saveFor(layer); err != nil { return err } } @@ -1532,7 +1654,7 @@ func (r *layerStore) Delete(id string) error { if err := r.deleteInternal(id); err != nil { return err } - return r.Save() + return r.saveFor(layer) } func (r *layerStore) Exists(id string) bool { @@ -1549,7 +1671,7 @@ func (r *layerStore) Get(id string) (*Layer, error) { func (r *layerStore) Wipe() error { if !r.lockfile.IsReadWrite() { - return fmt.Errorf("not allowed to delete layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly) + return fmt.Errorf("not allowed to delete layers at %q: %w", r.layerdir, ErrStoreIsReadOnly) } ids := make([]string, 0, len(r.byid)) for id := range r.byid { @@ -1814,7 +1936,7 @@ func (r *layerStore) ApplyDiff(to string, diff io.Reader) (size int64, err error func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, diff io.Reader) (size int64, err error) { if !r.lockfile.IsReadWrite() { - return -1, fmt.Errorf("not allowed to modify layer contents at %q: %w", r.layerspath(), ErrStoreIsReadOnly) + return -1, fmt.Errorf("not allowed to modify layer contents at %q: %w", r.layerdir, ErrStoreIsReadOnly) } layer, ok := r.lookup(to) @@ -1953,7 +2075,7 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, return layer.GIDs[i] < layer.GIDs[j] }) - err = r.Save() + err = r.saveFor(layer) return size, err } @@ -1994,7 +2116,7 @@ func (r *layerStore) ApplyDiffFromStagingDirectory(id, stagingDirectory string, layer.UncompressedDigest = diffOutput.UncompressedDigest layer.UncompressedSize = diffOutput.Size layer.Metadata = diffOutput.Metadata - if err = r.Save(); err != nil { + if err = r.saveFor(layer); err != nil { return err } for k, v := range diffOutput.BigData { @@ -2035,7 +2157,7 @@ func (r *layerStore) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOp } layer.UIDs = output.UIDs layer.GIDs = output.GIDs - err = r.Save() + err = r.saveFor(layer) return &output, err } diff --git a/pkg/ioutils/fswriters.go b/pkg/ioutils/fswriters.go index 17eed4e6f9..4b28c16ee5 100644 --- a/pkg/ioutils/fswriters.go +++ b/pkg/ioutils/fswriters.go @@ -61,8 +61,8 @@ func NewAtomicFileWriter(filename string, perm os.FileMode) (io.WriteCloser, err } // AtomicWriteFile atomically writes data to a file named by filename. -func AtomicWriteFile(filename string, data []byte, perm os.FileMode) error { - f, err := newAtomicFileWriter(filename, perm, nil) +func AtomicWriteFileWithOpts(filename string, data []byte, perm os.FileMode, opts *AtomicFileWriterOptions) error { + f, err := newAtomicFileWriter(filename, perm, opts) if err != nil { return err } @@ -77,6 +77,10 @@ func AtomicWriteFile(filename string, data []byte, perm os.FileMode) error { return err } +func AtomicWriteFile(filename string, data []byte, perm os.FileMode) error { + return AtomicWriteFileWithOpts(filename, data, perm, nil) +} + type atomicFileWriter struct { f *os.File fn string diff --git a/storage.conf b/storage.conf index 1294f6a9ab..6c419a952f 100644 --- a/storage.conf +++ b/storage.conf @@ -32,6 +32,10 @@ graphroot = "/var/lib/containers/storage" # # rootless_storage_path = "$HOME/.local/share/containers/storage" +# Transient store mode makes all container metadata be saved in temporary storage +# (i.e. runroot above). This is faster, but doesn't persist across reboots. +# transient_store = true + [storage.options] # Storage options to be passed to underlying storage drivers diff --git a/store.go b/store.go index b2d46128b6..9cb62d9b3d 100644 --- a/store.go +++ b/store.go @@ -141,6 +141,7 @@ type Store interface { // settings that were passed to GetStore() when the object was created. RunRoot() string GraphRoot() string + TransientStore() bool GraphDriverName() string GraphOptions() []string PullOptions() map[string]string @@ -502,6 +503,11 @@ type Store interface { // Releasing AdditionalLayer handler is caller's responsibility. // This API is experimental and can be changed without bumping the major version number. LookupAdditionalLayer(d digest.Digest, imageref string) (AdditionalLayer, error) + + // Tries to clean up remainders of previous containers or layers that are not + // references in the json files. These can happen in the case of unclean + // shutdowns or regular restarts in transient store mode. + GarbageCollect() error } // AdditionalLayer reprents a layer that is contained in the additional layer store @@ -545,6 +551,8 @@ type LayerOptions struct { // and reliably known by the caller. // Use the default "" if this fields is not applicable or the value is not known. UncompressedDigest digest.Digest + // True is the layer info can be treated as volatile + Volatile bool } // ImageOptions is used for passing options to a Store's CreateImage() method. @@ -594,6 +602,7 @@ type store struct { containerStore rwContainerStore digestLockRoot string disableVolatile bool + transientStore bool } // GetStore attempts to find an already-created Store object matching the @@ -701,6 +710,7 @@ func GetStore(options types.StoreOptions) (Store, error) { additionalGIDs: nil, usernsLock: usernsLock, disableVolatile: options.DisableVolatile, + transientStore: options.TransientStore, pullOptions: options.PullOptions, } if err := s.load(); err != nil { @@ -748,6 +758,10 @@ func (s *store) GraphRoot() string { return s.graphRoot } +func (s *store) TransientStore() bool { + return s.transientStore +} + func (s *store) GraphOptions() []string { return s.graphOptions } @@ -794,14 +808,16 @@ func (s *store) load() error { if err := os.MkdirAll(gcpath, 0700); err != nil { return err } - rcs, err := newContainerStore(gcpath) - if err != nil { - return err - } rcpath := filepath.Join(s.runRoot, driverPrefix+"containers") if err := os.MkdirAll(rcpath, 0700); err != nil { return err } + + rcs, err := newContainerStore(gcpath, rcpath, s.transientStore) + if err != nil { + return err + } + s.containerStore = rcs for _, store := range driver.AdditionalImageStores() { @@ -883,7 +899,7 @@ func (s *store) getLayerStore() (rwLayerStore, error) { if err := os.MkdirAll(glpath, 0700); err != nil { return nil, err } - rls, err := s.newLayerStore(rlpath, glpath, driver) + rls, err := s.newLayerStore(rlpath, glpath, driver, s.transientStore) if err != nil { return nil, err } @@ -1512,25 +1528,28 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat gidMap = s.gidMap } } - var layerOptions *LayerOptions + layerOptions := &LayerOptions{ + // Normally layers for containers are volatile only if the container is. + // But in transient store mode, all container layers are volatile. + Volatile: options.Volatile || s.transientStore, + } if s.canUseShifting(uidMap, gidMap) { - layerOptions = &LayerOptions{ - IDMappingOptions: types.IDMappingOptions{ + layerOptions.IDMappingOptions = + types.IDMappingOptions{ HostUIDMapping: true, HostGIDMapping: true, UIDMap: nil, GIDMap: nil, - }, - } + } } else { - layerOptions = &LayerOptions{ - IDMappingOptions: types.IDMappingOptions{ + layerOptions.IDMappingOptions = + types.IDMappingOptions{ HostUIDMapping: idMappingsOptions.HostUIDMapping, HostGIDMapping: idMappingsOptions.HostGIDMapping, UIDMap: copyIDMap(uidMap), GIDMap: copyIDMap(gidMap), - }, - } + } + } if options.Flags == nil { options.Flags = make(map[string]interface{}) @@ -1557,6 +1576,11 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat } layer = clayer.ID + // Normally only `--rm` containers are volatile, but in transient store mode all containers are volatile + if s.transientStore { + options.Volatile = true + } + var container *Container err = s.writeToContainerStore(func(rcstore rwContainerStore) error { options.IDMappingOptions = types.IDMappingOptions{ @@ -3346,3 +3370,20 @@ func (s *store) Free() { } } } + +// Tries to clean up old unreferenced container leftovers. returns the first error +// but continues as far as it can +func (s *store) GarbageCollect() error { + firstErr := s.writeToContainerStore(func(rcstore rwContainerStore) error { + return rcstore.GarbageCollect() + }) + + moreErr := s.writeToLayerStore(func(rlstore rwLayerStore) error { + return rlstore.GarbageCollect() + }) + if firstErr == nil { + firstErr = moreErr + } + + return firstErr +} diff --git a/tests/create-container.bats b/tests/create-container.bats index cec3b5c83f..76054c23f8 100644 --- a/tests/create-container.bats +++ b/tests/create-container.bats @@ -40,7 +40,7 @@ load helpers [ "$output" != "" ] firstcontainer=${output%% *} - firstwriter=$(cat ${TESTDIR}/root/${STORAGE_DRIVER}-containers/containers.lock) + firstwriter=$(cat ${TESTDIR}/${CONTAINERS_LOCK_ROOT}/${STORAGE_DRIVER}-containers/containers.lock) [ "$firstwriter" != "" ] # Check that the container can be found. @@ -52,7 +52,7 @@ load helpers [ "$output" != "" ] secondcontainer=${output%% *} - secondwriter=$(cat ${TESTDIR}/root/${STORAGE_DRIVER}-containers/containers.lock) + secondwriter=$(cat ${TESTDIR}/${CONTAINERS_LOCK_ROOT}/${STORAGE_DRIVER}-containers/containers.lock) [ "$secondwriter" != "" ] [ "$firstwriter" != "$secondwriter" ] diff --git a/tests/helpers.bash b/tests/helpers.bash index 884895c357..a1b32348a3 100755 --- a/tests/helpers.bash +++ b/tests/helpers.bash @@ -3,9 +3,17 @@ STORAGE_BINARY=${STORAGE_BINARY:-$(dirname ${BASH_SOURCE})/../containers-storage} TESTSDIR=${TESTSDIR:-$(dirname ${BASH_SOURCE})} STORAGE_DRIVER=${STORAGE_DRIVER:-vfs} +STORAGE_TRANSIENT=${STORAGE_TRANSIENT:-} STORAGE_OPTION=${STORAGE_OPTION:-} PATH=$(dirname ${BASH_SOURCE})/..:${PATH} OS=$(uname -s) +if [ "$STORAGE_TRANSIENT" -eq 1 ]; then + CONTAINERS_LOCK_ROOT=runroot + STORAGE_TRANSIENT_OPT=--transient-store +else + CONTAINERS_LOCK_ROOT=root + STORAGE_TRANSIENT_OPT="" +fi # Create a unique root directory and a runroot directory. function setup() { @@ -39,7 +47,7 @@ function createrandom() { # Run the CLI with the specified options. function storage() { - ${STORAGE_BINARY} --debug --graph ${TESTDIR}/root --run ${TESTDIR}/runroot --storage-driver ${STORAGE_DRIVER} ${STORAGE_OPTION:+--storage-opt=${STORAGE_OPTION}} "$@" + ${STORAGE_BINARY} --debug --graph ${TESTDIR}/root --run ${TESTDIR}/runroot ${STORAGE_TRANSIENT_OPT} --storage-driver ${STORAGE_DRIVER} ${STORAGE_OPTION:+--storage-opt=${STORAGE_OPTION}} "$@" } # Run the CLI with the specified options, and sort its output lines. diff --git a/types/options.go b/types/options.go index 45daa24edd..361cbf24f9 100644 --- a/types/options.go +++ b/types/options.go @@ -22,6 +22,7 @@ type TomlConfig struct { RunRoot string `toml:"runroot,omitempty"` GraphRoot string `toml:"graphroot,omitempty"` RootlessStoragePath string `toml:"rootless_storage_path,omitempty"` + TransientStore bool `toml:"transient_store,omitempty"` Options cfg.OptionsConfig `toml:"options,omitempty"` } `toml:"storage"` } @@ -234,6 +235,8 @@ type StoreOptions struct { PullOptions map[string]string `toml:"pull_options"` // DisableVolatile doesn't allow volatile mounts when it is set. DisableVolatile bool `json:"disable-volatile,omitempty"` + // If transient, don't persist containers over boot (stores db in runroot) + TransientStore bool `json:"transient_store,omitempty"` } // isRootlessDriver returns true if the given storage driver is valid for containers running as non root @@ -452,6 +455,7 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) erro } storeOptions.DisableVolatile = config.Storage.Options.DisableVolatile + storeOptions.TransientStore = config.Storage.TransientStore storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, cfg.GetGraphDriverOptions(storeOptions.GraphDriverName, config.Storage.Options)...) diff --git a/utils.go b/utils.go index ae9600e687..7f9e92b930 100644 --- a/utils.go +++ b/utils.go @@ -2,6 +2,7 @@ package storage import ( "fmt" + "unicode" "github.com/containers/storage/types" ) @@ -72,3 +73,15 @@ func applyNameOperation(oldNames []string, opParameters []string, op updateNameO } return dedupeNames(result), nil } + +func nameLooksLikeID(name string) bool { + if len(name) != 64 { + return false + } + for _, c := range name { + if !unicode.Is(unicode.ASCII_Hex_Digit, c) { + return false + } + } + return true +}