diff --git a/cmds/ocm/commands/cachecmds/clean/cmd.go b/cmds/ocm/commands/cachecmds/clean/cmd.go index b19108991d..c0a58b6acb 100644 --- a/cmds/ocm/commands/cachecmds/clean/cmd.go +++ b/cmds/ocm/commands/cachecmds/clean/cmd.go @@ -6,19 +6,21 @@ package clean import ( "fmt" - "sync" + "time" - "github.com/mandelsoft/vfs/pkg/vfs" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/open-component-model/ocm/cmds/ocm/commands/cachecmds/names" "github.com/open-component-model/ocm/cmds/ocm/commands/verbs" "github.com/open-component-model/ocm/cmds/ocm/pkg/utils" + "github.com/open-component-model/ocm/pkg/common" "github.com/open-component-model/ocm/pkg/common/accessio" "github.com/open-component-model/ocm/pkg/contexts/clictx" "github.com/open-component-model/ocm/pkg/contexts/oci/attrs/cacheattr" "github.com/open-component-model/ocm/pkg/errors" "github.com/open-component-model/ocm/pkg/out" + utils2 "github.com/open-component-model/ocm/pkg/utils" ) var ( @@ -33,7 +35,11 @@ type Cache interface { type Command struct { utils.BaseCommand - cache Cache + cache accessio.CleanupCache + + duration string + before time.Time + dryrun bool } // NewCommand creates a new artifact command. @@ -55,54 +61,65 @@ $ ocm clean cache } } +func (o *Command) AddFlags(fs *pflag.FlagSet) { + o.BaseCommand.AddFlags(fs) + fs.StringVarP(&o.duration, "before", "b", "", "time since last usage") + fs.BoolVarP(&o.dryrun, "dry-run", "s", false, "show size to be removed") +} + func (o *Command) Complete(args []string) error { c := cacheattr.Get(o.Context) if c == nil { return errors.Newf("no blob cache configured") } - r, ok := c.(Cache) + r, ok := c.(accessio.CleanupCache) if !ok { - return errors.Newf("only filesystem based caches are supported") + return errors.Newf("cache implementation does not support cleanup") } o.cache = r + if o.duration != "" { + if t, err := utils2.ParseDeltaTime(o.duration, true); err == nil { + o.before = t + } else { + t, err := time.Parse(time.RFC3339, o.duration) + if err != nil { + t, err = time.Parse(o.duration, o.duration) + } + if err != nil { + return fmt.Errorf("invalid lifetime %q", o.duration) + } + o.before = t + } + } return nil } func (o *Command) Run() error { - var size int64 - var fsize int64 - cnt := 0 - errs := 0 - if l, ok := o.cache.(sync.Locker); ok { - l.Lock() - defer l.Unlock() - } - path, fs := o.cache.Root() + cnt, ncnt, fcnt, size, nsize, fsize, err := o.cache.Cleanup(common.NewPrinter(o.Context.StdErr()), &o.before, o.dryrun) - entries, err := vfs.ReadDir(fs, path) if err != nil { return err } - for _, e := range entries { - err := fs.RemoveAll(vfs.Join(fs, path, e.Name())) - if err != nil { - out.Errf(o.Context, "cannot delete %q: %s\n", e.Name(), err) - errs++ - fsize += e.Size() + if !o.before.IsZero() { + if o.dryrun { + out.Outf(o.Context, "Matching %d/%d entries [%.3f/%.3f MB]\n", cnt, ncnt+cnt, float64(size)/1024/1024, float64(size+nsize)/1024/1024) } else { - cnt++ - size += e.Size() + out.Outf(o.Context, "Successfully deleted %d/%d entries [%.2f/%.3f MB]\n", cnt, ncnt+cnt, float64(size)/1024/1024, float64(size+nsize)/1024/1024) } - } - if cnt == 0 && errs > 0 { - return fmt.Errorf("Failed to delete %d entries [%.2f MB]\n", cnt, float64(fsize)/1024/1024) - } - if errs == 0 { - out.Outf(o.Context, "Successfully deleted %d entries [%.2f MB]\n", cnt, float64(size)/1024/1024) } else { - out.Outf(o.Context, "Deleted %d entries [%.2f MB]\n", cnt, float64(size)/1024/1024) - out.Outf(o.Context, "Failed to delete %d entries [%.2f MB]\n", cnt, float64(fsize)/1024/1024) + if o.dryrun { + out.Outf(o.Context, "Would remove %d entries [%.3f MB]\n", cnt, float64(size)/1024/1024) + } else { + out.Outf(o.Context, "Successfully deleted %d entries [%.3f MB]\n", cnt, float64(size)/1024/1024) + } + } + if fcnt > 0 { + if o.dryrun { + out.Outf(o.Context, "Failed to check %d entries [%.3f MB]\n", fcnt, float64(fsize)/1024/1024) + } else { + out.Outf(o.Context, "Failed to delete %d entries [%.3f MB]\n", fcnt, float64(fsize)/1024/1024) + } } return nil } diff --git a/cmds/ocm/commands/cachecmds/info/cmd.go b/cmds/ocm/commands/cachecmds/info/cmd.go index 665f1a8913..d896280dcc 100644 --- a/cmds/ocm/commands/cachecmds/info/cmd.go +++ b/cmds/ocm/commands/cachecmds/info/cmd.go @@ -5,9 +5,6 @@ package info import ( - "sync" - - "github.com/mandelsoft/vfs/pkg/vfs" "github.com/spf13/cobra" "github.com/open-component-model/ocm/cmds/ocm/commands/cachecmds/names" @@ -25,14 +22,9 @@ var ( Verb = verbs.Info ) -type Cache interface { - accessio.BlobCache - accessio.RootedCache -} - type Command struct { utils.BaseCommand - cache Cache + cache accessio.BlobCache } // NewCommand creates a new artifact command. @@ -55,38 +47,28 @@ $ ocm cache info } func (o *Command) Complete(args []string) error { - c := cacheattr.Get(o.Context) - if c == nil { + o.cache = cacheattr.Get(o.Context) + if o.cache == nil { return errors.Newf("no blob cache configured") } - r, ok := c.(Cache) - if !ok { - return errors.Newf("only filesystem based caches are supported") - } - o.cache = r return nil } func (o *Command) Run() error { - var size int64 - cnt := 0 - - if l, ok := o.cache.(sync.Locker); ok { - l.Lock() - defer l.Unlock() + if r, ok := o.cache.(accessio.RootedCache); ok { + path, fs := r.Root() + out.Outf(o.Context, "Used cache directory %s [%s]\n", path, fs.Name()) } - path, fs := o.cache.Root() - entries, err := vfs.ReadDir(fs, path) - if err != nil { - return err - } - for _, e := range entries { - cnt++ - size += e.Size() + if r, ok := o.cache.(accessio.CleanupCache); ok { + cnt, _, _, size, _, _, err := r.Cleanup(nil, nil, true) + if err != nil { + return err + } + out.Outf(o.Context, "Total cache size %d entries [%.3f MB]\n", cnt, float64(size)/1024/1024) + } else { + out.Outf(o.Context, "Cache does not support more info\n") } - out.Outf(o.Context, "Used cache directory %s [%s]\n", path, fs.Name()) - out.Outf(o.Context, "Total cache size %d entries [%.2f MB]\n", cnt, float64(size)/1024/1024) return nil } diff --git a/docs/reference/ocm_clean_cache.md b/docs/reference/ocm_clean_cache.md index 8e56c0dc21..133dd3105a 100644 --- a/docs/reference/ocm_clean_cache.md +++ b/docs/reference/ocm_clean_cache.md @@ -9,7 +9,9 @@ ocm clean cache [] ### Options ``` - -h, --help help for cache + -b, --before string time since last usage + -s, --dry-run show size to be removed + -h, --help help for cache ``` ### Description diff --git a/pkg/common/accessio/cache.go b/pkg/common/accessio/cache.go index 8666dce9ac..6f5e40f835 100644 --- a/pkg/common/accessio/cache.go +++ b/pkg/common/accessio/cache.go @@ -8,7 +8,9 @@ import ( "fmt" "io" "os" + "strings" "sync" + "time" "github.com/mandelsoft/vfs/pkg/osfs" "github.com/mandelsoft/vfs/pkg/projectionfs" @@ -132,6 +134,15 @@ type RootedCache interface { Root() (string, vfs.FileSystem) } +type CleanupCache interface { + // Cleanup can be implemented to offer a cache reorg. + // It returns the number and size of + // - handled entries (cnt, size) + // - not handled entries (ncnt, nsize) + // - failing entries (fcnt, fsize) + Cleanup(p common.Printer, before *time.Time, dryrun bool) (cnt int, ncnt int, fcnt int, size int64, nsize int64, fsize int64, err error) +} + type BlobCache interface { BlobSource BlobSink @@ -149,6 +160,12 @@ var ( _ RootedCache = (*blobCache)(nil) ) +// ACCESS_SUFFIX is the suffix of an additional blob related +// file used to track the last access time by its modification time, +// because Go does not support a platform independent way to access the +// last access time attribute of a filesystem. +const ACCESS_SUFFIX = ".acc" + func NewDefaultBlobCache(fss ...vfs.FileSystem) (BlobCache, error) { var err error fs := DefaultedFileSystem(nil, fss...) @@ -190,6 +207,61 @@ func (c *blobCache) Unlock() { c.lock.Unlock() } +func (c *blobCache) Cleanup(p common.Printer, before *time.Time, dryrun bool) (cnt int, ncnt int, fcnt int, size int64, nsize int64, fsize int64, err error) { + c.Lock() + defer c.Unlock() + + if p == nil { + p = common.NewPrinter(nil) + } + path, fs := c.Root() + + entries, err := vfs.ReadDir(fs, path) + if err != nil { + return 0, 0, 0, 0, 0, 0, err + } + for _, e := range entries { + if strings.HasSuffix(e.Name(), ACCESS_SUFFIX) { + continue + } + base := vfs.Join(fs, path, e.Name()) + if before != nil && !before.IsZero() { + fi, err := fs.Stat(base + ACCESS_SUFFIX) + if err != nil { + if !vfs.IsErrNotExist(err) { + if p != nil { + p.Printf("cannot stat %q: %s", e.Name(), err) + } + fcnt++ + fsize += e.Size() + continue + } + } else { + if fi.ModTime().After(*before) { + ncnt++ + nsize += e.Size() + continue + } + } + } + if !dryrun { + err := fs.RemoveAll(base) + if err != nil { + if p != nil { + p.Printf("cannot delete %q: %s", e.Name(), err) + } + fcnt++ + fsize += e.Size() + continue + } + fs.RemoveAll(base + ACCESS_SUFFIX) + } + cnt++ + size += e.Size() + } + return cnt, ncnt, fcnt, size, nsize, fsize, nil +} + func (c *blobCache) cleanup() error { return vfs.Cleanup(c.cache) } @@ -204,6 +276,9 @@ func (c *blobCache) GetBlobData(digest digest.Digest) (int64, DataAccess, error) path := common.DigestToFileName(digest) fi, err := c.cache.Stat(path) if err == nil { + vfs.WriteFile(c.cache, path+ACCESS_SUFFIX, []byte{}, 0o600) + // now := time.Now() + // c.cache.Chtimes(path+ACCESS_SUFFIX, now, now) return fi.Size(), DataAccessForFile(c.cache, path), nil } if os.IsNotExist(err) { @@ -272,6 +347,7 @@ func (c *blobCache) AddBlob(blob BlobAccess) (int64, digest.Digest, error) { err = c.cache.Rename(tmp, target) } c.cache.Remove(tmp) + vfs.WriteFile(c.cache, target+ACCESS_SUFFIX, []byte{}, 0o600) return size, digest, err } diff --git a/pkg/errors/list.go b/pkg/errors/list.go index dc7458c418..99b4ada2ff 100644 --- a/pkg/errors/list.go +++ b/pkg/errors/list.go @@ -60,6 +60,10 @@ func (l *ErrorList) Len() int { return len(l.errors) } +func (l *ErrorList) Entries() []error { + return l.errors +} + func (l *ErrorList) Result() error { if l == nil || len(l.errors) == 0 { return nil diff --git a/pkg/utils/time.go b/pkg/utils/time.go new file mode 100644 index 0000000000..e3df91e05e --- /dev/null +++ b/pkg/utils/time.go @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "strconv" + "time" + + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/generics" +) + +// ParseDeltaTime parses a time diff relative to the actual +// time and returns the resulting time. +func ParseDeltaTime(s string, past bool) (time.Time, error) { + var t time.Time + + f := int64(generics.Conditional(past, -1, 1)) + + if len(s) < 2 { + return t, errors.Newf("invalid time diff %q", s) + } + i, err := strconv.ParseInt(s[:len(s)-1], 10, 64) + if err != nil { + return t, errors.Wrapf(err, "invalid time diff %q", s) + } + + d := scale[s[len(s)-1:]] + if d == nil { + return t, errors.Newf("invalid time diff %q", s) + } + return d(i*f, time.Now()), nil +} + +type timeModifier func(d int64, t time.Time) time.Time + +var scale = map[string]timeModifier{ + "s": func(d int64, t time.Time) time.Time { return t.Add(time.Duration(d) * time.Second) }, + "m": func(d int64, t time.Time) time.Time { return t.Add(time.Duration(d) * time.Minute) }, + "h": func(d int64, t time.Time) time.Time { return t.Add(time.Duration(d) * time.Hour) }, + "d": func(d int64, t time.Time) time.Time { return t.AddDate(0, 0, int(d)) }, + "M": func(d int64, t time.Time) time.Time { return t.AddDate(0, int(d), 0) }, + "y": func(d int64, t time.Time) time.Time { return t.AddDate(int(d), 0, 0) }, +}