From 8a08c9f9b871c32c84f394abe513a8f6562297f0 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 22 May 2024 16:08:51 +0200 Subject: [PATCH 01/14] feat(ocis): Add backup consistency command Signed-off-by: jkoberg Co-authored-by: dragonchaser --- .../add-consistency-check-command.md | 5 + ocis/pkg/backup/backup.go | 189 ++++++++++++++++++ ocis/pkg/command/backup.go | 63 ++++++ 3 files changed, 257 insertions(+) create mode 100644 changelog/unreleased/add-consistency-check-command.md create mode 100644 ocis/pkg/backup/backup.go create mode 100644 ocis/pkg/command/backup.go diff --git a/changelog/unreleased/add-consistency-check-command.md b/changelog/unreleased/add-consistency-check-command.md new file mode 100644 index 00000000000..1f44a717518 --- /dev/null +++ b/changelog/unreleased/add-consistency-check-command.md @@ -0,0 +1,5 @@ +Enhancement: Add command to check ocis backup consistency + +Adds a command that checks the consistency of an ocis backup. + +https://github.com/owncloud/ocis/pull/9238 diff --git a/ocis/pkg/backup/backup.go b/ocis/pkg/backup/backup.go new file mode 100644 index 00000000000..8027984a358 --- /dev/null +++ b/ocis/pkg/backup/backup.go @@ -0,0 +1,189 @@ +// Package backup contains ocis backup functionality. +package backup + +import ( + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" + "github.com/shamaton/msgpack/v2" +) + +// Inconsistency describes the type of incosistency +type Inconsistency string + +var ( + InconsistencyBlobMissing Inconsistency = "blob missing" + InconsistencyBlobOrphaned Inconsistency = "blob orphaned" + InconsistencyNodeMissing Inconsistency = "node missing" + InconsistencyMetadataMissing Inconsistency = "metadata missing" + InconsistencySymlinkMissing Inconsistency = "symlink missing" +) + +// Consistency holds the node and blob data of a space +type Consistency struct { + Nodes map[string]Inconsistency + Links map[string]Inconsistency + Blobs map[string]Inconsistency + + fsys fs.FS + discpath string +} + +// ListBlobstore required to check blob consistency +type ListBlobstore interface { + Upload(node *node.Node, source string) error + Download(node *node.Node) (io.ReadCloser, error) + Delete(node *node.Node) error + List() ([]string, error) +} + +// New creates a new Consistency object +func New(fsys fs.FS, discpath string) *Consistency { + return &Consistency{ + Nodes: make(map[string]Inconsistency), + Links: make(map[string]Inconsistency), + Blobs: make(map[string]Inconsistency), + + fsys: fsys, + discpath: discpath, + } +} + +// CheckSpaceConsistency checks the consistency of a space +func CheckSpaceConsistency(pathToSpace string, lbs ListBlobstore) error { + fsys := os.DirFS(pathToSpace) + + con := New(fsys, pathToSpace) + err := con.Initialize() + if err != nil { + return err + } + + for n := range con.Nodes { + if _, ok := con.Links[n]; !ok { + // TODO: This is no inconsistency if + // * this is the spaceroot + // * this is a trashed node + con.Nodes[n] = InconsistencySymlinkMissing + continue + } + delete(con.Links, n) + delete(con.Nodes, n) + } + + for l := range con.Links { + con.Links[l] = InconsistencyNodeMissing + } + + blobs, err := lbs.List() + if err != nil { + return err + } + + deadBlobs := make(map[string]Inconsistency) + for _, b := range blobs { + if _, ok := con.Blobs[b]; !ok { + deadBlobs[b] = InconsistencyBlobOrphaned + continue + } + delete(con.Blobs, b) + delete(deadBlobs, b) + } + + for b := range con.Blobs { + con.Blobs[b] = InconsistencyBlobMissing + } + // get blobs from blobstore. + // initialize blobstore (s3, local) + // compare con.Blobs with blobstore.GetBlobs() + + for n := range con.Nodes { + fmt.Println("Inconsistency", n, con.Nodes[n]) + } + for l := range con.Links { + fmt.Println("Inconsistency", l, con.Links[l]) + } + for b := range con.Blobs { + fmt.Println("Inconsistency", b, con.Blobs[b]) + } + for b := range deadBlobs { + fmt.Println("Inconsistency", b, deadBlobs[b]) + } + + return nil +} + +func (c *Consistency) Initialize() error { + dirs, err := fs.Glob(c.fsys, "nodes/*/*/*/*") + if err != nil { + return err + } + + for _, d := range dirs { + entries, err := fs.ReadDir(c.fsys, d) + if err != nil { + return err + } + for _, e := range entries { + switch { + case e.IsDir(): + ls, err := fs.ReadDir(c.fsys, filepath.Join(d, e.Name())) + if err != nil { + fmt.Println("error reading dir", err) + continue + } + for _, l := range ls { + p, err := filepath.EvalSymlinks(filepath.Join(c.discpath, d, e.Name(), l.Name())) + if err != nil { + fmt.Println("error evaluating symlink", filepath.Join(d, e.Name(), l.Name()), err) + continue + } + c.Links[p] = "" + } + case filepath.Ext(e.Name()) == ".mpk": + inc, err := c.checkNode(filepath.Join(d, e.Name())) + if err != nil { + fmt.Println("error checking node", err) + continue + } + c.Nodes[filepath.Join(c.discpath, d, strings.TrimSuffix(e.Name(), ".mpk"))] = inc + default: + // fmt.Println("unknown", e.Name()) + } + } + } + return nil +} + +func (c *Consistency) checkNode(path string) (Inconsistency, error) { + b, err := fs.ReadFile(c.fsys, path) + if err != nil { + return "", err + } + + m := map[string][]byte{} + if err := msgpack.Unmarshal(b, &m); err != nil { + return "", err + } + + if bid := m["user.ocis.blobid"]; string(bid) != "" { + c.Blobs[string(bid)] = "" + } + + return "", nil +} + +func iterate(fsys fs.FS, path string, d *Consistency) ([]string, error) { + // open symlink -> NodeMissing + // remove node from data.Nodes + // check blob -> BlobMissing + // remove blob from data.Blobs + // list children (symlinks!) + // return children (symlinks!) + return nil, nil +} diff --git a/ocis/pkg/command/backup.go b/ocis/pkg/command/backup.go new file mode 100644 index 00000000000..3353b9995b7 --- /dev/null +++ b/ocis/pkg/command/backup.go @@ -0,0 +1,63 @@ +package command + +import ( + "fmt" + "path/filepath" + + "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" + "github.com/owncloud/ocis/v2/ocis-pkg/config" + "github.com/owncloud/ocis/v2/ocis/pkg/backup" + "github.com/owncloud/ocis/v2/ocis/pkg/register" + "github.com/urfave/cli/v2" +) + +// BackupCommand is the entrypoint for the backup command +func BackupCommand(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "backup", + Usage: "ocis backup functionality", + Subcommands: []*cli.Command{ + ConsistencyCommand(cfg), + }, + Action: func(c *cli.Context) error { + fmt.Println("Read the docs") + return nil + }, + } +} + +// ConsistencyCommand is the entrypoint for the consistency Command +func ConsistencyCommand(_ *config.Config) *cli.Command { + return &cli.Command{ + Name: "consistency", + Usage: "check backup consistency", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "blobstore", + Aliases: []string{"b"}, + Usage: "the blobstore type. Can be (ocis, s3ng). Default ocis", + }, + }, + Action: func(c *cli.Context) error { + basePath := "/home/jkoberg/.ocis/storage/users" + + // TODO: switch for s3ng blobstore + bs, err := blobstore.New(basePath) + if err != nil { + fmt.Println(err) + return err + } + + if err := backup.CheckSpaceConsistency(filepath.Join(basePath, "spaces/23/ebf113-76d4-43c0-8594-df974b02cd74"), bs); err != nil { + fmt.Println(err) + return err + } + + return nil + }, + } +} + +func init() { + register.AddCommand(BackupCommand) +} From ca12c2faa23ff43cb25661cc0d203607995b3eed Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 23 May 2024 12:03:31 +0200 Subject: [PATCH 02/14] feat(ocis): improve consistency command Signed-off-by: jkoberg Co-authored-by: dragonchaser Signed-off-by: jkoberg --- ocis/pkg/backup/backup.go | 264 ++++++++++++++++++++++++------------- ocis/pkg/command/backup.go | 42 ++++-- 2 files changed, 206 insertions(+), 100 deletions(-) diff --git a/ocis/pkg/backup/backup.go b/ocis/pkg/backup/backup.go index 8027984a358..a706080977a 100644 --- a/ocis/pkg/backup/backup.go +++ b/ocis/pkg/backup/backup.go @@ -7,28 +7,45 @@ import ( "io/fs" "os" "path/filepath" + "regexp" "strings" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/shamaton/msgpack/v2" ) -// Inconsistency describes the type of incosistency +// Inconsistency describes the type of inconsistency type Inconsistency string var ( - InconsistencyBlobMissing Inconsistency = "blob missing" - InconsistencyBlobOrphaned Inconsistency = "blob orphaned" - InconsistencyNodeMissing Inconsistency = "node missing" + // InconsistencyBlobMissing is an inconsistency where a blob is missing in the blobstore + InconsistencyBlobMissing Inconsistency = "blob missing" + // InconsistencyBlobOrphaned is an inconsistency where a blob in the blobstore has no reference + InconsistencyBlobOrphaned Inconsistency = "blob orphaned" + // InconsistencyNodeMissing is an inconsistency where a symlink points to a non-existing node + InconsistencyNodeMissing Inconsistency = "node missing" + // InconsistencyMetadataMissing is an inconsistency where a node is missing metadata InconsistencyMetadataMissing Inconsistency = "metadata missing" - InconsistencySymlinkMissing Inconsistency = "symlink missing" + // InconsistencySymlinkMissing is an inconsistency where a node is missing a symlink + InconsistencySymlinkMissing Inconsistency = "symlink missing" + // InconsistencyFilesMissing is an inconsistency where a node is missing metadata files like .mpk or .mlock + InconsistencyFilesMissing Inconsistency = "files missing" + // InconsistencyMalformedFile is an inconsistency where a node has a malformed metadata file + InconsistencyMalformedFile Inconsistency = "malformed file" + + // regex to determine if a node is trashed or versioned. + // 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z + _versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z$`) + // 9113a718-8285-4b32-9042-f930f1a58ac2.T.2024-05-23T08:25:20.006571811Z <- this HAS a symlink + _trashRegex = regexp.MustCompile(`\.T\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z$`) ) // Consistency holds the node and blob data of a space type Consistency struct { - Nodes map[string]Inconsistency - Links map[string]Inconsistency - Blobs map[string]Inconsistency + Nodes map[string][]Inconsistency + Links map[string][]Inconsistency + BlobReferences map[string][]Inconsistency + Blobs map[string][]Inconsistency fsys fs.FS discpath string @@ -45,9 +62,10 @@ type ListBlobstore interface { // New creates a new Consistency object func New(fsys fs.FS, discpath string) *Consistency { return &Consistency{ - Nodes: make(map[string]Inconsistency), - Links: make(map[string]Inconsistency), - Blobs: make(map[string]Inconsistency), + Nodes: make(map[string][]Inconsistency), + Links: make(map[string][]Inconsistency), + BlobReferences: make(map[string][]Inconsistency), + Blobs: make(map[string][]Inconsistency), fsys: fsys, discpath: discpath, @@ -55,71 +73,24 @@ func New(fsys fs.FS, discpath string) *Consistency { } // CheckSpaceConsistency checks the consistency of a space -func CheckSpaceConsistency(pathToSpace string, lbs ListBlobstore) error { - fsys := os.DirFS(pathToSpace) +func CheckSpaceConsistency(storagepath string, lbs ListBlobstore) error { + fsys := os.DirFS(storagepath) - con := New(fsys, pathToSpace) - err := con.Initialize() - if err != nil { + c := New(fsys, storagepath) + if err := c.Initialize(); err != nil { return err } - for n := range con.Nodes { - if _, ok := con.Links[n]; !ok { - // TODO: This is no inconsistency if - // * this is the spaceroot - // * this is a trashed node - con.Nodes[n] = InconsistencySymlinkMissing - continue - } - delete(con.Links, n) - delete(con.Nodes, n) - } - - for l := range con.Links { - con.Links[l] = InconsistencyNodeMissing - } - - blobs, err := lbs.List() - if err != nil { + if err := c.Evaluate(lbs); err != nil { return err } - deadBlobs := make(map[string]Inconsistency) - for _, b := range blobs { - if _, ok := con.Blobs[b]; !ok { - deadBlobs[b] = InconsistencyBlobOrphaned - continue - } - delete(con.Blobs, b) - delete(deadBlobs, b) - } - - for b := range con.Blobs { - con.Blobs[b] = InconsistencyBlobMissing - } - // get blobs from blobstore. - // initialize blobstore (s3, local) - // compare con.Blobs with blobstore.GetBlobs() - - for n := range con.Nodes { - fmt.Println("Inconsistency", n, con.Nodes[n]) - } - for l := range con.Links { - fmt.Println("Inconsistency", l, con.Links[l]) - } - for b := range con.Blobs { - fmt.Println("Inconsistency", b, con.Blobs[b]) - } - for b := range deadBlobs { - fmt.Println("Inconsistency", b, deadBlobs[b]) - } - - return nil + return c.PrintResults() } +// Initialize initializes the Consistency object func (c *Consistency) Initialize() error { - dirs, err := fs.Glob(c.fsys, "nodes/*/*/*/*") + dirs, err := fs.Glob(c.fsys, "spaces/*/*/nodes/*/*/*/*") if err != nil { return err } @@ -129,6 +100,7 @@ func (c *Consistency) Initialize() error { if err != nil { return err } + for _, e := range entries { switch { case e.IsDir(): @@ -138,52 +110,162 @@ func (c *Consistency) Initialize() error { continue } for _, l := range ls { - p, err := filepath.EvalSymlinks(filepath.Join(c.discpath, d, e.Name(), l.Name())) + p, err := os.Readlink(filepath.Join(c.discpath, d, e.Name(), l.Name())) if err != nil { - fmt.Println("error evaluating symlink", filepath.Join(d, e.Name(), l.Name()), err) - continue + fmt.Println("error reading symlink", err) } - c.Links[p] = "" + p = filepath.Join(c.discpath, d, e.Name(), p) + c.Links[p] = []Inconsistency{} } - case filepath.Ext(e.Name()) == ".mpk": - inc, err := c.checkNode(filepath.Join(d, e.Name())) - if err != nil { - fmt.Println("error checking node", err) - continue + fallthrough + case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()): + if !c.filesExist(filepath.Join(d, e.Name())) { + dp := filepath.Join(c.discpath, d, e.Name()) + c.Nodes[dp] = append(c.Nodes[dp], InconsistencyFilesMissing) + } + inc := c.checkNode(filepath.Join(d, e.Name()+".mpk")) + dp := filepath.Join(c.discpath, d, e.Name()) + if inc != "" { + c.Nodes[dp] = append(c.Nodes[dp], inc) + } else if len(c.Nodes[dp]) == 0 { + c.Nodes[dp] = []Inconsistency{} } - c.Nodes[filepath.Join(c.discpath, d, strings.TrimSuffix(e.Name(), ".mpk"))] = inc - default: - // fmt.Println("unknown", e.Name()) } } } + + links, err := fs.Glob(c.fsys, "spaces/*/*/trash/*/*/*/*/*") + if err != nil { + return err + } + for _, l := range links { + p, err := os.Readlink(filepath.Join(c.discpath, l)) + if err != nil { + fmt.Println("error reading symlink", err) + } + p = filepath.Join(c.discpath, l, "..", p) + c.Links[p] = []Inconsistency{} + } return nil } -func (c *Consistency) checkNode(path string) (Inconsistency, error) { +// Evaluate evaluates inconsistencies +func (c *Consistency) Evaluate(lbs ListBlobstore) error { + for n := range c.Nodes { + if _, ok := c.Links[n]; !ok && c.requiresSymlink(n) { + c.Nodes[n] = append(c.Nodes[n], InconsistencySymlinkMissing) + continue + } + + deleteInconsistency(c.Links, n) + deleteInconsistency(c.Nodes, n) + } + + for l := range c.Links { + c.Links[l] = append(c.Links[l], InconsistencyNodeMissing) + } + + blobs, err := lbs.List() + if err != nil { + return err + } + + for _, b := range blobs { + if _, ok := c.BlobReferences[b]; !ok { + c.Blobs[b] = append(c.Blobs[b], InconsistencyBlobOrphaned) + continue + } + deleteInconsistency(c.BlobReferences, b) + } + + for b := range c.BlobReferences { + c.BlobReferences[b] = append(c.BlobReferences[b], InconsistencyBlobMissing) + } + + return nil +} + +// PrintResults prints the results of the evaluation +func (c *Consistency) PrintResults() error { + if len(c.Nodes) != 0 { + fmt.Println("šŸšØ Inconsistent Nodes:") + } + for n := range c.Nodes { + fmt.Printf("\tšŸ‘‰ļø %v\tpath: %s\n", c.Nodes[n], n) + } + if len(c.Links) != 0 { + fmt.Println("šŸšØ Inconsistent Links:") + } + for l := range c.Links { + fmt.Printf("\tšŸ‘‰ļø %v\tpath: %s\n", c.Links[l], l) + } + if len(c.Blobs) != 0 { + fmt.Println("šŸšØ Inconsistent Blobs:") + } + for b := range c.Blobs { + fmt.Printf("\tšŸ‘‰ļø %v\tblob: %s\n", c.Blobs[b], b) + } + if len(c.BlobReferences) != 0 { + fmt.Println("šŸšØ Inconsistent BlobReferences:") + } + for b := range c.BlobReferences { + fmt.Printf("\tšŸ‘‰ļø %v\tblob: %s\n", c.BlobReferences[b], b) + } + if len(c.Nodes) == 0 && len(c.Links) == 0 && len(c.Blobs) == 0 && len(c.BlobReferences) == 0 { + fmt.Printf("šŸ’š No inconsistency found. The backup in '%s' seems to be valid.\n", c.discpath) + } + return nil + +} + +func (c *Consistency) checkNode(path string) Inconsistency { b, err := fs.ReadFile(c.fsys, path) if err != nil { - return "", err + return InconsistencyFilesMissing } m := map[string][]byte{} if err := msgpack.Unmarshal(b, &m); err != nil { - return "", err + return InconsistencyMalformedFile } if bid := m["user.ocis.blobid"]; string(bid) != "" { - c.Blobs[string(bid)] = "" + c.BlobReferences[string(bid)] = []Inconsistency{} + } + + return "" +} + +func (c *Consistency) requiresSymlink(path string) bool { + rawIDs := strings.Split(path, "/nodes/") + if len(rawIDs) != 2 { + return true + } + + s := strings.Split(rawIDs[0], "/spaces/") + if len(s) != 2 { + return true + } + + spaceID := strings.Replace(s[1], "/", "", -1) + nodeID := strings.Replace(rawIDs[1], "/", "", -1) + if spaceID == nodeID || _versionRegex.MatchString(nodeID) { + return false } - return "", nil + return true } -func iterate(fsys fs.FS, path string, d *Consistency) ([]string, error) { - // open symlink -> NodeMissing - // remove node from data.Nodes - // check blob -> BlobMissing - // remove blob from data.Blobs - // list children (symlinks!) - // return children (symlinks!) - return nil, nil +func (c *Consistency) filesExist(path string) bool { + check := func(p string) bool { + _, err := fs.Stat(c.fsys, p) + return err == nil + } + return check(path) && check(path+".mpk") +} + +func deleteInconsistency(incs map[string][]Inconsistency, path string) { + if len(incs[path]) == 0 { + delete(incs, path) + } } diff --git a/ocis/pkg/command/backup.go b/ocis/pkg/command/backup.go index 3353b9995b7..4f594579d42 100644 --- a/ocis/pkg/command/backup.go +++ b/ocis/pkg/command/backup.go @@ -2,9 +2,9 @@ package command import ( "fmt" - "path/filepath" - "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" + ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" + s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore" "github.com/owncloud/ocis/v2/ocis-pkg/config" "github.com/owncloud/ocis/v2/ocis/pkg/backup" "github.com/owncloud/ocis/v2/ocis/pkg/register" @@ -19,7 +19,7 @@ func BackupCommand(cfg *config.Config) *cli.Command { Subcommands: []*cli.Command{ ConsistencyCommand(cfg), }, - Action: func(c *cli.Context) error { + Action: func(_ *cli.Context) error { fmt.Println("Read the docs") return nil }, @@ -27,11 +27,17 @@ func BackupCommand(cfg *config.Config) *cli.Command { } // ConsistencyCommand is the entrypoint for the consistency Command -func ConsistencyCommand(_ *config.Config) *cli.Command { +func ConsistencyCommand(cfg *config.Config) *cli.Command { return &cli.Command{ Name: "consistency", Usage: "check backup consistency", Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "basepath", + Aliases: []string{"p"}, + Usage: "the basepath of the decomposedfs (e.g. /var/tmp/ocis/storage/users)", + Required: true, + }, &cli.StringFlag{ Name: "blobstore", Aliases: []string{"b"}, @@ -39,16 +45,34 @@ func ConsistencyCommand(_ *config.Config) *cli.Command { }, }, Action: func(c *cli.Context) error { - basePath := "/home/jkoberg/.ocis/storage/users" + basePath := c.String("basepath") + if basePath == "" { + fmt.Println("basepath is required") + return cli.ShowCommandHelp(c, "consistency") + } - // TODO: switch for s3ng blobstore - bs, err := blobstore.New(basePath) + var ( + bs backup.ListBlobstore + err error + ) + switch c.String("blobstore") { + case "s3ng": + bs, err = s3bs.New( + cfg.StorageUsers.Drivers.S3NG.Endpoint, + cfg.StorageUsers.Drivers.S3NG.Region, + cfg.StorageUsers.Drivers.S3NG.Bucket, + cfg.StorageUsers.Drivers.S3NG.AccessKey, + cfg.StorageUsers.Drivers.S3NG.SecretKey, + s3bs.Options{}, + ) + case "ocis": + bs, err = ocisbs.New(basePath) + } if err != nil { fmt.Println(err) return err } - - if err := backup.CheckSpaceConsistency(filepath.Join(basePath, "spaces/23/ebf113-76d4-43c0-8594-df974b02cd74"), bs); err != nil { + if err := backup.CheckSpaceConsistency(basePath, bs); err != nil { fmt.Println(err) return err } From ada83f93384288536a4127fe97948062984567c8 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 23 May 2024 15:49:46 +0200 Subject: [PATCH 03/14] feat(ci): test consistency command Signed-off-by: jkoberg Co-authored-by: dragonchaser --- .drone.star | 1 + ocis/pkg/command/backup.go | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.drone.star b/.drone.star index cdb55be3ea1..2c8de7e824b 100644 --- a/.drone.star +++ b/.drone.star @@ -1163,6 +1163,7 @@ def e2eTestPipeline(ctx): }, "commands": [ "cd %s/tests/e2e" % dirs["web"], + "ocis backup consistency -p /var/lib/ocis/storage/users", ], } diff --git a/ocis/pkg/command/backup.go b/ocis/pkg/command/backup.go index 4f594579d42..af4b2f6670f 100644 --- a/ocis/pkg/command/backup.go +++ b/ocis/pkg/command/backup.go @@ -1,11 +1,14 @@ package command import ( + "errors" "fmt" ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore" "github.com/owncloud/ocis/v2/ocis-pkg/config" + "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" + "github.com/owncloud/ocis/v2/ocis-pkg/config/parser" "github.com/owncloud/ocis/v2/ocis/pkg/backup" "github.com/owncloud/ocis/v2/ocis/pkg/register" "github.com/urfave/cli/v2" @@ -19,6 +22,9 @@ func BackupCommand(cfg *config.Config) *cli.Command { Subcommands: []*cli.Command{ ConsistencyCommand(cfg), }, + Before: func(c *cli.Context) error { + return configlog.ReturnError(parser.ParseConfig(cfg, false)) + }, Action: func(_ *cli.Context) error { fmt.Println("Read the docs") return nil @@ -42,6 +48,7 @@ func ConsistencyCommand(cfg *config.Config) *cli.Command { Name: "blobstore", Aliases: []string{"b"}, Usage: "the blobstore type. Can be (ocis, s3ng). Default ocis", + Value: "ocis", }, }, Action: func(c *cli.Context) error { @@ -67,6 +74,8 @@ func ConsistencyCommand(cfg *config.Config) *cli.Command { ) case "ocis": bs, err = ocisbs.New(basePath) + default: + err = errors.New("blobstore type not supported") } if err != nil { fmt.Println(err) From 6703dab9137a4c49cbcaedeb3ec5a10b05635b96 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 24 May 2024 10:32:18 +0200 Subject: [PATCH 04/14] feat(ocis): show full blob path when inconsistent Signed-off-by: jkoberg --- .drone.star | 4 ++- ocis/pkg/backup/backup.go | 59 +++++++++++++++++++++------------------ 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/.drone.star b/.drone.star index 2c8de7e824b..4ef44128ebe 100644 --- a/.drone.star +++ b/.drone.star @@ -1133,6 +1133,8 @@ def e2eTestPipeline(ctx): for item in defaults: params[item] = suite[item] if item in suite else defaults[item] + ocis_bin = "ocis/bin/ocis" + e2e_args = "" if params["totalParts"] > 0: e2e_args = "--total-parts %d" % params["totalParts"] @@ -1163,7 +1165,7 @@ def e2eTestPipeline(ctx): }, "commands": [ "cd %s/tests/e2e" % dirs["web"], - "ocis backup consistency -p /var/lib/ocis/storage/users", + "%s backup consistency -p /var/lib/ocis/storage/users" % ocis_bin, ], } diff --git a/ocis/pkg/backup/backup.go b/ocis/pkg/backup/backup.go index a706080977a..e5d3965e332 100644 --- a/ocis/pkg/backup/backup.go +++ b/ocis/pkg/backup/backup.go @@ -3,7 +3,6 @@ package backup import ( "fmt" - "io" "io/fs" "os" "path/filepath" @@ -53,10 +52,8 @@ type Consistency struct { // ListBlobstore required to check blob consistency type ListBlobstore interface { - Upload(node *node.Node, source string) error - Download(node *node.Node) (io.ReadCloser, error) - Delete(node *node.Node) error - List() ([]string, error) + List() ([]*node.Node, error) + Path(node *node.Node) string } // New creates a new Consistency object @@ -77,7 +74,7 @@ func CheckSpaceConsistency(storagepath string, lbs ListBlobstore) error { fsys := os.DirFS(storagepath) c := New(fsys, storagepath) - if err := c.Initialize(); err != nil { + if err := c.Initialize(lbs); err != nil { return err } @@ -89,7 +86,7 @@ func CheckSpaceConsistency(storagepath string, lbs ListBlobstore) error { } // Initialize initializes the Consistency object -func (c *Consistency) Initialize() error { +func (c *Consistency) Initialize(lbs ListBlobstore) error { dirs, err := fs.Glob(c.fsys, "spaces/*/*/nodes/*/*/*/*") if err != nil { return err @@ -123,7 +120,7 @@ func (c *Consistency) Initialize() error { dp := filepath.Join(c.discpath, d, e.Name()) c.Nodes[dp] = append(c.Nodes[dp], InconsistencyFilesMissing) } - inc := c.checkNode(filepath.Join(d, e.Name()+".mpk")) + inc := c.checkNode(filepath.Join(d, e.Name()+".mpk"), lbs) dp := filepath.Join(c.discpath, d, e.Name()) if inc != "" { c.Nodes[dp] = append(c.Nodes[dp], inc) @@ -170,12 +167,13 @@ func (c *Consistency) Evaluate(lbs ListBlobstore) error { return err } - for _, b := range blobs { - if _, ok := c.BlobReferences[b]; !ok { - c.Blobs[b] = append(c.Blobs[b], InconsistencyBlobOrphaned) + for _, bn := range blobs { + p := lbs.Path(bn) + if _, ok := c.BlobReferences[p]; !ok { + c.Blobs[p] = append(c.Blobs[p], InconsistencyBlobOrphaned) continue } - deleteInconsistency(c.BlobReferences, b) + deleteInconsistency(c.BlobReferences, p) } for b := range c.BlobReferences { @@ -218,7 +216,7 @@ func (c *Consistency) PrintResults() error { } -func (c *Consistency) checkNode(path string) Inconsistency { +func (c *Consistency) checkNode(path string, lbs ListBlobstore) Inconsistency { b, err := fs.ReadFile(c.fsys, path) if err != nil { return InconsistencyFilesMissing @@ -230,26 +228,17 @@ func (c *Consistency) checkNode(path string) Inconsistency { } if bid := m["user.ocis.blobid"]; string(bid) != "" { - c.BlobReferences[string(bid)] = []Inconsistency{} + spaceID, _ := getIDsFromPath(filepath.Join(c.discpath, path)) + p := lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}) + c.BlobReferences[p] = []Inconsistency{} } return "" } func (c *Consistency) requiresSymlink(path string) bool { - rawIDs := strings.Split(path, "/nodes/") - if len(rawIDs) != 2 { - return true - } - - s := strings.Split(rawIDs[0], "/spaces/") - if len(s) != 2 { - return true - } - - spaceID := strings.Replace(s[1], "/", "", -1) - nodeID := strings.Replace(rawIDs[1], "/", "", -1) - if spaceID == nodeID || _versionRegex.MatchString(nodeID) { + spaceID, nodeID := getIDsFromPath(path) + if nodeID != "" && spaceID != "" && (spaceID == nodeID || _versionRegex.MatchString(nodeID)) { return false } @@ -269,3 +258,19 @@ func deleteInconsistency(incs map[string][]Inconsistency, path string) { delete(incs, path) } } + +func getIDsFromPath(path string) (string, string) { + rawIDs := strings.Split(path, "/nodes/") + if len(rawIDs) != 2 { + return "", "" + } + + s := strings.Split(rawIDs[0], "/spaces/") + if len(s) != 2 { + return "", "" + } + + spaceID := strings.Replace(s[1], "/", "", -1) + nodeID := strings.Replace(rawIDs[1], "/", "", -1) + return spaceID, nodeID +} From 5086f55c75e8c34e7c184eca34da495ca9ea14cf Mon Sep 17 00:00:00 2001 From: jkoberg Date: Mon, 27 May 2024 13:53:37 +0200 Subject: [PATCH 05/14] feat(ci): test consistency command in ci II Signed-off-by: jkoberg Co-authored-by: dragonchaser --- .drone.star | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.drone.star b/.drone.star index 4ef44128ebe..4b5e36b63c8 100644 --- a/.drone.star +++ b/.drone.star @@ -1133,8 +1133,6 @@ def e2eTestPipeline(ctx): for item in defaults: params[item] = suite[item] if item in suite else defaults[item] - ocis_bin = "ocis/bin/ocis" - e2e_args = "" if params["totalParts"] > 0: e2e_args = "--total-parts %d" % params["totalParts"] @@ -1165,7 +1163,6 @@ def e2eTestPipeline(ctx): }, "commands": [ "cd %s/tests/e2e" % dirs["web"], - "%s backup consistency -p /var/lib/ocis/storage/users" % ocis_bin, ], } @@ -1992,6 +1989,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "commands": [ "%s init --insecure true" % ocis_bin, "cat $OCIS_CONFIG_DIR/ocis.yaml", + "%s backup consistency -p /var/lib/ocis/storage/users" % ocis_bin, ] + (wrapper_commands), "volumes": volumes, "depends_on": depends_on, From 2479a30babeb4b1bcce8bda8be9de6e6b510e538 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 29 May 2024 09:55:21 +0200 Subject: [PATCH 06/14] feat(ocis): list link location on inconsistency Signed-off-by: jkoberg --- ocis/pkg/backup/backup.go | 80 +++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 36 deletions(-) diff --git a/ocis/pkg/backup/backup.go b/ocis/pkg/backup/backup.go index e5d3965e332..2293bca0b17 100644 --- a/ocis/pkg/backup/backup.go +++ b/ocis/pkg/backup/backup.go @@ -39,33 +39,37 @@ var ( _trashRegex = regexp.MustCompile(`\.T\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z$`) ) +// ListBlobstore required to check blob consistency +type ListBlobstore interface { + List() ([]*node.Node, error) + Path(node *node.Node) string +} + // Consistency holds the node and blob data of a space type Consistency struct { Nodes map[string][]Inconsistency - Links map[string][]Inconsistency + LinkedNodes map[string][]Inconsistency BlobReferences map[string][]Inconsistency Blobs map[string][]Inconsistency - fsys fs.FS - discpath string -} - -// ListBlobstore required to check blob consistency -type ListBlobstore interface { - List() ([]*node.Node, error) - Path(node *node.Node) string + nodeToLink map[string]string + fsys fs.FS + discpath string + lbs ListBlobstore } // New creates a new Consistency object -func New(fsys fs.FS, discpath string) *Consistency { +func New(fsys fs.FS, discpath string, lbs ListBlobstore) *Consistency { return &Consistency{ Nodes: make(map[string][]Inconsistency), - Links: make(map[string][]Inconsistency), + LinkedNodes: make(map[string][]Inconsistency), BlobReferences: make(map[string][]Inconsistency), Blobs: make(map[string][]Inconsistency), - fsys: fsys, - discpath: discpath, + nodeToLink: make(map[string]string), + fsys: fsys, + discpath: discpath, + lbs: lbs, } } @@ -73,12 +77,12 @@ func New(fsys fs.FS, discpath string) *Consistency { func CheckSpaceConsistency(storagepath string, lbs ListBlobstore) error { fsys := os.DirFS(storagepath) - c := New(fsys, storagepath) - if err := c.Initialize(lbs); err != nil { + c := New(fsys, storagepath, lbs) + if err := c.Initialize(); err != nil { return err } - if err := c.Evaluate(lbs); err != nil { + if err := c.Evaluate(); err != nil { return err } @@ -86,7 +90,7 @@ func CheckSpaceConsistency(storagepath string, lbs ListBlobstore) error { } // Initialize initializes the Consistency object -func (c *Consistency) Initialize(lbs ListBlobstore) error { +func (c *Consistency) Initialize() error { dirs, err := fs.Glob(c.fsys, "spaces/*/*/nodes/*/*/*/*") if err != nil { return err @@ -107,12 +111,16 @@ func (c *Consistency) Initialize(lbs ListBlobstore) error { continue } for _, l := range ls { - p, err := os.Readlink(filepath.Join(c.discpath, d, e.Name(), l.Name())) + linkpath := filepath.Join(d, e.Name(), l.Name()) + r, err := os.Readlink(linkpath) if err != nil { - fmt.Println("error reading symlink", err) + // this can not happen as we just listed the directory + fmt.Println("directory entries changed while checking. Exiting.", err) + // return err } - p = filepath.Join(c.discpath, d, e.Name(), p) - c.Links[p] = []Inconsistency{} + nodePath := filepath.Join(c.discpath, d, e.Name(), r) + c.LinkedNodes[nodePath] = []Inconsistency{} + c.nodeToLink[nodePath] = linkpath } fallthrough case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()): @@ -120,7 +128,7 @@ func (c *Consistency) Initialize(lbs ListBlobstore) error { dp := filepath.Join(c.discpath, d, e.Name()) c.Nodes[dp] = append(c.Nodes[dp], InconsistencyFilesMissing) } - inc := c.checkNode(filepath.Join(d, e.Name()+".mpk"), lbs) + inc := c.checkNode(filepath.Join(d, e.Name()+".mpk")) dp := filepath.Join(c.discpath, d, e.Name()) if inc != "" { c.Nodes[dp] = append(c.Nodes[dp], inc) @@ -141,34 +149,34 @@ func (c *Consistency) Initialize(lbs ListBlobstore) error { fmt.Println("error reading symlink", err) } p = filepath.Join(c.discpath, l, "..", p) - c.Links[p] = []Inconsistency{} + c.LinkedNodes[p] = []Inconsistency{} } return nil } // Evaluate evaluates inconsistencies -func (c *Consistency) Evaluate(lbs ListBlobstore) error { +func (c *Consistency) Evaluate() error { for n := range c.Nodes { - if _, ok := c.Links[n]; !ok && c.requiresSymlink(n) { + if _, ok := c.LinkedNodes[n]; !ok && c.requiresSymlink(n) { c.Nodes[n] = append(c.Nodes[n], InconsistencySymlinkMissing) continue } - deleteInconsistency(c.Links, n) + deleteInconsistency(c.LinkedNodes, n) deleteInconsistency(c.Nodes, n) } - for l := range c.Links { - c.Links[l] = append(c.Links[l], InconsistencyNodeMissing) + for l := range c.LinkedNodes { + c.LinkedNodes[l] = append(c.LinkedNodes[l], InconsistencyNodeMissing) } - blobs, err := lbs.List() + blobs, err := c.lbs.List() if err != nil { return err } for _, bn := range blobs { - p := lbs.Path(bn) + p := c.lbs.Path(bn) if _, ok := c.BlobReferences[p]; !ok { c.Blobs[p] = append(c.Blobs[p], InconsistencyBlobOrphaned) continue @@ -191,11 +199,11 @@ func (c *Consistency) PrintResults() error { for n := range c.Nodes { fmt.Printf("\tšŸ‘‰ļø %v\tpath: %s\n", c.Nodes[n], n) } - if len(c.Links) != 0 { + if len(c.LinkedNodes) != 0 { fmt.Println("šŸšØ Inconsistent Links:") } - for l := range c.Links { - fmt.Printf("\tšŸ‘‰ļø %v\tpath: %s\n", c.Links[l], l) + for l := range c.LinkedNodes { + fmt.Printf("\tšŸ‘‰ļø %v\tpath: %s\tmissing node:%s\n", c.LinkedNodes[l], c.nodeToLink[l], l) } if len(c.Blobs) != 0 { fmt.Println("šŸšØ Inconsistent Blobs:") @@ -209,14 +217,14 @@ func (c *Consistency) PrintResults() error { for b := range c.BlobReferences { fmt.Printf("\tšŸ‘‰ļø %v\tblob: %s\n", c.BlobReferences[b], b) } - if len(c.Nodes) == 0 && len(c.Links) == 0 && len(c.Blobs) == 0 && len(c.BlobReferences) == 0 { + if len(c.Nodes) == 0 && len(c.LinkedNodes) == 0 && len(c.Blobs) == 0 && len(c.BlobReferences) == 0 { fmt.Printf("šŸ’š No inconsistency found. The backup in '%s' seems to be valid.\n", c.discpath) } return nil } -func (c *Consistency) checkNode(path string, lbs ListBlobstore) Inconsistency { +func (c *Consistency) checkNode(path string) Inconsistency { b, err := fs.ReadFile(c.fsys, path) if err != nil { return InconsistencyFilesMissing @@ -229,7 +237,7 @@ func (c *Consistency) checkNode(path string, lbs ListBlobstore) Inconsistency { if bid := m["user.ocis.blobid"]; string(bid) != "" { spaceID, _ := getIDsFromPath(filepath.Join(c.discpath, path)) - p := lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}) + p := c.lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}) c.BlobReferences[p] = []Inconsistency{} } From e4a46520f893c242eda12e1cbeaeece46d1ef815 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 29 May 2024 10:57:20 +0200 Subject: [PATCH 07/14] feat(ocis): improve output of backup command Signed-off-by: jkoberg Co-authored-by: dragonchaser --- ocis/pkg/backup/backup.go | 50 +++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/ocis/pkg/backup/backup.go b/ocis/pkg/backup/backup.go index 2293bca0b17..015702b0fa7 100644 --- a/ocis/pkg/backup/backup.go +++ b/ocis/pkg/backup/backup.go @@ -47,15 +47,19 @@ type ListBlobstore interface { // Consistency holds the node and blob data of a space type Consistency struct { + // Storing the data like this might take a lot of memory + // we might need to optimize this if we run into memory issues Nodes map[string][]Inconsistency LinkedNodes map[string][]Inconsistency BlobReferences map[string][]Inconsistency Blobs map[string][]Inconsistency nodeToLink map[string]string - fsys fs.FS - discpath string - lbs ListBlobstore + blobToNode map[string]string + + fsys fs.FS + discpath string + lbs ListBlobstore } // New creates a new Consistency object @@ -67,9 +71,11 @@ func New(fsys fs.FS, discpath string, lbs ListBlobstore) *Consistency { Blobs: make(map[string][]Inconsistency), nodeToLink: make(map[string]string), - fsys: fsys, - discpath: discpath, - lbs: lbs, + blobToNode: make(map[string]string), + + fsys: fsys, + discpath: discpath, + lbs: lbs, } } @@ -112,14 +118,11 @@ func (c *Consistency) Initialize() error { } for _, l := range ls { linkpath := filepath.Join(d, e.Name(), l.Name()) - r, err := os.Readlink(linkpath) - if err != nil { - // this can not happen as we just listed the directory - fmt.Println("directory entries changed while checking. Exiting.", err) - // return err - } + + // we always set InconsistencyNodeMissing as we later delete all referenced nodes from LinkedNodes + r, _ := os.Readlink(linkpath) nodePath := filepath.Join(c.discpath, d, e.Name(), r) - c.LinkedNodes[nodePath] = []Inconsistency{} + c.LinkedNodes[nodePath] = []Inconsistency{InconsistencyNodeMissing} c.nodeToLink[nodePath] = linkpath } fallthrough @@ -128,7 +131,7 @@ func (c *Consistency) Initialize() error { dp := filepath.Join(c.discpath, d, e.Name()) c.Nodes[dp] = append(c.Nodes[dp], InconsistencyFilesMissing) } - inc := c.checkNode(filepath.Join(d, e.Name()+".mpk")) + inc := c.checkNode(filepath.Join(d, e.Name())) dp := filepath.Join(c.discpath, d, e.Name()) if inc != "" { c.Nodes[dp] = append(c.Nodes[dp], inc) @@ -166,10 +169,6 @@ func (c *Consistency) Evaluate() error { deleteInconsistency(c.Nodes, n) } - for l := range c.LinkedNodes { - c.LinkedNodes[l] = append(c.LinkedNodes[l], InconsistencyNodeMissing) - } - blobs, err := c.lbs.List() if err != nil { return err @@ -194,28 +193,28 @@ func (c *Consistency) Evaluate() error { // PrintResults prints the results of the evaluation func (c *Consistency) PrintResults() error { if len(c.Nodes) != 0 { - fmt.Println("šŸšØ Inconsistent Nodes:") + fmt.Println("\nšŸšØ Inconsistent Nodes:") } for n := range c.Nodes { fmt.Printf("\tšŸ‘‰ļø %v\tpath: %s\n", c.Nodes[n], n) } if len(c.LinkedNodes) != 0 { - fmt.Println("šŸšØ Inconsistent Links:") + fmt.Println("\nšŸšØ Inconsistent Links:") } for l := range c.LinkedNodes { - fmt.Printf("\tšŸ‘‰ļø %v\tpath: %s\tmissing node:%s\n", c.LinkedNodes[l], c.nodeToLink[l], l) + fmt.Printf("\tšŸ‘‰ļø %v\tpath: %s\n\t\t\t\tmissing node:%s\n", c.LinkedNodes[l], c.nodeToLink[l], l) } if len(c.Blobs) != 0 { - fmt.Println("šŸšØ Inconsistent Blobs:") + fmt.Println("\nšŸšØ Inconsistent Blobs:") } for b := range c.Blobs { fmt.Printf("\tšŸ‘‰ļø %v\tblob: %s\n", c.Blobs[b], b) } if len(c.BlobReferences) != 0 { - fmt.Println("šŸšØ Inconsistent BlobReferences:") + fmt.Println("\nšŸšØ Inconsistent BlobReferences:") } for b := range c.BlobReferences { - fmt.Printf("\tšŸ‘‰ļø %v\tblob: %s\n", c.BlobReferences[b], b) + fmt.Printf("\tšŸ‘‰ļø %v\tblob: %s\n\t\t\t\treferencing node:%s\n", c.BlobReferences[b], b, c.blobToNode[b]) } if len(c.Nodes) == 0 && len(c.LinkedNodes) == 0 && len(c.Blobs) == 0 && len(c.BlobReferences) == 0 { fmt.Printf("šŸ’š No inconsistency found. The backup in '%s' seems to be valid.\n", c.discpath) @@ -225,7 +224,7 @@ func (c *Consistency) PrintResults() error { } func (c *Consistency) checkNode(path string) Inconsistency { - b, err := fs.ReadFile(c.fsys, path) + b, err := fs.ReadFile(c.fsys, path+".mpk") if err != nil { return InconsistencyFilesMissing } @@ -239,6 +238,7 @@ func (c *Consistency) checkNode(path string) Inconsistency { spaceID, _ := getIDsFromPath(filepath.Join(c.discpath, path)) p := c.lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}) c.BlobReferences[p] = []Inconsistency{} + c.blobToNode[p] = filepath.Join(c.discpath, path) } return "" From 969dabae78e0d4b509c3d53b5e211982c310e0a6 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 31 May 2024 10:09:54 +0200 Subject: [PATCH 08/14] fest(ocis): fix backup command Signed-off-by: jkoberg --- ocis/pkg/backup/backup.go | 34 ++++++++++++++++++++++++---------- ocis/pkg/command/backup.go | 2 +- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/ocis/pkg/backup/backup.go b/ocis/pkg/backup/backup.go index 015702b0fa7..946d07640ab 100644 --- a/ocis/pkg/backup/backup.go +++ b/ocis/pkg/backup/backup.go @@ -2,6 +2,7 @@ package backup import ( + "errors" "fmt" "io/fs" "os" @@ -79,8 +80,8 @@ func New(fsys fs.FS, discpath string, lbs ListBlobstore) *Consistency { } } -// CheckSpaceConsistency checks the consistency of a space -func CheckSpaceConsistency(storagepath string, lbs ListBlobstore) error { +// CheckProviderConsistency checks the consistency of a space +func CheckProviderConsistency(storagepath string, lbs ListBlobstore) error { fsys := os.DirFS(storagepath) c := New(fsys, storagepath, lbs) @@ -102,12 +103,21 @@ func (c *Consistency) Initialize() error { return err } + if len(dirs) == 0 { + return errors.New("no backup found. Double check storage path") + } + for _, d := range dirs { entries, err := fs.ReadDir(c.fsys, d) if err != nil { return err } + if len(entries) == 0 { + fmt.Println("empty dir", filepath.Join(c.discpath, d)) + continue + } + for _, e := range entries { switch { case e.IsDir(): @@ -117,12 +127,11 @@ func (c *Consistency) Initialize() error { continue } for _, l := range ls { - linkpath := filepath.Join(d, e.Name(), l.Name()) + linkpath := filepath.Join(c.discpath, d, e.Name(), l.Name()) - // we always set InconsistencyNodeMissing as we later delete all referenced nodes from LinkedNodes r, _ := os.Readlink(linkpath) nodePath := filepath.Join(c.discpath, d, e.Name(), r) - c.LinkedNodes[nodePath] = []Inconsistency{InconsistencyNodeMissing} + c.LinkedNodes[nodePath] = []Inconsistency{} c.nodeToLink[nodePath] = linkpath } fallthrough @@ -147,12 +156,11 @@ func (c *Consistency) Initialize() error { return err } for _, l := range links { - p, err := os.Readlink(filepath.Join(c.discpath, l)) - if err != nil { - fmt.Println("error reading symlink", err) - } - p = filepath.Join(c.discpath, l, "..", p) + linkpath := filepath.Join(c.discpath, l) + r, _ := os.Readlink(linkpath) + p := filepath.Join(c.discpath, l, "..", r) c.LinkedNodes[p] = []Inconsistency{} + c.nodeToLink[p] = linkpath } return nil } @@ -169,6 +177,11 @@ func (c *Consistency) Evaluate() error { deleteInconsistency(c.Nodes, n) } + // LinkedNodes should be empty now + for l := range c.LinkedNodes { + c.LinkedNodes[l] = append(c.LinkedNodes[l], InconsistencyNodeMissing) + } + blobs, err := c.lbs.List() if err != nil { return err @@ -183,6 +196,7 @@ func (c *Consistency) Evaluate() error { deleteInconsistency(c.BlobReferences, p) } + // BlobReferences should be empty now for b := range c.BlobReferences { c.BlobReferences[b] = append(c.BlobReferences[b], InconsistencyBlobMissing) } diff --git a/ocis/pkg/command/backup.go b/ocis/pkg/command/backup.go index af4b2f6670f..2c1031ab953 100644 --- a/ocis/pkg/command/backup.go +++ b/ocis/pkg/command/backup.go @@ -81,7 +81,7 @@ func ConsistencyCommand(cfg *config.Config) *cli.Command { fmt.Println(err) return err } - if err := backup.CheckSpaceConsistency(basePath, bs); err != nil { + if err := backup.CheckProviderConsistency(basePath, bs); err != nil { fmt.Println(err) return err } From e96921708eb5cc92c4023766dadb30813504dc56 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 31 May 2024 16:35:18 +0200 Subject: [PATCH 09/14] feat(ocis): concurrency for consistency command Signed-off-by: jkoberg --- ocis/pkg/backup/backup.go | 210 +++++++++++------------------------- ocis/pkg/backup/provider.go | 181 +++++++++++++++++++++++++++++++ 2 files changed, 245 insertions(+), 146 deletions(-) create mode 100644 ocis/pkg/backup/provider.go diff --git a/ocis/pkg/backup/backup.go b/ocis/pkg/backup/backup.go index 946d07640ab..8ca84fbbdbd 100644 --- a/ocis/pkg/backup/backup.go +++ b/ocis/pkg/backup/backup.go @@ -2,16 +2,11 @@ package backup import ( - "errors" "fmt" "io/fs" "os" - "path/filepath" "regexp" "strings" - - "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/shamaton/msgpack/v2" ) // Inconsistency describes the type of inconsistency @@ -40,13 +35,7 @@ var ( _trashRegex = regexp.MustCompile(`\.T\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z$`) ) -// ListBlobstore required to check blob consistency -type ListBlobstore interface { - List() ([]*node.Node, error) - Path(node *node.Node) string -} - -// Consistency holds the node and blob data of a space +// Consistency holds the node and blob data of a storage provider type Consistency struct { // Storing the data like this might take a lot of memory // we might need to optimize this if we run into memory issues @@ -57,14 +46,10 @@ type Consistency struct { nodeToLink map[string]string blobToNode map[string]string - - fsys fs.FS - discpath string - lbs ListBlobstore } -// New creates a new Consistency object -func New(fsys fs.FS, discpath string, lbs ListBlobstore) *Consistency { +// NewConsistency creates a new Consistency object +func NewConsistency() *Consistency { return &Consistency{ Nodes: make(map[string][]Inconsistency), LinkedNodes: make(map[string][]Inconsistency), @@ -73,10 +58,6 @@ func New(fsys fs.FS, discpath string, lbs ListBlobstore) *Consistency { nodeToLink: make(map[string]string), blobToNode: make(map[string]string), - - fsys: fsys, - discpath: discpath, - lbs: lbs, } } @@ -84,128 +65,86 @@ func New(fsys fs.FS, discpath string, lbs ListBlobstore) *Consistency { func CheckProviderConsistency(storagepath string, lbs ListBlobstore) error { fsys := os.DirFS(storagepath) - c := New(fsys, storagepath, lbs) - if err := c.Initialize(); err != nil { - return err - } - - if err := c.Evaluate(); err != nil { - return err - } - - return c.PrintResults() -} - -// Initialize initializes the Consistency object -func (c *Consistency) Initialize() error { - dirs, err := fs.Glob(c.fsys, "spaces/*/*/nodes/*/*/*/*") + nodes, links, blobs, quit, err := NewProvider(fsys, storagepath, lbs).ProduceData() if err != nil { return err } - if len(dirs) == 0 { - return errors.New("no backup found. Double check storage path") - } - - for _, d := range dirs { - entries, err := fs.ReadDir(c.fsys, d) - if err != nil { - return err - } - - if len(entries) == 0 { - fmt.Println("empty dir", filepath.Join(c.discpath, d)) - continue - } - - for _, e := range entries { - switch { - case e.IsDir(): - ls, err := fs.ReadDir(c.fsys, filepath.Join(d, e.Name())) - if err != nil { - fmt.Println("error reading dir", err) - continue - } - for _, l := range ls { - linkpath := filepath.Join(c.discpath, d, e.Name(), l.Name()) - - r, _ := os.Readlink(linkpath) - nodePath := filepath.Join(c.discpath, d, e.Name(), r) - c.LinkedNodes[nodePath] = []Inconsistency{} - c.nodeToLink[nodePath] = linkpath - } - fallthrough - case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()): - if !c.filesExist(filepath.Join(d, e.Name())) { - dp := filepath.Join(c.discpath, d, e.Name()) - c.Nodes[dp] = append(c.Nodes[dp], InconsistencyFilesMissing) - } - inc := c.checkNode(filepath.Join(d, e.Name())) - dp := filepath.Join(c.discpath, d, e.Name()) - if inc != "" { - c.Nodes[dp] = append(c.Nodes[dp], inc) - } else if len(c.Nodes[dp]) == 0 { - c.Nodes[dp] = []Inconsistency{} - } - } - } - } + c := NewConsistency() + c.GatherData(nodes, links, blobs, quit) - links, err := fs.Glob(c.fsys, "spaces/*/*/trash/*/*/*/*/*") - if err != nil { - return err - } - for _, l := range links { - linkpath := filepath.Join(c.discpath, l) - r, _ := os.Readlink(linkpath) - p := filepath.Join(c.discpath, l, "..", r) - c.LinkedNodes[p] = []Inconsistency{} - c.nodeToLink[p] = linkpath - } - return nil + return c.PrintResults(storagepath) } -// Evaluate evaluates inconsistencies -func (c *Consistency) Evaluate() error { +// GatherData gathers and evaluates data produced by the DataProvider +func (c *Consistency) GatherData(nodes chan NodeData, links chan LinkData, blobs chan BlobData, quit chan struct{}) { + c.gatherData(nodes, links, blobs, quit) + for n := range c.Nodes { - if _, ok := c.LinkedNodes[n]; !ok && c.requiresSymlink(n) { + if len(c.Nodes[n]) == 0 { c.Nodes[n] = append(c.Nodes[n], InconsistencySymlinkMissing) - continue } - - deleteInconsistency(c.LinkedNodes, n) - deleteInconsistency(c.Nodes, n) } - - // LinkedNodes should be empty now for l := range c.LinkedNodes { c.LinkedNodes[l] = append(c.LinkedNodes[l], InconsistencyNodeMissing) } - - blobs, err := c.lbs.List() - if err != nil { - return err - } - - for _, bn := range blobs { - p := c.lbs.Path(bn) - if _, ok := c.BlobReferences[p]; !ok { - c.Blobs[p] = append(c.Blobs[p], InconsistencyBlobOrphaned) - continue - } - deleteInconsistency(c.BlobReferences, p) + for b := range c.Blobs { + c.Blobs[b] = append(c.Blobs[b], InconsistencyBlobOrphaned) } - - // BlobReferences should be empty now for b := range c.BlobReferences { c.BlobReferences[b] = append(c.BlobReferences[b], InconsistencyBlobMissing) } +} + +func (c *Consistency) gatherData(nodes chan NodeData, links chan LinkData, blobs chan BlobData, quit chan struct{}) { + for { + select { + case n := <-nodes: + // does it have inconsistencies? + if len(n.Inconsistencies) != 0 { + c.Nodes[n.NodePath] = append(c.Nodes[n.NodePath], n.Inconsistencies...) + } + // is it linked? + if _, ok := c.LinkedNodes[n.NodePath]; ok { + deleteInconsistency(c.LinkedNodes, n.NodePath) + deleteInconsistency(c.Nodes, n.NodePath) + } else if requiresSymlink(n.NodePath) { + c.Nodes[n.NodePath] = c.Nodes[n.NodePath] + } + // does it have a blob? + if n.BlobPath != "" { + if _, ok := c.Blobs[n.BlobPath]; ok { + deleteInconsistency(c.Blobs, n.BlobPath) + } else { + c.BlobReferences[n.BlobPath] = []Inconsistency{} + c.blobToNode[n.BlobPath] = n.NodePath + } + } + case l := <-links: + // does it have a node? + if _, ok := c.Nodes[l.NodePath]; ok { + deleteInconsistency(c.Nodes, l.NodePath) + } else { + c.LinkedNodes[l.NodePath] = []Inconsistency{} + c.nodeToLink[l.NodePath] = l.LinkPath + } + case b := <-blobs: + // does it have a reference? + if _, ok := c.BlobReferences[b.BlobPath]; ok { + deleteInconsistency(c.BlobReferences, b.BlobPath) + } else { + c.Blobs[b.BlobPath] = []Inconsistency{} + } + case <-quit: + return + + } + } - return nil } // PrintResults prints the results of the evaluation -func (c *Consistency) PrintResults() error { +func (c *Consistency) PrintResults(discpath string) error { if len(c.Nodes) != 0 { fmt.Println("\nšŸšØ Inconsistent Nodes:") } @@ -231,34 +170,13 @@ func (c *Consistency) PrintResults() error { fmt.Printf("\tšŸ‘‰ļø %v\tblob: %s\n\t\t\t\treferencing node:%s\n", c.BlobReferences[b], b, c.blobToNode[b]) } if len(c.Nodes) == 0 && len(c.LinkedNodes) == 0 && len(c.Blobs) == 0 && len(c.BlobReferences) == 0 { - fmt.Printf("šŸ’š No inconsistency found. The backup in '%s' seems to be valid.\n", c.discpath) + fmt.Printf("šŸ’š No inconsistency found. The backup in '%s' seems to be valid.\n", discpath) } return nil } -func (c *Consistency) checkNode(path string) Inconsistency { - b, err := fs.ReadFile(c.fsys, path+".mpk") - if err != nil { - return InconsistencyFilesMissing - } - - m := map[string][]byte{} - if err := msgpack.Unmarshal(b, &m); err != nil { - return InconsistencyMalformedFile - } - - if bid := m["user.ocis.blobid"]; string(bid) != "" { - spaceID, _ := getIDsFromPath(filepath.Join(c.discpath, path)) - p := c.lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}) - c.BlobReferences[p] = []Inconsistency{} - c.blobToNode[p] = filepath.Join(c.discpath, path) - } - - return "" -} - -func (c *Consistency) requiresSymlink(path string) bool { +func requiresSymlink(path string) bool { spaceID, nodeID := getIDsFromPath(path) if nodeID != "" && spaceID != "" && (spaceID == nodeID || _versionRegex.MatchString(nodeID)) { return false @@ -267,7 +185,7 @@ func (c *Consistency) requiresSymlink(path string) bool { return true } -func (c *Consistency) filesExist(path string) bool { +func (c *DataProvider) filesExist(path string) bool { check := func(p string) bool { _, err := fs.Stat(c.fsys, p) return err == nil diff --git a/ocis/pkg/backup/provider.go b/ocis/pkg/backup/provider.go new file mode 100644 index 00000000000..917be0ad27b --- /dev/null +++ b/ocis/pkg/backup/provider.go @@ -0,0 +1,181 @@ +package backup + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sync" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" + "github.com/shamaton/msgpack/v2" +) + +// ListBlobstore required to check blob consistency +type ListBlobstore interface { + List() ([]*node.Node, error) + Path(node *node.Node) string +} + +// DataProvider provides data for the consistency check +type DataProvider struct { + fsys fs.FS + discpath string + lbs ListBlobstore +} + +// NodeData holds data about the nodes +type NodeData struct { + NodePath string + BlobPath string + Inconsistencies []Inconsistency +} + +// LinkData about the symlinks +type LinkData struct { + LinkPath string + NodePath string +} + +// BlobData about the blobs in the blobstore +type BlobData struct { + BlobPath string +} + +// NewProvider creates a new DataProvider object +func NewProvider(fsys fs.FS, discpath string, lbs ListBlobstore) *DataProvider { + return &DataProvider{ + fsys: fsys, + discpath: discpath, + lbs: lbs, + } +} + +// ProduceData produces data for the consistency check +func (c *DataProvider) ProduceData() (chan NodeData, chan LinkData, chan BlobData, chan struct{}, error) { + dirs, err := fs.Glob(c.fsys, "spaces/*/*/nodes/*/*/*/*") + if err != nil { + return nil, nil, nil, nil, err + } + + if len(dirs) == 0 { + return nil, nil, nil, nil, errors.New("no backup found. Double check storage path") + } + + nodes := make(chan NodeData) + links := make(chan LinkData) + blobs := make(chan BlobData) + quit := make(chan struct{}) + wg := sync.WaitGroup{} + + // crawl spaces + wg.Add(1) + go func() { + for _, d := range dirs { + entries, err := fs.ReadDir(c.fsys, d) + if err != nil { + fmt.Println("error reading dir", err) + continue + } + + if len(entries) == 0 { + fmt.Println("empty dir", filepath.Join(c.discpath, d)) + continue + } + + for _, e := range entries { + switch { + case e.IsDir(): + ls, err := fs.ReadDir(c.fsys, filepath.Join(d, e.Name())) + if err != nil { + fmt.Println("error reading dir", err) + continue + } + for _, l := range ls { + linkpath := filepath.Join(c.discpath, d, e.Name(), l.Name()) + + r, _ := os.Readlink(linkpath) + nodePath := filepath.Join(c.discpath, d, e.Name(), r) + links <- LinkData{LinkPath: linkpath, NodePath: nodePath} + } + fallthrough + case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()): + np := filepath.Join(c.discpath, d, e.Name()) + var inc []Inconsistency + if !c.filesExist(filepath.Join(d, e.Name())) { + inc = append(inc, InconsistencyFilesMissing) + } + bp, i := c.getBlobPath(filepath.Join(d, e.Name())) + if i != "" { + inc = append(inc, i) + } + + nodes <- NodeData{NodePath: np, BlobPath: bp, Inconsistencies: inc} + } + } + } + wg.Done() + }() + + // crawl trash + wg.Add(1) + go func() { + linkpaths, err := fs.Glob(c.fsys, "spaces/*/*/trash/*/*/*/*/*") + if err != nil { + fmt.Println("error reading trash", err) + } + for _, l := range linkpaths { + linkpath := filepath.Join(c.discpath, l) + r, _ := os.Readlink(linkpath) + p := filepath.Join(c.discpath, l, "..", r) + links <- LinkData{LinkPath: linkpath, NodePath: p} + } + wg.Done() + }() + + // crawl blobstore + wg.Add(1) + go func() { + bs, err := c.lbs.List() + if err != nil { + fmt.Println("error listing blobs", err) + } + + for _, bn := range bs { + blobs <- BlobData{BlobPath: c.lbs.Path(bn)} + } + wg.Done() + }() + + // wait for all crawlers to finish + go func() { + wg.Wait() + quit <- struct{}{} + close(nodes) + close(links) + close(blobs) + close(quit) + }() + + return nodes, links, blobs, quit, nil +} + +func (c *DataProvider) getBlobPath(path string) (string, Inconsistency) { + b, err := fs.ReadFile(c.fsys, path+".mpk") + if err != nil { + return "", InconsistencyFilesMissing + } + + m := map[string][]byte{} + if err := msgpack.Unmarshal(b, &m); err != nil { + return "", InconsistencyMalformedFile + } + + if bid := m["user.ocis.blobid"]; string(bid) != "" { + spaceID, _ := getIDsFromPath(filepath.Join(c.discpath, path)) + return c.lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}), "" + } + + return "", "" +} From d655c8140a6ef8ab2a9aae3f0ba905afc435ed48 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 31 May 2024 17:45:17 +0200 Subject: [PATCH 10/14] feat(ocis): finally unit tests for backup consistency Signed-off-by: jkoberg --- ocis/pkg/backup/backup.go | 48 ++-------- ocis/pkg/backup/backup_test.go | 169 +++++++++++++++++++++++++++++++++ ocis/pkg/backup/provider.go | 116 +++++++++++++++------- 3 files changed, 257 insertions(+), 76 deletions(-) create mode 100644 ocis/pkg/backup/backup_test.go diff --git a/ocis/pkg/backup/backup.go b/ocis/pkg/backup/backup.go index 8ca84fbbdbd..37a57c71e81 100644 --- a/ocis/pkg/backup/backup.go +++ b/ocis/pkg/backup/backup.go @@ -3,10 +3,8 @@ package backup import ( "fmt" - "io/fs" "os" "regexp" - "strings" ) // Inconsistency describes the type of inconsistency @@ -65,19 +63,19 @@ func NewConsistency() *Consistency { func CheckProviderConsistency(storagepath string, lbs ListBlobstore) error { fsys := os.DirFS(storagepath) - nodes, links, blobs, quit, err := NewProvider(fsys, storagepath, lbs).ProduceData() - if err != nil { + p := NewProvider(fsys, storagepath, lbs) + if err := p.ProduceData(); err != nil { return err } c := NewConsistency() - c.GatherData(nodes, links, blobs, quit) + c.GatherData(p.Nodes, p.Links, p.Blobs, p.Quit) return c.PrintResults(storagepath) } // GatherData gathers and evaluates data produced by the DataProvider -func (c *Consistency) GatherData(nodes chan NodeData, links chan LinkData, blobs chan BlobData, quit chan struct{}) { +func (c *Consistency) GatherData(nodes <-chan NodeData, links <-chan LinkData, blobs <-chan BlobData, quit <-chan struct{}) { c.gatherData(nodes, links, blobs, quit) for n := range c.Nodes { @@ -96,7 +94,7 @@ func (c *Consistency) GatherData(nodes chan NodeData, links chan LinkData, blobs } } -func (c *Consistency) gatherData(nodes chan NodeData, links chan LinkData, blobs chan BlobData, quit chan struct{}) { +func (c *Consistency) gatherData(nodes <-chan NodeData, links <-chan LinkData, blobs <-chan BlobData, quit <-chan struct{}) { for { select { case n := <-nodes: @@ -107,8 +105,7 @@ func (c *Consistency) gatherData(nodes chan NodeData, links chan LinkData, blobs // is it linked? if _, ok := c.LinkedNodes[n.NodePath]; ok { deleteInconsistency(c.LinkedNodes, n.NodePath) - deleteInconsistency(c.Nodes, n.NodePath) - } else if requiresSymlink(n.NodePath) { + } else if n.RequiresSymlink { c.Nodes[n.NodePath] = c.Nodes[n.NodePath] } // does it have a blob? @@ -176,41 +173,8 @@ func (c *Consistency) PrintResults(discpath string) error { } -func requiresSymlink(path string) bool { - spaceID, nodeID := getIDsFromPath(path) - if nodeID != "" && spaceID != "" && (spaceID == nodeID || _versionRegex.MatchString(nodeID)) { - return false - } - - return true -} - -func (c *DataProvider) filesExist(path string) bool { - check := func(p string) bool { - _, err := fs.Stat(c.fsys, p) - return err == nil - } - return check(path) && check(path+".mpk") -} - func deleteInconsistency(incs map[string][]Inconsistency, path string) { if len(incs[path]) == 0 { delete(incs, path) } } - -func getIDsFromPath(path string) (string, string) { - rawIDs := strings.Split(path, "/nodes/") - if len(rawIDs) != 2 { - return "", "" - } - - s := strings.Split(rawIDs[0], "/spaces/") - if len(s) != 2 { - return "", "" - } - - spaceID := strings.Replace(s[1], "/", "", -1) - nodeID := strings.Replace(rawIDs[1], "/", "", -1) - return spaceID, nodeID -} diff --git a/ocis/pkg/backup/backup_test.go b/ocis/pkg/backup/backup_test.go new file mode 100644 index 00000000000..6aff566f371 --- /dev/null +++ b/ocis/pkg/backup/backup_test.go @@ -0,0 +1,169 @@ +package backup_test + +import ( + "testing" + + "github.com/owncloud/ocis/v2/ocis/pkg/backup" + "github.com/test-go/testify/require" +) + +func TestGatherData(t *testing.T) { + testcases := []struct { + Name string + Events []interface{} + Expected *backup.Consistency + }{ + { + Name: "no symlinks - no blobs", + Events: []interface{}{ + nodeData("nodepath", "blobpath", true), + }, + Expected: consistency(func(c *backup.Consistency) { + node(c, "nodepath", backup.InconsistencySymlinkMissing) + blobReference(c, "blobpath", backup.InconsistencyBlobMissing) + }), + }, + { + Name: "symlink not required - no blobs", + Events: []interface{}{ + nodeData("nodepath", "blobpath", false), + }, + Expected: consistency(func(c *backup.Consistency) { + blobReference(c, "blobpath", backup.InconsistencyBlobMissing) + }), + }, + { + Name: "no inconsistencies", + Events: []interface{}{ + nodeData("nodepath", "blobpath", true), + linkData("linkpath", "nodepath"), + blobData("blobpath"), + }, + Expected: consistency(func(c *backup.Consistency) { + }), + }, + { + Name: "orphaned blob", + Events: []interface{}{ + nodeData("nodepath", "blobpath", true), + linkData("linkpath", "nodepath"), + blobData("blobpath"), + blobData("anotherpath"), + }, + Expected: consistency(func(c *backup.Consistency) { + blob(c, "anotherpath", backup.InconsistencyBlobOrphaned) + }), + }, + { + Name: "missing node", + Events: []interface{}{ + linkData("linkpath", "nodepath"), + blobData("blobpath"), + }, + Expected: consistency(func(c *backup.Consistency) { + linkedNode(c, "nodepath", backup.InconsistencyNodeMissing) + blob(c, "blobpath", backup.InconsistencyBlobOrphaned) + }), + }, + { + Name: "corrupt metadata", + Events: []interface{}{ + nodeData("nodepath", "blobpath", true, backup.InconsistencyMetadataMissing), + linkData("linkpath", "nodepath"), + blobData("blobpath"), + }, + Expected: consistency(func(c *backup.Consistency) { + node(c, "nodepath", backup.InconsistencyMetadataMissing) + }), + }, + { + Name: "corrupt metadata, no blob", + Events: []interface{}{ + nodeData("nodepath", "blobpath", true, backup.InconsistencyMetadataMissing), + linkData("linkpath", "nodepath"), + }, + Expected: consistency(func(c *backup.Consistency) { + node(c, "nodepath", backup.InconsistencyMetadataMissing) + blobReference(c, "blobpath", backup.InconsistencyBlobMissing) + }), + }, + } + + for _, tc := range testcases { + nodes := make(chan backup.NodeData) + links := make(chan backup.LinkData) + blobs := make(chan backup.BlobData) + quit := make(chan struct{}) + + go func() { + for _, ev := range tc.Events { + switch e := ev.(type) { + case backup.NodeData: + nodes <- e + case backup.LinkData: + links <- e + case backup.BlobData: + blobs <- e + } + } + quit <- struct{}{} + close(nodes) + close(links) + close(blobs) + close(quit) + }() + + c := backup.NewConsistency() + c.GatherData(nodes, links, blobs, quit) + + require.Equal(t, tc.Expected.Nodes, c.Nodes) + require.Equal(t, tc.Expected.LinkedNodes, c.LinkedNodes) + require.Equal(t, tc.Expected.Blobs, c.Blobs) + require.Equal(t, tc.Expected.BlobReferences, c.BlobReferences) + } + +} + +func nodeData(nodePath, blobPath string, requiresSymlink bool, incs ...backup.Inconsistency) backup.NodeData { + return backup.NodeData{ + NodePath: nodePath, + BlobPath: blobPath, + RequiresSymlink: requiresSymlink, + Inconsistencies: incs, + } +} + +func linkData(linkPath, nodePath string) backup.LinkData { + return backup.LinkData{ + LinkPath: linkPath, + NodePath: nodePath, + } +} + +func blobData(blobPath string) backup.BlobData { + return backup.BlobData{ + BlobPath: blobPath, + } +} + +func consistency(f func(*backup.Consistency)) *backup.Consistency { + c := backup.NewConsistency() + f(c) + return c +} + +func node(c *backup.Consistency, path string, inc ...backup.Inconsistency) { + c.Nodes[path] = inc +} + +func linkedNode(c *backup.Consistency, path string, inc ...backup.Inconsistency) { + c.LinkedNodes[path] = inc +} + +func blob(c *backup.Consistency, path string, inc ...backup.Inconsistency) { + c.Blobs[path] = inc +} + +func blobReference(c *backup.Consistency, path string, inc ...backup.Inconsistency) { + c.BlobReferences[path] = inc +} diff --git a/ocis/pkg/backup/provider.go b/ocis/pkg/backup/provider.go index 917be0ad27b..7f3e98529a8 100644 --- a/ocis/pkg/backup/provider.go +++ b/ocis/pkg/backup/provider.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" "sync" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" @@ -20,6 +21,11 @@ type ListBlobstore interface { // DataProvider provides data for the consistency check type DataProvider struct { + Nodes chan NodeData + Links chan LinkData + Blobs chan BlobData + Quit chan struct{} + fsys fs.FS discpath string lbs ListBlobstore @@ -29,6 +35,7 @@ type DataProvider struct { type NodeData struct { NodePath string BlobPath string + RequiresSymlink bool Inconsistencies []Inconsistency } @@ -46,6 +53,11 @@ type BlobData struct { // NewProvider creates a new DataProvider object func NewProvider(fsys fs.FS, discpath string, lbs ListBlobstore) *DataProvider { return &DataProvider{ + Nodes: make(chan NodeData), + Links: make(chan LinkData), + Blobs: make(chan BlobData), + Quit: make(chan struct{}), + fsys: fsys, discpath: discpath, lbs: lbs, @@ -53,65 +65,62 @@ func NewProvider(fsys fs.FS, discpath string, lbs ListBlobstore) *DataProvider { } // ProduceData produces data for the consistency check -func (c *DataProvider) ProduceData() (chan NodeData, chan LinkData, chan BlobData, chan struct{}, error) { - dirs, err := fs.Glob(c.fsys, "spaces/*/*/nodes/*/*/*/*") +// Spawns 4 go-routines at the moment. If needed, this can be optimized. +func (dp *DataProvider) ProduceData() error { + dirs, err := fs.Glob(dp.fsys, "spaces/*/*/nodes/*/*/*/*") if err != nil { - return nil, nil, nil, nil, err + return err } if len(dirs) == 0 { - return nil, nil, nil, nil, errors.New("no backup found. Double check storage path") + return errors.New("no backup found. Double check storage path") } - nodes := make(chan NodeData) - links := make(chan LinkData) - blobs := make(chan BlobData) - quit := make(chan struct{}) wg := sync.WaitGroup{} // crawl spaces wg.Add(1) go func() { for _, d := range dirs { - entries, err := fs.ReadDir(c.fsys, d) + entries, err := fs.ReadDir(dp.fsys, d) if err != nil { fmt.Println("error reading dir", err) continue } if len(entries) == 0 { - fmt.Println("empty dir", filepath.Join(c.discpath, d)) + fmt.Println("empty dir", filepath.Join(dp.discpath, d)) continue } for _, e := range entries { switch { case e.IsDir(): - ls, err := fs.ReadDir(c.fsys, filepath.Join(d, e.Name())) + ls, err := fs.ReadDir(dp.fsys, filepath.Join(d, e.Name())) if err != nil { fmt.Println("error reading dir", err) continue } for _, l := range ls { - linkpath := filepath.Join(c.discpath, d, e.Name(), l.Name()) + linkpath := filepath.Join(dp.discpath, d, e.Name(), l.Name()) r, _ := os.Readlink(linkpath) - nodePath := filepath.Join(c.discpath, d, e.Name(), r) - links <- LinkData{LinkPath: linkpath, NodePath: nodePath} + nodePath := filepath.Join(dp.discpath, d, e.Name(), r) + dp.Links <- LinkData{LinkPath: linkpath, NodePath: nodePath} } fallthrough case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()): - np := filepath.Join(c.discpath, d, e.Name()) + np := filepath.Join(dp.discpath, d, e.Name()) var inc []Inconsistency - if !c.filesExist(filepath.Join(d, e.Name())) { + if !dp.filesExist(filepath.Join(d, e.Name())) { inc = append(inc, InconsistencyFilesMissing) } - bp, i := c.getBlobPath(filepath.Join(d, e.Name())) + bp, i := dp.getBlobPath(filepath.Join(d, e.Name())) if i != "" { inc = append(inc, i) } - nodes <- NodeData{NodePath: np, BlobPath: bp, Inconsistencies: inc} + dp.Nodes <- NodeData{NodePath: np, BlobPath: bp, RequiresSymlink: requiresSymlink(np), Inconsistencies: inc} } } } @@ -121,15 +130,15 @@ func (c *DataProvider) ProduceData() (chan NodeData, chan LinkData, chan BlobDat // crawl trash wg.Add(1) go func() { - linkpaths, err := fs.Glob(c.fsys, "spaces/*/*/trash/*/*/*/*/*") + linkpaths, err := fs.Glob(dp.fsys, "spaces/*/*/trash/*/*/*/*/*") if err != nil { fmt.Println("error reading trash", err) } for _, l := range linkpaths { - linkpath := filepath.Join(c.discpath, l) + linkpath := filepath.Join(dp.discpath, l) r, _ := os.Readlink(linkpath) - p := filepath.Join(c.discpath, l, "..", r) - links <- LinkData{LinkPath: linkpath, NodePath: p} + p := filepath.Join(dp.discpath, l, "..", r) + dp.Links <- LinkData{LinkPath: linkpath, NodePath: p} } wg.Done() }() @@ -137,13 +146,13 @@ func (c *DataProvider) ProduceData() (chan NodeData, chan LinkData, chan BlobDat // crawl blobstore wg.Add(1) go func() { - bs, err := c.lbs.List() + bs, err := dp.lbs.List() if err != nil { fmt.Println("error listing blobs", err) } for _, bn := range bs { - blobs <- BlobData{BlobPath: c.lbs.Path(bn)} + dp.Blobs <- BlobData{BlobPath: dp.lbs.Path(bn)} } wg.Done() }() @@ -151,18 +160,14 @@ func (c *DataProvider) ProduceData() (chan NodeData, chan LinkData, chan BlobDat // wait for all crawlers to finish go func() { wg.Wait() - quit <- struct{}{} - close(nodes) - close(links) - close(blobs) - close(quit) + dp.quit() }() - return nodes, links, blobs, quit, nil + return nil } -func (c *DataProvider) getBlobPath(path string) (string, Inconsistency) { - b, err := fs.ReadFile(c.fsys, path+".mpk") +func (dp *DataProvider) getBlobPath(path string) (string, Inconsistency) { + b, err := fs.ReadFile(dp.fsys, path+".mpk") if err != nil { return "", InconsistencyFilesMissing } @@ -172,10 +177,53 @@ func (c *DataProvider) getBlobPath(path string) (string, Inconsistency) { return "", InconsistencyMalformedFile } + // FIXME: how to check if metadata is complete? + if bid := m["user.ocis.blobid"]; string(bid) != "" { - spaceID, _ := getIDsFromPath(filepath.Join(c.discpath, path)) - return c.lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}), "" + spaceID, _ := getIDsFromPath(filepath.Join(dp.discpath, path)) + return dp.lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}), "" } return "", "" } + +func (dp *DataProvider) filesExist(path string) bool { + check := func(p string) bool { + _, err := fs.Stat(dp.fsys, p) + return err == nil + } + return check(path) && check(path+".mpk") +} + +func (dp *DataProvider) quit() { + dp.Quit <- struct{}{} + close(dp.Nodes) + close(dp.Links) + close(dp.Blobs) + close(dp.Quit) +} + +func requiresSymlink(path string) bool { + spaceID, nodeID := getIDsFromPath(path) + if nodeID != "" && spaceID != "" && (spaceID == nodeID || _versionRegex.MatchString(nodeID)) { + return false + } + + return true +} + +func getIDsFromPath(path string) (string, string) { + rawIDs := strings.Split(path, "/nodes/") + if len(rawIDs) != 2 { + return "", "" + } + + s := strings.Split(rawIDs[0], "/spaces/") + if len(s) != 2 { + return "", "" + } + + spaceID := strings.Replace(s[1], "/", "", -1) + nodeID := strings.Replace(rawIDs[1], "/", "", -1) + return spaceID, nodeID +} From 1525f7ec2a43602194aa54cb7cb8a537faeab602 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 31 May 2024 18:15:54 +0200 Subject: [PATCH 11/14] feat(ocis): use only one channel for backup command Signed-off-by: jkoberg --- .drone.star | 1 - ocis/pkg/backup/backup.go | 85 ++++++++++----------- ocis/pkg/backup/backup_test.go | 19 ++--- ocis/pkg/backup/provider.go | 130 ++++++++++++++++----------------- 4 files changed, 110 insertions(+), 125 deletions(-) diff --git a/.drone.star b/.drone.star index 4b5e36b63c8..cdb55be3ea1 100644 --- a/.drone.star +++ b/.drone.star @@ -1989,7 +1989,6 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "commands": [ "%s init --insecure true" % ocis_bin, "cat $OCIS_CONFIG_DIR/ocis.yaml", - "%s backup consistency -p /var/lib/ocis/storage/users" % ocis_bin, ] + (wrapper_commands), "volumes": volumes, "depends_on": depends_on, diff --git a/ocis/pkg/backup/backup.go b/ocis/pkg/backup/backup.go index 37a57c71e81..27186a6f667 100644 --- a/ocis/pkg/backup/backup.go +++ b/ocis/pkg/backup/backup.go @@ -69,75 +69,68 @@ func CheckProviderConsistency(storagepath string, lbs ListBlobstore) error { } c := NewConsistency() - c.GatherData(p.Nodes, p.Links, p.Blobs, p.Quit) + c.GatherData(p.Events) return c.PrintResults(storagepath) } // GatherData gathers and evaluates data produced by the DataProvider -func (c *Consistency) GatherData(nodes <-chan NodeData, links <-chan LinkData, blobs <-chan BlobData, quit <-chan struct{}) { - c.gatherData(nodes, links, blobs, quit) - - for n := range c.Nodes { - if len(c.Nodes[n]) == 0 { - c.Nodes[n] = append(c.Nodes[n], InconsistencySymlinkMissing) - } - } - for l := range c.LinkedNodes { - c.LinkedNodes[l] = append(c.LinkedNodes[l], InconsistencyNodeMissing) - } - for b := range c.Blobs { - c.Blobs[b] = append(c.Blobs[b], InconsistencyBlobOrphaned) - } - for b := range c.BlobReferences { - c.BlobReferences[b] = append(c.BlobReferences[b], InconsistencyBlobMissing) - } -} - -func (c *Consistency) gatherData(nodes <-chan NodeData, links <-chan LinkData, blobs <-chan BlobData, quit <-chan struct{}) { - for { - select { - case n := <-nodes: +func (c *Consistency) GatherData(events <-chan interface{}) { + for ev := range events { + switch d := ev.(type) { + case NodeData: // does it have inconsistencies? - if len(n.Inconsistencies) != 0 { - c.Nodes[n.NodePath] = append(c.Nodes[n.NodePath], n.Inconsistencies...) + if len(d.Inconsistencies) != 0 { + c.Nodes[d.NodePath] = append(c.Nodes[d.NodePath], d.Inconsistencies...) } // is it linked? - if _, ok := c.LinkedNodes[n.NodePath]; ok { - deleteInconsistency(c.LinkedNodes, n.NodePath) - } else if n.RequiresSymlink { - c.Nodes[n.NodePath] = c.Nodes[n.NodePath] + if _, ok := c.LinkedNodes[d.NodePath]; ok { + deleteInconsistency(c.LinkedNodes, d.NodePath) + } else if d.RequiresSymlink && c.Nodes[d.NodePath] == nil { + c.Nodes[d.NodePath] = []Inconsistency{} } // does it have a blob? - if n.BlobPath != "" { - if _, ok := c.Blobs[n.BlobPath]; ok { - deleteInconsistency(c.Blobs, n.BlobPath) + if d.BlobPath != "" { + if _, ok := c.Blobs[d.BlobPath]; ok { + deleteInconsistency(c.Blobs, d.BlobPath) } else { - c.BlobReferences[n.BlobPath] = []Inconsistency{} - c.blobToNode[n.BlobPath] = n.NodePath + c.BlobReferences[d.BlobPath] = []Inconsistency{} + c.blobToNode[d.BlobPath] = d.NodePath } } - case l := <-links: + case LinkData: // does it have a node? - if _, ok := c.Nodes[l.NodePath]; ok { - deleteInconsistency(c.Nodes, l.NodePath) + if _, ok := c.Nodes[d.NodePath]; ok { + deleteInconsistency(c.Nodes, d.NodePath) } else { - c.LinkedNodes[l.NodePath] = []Inconsistency{} - c.nodeToLink[l.NodePath] = l.LinkPath + c.LinkedNodes[d.NodePath] = []Inconsistency{} + c.nodeToLink[d.NodePath] = d.LinkPath } - case b := <-blobs: + case BlobData: // does it have a reference? - if _, ok := c.BlobReferences[b.BlobPath]; ok { - deleteInconsistency(c.BlobReferences, b.BlobPath) + if _, ok := c.BlobReferences[d.BlobPath]; ok { + deleteInconsistency(c.BlobReferences, d.BlobPath) } else { - c.Blobs[b.BlobPath] = []Inconsistency{} + c.Blobs[d.BlobPath] = []Inconsistency{} } - case <-quit: - return } } + for n := range c.Nodes { + if len(c.Nodes[n]) == 0 { + c.Nodes[n] = append(c.Nodes[n], InconsistencySymlinkMissing) + } + } + for l := range c.LinkedNodes { + c.LinkedNodes[l] = append(c.LinkedNodes[l], InconsistencyNodeMissing) + } + for b := range c.Blobs { + c.Blobs[b] = append(c.Blobs[b], InconsistencyBlobOrphaned) + } + for b := range c.BlobReferences { + c.BlobReferences[b] = append(c.BlobReferences[b], InconsistencyBlobMissing) + } } // PrintResults prints the results of the evaluation diff --git a/ocis/pkg/backup/backup_test.go b/ocis/pkg/backup/backup_test.go index 6aff566f371..21e20842d7f 100644 --- a/ocis/pkg/backup/backup_test.go +++ b/ocis/pkg/backup/backup_test.go @@ -90,31 +90,24 @@ func TestGatherData(t *testing.T) { } for _, tc := range testcases { - nodes := make(chan backup.NodeData) - links := make(chan backup.LinkData) - blobs := make(chan backup.BlobData) - quit := make(chan struct{}) + events := make(chan interface{}) go func() { for _, ev := range tc.Events { switch e := ev.(type) { case backup.NodeData: - nodes <- e + events <- e case backup.LinkData: - links <- e + events <- e case backup.BlobData: - blobs <- e + events <- e } } - quit <- struct{}{} - close(nodes) - close(links) - close(blobs) - close(quit) + close(events) }() c := backup.NewConsistency() - c.GatherData(nodes, links, blobs, quit) + c.GatherData(events) require.Equal(t, tc.Expected.Nodes, c.Nodes) require.Equal(t, tc.Expected.LinkedNodes, c.LinkedNodes) diff --git a/ocis/pkg/backup/provider.go b/ocis/pkg/backup/provider.go index 7f3e98529a8..b9af5c8158f 100644 --- a/ocis/pkg/backup/provider.go +++ b/ocis/pkg/backup/provider.go @@ -21,10 +21,7 @@ type ListBlobstore interface { // DataProvider provides data for the consistency check type DataProvider struct { - Nodes chan NodeData - Links chan LinkData - Blobs chan BlobData - Quit chan struct{} + Events chan interface{} fsys fs.FS discpath string @@ -53,10 +50,7 @@ type BlobData struct { // NewProvider creates a new DataProvider object func NewProvider(fsys fs.FS, discpath string, lbs ListBlobstore) *DataProvider { return &DataProvider{ - Nodes: make(chan NodeData), - Links: make(chan LinkData), - Blobs: make(chan BlobData), - Quit: make(chan struct{}), + Events: make(chan interface{}), fsys: fsys, discpath: discpath, @@ -82,47 +76,7 @@ func (dp *DataProvider) ProduceData() error { wg.Add(1) go func() { for _, d := range dirs { - entries, err := fs.ReadDir(dp.fsys, d) - if err != nil { - fmt.Println("error reading dir", err) - continue - } - - if len(entries) == 0 { - fmt.Println("empty dir", filepath.Join(dp.discpath, d)) - continue - } - - for _, e := range entries { - switch { - case e.IsDir(): - ls, err := fs.ReadDir(dp.fsys, filepath.Join(d, e.Name())) - if err != nil { - fmt.Println("error reading dir", err) - continue - } - for _, l := range ls { - linkpath := filepath.Join(dp.discpath, d, e.Name(), l.Name()) - - r, _ := os.Readlink(linkpath) - nodePath := filepath.Join(dp.discpath, d, e.Name(), r) - dp.Links <- LinkData{LinkPath: linkpath, NodePath: nodePath} - } - fallthrough - case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()): - np := filepath.Join(dp.discpath, d, e.Name()) - var inc []Inconsistency - if !dp.filesExist(filepath.Join(d, e.Name())) { - inc = append(inc, InconsistencyFilesMissing) - } - bp, i := dp.getBlobPath(filepath.Join(d, e.Name())) - if i != "" { - inc = append(inc, i) - } - - dp.Nodes <- NodeData{NodePath: np, BlobPath: bp, RequiresSymlink: requiresSymlink(np), Inconsistencies: inc} - } - } + dp.evaluateNodeDir(d) } wg.Done() }() @@ -130,16 +84,7 @@ func (dp *DataProvider) ProduceData() error { // crawl trash wg.Add(1) go func() { - linkpaths, err := fs.Glob(dp.fsys, "spaces/*/*/trash/*/*/*/*/*") - if err != nil { - fmt.Println("error reading trash", err) - } - for _, l := range linkpaths { - linkpath := filepath.Join(dp.discpath, l) - r, _ := os.Readlink(linkpath) - p := filepath.Join(dp.discpath, l, "..", r) - dp.Links <- LinkData{LinkPath: linkpath, NodePath: p} - } + dp.evaluateTrashDir() wg.Done() }() @@ -152,7 +97,7 @@ func (dp *DataProvider) ProduceData() error { } for _, bn := range bs { - dp.Blobs <- BlobData{BlobPath: dp.lbs.Path(bn)} + dp.Events <- BlobData{BlobPath: dp.lbs.Path(bn)} } wg.Done() }() @@ -187,6 +132,65 @@ func (dp *DataProvider) getBlobPath(path string) (string, Inconsistency) { return "", "" } +func (dp *DataProvider) evaluateNodeDir(d string) { + // d is something like spaces/a8/e5d981-41e4-4468-b532-258d5fb457d3/nodes/2d/08/8d/24 + // we could have multiple nodes under this, but we are only interested in one file per node - the one with "" extension + entries, err := fs.ReadDir(dp.fsys, d) + if err != nil { + fmt.Println("error reading dir", err) + return + } + + if len(entries) == 0 { + fmt.Println("empty dir", filepath.Join(dp.discpath, d)) + return + } + + for _, e := range entries { + switch { + case e.IsDir(): + ls, err := fs.ReadDir(dp.fsys, filepath.Join(d, e.Name())) + if err != nil { + fmt.Println("error reading dir", err) + continue + } + for _, l := range ls { + linkpath := filepath.Join(dp.discpath, d, e.Name(), l.Name()) + + r, _ := os.Readlink(linkpath) + nodePath := filepath.Join(dp.discpath, d, e.Name(), r) + dp.Events <- LinkData{LinkPath: linkpath, NodePath: nodePath} + } + fallthrough + case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()): + np := filepath.Join(dp.discpath, d, e.Name()) + var inc []Inconsistency + if !dp.filesExist(filepath.Join(d, e.Name())) { + inc = append(inc, InconsistencyFilesMissing) + } + bp, i := dp.getBlobPath(filepath.Join(d, e.Name())) + if i != "" { + inc = append(inc, i) + } + + dp.Events <- NodeData{NodePath: np, BlobPath: bp, RequiresSymlink: requiresSymlink(np), Inconsistencies: inc} + } + } +} + +func (dp *DataProvider) evaluateTrashDir() { + linkpaths, err := fs.Glob(dp.fsys, "spaces/*/*/trash/*/*/*/*/*") + if err != nil { + fmt.Println("error reading trash", err) + } + for _, l := range linkpaths { + linkpath := filepath.Join(dp.discpath, l) + r, _ := os.Readlink(linkpath) + p := filepath.Join(dp.discpath, l, "..", r) + dp.Events <- LinkData{LinkPath: linkpath, NodePath: p} + } +} + func (dp *DataProvider) filesExist(path string) bool { check := func(p string) bool { _, err := fs.Stat(dp.fsys, p) @@ -196,11 +200,7 @@ func (dp *DataProvider) filesExist(path string) bool { } func (dp *DataProvider) quit() { - dp.Quit <- struct{}{} - close(dp.Nodes) - close(dp.Links) - close(dp.Blobs) - close(dp.Quit) + close(dp.Events) } func requiresSymlink(path string) bool { From e046768aa6ab270c94fc8b445bb65f422a147572 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 6 Jun 2024 15:21:50 +0200 Subject: [PATCH 12/14] feat(docs): extend backup docu to mention command Signed-off-by: jkoberg --- docs/ocis/backup.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/ocis/backup.md b/docs/ocis/backup.md index 093e180b460..0a0a88ea3ae 100644 --- a/docs/ocis/backup.md +++ b/docs/ocis/backup.md @@ -217,3 +217,12 @@ BACKUP RECOMMENDED/OMITABLE. This folder contains custom web assets. Can be spec When using an external idp/idm/nats or blobstore, its data needs to be backed up separately. Refer to your idp/idm/nats/blobstore documentation for backup details. +## Backup Consistency command + +Infinite Scale now allows checking an existing backup for consistency. Use command +```bash +ocis backup consistency -p "" +``` + +`path-to-base-folder` needs to be replaced with the path to the storage providers base path. For example: "/var/lib/ocis/storage/users" or "$HOME/storage/users". +Use the `-b s3ng` when using an external (s3) blobstore. Note: When using this flag the blobstore must be configured via envvars or yaml file the same as in ocis. Consistency checks for other blobstores than `ocis` and `s3ng` are not supported at the moment. From 222ead44e6401f87e018f01624274a15818eff8f Mon Sep 17 00:00:00 2001 From: kobergj Date: Fri, 7 Jun 2024 12:38:38 +0200 Subject: [PATCH 13/14] feat(ocis): Better documentation for backup command Co-authored-by: Martin --- docs/ocis/backup.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/ocis/backup.md b/docs/ocis/backup.md index 0a0a88ea3ae..07965a82c2d 100644 --- a/docs/ocis/backup.md +++ b/docs/ocis/backup.md @@ -217,12 +217,13 @@ BACKUP RECOMMENDED/OMITABLE. This folder contains custom web assets. Can be spec When using an external idp/idm/nats or blobstore, its data needs to be backed up separately. Refer to your idp/idm/nats/blobstore documentation for backup details. -## Backup Consistency command +## Backup Consistency Command -Infinite Scale now allows checking an existing backup for consistency. Use command +Infinite Scale now allows checking an existing backup for consistency. Use the command: ```bash ocis backup consistency -p "" ``` -`path-to-base-folder` needs to be replaced with the path to the storage providers base path. For example: "/var/lib/ocis/storage/users" or "$HOME/storage/users". -Use the `-b s3ng` when using an external (s3) blobstore. Note: When using this flag the blobstore must be configured via envvars or yaml file the same as in ocis. Consistency checks for other blobstores than `ocis` and `s3ng` are not supported at the moment. +`path-to-base-folder` needs to be replaced with the path to the storage providers base path. Should be same as the `STORAGE_USERS_OCIS_ROOT` + +Use the `-b s3ng` option when using an external (s3) blobstore. Note: When using this flag, the path to the blobstore must be configured via envvars or a yaml file to match the configuration of the original instance. Consistency checks for other blobstores than `ocis` and `s3ng` are not supported at the moment. From 9a84284372f004fd8b673c4e79def72cce81a02a Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 11 Jun 2024 16:51:30 +0200 Subject: [PATCH 14/14] feat(ocis): allow skiping blobcheck in consistency command Signed-off-by: jkoberg --- ocis/pkg/backup/provider.go | 42 ++++++++++++++++++++++--------------- ocis/pkg/command/backup.go | 6 ++++-- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/ocis/pkg/backup/provider.go b/ocis/pkg/backup/provider.go index b9af5c8158f..02d4fbec81b 100644 --- a/ocis/pkg/backup/provider.go +++ b/ocis/pkg/backup/provider.go @@ -23,9 +23,10 @@ type ListBlobstore interface { type DataProvider struct { Events chan interface{} - fsys fs.FS - discpath string - lbs ListBlobstore + fsys fs.FS + discpath string + lbs ListBlobstore + skipBlobs bool } // NodeData holds data about the nodes @@ -52,9 +53,10 @@ func NewProvider(fsys fs.FS, discpath string, lbs ListBlobstore) *DataProvider { return &DataProvider{ Events: make(chan interface{}), - fsys: fsys, - discpath: discpath, - lbs: lbs, + fsys: fsys, + discpath: discpath, + lbs: lbs, + skipBlobs: lbs == nil, } } @@ -89,18 +91,20 @@ func (dp *DataProvider) ProduceData() error { }() // crawl blobstore - wg.Add(1) - go func() { - bs, err := dp.lbs.List() - if err != nil { - fmt.Println("error listing blobs", err) - } + if !dp.skipBlobs { + wg.Add(1) + go func() { + bs, err := dp.lbs.List() + if err != nil { + fmt.Println("error listing blobs", err) + } - for _, bn := range bs { - dp.Events <- BlobData{BlobPath: dp.lbs.Path(bn)} - } - wg.Done() - }() + for _, bn := range bs { + dp.Events <- BlobData{BlobPath: dp.lbs.Path(bn)} + } + wg.Done() + }() + } // wait for all crawlers to finish go func() { @@ -112,6 +116,10 @@ func (dp *DataProvider) ProduceData() error { } func (dp *DataProvider) getBlobPath(path string) (string, Inconsistency) { + if dp.skipBlobs { + return "", "" + } + b, err := fs.ReadFile(dp.fsys, path+".mpk") if err != nil { return "", InconsistencyFilesMissing diff --git a/ocis/pkg/command/backup.go b/ocis/pkg/command/backup.go index 2c1031ab953..d985a020a3f 100644 --- a/ocis/pkg/command/backup.go +++ b/ocis/pkg/command/backup.go @@ -23,7 +23,7 @@ func BackupCommand(cfg *config.Config) *cli.Command { ConsistencyCommand(cfg), }, Before: func(c *cli.Context) error { - return configlog.ReturnError(parser.ParseConfig(cfg, false)) + return configlog.ReturnError(parser.ParseConfig(cfg, true)) }, Action: func(_ *cli.Context) error { fmt.Println("Read the docs") @@ -47,7 +47,7 @@ func ConsistencyCommand(cfg *config.Config) *cli.Command { &cli.StringFlag{ Name: "blobstore", Aliases: []string{"b"}, - Usage: "the blobstore type. Can be (ocis, s3ng). Default ocis", + Usage: "the blobstore type. Can be (none, ocis, s3ng). Default ocis", Value: "ocis", }, }, @@ -74,6 +74,8 @@ func ConsistencyCommand(cfg *config.Config) *cli.Command { ) case "ocis": bs, err = ocisbs.New(basePath) + case "none": + bs = nil default: err = errors.New("blobstore type not supported") }