Migrate to msgpack

changelog/unreleased/migrate-file-metadata.md

@@ -0,0 +1,5 @@
+Enhancement: Automatically migrate file metadata from xattrs to messagepack
+
+We added a migration which transparently migrates existig file metadata from xattrs to the new messagepack format.
+
+https://github.com/cs3org/reva/pull/3728

pkg/storage/utils/decomposedfs/decomposedfs.go

@@ -49,6 +49,7 @@ import (
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup"
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata"
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes"
+	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator"
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node"
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options"
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree"
@@ -135,6 +136,14 @@ func New(o *options.Options, lu *lookup.Lookup, p Permissions, tp Tree, es event
 		return nil, errors.Wrap(err, "could not setup tree")
 	}
 
+	// Run migrations & return
+	m := migrator.New(lu, log)
+	err = m.RunMigrations()
+	if err != nil {
+		log.Error().Err(err).Msg("could not migrate tree")
+		return nil, errors.Wrap(err, "could not migrate tree")
+	}
+
 	if o.MaxAcquireLockCycles != 0 {
 		filelocks.SetMaxLockCycles(o.MaxAcquireLockCycles)
 	}

pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go

@@ -53,6 +53,9 @@ func NewMessagePackBackend(rootPath string, o options.CacheOptions) MessagePackB
 	}
 }
 
+// Name returns the name of the backend
+func (MessagePackBackend) Name() string { return "messagepack" }
+
 // All reads all extended attributes for a node
 func (b MessagePackBackend) All(path string) (map[string][]byte, error) {
 	path = b.MetadataPath(path)

pkg/storage/utils/decomposedfs/metadata/metadata.go

@@ -27,8 +27,11 @@ var errUnconfiguredError = errors.New("no metadata backend configured. Bailing o
 
 // Backend defines the interface for file attribute backends
 type Backend interface {
+	Name() string
+
 	All(path string) (map[string][]byte, error)
 	Get(path, key string) ([]byte, error)
+
 	GetInt64(path, key string) (int64, error)
 	List(path string) (attribs []string, err error)
 	Set(path, key string, val []byte) error
@@ -46,6 +49,9 @@ type Backend interface {
 // NullBackend is the default stub backend, used to enforce the configuration of a proper backend
 type NullBackend struct{}
 
+// Name returns the name of the backend
+func (NullBackend) Name() string { return "null" }
+
 // All reads all extended attributes for a node
 func (NullBackend) All(path string) (map[string][]byte, error) { return nil, errUnconfiguredError }
 

pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go

@@ -33,6 +33,9 @@ import (
 // XattrsBackend stores the file attributes in extended attributes
 type XattrsBackend struct{}
 
+// Name returns the name of the backend
+func (XattrsBackend) Name() string { return "xattrs" }
+
 // Get an extended attribute value for the given key
 // No file locking is involved here as reading a single xattr is
 // considered to be atomic.

pkg/storage/utils/decomposedfs/migrator/0001_create_spaces_directory_structure.go

@@ -0,0 +1,126 @@
+// Copyright 2018-2023 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package migrator
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup"
+	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes"
+	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node"
+)
+
+// Migration0001 creates the spaces directory structure
+func (m *Migrator) Migration0001() (Result, error) {
+	m.log.Info().Msg("Migrating spaces directory structure...")
+
+	// create spaces folder and iterate over existing nodes to populate it
+	nodesPath := filepath.Join(m.lu.InternalRoot(), "nodes")
+	fi, err := os.Stat(nodesPath)
+	if err == nil && fi.IsDir() {
+		f, err := os.Open(nodesPath)
+		if err != nil {
+			return resultFailed, err
+		}
+		nodes, err := f.Readdir(0)
+		if err != nil {
+			return resultFailed, err
+		}
+
+		for _, n := range nodes {
+			nodePath := filepath.Join(nodesPath, n.Name())
+
+			attr, err := m.lu.MetadataBackend().Get(nodePath, prefixes.ParentidAttr)
+			if err == nil && string(attr) == node.RootID {
+				if err := m.moveNode(n.Name(), n.Name()); err != nil {
+					m.log.Error().Err(err).
+						Str("space", n.Name()).
+						Msg("could not move space")
+					continue
+				}
+				m.linkSpaceNode("personal", n.Name())
+			}
+		}
+		// TODO delete nodesPath if empty
+	}
+	return resultSucceeded, nil
+}
+
+func (m *Migrator) moveNode(spaceID, nodeID string) error {
+	dirPath := filepath.Join(m.lu.InternalRoot(), "nodes", nodeID)
+	f, err := os.Open(dirPath)
+	if err != nil {
+		return err
+	}
+	children, err := f.Readdir(0)
+	if err != nil {
+		return err
+	}
+	for _, child := range children {
+		old := filepath.Join(m.lu.InternalRoot(), "nodes", child.Name())
+		new := filepath.Join(m.lu.InternalRoot(), "spaces", lookup.Pathify(spaceID, 1, 2), "nodes", lookup.Pathify(child.Name(), 4, 2))
+		if err := os.Rename(old, new); err != nil {
+			m.log.Error().Err(err).
+				Str("space", spaceID).
+				Str("nodes", child.Name()).
+				Str("oldpath", old).
+				Str("newpath", new).
+				Msg("could not rename node")
+		}
+		if child.IsDir() {
+			if err := m.moveNode(spaceID, child.Name()); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// linkSpace creates a new symbolic link for a space with the given type st, and node id
+func (m *Migrator) linkSpaceNode(spaceType, spaceID string) {
+	spaceTypesPath := filepath.Join(m.lu.InternalRoot(), "spacetypes", spaceType, spaceID)
+	expectedTarget := "../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2)
+	linkTarget, err := os.Readlink(spaceTypesPath)
+	if errors.Is(err, os.ErrNotExist) {
+		err = os.Symlink(expectedTarget, spaceTypesPath)
+		if err != nil {
+			m.log.Error().Err(err).
+				Str("space_type", spaceType).
+				Str("space", spaceID).
+				Msg("could not create symlink")
+		}
+	} else {
+		if err != nil {
+			m.log.Error().Err(err).
+				Str("space_type", spaceType).
+				Str("space", spaceID).
+				Msg("could not read symlink")
+		}
+		if linkTarget != expectedTarget {
+			m.log.Warn().
+				Str("space_type", spaceType).
+				Str("space", spaceID).
+				Str("expected", expectedTarget).
+				Str("actual", linkTarget).
+				Msg("expected a different link target")
+		}
+	}
+}

pkg/storage/utils/decomposedfs/migrator/0002_move_spacetypes_to_indexes.go

@@ -0,0 +1,138 @@
+// Copyright 2018-2023 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package migrator
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/cs3org/reva/v2/pkg/logger"
+)
+
+// Migration0002 migrates spacetypes to indexes
+func (m *Migrator) Migration0002() (Result, error) {
+	m.log.Info().Msg("Migrating space types indexes...")
+
+	spaceTypesPath := filepath.Join(m.lu.InternalRoot(), "spacetypes")
+	fi, err := os.Stat(spaceTypesPath)
+	if err == nil && fi.IsDir() {
+
+		f, err := os.Open(spaceTypesPath)
+		if err != nil {
+			return resultFailed, err
+		}
+		spaceTypes, err := f.Readdir(0)
+		if err != nil {
+			return resultFailed, err
+		}
+
+		for _, st := range spaceTypes {
+			err := m.moveSpaceType(st.Name())
+			if err != nil {
+				logger.New().Error().Err(err).
+					Str("space", st.Name()).
+					Msg("could not move space")
+				continue
+			}
+		}
+
+		// delete spacetypespath
+		d, err := os.Open(spaceTypesPath)
+		if err != nil {
+			logger.New().Error().Err(err).
+				Str("spacetypesdir", spaceTypesPath).
+				Msg("could not open spacetypesdir")
+			return resultFailed, nil
+		}
+		defer d.Close()
+		_, err = d.Readdirnames(1) // Or f.Readdir(1)
+		if err == io.EOF {
+			// directory is empty we can delete
+			err := os.Remove(spaceTypesPath)
+			if err != nil {
+				logger.New().Error().Err(err).
+					Str("spacetypesdir", d.Name()).
+					Msg("could not delete")
+			}
+		} else {
+			logger.New().Error().Err(err).
+				Str("spacetypesdir", d.Name()).
+				Msg("could not delete, not empty")
+		}
+	}
+	return resultSucceeded, nil
+}
+
+func (m *Migrator) moveSpaceType(spaceType string) error {
+	dirPath := filepath.Join(m.lu.InternalRoot(), "spacetypes", spaceType)
+	f, err := os.Open(dirPath)
+	if err != nil {
+		return err
+	}
+	children, err := f.Readdir(0)
+	if err != nil {
+		return err
+	}
+	for _, child := range children {
+		old := filepath.Join(m.lu.InternalRoot(), "spacetypes", spaceType, child.Name())
+		target, err := os.Readlink(old)
+		if err != nil {
+			logger.New().Error().Err(err).
+				Str("space", spaceType).
+				Str("nodes", child.Name()).
+				Str("oldLink", old).
+				Msg("could not read old symlink")
+			continue
+		}
+		newDir := filepath.Join(m.lu.InternalRoot(), "indexes", "by-type", spaceType)
+		if err := os.MkdirAll(newDir, 0700); err != nil {
+			logger.New().Error().Err(err).
+				Str("space", spaceType).
+				Str("nodes", child.Name()).
+				Str("targetDir", newDir).
+				Msg("could not read old symlink")
+		}
+		newLink := filepath.Join(newDir, child.Name())
+		if err := os.Symlink(filepath.Join("..", target), newLink); err != nil {
+			logger.New().Error().Err(err).
+				Str("space", spaceType).
+				Str("nodes", child.Name()).
+				Str("oldpath", old).
+				Str("newpath", newLink).
+				Msg("could not rename node")
+			continue
+		}
+		if err := os.Remove(old); err != nil {
+			logger.New().Error().Err(err).
+				Str("space", spaceType).
+				Str("nodes", child.Name()).
+				Str("oldLink", old).
+				Msg("could not remove old symlink")
+			continue
+		}
+	}
+	if err := os.Remove(dirPath); err != nil {
+		logger.New().Error().Err(err).
+			Str("space", spaceType).
+			Str("dir", dirPath).
+			Msg("could not remove spaces folder, folder probably not empty")
+	}
+	return nil
+}