diff --git a/changelog/unreleased/migrate-file-metadata.md b/changelog/unreleased/migrate-file-metadata.md new file mode 100644 index 0000000000..14630b2370 --- /dev/null +++ b/changelog/unreleased/migrate-file-metadata.md @@ -0,0 +1,5 @@ +Enhancement: Automatically migrate file metadata from xattrs to messagepack + +We added a migration which transparently migrates existig file metadata from xattrs to the new messagepack format. + +https://github.com/cs3org/reva/pull/3728 diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 1273caeee9..95ab5fc5c7 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -49,6 +49,7 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree" @@ -135,6 +136,14 @@ func New(o *options.Options, lu *lookup.Lookup, p Permissions, tp Tree, es event return nil, errors.Wrap(err, "could not setup tree") } + // Run migrations & return + m := migrator.New(lu, log) + err = m.RunMigrations() + if err != nil { + log.Error().Err(err).Msg("could not migrate tree") + return nil, errors.Wrap(err, "could not migrate tree") + } + if o.MaxAcquireLockCycles != 0 { filelocks.SetMaxLockCycles(o.MaxAcquireLockCycles) } diff --git a/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go b/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go index 85f08c7ed3..e180408211 100644 --- a/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go +++ b/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go @@ -53,6 +53,9 @@ func NewMessagePackBackend(rootPath string, o options.CacheOptions) MessagePackB } } +// Name returns the name of the backend +func (MessagePackBackend) Name() string { return "messagepack" } + // All reads all extended attributes for a node func (b MessagePackBackend) All(path string) (map[string][]byte, error) { path = b.MetadataPath(path) diff --git a/pkg/storage/utils/decomposedfs/metadata/metadata.go b/pkg/storage/utils/decomposedfs/metadata/metadata.go index 23e2598441..8bb2663124 100644 --- a/pkg/storage/utils/decomposedfs/metadata/metadata.go +++ b/pkg/storage/utils/decomposedfs/metadata/metadata.go @@ -27,8 +27,11 @@ var errUnconfiguredError = errors.New("no metadata backend configured. Bailing o // Backend defines the interface for file attribute backends type Backend interface { + Name() string + All(path string) (map[string][]byte, error) Get(path, key string) ([]byte, error) + GetInt64(path, key string) (int64, error) List(path string) (attribs []string, err error) Set(path, key string, val []byte) error @@ -46,6 +49,9 @@ type Backend interface { // NullBackend is the default stub backend, used to enforce the configuration of a proper backend type NullBackend struct{} +// Name returns the name of the backend +func (NullBackend) Name() string { return "null" } + // All reads all extended attributes for a node func (NullBackend) All(path string) (map[string][]byte, error) { return nil, errUnconfiguredError } diff --git a/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go b/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go index a13ec2dc02..eed7c122ab 100644 --- a/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go +++ b/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go @@ -33,6 +33,9 @@ import ( // XattrsBackend stores the file attributes in extended attributes type XattrsBackend struct{} +// Name returns the name of the backend +func (XattrsBackend) Name() string { return "xattrs" } + // Get an extended attribute value for the given key // No file locking is involved here as reading a single xattr is // considered to be atomic. diff --git a/pkg/storage/utils/decomposedfs/migrator/0001_create_spaces_directory_structure.go b/pkg/storage/utils/decomposedfs/migrator/0001_create_spaces_directory_structure.go new file mode 100644 index 0000000000..9de83402dc --- /dev/null +++ b/pkg/storage/utils/decomposedfs/migrator/0001_create_spaces_directory_structure.go @@ -0,0 +1,126 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package migrator + +import ( + "errors" + "os" + "path/filepath" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" +) + +// Migration0001 creates the spaces directory structure +func (m *Migrator) Migration0001() (Result, error) { + m.log.Info().Msg("Migrating spaces directory structure...") + + // create spaces folder and iterate over existing nodes to populate it + nodesPath := filepath.Join(m.lu.InternalRoot(), "nodes") + fi, err := os.Stat(nodesPath) + if err == nil && fi.IsDir() { + f, err := os.Open(nodesPath) + if err != nil { + return resultFailed, err + } + nodes, err := f.Readdir(0) + if err != nil { + return resultFailed, err + } + + for _, n := range nodes { + nodePath := filepath.Join(nodesPath, n.Name()) + + attr, err := m.lu.MetadataBackend().Get(nodePath, prefixes.ParentidAttr) + if err == nil && string(attr) == node.RootID { + if err := m.moveNode(n.Name(), n.Name()); err != nil { + m.log.Error().Err(err). + Str("space", n.Name()). + Msg("could not move space") + continue + } + m.linkSpaceNode("personal", n.Name()) + } + } + // TODO delete nodesPath if empty + } + return resultSucceeded, nil +} + +func (m *Migrator) moveNode(spaceID, nodeID string) error { + dirPath := filepath.Join(m.lu.InternalRoot(), "nodes", nodeID) + f, err := os.Open(dirPath) + if err != nil { + return err + } + children, err := f.Readdir(0) + if err != nil { + return err + } + for _, child := range children { + old := filepath.Join(m.lu.InternalRoot(), "nodes", child.Name()) + new := filepath.Join(m.lu.InternalRoot(), "spaces", lookup.Pathify(spaceID, 1, 2), "nodes", lookup.Pathify(child.Name(), 4, 2)) + if err := os.Rename(old, new); err != nil { + m.log.Error().Err(err). + Str("space", spaceID). + Str("nodes", child.Name()). + Str("oldpath", old). + Str("newpath", new). + Msg("could not rename node") + } + if child.IsDir() { + if err := m.moveNode(spaceID, child.Name()); err != nil { + return err + } + } + } + return nil +} + +// linkSpace creates a new symbolic link for a space with the given type st, and node id +func (m *Migrator) linkSpaceNode(spaceType, spaceID string) { + spaceTypesPath := filepath.Join(m.lu.InternalRoot(), "spacetypes", spaceType, spaceID) + expectedTarget := "../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2) + linkTarget, err := os.Readlink(spaceTypesPath) + if errors.Is(err, os.ErrNotExist) { + err = os.Symlink(expectedTarget, spaceTypesPath) + if err != nil { + m.log.Error().Err(err). + Str("space_type", spaceType). + Str("space", spaceID). + Msg("could not create symlink") + } + } else { + if err != nil { + m.log.Error().Err(err). + Str("space_type", spaceType). + Str("space", spaceID). + Msg("could not read symlink") + } + if linkTarget != expectedTarget { + m.log.Warn(). + Str("space_type", spaceType). + Str("space", spaceID). + Str("expected", expectedTarget). + Str("actual", linkTarget). + Msg("expected a different link target") + } + } +} diff --git a/pkg/storage/utils/decomposedfs/migrator/0002_move_spacetypes_to_indexes.go b/pkg/storage/utils/decomposedfs/migrator/0002_move_spacetypes_to_indexes.go new file mode 100644 index 0000000000..689f736f35 --- /dev/null +++ b/pkg/storage/utils/decomposedfs/migrator/0002_move_spacetypes_to_indexes.go @@ -0,0 +1,138 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package migrator + +import ( + "io" + "os" + "path/filepath" + + "github.com/cs3org/reva/v2/pkg/logger" +) + +// Migration0002 migrates spacetypes to indexes +func (m *Migrator) Migration0002() (Result, error) { + m.log.Info().Msg("Migrating space types indexes...") + + spaceTypesPath := filepath.Join(m.lu.InternalRoot(), "spacetypes") + fi, err := os.Stat(spaceTypesPath) + if err == nil && fi.IsDir() { + + f, err := os.Open(spaceTypesPath) + if err != nil { + return resultFailed, err + } + spaceTypes, err := f.Readdir(0) + if err != nil { + return resultFailed, err + } + + for _, st := range spaceTypes { + err := m.moveSpaceType(st.Name()) + if err != nil { + logger.New().Error().Err(err). + Str("space", st.Name()). + Msg("could not move space") + continue + } + } + + // delete spacetypespath + d, err := os.Open(spaceTypesPath) + if err != nil { + logger.New().Error().Err(err). + Str("spacetypesdir", spaceTypesPath). + Msg("could not open spacetypesdir") + return resultFailed, nil + } + defer d.Close() + _, err = d.Readdirnames(1) // Or f.Readdir(1) + if err == io.EOF { + // directory is empty we can delete + err := os.Remove(spaceTypesPath) + if err != nil { + logger.New().Error().Err(err). + Str("spacetypesdir", d.Name()). + Msg("could not delete") + } + } else { + logger.New().Error().Err(err). + Str("spacetypesdir", d.Name()). + Msg("could not delete, not empty") + } + } + return resultSucceeded, nil +} + +func (m *Migrator) moveSpaceType(spaceType string) error { + dirPath := filepath.Join(m.lu.InternalRoot(), "spacetypes", spaceType) + f, err := os.Open(dirPath) + if err != nil { + return err + } + children, err := f.Readdir(0) + if err != nil { + return err + } + for _, child := range children { + old := filepath.Join(m.lu.InternalRoot(), "spacetypes", spaceType, child.Name()) + target, err := os.Readlink(old) + if err != nil { + logger.New().Error().Err(err). + Str("space", spaceType). + Str("nodes", child.Name()). + Str("oldLink", old). + Msg("could not read old symlink") + continue + } + newDir := filepath.Join(m.lu.InternalRoot(), "indexes", "by-type", spaceType) + if err := os.MkdirAll(newDir, 0700); err != nil { + logger.New().Error().Err(err). + Str("space", spaceType). + Str("nodes", child.Name()). + Str("targetDir", newDir). + Msg("could not read old symlink") + } + newLink := filepath.Join(newDir, child.Name()) + if err := os.Symlink(filepath.Join("..", target), newLink); err != nil { + logger.New().Error().Err(err). + Str("space", spaceType). + Str("nodes", child.Name()). + Str("oldpath", old). + Str("newpath", newLink). + Msg("could not rename node") + continue + } + if err := os.Remove(old); err != nil { + logger.New().Error().Err(err). + Str("space", spaceType). + Str("nodes", child.Name()). + Str("oldLink", old). + Msg("could not remove old symlink") + continue + } + } + if err := os.Remove(dirPath); err != nil { + logger.New().Error().Err(err). + Str("space", spaceType). + Str("dir", dirPath). + Msg("could not remove spaces folder, folder probably not empty") + } + return nil +} diff --git a/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go b/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go new file mode 100644 index 0000000000..6df6f60702 --- /dev/null +++ b/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go @@ -0,0 +1,130 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package migrator + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" +) + +// Migration0003 migrates the file metadata to the current backend. +// Only the xattrs -> messagepack path is supported. +func (m *Migrator) Migration0003() (Result, error) { + bod := m.detectBackendOnDisk() + if bod == "" { + return resultFailed, errors.New("could not detect metadata backend on disk") + } + + if bod != "xattrs" || m.lu.MetadataBackend().Name() != "messagepack" { + return resultSucceededRunAgain, nil + } + + m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating to messagepack metadata backend...") + xattrs := metadata.XattrsBackend{} + mpk := metadata.NewMessagePackBackend(m.lu.InternalRoot(), options.CacheOptions{}) + + spaces, _ := filepath.Glob(filepath.Join(m.lu.InternalRoot(), "spaces", "*", "*")) + for _, space := range spaces { + err := filepath.WalkDir(filepath.Join(space, "nodes"), func(path string, _ fs.DirEntry, err error) error { + // Do not continue on error + if err != nil { + return err + } + + if strings.HasSuffix(path, ".mpk") || strings.HasSuffix(path, ".flock") { + // None of our business + return nil + } + + fi, err := os.Lstat(path) + if err != nil { + return err + } + + if !fi.IsDir() && !fi.Mode().IsRegular() { + return nil + } + + mpkPath := mpk.MetadataPath(path) + _, err = os.Stat(mpkPath) + if err == nil { + return nil + } + + attribs, err := xattrs.All(path) + if err != nil { + m.log.Error().Err(err).Str("path", path).Msg("error converting file") + return err + } + if len(attribs) == 0 { + return nil + } + + err = mpk.SetMultiple(path, attribs, false) + if err != nil { + m.log.Error().Err(err).Str("path", path).Msg("error setting attributes") + return err + } + + for k := range attribs { + err = xattrs.Remove(path, k) + if err != nil { + m.log.Debug().Err(err).Str("path", path).Msg("error removing xattr") + } + } + + return nil + }) + if err != nil { + m.log.Error().Err(err).Msg("error migrating nodes to messagepack metadata backend") + } + } + + m.log.Info().Msg("done.") + return resultSucceeded, nil +} + +func (m *Migrator) detectBackendOnDisk() string { + root := m.lu.InternalRoot() + + matches, _ := filepath.Glob(filepath.Join(root, "spaces", "*", "*")) + if len(matches) > 0 { + base := matches[len(matches)-1] + spaceid := strings.ReplaceAll( + strings.TrimPrefix(base, filepath.Join(root, "spaces")), + "/", "") + spaceRoot := lookup.Pathify(spaceid, 4, 2) + _, err := os.Stat(filepath.Join(base, "nodes", spaceRoot+".mpk")) + if err == nil { + return "mpk" + } + _, err = os.Stat(filepath.Join(base, "nodes", spaceRoot+".ini")) + if err == nil { + return "ini" + } + } + return "xattrs" +} diff --git a/pkg/storage/utils/decomposedfs/migrator/migrator.go b/pkg/storage/utils/decomposedfs/migrator/migrator.go new file mode 100644 index 0000000000..2440cf6ad8 --- /dev/null +++ b/pkg/storage/utils/decomposedfs/migrator/migrator.go @@ -0,0 +1,129 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package migrator + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" + "github.com/rogpeppe/go-internal/lockedfile" + "github.com/rs/zerolog" +) + +var allMigrations = []string{"0001", "0002", "0003"} + +const ( + resultFailed = "failed" + resultSucceeded = "succeeded" + resultSucceededRunAgain = "runagain" +) + +type migrationState struct { + State string + Message string +} +type migrationStates map[string]migrationState + +// Result represents the result of a migration run +type Result string + +// Migrator runs migrations on an existing decomposedfs +type Migrator struct { + lu *lookup.Lookup + states migrationStates + log *zerolog.Logger +} + +// New returns a new Migrator instance +func New(lu *lookup.Lookup, log *zerolog.Logger) Migrator { + return Migrator{ + lu: lu, + log: log, + } +} + +// RunMigrations runs all migrations in sequence. Note this sequence must not be changed or it might +// damage existing decomposed fs. +func (m *Migrator) RunMigrations() error { + lock, err := lockedfile.OpenFile(filepath.Join(m.lu.InternalRoot(), ".migrations.lock"), os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return err + } + defer lock.Close() + + err = m.readStates() + if err != nil { + return err + } + + for _, migration := range allMigrations { + s := m.states[migration] + if s.State == "succeeded" { + continue + } + + migrateMethod := reflect.ValueOf(m).MethodByName("Migration" + migration) + v := migrateMethod.Call(nil) + s.State = string(v[0].Interface().(Result)) + if v[1].Interface() != nil { + err := v[1].Interface().(error) + m.log.Error().Err(err).Msg("migration " + migration + " failed") + s.Message = err.Error() + } + + m.states[migration] = s + err = m.writeStates() + if err != nil { + return err + } + } + return nil +} + +func (m *Migrator) readStates() error { + m.states = migrationStates{} + + d, err := os.ReadFile(filepath.Join(m.lu.InternalRoot(), ".migrations")) + if err != nil { + if !os.IsNotExist(err) { + return err + } + } + + if len(d) > 0 { + err = json.Unmarshal(d, &m.states) + if err != nil { + return err + } + } + + return nil +} + +func (m *Migrator) writeStates() error { + d, err := json.Marshal(m.states) + if err != nil { + m.log.Error().Err(err).Msg("could not marshal migration states") + return nil + } + return os.WriteFile(filepath.Join(m.lu.InternalRoot(), ".migrations"), d, 0600) +} diff --git a/pkg/storage/utils/decomposedfs/migrator/migrator_suite_test.go b/pkg/storage/utils/decomposedfs/migrator/migrator_suite_test.go new file mode 100644 index 0000000000..3748b97a26 --- /dev/null +++ b/pkg/storage/utils/decomposedfs/migrator/migrator_suite_test.go @@ -0,0 +1,13 @@ +package migrator_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMigrator(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Migrator Suite") +} diff --git a/pkg/storage/utils/decomposedfs/migrator/migrator_test.go b/pkg/storage/utils/decomposedfs/migrator/migrator_test.go new file mode 100644 index 0000000000..c3e29419df --- /dev/null +++ b/pkg/storage/utils/decomposedfs/migrator/migrator_test.go @@ -0,0 +1,110 @@ +package migrator_test + +import ( + "io" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pkg/xattr" + "github.com/rs/zerolog" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator" + helpers "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/testhelpers" +) + +var _ = Describe("Migrator", func() { + var ( + env *helpers.TestEnv + + nullLogger = zerolog.New(io.Discard).With().Logger() + ) + + BeforeEach(func() { + var err error + env, err = helpers.NewTestEnv(nil) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + if env != nil { + env.Cleanup() + } + }) + + Describe("migrating metadata - migration 0003", func() { + When("staying at xattrs", func() { + JustBeforeEach(func() { + var err error + env, err = helpers.NewTestEnv(map[string]interface{}{ + "metadata_backend": "xattrs", + }) + Expect(err).ToNot(HaveOccurred()) + }) + + It("doesn't migrate", func() { + m := migrator.New(env.Lookup, &nullLogger) + err := m.RunMigrations() + Expect(err).ToNot(HaveOccurred()) + + nRef, err := env.Lookup.NodeFromResource(env.Ctx, &provider.Reference{ + ResourceId: env.SpaceRootRes, + Path: "/dir1/file1", + }) + Expect(err).ToNot(HaveOccurred()) + nameAttr, err := xattr.Get(nRef.InternalPath(), prefixes.NameAttr) + Expect(err).ToNot(HaveOccurred()) + Expect(string(nameAttr)).To(Equal("file1")) + + _, err = os.Stat(nRef.InternalPath() + ".mpk") + Expect(err).To(HaveOccurred()) + }) + }) + + When("going from xattrs to messagepack", func() { + var ( + path string + backend metadata.Backend + ) + + JustBeforeEach(func() { + backend = metadata.NewMessagePackBackend(env.Root, env.Options.FileMetadataCache) + + var err error + env, err = helpers.NewTestEnv(map[string]interface{}{ + "metadata_backend": "xattrs", + }) + Expect(err).ToNot(HaveOccurred()) + + nRef, err := env.Lookup.NodeFromResource(env.Ctx, &provider.Reference{ + ResourceId: env.SpaceRootRes, + Path: "/dir1/file1", + }) + Expect(err).ToNot(HaveOccurred()) + path = nRef.InternalPath() + + // Change backend to messagepack + env.Lookup = lookup.New(backend, env.Options) + }) + + It("migrates", func() { + m := migrator.New(env.Lookup, &nullLogger) + err := m.RunMigrations() + Expect(err).ToNot(HaveOccurred()) + + nameAttr, _ := xattr.Get(path, prefixes.NameAttr) + Expect(nameAttr).To(Equal([]byte{})) + + _, err = os.Stat(path + ".mpk") + Expect(err).ToNot(HaveOccurred()) + nameAttr, err = backend.Get(path, prefixes.NameAttr) + Expect(err).ToNot(HaveOccurred()) + Expect(string(nameAttr)).To(Equal("file1")) + }) + }) + }) +}) diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index 2306edd85c..326fb30cb0 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -314,7 +314,7 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis if n.ParentID == "" { d, _ := os.ReadFile(lu.MetadataBackend().MetadataPath(n.InternalPath())) if _, ok := lu.MetadataBackend().(metadata.MessagePackBackend); ok { - appctx.GetLogger(ctx).Error().Str("nodeid", n.ID).Interface("attrs", attrs).Bytes("messagepack", d).Msg("missing parent id") + appctx.GetLogger(ctx).Error().Str("path", n.InternalPath()).Str("nodeid", n.ID).Interface("attrs", attrs).Bytes("messagepack", d).Msg("missing parent id") } return nil, errtypes.InternalError("Missing parent ID on node") } diff --git a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go index 26a6d61f48..1984fad8af 100644 --- a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go +++ b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go @@ -20,6 +20,7 @@ package helpers import ( "context" + "fmt" "os" "path/filepath" @@ -59,6 +60,7 @@ type TestEnv struct { Ctx context.Context SpaceRootRes *providerv1beta1.ResourceId PermissionsClient *mocks.CS3PermissionsClient + Options *options.Options } // Constant UUIDs for the space users @@ -141,12 +143,21 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) { }, }, } - lookup := lookup.New(metadata.XattrsBackend{}, o) + var lu *lookup.Lookup + switch o.MetadataBackend { + case "xattrs": + lu = lookup.New(metadata.XattrsBackend{}, o) + case "messagepack": + lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), o) + default: + return nil, fmt.Errorf("unknown metadata backend %s", o.MetadataBackend) + } + permissions := &mocks.PermissionsChecker{} cs3permissionsclient := &mocks.CS3PermissionsClient{} bs := &treemocks.Blobstore{} - tree := tree.New(o.Root, true, true, lookup, bs) - fs, err := decomposedfs.New(o, lookup, decomposedfs.NewPermissions(permissions, cs3permissionsclient), tree, nil) + tree := tree.New(o.Root, true, true, lu, bs) + fs, err := decomposedfs.New(o, lu, decomposedfs.NewPermissions(permissions, cs3permissionsclient), tree, nil) if err != nil { return nil, err } @@ -158,7 +169,7 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) { Root: tmpRoot, Fs: tmpFs, Tree: tree, - Lookup: lookup, + Lookup: lu, Permissions: permissions, Blobstore: bs, Owner: owner, @@ -167,6 +178,7 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) { Users: users, Ctx: ctx, PermissionsClient: cs3permissionsclient, + Options: o, } env.SpaceRootRes, err = env.CreateTestStorageSpace("personal", nil) diff --git a/pkg/storage/utils/decomposedfs/tree/migrations.go b/pkg/storage/utils/decomposedfs/tree/migrations.go deleted file mode 100644 index 61235ba113..0000000000 --- a/pkg/storage/utils/decomposedfs/tree/migrations.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package tree - -import ( - "io" - "os" - "path/filepath" - - "github.com/cs3org/reva/v2/pkg/logger" - "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" -) - -/** - * This function runs all migrations in sequence. - * Note this sequence must not be changed or it might - * damage existing decomposed fs. - */ -func (t *Tree) runMigrations() error { - if err := t.migration0001Nodes(); err != nil { - return err - } - return t.migration0002SpaceTypes() -} - -func (t *Tree) migration0001Nodes() error { - // create spaces folder and iterate over existing nodes to populate it - nodesPath := filepath.Join(t.root, "nodes") - fi, err := os.Stat(nodesPath) - if err == nil && fi.IsDir() { - - f, err := os.Open(nodesPath) - if err != nil { - return err - } - nodes, err := f.Readdir(0) - if err != nil { - return err - } - - for _, n := range nodes { - nodePath := filepath.Join(nodesPath, n.Name()) - - attr, err := t.lookup.MetadataBackend().Get(nodePath, prefixes.ParentidAttr) - if err == nil && string(attr) == node.RootID { - if err := t.moveNode(n.Name(), n.Name()); err != nil { - logger.New().Error().Err(err). - Str("space", n.Name()). - Msg("could not move space") - continue - } - t.linkSpaceNode("personal", n.Name()) - } - } - // TODO delete nodesPath if empty - } - return nil -} - -func (t *Tree) migration0002SpaceTypes() error { - spaceTypesPath := filepath.Join(t.root, "spacetypes") - fi, err := os.Stat(spaceTypesPath) - if err == nil && fi.IsDir() { - - f, err := os.Open(spaceTypesPath) - if err != nil { - return err - } - spaceTypes, err := f.Readdir(0) - if err != nil { - return err - } - - for _, st := range spaceTypes { - err := t.moveSpaceType(st.Name()) - if err != nil { - logger.New().Error().Err(err). - Str("space", st.Name()). - Msg("could not move space") - continue - } - } - - // delete spacetypespath - d, err := os.Open(spaceTypesPath) - if err != nil { - logger.New().Error().Err(err). - Str("spacetypesdir", spaceTypesPath). - Msg("could not open spacetypesdir") - return nil - } - defer d.Close() - _, err = d.Readdirnames(1) // Or f.Readdir(1) - if err == io.EOF { - // directory is empty we can delete - err := os.Remove(spaceTypesPath) - if err != nil { - logger.New().Error().Err(err). - Str("spacetypesdir", d.Name()). - Msg("could not delete") - } - } else { - logger.New().Error().Err(err). - Str("spacetypesdir", d.Name()). - Msg("could not delete, not empty") - } - } - return nil -} diff --git a/pkg/storage/utils/decomposedfs/tree/tree.go b/pkg/storage/utils/decomposedfs/tree/tree.go index 8b2aac59d4..5881d805a6 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/pkg/storage/utils/decomposedfs/tree/tree.go @@ -35,7 +35,6 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/appctx" "github.com/cs3org/reva/v2/pkg/errtypes" - "github.com/cs3org/reva/v2/pkg/logger" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" @@ -110,128 +109,9 @@ func (t *Tree) Setup() error { return err } } - // Run migrations & return - return t.runMigrations() -} - -func (t *Tree) moveNode(spaceID, nodeID string) error { - dirPath := filepath.Join(t.root, "nodes", nodeID) - f, err := os.Open(dirPath) - if err != nil { - return err - } - children, err := f.Readdir(0) - if err != nil { - return err - } - for _, child := range children { - old := filepath.Join(t.root, "nodes", child.Name()) - new := filepath.Join(t.root, "spaces", lookup.Pathify(spaceID, 1, 2), "nodes", lookup.Pathify(child.Name(), 4, 2)) - if err := os.Rename(old, new); err != nil { - logger.New().Error().Err(err). - Str("space", spaceID). - Str("nodes", child.Name()). - Str("oldpath", old). - Str("newpath", new). - Msg("could not rename node") - } - if child.IsDir() { - if err := t.moveNode(spaceID, child.Name()); err != nil { - return err - } - } - } return nil } -func (t *Tree) moveSpaceType(spaceType string) error { - dirPath := filepath.Join(t.root, "spacetypes", spaceType) - f, err := os.Open(dirPath) - if err != nil { - return err - } - children, err := f.Readdir(0) - if err != nil { - return err - } - for _, child := range children { - old := filepath.Join(t.root, "spacetypes", spaceType, child.Name()) - target, err := os.Readlink(old) - if err != nil { - logger.New().Error().Err(err). - Str("space", spaceType). - Str("nodes", child.Name()). - Str("oldLink", old). - Msg("could not read old symlink") - continue - } - newDir := filepath.Join(t.root, "indexes", "by-type", spaceType) - if err := os.MkdirAll(newDir, 0700); err != nil { - logger.New().Error().Err(err). - Str("space", spaceType). - Str("nodes", child.Name()). - Str("targetDir", newDir). - Msg("could not read old symlink") - } - newLink := filepath.Join(newDir, child.Name()) - if err := os.Symlink(filepath.Join("..", target), newLink); err != nil { - logger.New().Error().Err(err). - Str("space", spaceType). - Str("nodes", child.Name()). - Str("oldpath", old). - Str("newpath", newLink). - Msg("could not rename node") - continue - } - if err := os.Remove(old); err != nil { - logger.New().Error().Err(err). - Str("space", spaceType). - Str("nodes", child.Name()). - Str("oldLink", old). - Msg("could not remove old symlink") - continue - } - } - if err := os.Remove(dirPath); err != nil { - logger.New().Error().Err(err). - Str("space", spaceType). - Str("dir", dirPath). - Msg("could not remove spaces folder, folder probably not empty") - } - return nil -} - -// linkSpace creates a new symbolic link for a space with the given type st, and node id -func (t *Tree) linkSpaceNode(spaceType, spaceID string) { - spaceTypesPath := filepath.Join(t.root, "spacetypes", spaceType, spaceID) - expectedTarget := "../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2) - linkTarget, err := os.Readlink(spaceTypesPath) - if errors.Is(err, os.ErrNotExist) { - err = os.Symlink(expectedTarget, spaceTypesPath) - if err != nil { - logger.New().Error().Err(err). - Str("space_type", spaceType). - Str("space", spaceID). - Msg("could not create symlink") - } - } else { - if err != nil { - logger.New().Error().Err(err). - Str("space_type", spaceType). - Str("space", spaceID). - Msg("could not read symlink") - } - if linkTarget != expectedTarget { - logger.New().Warn(). - Str("space_type", spaceType). - Str("space", spaceID). - Str("expected", expectedTarget). - Str("actual", linkTarget). - Msg("expected a different link target") - } - } -} - // GetMD returns the metadata of a node in the tree func (t *Tree) GetMD(ctx context.Context, n *node.Node) (os.FileInfo, error) { md, err := os.Stat(n.InternalPath())