From 8de5c01f4fb519669909ed91c395aff3a484bce4 Mon Sep 17 00:00:00 2001 From: Christian Richter <1058116+dragonchaser@users.noreply.github.com> Date: Fri, 15 Jul 2022 14:11:59 +0200 Subject: [PATCH] Add user filter (#3046) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add index for user filter Signed-off-by: Christian Richter * implement filter & refactor spacetypes handling Signed-off-by: Christian Richter * bump go-cs3apis Signed-off-by: Christian Richter * fix wrong call to storageSpaceFromNode Signed-off-by: Christian Richter * fix tests Signed-off-by: Christian Richter * remove empty spacetypes folder Signed-off-by: Christian Richter * add missing error output Signed-off-by: Christian Richter * fix nil in error case Signed-off-by: Jörn Friedrich Dreyer * fix typo Signed-off-by: Jörn Friedrich Dreyer * only try clean up if spacetypes dir is empty Signed-off-by: Jörn Friedrich Dreyer * simplify checkNodePermissions logic Signed-off-by: Jörn Friedrich Dreyer * incorporate requested changes Signed-off-by: Christian Richter Co-authored-by: Jörn Friedrich Dreyer --- .drone.star | 8 +- changelog/unreleased/add-user-filter.md | 5 + go.mod | 2 +- go.sum | 4 +- .../grpc/services/gateway/storageprovider.go | 3 + pkg/storage/fs/owncloudsql/spaces.go | 2 + pkg/storage/registry/spaces/spaces.go | 11 ++ .../utils/decomposedfs/decomposedfs.go | 2 +- pkg/storage/utils/decomposedfs/grants.go | 2 +- pkg/storage/utils/decomposedfs/grants_test.go | 2 +- pkg/storage/utils/decomposedfs/spaces.go | 165 +++++++++++++----- pkg/storage/utils/decomposedfs/spaces_test.go | 25 ++- .../utils/decomposedfs/tree/migrations.go | 123 +++++++++++++ pkg/storage/utils/decomposedfs/tree/tree.go | 95 ++++++---- .../grpc/gateway_storageprovider_test.go | 4 +- 15 files changed, 345 insertions(+), 108 deletions(-) create mode 100644 changelog/unreleased/add-user-filter.md create mode 100644 pkg/storage/utils/decomposedfs/tree/migrations.go diff --git a/.drone.star b/.drone.star index e39e1df7e1..64dd13eedc 100644 --- a/.drone.star +++ b/.drone.star @@ -577,7 +577,7 @@ def virtualViews(): "PATH_TO_CORE": "/drone/src/tmp/testrunner", "TEST_SERVER_URL": "http://revad-services:20180", "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", - "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/spaces/* /drone/src/tmp/reva/data/blobs/* /drone/src/tmp/reva/data/spacetypes/*", + "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/spaces/* /drone/src/tmp/reva/data/blobs/* /drone/src/tmp/reva/data/indexes", "STORAGE_DRIVER": "OCIS", "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", "TEST_REVA": "true", @@ -756,7 +756,7 @@ def litmusOcisSpacesDav(): "commands": [ # The spaceid is randomly generated during the first login so we need this hack to construct the correct url. "curl -s -k -u einstein:relativity -I http://revad-services:20080/remote.php/dav/files/einstein", - "export LITMUS_URL=http://revad-services:20080/remote.php/dav/spaces/$(ls /drone/src/tmp/reva/data/spacetypes/personal/)", + "export LITMUS_URL=http://revad-services:20080/remote.php/dav/spaces/$(ls /drone/src/tmp/reva/data/indexes/by-type/personal/)", "/usr/local/bin/litmus-wrapper", ], }, @@ -929,7 +929,7 @@ def ocisIntegrationTests(parallelRuns, skipExceptParts = []): "environment": { "TEST_SERVER_URL": "http://revad-services:20080", "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", - "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/spaces/* /drone/src/tmp/reva/data/blobs/* /drone/src/tmp/reva/data/spacetypes/*", + "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/spaces/* /drone/src/tmp/reva/data/blobs/* /drone/src/tmp/reva/data/indexes/by-type/*", "STORAGE_DRIVER": "OCIS", "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", "TEST_WITH_LDAP": "true", @@ -1008,7 +1008,7 @@ def s3ngIntegrationTests(parallelRuns, skipExceptParts = []): "environment": { "TEST_SERVER_URL": "http://revad-services:20080", "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", - "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/spaces/* /drone/src/tmp/reva/data/blobs/* /drone/src/tmp/reva/data/spacetypes/*", + "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/spaces/* /drone/src/tmp/reva/data/blobs/* /drone/src/tmp/reva/data/indexes/by-type/*", "STORAGE_DRIVER": "S3NG", "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", "TEST_WITH_LDAP": "true", diff --git a/changelog/unreleased/add-user-filter.md b/changelog/unreleased/add-user-filter.md new file mode 100644 index 0000000000..9f8ebe91b0 --- /dev/null +++ b/changelog/unreleased/add-user-filter.md @@ -0,0 +1,5 @@ +Enhancement: Add user filter + +This PR adds the ability to filter spaces by user-id + +https://github.com/owncloud/ocis/pull/4072 \ No newline at end of file diff --git a/go.mod b/go.mod index 8521c3a161..92a8ea3e27 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e - github.com/cs3org/go-cs3apis v0.0.0-20220621145831-c38cca0796c2 + github.com/cs3org/go-cs3apis v0.0.0-20220711084433-8f71d4e812a3 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 github.com/dgraph-io/ristretto v0.1.0 github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 diff --git a/go.sum b/go.sum index 3c58365cbd..29f55774f9 100644 --- a/go.sum +++ b/go.sum @@ -216,8 +216,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20220621145831-c38cca0796c2 h1:o/ovJzS4pyL/rZgp0MtC4Q7JIle5DikimilTLBw2TjY= -github.com/cs3org/go-cs3apis v0.0.0-20220621145831-c38cca0796c2/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20220711084433-8f71d4e812a3 h1:QSQ2DGKPMChB4vHSs1Os9TnOJl21BrzKX9D5EtQfDog= +github.com/cs3org/go-cs3apis v0.0.0-20220711084433-8f71d4e812a3/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index 08181d8b36..cab0abbf7d 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -241,6 +241,9 @@ func (s *svc) ListStorageSpaces(ctx context.Context, req *provider.ListStorageSp filters["owner_id"] = f.GetOwner().OpaqueId case provider.ListStorageSpacesRequest_Filter_TYPE_SPACE_TYPE: filters["space_type"] = f.GetSpaceType() + case provider.ListStorageSpacesRequest_Filter_TYPE_USER: + filters["user_idp"] = f.GetUser().GetIdp() + filters["user_id"] = f.GetUser().GetOpaqueId() default: return &provider.ListStorageSpacesResponse{ Status: status.NewInvalidArg(ctx, fmt.Sprintf("unknown filter %v", f.Type)), diff --git a/pkg/storage/fs/owncloudsql/spaces.go b/pkg/storage/fs/owncloudsql/spaces.go index ab781565ac..05bd1644a2 100644 --- a/pkg/storage/fs/owncloudsql/spaces.go +++ b/pkg/storage/fs/owncloudsql/spaces.go @@ -48,6 +48,8 @@ func (fs *owncloudsqlfs) ListStorageSpaces(ctx context.Context, filter []*provid filteringUnsupportedSpaceTypes = (t != "personal" && !strings.HasPrefix(t, "+")) case provider.ListStorageSpacesRequest_Filter_TYPE_ID: _, spaceID, _, _ = storagespace.SplitID(filter[i].GetId().OpaqueId) + case provider.ListStorageSpacesRequest_Filter_TYPE_USER: + _, spaceID, _, _ = storagespace.SplitID(filter[i].GetId().OpaqueId) } } if filteringUnsupportedSpaceTypes { diff --git a/pkg/storage/registry/spaces/spaces.go b/pkg/storage/registry/spaces/spaces.go index 8b36d1d924..6aaaf30e1f 100644 --- a/pkg/storage/registry/spaces/spaces.go +++ b/pkg/storage/registry/spaces/spaces.go @@ -316,6 +316,17 @@ func (r *registry) buildFilters(filterMap map[string]string) []*providerpb.ListS }) } } + if filterMap["user_id"] != "" { + filters = append(filters, &providerpb.ListStorageSpacesRequest_Filter{ + Type: providerpb.ListStorageSpacesRequest_Filter_TYPE_USER, + Term: &providerpb.ListStorageSpacesRequest_Filter_User{ + User: &userpb.UserId{ + Idp: filterMap["user_idp"], + OpaqueId: filterMap["user_id"], + }, + }, + }) + } if filterMap["owner_id"] != "" && filterMap["owner_idp"] != "" { filters = append(filters, &providerpb.ListStorageSpacesRequest_Filter{ Type: providerpb.ListStorageSpacesRequest_Filter_TYPE_OWNER, diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 09f7548234..d4f151eb46 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -545,7 +545,7 @@ func (fs *Decomposedfs) GetMD(ctx context.Context, ref *provider.Reference, mdKe } } if addSpace { - if md.Space, err = fs.storageSpaceFromNode(ctx, node, node.InternalPath(), false, false); err != nil { + if md.Space, err = fs.storageSpaceFromNode(ctx, node, true); err != nil { return nil, err } } diff --git a/pkg/storage/utils/decomposedfs/grants.go b/pkg/storage/utils/decomposedfs/grants.go index b246a39bc8..75dfae9274 100644 --- a/pkg/storage/utils/decomposedfs/grants.go +++ b/pkg/storage/utils/decomposedfs/grants.go @@ -248,7 +248,7 @@ func (fs *Decomposedfs) storeGrant(ctx context.Context, n *node.Node, g *provide // when a grant is added to a space, do not add a new space under "shares" if spaceGrant := ctx.Value(utils.SpaceGrant); spaceGrant == nil { - err := fs.linkStorageSpaceType(ctx, spaceTypeShare, n.ID) + err := fs.updateIndexes(ctx, g.GetGrantee().GetUserId().GetOpaqueId(), spaceTypeShare, n.ID) if err != nil { return err } diff --git a/pkg/storage/utils/decomposedfs/grants_test.go b/pkg/storage/utils/decomposedfs/grants_test.go index 387e5b1707..460cad10a6 100644 --- a/pkg/storage/utils/decomposedfs/grants_test.go +++ b/pkg/storage/utils/decomposedfs/grants_test.go @@ -126,7 +126,7 @@ var _ = Describe("Grants", func() { err := env.Fs.AddGrant(env.Ctx, ref, grant) Expect(err).ToNot(HaveOccurred()) - spaceTypesPath := filepath.Join(env.Root, "spacetypes") + spaceTypesPath := filepath.Join(env.Root, "indexes", "by-type") tfs.root = spaceTypesPath entries, err := fs.ReadDir(tfs, "share") Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go index 4a94498098..2491f782fb 100644 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ b/pkg/storage/utils/decomposedfs/spaces.go @@ -105,7 +105,7 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr } } - err = fs.linkStorageSpaceType(ctx, req.Type, root.ID) + err = fs.updateIndexes(ctx, req.GetOwner().GetId().GetOpaqueId(), req.Type, root.ID) if err != nil { return nil, err } @@ -160,7 +160,7 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr } } - space, err := fs.storageSpaceFromNode(ctx, root, root.InternalPath(), false, false) + space, err := fs.storageSpaceFromNode(ctx, root, true) if err != nil { return nil, err } @@ -215,21 +215,17 @@ func (fs *Decomposedfs) canCreateSpace(ctx context.Context, spaceID string) bool return checkRes.Status.Code == v1beta11.Code_CODE_OK } -// ReadSpaceAndNodeFromSpaceTypeLink reads a symlink and parses space and node id if the link has the correct format, eg: +// ReadSpaceAndNodeFromIndexLink reads a symlink and parses space and node id if the link has the correct format, eg: // ../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51 // ../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51.T.2022-02-24T12:35:18.196484592Z -func ReadSpaceAndNodeFromSpaceTypeLink(path string) (string, string, error) { - link, err := os.Readlink(path) - if err != nil { - return "", "", err - } - // ../../spaces/sp/ace-id/nodes/sh/or/tn/od/eid - // 0 1 2 3 4 5 6 7 8 9 10 +func ReadSpaceAndNodeFromIndexLink(link string) (string, string, error) { + // ../../../spaces/sp/ace-id/nodes/sh/or/tn/od/eid + // 0 1 2 3 4 5 6 7 8 9 10 11 parts := strings.Split(link, string(filepath.Separator)) - if len(parts) != 11 || parts[0] != ".." || parts[1] != ".." || parts[2] != "spaces" || parts[5] != "nodes" { + if len(parts) != 12 || parts[0] != ".." || parts[1] != ".." || parts[2] != ".." || parts[3] != "spaces" || parts[6] != "nodes" { return "", "", errtypes.InternalError("malformed link") } - return strings.Join(parts[3:5], ""), strings.Join(parts[6:11], ""), nil + return strings.Join(parts[4:6], ""), strings.Join(parts[7:12], ""), nil } // ListStorageSpaces returns a list of StorageSpaces. @@ -250,9 +246,10 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide var ( spaceID = spaceIDAny nodeID = spaceIDAny + userID = spaceIDAny ) - spaceTypes := []string{} + spaceTypes := map[string]struct{}{} for i := range filter { switch filter[i].Type { @@ -263,21 +260,30 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide case "+grant": // TODO include grants default: - spaceTypes = append(spaceTypes, filter[i].GetSpaceType()) + spaceTypes[filter[i].GetSpaceType()] = struct{}{} } case provider.ListStorageSpacesRequest_Filter_TYPE_ID: _, spaceID, nodeID, _ = storagespace.SplitID(filter[i].GetId().OpaqueId) if strings.Contains(nodeID, "/") { return []*provider.StorageSpace{}, nil } + case provider.ListStorageSpacesRequest_Filter_TYPE_USER: + // TODO: refactor this to GetUserId() in cs3 + userID = filter[i].GetUser().GetOpaqueId() } } if len(spaceTypes) == 0 { - spaceTypes = []string{spaceTypeAny} + spaceTypes[spaceTypeAny] = struct{}{} } canListAllSpaces := fs.canListAllSpaces(ctx) + if userID != spaceTypeAny && !canListAllSpaces { + return nil, errtypes.PermissionDenied(fmt.Sprintf("user %s is not allowed to list spaces of other users", ctxpkg.ContextMustGetUser(ctx).GetId().GetOpaqueId())) + } + + checkNodePermissions := !canListAllSpaces && !unrestricted + spaces := []*provider.StorageSpace{} // build the glob path, eg. // /path/to/root/spaces/{spaceType}/{spaceId} @@ -295,28 +301,52 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide // return empty list return spaces, nil } - space, err := fs.storageSpaceFromNode(ctx, n, n.InternalPath(), canListAllSpaces, unrestricted) + space, err := fs.storageSpaceFromNode(ctx, n, checkNodePermissions) if err != nil { return nil, err } // filter space types - for _, spaceType := range spaceTypes { - if spaceType == spaceTypeAny || spaceType == space.SpaceType { - spaces = append(spaces, space) - } + _, ok1 := spaceTypes[spaceTypeAny] + _, ok2 := spaceTypes[space.SpaceType] + if ok1 || ok2 { + spaces = append(spaces, space) } - + // TODO: filter user id return spaces, nil } - matches := []string{} - for _, spaceType := range spaceTypes { - path := filepath.Join(fs.o.Root, "spacetypes", spaceType, nodeID) + matches := map[string]struct{}{} + + if userID != spaceTypeAny { + path := filepath.Join(fs.o.Root, "indexes", "by-user-id", userID, nodeID) m, err := filepath.Glob(path) if err != nil { return nil, err } - matches = append(matches, m...) + for _, match := range m { + link, err := os.Readlink(match) + if err != nil { + continue + } + matches[link] = struct{}{} + } + } + + if userID == spaceTypeAny { + for spaceType := range spaceTypes { + path := filepath.Join(fs.o.Root, "indexes", "by-type", spaceType, nodeID) + m, err := filepath.Glob(path) + if err != nil { + return nil, err + } + for _, match := range m { + link, err := os.Readlink(match) + if err != nil { + continue + } + matches[link] = struct{}{} + } + } } // FIXME if the space does not exist try a node as the space root. @@ -332,16 +362,16 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide numShares := 0 - for i := range matches { + for match := range matches { var err error // do not investigate flock files any further. They indicate file locks but are not relevant here. - if strings.HasSuffix(matches[i], ".flock") { + if strings.HasSuffix(match, ".flock") { continue } // always read link in case storage space id != node id - spaceID, nodeID, err = ReadSpaceAndNodeFromSpaceTypeLink(matches[i]) + spaceID, nodeID, err = ReadSpaceAndNodeFromIndexLink(match) if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("match", matches[i]).Msg("could not read link, skipping") + appctx.GetLogger(ctx).Error().Err(err).Str("match", match).Msg("could not read link, skipping") continue } @@ -355,25 +385,27 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide continue } - spaceType := filepath.Base(filepath.Dir(matches[i])) + space, err := fs.storageSpaceFromNode(ctx, n, checkNodePermissions) + if err != nil { + if _, ok := err.(errtypes.IsPermissionDenied); !ok { + appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not convert to storage space") + } + continue + } // FIXME type share evolved to grant on the edge branch ... make it configurable if the driver should support them or not for now ... ignore type share - if spaceType == spaceTypeShare { + if space.SpaceType == spaceTypeShare { numShares++ // do not list shares as spaces for the owner continue } // TODO apply more filters - space, err := fs.storageSpaceFromNode(ctx, n, matches[i], canListAllSpaces, unrestricted) - if err != nil { - if _, ok := err.(errtypes.IsPermissionDenied); !ok { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not convert to storage space") - } - continue + _, ok1 := spaceTypes[spaceTypeAny] + _, ok2 := spaceTypes[space.SpaceType] + if ok1 || ok2 { + spaces = append(spaces, space) } - spaces = append(spaces, space) - } // if there are no matches (or they happened to be spaces for the owner) and the node is a child return a space if len(matches) <= numShares && nodeID != spaceID { @@ -383,7 +415,7 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide return nil, err } if n.Exists { - space, err := fs.storageSpaceFromNode(ctx, n, n.InternalPath(), canListAllSpaces, unrestricted) + space, err := fs.storageSpaceFromNode(ctx, n, checkNodePermissions) if err != nil { return nil, err } @@ -491,7 +523,7 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up } // send back the updated data from the storage - updatedSpace, err := fs.storageSpaceFromNode(ctx, node, node.InternalPath(), false, false) + updatedSpace, err := fs.storageSpaceFromNode(ctx, node, true) if err != nil { return nil, err } @@ -535,7 +567,7 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De return err } // remove type index - spaceTypePath := filepath.Join(fs.o.Root, "spacetypes", spaceType, spaceID) + spaceTypePath := filepath.Join(fs.o.Root, "indexes", "by-type", spaceType, spaceID) if err := os.Remove(spaceTypePath); err != nil { return err } @@ -555,14 +587,51 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De return n.SetDTime(&dtime) } +func (fs *Decomposedfs) updateIndexes(ctx context.Context, userID, spaceType, spaceID string) error { + err := fs.linkStorageSpaceType(ctx, spaceType, spaceID) + if err != nil { + return err + } + return fs.linkSpaceByUser(ctx, userID, spaceID) +} + +func (fs *Decomposedfs) linkSpaceByUser(ctx context.Context, userID, spaceID string) error { + if userID == "" { + return nil + } + // create user index dir + // TODO: pathify userID + if err := os.MkdirAll(filepath.Join(fs.o.Root, "indexes", "by-user-id", userID), 0700); err != nil { + return err + } + + err := os.Symlink("../../../spaces/"+lookup.Pathify(spaceID, 1, 2)+"/nodes/"+lookup.Pathify(spaceID, 4, 2), filepath.Join(fs.o.Root, "indexes/by-user-id", userID, spaceID)) + if err != nil { + if isAlreadyExists(err) { + appctx.GetLogger(ctx).Debug().Err(err).Str("space", spaceID).Str("user-id", userID).Msg("symlink already exists") + // FIXME: is it ok to wipe this err if the symlink already exists? + err = nil //nolint + } else { + // TODO how should we handle error cases here? + appctx.GetLogger(ctx).Error().Err(err).Str("space", spaceID).Str("user-id", userID).Msg("could not create symlink") + } + } + return nil +} + +// TODO: implement linkSpaceByGroup + func (fs *Decomposedfs) linkStorageSpaceType(ctx context.Context, spaceType string, spaceID string) error { + if spaceType == "" { + return nil + } // create space type dir - if err := os.MkdirAll(filepath.Join(fs.o.Root, "spacetypes", spaceType), 0700); err != nil { + if err := os.MkdirAll(filepath.Join(fs.o.Root, "indexes", "by-type", spaceType), 0700); err != nil { return err } // link space in spacetypes - err := os.Symlink("../../spaces/"+lookup.Pathify(spaceID, 1, 2)+"/nodes/"+lookup.Pathify(spaceID, 4, 2), filepath.Join(fs.o.Root, "spacetypes", spaceType, spaceID)) + err := os.Symlink("../../../spaces/"+lookup.Pathify(spaceID, 1, 2)+"/nodes/"+lookup.Pathify(spaceID, 4, 2), filepath.Join(fs.o.Root, "indexes", "by-type", spaceType, spaceID)) if err != nil { if isAlreadyExists(err) { appctx.GetLogger(ctx).Debug().Err(err).Str("space", spaceID).Str("spacetype", spaceType).Msg("symlink already exists") @@ -577,9 +646,9 @@ func (fs *Decomposedfs) linkStorageSpaceType(ctx context.Context, spaceType stri return err } -func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, nodePath string, canListAllSpaces bool, unrestricted bool) (*provider.StorageSpace, error) { +func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, checkPermissions bool) (*provider.StorageSpace, error) { user := ctxpkg.ContextMustGetUser(ctx) - if !canListAllSpaces || !unrestricted { + if checkPermissions { ok, err := node.NewPermissions(fs.lu).HasPermission(ctx, n, func(p *provider.ResourcePermissions) bool { return p.Stat }) @@ -685,7 +754,7 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, Seconds: uint64(un / 1000000000), Nanos: uint32(un % 1000000000), } - } else if fi, err := os.Stat(nodePath); err == nil { + } else if fi, err := os.Stat(n.InternalPath()); err == nil { // fall back to stat mtime tmtime = fi.ModTime() un := fi.ModTime().UnixNano() @@ -704,7 +773,7 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, Value: []byte(etag), } - spaceAttributes, err := xattrs.All(nodePath) + spaceAttributes, err := xattrs.All(n.InternalPath()) if err != nil { return nil, err } diff --git a/pkg/storage/utils/decomposedfs/spaces_test.go b/pkg/storage/utils/decomposedfs/spaces_test.go index d4bfbd0cef..9841eeb9f1 100644 --- a/pkg/storage/utils/decomposedfs/spaces_test.go +++ b/pkg/storage/utils/decomposedfs/spaces_test.go @@ -20,7 +20,6 @@ package decomposedfs_test import ( "os" - "path/filepath" permissionsv1beta1 "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -130,10 +129,7 @@ var _ = Describe("Spaces", func() { DescribeTable("ReadSpaceAndNodeFromSpaceTypeLink", func(link string, expectSpace string, expectedNode string, shouldErr bool) { - err := os.Symlink(link, filepath.Join(tmpdir, "link")) - Expect(err).ToNot(HaveOccurred()) - - space, node, err := decomposedfs.ReadSpaceAndNodeFromSpaceTypeLink(filepath.Join(tmpdir, "link")) + space, node, err := decomposedfs.ReadSpaceAndNodeFromIndexLink(link) if shouldErr { Expect(err).To(HaveOccurred()) } else { @@ -143,15 +139,16 @@ var _ = Describe("Spaces", func() { Expect(node).To(Equal(expectedNode)) }, - Entry("invalid number of slashes", "../../spaces/sp_ace-id/nodes/sh/or/tn/od/eid", "", "", true), - Entry("does not contain spaces", "../../spac_s/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), - Entry("does not contain nodes", "../../spaces/sp/ace-id/nod_s/sh/or/tn/od/eid", "", "", true), - Entry("does not start with ..", "_./../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), - Entry("does not start with ../..", "../_./spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), - Entry("invalid", "../../spaces/space-id/nodes/sh/or/tn/od/eid", "", "", true), - Entry("uuid", "../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51", "4c510ada-c86b-4815-8820-42cdf82c3d51", "4c510ada-c86b-4815-8820-42cdf82c3d51", false), - Entry("uuid", "../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51.T.2022-02-24T12:35:18.196484592Z", "4c510ada-c86b-4815-8820-42cdf82c3d51", "4c510ada-c86b-4815-8820-42cdf82c3d51.T.2022-02-24T12:35:18.196484592Z", false), - Entry("short", "../../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "space-id", "shortnodeid", false), + Entry("invalid number of slashes", "../../../spaces/sp_ace-id/nodes/sh/or/tn/od/eid", "", "", true), + Entry("does not contain spaces", "../../../spac_s/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), + Entry("does not contain nodes", "../../../spaces/sp/ace-id/nod_s/sh/or/tn/od/eid", "", "", true), + Entry("does not start with ..", "_./../../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), + Entry("does not start with ../..", "../_./../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), + Entry("does not start with ../../..", "../_./../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), + Entry("invalid", "../../../spaces/space-id/nodes/sh/or/tn/od/eid", "", "", true), + Entry("uuid", "../../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51", "4c510ada-c86b-4815-8820-42cdf82c3d51", "4c510ada-c86b-4815-8820-42cdf82c3d51", false), + Entry("uuid", "../../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51.T.2022-02-24T12:35:18.196484592Z", "4c510ada-c86b-4815-8820-42cdf82c3d51", "4c510ada-c86b-4815-8820-42cdf82c3d51.T.2022-02-24T12:35:18.196484592Z", false), + Entry("short", "../../../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "space-id", "shortnodeid", false), ) }) }) diff --git a/pkg/storage/utils/decomposedfs/tree/migrations.go b/pkg/storage/utils/decomposedfs/tree/migrations.go new file mode 100644 index 0000000000..c6bce455a4 --- /dev/null +++ b/pkg/storage/utils/decomposedfs/tree/migrations.go @@ -0,0 +1,123 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package tree + +import ( + "io" + "os" + "path/filepath" + + "github.com/cs3org/reva/v2/pkg/logger" +) + +/** + * This function runs all migrations in sequence. + * Note this sequence must not be changed or it might + * damage existing decomposed fs. + */ +func (t *Tree) runMigrations() error { + if err := t.migration0001Nodes(); err != nil { + return err + } + return t.migration0002SpaceTypes() +} + +func (t *Tree) migration0001Nodes() error { + // create spaces folder and iterate over existing nodes to populate it + nodesPath := filepath.Join(t.root, "nodes") + fi, err := os.Stat(nodesPath) + if err == nil && fi.IsDir() { + + f, err := os.Open(nodesPath) + if err != nil { + return err + } + nodes, err := f.Readdir(0) + if err != nil { + return err + } + + for _, node := range nodes { + nodePath := filepath.Join(nodesPath, node.Name()) + + if isRootNode(nodePath) { + if err := t.moveNode(node.Name(), node.Name()); err != nil { + logger.New().Error().Err(err). + Str("space", node.Name()). + Msg("could not move space") + continue + } + t.linkSpaceNode("personal", node.Name()) + } + } + // TODO delete nodesPath if empty + } + return nil +} + +func (t *Tree) migration0002SpaceTypes() error { + spaceTypesPath := filepath.Join(t.root, "spacetypes") + fi, err := os.Stat(spaceTypesPath) + if err == nil && fi.IsDir() { + + f, err := os.Open(spaceTypesPath) + if err != nil { + return err + } + spaceTypes, err := f.Readdir(0) + if err != nil { + return err + } + + for _, st := range spaceTypes { + err := t.moveSpaceType(st.Name()) + if err != nil { + logger.New().Error().Err(err). + Str("space", st.Name()). + Msg("could not move space") + continue + } + } + + // delete spacetypespath + d, err := os.Open(spaceTypesPath) + if err != nil { + logger.New().Error().Err(err). + Str("spacetypesdir", spaceTypesPath). + Msg("could not open spacetypesdir") + return nil + } + defer d.Close() + _, err = d.Readdirnames(1) // Or f.Readdir(1) + if err == io.EOF { + // directory is empty we can delete + err := os.Remove(spaceTypesPath) + if err != nil { + logger.New().Error().Err(err). + Str("spacetypesdir", d.Name()). + Msg("could not delete") + } + } else { + logger.New().Error().Err(err). + Str("spacetypesdir", d.Name()). + Msg("could not delete, not empty") + } + } + return nil +} diff --git a/pkg/storage/utils/decomposedfs/tree/tree.go b/pkg/storage/utils/decomposedfs/tree/tree.go index bcdbbf9cf2..bea5b8b399 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/pkg/storage/utils/decomposedfs/tree/tree.go @@ -102,40 +102,10 @@ func (t *Tree) Setup() error { return err } } - - // create spaces folder and iterate over existing nodes to populate it - nodesPath := filepath.Join(t.root, "nodes") - fi, err := os.Stat(nodesPath) - if err == nil && fi.IsDir() { - - f, err := os.Open(nodesPath) - if err != nil { - return err - } - nodes, err := f.Readdir(0) - if err != nil { - return err - } - - for _, node := range nodes { - nodePath := filepath.Join(nodesPath, node.Name()) - - if isRootNode(nodePath) { - if err := t.moveNode(node.Name(), node.Name()); err != nil { - logger.New().Error().Err(err). - Str("space", node.Name()). - Msg("could not move space") - continue - } - t.linkSpace("personal", node.Name()) - } - } - // TODO delete nodesPath if empty - - } - - return nil + // Run migrations & return + return t.runMigrations() } + func (t *Tree) moveNode(spaceID, nodeID string) error { dirPath := filepath.Join(t.root, "nodes", nodeID) f, err := os.Open(dirPath) @@ -166,8 +136,65 @@ func (t *Tree) moveNode(spaceID, nodeID string) error { return nil } +func (t *Tree) moveSpaceType(spaceType string) error { + dirPath := filepath.Join(t.root, "spacetypes", spaceType) + f, err := os.Open(dirPath) + if err != nil { + return err + } + children, err := f.Readdir(0) + if err != nil { + return err + } + for _, child := range children { + old := filepath.Join(t.root, "spacetypes", spaceType, child.Name()) + target, err := os.Readlink(old) + if err != nil { + logger.New().Error().Err(err). + Str("space", spaceType). + Str("nodes", child.Name()). + Str("oldLink", old). + Msg("could not read old symplink") + continue + } + newDir := filepath.Join(t.root, "indexes", "by-type", spaceType) + if err := os.MkdirAll(newDir, 0700); err != nil { + logger.New().Error().Err(err). + Str("space", spaceType). + Str("nodes", child.Name()). + Str("targetDir", newDir). + Msg("could not read old symplink") + } + newLink := filepath.Join(newDir, child.Name()) + if err := os.Symlink(filepath.Join("..", target), newLink); err != nil { + logger.New().Error().Err(err). + Str("space", spaceType). + Str("nodes", child.Name()). + Str("oldpath", old). + Str("newpath", newLink). + Msg("could not rename node") + continue + } + if err := os.Remove(old); err != nil { + logger.New().Error().Err(err). + Str("space", spaceType). + Str("nodes", child.Name()). + Str("oldLink", old). + Msg("could not remove old symlink") + continue + } + } + if err := os.Remove(dirPath); err != nil { + logger.New().Error().Err(err). + Str("space", spaceType). + Str("dir", dirPath). + Msg("could not remove spaces folder, folder probably not empty") + } + return nil +} + // linkSpace creates a new symbolic link for a space with the given type st, and node id -func (t *Tree) linkSpace(spaceType, spaceID string) { +func (t *Tree) linkSpaceNode(spaceType, spaceID string) { spaceTypesPath := filepath.Join(t.root, "spacetypes", spaceType, spaceID) expectedTarget := "../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2) linkTarget, err := os.Readlink(spaceTypesPath) diff --git a/tests/integration/grpc/gateway_storageprovider_test.go b/tests/integration/grpc/gateway_storageprovider_test.go index f73dc19958..60df3f8b8d 100644 --- a/tests/integration/grpc/gateway_storageprovider_test.go +++ b/tests/integration/grpc/gateway_storageprovider_test.go @@ -308,9 +308,9 @@ var _ = Describe("gateway", func() { Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) ssid, err := storagespace.ParseID(space.Id.OpaqueId) Expect(err).ToNot(HaveOccurred()) - _, err = os.Stat(path.Join(revads["storage"].StorageRoot, "/spacetypes/project", ssid.SpaceId)) + _, err = os.Stat(path.Join(revads["storage"].StorageRoot, "/indexes/by-type/project", ssid.SpaceId)) Expect(err).To(HaveOccurred()) - _, err = os.Stat(path.Join(revads["storage2"].StorageRoot, "/spacetypes/project", ssid.SpaceId)) + _, err = os.Stat(path.Join(revads["storage2"].StorageRoot, "/indexes/by-type/project", ssid.SpaceId)) Expect(err).ToNot(HaveOccurred()) })