From 733d40a10519c4227508f401738d9b666653918c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 8 Jan 2021 17:33:40 +0000 Subject: [PATCH] allow setting favorites, mtime and a temporary etag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/ocis/metadata.go | 142 +++++++++++++++++++++--- pkg/storage/fs/ocis/node.go | 188 ++++++++++++++++++++++++++------ pkg/storage/fs/ocis/ocis.go | 2 +- 3 files changed, 282 insertions(+), 50 deletions(-) diff --git a/pkg/storage/fs/ocis/metadata.go b/pkg/storage/fs/ocis/metadata.go index ff167a8b419..47cdbc885c3 100644 --- a/pkg/storage/fs/ocis/metadata.go +++ b/pkg/storage/fs/ocis/metadata.go @@ -20,20 +20,37 @@ package ocis import ( "context" + "fmt" "path/filepath" + "strconv" + "strings" + "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/user" "github.com/pkg/errors" "github.com/pkg/xattr" ) +func parseMTime(v string) (t time.Time, err error) { + p := strings.SplitN(v, ".", 2) + var sec, nsec int64 + if sec, err = strconv.ParseInt(p[0], 10, 64); err == nil { + if len(p) > 1 { + nsec, err = strconv.ParseInt(p[1], 10, 64) + } + } + return time.Unix(sec, nsec), err +} + func (fs *ocisfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) (err error) { n, err := fs.lu.NodeFromResource(ctx, ref) if err != nil { return errors.Wrap(err, "ocisfs: error resolving ref") } + sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() if !n.Exists { err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) @@ -52,14 +69,67 @@ func (fs *ocisfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Refere } nodePath := n.lu.toInternalPath(n.ID) + + errs := []error{} + // TODO should we really continue updating when an error occurs? + if md.Metadata != nil { + if val, ok := md.Metadata["mtime"]; ok { + delete(md.Metadata, "mtime") + err := n.SetMtime(ctx, val) + if err != nil { + errs = append(errs, errors.Wrap(err, "could not set mtime")) + } + } + // TODO(jfd) special handling for atime? + // TODO(jfd) allow setting birth time (btime)? + // TODO(jfd) any other metadata that is interesting? fileid? + // TODO unset when file is updated + // TODO unset when folder is updated or add timestamp to etag? + if val, ok := md.Metadata["etag"]; ok { + delete(md.Metadata, "etag") + err := n.SetEtag(ctx, val) + if err != nil { + errs = append(errs, errors.Wrap(err, "could not set etag")) + } + } + if val, ok := md.Metadata[_favoriteKey]; ok { + delete(md.Metadata, _favoriteKey) + if u, ok := user.ContextGetUser(ctx); ok { + if uid := u.GetId(); uid != nil { + if err := n.SetFavorite(uid, val); err != nil { + sublog.Error().Err(err). + Interface("user", u). + Msg("could not set favorite flag") + errs = append(errs, errors.Wrap(err, "could not set favorite flag")) + } + } else { + sublog.Error().Interface("user", u).Msg("user has no id") + errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "user has no id")) + } + } else { + sublog.Error().Interface("user", u).Msg("error getting user from ctx") + errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx")) + } + } + } for k, v := range md.Metadata { - // TODO set etag as temporary etag tmpEtagAttr attrName := metadataPrefix + k if err = xattr.Set(nodePath, attrName, []byte(v)); err != nil { - return errors.Wrap(err, "ocisfs: could not set metadata attribute "+attrName+" to "+k) + errs = append(errs, errors.Wrap(err, "ocisfs: could not set metadata attribute "+attrName+" to "+k)) } } - return + + switch len(errs) { + case 0: + return fs.tp.Propagate(ctx, n) + case 1: + // TODO Propagate if anything changed + return errs[0] + default: + // TODO Propagate if anything changed + // TODO how to return multiple errors? + return errors.New("multiple errors occurred, see log for details") + } } func (fs *ocisfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) { @@ -67,6 +137,7 @@ func (fs *ocisfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refe if err != nil { return errors.Wrap(err, "ocisfs: error resolving ref") } + sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() if !n.Exists { err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) @@ -74,7 +145,7 @@ func (fs *ocisfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refe } ok, err := fs.p.HasPermission(ctx, n, func(rp *provider.ResourcePermissions) bool { - // TODO use SetArbitraryMetadata grant to CS3 api, tracked in https://github.com/cs3org/cs3apis/issues/91 + // TODO use SetArbitraryMetadata grant to CS3 api, tracked in https://github.com/cs3org/cs3apis/issues/91 return rp.InitiateFileUpload }) switch { @@ -85,22 +156,57 @@ func (fs *ocisfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refe } nodePath := n.lu.toInternalPath(n.ID) - for i := range keys { - attrName := metadataPrefix + keys[i] - if err = xattr.Remove(nodePath, attrName); err != nil { - // a non-existing attribute will return an error, which we can ignore - // (using string compare because the error type is syscall.Errno and not wrapped/recognizable) - if e, ok := err.(*xattr.Error); !ok || !(e.Err.Error() == "no data available" || - // darwin - e.Err.Error() == "attribute not found") { - appctx.GetLogger(ctx).Error().Err(err). - Interface("node", n). - Str("key", keys[i]). - Msg("could not unset metadata") + errs := []error{} + for _, k := range keys { + switch k { + case _favoriteKey: + if u, ok := user.ContextGetUser(ctx); ok { + // the favorite flag is specific to the user, so we need to incorporate the userid + if uid := u.GetId(); uid != nil { + fa := fmt.Sprintf("%s%s@%s", favPrefix, uid.GetOpaqueId(), uid.GetIdp()) + if err := xattr.Remove(nodePath, fa); err != nil { + sublog.Error().Err(err). + Interface("user", u). + Str("key", fa). + Msg("could not unset favorite flag") + errs = append(errs, errors.Wrap(err, "could not unset favorite flag")) + } + } else { + sublog.Error(). + Interface("user", u). + Msg("user has no id") + errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "user has no id")) + } } else { - err = nil + sublog.Error(). + Interface("user", u). + Msg("error getting user from ctx") + errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx")) + } + default: + if err = xattr.Remove(nodePath, metadataPrefix+k); err != nil { + // a non-existing attribute will return an error, which we can ignore + // (using string compare because the error type is syscall.Errno and not wrapped/recognizable) + if e, ok := err.(*xattr.Error); !ok || !(e.Err.Error() == "no data available" || + // darwin + e.Err.Error() == "attribute not found") { + sublog.Error().Err(err). + Str("key", k). + Msg("could not unset metadata") + errs = append(errs, errors.Wrap(err, "could not unset metadata")) + } } } } - return + switch len(errs) { + case 0: + return fs.tp.Propagate(ctx, n) + case 1: + // TODO Propagate if anything changed + return errs[0] + default: + // TODO Propagate if anything changed + // TODO how to return multiple errors? + return errors.New("multiple errors occurred, see log for details") + } } diff --git a/pkg/storage/fs/ocis/node.go b/pkg/storage/fs/ocis/node.go index 851597f12e4..259780d6aed 100644 --- a/pkg/storage/fs/ocis/node.go +++ b/pkg/storage/fs/ocis/node.go @@ -34,7 +34,6 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/mime" - "github.com/cs3org/reva/pkg/sdk/common" "github.com/cs3org/reva/pkg/storage/utils/ace" "github.com/cs3org/reva/pkg/user" "github.com/pkg/errors" @@ -45,6 +44,8 @@ import ( const ( _shareTypesKey = "http://owncloud.org/ns/share-types" _userShareType = "0" + + _favoriteKey = "http://owncloud.org/ns/favorite" ) // Node represents a node in the tree and provides methods to get a Parent or Child instance @@ -328,6 +329,97 @@ func (n *Node) PermissionSet(ctx context.Context) *provider.ResourcePermissions return noPermissions } +// calculateEtag returns a hash of fileid + tmtime (or mtime) +func calculateEtag(nodeID string, tmTime time.Time) (string, error) { + h := md5.New() + if _, err := io.WriteString(h, nodeID); err != nil { + return "", err + } + if tb, err := tmTime.UTC().MarshalBinary(); err == nil { + if _, err := h.Write(tb); err != nil { + return "", err + } + } else { + return "", err + } + return fmt.Sprintf(`"%x"`, h.Sum(nil)), nil +} + +// SetMtime sets the mtime and atime of a node +func (n *Node) SetMtime(ctx context.Context, mtime string) error { + sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() + if mt, err := parseMTime(mtime); err == nil { + nodePath := n.lu.toInternalPath(n.ID) + // updating mtime also updates atime + if err := os.Chtimes(nodePath, mt, mt); err != nil { + sublog.Error().Err(err). + Time("mtime", mt). + Msg("could not set mtime") + return errors.Wrap(err, "could not set mtime") + } + } else { + sublog.Error().Err(err). + Str("mtime", mtime). + Msg("could not parse mtime") + return errors.Wrap(err, "could not parse mtime") + } + return nil +} + +// SetEtag sets the temporary etag of a node if it differs from the current etag +func (n *Node) SetEtag(ctx context.Context, val string) (err error) { + sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() + nodePath := n.lu.toInternalPath(n.ID) + var tmTime time.Time + if tmTime, err = n.GetTMTime(); err != nil { + // no tmtime, use mtime + var fi os.FileInfo + if fi, err = os.Lstat(nodePath); err != nil { + return + } + tmTime = fi.ModTime() + } + var etag string + if etag, err = calculateEtag(n.ID, tmTime); err != nil { + return + } + + // sanitize etag + val = fmt.Sprintf("\"%s\"", strings.Trim(val, "\"")) + if etag == val { + sublog.Debug(). + Str("etag", val). + Msg("ignoring request to update identical etag") + return nil + } + // etag is only valid until the calculated etag changes, is part of propagation + return xattr.Set(nodePath, tmpEtagAttr, []byte(val)) +} + +// SetFavorite sets the favorite for the current user +// TODO we should not mess with the user here ... the favorites is now a user specific property for a file +// that cannot be mapped to extended attributes without leaking who has marked a file as a favorite +// it is a specific case of a tag, which is user individual as well +// TODO there are different types of tags +// 1. public that are managed by everyone +// 2. private tags that are only visible to the user +// 3. system tags that are only visible to the system +// 4. group tags that are only visible to a group ... +// urgh ... well this can be solved using different namespaces +// 1. public = p: +// 2. private = u:: for user specific +// 3. system = s: for system +// 4. group = g:: +// 5. app? = a:: for apps? +// obviously this only is secure when the u/s/g/a namespaces are not accessible by users in the filesystem +// public tags can be mapped to extended attributes +func (n *Node) SetFavorite(uid *userpb.UserId, val string) error { + nodePath := n.lu.toInternalPath(n.ID) + // the favorite flag is specific to the user, so we need to incorporate the userid + fa := fmt.Sprintf("%s%s@%s", favPrefix, uid.GetOpaqueId(), uid.GetIdp()) + return xattr.Set(nodePath, fa, []byte(val)) +} + // AsResourceInfo return the node as CS3 ResourceInfo func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissions, mdKeys []string) (ri *provider.ResourceInfo, err error) { sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() @@ -379,31 +471,21 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi sublog.Debug().Err(err).Msg("could not determine owner") } - // etag currently is a hash of fileid + tmtime (or mtime) // TODO make etag of files use fileid and checksum - // TODO implment adding temporery etag in an attribute to restore backups - h := md5.New() - if _, err := io.WriteString(h, n.ID); err != nil { - return nil, err - } + var tmTime time.Time if tmTime, err = n.GetTMTime(); err != nil { // no tmtime, use mtime tmTime = fi.ModTime() } - if tb, err := tmTime.UTC().MarshalBinary(); err == nil { - if _, err := h.Write(tb); err != nil { - return nil, err - } - } else { - return nil, err - } // use temporary etag if it is set if b, err := xattr.Get(nodePath, tmpEtagAttr); err == nil { ri.Etag = fmt.Sprintf(`"%x"`, string(b)) } else { - ri.Etag = fmt.Sprintf(`"%x"`, h.Sum(nil)) + if ri.Etag, err = calculateEtag(n.ID, tmTime); err != nil { + sublog.Debug().Err(err).Msg("could not calculate etag") + } } // mtime uses tmtime if present @@ -414,30 +496,74 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi Nanos: uint32(un % 1000000000), } - // TODO only read the requested metadata attributes - if attrs, err := xattr.List(nodePath); err == nil { - ri.ArbitraryMetadata = &provider.ArbitraryMetadata{ - Metadata: map[string]string{}, - } - for i := range attrs { - if strings.HasPrefix(attrs[i], metadataPrefix) { - k := strings.TrimPrefix(attrs[i], metadataPrefix) - if v, err := xattr.Get(nodePath, attrs[i]); err == nil { - ri.ArbitraryMetadata.Metadata[k] = string(v) - } else { - sublog.Error().Err(err).Str("attr", attrs[i]).Msg("could not get attribute value") + mdKeysMap := make(map[string]struct{}) + for _, k := range mdKeys { + mdKeysMap[k] = struct{}{} + } + + var returnAllKeys bool + if _, ok := mdKeysMap["*"]; len(mdKeys) == 0 || ok { + returnAllKeys = true + } + + metadata := map[string]string{} + + // read favorite flag for the current user + if _, ok := mdKeysMap[_favoriteKey]; returnAllKeys || ok { + favorite := "" + if u, ok := user.ContextGetUser(ctx); ok { + // the favorite flag is specific to the user, so we need to incorporate the userid + if uid := u.GetId(); uid != nil { + fa := fmt.Sprintf("%s%s@%s", favPrefix, uid.GetOpaqueId(), uid.GetIdp()) + if val, err := xattr.Get(nodePath, fa); err == nil { + sublog.Debug(). + Str("favorite", fa). + Msg("found favorite flag") + favorite = string(val) } + } else { + sublog.Error().Err(errtypes.UserRequired("userrequired")).Msg("user has no id") } + } else { + sublog.Error().Err(errtypes.UserRequired("userrequired")).Msg("error getting user from ctx") } - } else { - sublog.Error().Err(err).Msg("could not list attributes") + metadata[_favoriteKey] = favorite } - if common.FindString(mdKeys, _shareTypesKey) != -1 { + // share indicator + if _, ok := mdKeysMap[_shareTypesKey]; returnAllKeys || ok { if n.hasUserShares(ctx) { - ri.ArbitraryMetadata.Metadata[_shareTypesKey] = _userShareType + metadata[_shareTypesKey] = _userShareType + } + } + + // only read the requested metadata attributes + list, err := xattr.List(nodePath) + if err != nil { + sublog.Error().Err(err).Msg("error getting list of extended attributes") + } else { + for _, entry := range list { + // filter out non-custom properties + if !strings.HasPrefix(entry, metadataPrefix) { + continue + } + // only read when key was requested + k := entry[len(metadataPrefix):] + if _, ok := mdKeysMap[k]; returnAllKeys || ok { + if val, err := xattr.Get(nodePath, entry); err == nil { + metadata[k] = string(val) + } else { + sublog.Error().Err(err). + Str("entry", entry). + Msg("error retrieving xattr metadata") + } + } + } } + ri.ArbitraryMetadata = &provider.ArbitraryMetadata{ + Metadata: metadata, + } sublog.Debug(). Interface("ri", ri). diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index 1c3e211511c..3e59a70bf3c 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -63,7 +63,7 @@ const ( grantPrefix string = ocisPrefix + "grant." metadataPrefix string = ocisPrefix + "md." // TODO implement favorites metadata flag - //favPrefix string = ocisPrefix + "fav." // favorite flag, per user + favPrefix string = ocisPrefix + "fav." // favorite flag, per user // a temporary etag for a folder that is removed when the mtime propagation happens tmpEtagAttr string = ocisPrefix + "tmp.etag"