Skip to content

Commit

Permalink
store nodes of a space inside of the space
Browse files Browse the repository at this point in the history 
This is still work in progress. In this state the node_test.go run
successfully.
  • Loading branch information
David Christofas committed Jan 12, 2022
1 parent 3d66054 commit 2469457
Show file tree
Hide file tree
Showing 12 changed files with 132 additions and 93 deletions.
8 changes: 3 additions & 5 deletions pkg/storage/utils/decomposedfs/grants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,9 @@ func (fs *Decomposedfs) AddGrant(ctx context.Context, ref *provider.Reference, g
return errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name))
}

np := fs.lu.InternalPath(node.ID)
e := ace.FromGrant(g)
principal, value := e.Marshal()
if err := xattr.Set(np, xattrs.GrantPrefix+principal, value); err != nil {
if err := xattr.Set(node.InternalPath(), xattrs.GrantPrefix+principal, value); err != nil {
return err
}

Expand Down Expand Up @@ -104,7 +103,7 @@ func (fs *Decomposedfs) ListGrants(ctx context.Context, ref *provider.Reference)
}

log := appctx.GetLogger(ctx)
np := fs.lu.InternalPath(node.ID)
np := node.InternalPath()
var attrs []string
if attrs, err = xattr.List(np); err != nil {
log.Error().Err(err).Msg("error listing attributes")
Expand Down Expand Up @@ -151,8 +150,7 @@ func (fs *Decomposedfs) RemoveGrant(ctx context.Context, ref *provider.Reference
attr = xattrs.GrantUserAcePrefix + g.Grantee.GetUserId().OpaqueId
}

np := fs.lu.InternalPath(node.ID)
if err = xattr.Remove(np, attr); err != nil {
if err = xattr.Remove(node.InternalPath(), attr); err != nil {
return
}

Expand Down
19 changes: 14 additions & 5 deletions pkg/storage/utils/decomposedfs/lookup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ func (lu *Lookup) NodeFromResource(ctx context.Context, ref *provider.Reference)
if err != nil {
return nil, err
}
n.SpaceID = ref.ResourceId.StorageId
}
}
return n, nil
Expand All @@ -77,10 +78,18 @@ func (lu *Lookup) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *n
// The Resource references the root of a space
return lu.NodeFromSpaceID(ctx, id)
}
n, err = node.ReadNode(ctx, lu, id.OpaqueId)
var spaceID string
// If the ids are the same then both are the spaceID
if id.StorageId == id.OpaqueId {
spaceID = node.NoSpaceID
} else {
spaceID = id.StorageId
}
n, err = node.ReadNode(ctx, lu, spaceID, id.OpaqueId)
if err != nil {
return nil, err
}
n.SpaceID = spaceID

return n, n.FindStorageSpaceRoot()
}
Expand All @@ -102,7 +111,7 @@ func (lu *Lookup) NodeFromSpaceID(ctx context.Context, id *provider.ResourceId)
appctx.GetLogger(ctx).Error().Err(err).Str("match", matches[0]).Msg("could not read link, skipping")
}

node, err := node.ReadNode(ctx, lu, filepath.Base(target))
node, err := node.ReadNode(ctx, lu, id.StorageId, filepath.Base(target))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -131,7 +140,7 @@ func (lu *Lookup) Path(ctx context.Context, n *node.Node) (p string, err error)

// RootNode returns the root node of the storage
func (lu *Lookup) RootNode(ctx context.Context) (*node.Node, error) {
n := node.New("root", "", "", 0, "", nil, lu)
n := node.New(node.NoSpaceID, "root", "", "", 0, "", nil, lu)
n.Exists = true
return n, nil
}
Expand Down Expand Up @@ -183,8 +192,8 @@ func (lu *Lookup) InternalRoot() string {
}

// InternalPath returns the internal path for a given ID
func (lu *Lookup) InternalPath(id string) string {
return filepath.Join(lu.Options.Root, "nodes", id)
func (lu *Lookup) InternalPath(spaceID, nodeID string) string {
return filepath.Join(lu.Options.Root, "nodes", spaceID, nodeID)
}

func (lu *Lookup) mustGetUserLayout(ctx context.Context) string {
Expand Down
79 changes: 52 additions & 27 deletions pkg/storage/utils/decomposedfs/node/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,13 @@ const (
QuotaUncalculated = "-1"
QuotaUnknown = "-2"
QuotaUnlimited = "-3"

NoSpaceID = ""
)

// Node represents a node in the tree and provides methods to get a Parent or Child instance
type Node struct {
SpaceID string
ParentID string
ID string
Name string
Expand All @@ -82,17 +85,18 @@ type PathLookup interface {
RootNode(ctx context.Context) (node *Node, err error)

InternalRoot() string
InternalPath(ID string) string
InternalPath(spaceID, nodeID string) string
Path(ctx context.Context, n *Node) (path string, err error)
ShareFolder() string
}

// New returns a new instance of Node
func New(id, parentID, name string, blobsize int64, blobID string, owner *userpb.UserId, lu PathLookup) *Node {
func New(spaceID, id, parentID, name string, blobsize int64, blobID string, owner *userpb.UserId, lu PathLookup) *Node {
if blobID == "" {
blobID = uuid.New().String()
}
return &Node{
SpaceID: spaceID,
ID: id,
ParentID: parentID,
Name: name,
Expand Down Expand Up @@ -170,10 +174,11 @@ func (n *Node) WriteMetadata(owner *userpb.UserId) (err error) {
}

// ReadNode creates a new instance from an id and checks if it exists
func ReadNode(ctx context.Context, lu PathLookup, id string) (n *Node, err error) {
func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string) (n *Node, err error) {
n = &Node{
lu: lu,
ID: id,
SpaceID: spaceID,
lu: lu,
ID: nodeID,
}

nodePath := n.InternalPath()
Expand Down Expand Up @@ -216,8 +221,7 @@ func ReadNode(ctx context.Context, lu PathLookup, id string) (n *Node, err error
return
}

// Check if parent exists. Otherwise this node is part of a deleted subtree
_, err = os.Stat(lu.InternalPath(n.ParentID))
_, err = os.Stat(n.ParentInternalPath())
if err != nil {
if isNotFound(err) {
return nil, errtypes.NotFound(err.Error())
Expand All @@ -240,10 +244,18 @@ func isNotDir(err error) bool {

// Child returns the child node with the given name
func (n *Node) Child(ctx context.Context, name string) (*Node, error) {
spaceID := n.SpaceID
if spaceID == "" && n.ParentID == "root" {
spaceID = n.ID
} else if n.SpaceRoot != nil {
spaceID = n.SpaceRoot.ID
}
link, err := os.Readlink(filepath.Join(n.InternalPath(), filepath.Join("/", name)))
if err != nil {
if os.IsNotExist(err) || isNotDir(err) {

c := &Node{
SpaceID: spaceID,
lu: n.lu,
ParentID: n.ID,
Name: name,
Expand All @@ -255,16 +267,12 @@ func (n *Node) Child(ctx context.Context, name string) (*Node, error) {
return nil, errors.Wrap(err, "Decomposedfs: Wrap: readlink error")
}

var c *Node
if strings.HasPrefix(link, "../") {
c, err = ReadNode(ctx, n.lu, filepath.Base(link))
if err != nil {
return nil, errors.Wrap(err, "could not read child node")
}
c.SpaceRoot = n.SpaceRoot
} else {
return nil, fmt.Errorf("Decomposedfs: expected '../ prefix, got' %+v", link)
c, err := ReadNode(ctx, n.lu, spaceID, filepath.Base(link))
if err != nil {
return nil, errors.Wrap(err, "could not read child node")
}
c.SpaceRoot = n.SpaceRoot
c.SpaceID = spaceID

return c, nil
}
Expand All @@ -274,13 +282,21 @@ func (n *Node) Parent() (p *Node, err error) {
if n.ParentID == "" {
return nil, fmt.Errorf("Decomposedfs: root has no parent")
}
var spaceID string
if n.SpaceRoot != nil && n.SpaceRoot.ID != n.ParentID {
spaceID = n.SpaceID
} else {
spaceID = ""
}
p = &Node{
SpaceID: spaceID,
lu: n.lu,
ID: n.ParentID,
SpaceRoot: n.SpaceRoot,
}

parentPath := n.lu.InternalPath(n.ParentID)
// parentPath := n.lu.InternalPath(spaceID, n.ParentID)
parentPath := p.InternalPath()

// lookup parent id in extended attributes
var attrBytes []byte
Expand Down Expand Up @@ -378,7 +394,16 @@ func (n *Node) PermissionSet(ctx context.Context) provider.ResourcePermissions {

// InternalPath returns the internal path of the Node
func (n *Node) InternalPath() string {
return n.lu.InternalPath(n.ID)
return n.lu.InternalPath(n.SpaceID, n.ID)
}

func (n *Node) ParentInternalPath() string {
spaceID := n.SpaceID
// If the parent is the space root then we don't need the extra spaceID part in the path.
if n.SpaceID == n.ParentID {
spaceID = ""
}
return n.lu.InternalPath(spaceID, n.ParentID)
}

// CalculateEtag returns a hash of fileid + tmtime (or mtime)
Expand Down Expand Up @@ -406,7 +431,7 @@ func calculateEtag(nodeID string, tmTime time.Time) (string, error) {
func (n *Node) SetMtime(ctx context.Context, mtime string) error {
sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger()
if mt, err := parseMTime(mtime); err == nil {
nodePath := n.lu.InternalPath(n.ID)
nodePath := n.InternalPath()
// updating mtime also updates atime
if err := os.Chtimes(nodePath, mt, mt); err != nil {
sublog.Error().Err(err).
Expand All @@ -426,7 +451,7 @@ func (n *Node) SetMtime(ctx context.Context, mtime string) error {
// SetEtag sets the temporary etag of a node if it differs from the current etag
func (n *Node) SetEtag(ctx context.Context, val string) (err error) {
sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger()
nodePath := n.lu.InternalPath(n.ID)
nodePath := n.InternalPath()
var tmTime time.Time
if tmTime, err = n.GetTMTime(); err != nil {
// no tmtime, use mtime
Expand Down Expand Up @@ -471,7 +496,7 @@ func (n *Node) SetEtag(ctx context.Context, val string) (err error) {
// obviously this only is secure when the u/s/g/a namespaces are not accessible by users in the filesystem
// public tags can be mapped to extended attributes
func (n *Node) SetFavorite(uid *userpb.UserId, val string) error {
nodePath := n.lu.InternalPath(n.ID)
nodePath := n.InternalPath()
// the favorite flag is specific to the user, so we need to incorporate the userid
fa := fmt.Sprintf("%s:%s:%s@%s", xattrs.FavPrefix, utils.UserTypeToString(uid.GetType()), uid.GetOpaqueId(), uid.GetIdp())
return xattr.Set(nodePath, fa, []byte(val))
Expand All @@ -482,7 +507,7 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi
sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger()

var fn string
nodePath := n.lu.InternalPath(n.ID)
nodePath := n.InternalPath()

var fi os.FileInfo

Expand Down Expand Up @@ -740,7 +765,7 @@ func readQuotaIntoOpaque(ctx context.Context, nodePath string, ri *provider.Reso

// HasPropagation checks if the propagation attribute exists and is set to "1"
func (n *Node) HasPropagation() (propagation bool) {
if b, err := xattr.Get(n.lu.InternalPath(n.ID), xattrs.PropagationAttr); err == nil {
if b, err := xattr.Get(n.InternalPath(), xattrs.PropagationAttr); err == nil {
return string(b) == "1"
}
return false
Expand All @@ -749,15 +774,15 @@ func (n *Node) HasPropagation() (propagation bool) {
// GetTMTime reads the tmtime from the extended attributes
func (n *Node) GetTMTime() (tmTime time.Time, err error) {
var b []byte
if b, err = xattr.Get(n.lu.InternalPath(n.ID), xattrs.TreeMTimeAttr); err != nil {
if b, err = xattr.Get(n.InternalPath(), xattrs.TreeMTimeAttr); err != nil {
return
}
return time.Parse(time.RFC3339Nano, string(b))
}

// SetTMTime writes the tmtime to the extended attributes
func (n *Node) SetTMTime(t time.Time) (err error) {
return xattr.Set(n.lu.InternalPath(n.ID), xattrs.TreeMTimeAttr, []byte(t.UTC().Format(time.RFC3339Nano)))
return xattr.Set(n.InternalPath(), xattrs.TreeMTimeAttr, []byte(t.UTC().Format(time.RFC3339Nano)))
}

// GetTreeSize reads the treesize from the extended attributes
Expand All @@ -776,12 +801,12 @@ func (n *Node) SetTreeSize(ts uint64) (err error) {

// SetChecksum writes the checksum with the given checksum type to the extended attributes
func (n *Node) SetChecksum(csType string, h hash.Hash) (err error) {
return xattr.Set(n.lu.InternalPath(n.ID), xattrs.ChecksumPrefix+csType, h.Sum(nil))
return xattr.Set(n.InternalPath(), xattrs.ChecksumPrefix+csType, h.Sum(nil))
}

// UnsetTempEtag removes the temporary etag attribute
func (n *Node) UnsetTempEtag() (err error) {
if err = xattr.Remove(n.lu.InternalPath(n.ID), xattrs.TmpEtagAttr); err != nil {
if err = xattr.Remove(n.InternalPath(), xattrs.TmpEtagAttr); err != nil {
if e, ok := err.(*xattr.Error); ok && (e.Err.Error() == "no data available" ||
// darwin
e.Err.Error() == "attribute not found") {
Expand Down
6 changes: 3 additions & 3 deletions pkg/storage/utils/decomposedfs/node/node_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ var _ = Describe("Node", func() {

Describe("New", func() {
It("generates unique blob ids if none are given", func() {
n1 := node.New(id, "", name, 10, "", env.Owner.Id, env.Lookup)
n2 := node.New(id, "", name, 10, "", env.Owner.Id, env.Lookup)
n1 := node.New(env.SpaceRootRes.StorageId, id, "", name, 10, "", env.Owner.Id, env.Lookup)
n2 := node.New(env.SpaceRootRes.StorageId, id, "", name, 10, "", env.Owner.Id, env.Lookup)

Expect(len(n1.BlobID)).To(Equal(36))
Expect(n1.BlobID).ToNot(Equal(n2.BlobID))
Expand All @@ -71,7 +71,7 @@ var _ = Describe("Node", func() {
})
Expect(err).ToNot(HaveOccurred())

n, err := node.ReadNode(env.Ctx, env.Lookup, lookupNode.ID)
n, err := node.ReadNode(env.Ctx, env.Lookup, lookupNode.SpaceID, lookupNode.ID)
Expect(err).ToNot(HaveOccurred())
Expect(n.BlobID).To(Equal("file1-blobid"))
})
Expand Down
12 changes: 6 additions & 6 deletions pkg/storage/utils/decomposedfs/recycle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference
return nil, err
} else if !md.IsDir() {
// this is the case when we want to directly list a file in the trashbin
item, err := fs.createTrashItem(ctx, parentNode, filepath.Dir(relativePath), filepath.Join(trashRoot, key, relativePath))
item, err := fs.createTrashItem(ctx, spaceID, parentNode, filepath.Dir(relativePath), filepath.Join(trashRoot, key, relativePath))
if err != nil {
return items, err
}
Expand All @@ -108,14 +108,14 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference
return nil, err
}
for i := range names {
if item, err := fs.createTrashItem(ctx, parentNode, relativePath, filepath.Join(trashRoot, key, relativePath, names[i])); err == nil {
if item, err := fs.createTrashItem(ctx, spaceID, parentNode, relativePath, filepath.Join(trashRoot, key, relativePath, names[i])); err == nil {
items = append(items, item)
}
}
return items, nil
}

func (fs *Decomposedfs) createTrashItem(ctx context.Context, parentNode, intermediatePath, itemPath string) (*provider.RecycleItem, error) {
func (fs *Decomposedfs) createTrashItem(ctx context.Context, spaceID, parentNode, intermediatePath, itemPath string) (*provider.RecycleItem, error) {
log := appctx.GetLogger(ctx)
trashnode, err := os.Readlink(itemPath)
if err != nil {
Expand All @@ -128,7 +128,7 @@ func (fs *Decomposedfs) createTrashItem(ctx context.Context, parentNode, interme
return nil, errors.New("malformed trash link")
}

nodePath := fs.lu.InternalPath(filepath.Base(trashnode))
nodePath := fs.lu.InternalPath(spaceID, filepath.Base(trashnode))
md, err := os.Stat(nodePath)
if err != nil {
log.Error().Err(err).Str("trashnode", trashnode).Msg("could not stat trash item, skipping")
Expand All @@ -150,7 +150,7 @@ func (fs *Decomposedfs) createTrashItem(ctx context.Context, parentNode, interme
}

// lookup origin path in extended attributes
parentPath := fs.lu.InternalPath(filepath.Base(parentNode))
parentPath := fs.lu.InternalPath(spaceID, filepath.Base(parentNode))
if attrBytes, err := xattr.Get(parentPath, xattrs.TrashOriginAttr); err == nil {
item.Ref = &provider.Reference{Path: filepath.Join(string(attrBytes), intermediatePath, filepath.Base(itemPath))}
} else {
Expand Down Expand Up @@ -197,7 +197,7 @@ func (fs *Decomposedfs) listTrashRoot(ctx context.Context, spaceID string) ([]*p
continue
}

nodePath := fs.lu.InternalPath(filepath.Base(trashnode))
nodePath := fs.lu.InternalPath(spaceID, filepath.Base(trashnode))
md, err := os.Stat(nodePath)
if err != nil {
log.Error().Err(err).Str("trashRoot", trashRoot).Str("name", names[i]).Str("trashnode", trashnode). /*.Interface("parts", parts)*/ Msg("could not stat trash item, skipping")
Expand Down
2 changes: 1 addition & 1 deletion pkg/storage/utils/decomposedfs/recycle_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ var _ = Describe("Recycle", func() {
Expect(len(items)).To(Equal(1))

// use up 2000 byte quota
_, err = env.CreateTestFile("largefile", "largefile-blobid", 2000, projectID.OpaqueId)
_, err = env.CreateTestFile("largefile", "largefile-blobid", projectID.OpaqueId, projectID.StorageId, 2000)
Expect(err).ToNot(HaveOccurred())

err = env.Fs.RestoreRecycleItem(env.Ctx, &provider.Reference{ResourceId: projectID}, items[0].Key, "/", nil)
Expand Down
Loading

0 comments on commit 2469457

Please sign in to comment.