parent 692f38f

author Jörn Friedrich Dreyer <jfd@butonic.de> 1623945204 +0200 committer Jörn Friedrich Dreyer <jfd@butonic.de> 1623958936 +0000 switch references (cs3org#1721) Co-authored-by: A.Unger <zyxancf@gmail.com> Co-authored-by: David Christofas <dchristofas@owncloud.com> rebase Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> fix nil Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> update changelog Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> add storage provider list spaces interface Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> add stubs Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> initial ocis implementation for list storage spaces Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> minor fixes Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> add thoughts on proper spaces persistence layout Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> Initial spaces implementation. Signed-off-by: Klaas Freitag <kfreitag@owncloud.com> more spaces work Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> filter spaces based on permissions, return name Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> resolve linter issues implement storage space support into the storageprovider Co-authored-by: Jörn Friedrich Dreyer <jfd@butonic.de> fix dav spaces href Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> GET preparations Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> WIP: spaces datatx Add spaces.go distinguish spaces/simple datatx Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> make GET work for spaces Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> MKCol implementation for spaces WIP refactor CreateDir Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> make MKCOL work for spaces implement delete for the spaces api fix: unwrap the requested reference before using it further implement MOVE for spaces simplify check if request body is empty setup constants for webdav verbs fix listcontainers for spaces references implement PROPPATCH for spaces implement COPY for spaces add cases for lock, unlock and report for spaces implement PUT for spaces implement POST for spaces implement HEAD for spaces clean up and deduplicate webdav HEAD code clean up and deduplicate webdav DELETE code clean up and deduplicate webdav GET code clean up and deduplicate webdav PROPFIND code clean up and deduplicate webdav MKCOL code clean up and deduplicate webdav MOVE code clean up and deduplicate webdav PROPPATCH code clean up and deduplicate webdav COPY code clean up and deduplicate webdav TUS POST code clean up and deduplicate webdav PUT code try fixing tests Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> fix path checks before calling WalkPath update the owncloudsql storage driver fix spaceid handling Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> fix linter issues fix CreateDir in the owncloud storage driver fix non space path handling Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> allow creating empty files Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> no longer hardcode storageid in the driver Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> decomposedfs: create storage spaces Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> Id -> ID Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> move change-references changelog to unreleased add spaces changelog Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> add grant test for spaces goimports update to embedded reference Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> relative references should only return the basename Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> do not show shared resources as spaces for owners Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> make sure the usage of user provides paths is secure list spaces from all providers Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
butonic · Jun 17, 2021 · b222d4d · b222d4d
1 parent 9740eed
commit b222d4d
Show file tree

Hide file tree

Showing 53 changed files with 2,322 additions and 766 deletions.
diff --git a/changelog/unreleased/introduce-spaces.md b/changelog/unreleased/introduce-spaces.md
@@ -0,0 +1,5 @@
+Enhancement: Introduce spaces
+
+Clients can now use a new webdav endpoint `/dav/spaces/<storageid>/<nodeid>` to directly access storage spaces.
+
+https://github.com/cs3org/reva/pull/1776
diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go
@@ -115,37 +115,118 @@ func (s *svc) CreateStorageSpace(ctx context.Context, req *provider.CreateStorag
 
 func (s *svc) ListStorageSpaces(ctx context.Context, req *provider.ListStorageSpacesRequest) (*provider.ListStorageSpacesResponse, error) {
 	log := appctx.GetLogger(ctx)
-	// TODO: needs to be fixed
 	var id *provider.StorageSpaceId
 	for _, f := range req.Filters {
 		if f.Type == provider.ListStorageSpacesRequest_Filter_TYPE_ID {
 			id = f.GetId()
 		}
 	}
-	parts := strings.SplitN(id.OpaqueId, "!", 2)
-	if len(parts) != 2 {
-		return &provider.ListStorageSpacesResponse{
-			Status: status.NewInvalidArg(ctx, "space id must be separated by !"),
-		}, nil
+
+	var providers []*registry.ProviderInfo
+	var err error
+	c, err := pool.GetStorageRegistryClient(s.c.StorageRegistryEndpoint)
+	if err != nil {
+		return nil, errors.Wrap(err, "gateway: error getting storage registry client")
 	}
-	c, err := s.find(ctx, &provider.Reference{ResourceId: &provider.ResourceId{
-		StorageId: parts[0], // FIXME REFERENCE the StorageSpaceId is a storageid + a opaqueid
-		OpaqueId:  parts[1],
-	}})
+
+	if id != nil {
+		// query that specific story provider
+		parts := strings.SplitN(id.OpaqueId, "!", 2)
+		if len(parts) != 2 {
+			return &provider.ListStorageSpacesResponse{
+				Status: status.NewInvalidArg(ctx, "space id must be separated by !"),
+			}, nil
+		}
+		res, err := c.GetStorageProviders(ctx, &registry.GetStorageProvidersRequest{
+			Ref: &provider.Reference{ResourceId: &provider.ResourceId{
+				StorageId: parts[0], // FIXME REFERENCE the StorageSpaceId is a storageid + an opaqueid
+				OpaqueId:  parts[1],
+			}},
+		})
+		if err != nil {
+			return &provider.ListStorageSpacesResponse{
+				Status: status.NewStatusFromErrType(ctx, "ListStorageSpaces filters: req "+req.String(), err),
+			}, nil
+		}
+		if res.Status.Code != rpc.Code_CODE_OK {
+			return &provider.ListStorageSpacesResponse{
+				Status: res.Status,
+			}, nil
+		}
+		providers = res.Providers
+	} else {
+		// get list of all storage providers
+		res, err := c.ListStorageProviders(ctx, &registry.ListStorageProvidersRequest{})
+
+		if err != nil {
+			return &provider.ListStorageSpacesResponse{
+				Status: status.NewStatusFromErrType(ctx, "error listing providers", err),
+			}, nil
+		}
+		if res.Status.Code != rpc.Code_CODE_OK {
+			return &provider.ListStorageSpacesResponse{
+				Status: res.Status,
+			}, nil
+		}
+
+		providers = []*registry.ProviderInfo{}
+		// FIXME filter only providers that have an id set ... currently none have?
+		// bug? only ProviderPath is set
+		for i := range res.Providers {
+			// use only providers whose path does not start with a /?
+			if strings.HasPrefix(res.Providers[i].ProviderPath, "/") {
+				continue
+			}
+			providers = append(providers, res.Providers[i])
+		}
+	}
+
+	spacesFromProviders := make([][]*provider.StorageSpace, len(providers))
+	errors := make([]error, len(providers))
+	var wg sync.WaitGroup
+
+	for i, p := range providers {
+		wg.Add(1)
+		go s.listStorageSpacesOnProvider(ctx, req, &spacesFromProviders[i], p, &errors[i], &wg)
+	}
+	wg.Wait()
+
+	uniqueSpaces := map[string]*provider.StorageSpace{}
+	for i := range providers {
+		if errors[i] != nil {
+			log.Debug().Err(errors[i]).Msg("skipping provider")
+			continue
+		}
+		for j := range spacesFromProviders[i] {
+			uniqueSpaces[spacesFromProviders[i][j].Id.OpaqueId] = spacesFromProviders[i][j]
+		}
+	}
+	spaces := []*provider.StorageSpace{}
+	for spaceID := range uniqueSpaces {
+		spaces = append(spaces, uniqueSpaces[spaceID])
+	}
+
+	return &provider.ListStorageSpacesResponse{
+		Status:        status.NewOK(ctx),
+		StorageSpaces: spaces,
+	}, nil
+}
+
+func (s *svc) listStorageSpacesOnProvider(ctx context.Context, req *provider.ListStorageSpacesRequest, res *[]*provider.StorageSpace, p *registry.ProviderInfo, e *error, wg *sync.WaitGroup) {
+	defer wg.Done()
+	c, err := s.getStorageProviderClient(ctx, p)
 	if err != nil {
-		return &provider.ListStorageSpacesResponse{
-			Status: status.NewStatusFromErrType(ctx, "error finding path", err),
-		}, nil
+		*e = errors.Wrap(err, "error connecting to storage provider="+p.Address)
+		return
 	}
 
-	res, err := c.ListStorageSpaces(ctx, req)
+	r, err := c.ListStorageSpaces(ctx, req)
 	if err != nil {
-		log.Err(err).Msg("gateway: error listing storage space on storage provider")
-		return &provider.ListStorageSpacesResponse{
-			Status: status.NewInternal(ctx, err, "error calling ListStorageSpaces"),
-		}, nil
+		*e = errors.Wrap(err, "gateway: error calling ListStorageSpaces")
+		return
 	}
-	return res, nil
+
+	*res = r.StorageSpaces
 }
 
 func (s *svc) UpdateStorageSpace(ctx context.Context, req *provider.UpdateStorageSpaceRequest) (*provider.UpdateStorageSpaceResponse, error) {
@@ -210,6 +291,11 @@ func (s *svc) getHome(_ context.Context) string {
 
 func (s *svc) InitiateFileDownload(ctx context.Context, req *provider.InitiateFileDownloadRequest) (*gateway.InitiateFileDownloadResponse, error) {
 	log := appctx.GetLogger(ctx)
+
+	if utils.IsRelativeReference(req.Ref) {
+		return s.initiateFileDownload(ctx, req)
+	}
+
 	p, st := s.getPath(ctx, req.Ref)
 	if st.Code != rpc.Code_CODE_OK {
 		return &gateway.InitiateFileDownloadResponse{
@@ -420,6 +506,9 @@ func (s *svc) initiateFileDownload(ctx context.Context, req *provider.InitiateFi
 
 func (s *svc) InitiateFileUpload(ctx context.Context, req *provider.InitiateFileUploadRequest) (*gateway.InitiateFileUploadResponse, error) {
 	log := appctx.GetLogger(ctx)
+	if utils.IsRelativeReference(req.Ref) {
+		return s.initiateFileUpload(ctx, req)
+	}
 	p, st := s.getPath(ctx, req.Ref)
 	if st.Code != rpc.Code_CODE_OK {
 		return &gateway.InitiateFileUploadResponse{
@@ -642,6 +731,11 @@ func (s *svc) GetPath(ctx context.Context, req *provider.GetPathRequest) (*provi
 
 func (s *svc) CreateContainer(ctx context.Context, req *provider.CreateContainerRequest) (*provider.CreateContainerResponse, error) {
 	log := appctx.GetLogger(ctx)
+
+	if utils.IsRelativeReference(req.Ref) {
+		return s.createContainer(ctx, req)
+	}
+
 	p, st := s.getPath(ctx, req.Ref)
 	if st.Code != rpc.Code_CODE_OK {
 		return &provider.CreateContainerResponse{
@@ -1086,14 +1180,19 @@ func (s *svc) stat(ctx context.Context, req *provider.StatRequest) (*provider.St
 	}
 
 	resPath := req.Ref.GetPath()
-	if len(providers) == 1 && (resPath == "" || strings.HasPrefix(resPath, providers[0].ProviderPath)) {
+	if len(providers) == 1 && (utils.IsRelativeReference(req.Ref) || resPath == "" || strings.HasPrefix(resPath, providers[0].ProviderPath)) {
 		c, err := s.getStorageProviderClient(ctx, providers[0])
 		if err != nil {
 			return &provider.StatResponse{
 				Status: status.NewInternal(ctx, err, "error connecting to storage provider="+providers[0].Address),
 			}, nil
 		}
-		return c.Stat(ctx, req)
+		rsp, err := c.Stat(ctx, req)
+		if err != nil || rsp.Status.Code != rpc.Code_CODE_OK {
+			return rsp, err
+		}
+
+		return rsp, nil
 	}
 
 	infoFromProviders := make([]*provider.ResourceInfo, len(providers))
@@ -1141,12 +1240,16 @@ func (s *svc) statOnProvider(ctx context.Context, req *provider.StatRequest, res
 		return
 	}
 
-	resPath := path.Clean(req.Ref.GetPath())
-	newPath := req.Ref.GetPath()
-	if resPath != "" && !strings.HasPrefix(resPath, p.ProviderPath) {
-		newPath = p.ProviderPath
+	if utils.IsAbsoluteReference(req.Ref) {
+		resPath := path.Clean(req.Ref.GetPath())
+		newPath := req.Ref.GetPath()
+		if resPath != "" && !strings.HasPrefix(resPath, p.ProviderPath) {
+			newPath = p.ProviderPath
+		}
+		req.Ref = &provider.Reference{Path: newPath}
 	}
-	r, err := c.Stat(ctx, &provider.StatRequest{Ref: &provider.Reference{Path: newPath}})
+
+	r, err := c.Stat(ctx, req)
 	if err != nil {
 		*e = errors.Wrap(err, "gateway: error calling ListContainer")
 		return
@@ -1158,6 +1261,11 @@ func (s *svc) statOnProvider(ctx context.Context, req *provider.StatRequest, res
 }
 
 func (s *svc) Stat(ctx context.Context, req *provider.StatRequest) (*provider.StatResponse, error) {
+
+	if utils.IsRelativeReference(req.Ref) {
+		return s.stat(ctx, req)
+	}
+
 	p, st := s.getPath(ctx, req.Ref, req.ArbitraryMetadataKeys...)
 	if st.Code != rpc.Code_CODE_OK {
 		return &provider.StatResponse{
@@ -1457,14 +1565,15 @@ func (s *svc) listContainer(ctx context.Context, req *provider.ListContainerRequ
 
 	infos := []*provider.ResourceInfo{}
 	indirects := make(map[string][]*provider.ResourceInfo)
+	trimPrefix := utils.IsAbsoluteReference(req.Ref)
 	for i := range providers {
 		if errors[i] != nil {
 			return &provider.ListContainerResponse{
 				Status: status.NewStatusFromErrType(ctx, "listContainer ref: "+req.Ref.String(), errors[i]),
 			}, nil
 		}
 		for _, inf := range infoFromProviders[i] {
-			if parent := path.Dir(inf.Path); resPath != "" && resPath != parent {
+			if parent := path.Dir(inf.Path); trimPrefix && resPath != "." && resPath != parent {
 				parts := strings.Split(strings.TrimPrefix(inf.Path, resPath), "/")
 				p := path.Join(resPath, parts[1])
 				indirects[p] = append(indirects[p], inf)
@@ -1502,12 +1611,16 @@ func (s *svc) listContainerOnProvider(ctx context.Context, req *provider.ListCon
 		return
 	}
 
-	resPath := path.Clean(req.Ref.GetPath())
-	newPath := req.Ref.GetPath()
-	if resPath != "" && !strings.HasPrefix(resPath, p.ProviderPath) {
-		newPath = p.ProviderPath
+	if utils.IsAbsoluteReference(req.Ref) {
+		resPath := path.Clean(req.Ref.GetPath())
+		newPath := req.Ref.GetPath()
+		if resPath != "" && !strings.HasPrefix(resPath, p.ProviderPath) {
+			newPath = p.ProviderPath
+		}
+		req.Ref = &provider.Reference{Path: newPath}
 	}
-	r, err := c.ListContainer(ctx, &provider.ListContainerRequest{Ref: &provider.Reference{Path: newPath}})
+
+	r, err := c.ListContainer(ctx, req)
 	if err != nil {
 		*e = errors.Wrap(err, "gateway: error calling ListContainer")
 		return
@@ -1517,6 +1630,11 @@ func (s *svc) listContainerOnProvider(ctx context.Context, req *provider.ListCon
 
 func (s *svc) ListContainer(ctx context.Context, req *provider.ListContainerRequest) (*provider.ListContainerResponse, error) {
 	log := appctx.GetLogger(ctx)
+
+	if utils.IsRelativeReference(req.Ref) {
+		return s.listContainer(ctx, req)
+	}
+
 	p, st := s.getPath(ctx, req.Ref, req.ArbitraryMetadataKeys...)
 	if st.Code != rpc.Code_CODE_OK {
 		return &provider.ListContainerResponse{
@@ -1705,7 +1823,7 @@ func (s *svc) getPath(ctx context.Context, ref *provider.Reference, keys ...stri
 		return res.Info.Path, res.Status
 	}
 
-	if ref.Path != "" {
+	if ref.ResourceId == nil && strings.HasPrefix(ref.Path, "/") {
 		return ref.Path, &rpc.Status{Code: rpc.Code_CODE_OK}
 	}
 	return "", &rpc.Status{Code: rpc.Code_CODE_INTERNAL}

diff --git a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go
@@ -577,7 +577,7 @@ func (s *service) unwrap(ctx context.Context, ref *provider.Reference) (token st
 		return "", "", errtypes.BadRequest("need absolute path ref: got " + ref.String())
 	}
 
-	if ref.GetPath() == "" {
+	if !strings.HasPrefix(ref.GetPath(), "/") {
 		// abort, no valid id nor path
 		return "", "", errtypes.BadRequest("invalid ref: " + ref.String())
 	}