OCM: Add logic for resolving storage references over webdav

changelog/unreleased/webdav-storage.md

@@ -0,0 +1,7 @@
+Enhancement: Add logic for resolving storage references over webdav
+
+This PR adds the functionality to resolve webdav references using the ocs
+webdav service by passing the resource's owner's token. This would need to be
+changed in production.
+
+https://github.com/cs3org/reva/pull/1094

cmd/reva/download.go

@@ -28,10 +28,14 @@ import (
  "github.com/cheggaaa/pb"
  rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
  provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+ typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
  "github.com/cs3org/reva/internal/http/services/datagateway"
+ "github.com/cs3org/reva/pkg/errtypes"
  "github.com/cs3org/reva/pkg/rhttp"
+ tokenpkg "github.com/cs3org/reva/pkg/token"
  "github.com/cs3org/reva/pkg/utils"
  "github.com/pkg/errors"
+ "github.com/studio-b12/gowebdav"
 )
 
 func downloadCommand() *command {
@@ -85,51 +89,97 @@ func downloadCommand() *command {
  // TODO(labkode): upload to data server
  fmt.Printf("Downloading from: %s\n", res.DownloadEndpoint)
 
- dataServerURL := res.DownloadEndpoint
- // TODO(labkode): do a protocol switch
- httpReq, err := rhttp.NewRequest(ctx, "GET", dataServerURL, nil)
+ content, err := checkDownloadWebdavRef(res.DownloadEndpoint, res.Opaque)
  if err != nil {
- return err
- }
-
- httpReq.Header.Set(datagateway.TokenTransportHeader, res.Token)
- httpClient := rhttp.GetHTTPClient(
- rhttp.Context(ctx),
- // TODO make insecure configurable
- rhttp.Insecure(true),
- // TODO make timeout configurable
- rhttp.Timeout(time.Duration(24*int64(time.Hour))),
- )
-
- httpRes, err := httpClient.Do(httpReq)
- if err != nil {
- return err
- }
- defer httpRes.Body.Close()
-
- if httpRes.StatusCode != http.StatusOK {
- return err
+ if _, ok := err.(errtypes.IsNotSupported); !ok {
+ return err
+ }
+
+ dataServerURL := res.DownloadEndpoint
+ // TODO(labkode): do a protocol switch
+ httpReq, err := rhttp.NewRequest(ctx, "GET", dataServerURL, nil)
+ if err != nil {
+ return err
+ }
+
+ httpReq.Header.Set(datagateway.TokenTransportHeader, res.Token)
+ httpClient := rhttp.GetHTTPClient(
+ rhttp.Context(ctx),
+ // TODO make insecure configurable
+ rhttp.Insecure(true),
+ // TODO make timeout configurable
+ rhttp.Timeout(time.Duration(24*int64(time.Hour))),
+ )
+
+ httpRes, err := httpClient.Do(httpReq)
+ if err != nil {
+ return err
+ }
+ defer httpRes.Body.Close()
+
+ if httpRes.StatusCode != http.StatusOK {
+ return err
+ }
+ content = httpRes.Body
  }
 
  absPath, err := utils.ResolvePath(local)
  if err != nil {
  return err
  }
 
+ bar := pb.New(int(info.Size)).SetUnits(pb.U_BYTES)
+ bar.Start()
+ reader := bar.NewProxyReader(content)
+
  fd, err := os.OpenFile(absPath, os.O_CREATE|os.O_WRONLY, 0644)
  if err != nil {
  return err
  }
-
- bar := pb.New(int(info.Size)).SetUnits(pb.U_BYTES)
- bar.Start()
- reader := bar.NewProxyReader(httpRes.Body)
  if _, err := io.Copy(fd, reader); err != nil {
  return err
  }
  bar.Finish()
  return nil
-
  }
  return cmd
 }
+
+func checkDownloadWebdavRef(endpoint string, opaque *typespb.Opaque) (io.Reader, error) {
+ if opaque == nil {
+ return nil, errtypes.NotSupported("opaque object not defined")
+ }
+
+ var token string
+ tokenOpaque, ok := opaque.Map["webdav-token"]
+ if !ok {
+ return nil, errtypes.NotSupported("webdav token not defined")
+ }
+ switch tokenOpaque.Decoder {
+ case "plain":
+ token = string(tokenOpaque.Value)
+ default:
+ return nil, errors.New("opaque entry decoder not recognized: " + tokenOpaque.Decoder)
+ }
+
+ var filePath string
+ fileOpaque, ok := opaque.Map["webdav-file-path"]
+ if !ok {
+ return nil, errtypes.NotSupported("webdav file path not defined")
+ }
+ switch fileOpaque.Decoder {
+ case "plain":
+ filePath = string(fileOpaque.Value)
+ default:
+ return nil, errors.New("opaque entry decoder not recognized: " + fileOpaque.Decoder)
+ }
+
+ c := gowebdav.NewClient(endpoint, "", "")
+ c.SetHeader(tokenpkg.TokenHeader, token)
+
+ reader, err := c.ReadStream(filePath)
+ if err != nil {
+ return nil, err
+ }
+ return reader, nil
+}

cmd/reva/ocm-share-create.go

@@ -19,11 +19,9 @@
 package main
 
 import (
- "encoding/json"
  "fmt"
  "io"
  "os"
- "path"
  "strconv"
  "time"
 
@@ -119,20 +117,17 @@ func ocmShareCreateCommand() *command {
  },
  }
 
- permissionMap := map[string]string{"name": strconv.Itoa(pint)}
- val, err := json.Marshal(permissionMap)
- if err != nil {
- return err
- }
+ fmt.Println("res.Info.Path" + res.Info.Path)
+
  opaqueObj := &types.Opaque{
  Map: map[string]*types.OpaqueEntry{
  "permissions": &types.OpaqueEntry{
- Decoder: "json",
- Value: val,
+ Decoder: "plain",
+ Value: []byte(strconv.Itoa(pint)),
  },
  "name": &types.OpaqueEntry{
  Decoder: "plain",
- Value: []byte(path.Base(res.Info.Path)),
+ Value: []byte(res.Info.Path),
  },
  },
  }
@@ -148,12 +143,12 @@ func ocmShareCreateCommand() *command {
  if err != nil {
  return err
  }
- fmt.Println("create share done")
 
  if shareRes.Status.Code != rpc.Code_CODE_OK {
  return formatError(shareRes.Status)
  }
 
+ fmt.Println("create share done")
  t := table.NewWriter()
  t.SetOutputMirror(os.Stdout)
  t.AppendHeader(table.Row{"#", "Owner.Idp", "Owner.OpaqueId", "ResourceId", "Permissions", "Type", "Grantee.Idp", "Grantee.OpaqueId", "Created", "Updated"})

cmd/reva/upload.go

@@ -35,10 +35,11 @@ import (
  rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
  provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
  typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
-
+ "github.com/cs3org/reva/pkg/errtypes"
  tokenpkg "github.com/cs3org/reva/pkg/token"
  "github.com/eventials/go-tus"
  "github.com/eventials/go-tus/memorystore"
+ "github.com/studio-b12/gowebdav"
 
  // TODO(labkode): this should not come from this package.
  "github.com/cs3org/reva/internal/grpc/services/storageprovider"
@@ -51,11 +52,11 @@ func uploadCommand() *command {
  cmd := newCommand("upload")
  cmd.Description = func() string { return "upload a local file to the remote server" }
  cmd.Usage = func() string { return "Usage: upload [-flags] <file_name> <remote_target>" }
- disabletusFlag := cmd.Bool("disable-tus", false, "whether to disable tus protocol")
+ disableTusFlag := cmd.Bool("disable-tus", false, "whether to disable tus protocol")
  xsFlag := cmd.String("xs", "negotiate", "compute checksum")
 
  cmd.ResetFlags = func() {
- *disabletusFlag, *xsFlag = false, "negotiate"
+ *disableTusFlag, *xsFlag = false, "negotiate"
  }
 
  cmd.Action = func(w ...io.Writer) error {
@@ -121,6 +122,14 @@ func uploadCommand() *command {
  fmt.Printf("Data server: %s\n", res.UploadEndpoint)
  fmt.Printf("Allowed checksums: %+v\n", res.AvailableChecksums)
 
+ if err = checkUploadWebdavRef(res.UploadEndpoint, res.Opaque, md, fd); err != nil {
+ if _, ok := err.(errtypes.IsNotSupported); !ok {
+ return err
+ }
+ } else {
+ return nil
+ }
+
  xsType, err := guessXS(*xsFlag, res.AvailableChecksums)
  if err != nil {
  return err
@@ -144,7 +153,7 @@ func uploadCommand() *command {
  bar.Start()
  reader := bar.NewProxyReader(fd)
 
- if *disabletusFlag {
+ if *disableTusFlag {
  httpReq, err := rhttp.NewRequest(ctx, "PUT", dataServerURL, reader)
  if err != nil {
  bar.Finish()
@@ -253,6 +262,54 @@ func uploadCommand() *command {
  return cmd
 }
 
+func checkUploadWebdavRef(endpoint string, opaque *typespb.Opaque, md os.FileInfo, fd *os.File) error {
+ if opaque == nil {
+ return errtypes.NotSupported("opaque object not defined")
+ }
+
+ var token string
+ tokenOpaque, ok := opaque.Map["webdav-token"]
+ if !ok {
+ return errtypes.NotSupported("webdav token not defined")
+ }
+ switch tokenOpaque.Decoder {
+ case "plain":
+ token = string(tokenOpaque.Value)
+ default:
+ return errors.New("opaque entry decoder not recognized: " + tokenOpaque.Decoder)
+ }
+
+ var filePath string
+ fileOpaque, ok := opaque.Map["webdav-file-path"]
+ if !ok {
+ return errtypes.NotSupported("webdav file path not defined")
+ }
+ switch fileOpaque.Decoder {
+ case "plain":
+ filePath = string(fileOpaque.Value)
+ default:
+ return errors.New("opaque entry decoder not recognized: " + fileOpaque.Decoder)
+ }
+
+ bar := pb.New(int(md.Size())).SetUnits(pb.U_BYTES)
+ bar.Start()
+ reader := bar.NewProxyReader(fd)
+
+ c := gowebdav.NewClient(endpoint, "", "")
+ c.SetHeader(tokenpkg.TokenHeader, token)
+ c.SetHeader("Upload-Length", strconv.FormatInt(md.Size(), 10))
+
+ err := c.WriteStream(filePath, reader, 0700)
+ if err != nil {
+ bar.Finish()
+ return err
+ }
+
+ bar.Finish()
+ fmt.Println("File uploaded")
+ return nil
+}
+
 func computeXS(t provider.ResourceChecksumType, r io.Reader) (string, error) {
  switch t {
  case provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_ADLER32:

examples/ocm-partners/providers.demo.json

@@ -27,7 +27,7 @@
  "description": "CERNBox Webdav API"
  },
  "name": "CERNBox - Webdav API",
- "path": "https://sciencemesh.cernbox.cern.ch/iop/webdav/",
+ "path": "https://sciencemesh.cernbox.cern.ch/iop/remote.php/webdav/",
  "is_monitored": true
  },
  "api_version": "0.0.1",
@@ -63,7 +63,7 @@
  "description": "CESNET Webdav API"
  },
  "name": "CESNET - Webdav API",
- "path": "https://sciencemesh.cesnet.cz/iop/webdav/",
+ "path": "https://sciencemesh.cesnet.cz/iop/remote.php/webdav/",
  "is_monitored": true
  },
  "api_version": "0.0.1",
@@ -99,7 +99,7 @@
  "description": "WWU Webdav API"
  },
  "name": "WWU - Webdav API",
- "path": "https://sciencemesh-test.uni-muenster.de/api/webdav/",
+ "path": "https://sciencemesh-test.uni-muenster.de/api/remote.php/webdav/",
  "is_monitored": true
  },
  "api_version": "0.0.1",
@@ -135,7 +135,7 @@
  "description": "Cubbit Webdav API"
  },
  "name": "Cubbit - Webdav API",
- "path": "https://sciencemesh.cubbit.io/webdav/",
+ "path": "https://sciencemesh.cubbit.io/remote.php/webdav/",
  "is_monitored": true
  },
  "api_version": "0.0.1",
@@ -171,7 +171,7 @@
  "description": "Ailleron Webdav API"
  },
  "name": "Ailleron - Webdav API",
- "path": "https://sciencemesh.softwaremind.com/iop/webdav/",
+ "path": "https://sciencemesh.softwaremind.com/iop/remote.php/webdav/",
  "is_monitored": true
  },
  "api_version": "0.0.1",
@@ -207,7 +207,7 @@
  "description": "Surfsara Webdav API"
  },
  "name": "Surfsara - Webdav API",
- "path": "https://app.cs3mesh-iop.k8s.surfsara.nl/iop/webdav/",
+ "path": "https://app.cs3mesh-iop.k8s.surfsara.nl/iop/remote.php/webdav/",
  "is_monitored": true
  },
  "api_version": "0.0.1",

examples/ocmd/ocmd-server-1.toml

@@ -131,4 +131,6 @@ providers = "providers.demo.json"
 [http.services.ocs]
 prefix = "ocs"
 
+[http.services.ocdav]
+
 [http.middlewares.cors]

examples/ocmd/ocmd-server-2.toml

@@ -114,4 +114,6 @@ providers = "providers.demo.json"
 [http.services.ocs]
 prefix = "ocs"
 
+[http.services.ocdav]
+
 [http.middlewares.cors]