From a3dca1a4dd916083841c71bb7bd1711147f05480 Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Thu, 11 Jun 2020 17:31:35 +0200 Subject: [PATCH 1/3] Refactor EOS fs --- .../config/packages/storage/fs/eos/_index.md | 138 ++ .../packages/storage/fs/eoshome/_index.md | 146 ++ pkg/eosclient/eosclient.go | 2 +- pkg/storage/fs/eos/eos.go | 1400 +---------------- pkg/storage/fs/eoshome/eoshome.go | 129 ++ pkg/storage/{ => utils}/acl/acl.go | 0 pkg/storage/utils/eosfs/eosfs.go | 1341 ++++++++++++++++ pkg/storage/{fs/eos => utils/eosfs}/upload.go | 2 +- pkg/storage/utils/grants/grants.go | 137 ++ pkg/storage/utils/localfs/localfs.go | 96 +- 10 files changed, 1937 insertions(+), 1454 deletions(-) create mode 100644 docs/content/en/docs/config/packages/storage/fs/eos/_index.md create mode 100644 docs/content/en/docs/config/packages/storage/fs/eoshome/_index.md create mode 100644 pkg/storage/fs/eoshome/eoshome.go rename pkg/storage/{ => utils}/acl/acl.go (100%) create mode 100644 pkg/storage/utils/eosfs/eosfs.go rename pkg/storage/{fs/eos => utils/eosfs}/upload.go (99%) create mode 100644 pkg/storage/utils/grants/grants.go diff --git a/docs/content/en/docs/config/packages/storage/fs/eos/_index.md b/docs/content/en/docs/config/packages/storage/fs/eos/_index.md new file mode 100644 index 0000000000..d0bd0cd81c --- /dev/null +++ b/docs/content/en/docs/config/packages/storage/fs/eos/_index.md @@ -0,0 +1,138 @@ +--- +title: "eos" +linkTitle: "eos" +weight: 10 +description: > + Configuration for the eos service +--- + +# _struct: config_ + +{{% dir name="namespace" type="string" default="/" %}} +Namespace for metadata operations [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L38) +{{< highlight toml >}} +[storage.fs.eos] +namespace = "/" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="shadow_namespace" type="string" default="/.shadow" %}} +ShadowNamespace for storing shadow data [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L41) +{{< highlight toml >}} +[storage.fs.eos] +shadow_namespace = "/.shadow" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="uploads_namespace" type="string" default="/.uploads" %}} +UploadsNamespace for storing upload data [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L44) +{{< highlight toml >}} +[storage.fs.eos] +uploads_namespace = "/.uploads" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="share_folder" type="string" default="/MyShares" %}} +ShareFolder defines the name of the folder in the shadowed namespace. Ex: /eos/user/.shadow/h/hugo/MyShares [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L48) +{{< highlight toml >}} +[storage.fs.eos] +share_folder = "/MyShares" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="eos_binary" type="string" default="/usr/bin/eos" %}} +Location of the eos binary. Default is /usr/bin/eos. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L52) +{{< highlight toml >}} +[storage.fs.eos] +eos_binary = "/usr/bin/eos" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="xrdcopy_binary" type="string" default="/usr/bin/xrdcopy" %}} +Location of the xrdcopy binary. Default is /usr/bin/xrdcopy. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L56) +{{< highlight toml >}} +[storage.fs.eos] +xrdcopy_binary = "/usr/bin/xrdcopy" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="master_url" type="string" default="root://eos-example.org" %}} +URL of the Master EOS MGM. Default is root:eos-example.org [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L60) +{{< highlight toml >}} +[storage.fs.eos] +master_url = "root://eos-example.org" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="slave_url" type="string" default="root://eos-example.org" %}} +URL of the Slave EOS MGM. Default is root:eos-example.org [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L64) +{{< highlight toml >}} +[storage.fs.eos] +slave_url = "root://eos-example.org" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="cache_directory" type="string" default="/var/tmp/" %}} +Location on the local fs where to store reads. Defaults to os.TempDir() [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L68) +{{< highlight toml >}} +[storage.fs.eos] +cache_directory = "/var/tmp/" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="sec_protocol" type="string" default="-" %}} +SecProtocol specifies the xrootd security protocol to use between the server and EOS. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L73) +{{< highlight toml >}} +[storage.fs.eos] +sec_protocol = "-" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="keytab" type="string" default="-" %}} +Keytab specifies the location of the keytab to use to authenticate to EOS. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L76) +{{< highlight toml >}} +[storage.fs.eos] +keytab = "-" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="single_username" type="string" default="-" %}} +SingleUsername is the username to use when SingleUserMode is enabled [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L79) +{{< highlight toml >}} +[storage.fs.eos] +single_username = "-" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="enable_logging" type="bool" default=false %}} +Enables logging of the commands executed Defaults to false [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L83) +{{< highlight toml >}} +[storage.fs.eos] +enable_logging = false +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="show_hidden_sys_files" type="bool" default=false %}} +ShowHiddenSysFiles shows internal EOS files like .sys.v# and .sys.a# files. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L87) +{{< highlight toml >}} +[storage.fs.eos] +show_hidden_sys_files = false +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="force_single_user_mode" type="bool" default=false %}} +ForceSingleUserMode will force connections to EOS to use SingleUsername [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L90) +{{< highlight toml >}} +[storage.fs.eos] +force_single_user_mode = false +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="use_keytab" type="bool" default=false %}} +UseKeyTabAuth changes will authenticate requests by using an EOS keytab. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eos/eos.go#L94) +{{< highlight toml >}} +[storage.fs.eos] +use_keytab = false +{{< /highlight >}} +{{% /dir %}} + diff --git a/docs/content/en/docs/config/packages/storage/fs/eoshome/_index.md b/docs/content/en/docs/config/packages/storage/fs/eoshome/_index.md new file mode 100644 index 0000000000..0ad1fe63fd --- /dev/null +++ b/docs/content/en/docs/config/packages/storage/fs/eoshome/_index.md @@ -0,0 +1,146 @@ +--- +title: "eoshome" +linkTitle: "eoshome" +weight: 10 +description: > + Configuration for the eoshome service +--- + +# _struct: config_ + +{{% dir name="namespace" type="string" default="/" %}} +Namespace for metadata operations [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L38) +{{< highlight toml >}} +[storage.fs.eoshome] +namespace = "/" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="shadow_namespace" type="string" default="/.shadow" %}} +ShadowNamespace for storing shadow data [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L41) +{{< highlight toml >}} +[storage.fs.eoshome] +shadow_namespace = "/.shadow" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="uploads_namespace" type="string" default="/.uploads" %}} +UploadsNamespace for storing upload data [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L44) +{{< highlight toml >}} +[storage.fs.eoshome] +uploads_namespace = "/.uploads" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="share_folder" type="string" default="/MyShares" %}} +ShareFolder defines the name of the folder in the shadowed namespace. Ex: /eos/user/.shadow/h/hugo/MyShares [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L48) +{{< highlight toml >}} +[storage.fs.eoshome] +share_folder = "/MyShares" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="eos_binary" type="string" default="/usr/bin/eos" %}} +Location of the eos binary. Default is /usr/bin/eos. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L52) +{{< highlight toml >}} +[storage.fs.eoshome] +eos_binary = "/usr/bin/eos" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="xrdcopy_binary" type="string" default="/usr/bin/xrdcopy" %}} +Location of the xrdcopy binary. Default is /usr/bin/xrdcopy. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L56) +{{< highlight toml >}} +[storage.fs.eoshome] +xrdcopy_binary = "/usr/bin/xrdcopy" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="master_url" type="string" default="root://eos-example.org" %}} +URL of the Master EOS MGM. Default is root:eos-example.org [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L60) +{{< highlight toml >}} +[storage.fs.eoshome] +master_url = "root://eos-example.org" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="slave_url" type="string" default="root://eos-example.org" %}} +URL of the Slave EOS MGM. Default is root:eos-example.org [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L64) +{{< highlight toml >}} +[storage.fs.eoshome] +slave_url = "root://eos-example.org" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="cache_directory" type="string" default="/var/tmp/" %}} +Location on the local fs where to store reads. Defaults to os.TempDir() [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L68) +{{< highlight toml >}} +[storage.fs.eoshome] +cache_directory = "/var/tmp/" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="sec_protocol" type="string" default="-" %}} +SecProtocol specifies the xrootd security protocol to use between the server and EOS. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L73) +{{< highlight toml >}} +[storage.fs.eoshome] +sec_protocol = "-" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="keytab" type="string" default="-" %}} +Keytab specifies the location of the keytab to use to authenticate to EOS. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L76) +{{< highlight toml >}} +[storage.fs.eoshome] +keytab = "-" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="single_username" type="string" default="-" %}} +SingleUsername is the username to use when SingleUserMode is enabled [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L79) +{{< highlight toml >}} +[storage.fs.eoshome] +single_username = "-" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="user_layout" type="string" default="{{.Username}}" %}} +UserLayout wraps the internal path with user information. Example: if conf.Namespace is /eos/user and received path is /docs and the UserLayout is {{.Username}} the internal path will be: /eos/user//docs [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L85) +{{< highlight toml >}} +[storage.fs.eoshome] +user_layout = "{{.Username}}" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="enable_logging" type="bool" default=false %}} +Enables logging of the commands executed Defaults to false [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L89) +{{< highlight toml >}} +[storage.fs.eoshome] +enable_logging = false +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="show_hidden_sys_files" type="bool" default=false %}} +ShowHiddenSysFiles shows internal EOS files like .sys.v# and .sys.a# files. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L93) +{{< highlight toml >}} +[storage.fs.eoshome] +show_hidden_sys_files = false +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="force_single_user_mode" type="bool" default=false %}} +ForceSingleUserMode will force connections to EOS to use SingleUsername [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L96) +{{< highlight toml >}} +[storage.fs.eoshome] +force_single_user_mode = false +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="use_keytab" type="bool" default=false %}} +UseKeyTabAuth changes will authenticate requests by using an EOS keytab. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/eoshome/eoshome.go#L100) +{{< highlight toml >}} +[storage.fs.eoshome] +use_keytab = false +{{< /highlight >}} +{{% /dir %}} + diff --git a/pkg/eosclient/eosclient.go b/pkg/eosclient/eosclient.go index ed120aaa55..79257cb6ef 100644 --- a/pkg/eosclient/eosclient.go +++ b/pkg/eosclient/eosclient.go @@ -34,7 +34,7 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage/acl" + "github.com/cs3org/reva/pkg/storage/utils/acl" "github.com/gofrs/uuid" "github.com/pkg/errors" "go.opencensus.io/trace" diff --git a/pkg/storage/fs/eos/eos.go b/pkg/storage/fs/eos/eos.go index 8ea146e0ba..d615357351 100644 --- a/pkg/storage/fs/eos/eos.go +++ b/pkg/storage/fs/eos/eos.go @@ -19,174 +19,85 @@ package eos import ( - "context" - "encoding/json" - "fmt" - "io" - "net/url" - "os" - gouser "os/user" - "path" - "regexp" - "strconv" - "strings" + "bytes" + "encoding/gob" - "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/eosclient" - "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/acl" "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/cs3org/reva/pkg/storage/templates" - "github.com/cs3org/reva/pkg/user" + "github.com/cs3org/reva/pkg/storage/utils/eosfs" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/cs3org/reva/pkg/errtypes" -) - -const ( - refTargetAttrKey = "reva.target" ) func init() { registry.Register("eos", New) } -var hiddenReg = regexp.MustCompile(`\.sys\..#.`) - -type eosfs struct { - c *eosclient.Client - conf *config -} - -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - -// Options are the configuration options to pass to the New function. type config struct { // Namespace for metadata operations - Namespace string `mapstructure:"namespace"` + Namespace string `mapstructure:"namespace" docs:"/"` // ShadowNamespace for storing shadow data - ShadowNamespace string `mapstructure:"shadow_namespace"` + ShadowNamespace string `mapstructure:"shadow_namespace" docs:"/.shadow"` // UploadsNamespace for storing upload data - UploadsNamespace string `mapstructure:"uploads_namespace"` + UploadsNamespace string `mapstructure:"uploads_namespace" docs:"/.uploads"` // ShareFolder defines the name of the folder in the // shadowed namespace. Ex: /eos/user/.shadow/h/hugo/MyShares - ShareFolder string `mapstructure:"share_folder"` + ShareFolder string `mapstructure:"share_folder" docs:"/MyShares"` // Location of the eos binary. // Default is /usr/bin/eos. - EosBinary string `mapstructure:"eos_binary"` + EosBinary string `mapstructure:"eos_binary" docs:"/usr/bin/eos"` // Location of the xrdcopy binary. // Default is /usr/bin/xrdcopy. - XrdcopyBinary string `mapstructure:"xrdcopy_binary"` + XrdcopyBinary string `mapstructure:"xrdcopy_binary" docs:"/usr/bin/xrdcopy"` // URL of the Master EOS MGM. // Default is root://eos-example.org - MasterURL string `mapstructure:"master_url"` + MasterURL string `mapstructure:"master_url" docs:"root://eos-example.org"` // URL of the Slave EOS MGM. // Default is root://eos-example.org - SlaveURL string `mapstructure:"slave_url"` + SlaveURL string `mapstructure:"slave_url" docs:"root://eos-example.org"` // Location on the local fs where to store reads. // Defaults to os.TempDir() - CacheDirectory string `mapstructure:"cache_directory"` + CacheDirectory string `mapstructure:"cache_directory" docs:"/var/tmp/"` // SecProtocol specifies the xrootd security protocol to use between the server and EOS. - SecProtocol string `mapstructure:"sec_protocol"` + SecProtocol string `mapstructure:"sec_protocol" docs:"-"` // Keytab specifies the location of the keytab to use to authenticate to EOS. - Keytab string `mapstructure:"keytab"` + Keytab string `mapstructure:"keytab" docs:"-"` // SingleUsername is the username to use when SingleUserMode is enabled - SingleUsername string `mapstructure:"single_username"` - - // UserLayout wraps the internal path with user information. - // Example: if conf.Namespace is /eos/user and received path is /docs - // and the UserLayout is {{.Username}} the internal path will be: - // /eos/user//docs - UserLayout string `mapstructure:"user_layout"` + SingleUsername string `mapstructure:"single_username" docs:"-"` // Enables logging of the commands executed // Defaults to false - EnableLogging bool `mapstructure:"enable_logging"` + EnableLogging bool `mapstructure:"enable_logging" docs:"false"` // ShowHiddenSysFiles shows internal EOS files like // .sys.v# and .sys.a# files. - ShowHiddenSysFiles bool `mapstructure:"show_hidden_sys_files"` + ShowHiddenSysFiles bool `mapstructure:"show_hidden_sys_files" docs:"false"` // ForceSingleUserMode will force connections to EOS to use SingleUsername - ForceSingleUserMode bool `mapstructure:"force_single_user_mode"` + ForceSingleUserMode bool `mapstructure:"force_single_user_mode" docs:"false"` // UseKeyTabAuth changes will authenticate requests by using an EOS keytab. - UseKeytab bool `mapstructure:"use_keytab"` - - // EnableHome enables the creation of home directories. - EnableHome bool `mapstructure:"enable_home"` + UseKeytab bool `mapstructure:"use_keytab" docs:"false"` } -func getUser(ctx context.Context) (*userpb.User, error) { - u, ok := user.ContextGetUser(ctx) - if !ok { - err := errors.Wrap(errtypes.UserRequired(""), "eos: error getting user from ctx") +func parseConfig(m map[string]interface{}) (*config, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + err = errors.Wrap(err, "error decoding conf") return nil, err } - return u, nil -} - -func (c *config) init() { - c.Namespace = path.Clean(c.Namespace) - if !strings.HasPrefix(c.Namespace, "/") { - c.Namespace = "/" - } - - if c.ShadowNamespace == "" { - c.ShadowNamespace = path.Join(c.Namespace, ".shadow") - } - - if c.ShareFolder == "" { - c.ShareFolder = "/MyShares" - } - // ensure share folder always starts with slash - c.ShareFolder = path.Join("/", c.ShareFolder) - - if c.EosBinary == "" { - c.EosBinary = "/usr/bin/eos" - } - - if c.XrdcopyBinary == "" { - c.XrdcopyBinary = "/usr/bin/xrdcopy" - } - - if c.MasterURL == "" { - c.MasterURL = "root://eos-example.org" - } - - if c.SlaveURL == "" { - c.SlaveURL = c.MasterURL - } - - if c.CacheDirectory == "" { - c.CacheDirectory = os.TempDir() - } - - if c.UserLayout == "" { - c.UserLayout = "{{.Username}}" // TODO set better layout - } + return c, nil } // New returns a new implementation of the storage.FS interface that connects to EOS. @@ -195,1270 +106,17 @@ func New(m map[string]interface{}) (storage.FS, error) { if err != nil { return nil, err } - c.init() - - // bail out if keytab is not found. - if c.UseKeytab { - if _, err := os.Stat(c.Keytab); err != nil { - err = errors.Wrapf(err, "eos: keytab not accesible at location: %s", err) - return nil, err - } - } - - eosClientOpts := &eosclient.Options{ - XrdcopyBinary: c.XrdcopyBinary, - URL: c.MasterURL, - EosBinary: c.EosBinary, - CacheDirectory: c.CacheDirectory, - ForceSingleUserMode: c.ForceSingleUserMode, - SingleUsername: c.SingleUsername, - UseKeytab: c.UseKeytab, - Keytab: c.Keytab, - SecProtocol: c.SecProtocol, - } - - eosClient := eosclient.New(eosClientOpts) - - eosfs := &eosfs{ - c: eosClient, - conf: c, - } - - return eosfs, nil -} - -func (fs *eosfs) Shutdown(ctx context.Context) error { - // TODO(labkode): in a grpc implementation we can close connections. - return nil -} - -func (fs *eosfs) wrapShadow(ctx context.Context, fn string) (internal string) { - if fs.conf.EnableHome { - layout, err := fs.getInternalHome(ctx) - if err != nil { - panic(err) - } - internal = path.Join(fs.conf.ShadowNamespace, layout, fn) - } else { - internal = path.Join(fs.conf.ShadowNamespace, fn) - } - return -} - -func (fs *eosfs) wrap(ctx context.Context, fn string) (internal string) { - if fs.conf.EnableHome { - layout, err := fs.getInternalHome(ctx) - if err != nil { - panic(err) - } - internal = path.Join(fs.conf.Namespace, layout, fn) - } else { - internal = path.Join(fs.conf.Namespace, fn) - } - log := appctx.GetLogger(ctx) - log.Debug().Msg("eos: wrap external=" + fn + " internal=" + internal) - return -} - -func (fs *eosfs) unwrap(ctx context.Context, internal string) (external string) { - log := appctx.GetLogger(ctx) - layout := fs.getLayout(ctx) - ns := fs.getNsMatch(internal, []string{fs.conf.Namespace, fs.conf.ShadowNamespace}) - external = fs.unwrapInternal(ctx, ns, internal, layout) - log.Debug().Msgf("eos: unwrap: internal=%s external=%s", internal, external) - return -} - -func (fs *eosfs) getLayout(ctx context.Context) (layout string) { - if fs.conf.EnableHome { - u, err := getUser(ctx) - if err != nil { - panic(err) - } - layout = templates.WithUser(u, fs.conf.UserLayout) - } - return -} - -func (fs *eosfs) getNsMatch(internal string, nss []string) string { - var match string - - for _, ns := range nss { - if strings.HasPrefix(internal, ns) && len(ns) > len(match) { - match = ns - } - } - - if match == "" { - panic(fmt.Sprintf("eos: path is outside namespaces: path=%s namespaces=%+v", internal, nss)) - } - - return match -} - -func (fs *eosfs) unwrapInternal(ctx context.Context, ns, np, layout string) (external string) { - log := appctx.GetLogger(ctx) - trim := path.Join(ns, layout) - - if !strings.HasPrefix(np, trim) { - panic("eos: resource is outside the directory of the logged-in user: internal=" + np + " trim=" + trim + " namespace=" + ns) - } - - external = strings.TrimPrefix(np, trim) - - if external == "" { - external = "/" - } - - log.Debug().Msgf("eos: unwrapInternal: trim=%s external=%s ns=%s np=%s", trim, external, ns, np) - - return -} - -func (fs *eosfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) { - u, err := getUser(ctx) - if err != nil { - return "", errors.Wrap(err, "eos: no user in ctx") - } - - // parts[0] = 868317, parts[1] = photos, ... - parts := strings.Split(id.OpaqueId, "/") - fileID, err := strconv.ParseUint(parts[0], 10, 64) - if err != nil { - return "", errors.Wrap(err, "eos: error parsing fileid string") - } - - eosFileInfo, err := fs.c.GetFileInfoByInode(ctx, u.Username, fileID) - if err != nil { - return "", errors.Wrap(err, "eos: error getting file info by inode") - } - - fi := fs.convertToResourceInfo(ctx, eosFileInfo) - return fi.Path, nil -} - -// resolve takes in a request path or request id and returns the unwrappedNominal path. -func (fs *eosfs) resolve(ctx context.Context, u *userpb.User, ref *provider.Reference) (string, error) { - if ref.GetPath() != "" { - return ref.GetPath(), nil - } - - if ref.GetId() != nil { - p, err := fs.getPath(ctx, u, ref.GetId()) - if err != nil { - return "", err - } - - return p, nil - } - - // reference is invalid - return "", fmt.Errorf("invalid reference %+v. id and path are missing", ref) -} - -func (fs *eosfs) getPath(ctx context.Context, u *userpb.User, id *provider.ResourceId) (string, error) { - fid, err := strconv.ParseUint(id.OpaqueId, 10, 64) - if err != nil { - return "", fmt.Errorf("error converting string to int for eos fileid: %s", id.OpaqueId) - } - - eosFileInfo, err := fs.c.GetFileInfoByInode(ctx, u.Username, fid) - if err != nil { - return "", errors.Wrap(err, "eos: error getting file info by inode") - } - - return fs.unwrap(ctx, eosFileInfo.File), nil -} - -func (fs *eosfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return errors.Wrap(err, "eos: error resolving reference") - } - - fn := fs.wrap(ctx, p) - - eosACL, err := fs.getEosACL(g) - if err != nil { - return err - } - - err = fs.c.AddACL(ctx, u.Username, fn, eosACL) - if err != nil { - return errors.Wrap(err, "eos: error adding acl") - } - - return nil -} - -func getEosACLType(gt provider.GranteeType) (string, error) { - switch gt { - case provider.GranteeType_GRANTEE_TYPE_USER: - return "u", nil - case provider.GranteeType_GRANTEE_TYPE_GROUP: - return "g", nil - default: - return "", errors.New("no eos acl for grantee type: " + gt.String()) - } -} - -// TODO(labkode): fine grained permission controls. -func getEosACLPerm(set *provider.ResourcePermissions) (string, error) { - var b strings.Builder - - if set.Stat || set.InitiateFileDownload { - b.WriteString("r") - } - if set.CreateContainer || set.InitiateFileUpload || set.Delete || set.Move { - b.WriteString("w") - } - if set.ListContainer { - b.WriteString("x") - } - - if set.Delete { - b.WriteString("+d") - } else { - b.WriteString("!d") - } - - // TODO sharing - // TODO trash - // TODO versions - return b.String(), nil -} - -func (fs *eosfs) getEosACL(g *provider.Grant) (*acl.Entry, error) { - permissions, err := getEosACLPerm(g.Permissions) - if err != nil { - return nil, err - } - t, err := getEosACLType(g.Grantee.Type) - if err != nil { - return nil, err - } - eosACL := &acl.Entry{ - Qualifier: g.Grantee.Id.OpaqueId, - Permissions: permissions, - Type: t, - } - return eosACL, nil -} - -func (fs *eosfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error { - return errtypes.NotSupported("eos: operation not supported") -} - -func (fs *eosfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) error { - return errtypes.NotSupported("eos: operation not supported") -} - -func (fs *eosfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - - eosACLType, err := getEosACLType(g.Grantee.Type) - if err != nil { - return err - } - - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return errors.Wrap(err, "eos: error resolving reference") - } - - fn := fs.wrap(ctx, p) - - err = fs.c.RemoveACL(ctx, u.Username, fn, eosACLType, g.Grantee.Id.OpaqueId) - if err != nil { - return errors.Wrap(err, "eos: error removing acl") - } - return nil -} - -func (fs *eosfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - - eosACL, err := fs.getEosACL(g) - if err != nil { - return errors.Wrap(err, "eos: error mapping acl") - } - - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return errors.Wrap(err, "eos: error resolving reference") - } - fn := fs.wrap(ctx, p) - - err = fs.c.AddACL(ctx, u.Username, fn, eosACL) - if err != nil { - return errors.Wrap(err, "eos: error updating acl") - } - return nil -} - -func (fs *eosfs) ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) { - u, err := getUser(ctx) - if err != nil { - return nil, err - } - - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return nil, errors.Wrap(err, "eos: error resolving reference") - } - fn := fs.wrap(ctx, p) - - acls, err := fs.c.ListACLs(ctx, u.Username, fn) - if err != nil { - return nil, err - } - - grants := []*provider.Grant{} - for _, a := range acls { - grantee := &provider.Grantee{ - Id: &userpb.UserId{OpaqueId: a.Qualifier}, - Type: fs.getGranteeType(a.Type), - } - grants = append(grants, &provider.Grant{ - Grantee: grantee, - Permissions: fs.getGrantPermissionSet(a.Permissions), - }) - } - - return grants, nil -} - -func (fs *eosfs) getGranteeType(aclType string) provider.GranteeType { - switch aclType { - case "u": - return provider.GranteeType_GRANTEE_TYPE_USER - case "g": - return provider.GranteeType_GRANTEE_TYPE_GROUP - default: - return provider.GranteeType_GRANTEE_TYPE_INVALID - } -} - -// TODO(labkode): add more fine grained controls. -// EOS acls are a mix of ACLs and POSIX permissions. More details can be found in -// https://github.com/cern-eos/eos/blob/master/doc/configuration/permission.rst -// TODO we need to evaluate all acls in the list at once to properly forbid (!) and overwrite (+) permissions -// This is ugly, because those are actually negative permissions ... -func (fs *eosfs) getGrantPermissionSet(mode string) *provider.ResourcePermissions { - - // TODO also check unix permissions for read access - p := &provider.ResourcePermissions{} - // r - if strings.Contains(mode, "r") { - p.Stat = true - p.InitiateFileDownload = true - } - // w - if strings.Contains(mode, "w") { - p.CreateContainer = true - p.InitiateFileUpload = true - p.Delete = true - if p.InitiateFileDownload { - p.Move = true - } - } - if strings.Contains(mode, "wo") { - p.CreateContainer = true - // p.InitiateFileUpload = false // TODO only when the file exists - p.Delete = false - } - if strings.Contains(mode, "!d") { - p.Delete = false - } else if strings.Contains(mode, "+d") { - p.Delete = true - } - // x - if strings.Contains(mode, "x") { - p.ListContainer = true - } - - // sharing - // TODO AddGrant - // TODO ListGrants - // TODO RemoveGrant - // TODO UpdateGrant - - // trash - // TODO ListRecycle - // TODO RestoreRecycleItem - // TODO PurgeRecycle - - // versions - // TODO ListFileVersions - // TODO RestoreFileVersion - - // ? - // TODO GetPath - // TODO GetQuota - return p -} - -func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) { - u, err := getUser(ctx) - if err != nil { - return nil, err - } - - log := appctx.GetLogger(ctx) - log.Info().Msg("eos: get md for ref:" + ref.String()) - - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return nil, errors.Wrap(err, "eos: error resolving reference") - } - - // if path is home we need to add in the response any shadow folder in the shadow homedirectory. - if fs.conf.EnableHome { - if fs.isShareFolder(ctx, p) { - return fs.getMDShareFolder(ctx, p) - } - } - - fn := fs.wrap(ctx, p) - - eosFileInfo, err := fs.c.GetFileInfoByPath(ctx, u.Username, fn) - if err != nil { - return nil, err - } - - fi := fs.convertToResourceInfo(ctx, eosFileInfo) - return fi, nil -} -func (fs *eosfs) getMDShareFolder(ctx context.Context, p string) (*provider.ResourceInfo, error) { - u, err := getUser(ctx) + var buf bytes.Buffer + err = gob.NewEncoder(&buf).Encode(&c) if err != nil { return nil, err } - - fn := fs.wrapShadow(ctx, p) - eosFileInfo, err := fs.c.GetFileInfoByPath(ctx, u.Username, fn) + var conf eosfs.Config + err = gob.NewDecoder(&buf).Decode(&conf) if err != nil { return nil, err } - // TODO(labkode): diff between root (dir) and children (ref) - - if fs.isShareFolderRoot(ctx, p) { - return fs.convertToResourceInfo(ctx, eosFileInfo), nil - } - return fs.convertToFileReference(ctx, eosFileInfo), nil -} - -func (fs *eosfs) ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) { - log := appctx.GetLogger(ctx) - u, err := getUser(ctx) - if err != nil { - return nil, errors.Wrap(err, "eos: no user in ctx") - } - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return nil, errors.Wrap(err, "eos: error resolving reference") - } - - log.Debug().Msg("internal: " + p) - - // if path is home we need to add in the response any shadow folder in the shadow homedirectory. - if fs.conf.EnableHome { - log.Debug().Msg("home enabled") - if strings.HasPrefix(p, "/") { - return fs.listWithHome(ctx, "/", p) - } - } - - log.Debug().Msg("list with nominal home") - return fs.listWithNominalHome(ctx, p) -} - -func (fs *eosfs) listWithNominalHome(ctx context.Context, p string) (finfos []*provider.ResourceInfo, err error) { - log := appctx.GetLogger(ctx) - - u, err := getUser(ctx) - if err != nil { - return nil, errors.Wrap(err, "eos: no user in ctx") - } - - fn := fs.wrap(ctx, p) - - eosFileInfos, err := fs.c.List(ctx, u.Username, fn) - if err != nil { - return nil, errors.Wrap(err, "eos: error listing") - } - - for _, eosFileInfo := range eosFileInfos { - // filter out sys files - if !fs.conf.ShowHiddenSysFiles { - base := path.Base(eosFileInfo.File) - if hiddenReg.MatchString(base) { - log.Debug().Msgf("eos: path is filtered because is considered hidden: path=%s hiddenReg=%s", base, hiddenReg) - continue - } - } - - finfos = append(finfos, fs.convertToResourceInfo(ctx, eosFileInfo)) - } - - return finfos, nil -} - -func (fs *eosfs) listWithHome(ctx context.Context, home, p string) ([]*provider.ResourceInfo, error) { - log := appctx.GetLogger(ctx) - if p == home { - log.Debug().Msg("listing home") - return fs.listHome(ctx, home) - } - - if fs.isShareFolderRoot(ctx, p) { - log.Debug().Msg("listing share root folder") - return fs.listShareFolderRoot(ctx, p) - } - - if fs.isShareFolderChild(ctx, p) { - return nil, errtypes.PermissionDenied("eos: error listing folders inside the shared folder, only file references are stored inside") - } - - // path points to a resource in the nominal home - log.Debug().Msg("listting nominal home") - return fs.listWithNominalHome(ctx, p) + return eosfs.NewEOSFS(&conf) } - -func (fs *eosfs) listHome(ctx context.Context, home string) ([]*provider.ResourceInfo, error) { - u, err := getUser(ctx) - if err != nil { - return nil, errors.Wrap(err, "eos: no user in ctx") - } - - fns := []string{fs.wrap(ctx, home), fs.wrapShadow(ctx, home)} - - finfos := []*provider.ResourceInfo{} - for _, fn := range fns { - eosFileInfos, err := fs.c.List(ctx, u.Username, fn) - if err != nil { - return nil, errors.Wrap(err, "eos: error listing") - } - - for _, eosFileInfo := range eosFileInfos { - // filter out sys files - if !fs.conf.ShowHiddenSysFiles { - base := path.Base(eosFileInfo.File) - if hiddenReg.MatchString(base) { - continue - } - - } - finfos = append(finfos, fs.convertToResourceInfo(ctx, eosFileInfo)) - } - - } - return finfos, nil -} - -func (fs *eosfs) listShareFolderRoot(ctx context.Context, p string) (finfos []*provider.ResourceInfo, err error) { - u, err := getUser(ctx) - if err != nil { - return nil, errors.Wrap(err, "eos: no user in ctx") - } - - fn := fs.wrapShadow(ctx, p) - - eosFileInfos, err := fs.c.List(ctx, u.Username, fn) - if err != nil { - return nil, errors.Wrap(err, "eos: error listing") - } - - for _, eosFileInfo := range eosFileInfos { - // filter out sys files - if !fs.conf.ShowHiddenSysFiles { - base := path.Base(eosFileInfo.File) - if hiddenReg.MatchString(base) { - continue - } - } - - finfo := fs.convertToFileReference(ctx, eosFileInfo) - finfos = append(finfos, finfo) - } - - return finfos, nil -} - -func (fs *eosfs) GetQuota(ctx context.Context) (int, int, error) { - u, err := getUser(ctx) - if err != nil { - return 0, 0, errors.Wrap(err, "eos: no user in ctx") - } - - qi, err := fs.c.GetQuota(ctx, u.Username, fs.conf.Namespace) - if err != nil { - err := errors.Wrap(err, "eosfs: error getting quota") - return 0, 0, err - } - - return qi.AvailableBytes, qi.UsedBytes, nil -} - -func (fs *eosfs) getInternalHome(ctx context.Context) (string, error) { - if !fs.conf.EnableHome { - return "", errtypes.NotSupported("eos: get home not supported") - } - - u, err := getUser(ctx) - if err != nil { - err = errors.Wrap(err, "local: wrap: no user in ctx and home is enabled") - return "", err - } - - relativeHome := templates.WithUser(u, fs.conf.UserLayout) - return relativeHome, nil -} - -func (fs *eosfs) GetHome(ctx context.Context) (string, error) { - if !fs.conf.EnableHome { - return "", errtypes.NotSupported("eos: get home not supported") - } - - // eos drive for homes assumess root(/) points to the user home. - return "/", nil -} - -func (fs *eosfs) createShadowHome(ctx context.Context) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - - home := fs.wrapShadow(ctx, "/") - _, err = fs.c.GetFileInfoByPath(ctx, "root", home) - if err == nil { // home already exists - return nil - } - - // TODO(labkode): abort on any error that is not found - if _, ok := err.(errtypes.IsNotFound); !ok { - return errors.Wrap(err, "eos: error verifying if user home directory exists") - } - - err = fs.createUserDir(ctx, u.Username, home) - if err != nil { - return err - } - shadowFolders := []string{fs.conf.ShareFolder} - for _, sf := range shadowFolders { - err = fs.createUserDir(ctx, u.Username, path.Join(home, sf)) - if err != nil { - return err - } - } - - return nil -} - -func (fs *eosfs) createNominalHome(ctx context.Context) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - - home := fs.wrap(ctx, "/") - _, err = fs.c.GetFileInfoByPath(ctx, "root", home) - if err == nil { // home already exists - return nil - } - - // TODO(labkode): abort on any error that is not found - if _, ok := err.(errtypes.IsNotFound); !ok { - return errors.Wrap(err, "eos: error verifying if user home directory exists") - } - - err = fs.createUserDir(ctx, u.Username, home) - return err -} - -func (fs *eosfs) CreateHome(ctx context.Context) error { - if !fs.conf.EnableHome { - return errtypes.NotSupported("eos: create home not supported") - } - - if err := fs.createNominalHome(ctx); err != nil { - return errors.Wrap(err, "eos: error creating nominal home") - } - - if err := fs.createShadowHome(ctx); err != nil { - return errors.Wrap(err, "eos: error creating shadow home") - } - - return nil -} - -func (fs *eosfs) createUserDir(ctx context.Context, username string, path string) error { - err := fs.c.CreateDir(ctx, "root", path) - if err != nil { - // EOS will return success on mkdir over an existing directory. - return errors.Wrap(err, "eos: error creating dir") - } - - err = fs.c.Chown(ctx, "root", username, path) - if err != nil { - return errors.Wrap(err, "eos: error chowning directory") - } - - err = fs.c.Chmod(ctx, "root", "2770", path) - if err != nil { - return errors.Wrap(err, "eos: error chmoding directory") - } - - attrs := []*eosclient.Attribute{ - &eosclient.Attribute{ - Type: eosclient.SystemAttr, - Key: "mask", - Val: "700", - }, - &eosclient.Attribute{ - Type: eosclient.SystemAttr, - Key: "allow.oc.sync", - Val: "1", - }, - &eosclient.Attribute{ - Type: eosclient.SystemAttr, - Key: "mtime.propagation", - Val: "1", - }, - &eosclient.Attribute{ - Type: eosclient.SystemAttr, - Key: "forced.atomic", - Val: "1", - }, - } - - for _, attr := range attrs { - err = fs.c.SetAttr(ctx, "root", attr, true, path) - if err != nil { - return errors.Wrap(err, "eos: error setting attribute") - } - } - return nil -} - -func (fs *eosfs) CreateDir(ctx context.Context, p string) error { - log := appctx.GetLogger(ctx) - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - - log.Info().Msgf("eos: createdir: path=%s", p) - - if fs.isShareFolder(ctx, p) { - return errtypes.PermissionDenied("eos: cannot create folder under the share folder") - } - - fn := fs.wrap(ctx, p) - return fs.c.CreateDir(ctx, u.Username, fn) -} - -func (fs *eosfs) isShareFolder(ctx context.Context, p string) bool { - return strings.HasPrefix(p, fs.conf.ShareFolder) -} - -func (fs *eosfs) isShareFolderRoot(ctx context.Context, p string) bool { - return path.Clean(p) == fs.conf.ShareFolder -} - -func (fs *eosfs) isShareFolderChild(ctx context.Context, p string) bool { - p = path.Clean(p) - vals := strings.Split(p, fs.conf.ShareFolder+"/") - return len(vals) > 1 && vals[1] != "" -} - -func (fs *eosfs) CreateReference(ctx context.Context, p string, targetURI *url.URL) error { - // TODO(labkode): for the time being we only allow to create references - // on the virtual share folder to not pollute the nominal user tree. - - if !fs.isShareFolder(ctx, p) { - return errtypes.PermissionDenied("eos: cannot create references outside the share folder: share_folder=" + fs.conf.ShareFolder + " path=" + p) - } - - fn := fs.wrapShadow(ctx, p) - - // TODO(labkode): with grpc we can create a file touching with xattrs. - // Current mechanism is: touch to hidden dir, set xattr, rename. - dir, base := path.Split(fn) - tmp := path.Join(dir, fmt.Sprintf(".sys.reva#.%s", base)) - if err := fs.c.CreateDir(ctx, "root", tmp); err != nil { - err = errors.Wrapf(err, "eos: error creating temporary ref file") - return err - } - - // set xattr on ref - attr := &eosclient.Attribute{ - Type: eosclient.UserAttr, - Key: refTargetAttrKey, - Val: targetURI.String(), - } - - if err := fs.c.SetAttr(ctx, "root", attr, false, tmp); err != nil { - err = errors.Wrapf(err, "eos: error setting reva.ref attr on file: %q", tmp) - return err - } - - // rename to have the file visible in user space. - if err := fs.c.Rename(ctx, "root", tmp, fn); err != nil { - err = errors.Wrapf(err, "eos: error renaming from: %q to %q", tmp, fn) - return err - } - - return nil -} - -func (fs *eosfs) Delete(ctx context.Context, ref *provider.Reference) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return errors.Wrap(err, "eos: error resolving reference") - } - - if fs.isShareFolder(ctx, p) { - return fs.deleteShadow(ctx, p) - } - - fn := fs.wrap(ctx, p) - - return fs.c.Remove(ctx, u.Username, fn) -} - -func (fs *eosfs) deleteShadow(ctx context.Context, p string) error { - if fs.isShareFolderRoot(ctx, p) { - return errtypes.PermissionDenied("eos: cannot delete the virtual share folder") - } - - if fs.isShareFolderChild(ctx, p) { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - fn := fs.wrapShadow(ctx, p) - return fs.c.Remove(ctx, u.Username, fn) - } - - panic("eos: shadow delete of share folder that is neither root nor child. path=" + p) -} - -func (fs *eosfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - - oldPath, err := fs.resolve(ctx, u, oldRef) - if err != nil { - return errors.Wrap(err, "eos: error resolving reference") - } - - newPath, err := fs.resolve(ctx, u, newRef) - if err != nil { - return errors.Wrap(err, "eos: error resolving reference") - } - - if fs.isShareFolder(ctx, oldPath) || fs.isShareFolder(ctx, newPath) { - return fs.moveShadow(ctx, oldPath, newPath) - } - - oldFn := fs.wrap(ctx, oldPath) - newFn := fs.wrap(ctx, newPath) - return fs.c.Rename(ctx, u.Username, oldFn, newFn) -} - -func (fs *eosfs) moveShadow(ctx context.Context, oldPath, newPath string) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - - if fs.isShareFolderRoot(ctx, oldPath) || fs.isShareFolderRoot(ctx, newPath) { - return errtypes.PermissionDenied("eos: cannot move/rename the virtual share folder") - } - - // only rename of the reference is allowed, hence having the same basedir - bold, _ := path.Split(oldPath) - bnew, _ := path.Split(newPath) - - if bold != bnew { - return errtypes.PermissionDenied("eos: cannot move references under the virtual share folder") - } - - oldfn := fs.wrapShadow(ctx, oldPath) - newfn := fs.wrapShadow(ctx, newPath) - return fs.c.Rename(ctx, u.Username, oldfn, newfn) -} - -func (fs *eosfs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { - u, err := getUser(ctx) - if err != nil { - return nil, errors.Wrap(err, "eos: no user in ctx") - } - - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return nil, errors.Wrap(err, "eos: error resolving reference") - } - - if fs.isShareFolder(ctx, p) { - return nil, errtypes.PermissionDenied("eos: cannot download under the virtual share folder") - } - - fn := fs.wrap(ctx, p) - - return fs.c.Read(ctx, u.Username, fn) -} - -func (fs *eosfs) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { - u, err := getUser(ctx) - if err != nil { - return nil, errors.Wrap(err, "eos: no user in ctx") - } - - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return nil, errors.Wrap(err, "eos: error resolving reference") - } - - if fs.isShareFolder(ctx, p) { - return nil, errtypes.PermissionDenied("eos: cannot list revisions under the virtual share folder") - } - - fn := fs.wrap(ctx, p) - - eosRevisions, err := fs.c.ListVersions(ctx, u.Username, fn) - if err != nil { - return nil, errors.Wrap(err, "eos: error listing versions") - } - revisions := []*provider.FileVersion{} - for _, eosRev := range eosRevisions { - rev := fs.convertToRevision(ctx, eosRev) - revisions = append(revisions, rev) - } - return revisions, nil -} - -func (fs *eosfs) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { - u, err := getUser(ctx) - if err != nil { - return nil, errors.Wrap(err, "eos: no user in ctx") - } - - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return nil, errors.Wrap(err, "eos: error resolving reference") - } - - if fs.isShareFolder(ctx, p) { - return nil, errtypes.PermissionDenied("eos: cannot download revision under the virtual share folder") - } - - fn := fs.wrap(ctx, p) - - fn = fs.wrap(ctx, fn) - return fs.c.ReadVersion(ctx, u.Username, fn, revisionKey) -} - -func (fs *eosfs) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return errors.Wrap(err, "eos: error resolving reference") - } - - if fs.isShareFolder(ctx, p) { - return errtypes.PermissionDenied("eos: cannot restore revision under the virtual share folder") - } - - fn := fs.wrap(ctx, p) - - return fs.c.RollbackToVersion(ctx, u.Username, fn, revisionKey) -} - -func (fs *eosfs) PurgeRecycleItem(ctx context.Context, key string) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "storage_eos: no user in ctx") - } - return fs.c.RestoreDeletedEntry(ctx, u.Username, key) -} - -func (fs *eosfs) EmptyRecycle(ctx context.Context) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - return fs.c.PurgeDeletedEntries(ctx, u.Username) -} - -func (fs *eosfs) ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) { - u, err := getUser(ctx) - if err != nil { - return nil, errors.Wrap(err, "eos: no user in ctx") - } - eosDeletedEntries, err := fs.c.ListDeletedEntries(ctx, u.Username) - if err != nil { - return nil, errors.Wrap(err, "eos: error listing deleted entries") - } - recycleEntries := []*provider.RecycleItem{} - for _, entry := range eosDeletedEntries { - if !fs.conf.ShowHiddenSysFiles { - base := path.Base(entry.RestorePath) - if hiddenReg.MatchString(base) { - continue - } - - } - recycleItem := fs.convertToRecycleItem(ctx, entry) - recycleEntries = append(recycleEntries, recycleItem) - } - return recycleEntries, nil -} - -func (fs *eosfs) RestoreRecycleItem(ctx context.Context, key string) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - return fs.c.RestoreDeletedEntry(ctx, u.Username, key) -} - -func (fs *eosfs) convertToRecycleItem(ctx context.Context, eosDeletedItem *eosclient.DeletedEntry) *provider.RecycleItem { - path := fs.unwrap(ctx, eosDeletedItem.RestorePath) - recycleItem := &provider.RecycleItem{ - Path: path, - Key: eosDeletedItem.RestoreKey, - Size: eosDeletedItem.Size, - DeletionTime: &types.Timestamp{Seconds: eosDeletedItem.DeletionMTime / 1000}, // TODO(labkode): check if eos time is millis or nanos - } - if eosDeletedItem.IsDir { - recycleItem.Type = provider.ResourceType_RESOURCE_TYPE_CONTAINER - } else { - // TODO(labkode): if eos returns more types oin the future we need to map them. - recycleItem.Type = provider.ResourceType_RESOURCE_TYPE_FILE - } - return recycleItem -} - -func (fs *eosfs) convertToRevision(ctx context.Context, eosFileInfo *eosclient.FileInfo) *provider.FileVersion { - md := fs.convertToResourceInfo(ctx, eosFileInfo) - revision := &provider.FileVersion{ - Key: path.Base(md.Path), - Size: md.Size, - Mtime: md.Mtime.Seconds, // TODO do we need nanos here? - } - return revision -} - -func (fs *eosfs) convertToResourceInfo(ctx context.Context, eosFileInfo *eosclient.FileInfo) *provider.ResourceInfo { - return fs.convert(ctx, eosFileInfo) -} - -func (fs *eosfs) convertToFileReference(ctx context.Context, eosFileInfo *eosclient.FileInfo) *provider.ResourceInfo { - info := fs.convert(ctx, eosFileInfo) - info.Type = provider.ResourceType_RESOURCE_TYPE_REFERENCE - val, ok := eosFileInfo.Attrs["user.reva.target"] - if !ok || val == "" { - panic("eos: reference does not contain target: target=" + val + " file=" + eosFileInfo.File) - } - info.Target = val - return info -} - -func (fs *eosfs) convert(ctx context.Context, eosFileInfo *eosclient.FileInfo) *provider.ResourceInfo { - path := fs.unwrap(ctx, eosFileInfo.File) - - size := eosFileInfo.Size - if eosFileInfo.IsDir { - size = eosFileInfo.TreeSize - } - - username, err := getUsername(eosFileInfo.UID) - if err != nil { - log := appctx.GetLogger(ctx) - log.Warn().Uint64("uid", eosFileInfo.UID).Msg("could not lookup userid, leaving empty") - username = "" // TODO(labkode): should we abort here? - } - - info := &provider.ResourceInfo{ - Id: &provider.ResourceId{OpaqueId: fmt.Sprintf("%d", eosFileInfo.Inode)}, - Path: path, - Owner: &userpb.UserId{OpaqueId: username}, - Etag: eosFileInfo.ETag, - MimeType: mime.Detect(eosFileInfo.IsDir, path), - Size: size, - PermissionSet: &provider.ResourcePermissions{ListContainer: true, CreateContainer: true}, - Mtime: &types.Timestamp{ - Seconds: eosFileInfo.MTimeSec, - Nanos: eosFileInfo.MTimeNanos, - }, - Opaque: &types.Opaque{ - Map: map[string]*types.OpaqueEntry{ - "eos": &types.OpaqueEntry{ - Decoder: "json", - Value: fs.getEosMetadata(eosFileInfo), - }, - }, - }, - } - - info.Type = getResourceType(eosFileInfo.IsDir) - return info -} - -func getResourceType(isDir bool) provider.ResourceType { - if isDir { - return provider.ResourceType_RESOURCE_TYPE_CONTAINER - } - return provider.ResourceType_RESOURCE_TYPE_FILE -} - -func getUsername(uid uint64) (string, error) { - s := strconv.FormatUint(uid, 10) - user, err := gouser.LookupId(s) - if err != nil { - return "", err - } - return user.Username, nil -} - -type eosSysMetadata struct { - TreeSize uint64 `json:"tree_size"` - TreeCount uint64 `json:"tree_count"` - File string `json:"file"` - Instance string `json:"instance"` -} - -func (fs *eosfs) getEosMetadata(finfo *eosclient.FileInfo) []byte { - sys := &eosSysMetadata{ - File: finfo.File, - Instance: finfo.Instance, - } - - if finfo.IsDir { - sys.TreeCount = finfo.TreeCount - sys.TreeSize = finfo.TreeSize - } - - v, _ := json.Marshal(sys) - return v -} - -/* - Merge shadow on requests for /home ? - - No - GetHome(ctx context.Context) (string, error) - No -CreateHome(ctx context.Context) error - No - CreateDir(ctx context.Context, fn string) error - No -Delete(ctx context.Context, ref *provider.Reference) error - No -Move(ctx context.Context, oldRef, newRef *provider.Reference) error - No -GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) - Yes -ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) - No -Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error - No -Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) - No -ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) - No -DownloadRevision(ctx context.Context, ref *provider.Reference, key string) (io.ReadCloser, error) - No -RestoreRevision(ctx context.Context, ref *provider.Reference, key string) error - No ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) - No RestoreRecycleItem(ctx context.Context, key string) error - No PurgeRecycleItem(ctx context.Context, key string) error - No EmptyRecycle(ctx context.Context) error - ? GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) - No AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error - No RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error - No UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error - No ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) - No GetQuota(ctx context.Context) (int, int, error) - No CreateReference(ctx context.Context, path string, targetURI *url.URL) error - No Shutdown(ctx context.Context) error - No SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error - No UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) error -*/ - -/* - Merge shadow on requests for /home/MyShares ? - - No - GetHome(ctx context.Context) (string, error) - No -CreateHome(ctx context.Context) error - No - CreateDir(ctx context.Context, fn string) error - Maybe -Delete(ctx context.Context, ref *provider.Reference) error - No -Move(ctx context.Context, oldRef, newRef *provider.Reference) error - Yes -GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) - Yes -ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) - No -Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error - No -Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) - No -ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) - No -DownloadRevision(ctx context.Context, ref *provider.Reference, key string) (io.ReadCloser, error) - No -RestoreRevision(ctx context.Context, ref *provider.Reference, key string) error - No ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) - No RestoreRecycleItem(ctx context.Context, key string) error - No PurgeRecycleItem(ctx context.Context, key string) error - No EmptyRecycle(ctx context.Context) error - ? GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) - No AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error - No RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error - No UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error - No ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) - No GetQuota(ctx context.Context) (int, int, error) - No CreateReference(ctx context.Context, path string, targetURI *url.URL) error - No Shutdown(ctx context.Context) error - No SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error - No UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) error -*/ - -/* - Merge shadow on requests for /home/MyShares/file-reference ? - - No - GetHome(ctx context.Context) (string, error) - No -CreateHome(ctx context.Context) error - No - CreateDir(ctx context.Context, fn string) error - Maybe -Delete(ctx context.Context, ref *provider.Reference) error - Yes -Move(ctx context.Context, oldRef, newRef *provider.Reference) error - Yes -GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) - No -ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) - No -Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error - No -Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) - No -ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) - No -DownloadRevision(ctx context.Context, ref *provider.Reference, key string) (io.ReadCloser, error) - No -RestoreRevision(ctx context.Context, ref *provider.Reference, key string) error - No ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) - No RestoreRecycleItem(ctx context.Context, key string) error - No PurgeRecycleItem(ctx context.Context, key string) error - No EmptyRecycle(ctx context.Context) error - ? GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) - No AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error - No RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error - No UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error - No ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) - No GetQuota(ctx context.Context) (int, int, error) - No CreateReference(ctx context.Context, path string, targetURI *url.URL) error - No Shutdown(ctx context.Context) error - Maybe SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error - Maybe UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) error -*/ diff --git a/pkg/storage/fs/eoshome/eoshome.go b/pkg/storage/fs/eoshome/eoshome.go new file mode 100644 index 0000000000..b0c71a0879 --- /dev/null +++ b/pkg/storage/fs/eoshome/eoshome.go @@ -0,0 +1,129 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package eoshome + +import ( + "bytes" + "encoding/gob" + + "github.com/cs3org/reva/pkg/storage" + "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/cs3org/reva/pkg/storage/utils/eosfs" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +func init() { + registry.Register("eoshome", New) +} + +type config struct { + // Namespace for metadata operations + Namespace string `mapstructure:"namespace" docs:"/"` + + // ShadowNamespace for storing shadow data + ShadowNamespace string `mapstructure:"shadow_namespace" docs:"/.shadow"` + + // UploadsNamespace for storing upload data + UploadsNamespace string `mapstructure:"uploads_namespace" docs:"/.uploads"` + + // ShareFolder defines the name of the folder in the + // shadowed namespace. Ex: /eos/user/.shadow/h/hugo/MyShares + ShareFolder string `mapstructure:"share_folder" docs:"/MyShares"` + + // Location of the eos binary. + // Default is /usr/bin/eos. + EosBinary string `mapstructure:"eos_binary" docs:"/usr/bin/eos"` + + // Location of the xrdcopy binary. + // Default is /usr/bin/xrdcopy. + XrdcopyBinary string `mapstructure:"xrdcopy_binary" docs:"/usr/bin/xrdcopy"` + + // URL of the Master EOS MGM. + // Default is root://eos-example.org + MasterURL string `mapstructure:"master_url" docs:"root://eos-example.org"` + + // URL of the Slave EOS MGM. + // Default is root://eos-example.org + SlaveURL string `mapstructure:"slave_url" docs:"root://eos-example.org"` + + // Location on the local fs where to store reads. + // Defaults to os.TempDir() + CacheDirectory string `mapstructure:"cache_directory" docs:"/var/tmp/"` + + // SecProtocol specifies the xrootd security protocol to use between the server and EOS. + SecProtocol string `mapstructure:"sec_protocol" docs:"-"` + + // Keytab specifies the location of the keytab to use to authenticate to EOS. + Keytab string `mapstructure:"keytab" docs:"-"` + + // SingleUsername is the username to use when SingleUserMode is enabled + SingleUsername string `mapstructure:"single_username" docs:"-"` + + // UserLayout wraps the internal path with user information. + // Example: if conf.Namespace is /eos/user and received path is /docs + // and the UserLayout is {{.Username}} the internal path will be: + // /eos/user//docs + UserLayout string `mapstructure:"user_layout" docs:"{{.Username}}"` + + // Enables logging of the commands executed + // Defaults to false + EnableLogging bool `mapstructure:"enable_logging" docs:"false"` + + // ShowHiddenSysFiles shows internal EOS files like + // .sys.v# and .sys.a# files. + ShowHiddenSysFiles bool `mapstructure:"show_hidden_sys_files" docs:"false"` + + // ForceSingleUserMode will force connections to EOS to use SingleUsername + ForceSingleUserMode bool `mapstructure:"force_single_user_mode" docs:"false"` + + // UseKeyTabAuth changes will authenticate requests by using an EOS keytab. + UseKeytab bool `mapstructure:"use_keytab" docs:"false"` +} + +func parseConfig(m map[string]interface{}) (*config, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + err = errors.Wrap(err, "error decoding conf") + return nil, err + } + return c, nil +} + +// New returns a new implementation of the storage.FS interface that connects to EOS. +func New(m map[string]interface{}) (storage.FS, error) { + c, err := parseConfig(m) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + err = gob.NewEncoder(&buf).Encode(&c) + if err != nil { + return nil, err + } + var conf eosfs.Config + err = gob.NewDecoder(&buf).Decode(&conf) + if err != nil { + return nil, err + } + conf.EnableHome = true + + return eosfs.NewEOSFS(&conf) +} diff --git a/pkg/storage/acl/acl.go b/pkg/storage/utils/acl/acl.go similarity index 100% rename from pkg/storage/acl/acl.go rename to pkg/storage/utils/acl/acl.go diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go new file mode 100644 index 0000000000..7acb054140 --- /dev/null +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -0,0 +1,1341 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package eosfs + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/url" + "os" + gouser "os/user" + "path" + "regexp" + "strconv" + "strings" + + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/eosclient" + "github.com/cs3org/reva/pkg/mime" + "github.com/cs3org/reva/pkg/storage" + "github.com/cs3org/reva/pkg/storage/templates" + "github.com/cs3org/reva/pkg/storage/utils/acl" + "github.com/cs3org/reva/pkg/storage/utils/grants" + "github.com/cs3org/reva/pkg/user" + "github.com/pkg/errors" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/pkg/errtypes" +) + +const ( + refTargetAttrKey = "reva.target" +) + +var hiddenReg = regexp.MustCompile(`\.sys\..#.`) + +// Config holds the configuration details for the EOS fs. +type Config struct { + // Namespace for metadata operations + Namespace string `mapstructure:"namespace"` + + // ShadowNamespace for storing shadow data + ShadowNamespace string `mapstructure:"shadow_namespace"` + + // UploadsNamespace for storing upload data + UploadsNamespace string `mapstructure:"uploads_namespace"` + + // ShareFolder defines the name of the folder in the + // shadowed namespace. Ex: /eos/user/.shadow/h/hugo/MyShares + ShareFolder string `mapstructure:"share_folder"` + + // Location of the eos binary. + // Default is /usr/bin/eos. + EosBinary string `mapstructure:"eos_binary"` + + // Location of the xrdcopy binary. + // Default is /usr/bin/xrdcopy. + XrdcopyBinary string `mapstructure:"xrdcopy_binary"` + + // URL of the Master EOS MGM. + // Default is root://eos-example.org + MasterURL string `mapstructure:"master_url"` + + // URL of the Slave EOS MGM. + // Default is root://eos-example.org + SlaveURL string `mapstructure:"slave_url"` + + // Location on the local fs where to store reads. + // Defaults to os.TempDir() + CacheDirectory string `mapstructure:"cache_directory"` + + // SecProtocol specifies the xrootd security protocol to use between the server and EOS. + SecProtocol string `mapstructure:"sec_protocol"` + + // Keytab specifies the location of the keytab to use to authenticate to EOS. + Keytab string `mapstructure:"keytab"` + + // SingleUsername is the username to use when SingleUserMode is enabled + SingleUsername string `mapstructure:"single_username"` + + // UserLayout wraps the internal path with user information. + // Example: if conf.Namespace is /eos/user and received path is /docs + // and the UserLayout is {{.Username}} the internal path will be: + // /eos/user//docs + UserLayout string `mapstructure:"user_layout"` + + // Enables logging of the commands executed + // Defaults to false + EnableLogging bool `mapstructure:"enable_logging"` + + // ShowHiddenSysFiles shows internal EOS files like + // .sys.v# and .sys.a# files. + ShowHiddenSysFiles bool `mapstructure:"show_hidden_sys_files"` + + // ForceSingleUserMode will force connections to EOS to use SingleUsername + ForceSingleUserMode bool `mapstructure:"force_single_user_mode"` + + // UseKeyTabAuth changes will authenticate requests by using an EOS keytab. + UseKeytab bool `mapstructure:"use_keytab"` + + // EnableHome enables the creation of home directories. + EnableHome bool `mapstructure:"enable_home"` +} + +func (c *Config) init() { + c.Namespace = path.Clean(c.Namespace) + if !strings.HasPrefix(c.Namespace, "/") { + c.Namespace = "/" + } + + if c.ShadowNamespace == "" { + c.ShadowNamespace = path.Join(c.Namespace, ".shadow") + } + + if c.ShareFolder == "" { + c.ShareFolder = "/MyShares" + } + // ensure share folder always starts with slash + c.ShareFolder = path.Join("/", c.ShareFolder) + + if c.EosBinary == "" { + c.EosBinary = "/usr/bin/eos" + } + + if c.XrdcopyBinary == "" { + c.XrdcopyBinary = "/usr/bin/xrdcopy" + } + + if c.MasterURL == "" { + c.MasterURL = "root://eos-example.org" + } + + if c.SlaveURL == "" { + c.SlaveURL = c.MasterURL + } + + if c.CacheDirectory == "" { + c.CacheDirectory = os.TempDir() + } + + if c.UserLayout == "" { + c.UserLayout = "{{.Username}}" // TODO set better layout + } +} + +type eosfs struct { + c *eosclient.Client + conf *Config +} + +// NewEOSFS returns a storage.FS interface implementation that connects to an +// EOS instance +func NewEOSFS(c *Config) (storage.FS, error) { + c.init() + + // bail out if keytab is not found. + if c.UseKeytab { + if _, err := os.Stat(c.Keytab); err != nil { + err = errors.Wrapf(err, "eos: keytab not accesible at location: %s", err) + return nil, err + } + } + + eosClientOpts := &eosclient.Options{ + XrdcopyBinary: c.XrdcopyBinary, + URL: c.MasterURL, + EosBinary: c.EosBinary, + CacheDirectory: c.CacheDirectory, + ForceSingleUserMode: c.ForceSingleUserMode, + SingleUsername: c.SingleUsername, + UseKeytab: c.UseKeytab, + Keytab: c.Keytab, + SecProtocol: c.SecProtocol, + } + + eosClient := eosclient.New(eosClientOpts) + + eosfs := &eosfs{ + c: eosClient, + conf: c, + } + + return eosfs, nil +} + +func (fs *eosfs) Shutdown(ctx context.Context) error { + // TODO(labkode): in a grpc implementation we can close connections. + return nil +} + +func getUser(ctx context.Context) (*userpb.User, error) { + u, ok := user.ContextGetUser(ctx) + if !ok { + err := errors.Wrap(errtypes.UserRequired(""), "eos: error getting user from ctx") + return nil, err + } + return u, nil +} + +func (fs *eosfs) wrapShadow(ctx context.Context, fn string) (internal string) { + if fs.conf.EnableHome { + layout, err := fs.getInternalHome(ctx) + if err != nil { + panic(err) + } + internal = path.Join(fs.conf.ShadowNamespace, layout, fn) + } else { + internal = path.Join(fs.conf.ShadowNamespace, fn) + } + return +} + +func (fs *eosfs) wrap(ctx context.Context, fn string) (internal string) { + if fs.conf.EnableHome { + layout, err := fs.getInternalHome(ctx) + if err != nil { + panic(err) + } + internal = path.Join(fs.conf.Namespace, layout, fn) + } else { + internal = path.Join(fs.conf.Namespace, fn) + } + log := appctx.GetLogger(ctx) + log.Debug().Msg("eos: wrap external=" + fn + " internal=" + internal) + return +} + +func (fs *eosfs) unwrap(ctx context.Context, internal string) (external string) { + log := appctx.GetLogger(ctx) + layout := fs.getLayout(ctx) + ns := fs.getNsMatch(internal, []string{fs.conf.Namespace, fs.conf.ShadowNamespace}) + external = fs.unwrapInternal(ctx, ns, internal, layout) + log.Debug().Msgf("eos: unwrap: internal=%s external=%s", internal, external) + return +} + +func (fs *eosfs) getLayout(ctx context.Context) (layout string) { + if fs.conf.EnableHome { + u, err := getUser(ctx) + if err != nil { + panic(err) + } + layout = templates.WithUser(u, fs.conf.UserLayout) + } + return +} + +func (fs *eosfs) getNsMatch(internal string, nss []string) string { + var match string + + for _, ns := range nss { + if strings.HasPrefix(internal, ns) && len(ns) > len(match) { + match = ns + } + } + + if match == "" { + panic(fmt.Sprintf("eos: path is outside namespaces: path=%s namespaces=%+v", internal, nss)) + } + + return match +} + +func (fs *eosfs) unwrapInternal(ctx context.Context, ns, np, layout string) (external string) { + log := appctx.GetLogger(ctx) + trim := path.Join(ns, layout) + + if !strings.HasPrefix(np, trim) { + panic("eos: resource is outside the directory of the logged-in user: internal=" + np + " trim=" + trim + " namespace=" + ns) + } + + external = strings.TrimPrefix(np, trim) + + if external == "" { + external = "/" + } + + log.Debug().Msgf("eos: unwrapInternal: trim=%s external=%s ns=%s np=%s", trim, external, ns, np) + + return +} + +// resolve takes in a request path or request id and returns the unwrappedNominal path. +func (fs *eosfs) resolve(ctx context.Context, u *userpb.User, ref *provider.Reference) (string, error) { + if ref.GetPath() != "" { + return ref.GetPath(), nil + } + + if ref.GetId() != nil { + p, err := fs.getPath(ctx, u, ref.GetId()) + if err != nil { + return "", err + } + + return p, nil + } + + // reference is invalid + return "", fmt.Errorf("invalid reference %+v. id and path are missing", ref) +} + +func (fs *eosfs) getPath(ctx context.Context, u *userpb.User, id *provider.ResourceId) (string, error) { + fid, err := strconv.ParseUint(id.OpaqueId, 10, 64) + if err != nil { + return "", fmt.Errorf("error converting string to int for eos fileid: %s", id.OpaqueId) + } + + eosFileInfo, err := fs.c.GetFileInfoByInode(ctx, u.Username, fid) + if err != nil { + return "", errors.Wrap(err, "eos: error getting file info by inode") + } + + return fs.unwrap(ctx, eosFileInfo.File), nil +} + +func (fs *eosfs) isShareFolder(ctx context.Context, p string) bool { + return strings.HasPrefix(p, fs.conf.ShareFolder) +} + +func (fs *eosfs) isShareFolderRoot(ctx context.Context, p string) bool { + return path.Clean(p) == fs.conf.ShareFolder +} + +func (fs *eosfs) isShareFolderChild(ctx context.Context, p string) bool { + p = path.Clean(p) + vals := strings.Split(p, fs.conf.ShareFolder+"/") + return len(vals) > 1 && vals[1] != "" +} + +func (fs *eosfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) { + u, err := getUser(ctx) + if err != nil { + return "", errors.Wrap(err, "eos: no user in ctx") + } + + // parts[0] = 868317, parts[1] = photos, ... + parts := strings.Split(id.OpaqueId, "/") + fileID, err := strconv.ParseUint(parts[0], 10, 64) + if err != nil { + return "", errors.Wrap(err, "eos: error parsing fileid string") + } + + eosFileInfo, err := fs.c.GetFileInfoByInode(ctx, u.Username, fileID) + if err != nil { + return "", errors.Wrap(err, "eos: error getting file info by inode") + } + + fi := fs.convertToResourceInfo(ctx, eosFileInfo) + return fi.Path, nil +} + +func (fs *eosfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error { + return errtypes.NotSupported("eos: operation not supported") +} + +func (fs *eosfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) error { + return errtypes.NotSupported("eos: operation not supported") +} + +func (fs *eosfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + p, err := fs.resolve(ctx, u, ref) + if err != nil { + return errors.Wrap(err, "eos: error resolving reference") + } + + fn := fs.wrap(ctx, p) + + eosACL, err := fs.getEosACL(g) + if err != nil { + return err + } + + err = fs.c.AddACL(ctx, u.Username, fn, eosACL) + if err != nil { + return errors.Wrap(err, "eos: error adding acl") + } + + return nil +} + +func (fs *eosfs) getEosACL(g *provider.Grant) (*acl.Entry, error) { + permissions, err := grants.GetACLPerm(g.Permissions) + if err != nil { + return nil, err + } + t, err := grants.GetACLType(g.Grantee.Type) + if err != nil { + return nil, err + } + eosACL := &acl.Entry{ + Qualifier: g.Grantee.Id.OpaqueId, + Permissions: permissions, + Type: t, + } + return eosACL, nil +} + +func (fs *eosfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + eosACLType, err := grants.GetACLType(g.Grantee.Type) + if err != nil { + return err + } + + p, err := fs.resolve(ctx, u, ref) + if err != nil { + return errors.Wrap(err, "eos: error resolving reference") + } + + fn := fs.wrap(ctx, p) + + err = fs.c.RemoveACL(ctx, u.Username, fn, eosACLType, g.Grantee.Id.OpaqueId) + if err != nil { + return errors.Wrap(err, "eos: error removing acl") + } + return nil +} + +func (fs *eosfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + eosACL, err := fs.getEosACL(g) + if err != nil { + return errors.Wrap(err, "eos: error mapping acl") + } + + p, err := fs.resolve(ctx, u, ref) + if err != nil { + return errors.Wrap(err, "eos: error resolving reference") + } + fn := fs.wrap(ctx, p) + + err = fs.c.AddACL(ctx, u.Username, fn, eosACL) + if err != nil { + return errors.Wrap(err, "eos: error updating acl") + } + return nil +} + +func (fs *eosfs) ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) { + u, err := getUser(ctx) + if err != nil { + return nil, err + } + + p, err := fs.resolve(ctx, u, ref) + if err != nil { + return nil, errors.Wrap(err, "eos: error resolving reference") + } + fn := fs.wrap(ctx, p) + + acls, err := fs.c.ListACLs(ctx, u.Username, fn) + if err != nil { + return nil, err + } + + grantList := []*provider.Grant{} + for _, a := range acls { + grantee := &provider.Grantee{ + Id: &userpb.UserId{OpaqueId: a.Qualifier}, + Type: grants.GetGranteeType(a.Type), + } + grantList = append(grantList, &provider.Grant{ + Grantee: grantee, + Permissions: grants.GetGrantPermissionSet(a.Permissions), + }) + } + + return grantList, nil +} + +func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) { + u, err := getUser(ctx) + if err != nil { + return nil, err + } + + log := appctx.GetLogger(ctx) + log.Info().Msg("eos: get md for ref:" + ref.String()) + + p, err := fs.resolve(ctx, u, ref) + if err != nil { + return nil, errors.Wrap(err, "eos: error resolving reference") + } + + // if path is home we need to add in the response any shadow folder in the shadow homedirectory. + if fs.conf.EnableHome { + if fs.isShareFolder(ctx, p) { + return fs.getMDShareFolder(ctx, p) + } + } + + fn := fs.wrap(ctx, p) + + eosFileInfo, err := fs.c.GetFileInfoByPath(ctx, u.Username, fn) + if err != nil { + return nil, err + } + + fi := fs.convertToResourceInfo(ctx, eosFileInfo) + return fi, nil +} + +func (fs *eosfs) getMDShareFolder(ctx context.Context, p string) (*provider.ResourceInfo, error) { + u, err := getUser(ctx) + if err != nil { + return nil, err + } + + fn := fs.wrapShadow(ctx, p) + eosFileInfo, err := fs.c.GetFileInfoByPath(ctx, u.Username, fn) + if err != nil { + return nil, err + } + // TODO(labkode): diff between root (dir) and children (ref) + + if fs.isShareFolderRoot(ctx, p) { + return fs.convertToResourceInfo(ctx, eosFileInfo), nil + } + return fs.convertToFileReference(ctx, eosFileInfo), nil +} + +func (fs *eosfs) ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) { + log := appctx.GetLogger(ctx) + u, err := getUser(ctx) + if err != nil { + return nil, errors.Wrap(err, "eos: no user in ctx") + } + + p, err := fs.resolve(ctx, u, ref) + if err != nil { + return nil, errors.Wrap(err, "eos: error resolving reference") + } + + log.Debug().Msg("internal: " + p) + + // if path is home we need to add in the response any shadow folder in the shadow homedirectory. + if fs.conf.EnableHome { + log.Debug().Msg("home enabled") + if strings.HasPrefix(p, "/") { + return fs.listWithHome(ctx, "/", p) + } + } + + log.Debug().Msg("list with nominal home") + return fs.listWithNominalHome(ctx, p) +} + +func (fs *eosfs) listWithNominalHome(ctx context.Context, p string) (finfos []*provider.ResourceInfo, err error) { + log := appctx.GetLogger(ctx) + + u, err := getUser(ctx) + if err != nil { + return nil, errors.Wrap(err, "eos: no user in ctx") + } + + fn := fs.wrap(ctx, p) + + eosFileInfos, err := fs.c.List(ctx, u.Username, fn) + if err != nil { + return nil, errors.Wrap(err, "eos: error listing") + } + + for _, eosFileInfo := range eosFileInfos { + // filter out sys files + if !fs.conf.ShowHiddenSysFiles { + base := path.Base(eosFileInfo.File) + if hiddenReg.MatchString(base) { + log.Debug().Msgf("eos: path is filtered because is considered hidden: path=%s hiddenReg=%s", base, hiddenReg) + continue + } + } + + finfos = append(finfos, fs.convertToResourceInfo(ctx, eosFileInfo)) + } + + return finfos, nil +} + +func (fs *eosfs) listWithHome(ctx context.Context, home, p string) ([]*provider.ResourceInfo, error) { + log := appctx.GetLogger(ctx) + if p == home { + log.Debug().Msg("listing home") + return fs.listHome(ctx, home) + } + + if fs.isShareFolderRoot(ctx, p) { + log.Debug().Msg("listing share root folder") + return fs.listShareFolderRoot(ctx, p) + } + + if fs.isShareFolderChild(ctx, p) { + return nil, errtypes.PermissionDenied("eos: error listing folders inside the shared folder, only file references are stored inside") + } + + // path points to a resource in the nominal home + log.Debug().Msg("listting nominal home") + return fs.listWithNominalHome(ctx, p) +} + +func (fs *eosfs) listHome(ctx context.Context, home string) ([]*provider.ResourceInfo, error) { + u, err := getUser(ctx) + if err != nil { + return nil, errors.Wrap(err, "eos: no user in ctx") + } + + fns := []string{fs.wrap(ctx, home), fs.wrapShadow(ctx, home)} + + finfos := []*provider.ResourceInfo{} + for _, fn := range fns { + eosFileInfos, err := fs.c.List(ctx, u.Username, fn) + if err != nil { + return nil, errors.Wrap(err, "eos: error listing") + } + + for _, eosFileInfo := range eosFileInfos { + // filter out sys files + if !fs.conf.ShowHiddenSysFiles { + base := path.Base(eosFileInfo.File) + if hiddenReg.MatchString(base) { + continue + } + + } + finfos = append(finfos, fs.convertToResourceInfo(ctx, eosFileInfo)) + } + + } + return finfos, nil +} + +func (fs *eosfs) listShareFolderRoot(ctx context.Context, p string) (finfos []*provider.ResourceInfo, err error) { + u, err := getUser(ctx) + if err != nil { + return nil, errors.Wrap(err, "eos: no user in ctx") + } + + fn := fs.wrapShadow(ctx, p) + + eosFileInfos, err := fs.c.List(ctx, u.Username, fn) + if err != nil { + return nil, errors.Wrap(err, "eos: error listing") + } + + for _, eosFileInfo := range eosFileInfos { + // filter out sys files + if !fs.conf.ShowHiddenSysFiles { + base := path.Base(eosFileInfo.File) + if hiddenReg.MatchString(base) { + continue + } + } + + finfo := fs.convertToFileReference(ctx, eosFileInfo) + finfos = append(finfos, finfo) + } + + return finfos, nil +} + +func (fs *eosfs) GetQuota(ctx context.Context) (int, int, error) { + u, err := getUser(ctx) + if err != nil { + return 0, 0, errors.Wrap(err, "eos: no user in ctx") + } + + qi, err := fs.c.GetQuota(ctx, u.Username, fs.conf.Namespace) + if err != nil { + err := errors.Wrap(err, "eosfs: error getting quota") + return 0, 0, err + } + + return qi.AvailableBytes, qi.UsedBytes, nil +} + +func (fs *eosfs) getInternalHome(ctx context.Context) (string, error) { + if !fs.conf.EnableHome { + return "", errtypes.NotSupported("eos: get home not supported") + } + + u, err := getUser(ctx) + if err != nil { + err = errors.Wrap(err, "local: wrap: no user in ctx and home is enabled") + return "", err + } + + relativeHome := templates.WithUser(u, fs.conf.UserLayout) + return relativeHome, nil +} + +func (fs *eosfs) GetHome(ctx context.Context) (string, error) { + if !fs.conf.EnableHome { + return "", errtypes.NotSupported("eos: get home not supported") + } + + // eos drive for homes assumess root(/) points to the user home. + return "/", nil +} + +func (fs *eosfs) createShadowHome(ctx context.Context) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + home := fs.wrapShadow(ctx, "/") + _, err = fs.c.GetFileInfoByPath(ctx, "root", home) + if err == nil { // home already exists + return nil + } + + // TODO(labkode): abort on any error that is not found + if _, ok := err.(errtypes.IsNotFound); !ok { + return errors.Wrap(err, "eos: error verifying if user home directory exists") + } + + err = fs.createUserDir(ctx, u.Username, home) + if err != nil { + return err + } + shadowFolders := []string{fs.conf.ShareFolder} + for _, sf := range shadowFolders { + err = fs.createUserDir(ctx, u.Username, path.Join(home, sf)) + if err != nil { + return err + } + } + + return nil +} + +func (fs *eosfs) createNominalHome(ctx context.Context) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + home := fs.wrap(ctx, "/") + _, err = fs.c.GetFileInfoByPath(ctx, "root", home) + if err == nil { // home already exists + return nil + } + + // TODO(labkode): abort on any error that is not found + if _, ok := err.(errtypes.IsNotFound); !ok { + return errors.Wrap(err, "eos: error verifying if user home directory exists") + } + + err = fs.createUserDir(ctx, u.Username, home) + return err +} + +func (fs *eosfs) CreateHome(ctx context.Context) error { + if !fs.conf.EnableHome { + return errtypes.NotSupported("eos: create home not supported") + } + + if err := fs.createNominalHome(ctx); err != nil { + return errors.Wrap(err, "eos: error creating nominal home") + } + + if err := fs.createShadowHome(ctx); err != nil { + return errors.Wrap(err, "eos: error creating shadow home") + } + + return nil +} + +func (fs *eosfs) createUserDir(ctx context.Context, username string, path string) error { + err := fs.c.CreateDir(ctx, "root", path) + if err != nil { + // EOS will return success on mkdir over an existing directory. + return errors.Wrap(err, "eos: error creating dir") + } + + err = fs.c.Chown(ctx, "root", username, path) + if err != nil { + return errors.Wrap(err, "eos: error chowning directory") + } + + err = fs.c.Chmod(ctx, "root", "2770", path) + if err != nil { + return errors.Wrap(err, "eos: error chmoding directory") + } + + attrs := []*eosclient.Attribute{ + &eosclient.Attribute{ + Type: eosclient.SystemAttr, + Key: "mask", + Val: "700", + }, + &eosclient.Attribute{ + Type: eosclient.SystemAttr, + Key: "allow.oc.sync", + Val: "1", + }, + &eosclient.Attribute{ + Type: eosclient.SystemAttr, + Key: "mtime.propagation", + Val: "1", + }, + &eosclient.Attribute{ + Type: eosclient.SystemAttr, + Key: "forced.atomic", + Val: "1", + }, + } + + for _, attr := range attrs { + err = fs.c.SetAttr(ctx, "root", attr, true, path) + if err != nil { + return errors.Wrap(err, "eos: error setting attribute") + } + } + return nil +} + +func (fs *eosfs) CreateDir(ctx context.Context, p string) error { + log := appctx.GetLogger(ctx) + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + log.Info().Msgf("eos: createdir: path=%s", p) + + if fs.isShareFolder(ctx, p) { + return errtypes.PermissionDenied("eos: cannot create folder under the share folder") + } + + fn := fs.wrap(ctx, p) + return fs.c.CreateDir(ctx, u.Username, fn) +} + +func (fs *eosfs) CreateReference(ctx context.Context, p string, targetURI *url.URL) error { + // TODO(labkode): for the time being we only allow to create references + // on the virtual share folder to not pollute the nominal user tree. + + if !fs.isShareFolder(ctx, p) { + return errtypes.PermissionDenied("eos: cannot create references outside the share folder: share_folder=" + fs.conf.ShareFolder + " path=" + p) + } + + fn := fs.wrapShadow(ctx, p) + + // TODO(labkode): with grpc we can create a file touching with xattrs. + // Current mechanism is: touch to hidden dir, set xattr, rename. + dir, base := path.Split(fn) + tmp := path.Join(dir, fmt.Sprintf(".sys.reva#.%s", base)) + if err := fs.c.CreateDir(ctx, "root", tmp); err != nil { + err = errors.Wrapf(err, "eos: error creating temporary ref file") + return err + } + + // set xattr on ref + attr := &eosclient.Attribute{ + Type: eosclient.UserAttr, + Key: refTargetAttrKey, + Val: targetURI.String(), + } + + if err := fs.c.SetAttr(ctx, "root", attr, false, tmp); err != nil { + err = errors.Wrapf(err, "eos: error setting reva.ref attr on file: %q", tmp) + return err + } + + // rename to have the file visible in user space. + if err := fs.c.Rename(ctx, "root", tmp, fn); err != nil { + err = errors.Wrapf(err, "eos: error renaming from: %q to %q", tmp, fn) + return err + } + + return nil +} + +func (fs *eosfs) Delete(ctx context.Context, ref *provider.Reference) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + p, err := fs.resolve(ctx, u, ref) + if err != nil { + return errors.Wrap(err, "eos: error resolving reference") + } + + if fs.isShareFolder(ctx, p) { + return fs.deleteShadow(ctx, p) + } + + fn := fs.wrap(ctx, p) + + return fs.c.Remove(ctx, u.Username, fn) +} + +func (fs *eosfs) deleteShadow(ctx context.Context, p string) error { + if fs.isShareFolderRoot(ctx, p) { + return errtypes.PermissionDenied("eos: cannot delete the virtual share folder") + } + + if fs.isShareFolderChild(ctx, p) { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + fn := fs.wrapShadow(ctx, p) + return fs.c.Remove(ctx, u.Username, fn) + } + + panic("eos: shadow delete of share folder that is neither root nor child. path=" + p) +} + +func (fs *eosfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + oldPath, err := fs.resolve(ctx, u, oldRef) + if err != nil { + return errors.Wrap(err, "eos: error resolving reference") + } + + newPath, err := fs.resolve(ctx, u, newRef) + if err != nil { + return errors.Wrap(err, "eos: error resolving reference") + } + + if fs.isShareFolder(ctx, oldPath) || fs.isShareFolder(ctx, newPath) { + return fs.moveShadow(ctx, oldPath, newPath) + } + + oldFn := fs.wrap(ctx, oldPath) + newFn := fs.wrap(ctx, newPath) + return fs.c.Rename(ctx, u.Username, oldFn, newFn) +} + +func (fs *eosfs) moveShadow(ctx context.Context, oldPath, newPath string) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + if fs.isShareFolderRoot(ctx, oldPath) || fs.isShareFolderRoot(ctx, newPath) { + return errtypes.PermissionDenied("eos: cannot move/rename the virtual share folder") + } + + // only rename of the reference is allowed, hence having the same basedir + bold, _ := path.Split(oldPath) + bnew, _ := path.Split(newPath) + + if bold != bnew { + return errtypes.PermissionDenied("eos: cannot move references under the virtual share folder") + } + + oldfn := fs.wrapShadow(ctx, oldPath) + newfn := fs.wrapShadow(ctx, newPath) + return fs.c.Rename(ctx, u.Username, oldfn, newfn) +} + +func (fs *eosfs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { + u, err := getUser(ctx) + if err != nil { + return nil, errors.Wrap(err, "eos: no user in ctx") + } + + p, err := fs.resolve(ctx, u, ref) + if err != nil { + return nil, errors.Wrap(err, "eos: error resolving reference") + } + + if fs.isShareFolder(ctx, p) { + return nil, errtypes.PermissionDenied("eos: cannot download under the virtual share folder") + } + + fn := fs.wrap(ctx, p) + + return fs.c.Read(ctx, u.Username, fn) +} + +func (fs *eosfs) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { + u, err := getUser(ctx) + if err != nil { + return nil, errors.Wrap(err, "eos: no user in ctx") + } + + p, err := fs.resolve(ctx, u, ref) + if err != nil { + return nil, errors.Wrap(err, "eos: error resolving reference") + } + + if fs.isShareFolder(ctx, p) { + return nil, errtypes.PermissionDenied("eos: cannot list revisions under the virtual share folder") + } + + fn := fs.wrap(ctx, p) + + eosRevisions, err := fs.c.ListVersions(ctx, u.Username, fn) + if err != nil { + return nil, errors.Wrap(err, "eos: error listing versions") + } + revisions := []*provider.FileVersion{} + for _, eosRev := range eosRevisions { + rev := fs.convertToRevision(ctx, eosRev) + revisions = append(revisions, rev) + } + return revisions, nil +} + +func (fs *eosfs) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { + u, err := getUser(ctx) + if err != nil { + return nil, errors.Wrap(err, "eos: no user in ctx") + } + + p, err := fs.resolve(ctx, u, ref) + if err != nil { + return nil, errors.Wrap(err, "eos: error resolving reference") + } + + if fs.isShareFolder(ctx, p) { + return nil, errtypes.PermissionDenied("eos: cannot download revision under the virtual share folder") + } + + fn := fs.wrap(ctx, p) + + fn = fs.wrap(ctx, fn) + return fs.c.ReadVersion(ctx, u.Username, fn, revisionKey) +} + +func (fs *eosfs) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + p, err := fs.resolve(ctx, u, ref) + if err != nil { + return errors.Wrap(err, "eos: error resolving reference") + } + + if fs.isShareFolder(ctx, p) { + return errtypes.PermissionDenied("eos: cannot restore revision under the virtual share folder") + } + + fn := fs.wrap(ctx, p) + + return fs.c.RollbackToVersion(ctx, u.Username, fn, revisionKey) +} + +func (fs *eosfs) PurgeRecycleItem(ctx context.Context, key string) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "storage_eos: no user in ctx") + } + return fs.c.RestoreDeletedEntry(ctx, u.Username, key) +} + +func (fs *eosfs) EmptyRecycle(ctx context.Context) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + return fs.c.PurgeDeletedEntries(ctx, u.Username) +} + +func (fs *eosfs) ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) { + u, err := getUser(ctx) + if err != nil { + return nil, errors.Wrap(err, "eos: no user in ctx") + } + eosDeletedEntries, err := fs.c.ListDeletedEntries(ctx, u.Username) + if err != nil { + return nil, errors.Wrap(err, "eos: error listing deleted entries") + } + recycleEntries := []*provider.RecycleItem{} + for _, entry := range eosDeletedEntries { + if !fs.conf.ShowHiddenSysFiles { + base := path.Base(entry.RestorePath) + if hiddenReg.MatchString(base) { + continue + } + + } + recycleItem := fs.convertToRecycleItem(ctx, entry) + recycleEntries = append(recycleEntries, recycleItem) + } + return recycleEntries, nil +} + +func (fs *eosfs) RestoreRecycleItem(ctx context.Context, key string) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + return fs.c.RestoreDeletedEntry(ctx, u.Username, key) +} + +func (fs *eosfs) convertToRecycleItem(ctx context.Context, eosDeletedItem *eosclient.DeletedEntry) *provider.RecycleItem { + path := fs.unwrap(ctx, eosDeletedItem.RestorePath) + recycleItem := &provider.RecycleItem{ + Path: path, + Key: eosDeletedItem.RestoreKey, + Size: eosDeletedItem.Size, + DeletionTime: &types.Timestamp{Seconds: eosDeletedItem.DeletionMTime / 1000}, // TODO(labkode): check if eos time is millis or nanos + } + if eosDeletedItem.IsDir { + recycleItem.Type = provider.ResourceType_RESOURCE_TYPE_CONTAINER + } else { + // TODO(labkode): if eos returns more types oin the future we need to map them. + recycleItem.Type = provider.ResourceType_RESOURCE_TYPE_FILE + } + return recycleItem +} + +func (fs *eosfs) convertToRevision(ctx context.Context, eosFileInfo *eosclient.FileInfo) *provider.FileVersion { + md := fs.convertToResourceInfo(ctx, eosFileInfo) + revision := &provider.FileVersion{ + Key: path.Base(md.Path), + Size: md.Size, + Mtime: md.Mtime.Seconds, // TODO do we need nanos here? + } + return revision +} + +func (fs *eosfs) convertToResourceInfo(ctx context.Context, eosFileInfo *eosclient.FileInfo) *provider.ResourceInfo { + return fs.convert(ctx, eosFileInfo) +} + +func (fs *eosfs) convertToFileReference(ctx context.Context, eosFileInfo *eosclient.FileInfo) *provider.ResourceInfo { + info := fs.convert(ctx, eosFileInfo) + info.Type = provider.ResourceType_RESOURCE_TYPE_REFERENCE + val, ok := eosFileInfo.Attrs["user.reva.target"] + if !ok || val == "" { + panic("eos: reference does not contain target: target=" + val + " file=" + eosFileInfo.File) + } + info.Target = val + return info +} + +func (fs *eosfs) convert(ctx context.Context, eosFileInfo *eosclient.FileInfo) *provider.ResourceInfo { + path := fs.unwrap(ctx, eosFileInfo.File) + + size := eosFileInfo.Size + if eosFileInfo.IsDir { + size = eosFileInfo.TreeSize + } + + username, err := getUsername(eosFileInfo.UID) + if err != nil { + log := appctx.GetLogger(ctx) + log.Warn().Uint64("uid", eosFileInfo.UID).Msg("could not lookup userid, leaving empty") + username = "" // TODO(labkode): should we abort here? + } + + info := &provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: fmt.Sprintf("%d", eosFileInfo.Inode)}, + Path: path, + Owner: &userpb.UserId{OpaqueId: username}, + Etag: eosFileInfo.ETag, + MimeType: mime.Detect(eosFileInfo.IsDir, path), + Size: size, + PermissionSet: &provider.ResourcePermissions{ListContainer: true, CreateContainer: true}, + Mtime: &types.Timestamp{ + Seconds: eosFileInfo.MTimeSec, + Nanos: eosFileInfo.MTimeNanos, + }, + Opaque: &types.Opaque{ + Map: map[string]*types.OpaqueEntry{ + "eos": &types.OpaqueEntry{ + Decoder: "json", + Value: fs.getEosMetadata(eosFileInfo), + }, + }, + }, + } + + info.Type = getResourceType(eosFileInfo.IsDir) + return info +} + +func getResourceType(isDir bool) provider.ResourceType { + if isDir { + return provider.ResourceType_RESOURCE_TYPE_CONTAINER + } + return provider.ResourceType_RESOURCE_TYPE_FILE +} + +func getUsername(uid uint64) (string, error) { + s := strconv.FormatUint(uid, 10) + user, err := gouser.LookupId(s) + if err != nil { + return "", err + } + return user.Username, nil +} + +type eosSysMetadata struct { + TreeSize uint64 `json:"tree_size"` + TreeCount uint64 `json:"tree_count"` + File string `json:"file"` + Instance string `json:"instance"` +} + +func (fs *eosfs) getEosMetadata(finfo *eosclient.FileInfo) []byte { + sys := &eosSysMetadata{ + File: finfo.File, + Instance: finfo.Instance, + } + + if finfo.IsDir { + sys.TreeCount = finfo.TreeCount + sys.TreeSize = finfo.TreeSize + } + + v, _ := json.Marshal(sys) + return v +} + +/* + Merge shadow on requests for /home ? + + No - GetHome(ctx context.Context) (string, error) + No -CreateHome(ctx context.Context) error + No - CreateDir(ctx context.Context, fn string) error + No -Delete(ctx context.Context, ref *provider.Reference) error + No -Move(ctx context.Context, oldRef, newRef *provider.Reference) error + No -GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) + Yes -ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) + No -Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error + No -Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) + No -ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) + No -DownloadRevision(ctx context.Context, ref *provider.Reference, key string) (io.ReadCloser, error) + No -RestoreRevision(ctx context.Context, ref *provider.Reference, key string) error + No ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) + No RestoreRecycleItem(ctx context.Context, key string) error + No PurgeRecycleItem(ctx context.Context, key string) error + No EmptyRecycle(ctx context.Context) error + ? GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) + No AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error + No RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error + No UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error + No ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) + No GetQuota(ctx context.Context) (int, int, error) + No CreateReference(ctx context.Context, path string, targetURI *url.URL) error + No Shutdown(ctx context.Context) error + No SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error + No UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) error +*/ + +/* + Merge shadow on requests for /home/MyShares ? + + No - GetHome(ctx context.Context) (string, error) + No -CreateHome(ctx context.Context) error + No - CreateDir(ctx context.Context, fn string) error + Maybe -Delete(ctx context.Context, ref *provider.Reference) error + No -Move(ctx context.Context, oldRef, newRef *provider.Reference) error + Yes -GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) + Yes -ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) + No -Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error + No -Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) + No -ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) + No -DownloadRevision(ctx context.Context, ref *provider.Reference, key string) (io.ReadCloser, error) + No -RestoreRevision(ctx context.Context, ref *provider.Reference, key string) error + No ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) + No RestoreRecycleItem(ctx context.Context, key string) error + No PurgeRecycleItem(ctx context.Context, key string) error + No EmptyRecycle(ctx context.Context) error + ? GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) + No AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error + No RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error + No UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error + No ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) + No GetQuota(ctx context.Context) (int, int, error) + No CreateReference(ctx context.Context, path string, targetURI *url.URL) error + No Shutdown(ctx context.Context) error + No SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error + No UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) error +*/ + +/* + Merge shadow on requests for /home/MyShares/file-reference ? + + No - GetHome(ctx context.Context) (string, error) + No -CreateHome(ctx context.Context) error + No - CreateDir(ctx context.Context, fn string) error + Maybe -Delete(ctx context.Context, ref *provider.Reference) error + Yes -Move(ctx context.Context, oldRef, newRef *provider.Reference) error + Yes -GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) + No -ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) + No -Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error + No -Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) + No -ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) + No -DownloadRevision(ctx context.Context, ref *provider.Reference, key string) (io.ReadCloser, error) + No -RestoreRevision(ctx context.Context, ref *provider.Reference, key string) error + No ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) + No RestoreRecycleItem(ctx context.Context, key string) error + No PurgeRecycleItem(ctx context.Context, key string) error + No EmptyRecycle(ctx context.Context) error + ? GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) + No AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error + No RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error + No UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error + No ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) + No GetQuota(ctx context.Context) (int, int, error) + No CreateReference(ctx context.Context, path string, targetURI *url.URL) error + No Shutdown(ctx context.Context) error + Maybe SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error + Maybe UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) error +*/ diff --git a/pkg/storage/fs/eos/upload.go b/pkg/storage/utils/eosfs/upload.go similarity index 99% rename from pkg/storage/fs/eos/upload.go rename to pkg/storage/utils/eosfs/upload.go index f7fed5b781..f0a900d554 100644 --- a/pkg/storage/fs/eos/upload.go +++ b/pkg/storage/utils/eosfs/upload.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package eos +package eosfs import ( "context" diff --git a/pkg/storage/utils/grants/grants.go b/pkg/storage/utils/grants/grants.go new file mode 100644 index 0000000000..eadc80e7c5 --- /dev/null +++ b/pkg/storage/utils/grants/grants.go @@ -0,0 +1,137 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package grants + +import ( + "errors" + "strings" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" +) + +// GetACLPerm generates a string representation of CS3APIs' ResourcePermissions +// TODO(labkode): fine grained permission controls. +func GetACLPerm(set *provider.ResourcePermissions) (string, error) { + var b strings.Builder + + if set.Stat || set.InitiateFileDownload { + b.WriteString("r") + } + if set.CreateContainer || set.InitiateFileUpload || set.Delete || set.Move { + b.WriteString("w") + } + if set.ListContainer { + b.WriteString("x") + } + + if set.Delete { + b.WriteString("+d") + } else { + b.WriteString("!d") + } + + // TODO sharing + // TODO trash + // TODO versions + return b.String(), nil +} + +// GetGrantPermissionSet converts CSEAPIs' ResourcePermissions from a string +// TODO(labkode): add more fine grained controls. +// EOS acls are a mix of ACLs and POSIX permissions. More details can be found in +// https://github.com/cern-eos/eos/blob/master/doc/configuration/permission.rst +// TODO we need to evaluate all acls in the list at once to properly forbid (!) and overwrite (+) permissions +// This is ugly, because those are actually negative permissions ... +func GetGrantPermissionSet(mode string) *provider.ResourcePermissions { + + // TODO also check unix permissions for read access + p := &provider.ResourcePermissions{} + // r + if strings.Contains(mode, "r") { + p.Stat = true + p.InitiateFileDownload = true + } + // w + if strings.Contains(mode, "w") { + p.CreateContainer = true + p.InitiateFileUpload = true + p.Delete = true + if p.InitiateFileDownload { + p.Move = true + } + } + if strings.Contains(mode, "wo") { + p.CreateContainer = true + // p.InitiateFileUpload = false // TODO only when the file exists + p.Delete = false + } + if strings.Contains(mode, "!d") { + p.Delete = false + } else if strings.Contains(mode, "+d") { + p.Delete = true + } + // x + if strings.Contains(mode, "x") { + p.ListContainer = true + } + + // sharing + // TODO AddGrant + // TODO ListGrants + // TODO RemoveGrant + // TODO UpdateGrant + + // trash + // TODO ListRecycle + // TODO RestoreRecycleItem + // TODO PurgeRecycle + + // versions + // TODO ListFileVersions + // TODO RestoreFileVersion + + // ? + // TODO GetPath + // TODO GetQuota + return p +} + +// GetACLType returns a char representation of the type of grantee +func GetACLType(gt provider.GranteeType) (string, error) { + switch gt { + case provider.GranteeType_GRANTEE_TYPE_USER: + return "u", nil + case provider.GranteeType_GRANTEE_TYPE_GROUP: + return "g", nil + default: + return "", errors.New("no eos acl for grantee type: " + gt.String()) + } +} + +// GetGranteeType returns the grantee type from a char +func GetGranteeType(aclType string) provider.GranteeType { + switch aclType { + case "u": + return provider.GranteeType_GRANTEE_TYPE_USER + case "g": + return provider.GranteeType_GRANTEE_TYPE_GROUP + default: + return provider.GranteeType_GRANTEE_TYPE_INVALID + } +} diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go index da89d8cdf3..eeb2b07856 100644 --- a/pkg/storage/utils/localfs/localfs.go +++ b/pkg/storage/utils/localfs/localfs.go @@ -38,6 +38,7 @@ import ( "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/templates" + "github.com/cs3org/reva/pkg/storage/utils/grants" "github.com/cs3org/reva/pkg/user" "github.com/pkg/errors" ) @@ -319,64 +320,6 @@ func (fs *localfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (st return url.QueryUnescape(strings.TrimPrefix(id.OpaqueId, "fileid-")) } -func role2CS3Permissions(mode string) *provider.ResourcePermissions { - - // TODO also check unix permissions for read access - p := &provider.ResourcePermissions{} - // r - if strings.Contains(mode, "r") { - p.Stat = true - p.InitiateFileDownload = true - } - // w - if strings.Contains(mode, "w") { - p.CreateContainer = true - p.InitiateFileUpload = true - p.Delete = true - if p.InitiateFileDownload { - p.Move = true - } - } - if strings.Contains(mode, "wo") { - p.CreateContainer = true - // p.InitiateFileUpload = false // TODO only when the file exists - p.Delete = false - } - if strings.Contains(mode, "!d") { - p.Delete = false - } else if strings.Contains(mode, "+d") { - p.Delete = true - } - // x - if strings.Contains(mode, "x") { - p.ListContainer = true - } - - return p -} - -func cs3Permissions2Role(set *provider.ResourcePermissions) (string, error) { - var b strings.Builder - - if set.Stat || set.InitiateFileDownload { - b.WriteString("r") - } - if set.CreateContainer || set.InitiateFileUpload || set.Delete || set.Move { - b.WriteString("w") - } - if set.ListContainer { - b.WriteString("x") - } - - if set.Delete { - b.WriteString("+d") - } else { - b.WriteString("!d") - } - - return b.String(), nil -} - func (fs *localfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { fn, err := fs.resolve(ctx, ref) if err != nil { @@ -384,17 +327,16 @@ func (fs *localfs) AddGrant(ctx context.Context, ref *provider.Reference, g *pro } fn = fs.wrap(ctx, fn) - role, err := cs3Permissions2Role(g.Permissions) + role, err := grants.GetACLPerm(g.Permissions) if err != nil { return errors.Wrap(err, "localfs: unknown set permissions") } - var grantee string - if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { - grantee = fmt.Sprintf("g:%s@%s", g.Grantee.Id.OpaqueId, g.Grantee.Id.Idp) - } else { - grantee = fmt.Sprintf("u:%s@%s", g.Grantee.Id.OpaqueId, g.Grantee.Id.Idp) + granteeType, err := grants.GetACLType(g.Grantee.Type) + if err != nil { + return errors.Wrap(err, "localfs: error getting grantee type") } + grantee := fmt.Sprintf("%s:%s@%s", granteeType, g.Grantee.Id.OpaqueId, g.Grantee.Id.Idp) err = fs.addToACLDB(ctx, fn, grantee, role) if err != nil { @@ -411,23 +353,23 @@ func (fs *localfs) ListGrants(ctx context.Context, ref *provider.Reference) ([]* } fn = fs.wrap(ctx, fn) - grants, err := fs.getACLs(ctx, fn) + g, err := fs.getACLs(ctx, fn) if err != nil { return nil, errors.Wrap(err, "localfs: error listing grants") } var granteeID, role string var grantList []*provider.Grant - for grants.Next() { - err = grants.Scan(&granteeID, &role) + for g.Next() { + err = g.Scan(&granteeID, &role) if err != nil { return nil, errors.Wrap(err, "localfs: error scanning db rows") } grantee := &provider.Grantee{ Id: &userpb.UserId{OpaqueId: granteeID[2:]}, - Type: fs.getGranteeType(string(granteeID[0])), + Type: grants.GetGranteeType(string(granteeID[0])), } - permissions := role2CS3Permissions(role) + permissions := grants.GetGrantPermissionSet(role) grantList = append(grantList, &provider.Grant{ Grantee: grantee, @@ -438,13 +380,6 @@ func (fs *localfs) ListGrants(ctx context.Context, ref *provider.Reference) ([]* } -func (fs *localfs) getGranteeType(granteeType string) provider.GranteeType { - if granteeType == "g" { - return provider.GranteeType_GRANTEE_TYPE_GROUP - } - return provider.GranteeType_GRANTEE_TYPE_USER -} - func (fs *localfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { fn, err := fs.resolve(ctx, ref) if err != nil { @@ -452,12 +387,11 @@ func (fs *localfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g * } fn = fs.wrap(ctx, fn) - var grantee string - if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { - grantee = fmt.Sprintf("g:%s@%s", g.Grantee.Id.OpaqueId, g.Grantee.Id.Idp) - } else { - grantee = fmt.Sprintf("u:%s@%s", g.Grantee.Id.OpaqueId, g.Grantee.Id.Idp) + granteeType, err := grants.GetACLType(g.Grantee.Type) + if err != nil { + return errors.Wrap(err, "localfs: error getting grantee type") } + grantee := fmt.Sprintf("%s:%s@%s", granteeType, g.Grantee.Id.OpaqueId, g.Grantee.Id.Idp) err = fs.removeFromACLDB(ctx, fn, grantee) if err != nil { From d9afbb7c140f40c3697c6bc7c544e099eb254b64 Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Thu, 11 Jun 2020 18:16:05 +0200 Subject: [PATCH 2/3] Move templates to storage/utils --- .../http/services/owncloud/ocdav/ocdav.go | 2 +- pkg/storage/fs/owncloud/owncloud.go | 2 +- pkg/storage/utils/eosfs/eosfs.go | 2 +- pkg/storage/utils/localfs/localfs.go | 2 +- pkg/storage/utils/templates/templates.go | 93 ++++++++++++ pkg/storage/utils/templates/templates_test.go | 133 ++++++++++++++++++ 6 files changed, 230 insertions(+), 4 deletions(-) create mode 100644 pkg/storage/utils/templates/templates.go create mode 100644 pkg/storage/utils/templates/templates_test.go diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index 9c1d57b2f6..023d16b0ed 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -34,7 +34,7 @@ import ( "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/sharedconf" - "github.com/cs3org/reva/pkg/storage/templates" + "github.com/cs3org/reva/pkg/storage/utils/templates" ctxuser "github.com/cs3org/reva/pkg/user" "github.com/mitchellh/mapstructure" "github.com/rs/zerolog" diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index 499cc3a7f4..1218bb0c3c 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -41,7 +41,7 @@ import ( "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/cs3org/reva/pkg/storage/templates" + "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/cs3org/reva/pkg/user" "github.com/gofrs/uuid" "github.com/gomodule/redigo/redis" diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 7acb054140..3d5fb86eba 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -35,9 +35,9 @@ import ( "github.com/cs3org/reva/pkg/eosclient" "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/templates" "github.com/cs3org/reva/pkg/storage/utils/acl" "github.com/cs3org/reva/pkg/storage/utils/grants" + "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/cs3org/reva/pkg/user" "github.com/pkg/errors" diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go index eeb2b07856..b555ea6cd6 100644 --- a/pkg/storage/utils/localfs/localfs.go +++ b/pkg/storage/utils/localfs/localfs.go @@ -37,8 +37,8 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/templates" "github.com/cs3org/reva/pkg/storage/utils/grants" + "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/cs3org/reva/pkg/user" "github.com/pkg/errors" ) diff --git a/pkg/storage/utils/templates/templates.go b/pkg/storage/utils/templates/templates.go new file mode 100644 index 0000000000..df143a6323 --- /dev/null +++ b/pkg/storage/utils/templates/templates.go @@ -0,0 +1,93 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/* +Package templates contains data-driven templates for path layouts. + +Templates can use functions from the gitbub.com/Masterminds/sprig library. +All templates are cleaned with path.Clean(). +*/ +package templates + +import ( + "bytes" + "fmt" + "path" + "strings" + "text/template" + + "github.com/Masterminds/sprig" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + "github.com/pkg/errors" +) + +// UserData contains the template placeholders for a user. +// For example {{.Username}} or {{.Id.Idp}} +type UserData struct { + *userpb.User + Email EmailData +} + +// EmailData contains mail data +// split into local and domain part. +// It is extracted from spliting the username by @. +type EmailData struct { + Local string + Domain string +} + +// WithUser generates a layout based on user data. +func WithUser(u *userpb.User, tpl string) string { + tpl = clean(tpl) + ut := newUserData(u) + // compile given template tpl + t, err := template.New("tpl").Funcs(sprig.TxtFuncMap()).Parse(tpl) + if err != nil { + err := errors.Wrap(err, fmt.Sprintf("error parsing template: user_template:%+v tpl:%s", ut, tpl)) + panic(err) + } + b := bytes.Buffer{} + if err := t.Execute(&b, ut); err != nil { + err := errors.Wrap(err, fmt.Sprintf("error executing template: user_template:%+v tpl:%s", ut, tpl)) + panic(err) + } + return b.String() +} + +func newUserData(u *userpb.User) *UserData { + usernameSplit := strings.Split(u.Username, "@") + if len(usernameSplit) == 1 { + usernameSplit = append(usernameSplit, "_unknown") + } + if usernameSplit[1] == "" { + usernameSplit[1] = "_unknown" + } + + ut := &UserData{ + User: u, + Email: EmailData{ + Local: strings.ToLower(usernameSplit[0]), + Domain: strings.ToLower(usernameSplit[1]), + }, + } + return ut +} + +func clean(a string) string { + return path.Clean(a) +} diff --git a/pkg/storage/utils/templates/templates_test.go b/pkg/storage/utils/templates/templates_test.go new file mode 100644 index 0000000000..1e5c55ed59 --- /dev/null +++ b/pkg/storage/utils/templates/templates_test.go @@ -0,0 +1,133 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package templates + +import ( + "testing" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" +) + +type testUnit struct { + expected string + template string + user *userpb.User +} + +var tests = []*testUnit{ + &testUnit{ + expected: "alabasta", + user: &userpb.User{ + Username: "alabasta", + }, + template: "{{.Username}}", + }, + &testUnit{ + expected: "a/alabasta", + user: &userpb.User{ + Username: "alabasta", + }, + template: "{{substr 0 1 .Username}}/{{.Username}}", + }, + &testUnit{ + expected: "idp@opaque", + user: &userpb.User{ + Id: &userpb.UserId{ + Idp: "idp", + OpaqueId: "opaque", + }, + }, + template: "{{.Id.Idp}}@{{.Id.OpaqueId}}", + }, + &testUnit{ // test path clean + expected: "/alabasta", + user: &userpb.User{ + Username: "alabasta", + }, + template: "///{{.Username}}", + }, + &testUnit{ + expected: "michael", + user: &userpb.User{ + Username: "MICHAEL", + }, + template: "{{lower .Username}}", + }, + &testUnit{ + expected: "somewhere.com/michael@somewhere.com", + user: &userpb.User{ + Username: "michael@somewhere.com", + }, + template: "{{.Email.Domain}}/{{.Username}}", + }, + &testUnit{ + expected: "somewhere.com/michael", + user: &userpb.User{ + Username: "michael@somewhere.com", + }, + template: "{{.Email.Domain}}/{{.Email.Local}}", + }, + &testUnit{ + expected: "_unknown/michael", + user: &userpb.User{ + Username: "michael", + }, + template: "{{.Email.Domain}}/{{.Username}}", + }, +} + +func TestLayout(t *testing.T) { + for _, u := range tests { + got := WithUser(u.user, u.template) + if u.expected != got { + t.Fatal("expected: " + u.expected + " got: " + got) + } + } +} + +func TestLayoutPanic(t *testing.T) { + assertPanic(t, testBadLayout) +} + +func TestUserPanic(t *testing.T) { + assertPanic(t, testBadUser) +} + +// should panic +func testBadLayout() { + layout := "{{ bad layout sintax" + user := &userpb.User{} + WithUser(user, layout) +} + +//should panic +func testBadUser() { + layout := "{{ .DoesNotExist }}" + user := &userpb.User{} + WithUser(user, layout) +} + +func assertPanic(t *testing.T, f func()) { + defer func() { + if r := recover(); r == nil { + t.Errorf("the code did not panic") + } + }() + f() +} From e63c4e2a7990c02e083bf91ce2123c2ad8f66a2d Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Thu, 11 Jun 2020 18:45:47 +0200 Subject: [PATCH 3/3] Remove duplicate files --- pkg/storage/templates/templates.go | 93 ----------------- pkg/storage/templates/templates_test.go | 133 ------------------------ 2 files changed, 226 deletions(-) delete mode 100644 pkg/storage/templates/templates.go delete mode 100644 pkg/storage/templates/templates_test.go diff --git a/pkg/storage/templates/templates.go b/pkg/storage/templates/templates.go deleted file mode 100644 index df143a6323..0000000000 --- a/pkg/storage/templates/templates.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/* -Package templates contains data-driven templates for path layouts. - -Templates can use functions from the gitbub.com/Masterminds/sprig library. -All templates are cleaned with path.Clean(). -*/ -package templates - -import ( - "bytes" - "fmt" - "path" - "strings" - "text/template" - - "github.com/Masterminds/sprig" - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - "github.com/pkg/errors" -) - -// UserData contains the template placeholders for a user. -// For example {{.Username}} or {{.Id.Idp}} -type UserData struct { - *userpb.User - Email EmailData -} - -// EmailData contains mail data -// split into local and domain part. -// It is extracted from spliting the username by @. -type EmailData struct { - Local string - Domain string -} - -// WithUser generates a layout based on user data. -func WithUser(u *userpb.User, tpl string) string { - tpl = clean(tpl) - ut := newUserData(u) - // compile given template tpl - t, err := template.New("tpl").Funcs(sprig.TxtFuncMap()).Parse(tpl) - if err != nil { - err := errors.Wrap(err, fmt.Sprintf("error parsing template: user_template:%+v tpl:%s", ut, tpl)) - panic(err) - } - b := bytes.Buffer{} - if err := t.Execute(&b, ut); err != nil { - err := errors.Wrap(err, fmt.Sprintf("error executing template: user_template:%+v tpl:%s", ut, tpl)) - panic(err) - } - return b.String() -} - -func newUserData(u *userpb.User) *UserData { - usernameSplit := strings.Split(u.Username, "@") - if len(usernameSplit) == 1 { - usernameSplit = append(usernameSplit, "_unknown") - } - if usernameSplit[1] == "" { - usernameSplit[1] = "_unknown" - } - - ut := &UserData{ - User: u, - Email: EmailData{ - Local: strings.ToLower(usernameSplit[0]), - Domain: strings.ToLower(usernameSplit[1]), - }, - } - return ut -} - -func clean(a string) string { - return path.Clean(a) -} diff --git a/pkg/storage/templates/templates_test.go b/pkg/storage/templates/templates_test.go deleted file mode 100644 index 1e5c55ed59..0000000000 --- a/pkg/storage/templates/templates_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package templates - -import ( - "testing" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" -) - -type testUnit struct { - expected string - template string - user *userpb.User -} - -var tests = []*testUnit{ - &testUnit{ - expected: "alabasta", - user: &userpb.User{ - Username: "alabasta", - }, - template: "{{.Username}}", - }, - &testUnit{ - expected: "a/alabasta", - user: &userpb.User{ - Username: "alabasta", - }, - template: "{{substr 0 1 .Username}}/{{.Username}}", - }, - &testUnit{ - expected: "idp@opaque", - user: &userpb.User{ - Id: &userpb.UserId{ - Idp: "idp", - OpaqueId: "opaque", - }, - }, - template: "{{.Id.Idp}}@{{.Id.OpaqueId}}", - }, - &testUnit{ // test path clean - expected: "/alabasta", - user: &userpb.User{ - Username: "alabasta", - }, - template: "///{{.Username}}", - }, - &testUnit{ - expected: "michael", - user: &userpb.User{ - Username: "MICHAEL", - }, - template: "{{lower .Username}}", - }, - &testUnit{ - expected: "somewhere.com/michael@somewhere.com", - user: &userpb.User{ - Username: "michael@somewhere.com", - }, - template: "{{.Email.Domain}}/{{.Username}}", - }, - &testUnit{ - expected: "somewhere.com/michael", - user: &userpb.User{ - Username: "michael@somewhere.com", - }, - template: "{{.Email.Domain}}/{{.Email.Local}}", - }, - &testUnit{ - expected: "_unknown/michael", - user: &userpb.User{ - Username: "michael", - }, - template: "{{.Email.Domain}}/{{.Username}}", - }, -} - -func TestLayout(t *testing.T) { - for _, u := range tests { - got := WithUser(u.user, u.template) - if u.expected != got { - t.Fatal("expected: " + u.expected + " got: " + got) - } - } -} - -func TestLayoutPanic(t *testing.T) { - assertPanic(t, testBadLayout) -} - -func TestUserPanic(t *testing.T) { - assertPanic(t, testBadUser) -} - -// should panic -func testBadLayout() { - layout := "{{ bad layout sintax" - user := &userpb.User{} - WithUser(user, layout) -} - -//should panic -func testBadUser() { - layout := "{{ .DoesNotExist }}" - user := &userpb.User{} - WithUser(user, layout) -} - -func assertPanic(t *testing.T, f func()) { - defer func() { - if r := recover(); r == nil { - t.Errorf("the code did not panic") - } - }() - f() -}