From 9550f678e563d079821bf81d1e714e7e84d1af03 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 3 Mar 2022 15:01:46 +0100 Subject: [PATCH] Externalize custom mime types for storage providers --- changelog/unreleased/mimetypes-config.md | 5 ++ .../grpc/services/storageprovider/_index.md | 22 ++++---- .../custom-mime-types-demo.json | 10 ++++ examples/nextcloud-integration/revad.toml | 4 +- examples/ocmd/ocmd-server-1.toml | 3 -- .../storageprovider/storageprovider.go | 53 ++++++++++++++----- 6 files changed, 68 insertions(+), 29 deletions(-) create mode 100644 changelog/unreleased/mimetypes-config.md create mode 100644 examples/nextcloud-integration/custom-mime-types-demo.json diff --git a/changelog/unreleased/mimetypes-config.md b/changelog/unreleased/mimetypes-config.md new file mode 100644 index 0000000000..680115abaa --- /dev/null +++ b/changelog/unreleased/mimetypes-config.md @@ -0,0 +1,5 @@ +Enhancement: Externalize custom mime types configuration for storage providers + +Added ability to configure custom mime types in an external JSON file, such that it can be reused when many storage providers are deployed at the same time. + +https://github.com/cs3org/reva/pull/2613 diff --git a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md index 728c382f59..fbea4db9c4 100644 --- a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md +++ b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="mount_path" type="string" default="/" %}} -The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L56) +The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L58) {{< highlight toml >}} [grpc.services.storageprovider] mount_path = "/" @@ -17,7 +17,7 @@ mount_path = "/" {{% /dir %}} {{% dir name="mount_id" type="string" default="-" %}} -The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L57) +The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L59) {{< highlight toml >}} [grpc.services.storageprovider] mount_id = "-" @@ -25,7 +25,7 @@ mount_id = "-" {{% /dir %}} {{% dir name="driver" type="string" default="localhome" %}} -The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L58) +The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L60) {{< highlight toml >}} [grpc.services.storageprovider] driver = "localhome" @@ -33,7 +33,7 @@ driver = "localhome" {{% /dir %}} {{% dir name="drivers" type="map[string]map[string]interface{}" default="localhome" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L59) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L61) {{< highlight toml >}} [grpc.services.storageprovider.drivers.localhome] root = "/var/tmp/reva/" @@ -44,7 +44,7 @@ user_layout = "{{.Username}}" {{% /dir %}} {{% dir name="tmp_folder" type="string" default="/var/tmp" %}} -Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L60) +Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L62) {{< highlight toml >}} [grpc.services.storageprovider] tmp_folder = "/var/tmp" @@ -52,7 +52,7 @@ tmp_folder = "/var/tmp" {{% /dir %}} {{% dir name="data_server_url" type="string" default="http://localhost/data" %}} -The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L61) +The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63) {{< highlight toml >}} [grpc.services.storageprovider] data_server_url = "http://localhost/data" @@ -60,7 +60,7 @@ data_server_url = "http://localhost/data" {{% /dir %}} {{% dir name="expose_data_server" type="bool" default=false %}} -Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L62) +Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64) {{< highlight toml >}} [grpc.services.storageprovider] expose_data_server = false @@ -68,18 +68,18 @@ expose_data_server = false {{% /dir %}} {{% dir name="available_checksums" type="map[string]uint32" default=nil %}} -List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63) +List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L65) {{< highlight toml >}} [grpc.services.storageprovider] available_checksums = nil {{< /highlight >}} {{% /dir %}} -{{% dir name="mimetypes" type="map[string]string" default=nil %}} -List of supported mime types and corresponding file extensions. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64) +{{% dir name="custom_mimetypes" type="string" default="nil" %}} +An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L66) {{< highlight toml >}} [grpc.services.storageprovider] -mimetypes = nil +custom_mimetypes = "nil" {{< /highlight >}} {{% /dir %}} diff --git a/examples/nextcloud-integration/custom-mime-types-demo.json b/examples/nextcloud-integration/custom-mime-types-demo.json new file mode 100644 index 0000000000..828af503d9 --- /dev/null +++ b/examples/nextcloud-integration/custom-mime-types-demo.json @@ -0,0 +1,10 @@ +[ + { + "file_ext": ".zmd", + "mime_type": "application/compressed-markdown" + }, + { + "file_ext": ".zep", + "mime_type": "application/compressed-etherpad" + } +] diff --git a/examples/nextcloud-integration/revad.toml b/examples/nextcloud-integration/revad.toml index 6c9b0c0615..77bf14be86 100644 --- a/examples/nextcloud-integration/revad.toml +++ b/examples/nextcloud-integration/revad.toml @@ -90,9 +90,7 @@ expose_data_server = true data_server_url = "http://127.0.0.1:19001/data" enable_home_creation = true disable_tus = true - -[grpc.services.storageprovider.mimetypes] -".zmd" = "application/compressed-markdown" +custom_mime_types = "custom-mime-types-demo.json" [grpc.services.storageprovider.drivers.nextcloud] end_point = "http://localhost/apps/sciencemesh/" diff --git a/examples/ocmd/ocmd-server-1.toml b/examples/ocmd/ocmd-server-1.toml index dad77ce087..2e41678afb 100644 --- a/examples/ocmd/ocmd-server-1.toml +++ b/examples/ocmd/ocmd-server-1.toml @@ -97,9 +97,6 @@ expose_data_server = true data_server_url = "http://localhost:19001/data" enable_home_creation = true -[grpc.services.storageprovider.mimetypes] -".zmd" = "application/compressed-markdown" - [grpc.services.storageprovider.drivers.localhome] user_layout = "{{.Username}}" diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index fd72dbcb7c..2b1fdd5f46 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -20,7 +20,9 @@ package storageprovider import ( "context" + "encoding/json" "fmt" + "io/ioutil" "net/url" "os" "path" @@ -61,7 +63,12 @@ type config struct { DataServerURL string `mapstructure:"data_server_url" docs:"http://localhost/data;The URL for the data server."` ExposeDataServer bool `mapstructure:"expose_data_server" docs:"false;Whether to expose data server."` // if true the client will be able to upload/download directly to it AvailableXS map[string]uint32 `mapstructure:"available_checksums" docs:"nil;List of available checksums."` - MimeTypes map[string]string `mapstructure:"mimetypes" docs:"nil;List of supported mime types and corresponding file extensions."` + CustomMimeTypes string `mapstructure:"custom_mimetypes" docs:"nil;An optional mapping file with the list of supported custom file extensions and corresponding mime types."` +} + +type mimeType struct { + FileExt string `mapstructure:"file_ext" json:"file_ext"` + MimeType string `mapstructure:"mime_type" json:"mime_type"` } func (c *config) init() { @@ -94,10 +101,6 @@ func (c *config) init() { if len(c.AvailableXS) == 0 { c.AvailableXS = map[string]uint32{"md5": 100, "unset": 1000} } - if c.MimeTypes == nil || len(c.MimeTypes) == 0 { - c.MimeTypes = map[string]string{".zmd": "application/compressed-markdown"} - } - } type service struct { @@ -144,6 +147,34 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } +func registerMimeTypes(mappingFile string) error { + mimeTypes := map[string]string{} + + if mappingFile != "" { + f, err := ioutil.ReadFile(mappingFile) + if err != nil { + return fmt.Errorf("storageprovider: error reading the custom mime types file: +%v", err) + } + mimes := []*mimeType{} + err = json.Unmarshal(f, &mimes) + if err != nil { + return fmt.Errorf("storageprovider: error unmarshalling the custom mime types file: +%v", err) + } + for _, m := range mimes { + if _, found := mimeTypes[m.FileExt]; found { + return fmt.Errorf("storageprovider: mimetypes mapping error, file extension \"%s\" is mapped to multiple mime types", m.FileExt) + } + mimeTypes[m.FileExt] = m.MimeType + } + + // now register all mime types that were read + for k, v := range mimeTypes { + mime.RegisterMime(k, v) + } + } + return nil +} + // New creates a new storage provider svc func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { @@ -182,7 +213,11 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return nil, errtypes.NotFound("no available checksum, please set in config") } - registerMimeTypes(c.MimeTypes) + // read and register custom mime types if configured + err = registerMimeTypes(c.CustomMimeTypes) + if err != nil { + return nil, err + } service := &service{ conf: c, @@ -197,12 +232,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return service, nil } -func registerMimeTypes(mimes map[string]string) { - for k, v := range mimes { - mime.RegisterMime(k, v) - } -} - func (s *service) SetArbitraryMetadata(ctx context.Context, req *provider.SetArbitraryMetadataRequest) (*provider.SetArbitraryMetadataResponse, error) { newRef, err := s.unwrap(ctx, req.Ref) if err != nil {