Externalize custom mime types for storage providers

changelog/unreleased/mimetypes-config.md

@@ -0,0 +1,5 @@
+Enhancement: Externalize custom mime types configuration for storage providers
+
+Added ability to configure custom mime types in an external JSON file, such that it can be reused when many storage providers are deployed at the same time.
+
+https://github.com/cs3org/reva/pull/2613

docs/content/en/docs/config/grpc/services/storageprovider/_index.md

@@ -9,31 +9,31 @@ description: >
 # _struct: config_
 
 {{% dir name="mount_path" type="string" default="/" %}}
-The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L56)
+The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L58)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 mount_path = "/"
 {{< /highlight >}}
 {{% /dir %}}
 
 {{% dir name="mount_id" type="string" default="-" %}}
-The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L57)
+The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L59)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 mount_id = "-"
 {{< /highlight >}}
 {{% /dir %}}
 
 {{% dir name="driver" type="string" default="localhome" %}}
-The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L58)
+The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L60)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 driver = "localhome"
 {{< /highlight >}}
 {{% /dir %}}
 
 {{% dir name="drivers" type="map[string]map[string]interface{}" default="localhome" %}}
- [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L59)
+ [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L61)
 {{< highlight toml >}}
 [grpc.services.storageprovider.drivers.localhome]
 root = "/var/tmp/reva/"
@@ -44,42 +44,42 @@ user_layout = "{{.Username}}"
 {{% /dir %}}
 
 {{% dir name="tmp_folder" type="string" default="/var/tmp" %}}
-Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L60)
+Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L62)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 tmp_folder = "/var/tmp"
 {{< /highlight >}}
 {{% /dir %}}
 
 {{% dir name="data_server_url" type="string" default="http://localhost/data" %}}
-The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L61)
+The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 data_server_url = "http://localhost/data"
 {{< /highlight >}}
 {{% /dir %}}
 
 {{% dir name="expose_data_server" type="bool" default=false %}}
-Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L62)
+Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 expose_data_server = false
 {{< /highlight >}}
 {{% /dir %}}
 
 {{% dir name="available_checksums" type="map[string]uint32" default=nil %}}
-List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63)
+List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L65)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 available_checksums = nil
 {{< /highlight >}}
 {{% /dir %}}
 
-{{% dir name="mimetypes" type="map[string]string" default=nil %}}
-List of supported mime types and corresponding file extensions. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64)
+{{% dir name="custom_mimetypes" type="string" default="nil" %}}
+An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L66)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
-mimetypes = nil
+custom_mimetypes = "nil"
 {{< /highlight >}}
 {{% /dir %}}
 

examples/nextcloud-integration/custom-mime-types-demo.json

@@ -0,0 +1,10 @@
+[
+    {
+        "file_ext": ".zmd",
+        "mime_type": "application/compressed-markdown"
+    },
+    {
+        "file_ext": ".zep",
+        "mime_type": "application/compressed-etherpad"
+    }
+]

examples/nextcloud-integration/revad.toml

@@ -90,9 +90,7 @@ expose_data_server = true
 data_server_url = "http://127.0.0.1:19001/data"
 enable_home_creation = true
 disable_tus = true
-
-[grpc.services.storageprovider.mimetypes]
-".zmd" = "application/compressed-markdown"
+custom_mime_types = "custom-mime-types-demo.json"
 
 [grpc.services.storageprovider.drivers.nextcloud]
 end_point = "http://localhost/apps/sciencemesh/"

examples/ocmd/ocmd-server-1.toml

@@ -97,9 +97,6 @@ expose_data_server = true
 data_server_url = "http://localhost:19001/data"
 enable_home_creation = true
 
-[grpc.services.storageprovider.mimetypes]
-".zmd" = "application/compressed-markdown"
-
 [grpc.services.storageprovider.drivers.localhome]
 user_layout = "{{.Username}}"
 

internal/grpc/services/storageprovider/storageprovider.go

@@ -20,7 +20,9 @@ package storageprovider
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"path"
@@ -61,7 +63,12 @@ type config struct {
 	DataServerURL    string                            `mapstructure:"data_server_url" docs:"http://localhost/data;The URL for the data server."`
 	ExposeDataServer bool                              `mapstructure:"expose_data_server" docs:"false;Whether to expose data server."` // if true the client will be able to upload/download directly to it
 	AvailableXS      map[string]uint32                 `mapstructure:"available_checksums" docs:"nil;List of available checksums."`
-	MimeTypes        map[string]string                 `mapstructure:"mimetypes" docs:"nil;List of supported mime types and corresponding file extensions."`
+	CustomMimeTypes  string                            `mapstructure:"custom_mimetypes" docs:"nil;An optional mapping file with the list of supported custom file extensions and corresponding mime types."`
+}
+
+type mimeType struct {
+	FileExt  string `mapstructure:"file_ext" json:"file_ext"`
+	MimeType string `mapstructure:"mime_type" json:"mime_type"`
 }
 
 func (c *config) init() {
@@ -94,10 +101,6 @@ func (c *config) init() {
 	if len(c.AvailableXS) == 0 {
 		c.AvailableXS = map[string]uint32{"md5": 100, "unset": 1000}
 	}
-	if c.MimeTypes == nil || len(c.MimeTypes) == 0 {
-		c.MimeTypes = map[string]string{".zmd": "application/compressed-markdown"}
-	}
-
 }
 
 type service struct {
@@ -144,6 +147,34 @@ func parseConfig(m map[string]interface{}) (*config, error) {
 	return c, nil
 }
 
+func registerMimeTypes(mappingFile string) error {
+	mimeTypes := map[string]string{}
+
+	if mappingFile != "" {
+		f, err := ioutil.ReadFile(mappingFile)
+		if err != nil {
+			return fmt.Errorf("storageprovider: error reading the custom mime types file: +%v", err)
+		}
+		mimes := []*mimeType{}
+		err = json.Unmarshal(f, &mimes)
+		if err != nil {
+			return fmt.Errorf("storageprovider: error unmarshalling the custom mime types file: +%v", err)
+		}
+		for _, m := range mimes {
+			if _, found := mimeTypes[m.FileExt]; found {
+				return fmt.Errorf("storageprovider: mimetypes mapping error, file extension \"%s\" is mapped to multiple mime types", m.FileExt)
+			}
+			mimeTypes[m.FileExt] = m.MimeType
+		}
+
+		// now register all mime types that were read
+		for k, v := range mimeTypes {
+			mime.RegisterMime(k, v)
+		}
+	}
+	return nil
+}
+
 // New creates a new storage provider svc
 func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
 
@@ -182,7 +213,11 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
 		return nil, errtypes.NotFound("no available checksum, please set in config")
 	}
 
-	registerMimeTypes(c.MimeTypes)
+	// read and register custom mime types if configured
+	err = registerMimeTypes(c.CustomMimeTypes)
+	if err != nil {
+		return nil, err
+	}
 
 	service := &service{
 		conf:          c,
@@ -197,12 +232,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
 	return service, nil
 }
 
-func registerMimeTypes(mimes map[string]string) {
-	for k, v := range mimes {
-		mime.RegisterMime(k, v)
-	}
-}
-
 func (s *service) SetArbitraryMetadata(ctx context.Context, req *provider.SetArbitraryMetadataRequest) (*provider.SetArbitraryMetadataResponse, error) {
 	newRef, err := s.unwrap(ctx, req.Ref)
 	if err != nil {