From 6c6cb64364ebdf5da83313b8fb3b963b02af1a6e Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 11 Oct 2021 11:38:34 +0200 Subject: [PATCH] remove uneeded storages for metadata, add missing flagsets --- .../oc10_ocis_parallel/docker-compose.yml | 1 - storage/pkg/command/storagedrivers/home.go | 11 ++- .../pkg/command/storagedrivers/metadata.go | 50 +----------- .../pkg/flagset/metadatadrivers/drivereos.go | 59 +++++++------- .../flagset/metadatadrivers/driverowncloud.go | 55 -------------- .../metadatadrivers/driverowncloudsql.go | 76 ------------------- .../pkg/flagset/metadatadrivers/drivers3.go | 48 ++++++++++++ storage/pkg/flagset/storagehome.go | 1 + storage/pkg/flagset/storagemetadata.go | 2 +- storage/pkg/flagset/storageusers.go | 1 + storage/pkg/flagset/userdrivers/drivereos.go | 13 ++-- .../pkg/flagset/userdrivers/driverlocal.go | 14 ++++ storage/pkg/flagset/userdrivers/driverocis.go | 7 -- .../pkg/flagset/userdrivers/driverowncloud.go | 7 -- .../flagset/userdrivers/driverowncloudsql.go | 7 -- storage/pkg/flagset/userdrivers/drivers3.go | 48 ++++++++++++ storage/pkg/flagset/userdrivers/drivers3ng.go | 9 +-- 17 files changed, 162 insertions(+), 247 deletions(-) delete mode 100644 storage/pkg/flagset/metadatadrivers/driverowncloud.go delete mode 100644 storage/pkg/flagset/metadatadrivers/driverowncloudsql.go create mode 100644 storage/pkg/flagset/metadatadrivers/drivers3.go create mode 100644 storage/pkg/flagset/userdrivers/drivers3.go diff --git a/deployments/examples/oc10_ocis_parallel/docker-compose.yml b/deployments/examples/oc10_ocis_parallel/docker-compose.yml index 1ec6b5e9578..0440ac42592 100644 --- a/deployments/examples/oc10_ocis_parallel/docker-compose.yml +++ b/deployments/examples/oc10_ocis_parallel/docker-compose.yml @@ -96,7 +96,6 @@ services: STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR: /mnt/data/files STORAGE_USERS_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR: /tmp STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER: "/Shares" - STORAGE_USERS_DRIVER_OWNCLOUDSQL_ENABLE_HOME: "false" STORAGE_USERS_DRIVER_OWNCLOUDSQL_LAYOUT: "{{.Username}}" STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBUSERNAME: owncloud STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPASSWORD: owncloud diff --git a/storage/pkg/command/storagedrivers/home.go b/storage/pkg/command/storagedrivers/home.go index cdd4968a6d4..ccac2f85d1f 100644 --- a/storage/pkg/command/storagedrivers/home.go +++ b/storage/pkg/command/storagedrivers/home.go @@ -63,7 +63,7 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} { "show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles, "force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode, "use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab, - "enable_home": cfg.Reva.UserStorage.EOS.EnableHome, + "enable_home": true, "gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC, }, "local": map[string]interface{}{ @@ -81,7 +81,7 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} { "share_folder": cfg.Reva.UserStorage.OwnCloud.ShareFolder, "user_layout": cfg.Reva.UserStorage.OwnCloud.UserLayout, "redis": cfg.Reva.UserStorage.OwnCloud.Redis, - "enable_home": cfg.Reva.UserStorage.OwnCloud.EnableHome, + "enable_home": true, "scan": cfg.Reva.UserStorage.OwnCloud.Scan, "userprovidersvc": cfg.Reva.Users.Endpoint, }, @@ -90,7 +90,7 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} { "upload_info_dir": cfg.Reva.UserStorage.OwnCloudSQL.UploadInfoDir, "share_folder": cfg.Reva.UserStorage.OwnCloudSQL.ShareFolder, "user_layout": cfg.Reva.UserStorage.OwnCloudSQL.UserLayout, - "enable_home": cfg.Reva.UserStorage.OwnCloudSQL.EnableHome, + "enable_home": true, "dbusername": cfg.Reva.UserStorage.OwnCloudSQL.DBUsername, "dbpassword": cfg.Reva.UserStorage.OwnCloudSQL.DBPassword, "dbhost": cfg.Reva.UserStorage.OwnCloudSQL.DBHost, @@ -100,7 +100,7 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} { }, "ocis": map[string]interface{}{ "root": cfg.Reva.UserStorage.OCIS.Root, - "enable_home": cfg.Reva.UserStorage.OCIS.EnableHome, + "enable_home": true, "user_layout": cfg.Reva.UserStorage.OCIS.UserLayout, "share_folder": cfg.Reva.UserStorage.OCIS.ShareFolder, "treetime_accounting": true, @@ -113,11 +113,10 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} { "secret_key": cfg.Reva.UserStorage.S3.SecretKey, "endpoint": cfg.Reva.UserStorage.S3.Endpoint, "bucket": cfg.Reva.UserStorage.S3.Bucket, - "prefix": cfg.Reva.UserStorage.S3.Root, }, "s3ng": map[string]interface{}{ "root": cfg.Reva.UserStorage.S3NG.Root, - "enable_home": cfg.Reva.UserStorage.S3NG.EnableHome, + "enable_home": true, "user_layout": cfg.Reva.UserStorage.S3NG.UserLayout, "share_folder": cfg.Reva.UserStorage.S3NG.ShareFolder, "s3.region": cfg.Reva.UserStorage.S3NG.Region, diff --git a/storage/pkg/command/storagedrivers/metadata.go b/storage/pkg/command/storagedrivers/metadata.go index 3438c7043e1..578b02fc71f 100644 --- a/storage/pkg/command/storagedrivers/metadata.go +++ b/storage/pkg/command/storagedrivers/metadata.go @@ -23,25 +23,7 @@ func MetadataDrivers(cfg *config.Config) map[string]interface{} { "force_single_user_mode": cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode, "use_keytab": cfg.Reva.MetadataStorage.EOS.UseKeytab, "gatewaysvc": cfg.Reva.MetadataStorage.EOS.GatewaySVC, - }, - "eoshome": map[string]interface{}{ - "namespace": cfg.Reva.MetadataStorage.EOS.Root, - "shadow_namespace": cfg.Reva.MetadataStorage.EOS.ShadowNamespace, - "uploads_namespace": cfg.Reva.MetadataStorage.EOS.UploadsNamespace, - "eos_binary": cfg.Reva.MetadataStorage.EOS.EosBinary, - "xrdcopy_binary": cfg.Reva.MetadataStorage.EOS.XrdcopyBinary, - "master_url": cfg.Reva.MetadataStorage.EOS.MasterURL, - "slave_url": cfg.Reva.MetadataStorage.EOS.SlaveURL, - "cache_directory": cfg.Reva.MetadataStorage.EOS.CacheDirectory, - "sec_protocol": cfg.Reva.MetadataStorage.EOS.SecProtocol, - "keytab": cfg.Reva.MetadataStorage.EOS.Keytab, - "single_username": cfg.Reva.MetadataStorage.EOS.SingleUsername, - "user_layout": cfg.Reva.MetadataStorage.EOS.UserLayout, - "enable_logging": cfg.Reva.MetadataStorage.EOS.EnableLogging, - "show_hidden_sys_files": cfg.Reva.MetadataStorage.EOS.ShowHiddenSysFiles, - "force_single_user_mode": cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode, - "use_keytab": cfg.Reva.MetadataStorage.EOS.UseKeytab, - "gatewaysvc": cfg.Reva.MetadataStorage.EOS.GatewaySVC, + "enable_home": false, }, "eosgrpc": map[string]interface{}{ "namespace": cfg.Reva.MetadataStorage.EOS.Root, @@ -66,37 +48,12 @@ func MetadataDrivers(cfg *config.Config) map[string]interface{} { "local": map[string]interface{}{ "root": cfg.Reva.MetadataStorage.Local.Root, }, - "localhome": map[string]interface{}{ - "root": cfg.Reva.MetadataStorage.Local.Root, - "user_layout": cfg.Reva.MetadataStorage.Local.UserLayout, - }, - "owncloud": map[string]interface{}{ - "datadirectory": cfg.Reva.MetadataStorage.OwnCloud.Root, - "upload_info_dir": cfg.Reva.MetadataStorage.OwnCloud.UploadInfoDir, - "user_layout": cfg.Reva.MetadataStorage.OwnCloud.UserLayout, - "redis": cfg.Reva.MetadataStorage.OwnCloud.Redis, - "enable_home": false, - "scan": cfg.Reva.MetadataStorage.OwnCloud.Scan, - "userprovidersvc": cfg.Reva.Users.Endpoint, - }, - "owncloudsql": map[string]interface{}{ - "datadirectory": cfg.Reva.MetadataStorage.OwnCloudSQL.Root, - "upload_info_dir": cfg.Reva.MetadataStorage.OwnCloudSQL.UploadInfoDir, - "user_layout": cfg.Reva.MetadataStorage.OwnCloudSQL.UserLayout, - "enable_home": false, - "dbusername": cfg.Reva.MetadataStorage.OwnCloudSQL.DBUsername, - "dbpassword": cfg.Reva.MetadataStorage.OwnCloudSQL.DBPassword, - "dbhost": cfg.Reva.MetadataStorage.OwnCloudSQL.DBHost, - "dbport": cfg.Reva.MetadataStorage.OwnCloudSQL.DBPort, - "dbname": cfg.Reva.MetadataStorage.OwnCloudSQL.DBName, - "userprovidersvc": cfg.Reva.Users.Endpoint, - }, "ocis": map[string]interface{}{ "root": cfg.Reva.MetadataStorage.OCIS.Root, "enable_home": false, "user_layout": cfg.Reva.MetadataStorage.OCIS.UserLayout, - "treetime_accounting": true, - "treesize_accounting": true, + "treetime_accounting": false, + "treesize_accounting": false, "owner": cfg.Reva.MetadataStorage.OCIS.ServiceUserUUID, // the accounts service system account uuid }, "s3": map[string]interface{}{ @@ -105,7 +62,6 @@ func MetadataDrivers(cfg *config.Config) map[string]interface{} { "secret_key": cfg.Reva.MetadataStorage.S3.SecretKey, "endpoint": cfg.Reva.MetadataStorage.S3.Endpoint, "bucket": cfg.Reva.MetadataStorage.S3.Bucket, - "prefix": cfg.Reva.MetadataStorage.S3.Root, }, "s3ng": map[string]interface{}{ "root": cfg.Reva.MetadataStorage.S3NG.Root, diff --git a/storage/pkg/flagset/metadatadrivers/drivereos.go b/storage/pkg/flagset/metadatadrivers/drivereos.go index ec66e4200a2..c734e6c1294 100644 --- a/storage/pkg/flagset/metadatadrivers/drivereos.go +++ b/storage/pkg/flagset/metadatadrivers/drivereos.go @@ -15,115 +15,122 @@ func DriverEOSWithConfig(cfg *config.Config) []cli.Flag { &cli.StringFlag{ Name: "storage-eos-namespace", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.Root, "/eos/dockertest/reva"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.Root, "/eos/dockertest/reva"), Usage: "Namespace for metadata operations", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_NAMESPACE"}, - Destination: &cfg.Reva.UserStorage.EOS.Root, + Destination: &cfg.Reva.MetadataStorage.EOS.Root, }, &cli.StringFlag{ Name: "storage-eos-shadow-namespace", // Defaults to path.Join(c.Namespace, ".shadow") Usage: "Shadow namespace where share references are stored", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SHADOW_NAMESPACE"}, - Destination: &cfg.Reva.UserStorage.EOS.ShadowNamespace, + Destination: &cfg.Reva.MetadataStorage.EOS.ShadowNamespace, + }, + &cli.StringFlag{ + Name: "storage-eos-uploads-namespace", + // Defaults to path.Join(c.Namespace, ".uploads") + Usage: "Uploads namespace", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_UPLOADS_NAMESPACE"}, + Destination: &cfg.Reva.MetadataStorage.EOS.UploadsNamespace, }, &cli.StringFlag{ Name: "storage-eos-share-folder", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.ShareFolder, "/Shares"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.ShareFolder, "/Shares"), Usage: "name of the share folder", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SHARE_FOLDER"}, - Destination: &cfg.Reva.UserStorage.EOS.ShareFolder, + Destination: &cfg.Reva.MetadataStorage.EOS.ShareFolder, }, &cli.StringFlag{ Name: "storage-eos-binary", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.EosBinary, "/usr/bin/eos"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.EosBinary, "/usr/bin/eos"), Usage: "Location of the eos binary", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_BINARY"}, - Destination: &cfg.Reva.UserStorage.EOS.EosBinary, + Destination: &cfg.Reva.MetadataStorage.EOS.EosBinary, }, &cli.StringFlag{ Name: "storage-eos-xrdcopy-binary", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.XrdcopyBinary, "/usr/bin/xrdcopy"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.XrdcopyBinary, "/usr/bin/xrdcopy"), Usage: "Location of the xrdcopy binary", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_XRDCOPY_BINARY"}, - Destination: &cfg.Reva.UserStorage.EOS.XrdcopyBinary, + Destination: &cfg.Reva.MetadataStorage.EOS.XrdcopyBinary, }, &cli.StringFlag{ Name: "storage-eos-master-url", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.MasterURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.MasterURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), Usage: "URL of the Master EOS MGM", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_MASTER_URL"}, - Destination: &cfg.Reva.UserStorage.EOS.MasterURL, + Destination: &cfg.Reva.MetadataStorage.EOS.MasterURL, }, &cli.StringFlag{ Name: "storage-eos-slave-url", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.SlaveURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.SlaveURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), Usage: "URL of the Slave EOS MGM", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SLAVE_URL"}, - Destination: &cfg.Reva.UserStorage.EOS.SlaveURL, + Destination: &cfg.Reva.MetadataStorage.EOS.SlaveURL, }, &cli.StringFlag{ Name: "storage-eos-cache-directory", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.CacheDirectory, os.TempDir()), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.CacheDirectory, os.TempDir()), Usage: "Location on the local fs where to store reads", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_CACHE_DIRECTORY"}, - Destination: &cfg.Reva.UserStorage.EOS.CacheDirectory, + Destination: &cfg.Reva.MetadataStorage.EOS.CacheDirectory, }, &cli.BoolFlag{ Name: "storage-eos-enable-logging", Usage: "Enables logging of the commands executed", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_ENABLE_LOGGING"}, - Destination: &cfg.Reva.UserStorage.EOS.EnableLogging, + Destination: &cfg.Reva.MetadataStorage.EOS.EnableLogging, }, &cli.BoolFlag{ Name: "storage-eos-show-hidden-sysfiles", Usage: "show internal EOS files like .sys.v# and .sys.a# files.", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SHOW_HIDDEN_SYSFILES"}, - Destination: &cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles, + Destination: &cfg.Reva.MetadataStorage.EOS.ShowHiddenSysFiles, }, &cli.BoolFlag{ Name: "storage-eos-force-singleuser-mode", Usage: "force connections to EOS to use SingleUsername", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_FORCE_SINGLEUSER_MODE"}, - Destination: &cfg.Reva.UserStorage.EOS.ForceSingleUserMode, + Destination: &cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode, }, &cli.BoolFlag{ Name: "storage-eos-use-keytab", Usage: "authenticate requests by using an EOS keytab", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_USE_KEYTAB"}, - Destination: &cfg.Reva.UserStorage.EOS.UseKeytab, + Destination: &cfg.Reva.MetadataStorage.EOS.UseKeytab, }, &cli.StringFlag{ Name: "storage-eos-sec-protocol", Usage: "the xrootd security protocol to use between the server and EOS", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SEC_PROTOCOL"}, - Destination: &cfg.Reva.UserStorage.EOS.SecProtocol, + Destination: &cfg.Reva.MetadataStorage.EOS.SecProtocol, }, &cli.StringFlag{ Name: "storage-eos-keytab", Usage: "the location of the keytab to use to authenticate to EOS", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_KEYTAB"}, - Destination: &cfg.Reva.UserStorage.EOS.Keytab, + Destination: &cfg.Reva.MetadataStorage.EOS.Keytab, }, &cli.StringFlag{ Name: "storage-eos-single-username", Usage: "the username to use when SingleUserMode is enabled", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SINGLE_USERNAME"}, - Destination: &cfg.Reva.UserStorage.EOS.SingleUsername, + Destination: &cfg.Reva.MetadataStorage.EOS.SingleUsername, }, &cli.StringFlag{ Name: "storage-eos-layout", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.UserLayout, "{{substr 0 1 .Username}}/{{.Username}}"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.UserLayout, "{{substr 0 1 .Username}}/{{.Username}}"), Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.UsernameLower}} and {{.Provider}} also supports prefixing dirs: "{{.UsernamePrefixCount.2}}/{{.UsernameLower}}" will turn "Einstein" into "Ei/Einstein" `, EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_LAYOUT"}, - Destination: &cfg.Reva.UserStorage.EOS.UserLayout, + Destination: &cfg.Reva.MetadataStorage.EOS.UserLayout, }, &cli.StringFlag{ Name: "storage-eos-reva-gateway-addr", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.GatewaySVC, "127.0.0.1:9142"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.GatewaySVC, "127.0.0.1:9142"), Usage: "Address of REVA gateway endpoint", EnvVars: []string{"REVA_GATEWAY"}, - Destination: &cfg.Reva.UserStorage.EOS.GatewaySVC, + Destination: &cfg.Reva.MetadataStorage.EOS.GatewaySVC, }, } } diff --git a/storage/pkg/flagset/metadatadrivers/driverowncloud.go b/storage/pkg/flagset/metadatadrivers/driverowncloud.go deleted file mode 100644 index cf727526b0c..00000000000 --- a/storage/pkg/flagset/metadatadrivers/driverowncloud.go +++ /dev/null @@ -1,55 +0,0 @@ -package metadatadrivers - -import ( - "github.com/owncloud/ocis/ocis-pkg/flags" - "github.com/owncloud/ocis/storage/pkg/config" - "github.com/urfave/cli/v2" -) - -// DriverOwnCloudWithConfig applies cfg to the root flagset -func DriverOwnCloudWithConfig(cfg *config.Config) []cli.Flag { - return []cli.Flag{ - &cli.StringFlag{ - Name: "storage-owncloud-datadir", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.Root, "/var/tmp/ocis/storage/owncloud"), - Usage: "the path to the owncloud data directory", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_DATADIR"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloud.Root, - }, - &cli.StringFlag{ - Name: "storage-owncloud-uploadinfo-dir", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.UploadInfoDir, "/var/tmp/ocis/storage/uploadinfo"), - Usage: "the path to the tus upload info directory", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_UPLOADINFO_DIR"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloud.UploadInfoDir, - }, - &cli.StringFlag{ - Name: "storage-owncloud-share-folder", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.ShareFolder, "/Shares"), - Usage: "name of the shares folder", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_SHARE_FOLDER"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloud.ShareFolder, - }, - &cli.BoolFlag{ - Name: "storage-owncloud-scan", - Value: flags.OverrideDefaultBool(cfg.Reva.MetadataStorage.OwnCloud.Scan, true), - Usage: "scan files on startup to add fileids", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_SCAN"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloud.Scan, - }, - &cli.StringFlag{ - Name: "storage-owncloud-redis", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.Redis, ":6379"), - Usage: "the address of the redis server", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_REDIS_ADDR"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloud.Redis, - }, - &cli.StringFlag{ - Name: "storage-owncloud-layout", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.UserLayout, "{{.Id.OpaqueId}}"), - Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_LAYOUT"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloud.UserLayout, - }, - } -} diff --git a/storage/pkg/flagset/metadatadrivers/driverowncloudsql.go b/storage/pkg/flagset/metadatadrivers/driverowncloudsql.go deleted file mode 100644 index ca444348aaf..00000000000 --- a/storage/pkg/flagset/metadatadrivers/driverowncloudsql.go +++ /dev/null @@ -1,76 +0,0 @@ -package metadatadrivers - -import ( - "github.com/owncloud/ocis/ocis-pkg/flags" - "github.com/owncloud/ocis/storage/pkg/config" - "github.com/urfave/cli/v2" -) - -// DriverOwnCloudSQLWithConfig applies cfg to the root flagset -func DriverOwnCloudSQLWithConfig(cfg *config.Config) []cli.Flag { - return []cli.Flag{ - &cli.StringFlag{ - Name: "storage-owncloudsql-datadir", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.Root, "/var/tmp/ocis/storage/owncloud"), - Usage: "the path to the owncloudsql data directory", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DATADIR"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.Root, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-uploadinfo-dir", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.UploadInfoDir, "/var/tmp/ocis/storage/uploadinfo"), - Usage: "the path to the tus upload info directory", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.UploadInfoDir, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-share-folder", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.ShareFolder, "/Shares"), - Usage: "name of the shares folder", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_SHARE_FOLDER"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.ShareFolder, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-layout", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.UserLayout, "{{.Username}}"), - Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_LAYOUT"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.UserLayout, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-dbusername", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.DBUsername, "owncloud"), - Usage: `"username for accessing the database" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBUSERNAME"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBUsername, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-dbpassword", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.DBPassword, "owncloud"), - Usage: `"password for accessing the database" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBPASSWORD"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBPassword, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-dbhost", - Value: cfg.Reva.MetadataStorage.OwnCloudSQL.DBHost, - Usage: `"the database hostname or IP address" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBHOST"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBHost, - }, - &cli.IntFlag{ - Name: "storage-owncloudsql-dbport", - Value: flags.OverrideDefaultInt(cfg.Reva.MetadataStorage.OwnCloudSQL.DBPort, 3306), - Usage: `"port the database listens on" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBPORT"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBPort, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-dbname", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.DBName, "owncloud"), - Usage: `"the database name" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBNAME"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBName, - }, - } -} diff --git a/storage/pkg/flagset/metadatadrivers/drivers3.go b/storage/pkg/flagset/metadatadrivers/drivers3.go new file mode 100644 index 00000000000..56047756042 --- /dev/null +++ b/storage/pkg/flagset/metadatadrivers/drivers3.go @@ -0,0 +1,48 @@ +package metadatadrivers + +import ( + "github.com/owncloud/ocis/ocis-pkg/flags" + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverS3NGWithConfig applies cfg to the root flagset +func DriverS3WithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "storage-s3-region", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.S3.Region, "default"), + Usage: `"the s3 region" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3_REGION"}, + Destination: &cfg.Reva.MetadataStorage.S3.Region, + }, + &cli.StringFlag{ + Name: "storage-s3-accesskey", + Value: "", + Usage: `"the s3 access key" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3_ACCESS_KEY"}, + Destination: &cfg.Reva.MetadataStorage.S3.AccessKey, + }, + &cli.StringFlag{ + Name: "storage-s3-secretkey", + Value: "", + Usage: `"the secret s3 api key" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3_SECRET_KEY"}, + Destination: &cfg.Reva.MetadataStorage.S3.SecretKey, + }, + &cli.StringFlag{ + Name: "storage-s3-endpoint", + Value: "", + Usage: `"s3 compatible API endpoint" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3_ENDPOINT"}, + Destination: &cfg.Reva.MetadataStorage.S3.Endpoint, + }, + &cli.StringFlag{ + Name: "storage-s3-bucket", + Value: "", + Usage: `"bucket where the data will be stored in`, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3_BUCKET"}, + Destination: &cfg.Reva.MetadataStorage.S3.Bucket, + }, + } +} diff --git a/storage/pkg/flagset/storagehome.go b/storage/pkg/flagset/storagehome.go index 9c989668c0d..a23c9423409 100644 --- a/storage/pkg/flagset/storagehome.go +++ b/storage/pkg/flagset/storagehome.go @@ -159,6 +159,7 @@ func StorageHomeWithConfig(cfg *config.Config) []cli.Flag { flags = append(flags, userdrivers.DriverOwnCloudSQLWithConfig(cfg)...) flags = append(flags, userdrivers.DriverOCISWithConfig(cfg)...) flags = append(flags, userdrivers.DriverS3NGWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverS3WithConfig(cfg)...) return flags } diff --git a/storage/pkg/flagset/storagemetadata.go b/storage/pkg/flagset/storagemetadata.go index 644a0042124..ce5961c18c2 100644 --- a/storage/pkg/flagset/storagemetadata.go +++ b/storage/pkg/flagset/storagemetadata.go @@ -95,9 +95,9 @@ func StorageMetadata(cfg *config.Config) []cli.Flag { f = append(f, SecretWithConfig(cfg)...) f = append(f, metadatadrivers.DriverEOSWithConfig(cfg)...) f = append(f, metadatadrivers.DriverLocalWithConfig(cfg)...) - f = append(f, metadatadrivers.DriverOwnCloudWithConfig(cfg)...) f = append(f, metadatadrivers.DriverOCISWithConfig(cfg)...) f = append(f, metadatadrivers.DriverS3NGWithConfig(cfg)...) + f = append(f, metadatadrivers.DriverS3WithConfig(cfg)...) return f diff --git a/storage/pkg/flagset/storageusers.go b/storage/pkg/flagset/storageusers.go index b5bf0196857..ef8b4e0fc32 100644 --- a/storage/pkg/flagset/storageusers.go +++ b/storage/pkg/flagset/storageusers.go @@ -156,6 +156,7 @@ func StorageUsersWithConfig(cfg *config.Config) []cli.Flag { flags = append(flags, userdrivers.DriverOwnCloudSQLWithConfig(cfg)...) flags = append(flags, userdrivers.DriverOCISWithConfig(cfg)...) flags = append(flags, userdrivers.DriverS3NGWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverS3WithConfig(cfg)...) return flags } diff --git a/storage/pkg/flagset/userdrivers/drivereos.go b/storage/pkg/flagset/userdrivers/drivereos.go index 78fc81b93b7..a937307ef5d 100644 --- a/storage/pkg/flagset/userdrivers/drivereos.go +++ b/storage/pkg/flagset/userdrivers/drivereos.go @@ -27,6 +27,13 @@ func DriverEOSWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_SHADOW_NAMESPACE"}, Destination: &cfg.Reva.UserStorage.EOS.ShadowNamespace, }, + &cli.StringFlag{ + Name: "storage-eos-uploads-namespace", + // Defaults to path.Join(c.Namespace, ".uploads") + Usage: "Uploads namespace", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_UPLOADS_NAMESPACE"}, + Destination: &cfg.Reva.UserStorage.EOS.UploadsNamespace, + }, &cli.StringFlag{ Name: "storage-eos-share-folder", Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.ShareFolder, "/Shares"), @@ -93,12 +100,6 @@ func DriverEOSWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_USE_KEYTAB"}, Destination: &cfg.Reva.UserStorage.EOS.UseKeytab, }, - &cli.BoolFlag{ - Name: "storage-eos-enable-home", - Usage: "enable the creation of home directories", - EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_ENABLE_HOME"}, - Destination: &cfg.Reva.UserStorage.EOS.EnableHome, - }, &cli.StringFlag{ Name: "storage-eos-sec-protocol", Usage: "the xrootd security protocol to use between the server and EOS", diff --git a/storage/pkg/flagset/userdrivers/driverlocal.go b/storage/pkg/flagset/userdrivers/driverlocal.go index 7241b106497..049178a41e2 100644 --- a/storage/pkg/flagset/userdrivers/driverlocal.go +++ b/storage/pkg/flagset/userdrivers/driverlocal.go @@ -16,5 +16,19 @@ func DriverLocalWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_LOCAL_ROOT"}, Destination: &cfg.Reva.UserStorage.Local.Root, }, + &cli.StringFlag{ + Name: "storage-local-share-folder", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.Local.ShareFolder, "/Shares"), + Usage: "the path to the local share folder", + EnvVars: []string{"STORAGE_USERS_DRIVER_LOCAL_SHARE_FOLDER"}, + Destination: &cfg.Reva.UserStorage.Local.ShareFolder, + }, + &cli.StringFlag{ + Name: "storage-local-user-layout", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.Local.UserLayout, "{{.Username}}"), + Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.UsernameLower}} and {{.Provider}} also supports prefixing dirs: "{{.UsernamePrefixCount.2}}/{{.UsernameLower}}" will turn "Einstein" into "Ei/Einstein" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_LOCAL_USER_LAYOUT"}, + Destination: &cfg.Reva.UserStorage.Local.UserLayout, + }, } } diff --git a/storage/pkg/flagset/userdrivers/driverocis.go b/storage/pkg/flagset/userdrivers/driverocis.go index 3621f8578a3..fd7e0f22605 100644 --- a/storage/pkg/flagset/userdrivers/driverocis.go +++ b/storage/pkg/flagset/userdrivers/driverocis.go @@ -16,13 +16,6 @@ func DriverOCISWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_OCIS_ROOT"}, Destination: &cfg.Reva.UserStorage.OCIS.Root, }, - &cli.BoolFlag{ - Name: "storage-ocis-enable-home", - Value: flags.OverrideDefaultBool(cfg.Reva.UserStorage.OCIS.EnableHome, true), - Usage: "enable the creation of home storages", - EnvVars: []string{"STORAGE_USERS_DRIVER_OCIS_ENABLE_HOME"}, - Destination: &cfg.Reva.UserStorage.OCIS.EnableHome, - }, &cli.StringFlag{ Name: "storage-ocis-layout", Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OCIS.UserLayout, "{{.Id.OpaqueId}}"), diff --git a/storage/pkg/flagset/userdrivers/driverowncloud.go b/storage/pkg/flagset/userdrivers/driverowncloud.go index 865b0faff29..1ba06b2fddf 100644 --- a/storage/pkg/flagset/userdrivers/driverowncloud.go +++ b/storage/pkg/flagset/userdrivers/driverowncloud.go @@ -44,13 +44,6 @@ func DriverOwnCloudWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_REDIS_ADDR"}, Destination: &cfg.Reva.UserStorage.OwnCloud.Redis, }, - &cli.BoolFlag{ - Name: "storage-owncloud-enable-home", - Value: flags.OverrideDefaultBool(cfg.Reva.UserStorage.OwnCloud.EnableHome, false), - Usage: "enable the creation of home storages", - EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_ENABLE_HOME"}, - Destination: &cfg.Reva.UserStorage.OwnCloud.EnableHome, - }, &cli.StringFlag{ Name: "storage-owncloud-layout", Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloud.UserLayout, "{{.Id.OpaqueId}}"), diff --git a/storage/pkg/flagset/userdrivers/driverowncloudsql.go b/storage/pkg/flagset/userdrivers/driverowncloudsql.go index d4e3a4cc9aa..1b965af7d9e 100644 --- a/storage/pkg/flagset/userdrivers/driverowncloudsql.go +++ b/storage/pkg/flagset/userdrivers/driverowncloudsql.go @@ -30,13 +30,6 @@ func DriverOwnCloudSQLWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER"}, Destination: &cfg.Reva.UserStorage.OwnCloudSQL.ShareFolder, }, - &cli.BoolFlag{ - Name: "storage-owncloudsql-enable-home", - Value: flags.OverrideDefaultBool(cfg.Reva.UserStorage.OwnCloudSQL.EnableHome, true), - Usage: "enable the creation of home storages", - EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_ENABLE_HOME"}, - Destination: &cfg.Reva.UserStorage.OwnCloudSQL.EnableHome, - }, &cli.StringFlag{ Name: "storage-owncloudsql-layout", Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloudSQL.UserLayout, "{{.Username}}"), diff --git a/storage/pkg/flagset/userdrivers/drivers3.go b/storage/pkg/flagset/userdrivers/drivers3.go new file mode 100644 index 00000000000..72984e684bb --- /dev/null +++ b/storage/pkg/flagset/userdrivers/drivers3.go @@ -0,0 +1,48 @@ +package userdrivers + +import ( + "github.com/owncloud/ocis/ocis-pkg/flags" + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverS3NGWithConfig applies cfg to the root flagset +func DriverS3WithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "storage-s3-region", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.S3.Region, "default"), + Usage: `"the s3 region" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3_REGION"}, + Destination: &cfg.Reva.UserStorage.S3.Region, + }, + &cli.StringFlag{ + Name: "storage-s3-accesskey", + Value: "", + Usage: `"the s3 access key" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3_ACCESS_KEY"}, + Destination: &cfg.Reva.UserStorage.S3.AccessKey, + }, + &cli.StringFlag{ + Name: "storage-s3-secretkey", + Value: "", + Usage: `"the secret s3 api key" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3_SECRET_KEY"}, + Destination: &cfg.Reva.UserStorage.S3.SecretKey, + }, + &cli.StringFlag{ + Name: "storage-s3-endpoint", + Value: "", + Usage: `"s3 compatible API endpoint" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3_ENDPOINT"}, + Destination: &cfg.Reva.UserStorage.S3.Endpoint, + }, + &cli.StringFlag{ + Name: "storage-s3-bucket", + Value: "", + Usage: `"bucket where the data will be stored in`, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3_BUCKET"}, + Destination: &cfg.Reva.UserStorage.S3.Bucket, + }, + } +} diff --git a/storage/pkg/flagset/userdrivers/drivers3ng.go b/storage/pkg/flagset/userdrivers/drivers3ng.go index 12c61320552..434bb1b1e17 100644 --- a/storage/pkg/flagset/userdrivers/drivers3ng.go +++ b/storage/pkg/flagset/userdrivers/drivers3ng.go @@ -16,13 +16,6 @@ func DriverS3NGWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_ROOT"}, Destination: &cfg.Reva.UserStorage.S3NG.Root, }, - &cli.BoolFlag{ - Name: "storage-s3ng-enable-home", - Value: flags.OverrideDefaultBool(cfg.Reva.UserStorage.S3NG.EnableHome, true), - Usage: "enable the creation of home storages", - EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_ENABLE_HOME"}, - Destination: &cfg.Reva.UserStorage.S3NG.EnableHome, - }, &cli.StringFlag{ Name: "storage-s3ng-layout", Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.S3NG.UserLayout, "{{.Id.OpaqueId}}"), @@ -39,7 +32,7 @@ func DriverS3NGWithConfig(cfg *config.Config) []cli.Flag { }, &cli.StringFlag{ Name: "storage-s3ng-region", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.S3NG.Root, "default"), + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.S3NG.Region, "default"), Usage: `"the s3 region" `, EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_REGION"}, Destination: &cfg.Reva.UserStorage.S3NG.Region,