Skip to content

Commit

Permalink
remove uneeded storages for metadata, add missing flagsets
Browse files Browse the repository at this point in the history
  • Loading branch information
@wkloucek
wkloucek committed Oct 11, 2021
1 parent d74816f commit 6c6cb64
Show file tree
Hide file tree
Showing 17 changed files with 162 additions and 247 deletions.
1 change: 0 additions & 1 deletion deployments/examples/oc10_ocis_parallel/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ services:
STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR: /mnt/data/files
STORAGE_USERS_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR: /tmp
STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER: "/Shares"
STORAGE_USERS_DRIVER_OWNCLOUDSQL_ENABLE_HOME: "false"
STORAGE_USERS_DRIVER_OWNCLOUDSQL_LAYOUT: "{{.Username}}"
STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBUSERNAME: owncloud
STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPASSWORD: owncloud
Expand Down
11 changes: 5 additions & 6 deletions storage/pkg/command/storagedrivers/home.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} {
"show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab,
"enable_home": cfg.Reva.UserStorage.EOS.EnableHome,
"enable_home": true,
"gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC,
},
"local": map[string]interface{}{
Expand All @@ -81,7 +81,7 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} {
"share_folder": cfg.Reva.UserStorage.OwnCloud.ShareFolder,
"user_layout": cfg.Reva.UserStorage.OwnCloud.UserLayout,
"redis": cfg.Reva.UserStorage.OwnCloud.Redis,
"enable_home": cfg.Reva.UserStorage.OwnCloud.EnableHome,
"enable_home": true,
"scan": cfg.Reva.UserStorage.OwnCloud.Scan,
"userprovidersvc": cfg.Reva.Users.Endpoint,
},
Expand All @@ -90,7 +90,7 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} {
"upload_info_dir": cfg.Reva.UserStorage.OwnCloudSQL.UploadInfoDir,
"share_folder": cfg.Reva.UserStorage.OwnCloudSQL.ShareFolder,
"user_layout": cfg.Reva.UserStorage.OwnCloudSQL.UserLayout,
"enable_home": cfg.Reva.UserStorage.OwnCloudSQL.EnableHome,
"enable_home": true,
"dbusername": cfg.Reva.UserStorage.OwnCloudSQL.DBUsername,
"dbpassword": cfg.Reva.UserStorage.OwnCloudSQL.DBPassword,
"dbhost": cfg.Reva.UserStorage.OwnCloudSQL.DBHost,
Expand All @@ -100,7 +100,7 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} {
},
"ocis": map[string]interface{}{
"root": cfg.Reva.UserStorage.OCIS.Root,
"enable_home": cfg.Reva.UserStorage.OCIS.EnableHome,
"enable_home": true,
"user_layout": cfg.Reva.UserStorage.OCIS.UserLayout,
"share_folder": cfg.Reva.UserStorage.OCIS.ShareFolder,
"treetime_accounting": true,
Expand All @@ -113,11 +113,10 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} {
"secret_key": cfg.Reva.UserStorage.S3.SecretKey,
"endpoint": cfg.Reva.UserStorage.S3.Endpoint,
"bucket": cfg.Reva.UserStorage.S3.Bucket,
"prefix": cfg.Reva.UserStorage.S3.Root,
},
"s3ng": map[string]interface{}{
"root": cfg.Reva.UserStorage.S3NG.Root,
"enable_home": cfg.Reva.UserStorage.S3NG.EnableHome,
"enable_home": true,
"user_layout": cfg.Reva.UserStorage.S3NG.UserLayout,
"share_folder": cfg.Reva.UserStorage.S3NG.ShareFolder,
"s3.region": cfg.Reva.UserStorage.S3NG.Region,
Expand Down
50 changes: 3 additions & 47 deletions storage/pkg/command/storagedrivers/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,25 +23,7 @@ func MetadataDrivers(cfg *config.Config) map[string]interface{} {
"force_single_user_mode": cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.MetadataStorage.EOS.UseKeytab,
"gatewaysvc": cfg.Reva.MetadataStorage.EOS.GatewaySVC,
},
"eoshome": map[string]interface{}{
"namespace": cfg.Reva.MetadataStorage.EOS.Root,
"shadow_namespace": cfg.Reva.MetadataStorage.EOS.ShadowNamespace,
"uploads_namespace": cfg.Reva.MetadataStorage.EOS.UploadsNamespace,
"eos_binary": cfg.Reva.MetadataStorage.EOS.EosBinary,
"xrdcopy_binary": cfg.Reva.MetadataStorage.EOS.XrdcopyBinary,
"master_url": cfg.Reva.MetadataStorage.EOS.MasterURL,
"slave_url": cfg.Reva.MetadataStorage.EOS.SlaveURL,
"cache_directory": cfg.Reva.MetadataStorage.EOS.CacheDirectory,
"sec_protocol": cfg.Reva.MetadataStorage.EOS.SecProtocol,
"keytab": cfg.Reva.MetadataStorage.EOS.Keytab,
"single_username": cfg.Reva.MetadataStorage.EOS.SingleUsername,
"user_layout": cfg.Reva.MetadataStorage.EOS.UserLayout,
"enable_logging": cfg.Reva.MetadataStorage.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Reva.MetadataStorage.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.MetadataStorage.EOS.UseKeytab,
"gatewaysvc": cfg.Reva.MetadataStorage.EOS.GatewaySVC,
"enable_home": false,
},
"eosgrpc": map[string]interface{}{
"namespace": cfg.Reva.MetadataStorage.EOS.Root,
Expand All @@ -66,37 +48,12 @@ func MetadataDrivers(cfg *config.Config) map[string]interface{} {
"local": map[string]interface{}{
"root": cfg.Reva.MetadataStorage.Local.Root,
},
"localhome": map[string]interface{}{
"root": cfg.Reva.MetadataStorage.Local.Root,
"user_layout": cfg.Reva.MetadataStorage.Local.UserLayout,
},
"owncloud": map[string]interface{}{
"datadirectory": cfg.Reva.MetadataStorage.OwnCloud.Root,
"upload_info_dir": cfg.Reva.MetadataStorage.OwnCloud.UploadInfoDir,
"user_layout": cfg.Reva.MetadataStorage.OwnCloud.UserLayout,
"redis": cfg.Reva.MetadataStorage.OwnCloud.Redis,
"enable_home": false,
"scan": cfg.Reva.MetadataStorage.OwnCloud.Scan,
"userprovidersvc": cfg.Reva.Users.Endpoint,
},
"owncloudsql": map[string]interface{}{
"datadirectory": cfg.Reva.MetadataStorage.OwnCloudSQL.Root,
"upload_info_dir": cfg.Reva.MetadataStorage.OwnCloudSQL.UploadInfoDir,
"user_layout": cfg.Reva.MetadataStorage.OwnCloudSQL.UserLayout,
"enable_home": false,
"dbusername": cfg.Reva.MetadataStorage.OwnCloudSQL.DBUsername,
"dbpassword": cfg.Reva.MetadataStorage.OwnCloudSQL.DBPassword,
"dbhost": cfg.Reva.MetadataStorage.OwnCloudSQL.DBHost,
"dbport": cfg.Reva.MetadataStorage.OwnCloudSQL.DBPort,
"dbname": cfg.Reva.MetadataStorage.OwnCloudSQL.DBName,
"userprovidersvc": cfg.Reva.Users.Endpoint,
},
"ocis": map[string]interface{}{
"root": cfg.Reva.MetadataStorage.OCIS.Root,
"enable_home": false,
"user_layout": cfg.Reva.MetadataStorage.OCIS.UserLayout,
"treetime_accounting": true,
"treesize_accounting": true,
"treetime_accounting": false,
"treesize_accounting": false,
"owner": cfg.Reva.MetadataStorage.OCIS.ServiceUserUUID, // the accounts service system account uuid
},
"s3": map[string]interface{}{
Expand All @@ -105,7 +62,6 @@ func MetadataDrivers(cfg *config.Config) map[string]interface{} {
"secret_key": cfg.Reva.MetadataStorage.S3.SecretKey,
"endpoint": cfg.Reva.MetadataStorage.S3.Endpoint,
"bucket": cfg.Reva.MetadataStorage.S3.Bucket,
"prefix": cfg.Reva.MetadataStorage.S3.Root,
},
"s3ng": map[string]interface{}{
"root": cfg.Reva.MetadataStorage.S3NG.Root,
Expand Down
59 changes: 33 additions & 26 deletions storage/pkg/flagset/metadatadrivers/drivereos.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,115 +15,122 @@ func DriverEOSWithConfig(cfg *config.Config) []cli.Flag {

&cli.StringFlag{
Name: "storage-eos-namespace",
Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.Root, "/eos/dockertest/reva"),
Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.Root, "/eos/dockertest/reva"),
Usage: "Namespace for metadata operations",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_NAMESPACE"},
Destination: &cfg.Reva.UserStorage.EOS.Root,
Destination: &cfg.Reva.MetadataStorage.EOS.Root,
},
&cli.StringFlag{
Name: "storage-eos-shadow-namespace",
// Defaults to path.Join(c.Namespace, ".shadow")
Usage: "Shadow namespace where share references are stored",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SHADOW_NAMESPACE"},
Destination: &cfg.Reva.UserStorage.EOS.ShadowNamespace,
Destination: &cfg.Reva.MetadataStorage.EOS.ShadowNamespace,
},
&cli.StringFlag{
Name: "storage-eos-uploads-namespace",
// Defaults to path.Join(c.Namespace, ".uploads")
Usage: "Uploads namespace",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_UPLOADS_NAMESPACE"},
Destination: &cfg.Reva.MetadataStorage.EOS.UploadsNamespace,
},
&cli.StringFlag{
Name: "storage-eos-share-folder",
Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.ShareFolder, "/Shares"),
Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.ShareFolder, "/Shares"),
Usage: "name of the share folder",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SHARE_FOLDER"},
Destination: &cfg.Reva.UserStorage.EOS.ShareFolder,
Destination: &cfg.Reva.MetadataStorage.EOS.ShareFolder,
},
&cli.StringFlag{
Name: "storage-eos-binary",
Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.EosBinary, "/usr/bin/eos"),
Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.EosBinary, "/usr/bin/eos"),
Usage: "Location of the eos binary",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_BINARY"},
Destination: &cfg.Reva.UserStorage.EOS.EosBinary,
Destination: &cfg.Reva.MetadataStorage.EOS.EosBinary,
},
&cli.StringFlag{
Name: "storage-eos-xrdcopy-binary",
Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.XrdcopyBinary, "/usr/bin/xrdcopy"),
Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.XrdcopyBinary, "/usr/bin/xrdcopy"),
Usage: "Location of the xrdcopy binary",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_XRDCOPY_BINARY"},
Destination: &cfg.Reva.UserStorage.EOS.XrdcopyBinary,
Destination: &cfg.Reva.MetadataStorage.EOS.XrdcopyBinary,
},
&cli.StringFlag{
Name: "storage-eos-master-url",
Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.MasterURL, "root://eos-mgm1.eoscluster.cern.ch:1094"),
Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.MasterURL, "root://eos-mgm1.eoscluster.cern.ch:1094"),
Usage: "URL of the Master EOS MGM",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_MASTER_URL"},
Destination: &cfg.Reva.UserStorage.EOS.MasterURL,
Destination: &cfg.Reva.MetadataStorage.EOS.MasterURL,
},
&cli.StringFlag{
Name: "storage-eos-slave-url",
Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.SlaveURL, "root://eos-mgm1.eoscluster.cern.ch:1094"),
Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.SlaveURL, "root://eos-mgm1.eoscluster.cern.ch:1094"),
Usage: "URL of the Slave EOS MGM",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SLAVE_URL"},
Destination: &cfg.Reva.UserStorage.EOS.SlaveURL,
Destination: &cfg.Reva.MetadataStorage.EOS.SlaveURL,
},
&cli.StringFlag{
Name: "storage-eos-cache-directory",
Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.CacheDirectory, os.TempDir()),
Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.CacheDirectory, os.TempDir()),
Usage: "Location on the local fs where to store reads",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_CACHE_DIRECTORY"},
Destination: &cfg.Reva.UserStorage.EOS.CacheDirectory,
Destination: &cfg.Reva.MetadataStorage.EOS.CacheDirectory,
},
&cli.BoolFlag{
Name: "storage-eos-enable-logging",
Usage: "Enables logging of the commands executed",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_ENABLE_LOGGING"},
Destination: &cfg.Reva.UserStorage.EOS.EnableLogging,
Destination: &cfg.Reva.MetadataStorage.EOS.EnableLogging,
},
&cli.BoolFlag{
Name: "storage-eos-show-hidden-sysfiles",
Usage: "show internal EOS files like .sys.v# and .sys.a# files.",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SHOW_HIDDEN_SYSFILES"},
Destination: &cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
Destination: &cfg.Reva.MetadataStorage.EOS.ShowHiddenSysFiles,
},
&cli.BoolFlag{
Name: "storage-eos-force-singleuser-mode",
Usage: "force connections to EOS to use SingleUsername",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_FORCE_SINGLEUSER_MODE"},
Destination: &cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
Destination: &cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode,
},
&cli.BoolFlag{
Name: "storage-eos-use-keytab",
Usage: "authenticate requests by using an EOS keytab",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_USE_KEYTAB"},
Destination: &cfg.Reva.UserStorage.EOS.UseKeytab,
Destination: &cfg.Reva.MetadataStorage.EOS.UseKeytab,
},
&cli.StringFlag{
Name: "storage-eos-sec-protocol",
Usage: "the xrootd security protocol to use between the server and EOS",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SEC_PROTOCOL"},
Destination: &cfg.Reva.UserStorage.EOS.SecProtocol,
Destination: &cfg.Reva.MetadataStorage.EOS.SecProtocol,
},
&cli.StringFlag{
Name: "storage-eos-keytab",
Usage: "the location of the keytab to use to authenticate to EOS",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_KEYTAB"},
Destination: &cfg.Reva.UserStorage.EOS.Keytab,
Destination: &cfg.Reva.MetadataStorage.EOS.Keytab,
},
&cli.StringFlag{
Name: "storage-eos-single-username",
Usage: "the username to use when SingleUserMode is enabled",
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SINGLE_USERNAME"},
Destination: &cfg.Reva.UserStorage.EOS.SingleUsername,
Destination: &cfg.Reva.MetadataStorage.EOS.SingleUsername,
},
&cli.StringFlag{
Name: "storage-eos-layout",
Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.UserLayout, "{{substr 0 1 .Username}}/{{.Username}}"),
Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.UserLayout, "{{substr 0 1 .Username}}/{{.Username}}"),
Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.UsernameLower}} and {{.Provider}} also supports prefixing dirs: "{{.UsernamePrefixCount.2}}/{{.UsernameLower}}" will turn "Einstein" into "Ei/Einstein" `,
EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_LAYOUT"},
Destination: &cfg.Reva.UserStorage.EOS.UserLayout,
Destination: &cfg.Reva.MetadataStorage.EOS.UserLayout,
},
&cli.StringFlag{
Name: "storage-eos-reva-gateway-addr",
Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.GatewaySVC, "127.0.0.1:9142"),
Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.GatewaySVC, "127.0.0.1:9142"),
Usage: "Address of REVA gateway endpoint",
EnvVars: []string{"REVA_GATEWAY"},
Destination: &cfg.Reva.UserStorage.EOS.GatewaySVC,
Destination: &cfg.Reva.MetadataStorage.EOS.GatewaySVC,
},
}
}
55 changes: 0 additions & 55 deletions storage/pkg/flagset/metadatadrivers/driverowncloud.go
View file

This file was deleted.

Loading

0 comments on commit 6c6cb64

Please sign in to comment.