From d74816f242aadb0b80d38bb4dfbcdb83c16d8794 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 8 Oct 2021 17:19:52 +0200 Subject: [PATCH 1/2] configure users and metadata storage separately --- .drone.star | 12 +- ...igure-users-metadata-storage-separately.md | 9 ++ .../oc10_ocis_parallel/docker-compose.yml | 23 +-- .../examples/ocis_s3/docker-compose.yml | 10 +- storage/pkg/command/drivers.go | 128 ----------------- storage/pkg/command/frontend.go | 2 +- storage/pkg/command/storagedrivers/home.go | 130 +++++++++++++++++ .../pkg/command/storagedrivers/metadata.go | 121 ++++++++++++++++ storage/pkg/command/storagedrivers/user.go | 131 +++++++++++++++++ storage/pkg/command/storagehome.go | 15 +- storage/pkg/command/storagemetadata.go | 33 +---- storage/pkg/command/storageusers.go | 15 +- storage/pkg/config/config.go | 7 +- storage/pkg/flagset/drivereos.go | 135 ------------------ storage/pkg/flagset/driverocis.go | 41 ------ storage/pkg/flagset/driverowncloud.go | 62 -------- storage/pkg/flagset/driverowncloudsql.go | 83 ----------- storage/pkg/flagset/frontend.go | 3 +- .../pkg/flagset/metadatadrivers/drivereos.go | 129 +++++++++++++++++ .../flagset/metadatadrivers/driverlocal.go | 20 +++ .../pkg/flagset/metadatadrivers/driverocis.go | 34 +++++ .../flagset/metadatadrivers/driverowncloud.go | 55 +++++++ .../metadatadrivers/driverowncloudsql.go | 76 ++++++++++ .../{ => metadatadrivers}/drivers3ng.go | 41 +++--- storage/pkg/flagset/storagehome.go | 20 +-- storage/pkg/flagset/storagemetadata.go | 20 +-- storage/pkg/flagset/storageusers.go | 13 +- storage/pkg/flagset/userdrivers/drivereos.go | 135 ++++++++++++++++++ .../flagset/{ => userdrivers}/driverlocal.go | 8 +- storage/pkg/flagset/userdrivers/driverocis.go | 48 +++++++ .../pkg/flagset/userdrivers/driverowncloud.go | 62 ++++++++ .../flagset/userdrivers/driverowncloudsql.go | 83 +++++++++++ storage/pkg/flagset/userdrivers/drivers3ng.go | 76 ++++++++++ tests/acceptance/docker/src/ocis-base.yml | 21 ++- 34 files changed, 1196 insertions(+), 605 deletions(-) create mode 100644 changelog/unreleased/configure-users-metadata-storage-separately.md delete mode 100644 storage/pkg/command/drivers.go create mode 100644 storage/pkg/command/storagedrivers/home.go create mode 100644 storage/pkg/command/storagedrivers/metadata.go create mode 100644 storage/pkg/command/storagedrivers/user.go delete mode 100644 storage/pkg/flagset/drivereos.go delete mode 100644 storage/pkg/flagset/driverocis.go delete mode 100644 storage/pkg/flagset/driverowncloud.go delete mode 100644 storage/pkg/flagset/driverowncloudsql.go create mode 100644 storage/pkg/flagset/metadatadrivers/drivereos.go create mode 100644 storage/pkg/flagset/metadatadrivers/driverlocal.go create mode 100644 storage/pkg/flagset/metadatadrivers/driverocis.go create mode 100644 storage/pkg/flagset/metadatadrivers/driverowncloud.go create mode 100644 storage/pkg/flagset/metadatadrivers/driverowncloudsql.go rename storage/pkg/flagset/{ => metadatadrivers}/drivers3ng.go (51%) create mode 100644 storage/pkg/flagset/userdrivers/drivereos.go rename storage/pkg/flagset/{ => userdrivers}/driverlocal.go (61%) create mode 100644 storage/pkg/flagset/userdrivers/driverocis.go create mode 100644 storage/pkg/flagset/userdrivers/driverowncloud.go create mode 100644 storage/pkg/flagset/userdrivers/driverowncloudsql.go create mode 100644 storage/pkg/flagset/userdrivers/drivers3ng.go diff --git a/.drone.star b/.drone.star index 98f27960d0d..44432c87370 100644 --- a/.drone.star +++ b/.drone.star @@ -1504,17 +1504,13 @@ def notify(ctx): def ocisServer(storage, accounts_hash_difficulty = 4, volumes = []): environment = { - #'OCIS_LOG_LEVEL': 'debug', "OCIS_URL": "https://ocis-server:9200", "STORAGE_HOME_DRIVER": "%s" % (storage), "STORAGE_USERS_DRIVER": "%s" % (storage), - "STORAGE_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users", - "STORAGE_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root", - "STORAGE_METADATA_ROOT": "/srv/app/tmp/ocis/metadata", - "STORAGE_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data", - "STORAGE_DRIVER_OWNCLOUD_REDIS_ADDR": "redis:6379" if storage == "owncloud" else "", - "STORAGE_HOME_DATA_SERVER_URL": "http://ocis-server:9155/data", - "STORAGE_USERS_DATA_SERVER_URL": "http://ocis-server:9158/data", + "STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root", + "STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data", + "STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users", + "STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata", "STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json", "PROXY_ENABLE_BASIC_AUTH": True, "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json", diff --git a/changelog/unreleased/configure-users-metadata-storage-separately.md b/changelog/unreleased/configure-users-metadata-storage-separately.md new file mode 100644 index 00000000000..808e05250d4 --- /dev/null +++ b/changelog/unreleased/configure-users-metadata-storage-separately.md @@ -0,0 +1,9 @@ +Change: Configure users and metadata storage separately + +We've fixed the configuration behaviour of the user and metadata service writing in the same +directory when using oCIS storage. + +Therefore we needed to separate the configuration of the users and metadata storage so that they +now can be configured totally separate. + +https://github.com/owncloud/ocis/pull/2598 diff --git a/deployments/examples/oc10_ocis_parallel/docker-compose.yml b/deployments/examples/oc10_ocis_parallel/docker-compose.yml index 4678b7faed7..1ec6b5e9578 100644 --- a/deployments/examples/oc10_ocis_parallel/docker-compose.yml +++ b/deployments/examples/oc10_ocis_parallel/docker-compose.yml @@ -92,17 +92,18 @@ services: # ownCloud storage driver STORAGE_HOME_DRIVER: owncloudsql STORAGE_USERS_DRIVER: owncloudsql - STORAGE_DRIVER_OWNCLOUDSQL_DATADIR: /mnt/data/files - STORAGE_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR: /tmp - STORAGE_DRIVER_OWNCLOUDSQL_SHARE_FOLDER: "/Shares" - STORAGE_DRIVER_OWNCLOUDSQL_ENABLE_HOME: "false" - STORAGE_DRIVER_OWNCLOUDSQL_LAYOUT: "{{.Username}}" - STORAGE_DRIVER_OWNCLOUDSQL_DBUSERNAME: owncloud - STORAGE_DRIVER_OWNCLOUDSQL_DBPASSWORD: owncloud - STORAGE_DRIVER_OWNCLOUDSQL_DBHOST: oc10-db - STORAGE_DRIVER_OWNCLOUDSQL_DBPORT: 3306 - STORAGE_DRIVER_OWNCLOUDSQL_DBNAME: owncloud - STORAGE_DRIVER_OWNCLOUDSQL_REDIS_ADDR: redis:6379 # TODO: redis is not yet supported + STORAGE_METADATA_DRIVER: ocis # keep metadata on ocis storage since this are only small files atm + STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR: /mnt/data/files + STORAGE_USERS_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR: /tmp + STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER: "/Shares" + STORAGE_USERS_DRIVER_OWNCLOUDSQL_ENABLE_HOME: "false" + STORAGE_USERS_DRIVER_OWNCLOUDSQL_LAYOUT: "{{.Username}}" + STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBUSERNAME: owncloud + STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPASSWORD: owncloud + STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBHOST: oc10-db + STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPORT: 3306 + STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBNAME: owncloud + STORAGE_USERS_DRIVER_OWNCLOUDSQL_REDIS_ADDR: redis:6379 # TODO: redis is not yet supported # ownCloud storage readonly OCIS_STORAGE_READ_ONLY: "false" # TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303 # General oCIS config diff --git a/deployments/examples/ocis_s3/docker-compose.yml b/deployments/examples/ocis_s3/docker-compose.yml index 5ac5a0916d3..2964a58864e 100644 --- a/deployments/examples/ocis_s3/docker-compose.yml +++ b/deployments/examples/ocis_s3/docker-compose.yml @@ -65,11 +65,11 @@ services: STORAGE_USERS_DRIVER: s3ng STORAGE_METADATA_DRIVER: ocis # keep metadata on ocis storage since this are only small files atm # s3ng specific settings - STORAGE_DRIVER_S3NG_ENDPOINT: http://minio:9000 - STORAGE_DRIVER_S3NG_REGION: default - STORAGE_DRIVER_S3NG_ACCESS_KEY: ${MINIO_ACCESS_KEY:-ocis} - STORAGE_DRIVER_S3NG_SECRET_KEY: ${MINIO_SECRET_KEY:-ocis-secret-key} - STORAGE_DRIVER_S3NG_BUCKET: ${MINIO_BUCKET:-ocis-bucket} + STORAGE_USERS_DRIVER_S3NG_ENDPOINT: http://minio:9000 + STORAGE_USERS_DRIVER_S3NG_REGION: default + STORAGE_USERS_DRIVER_S3NG_ACCESS_KEY: ${MINIO_ACCESS_KEY:-ocis} + STORAGE_USERS_DRIVER_S3NG_SECRET_KEY: ${MINIO_SECRET_KEY:-ocis-secret-key} + STORAGE_USERS_DRIVER_S3NG_BUCKET: ${MINIO_BUCKET:-ocis-bucket} volumes: - ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh - ocis-data:/var/tmp/ocis diff --git a/storage/pkg/command/drivers.go b/storage/pkg/command/drivers.go deleted file mode 100644 index 8cf79b83d81..00000000000 --- a/storage/pkg/command/drivers.go +++ /dev/null @@ -1,128 +0,0 @@ -package command - -import ( - "github.com/owncloud/ocis/storage/pkg/config" -) - -func drivers(cfg *config.Config) map[string]interface{} { - return map[string]interface{}{ - "eos": map[string]interface{}{ - "namespace": cfg.Reva.Storages.EOS.Root, - "shadow_namespace": cfg.Reva.Storages.EOS.ShadowNamespace, - "uploads_namespace": cfg.Reva.Storages.EOS.UploadsNamespace, - "share_folder": cfg.Reva.Storages.EOS.ShareFolder, - "eos_binary": cfg.Reva.Storages.EOS.EosBinary, - "xrdcopy_binary": cfg.Reva.Storages.EOS.XrdcopyBinary, - "master_url": cfg.Reva.Storages.EOS.MasterURL, - "slave_url": cfg.Reva.Storages.EOS.SlaveURL, - "cache_directory": cfg.Reva.Storages.EOS.CacheDirectory, - "sec_protocol": cfg.Reva.Storages.EOS.SecProtocol, - "keytab": cfg.Reva.Storages.EOS.Keytab, - "single_username": cfg.Reva.Storages.EOS.SingleUsername, - "enable_logging": cfg.Reva.Storages.EOS.EnableLogging, - "show_hidden_sys_files": cfg.Reva.Storages.EOS.ShowHiddenSysFiles, - "force_single_user_mode": cfg.Reva.Storages.EOS.ForceSingleUserMode, - "use_keytab": cfg.Reva.Storages.EOS.UseKeytab, - "gatewaysvc": cfg.Reva.Storages.EOS.GatewaySVC, - }, - "eoshome": map[string]interface{}{ - "namespace": cfg.Reva.Storages.EOS.Root, - "shadow_namespace": cfg.Reva.Storages.EOS.ShadowNamespace, - "uploads_namespace": cfg.Reva.Storages.EOS.UploadsNamespace, - "share_folder": cfg.Reva.Storages.EOS.ShareFolder, - "eos_binary": cfg.Reva.Storages.EOS.EosBinary, - "xrdcopy_binary": cfg.Reva.Storages.EOS.XrdcopyBinary, - "master_url": cfg.Reva.Storages.EOS.MasterURL, - "slave_url": cfg.Reva.Storages.EOS.SlaveURL, - "cache_directory": cfg.Reva.Storages.EOS.CacheDirectory, - "sec_protocol": cfg.Reva.Storages.EOS.SecProtocol, - "keytab": cfg.Reva.Storages.EOS.Keytab, - "single_username": cfg.Reva.Storages.EOS.SingleUsername, - "user_layout": cfg.Reva.Storages.EOS.UserLayout, - "enable_logging": cfg.Reva.Storages.EOS.EnableLogging, - "show_hidden_sys_files": cfg.Reva.Storages.EOS.ShowHiddenSysFiles, - "force_single_user_mode": cfg.Reva.Storages.EOS.ForceSingleUserMode, - "use_keytab": cfg.Reva.Storages.EOS.UseKeytab, - "gatewaysvc": cfg.Reva.Storages.EOS.GatewaySVC, - }, - "eosgrpc": map[string]interface{}{ - "namespace": cfg.Reva.Storages.EOS.Root, - "shadow_namespace": cfg.Reva.Storages.EOS.ShadowNamespace, - "share_folder": cfg.Reva.Storages.EOS.ShareFolder, - "eos_binary": cfg.Reva.Storages.EOS.EosBinary, - "xrdcopy_binary": cfg.Reva.Storages.EOS.XrdcopyBinary, - "master_url": cfg.Reva.Storages.EOS.MasterURL, - "master_grpc_uri": cfg.Reva.Storages.EOS.GrpcURI, - "slave_url": cfg.Reva.Storages.EOS.SlaveURL, - "cache_directory": cfg.Reva.Storages.EOS.CacheDirectory, - "sec_protocol": cfg.Reva.Storages.EOS.SecProtocol, - "keytab": cfg.Reva.Storages.EOS.Keytab, - "single_username": cfg.Reva.Storages.EOS.SingleUsername, - "user_layout": cfg.Reva.Storages.EOS.UserLayout, - "enable_logging": cfg.Reva.Storages.EOS.EnableLogging, - "show_hidden_sys_files": cfg.Reva.Storages.EOS.ShowHiddenSysFiles, - "force_single_user_mode": cfg.Reva.Storages.EOS.ForceSingleUserMode, - "use_keytab": cfg.Reva.Storages.EOS.UseKeytab, - "enable_home": cfg.Reva.Storages.EOS.EnableHome, - "gatewaysvc": cfg.Reva.Storages.EOS.GatewaySVC, - }, - "local": map[string]interface{}{ - "root": cfg.Reva.Storages.Local.Root, - "share_folder": cfg.Reva.Storages.Local.ShareFolder, - }, - "localhome": map[string]interface{}{ - "root": cfg.Reva.Storages.Local.Root, - "share_folder": cfg.Reva.Storages.Local.ShareFolder, - "user_layout": cfg.Reva.Storages.Local.UserLayout, - }, - "owncloud": map[string]interface{}{ - "datadirectory": cfg.Reva.Storages.OwnCloud.Root, - "upload_info_dir": cfg.Reva.Storages.OwnCloud.UploadInfoDir, - "sharedirectory": cfg.Reva.Storages.OwnCloud.ShareFolder, - "user_layout": cfg.Reva.Storages.OwnCloud.UserLayout, - "redis": cfg.Reva.Storages.OwnCloud.Redis, - "enable_home": cfg.Reva.Storages.OwnCloud.EnableHome, - "scan": cfg.Reva.Storages.OwnCloud.Scan, - "userprovidersvc": cfg.Reva.Users.Endpoint, - }, - "owncloudsql": map[string]interface{}{ - "datadirectory": cfg.Reva.Storages.OwnCloudSQL.Root, - "upload_info_dir": cfg.Reva.Storages.OwnCloudSQL.UploadInfoDir, - "sharedirectory": cfg.Reva.Storages.OwnCloudSQL.ShareFolder, - "user_layout": cfg.Reva.Storages.OwnCloudSQL.UserLayout, - "enable_home": cfg.Reva.Storages.OwnCloudSQL.EnableHome, - "dbusername": cfg.Reva.Storages.OwnCloudSQL.DBUsername, - "dbpassword": cfg.Reva.Storages.OwnCloudSQL.DBPassword, - "dbhost": cfg.Reva.Storages.OwnCloudSQL.DBHost, - "dbport": cfg.Reva.Storages.OwnCloudSQL.DBPort, - "dbname": cfg.Reva.Storages.OwnCloudSQL.DBName, - "userprovidersvc": cfg.Reva.Users.Endpoint, - }, - "ocis": map[string]interface{}{ - "root": cfg.Reva.Storages.Common.Root, - "enable_home": cfg.Reva.Storages.Common.EnableHome, - "user_layout": cfg.Reva.Storages.Common.UserLayout, - "treetime_accounting": true, - "treesize_accounting": true, - "owner": cfg.Reva.Storages.OCIS.ServiceUserUUID, // the accounts service system account uuid - }, - "s3": map[string]interface{}{ - "region": cfg.Reva.Storages.S3.Region, - "access_key": cfg.Reva.Storages.S3.AccessKey, - "secret_key": cfg.Reva.Storages.S3.SecretKey, - "endpoint": cfg.Reva.Storages.S3.Endpoint, - "bucket": cfg.Reva.Storages.S3.Bucket, - "prefix": cfg.Reva.Storages.S3.Root, - }, - "s3ng": map[string]interface{}{ - "root": cfg.Reva.Storages.Common.Root, - "enable_home": cfg.Reva.Storages.Common.EnableHome, - "user_layout": cfg.Reva.Storages.Common.UserLayout, - "s3.region": cfg.Reva.Storages.S3NG.Region, - "s3.access_key": cfg.Reva.Storages.S3NG.AccessKey, - "s3.secret_key": cfg.Reva.Storages.S3NG.SecretKey, - "s3.endpoint": cfg.Reva.Storages.S3NG.Endpoint, - "s3.bucket": cfg.Reva.Storages.S3NG.Bucket, - }, - } -} diff --git a/storage/pkg/command/frontend.go b/storage/pkg/command/frontend.go index 96566f55f70..3aaa7e8621d 100644 --- a/storage/pkg/command/frontend.go +++ b/storage/pkg/command/frontend.go @@ -204,7 +204,7 @@ func frontendConfigFromStruct(c *cli.Context, cfg *config.Config, filesCfg map[s "db_host": cfg.Reva.Sharing.UserSQLHost, "db_port": cfg.Reva.Sharing.UserSQLPort, "db_name": cfg.Reva.Sharing.UserSQLName, - "namespace": cfg.Reva.Storages.EOS.Root, + "namespace": cfg.Reva.UserStorage.EOS.Root, "gatewaysvc": cfg.Reva.Gateway.Endpoint, }, }, diff --git a/storage/pkg/command/storagedrivers/home.go b/storage/pkg/command/storagedrivers/home.go new file mode 100644 index 00000000000..cdd4968a6d4 --- /dev/null +++ b/storage/pkg/command/storagedrivers/home.go @@ -0,0 +1,130 @@ +package storagedrivers + +import ( + "github.com/owncloud/ocis/storage/pkg/config" +) + +func HomeDrivers(cfg *config.Config) map[string]interface{} { + return map[string]interface{}{ + "eos": map[string]interface{}{ + "namespace": cfg.Reva.UserStorage.EOS.Root, + "shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace, + "uploads_namespace": cfg.Reva.UserStorage.EOS.UploadsNamespace, + "share_folder": cfg.Reva.UserStorage.EOS.ShareFolder, + "eos_binary": cfg.Reva.UserStorage.EOS.EosBinary, + "xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary, + "master_url": cfg.Reva.UserStorage.EOS.MasterURL, + "slave_url": cfg.Reva.UserStorage.EOS.SlaveURL, + "cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory, + "sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol, + "keytab": cfg.Reva.UserStorage.EOS.Keytab, + "single_username": cfg.Reva.UserStorage.EOS.SingleUsername, + "enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging, + "show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles, + "force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode, + "use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab, + "gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC, + }, + "eoshome": map[string]interface{}{ + "namespace": cfg.Reva.UserStorage.EOS.Root, + "shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace, + "uploads_namespace": cfg.Reva.UserStorage.EOS.UploadsNamespace, + "share_folder": cfg.Reva.UserStorage.EOS.ShareFolder, + "eos_binary": cfg.Reva.UserStorage.EOS.EosBinary, + "xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary, + "master_url": cfg.Reva.UserStorage.EOS.MasterURL, + "slave_url": cfg.Reva.UserStorage.EOS.SlaveURL, + "cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory, + "sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol, + "keytab": cfg.Reva.UserStorage.EOS.Keytab, + "single_username": cfg.Reva.UserStorage.EOS.SingleUsername, + "user_layout": cfg.Reva.UserStorage.EOS.UserLayout, + "enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging, + "show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles, + "force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode, + "use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab, + "gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC, + }, + "eosgrpc": map[string]interface{}{ + "namespace": cfg.Reva.UserStorage.EOS.Root, + "shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace, + "share_folder": cfg.Reva.UserStorage.EOS.ShareFolder, + "eos_binary": cfg.Reva.UserStorage.EOS.EosBinary, + "xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary, + "master_url": cfg.Reva.UserStorage.EOS.MasterURL, + "master_grpc_uri": cfg.Reva.UserStorage.EOS.GrpcURI, + "slave_url": cfg.Reva.UserStorage.EOS.SlaveURL, + "cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory, + "sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol, + "keytab": cfg.Reva.UserStorage.EOS.Keytab, + "single_username": cfg.Reva.UserStorage.EOS.SingleUsername, + "user_layout": cfg.Reva.UserStorage.EOS.UserLayout, + "enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging, + "show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles, + "force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode, + "use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab, + "enable_home": cfg.Reva.UserStorage.EOS.EnableHome, + "gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC, + }, + "local": map[string]interface{}{ + "root": cfg.Reva.UserStorage.Local.Root, + "share_folder": cfg.Reva.UserStorage.Local.ShareFolder, + }, + "localhome": map[string]interface{}{ + "root": cfg.Reva.UserStorage.Local.Root, + "share_folder": cfg.Reva.UserStorage.Local.ShareFolder, + "user_layout": cfg.Reva.UserStorage.Local.UserLayout, + }, + "owncloud": map[string]interface{}{ + "datadirectory": cfg.Reva.UserStorage.OwnCloud.Root, + "upload_info_dir": cfg.Reva.UserStorage.OwnCloud.UploadInfoDir, + "share_folder": cfg.Reva.UserStorage.OwnCloud.ShareFolder, + "user_layout": cfg.Reva.UserStorage.OwnCloud.UserLayout, + "redis": cfg.Reva.UserStorage.OwnCloud.Redis, + "enable_home": cfg.Reva.UserStorage.OwnCloud.EnableHome, + "scan": cfg.Reva.UserStorage.OwnCloud.Scan, + "userprovidersvc": cfg.Reva.Users.Endpoint, + }, + "owncloudsql": map[string]interface{}{ + "datadirectory": cfg.Reva.UserStorage.OwnCloudSQL.Root, + "upload_info_dir": cfg.Reva.UserStorage.OwnCloudSQL.UploadInfoDir, + "share_folder": cfg.Reva.UserStorage.OwnCloudSQL.ShareFolder, + "user_layout": cfg.Reva.UserStorage.OwnCloudSQL.UserLayout, + "enable_home": cfg.Reva.UserStorage.OwnCloudSQL.EnableHome, + "dbusername": cfg.Reva.UserStorage.OwnCloudSQL.DBUsername, + "dbpassword": cfg.Reva.UserStorage.OwnCloudSQL.DBPassword, + "dbhost": cfg.Reva.UserStorage.OwnCloudSQL.DBHost, + "dbport": cfg.Reva.UserStorage.OwnCloudSQL.DBPort, + "dbname": cfg.Reva.UserStorage.OwnCloudSQL.DBName, + "userprovidersvc": cfg.Reva.Users.Endpoint, + }, + "ocis": map[string]interface{}{ + "root": cfg.Reva.UserStorage.OCIS.Root, + "enable_home": cfg.Reva.UserStorage.OCIS.EnableHome, + "user_layout": cfg.Reva.UserStorage.OCIS.UserLayout, + "share_folder": cfg.Reva.UserStorage.OCIS.ShareFolder, + "treetime_accounting": true, + "treesize_accounting": true, + "owner": cfg.Reva.UserStorage.OCIS.ServiceUserUUID, // the accounts service system account uuid + }, + "s3": map[string]interface{}{ + "region": cfg.Reva.UserStorage.S3.Region, + "access_key": cfg.Reva.UserStorage.S3.AccessKey, + "secret_key": cfg.Reva.UserStorage.S3.SecretKey, + "endpoint": cfg.Reva.UserStorage.S3.Endpoint, + "bucket": cfg.Reva.UserStorage.S3.Bucket, + "prefix": cfg.Reva.UserStorage.S3.Root, + }, + "s3ng": map[string]interface{}{ + "root": cfg.Reva.UserStorage.S3NG.Root, + "enable_home": cfg.Reva.UserStorage.S3NG.EnableHome, + "user_layout": cfg.Reva.UserStorage.S3NG.UserLayout, + "share_folder": cfg.Reva.UserStorage.S3NG.ShareFolder, + "s3.region": cfg.Reva.UserStorage.S3NG.Region, + "s3.access_key": cfg.Reva.UserStorage.S3NG.AccessKey, + "s3.secret_key": cfg.Reva.UserStorage.S3NG.SecretKey, + "s3.endpoint": cfg.Reva.UserStorage.S3NG.Endpoint, + "s3.bucket": cfg.Reva.UserStorage.S3NG.Bucket, + }, + } +} diff --git a/storage/pkg/command/storagedrivers/metadata.go b/storage/pkg/command/storagedrivers/metadata.go new file mode 100644 index 00000000000..3438c7043e1 --- /dev/null +++ b/storage/pkg/command/storagedrivers/metadata.go @@ -0,0 +1,121 @@ +package storagedrivers + +import ( + "github.com/owncloud/ocis/storage/pkg/config" +) + +func MetadataDrivers(cfg *config.Config) map[string]interface{} { + return map[string]interface{}{ + "eos": map[string]interface{}{ + "namespace": cfg.Reva.MetadataStorage.EOS.Root, + "shadow_namespace": cfg.Reva.MetadataStorage.EOS.ShadowNamespace, + "uploads_namespace": cfg.Reva.MetadataStorage.EOS.UploadsNamespace, + "eos_binary": cfg.Reva.MetadataStorage.EOS.EosBinary, + "xrdcopy_binary": cfg.Reva.MetadataStorage.EOS.XrdcopyBinary, + "master_url": cfg.Reva.MetadataStorage.EOS.MasterURL, + "slave_url": cfg.Reva.MetadataStorage.EOS.SlaveURL, + "cache_directory": cfg.Reva.MetadataStorage.EOS.CacheDirectory, + "sec_protocol": cfg.Reva.MetadataStorage.EOS.SecProtocol, + "keytab": cfg.Reva.MetadataStorage.EOS.Keytab, + "single_username": cfg.Reva.MetadataStorage.EOS.SingleUsername, + "enable_logging": cfg.Reva.MetadataStorage.EOS.EnableLogging, + "show_hidden_sys_files": cfg.Reva.MetadataStorage.EOS.ShowHiddenSysFiles, + "force_single_user_mode": cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode, + "use_keytab": cfg.Reva.MetadataStorage.EOS.UseKeytab, + "gatewaysvc": cfg.Reva.MetadataStorage.EOS.GatewaySVC, + }, + "eoshome": map[string]interface{}{ + "namespace": cfg.Reva.MetadataStorage.EOS.Root, + "shadow_namespace": cfg.Reva.MetadataStorage.EOS.ShadowNamespace, + "uploads_namespace": cfg.Reva.MetadataStorage.EOS.UploadsNamespace, + "eos_binary": cfg.Reva.MetadataStorage.EOS.EosBinary, + "xrdcopy_binary": cfg.Reva.MetadataStorage.EOS.XrdcopyBinary, + "master_url": cfg.Reva.MetadataStorage.EOS.MasterURL, + "slave_url": cfg.Reva.MetadataStorage.EOS.SlaveURL, + "cache_directory": cfg.Reva.MetadataStorage.EOS.CacheDirectory, + "sec_protocol": cfg.Reva.MetadataStorage.EOS.SecProtocol, + "keytab": cfg.Reva.MetadataStorage.EOS.Keytab, + "single_username": cfg.Reva.MetadataStorage.EOS.SingleUsername, + "user_layout": cfg.Reva.MetadataStorage.EOS.UserLayout, + "enable_logging": cfg.Reva.MetadataStorage.EOS.EnableLogging, + "show_hidden_sys_files": cfg.Reva.MetadataStorage.EOS.ShowHiddenSysFiles, + "force_single_user_mode": cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode, + "use_keytab": cfg.Reva.MetadataStorage.EOS.UseKeytab, + "gatewaysvc": cfg.Reva.MetadataStorage.EOS.GatewaySVC, + }, + "eosgrpc": map[string]interface{}{ + "namespace": cfg.Reva.MetadataStorage.EOS.Root, + "shadow_namespace": cfg.Reva.MetadataStorage.EOS.ShadowNamespace, + "eos_binary": cfg.Reva.MetadataStorage.EOS.EosBinary, + "xrdcopy_binary": cfg.Reva.MetadataStorage.EOS.XrdcopyBinary, + "master_url": cfg.Reva.MetadataStorage.EOS.MasterURL, + "master_grpc_uri": cfg.Reva.MetadataStorage.EOS.GrpcURI, + "slave_url": cfg.Reva.MetadataStorage.EOS.SlaveURL, + "cache_directory": cfg.Reva.MetadataStorage.EOS.CacheDirectory, + "sec_protocol": cfg.Reva.MetadataStorage.EOS.SecProtocol, + "keytab": cfg.Reva.MetadataStorage.EOS.Keytab, + "single_username": cfg.Reva.MetadataStorage.EOS.SingleUsername, + "user_layout": cfg.Reva.MetadataStorage.EOS.UserLayout, + "enable_logging": cfg.Reva.MetadataStorage.EOS.EnableLogging, + "show_hidden_sys_files": cfg.Reva.MetadataStorage.EOS.ShowHiddenSysFiles, + "force_single_user_mode": cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode, + "use_keytab": cfg.Reva.MetadataStorage.EOS.UseKeytab, + "enable_home": false, + "gatewaysvc": cfg.Reva.MetadataStorage.EOS.GatewaySVC, + }, + "local": map[string]interface{}{ + "root": cfg.Reva.MetadataStorage.Local.Root, + }, + "localhome": map[string]interface{}{ + "root": cfg.Reva.MetadataStorage.Local.Root, + "user_layout": cfg.Reva.MetadataStorage.Local.UserLayout, + }, + "owncloud": map[string]interface{}{ + "datadirectory": cfg.Reva.MetadataStorage.OwnCloud.Root, + "upload_info_dir": cfg.Reva.MetadataStorage.OwnCloud.UploadInfoDir, + "user_layout": cfg.Reva.MetadataStorage.OwnCloud.UserLayout, + "redis": cfg.Reva.MetadataStorage.OwnCloud.Redis, + "enable_home": false, + "scan": cfg.Reva.MetadataStorage.OwnCloud.Scan, + "userprovidersvc": cfg.Reva.Users.Endpoint, + }, + "owncloudsql": map[string]interface{}{ + "datadirectory": cfg.Reva.MetadataStorage.OwnCloudSQL.Root, + "upload_info_dir": cfg.Reva.MetadataStorage.OwnCloudSQL.UploadInfoDir, + "user_layout": cfg.Reva.MetadataStorage.OwnCloudSQL.UserLayout, + "enable_home": false, + "dbusername": cfg.Reva.MetadataStorage.OwnCloudSQL.DBUsername, + "dbpassword": cfg.Reva.MetadataStorage.OwnCloudSQL.DBPassword, + "dbhost": cfg.Reva.MetadataStorage.OwnCloudSQL.DBHost, + "dbport": cfg.Reva.MetadataStorage.OwnCloudSQL.DBPort, + "dbname": cfg.Reva.MetadataStorage.OwnCloudSQL.DBName, + "userprovidersvc": cfg.Reva.Users.Endpoint, + }, + "ocis": map[string]interface{}{ + "root": cfg.Reva.MetadataStorage.OCIS.Root, + "enable_home": false, + "user_layout": cfg.Reva.MetadataStorage.OCIS.UserLayout, + "treetime_accounting": true, + "treesize_accounting": true, + "owner": cfg.Reva.MetadataStorage.OCIS.ServiceUserUUID, // the accounts service system account uuid + }, + "s3": map[string]interface{}{ + "region": cfg.Reva.MetadataStorage.S3.Region, + "access_key": cfg.Reva.MetadataStorage.S3.AccessKey, + "secret_key": cfg.Reva.MetadataStorage.S3.SecretKey, + "endpoint": cfg.Reva.MetadataStorage.S3.Endpoint, + "bucket": cfg.Reva.MetadataStorage.S3.Bucket, + "prefix": cfg.Reva.MetadataStorage.S3.Root, + }, + "s3ng": map[string]interface{}{ + "root": cfg.Reva.MetadataStorage.S3NG.Root, + "enable_home": false, + "user_layout": cfg.Reva.MetadataStorage.S3NG.UserLayout, + "s3.region": cfg.Reva.MetadataStorage.S3NG.Region, + "s3.access_key": cfg.Reva.MetadataStorage.S3NG.AccessKey, + "s3.secret_key": cfg.Reva.MetadataStorage.S3NG.SecretKey, + "s3.endpoint": cfg.Reva.MetadataStorage.S3NG.Endpoint, + "s3.bucket": cfg.Reva.MetadataStorage.S3NG.Bucket, + }, + } +} diff --git a/storage/pkg/command/storagedrivers/user.go b/storage/pkg/command/storagedrivers/user.go new file mode 100644 index 00000000000..9fe3f8b9820 --- /dev/null +++ b/storage/pkg/command/storagedrivers/user.go @@ -0,0 +1,131 @@ +package storagedrivers + +import ( + "github.com/owncloud/ocis/storage/pkg/config" +) + +func UserDrivers(cfg *config.Config) map[string]interface{} { + return map[string]interface{}{ + "eos": map[string]interface{}{ + "namespace": cfg.Reva.UserStorage.EOS.Root, + "shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace, + "uploads_namespace": cfg.Reva.UserStorage.EOS.UploadsNamespace, + "share_folder": cfg.Reva.UserStorage.EOS.ShareFolder, + "eos_binary": cfg.Reva.UserStorage.EOS.EosBinary, + "xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary, + "master_url": cfg.Reva.UserStorage.EOS.MasterURL, + "slave_url": cfg.Reva.UserStorage.EOS.SlaveURL, + "cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory, + "sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol, + "keytab": cfg.Reva.UserStorage.EOS.Keytab, + "single_username": cfg.Reva.UserStorage.EOS.SingleUsername, + "enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging, + "show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles, + "force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode, + "use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab, + "gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC, + }, + "eoshome": map[string]interface{}{ + "namespace": cfg.Reva.UserStorage.EOS.Root, + "shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace, + "uploads_namespace": cfg.Reva.UserStorage.EOS.UploadsNamespace, + "share_folder": cfg.Reva.UserStorage.EOS.ShareFolder, + "eos_binary": cfg.Reva.UserStorage.EOS.EosBinary, + "xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary, + "master_url": cfg.Reva.UserStorage.EOS.MasterURL, + "slave_url": cfg.Reva.UserStorage.EOS.SlaveURL, + "cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory, + "sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol, + "keytab": cfg.Reva.UserStorage.EOS.Keytab, + "single_username": cfg.Reva.UserStorage.EOS.SingleUsername, + "user_layout": cfg.Reva.UserStorage.EOS.UserLayout, + "enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging, + "show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles, + "force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode, + "use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab, + "gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC, + }, + "eosgrpc": map[string]interface{}{ + "namespace": cfg.Reva.UserStorage.EOS.Root, + "shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace, + "share_folder": cfg.Reva.UserStorage.EOS.ShareFolder, + "eos_binary": cfg.Reva.UserStorage.EOS.EosBinary, + "xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary, + "master_url": cfg.Reva.UserStorage.EOS.MasterURL, + "master_grpc_uri": cfg.Reva.UserStorage.EOS.GrpcURI, + "slave_url": cfg.Reva.UserStorage.EOS.SlaveURL, + "cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory, + "sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol, + "keytab": cfg.Reva.UserStorage.EOS.Keytab, + "single_username": cfg.Reva.UserStorage.EOS.SingleUsername, + "user_layout": cfg.Reva.UserStorage.EOS.UserLayout, + "enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging, + "show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles, + "force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode, + "use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab, + "enable_home": false, + "gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC, + }, + "local": map[string]interface{}{ + "root": cfg.Reva.UserStorage.Local.Root, + "share_folder": cfg.Reva.UserStorage.Local.ShareFolder, + }, + "localhome": map[string]interface{}{ + "root": cfg.Reva.UserStorage.Local.Root, + "share_folder": cfg.Reva.UserStorage.Local.ShareFolder, + "user_layout": cfg.Reva.UserStorage.Local.UserLayout, + }, + "owncloud": map[string]interface{}{ + "datadirectory": cfg.Reva.UserStorage.OwnCloud.Root, + "upload_info_dir": cfg.Reva.UserStorage.OwnCloud.UploadInfoDir, + "share_folder": cfg.Reva.UserStorage.OwnCloud.ShareFolder, + "user_layout": cfg.Reva.UserStorage.OwnCloud.UserLayout, + "redis": cfg.Reva.UserStorage.OwnCloud.Redis, + "enable_home": false, + "scan": cfg.Reva.UserStorage.OwnCloud.Scan, + "userprovidersvc": cfg.Reva.Users.Endpoint, + }, + "owncloudsql": map[string]interface{}{ + "datadirectory": cfg.Reva.UserStorage.OwnCloudSQL.Root, + "upload_info_dir": cfg.Reva.UserStorage.OwnCloudSQL.UploadInfoDir, + "share_folder": cfg.Reva.UserStorage.OwnCloudSQL.ShareFolder, + "user_layout": cfg.Reva.UserStorage.OwnCloudSQL.UserLayout, + "enable_home": false, + "dbusername": cfg.Reva.UserStorage.OwnCloudSQL.DBUsername, + "dbpassword": cfg.Reva.UserStorage.OwnCloudSQL.DBPassword, + "dbhost": cfg.Reva.UserStorage.OwnCloudSQL.DBHost, + "dbport": cfg.Reva.UserStorage.OwnCloudSQL.DBPort, + "dbname": cfg.Reva.UserStorage.OwnCloudSQL.DBName, + "userprovidersvc": cfg.Reva.Users.Endpoint, + }, + "ocis": map[string]interface{}{ + "root": cfg.Reva.UserStorage.OCIS.Root, + "enable_home": false, + "user_layout": cfg.Reva.UserStorage.OCIS.UserLayout, + "share_folder": cfg.Reva.UserStorage.OCIS.ShareFolder, + "treetime_accounting": true, + "treesize_accounting": true, + "owner": cfg.Reva.UserStorage.OCIS.ServiceUserUUID, // the accounts service system account uuid + }, + "s3": map[string]interface{}{ + "enable_home": false, + "region": cfg.Reva.UserStorage.S3.Region, + "access_key": cfg.Reva.UserStorage.S3.AccessKey, + "secret_key": cfg.Reva.UserStorage.S3.SecretKey, + "endpoint": cfg.Reva.UserStorage.S3.Endpoint, + "bucket": cfg.Reva.UserStorage.S3.Bucket, + "prefix": cfg.Reva.UserStorage.S3.Root, + }, + "s3ng": map[string]interface{}{ + "root": cfg.Reva.UserStorage.S3NG.Root, + "enable_home": false, + "user_layout": cfg.Reva.UserStorage.S3NG.UserLayout, + "share_folder": cfg.Reva.UserStorage.S3NG.ShareFolder, + "s3.region": cfg.Reva.UserStorage.S3NG.Region, + "s3.access_key": cfg.Reva.UserStorage.S3NG.AccessKey, + "s3.secret_key": cfg.Reva.UserStorage.S3NG.SecretKey, + "s3.endpoint": cfg.Reva.UserStorage.S3NG.Endpoint, + "s3.bucket": cfg.Reva.UserStorage.S3NG.Bucket, + }, + } +} diff --git a/storage/pkg/command/storagehome.go b/storage/pkg/command/storagehome.go index 547291faa65..bcea78c22ca 100644 --- a/storage/pkg/command/storagehome.go +++ b/storage/pkg/command/storagehome.go @@ -12,6 +12,7 @@ import ( "github.com/gofrs/uuid" "github.com/oklog/run" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" + "github.com/owncloud/ocis/storage/pkg/command/storagedrivers" "github.com/owncloud/ocis/storage/pkg/config" "github.com/owncloud/ocis/storage/pkg/flagset" "github.com/owncloud/ocis/storage/pkg/server/debug" @@ -43,16 +44,6 @@ func StorageHome(cfg *config.Config) *cli.Command { uuid := uuid.Must(uuid.NewV4()) pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid") - // override driver enable home option with home config - if cfg.Reva.Storages.Home.EnableHome { - cfg.Reva.Storages.Common.EnableHome = true - cfg.Reva.Storages.EOS.EnableHome = true - cfg.Reva.Storages.Local.EnableHome = true - cfg.Reva.Storages.OwnCloud.EnableHome = true - cfg.Reva.Storages.OwnCloudSQL.EnableHome = true - cfg.Reva.Storages.S3.EnableHome = true - cfg.Reva.Storages.S3NG.EnableHome = true - } rcfg := storageHomeConfigFromStruct(c, cfg) gr.Add(func() error { @@ -117,7 +108,7 @@ func storageHomeConfigFromStruct(c *cli.Context, cfg *config.Config) map[string] "services": map[string]interface{}{ "storageprovider": map[string]interface{}{ "driver": cfg.Reva.StorageHome.Driver, - "drivers": drivers(cfg), + "drivers": storagedrivers.HomeDrivers(cfg), "mount_path": cfg.Reva.StorageHome.MountPath, "mount_id": cfg.Reva.StorageHome.MountID, "expose_data_server": cfg.Reva.StorageHome.ExposeDataServer, @@ -134,7 +125,7 @@ func storageHomeConfigFromStruct(c *cli.Context, cfg *config.Config) map[string] "dataprovider": map[string]interface{}{ "prefix": cfg.Reva.StorageHome.HTTPPrefix, "driver": cfg.Reva.StorageHome.Driver, - "drivers": drivers(cfg), + "drivers": storagedrivers.HomeDrivers(cfg), "timeout": 86400, "insecure": true, "disable_tus": false, diff --git a/storage/pkg/command/storagemetadata.go b/storage/pkg/command/storagemetadata.go index 791df905f07..66b51988566 100644 --- a/storage/pkg/command/storagemetadata.go +++ b/storage/pkg/command/storagemetadata.go @@ -13,6 +13,7 @@ import ( "github.com/oklog/run" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/ocis-pkg/version" + "github.com/owncloud/ocis/storage/pkg/command/storagedrivers" "github.com/owncloud/ocis/storage/pkg/config" "github.com/owncloud/ocis/storage/pkg/flagset" "github.com/owncloud/ocis/storage/pkg/server/debug" @@ -32,17 +33,6 @@ func StorageMetadata(cfg *config.Config) *cli.Command { // TODO(refs) at this point it might make sense delegate log flags to each individual storage command. Flags: append(flagset.StorageMetadata(cfg), flagset.RootWithConfig(cfg)...), Category: "Extensions", - Before: func(c *cli.Context) error { - storageRoot := c.String("storage-root") - - cfg.Reva.Storages.OwnCloud.Root = storageRoot - cfg.Reva.Storages.EOS.Root = storageRoot - cfg.Reva.Storages.Local.Root = storageRoot - cfg.Reva.Storages.S3.Root = storageRoot - cfg.Reva.Storages.Home.Root = storageRoot - - return nil - }, Action: func(c *cli.Context) error { logger := NewLogger(cfg) tracing.Configure(cfg, logger) @@ -58,23 +48,6 @@ func StorageMetadata(cfg *config.Config) *cli.Command { defer cancel() pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.Must(uuid.NewV4()).String()+".pid") - - // Disable home because the metadata is stored independently - // of the user. This also means that a valid-token without any user-id - // is allowed to write to the metadata-storage. - cfg.Reva.Storages.Common.EnableHome = false - cfg.Reva.Storages.EOS.EnableHome = false - cfg.Reva.Storages.Local.EnableHome = false - cfg.Reva.Storages.OwnCloud.EnableHome = false - cfg.Reva.Storages.S3.EnableHome = false - - // We need this hack because the metadata storage can define STORAGE_METADATA_ROOT which has the same destination as - // STORAGE_DRIVER_OCIS_ROOT. When both variables are set one storage will always be out of sync. Ensure the - // metadata storage root is never overridden. This is the kind of stateful code that make you want to cry blood. - if os.Getenv("STORAGE_METADATA_ROOT") != "" && os.Getenv("STORAGE_DRIVER_OCIS_ROOT") != "" { - cfg.Reva.Storages.Common.Root = os.Getenv("STORAGE_METADATA_ROOT") - } - rcfg := storageMetadataFromStruct(c, cfg) gr.Add(func() error { @@ -160,7 +133,7 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in "storageprovider": map[string]interface{}{ "mount_path": "/meta", "driver": cfg.Reva.StorageMetadata.Driver, - "drivers": drivers(cfg), + "drivers": storagedrivers.MetadataDrivers(cfg), "data_server_url": cfg.Reva.StorageMetadata.DataServerURL, "tmp_folder": cfg.Reva.StorageMetadata.TempFolder, }, @@ -174,7 +147,7 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in "dataprovider": map[string]interface{}{ "prefix": "data", "driver": cfg.Reva.StorageMetadata.Driver, - "drivers": drivers(cfg), + "drivers": storagedrivers.MetadataDrivers(cfg), "timeout": 86400, "insecure": true, "disable_tus": true, diff --git a/storage/pkg/command/storageusers.go b/storage/pkg/command/storageusers.go index 581b79c61f7..85b1dfd95b9 100644 --- a/storage/pkg/command/storageusers.go +++ b/storage/pkg/command/storageusers.go @@ -11,6 +11,7 @@ import ( "github.com/oklog/run" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/ocis-pkg/sync" + "github.com/owncloud/ocis/storage/pkg/command/storagedrivers" "github.com/owncloud/ocis/storage/pkg/config" "github.com/owncloud/ocis/storage/pkg/flagset" "github.com/owncloud/ocis/storage/pkg/server/debug" @@ -43,16 +44,6 @@ func StorageUsers(cfg *config.Config) *cli.Command { uuid := uuid.Must(uuid.NewV4()) pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid") - // override driver enable home option with home config - if cfg.Reva.Storages.Home.EnableHome { - cfg.Reva.Storages.Common.EnableHome = true - cfg.Reva.Storages.EOS.EnableHome = true - cfg.Reva.Storages.Local.EnableHome = true - cfg.Reva.Storages.OwnCloud.EnableHome = true - cfg.Reva.Storages.OwnCloudSQL.EnableHome = true - cfg.Reva.Storages.S3.EnableHome = true - cfg.Reva.Storages.S3NG.EnableHome = true - } rcfg := storageUsersConfigFromStruct(c, cfg) gr.Add(func() error { @@ -117,7 +108,7 @@ func storageUsersConfigFromStruct(c *cli.Context, cfg *config.Config) map[string "services": map[string]interface{}{ "storageprovider": map[string]interface{}{ "driver": cfg.Reva.StorageUsers.Driver, - "drivers": drivers(cfg), + "drivers": storagedrivers.UserDrivers(cfg), "mount_path": cfg.Reva.StorageUsers.MountPath, "mount_id": cfg.Reva.StorageUsers.MountID, "expose_data_server": cfg.Reva.StorageUsers.ExposeDataServer, @@ -134,7 +125,7 @@ func storageUsersConfigFromStruct(c *cli.Context, cfg *config.Config) map[string "dataprovider": map[string]interface{}{ "prefix": cfg.Reva.StorageUsers.HTTPPrefix, "driver": cfg.Reva.StorageUsers.Driver, - "drivers": drivers(cfg), + "drivers": storagedrivers.UserDrivers(cfg), "timeout": 86400, "insecure": true, "disable_tus": false, diff --git a/storage/pkg/config/config.go b/storage/pkg/config/config.go index 7d34e718716..de770d06d61 100644 --- a/storage/pkg/config/config.go +++ b/storage/pkg/config/config.go @@ -201,14 +201,12 @@ type PublicStorage struct { // StorageConfig combines all available storage driver configuration parts. type StorageConfig struct { - Home DriverCommon EOS DriverEOS Local DriverCommon OwnCloud DriverOwnCloud OwnCloudSQL DriverOwnCloudSQL S3 DriverS3 S3NG DriverS3NG - Common DriverCommon OCIS DriverOCIS // TODO checksums ... figure out what that is supposed to do } @@ -289,6 +287,8 @@ type DriverEOS struct { // DriverOCIS defines the available oCIS storage driver configuration. type DriverOCIS struct { + DriverCommon + ServiceUserUUID string } @@ -438,7 +438,8 @@ type Reva struct { UserOwnCloudSQL UserOwnCloudSQL OCDav OCDav Archiver Archiver - Storages StorageConfig + UserStorage StorageConfig + MetadataStorage StorageConfig // Ports are used to configure which services to start on which port Frontend FrontendPort DataGateway DataGatewayPort diff --git a/storage/pkg/flagset/drivereos.go b/storage/pkg/flagset/drivereos.go deleted file mode 100644 index b6ce8c7548b..00000000000 --- a/storage/pkg/flagset/drivereos.go +++ /dev/null @@ -1,135 +0,0 @@ -package flagset - -import ( - "os" - - "github.com/owncloud/ocis/ocis-pkg/flags" - - "github.com/owncloud/ocis/storage/pkg/config" - "github.com/urfave/cli/v2" -) - -// DriverEOSWithConfig applies cfg to the root flagset -func DriverEOSWithConfig(cfg *config.Config) []cli.Flag { - return []cli.Flag{ - - &cli.StringFlag{ - Name: "storage-eos-namespace", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.EOS.Root, "/eos/dockertest/reva"), - Usage: "Namespace for metadata operations", - EnvVars: []string{"STORAGE_DRIVER_EOS_NAMESPACE"}, - Destination: &cfg.Reva.Storages.EOS.Root, - }, - &cli.StringFlag{ - Name: "storage-eos-shadow-namespace", - // Defaults to path.Join(c.Namespace, ".shadow") - Usage: "Shadow namespace where share references are stored", - EnvVars: []string{"STORAGE_DRIVER_EOS_SHADOW_NAMESPACE"}, - Destination: &cfg.Reva.Storages.EOS.ShadowNamespace, - }, - &cli.StringFlag{ - Name: "storage-eos-share-folder", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.EOS.ShareFolder, "/Shares"), - Usage: "name of the share folder", - EnvVars: []string{"STORAGE_DRIVER_EOS_SHARE_FOLDER"}, - Destination: &cfg.Reva.Storages.EOS.ShareFolder, - }, - &cli.StringFlag{ - Name: "storage-eos-binary", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.EOS.EosBinary, "/usr/bin/eos"), - Usage: "Location of the eos binary", - EnvVars: []string{"STORAGE_DRIVER_EOS_BINARY"}, - Destination: &cfg.Reva.Storages.EOS.EosBinary, - }, - &cli.StringFlag{ - Name: "storage-eos-xrdcopy-binary", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.EOS.XrdcopyBinary, "/usr/bin/xrdcopy"), - Usage: "Location of the xrdcopy binary", - EnvVars: []string{"STORAGE_DRIVER_EOS_XRDCOPY_BINARY"}, - Destination: &cfg.Reva.Storages.EOS.XrdcopyBinary, - }, - &cli.StringFlag{ - Name: "storage-eos-master-url", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.EOS.MasterURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), - Usage: "URL of the Master EOS MGM", - EnvVars: []string{"STORAGE_DRIVER_EOS_MASTER_URL"}, - Destination: &cfg.Reva.Storages.EOS.MasterURL, - }, - &cli.StringFlag{ - Name: "storage-eos-slave-url", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.EOS.SlaveURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), - Usage: "URL of the Slave EOS MGM", - EnvVars: []string{"STORAGE_DRIVER_EOS_SLAVE_URL"}, - Destination: &cfg.Reva.Storages.EOS.SlaveURL, - }, - &cli.StringFlag{ - Name: "storage-eos-cache-directory", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.EOS.CacheDirectory, os.TempDir()), - Usage: "Location on the local fs where to store reads", - EnvVars: []string{"STORAGE_DRIVER_EOS_CACHE_DIRECTORY"}, - Destination: &cfg.Reva.Storages.EOS.CacheDirectory, - }, - &cli.BoolFlag{ - Name: "storage-eos-enable-logging", - Usage: "Enables logging of the commands executed", - EnvVars: []string{"STORAGE_DRIVER_EOS_ENABLE_LOGGING"}, - Destination: &cfg.Reva.Storages.EOS.EnableLogging, - }, - &cli.BoolFlag{ - Name: "storage-eos-show-hidden-sysfiles", - Usage: "show internal EOS files like .sys.v# and .sys.a# files.", - EnvVars: []string{"STORAGE_DRIVER_EOS_SHOW_HIDDEN_SYSFILES"}, - Destination: &cfg.Reva.Storages.EOS.ShowHiddenSysFiles, - }, - &cli.BoolFlag{ - Name: "storage-eos-force-singleuser-mode", - Usage: "force connections to EOS to use SingleUsername", - EnvVars: []string{"STORAGE_DRIVER_EOS_FORCE_SINGLEUSER_MODE"}, - Destination: &cfg.Reva.Storages.EOS.ForceSingleUserMode, - }, - &cli.BoolFlag{ - Name: "storage-eos-use-keytab", - Usage: "authenticate requests by using an EOS keytab", - EnvVars: []string{"STORAGE_DRIVER_EOS_USE_KEYTAB"}, - Destination: &cfg.Reva.Storages.EOS.UseKeytab, - }, - &cli.BoolFlag{ - Name: "storage-eos-enable-home", - Usage: "enable the creation of home directories", - EnvVars: []string{"STORAGE_DRIVER_EOS_ENABLE_HOME"}, - Destination: &cfg.Reva.Storages.EOS.EnableHome, - }, - &cli.StringFlag{ - Name: "storage-eos-sec-protocol", - Usage: "the xrootd security protocol to use between the server and EOS", - EnvVars: []string{"STORAGE_DRIVER_EOS_SEC_PROTOCOL"}, - Destination: &cfg.Reva.Storages.EOS.SecProtocol, - }, - &cli.StringFlag{ - Name: "storage-eos-keytab", - Usage: "the location of the keytab to use to authenticate to EOS", - EnvVars: []string{"STORAGE_DRIVER_EOS_KEYTAB"}, - Destination: &cfg.Reva.Storages.EOS.Keytab, - }, - &cli.StringFlag{ - Name: "storage-eos-single-username", - Usage: "the username to use when SingleUserMode is enabled", - EnvVars: []string{"STORAGE_DRIVER_EOS_SINGLE_USERNAME"}, - Destination: &cfg.Reva.Storages.EOS.SingleUsername, - }, - &cli.StringFlag{ - Name: "storage-eos-layout", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.EOS.UserLayout, "{{substr 0 1 .Username}}/{{.Username}}"), - Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.UsernameLower}} and {{.Provider}} also supports prefixing dirs: "{{.UsernamePrefixCount.2}}/{{.UsernameLower}}" will turn "Einstein" into "Ei/Einstein" `, - EnvVars: []string{"STORAGE_DRIVER_EOS_LAYOUT"}, - Destination: &cfg.Reva.Storages.EOS.UserLayout, - }, - &cli.StringFlag{ - Name: "storage-eos-reva-gateway-addr", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.EOS.GatewaySVC, "127.0.0.1:9142"), - Usage: "Address of REVA gateway endpoint", - EnvVars: []string{"REVA_GATEWAY"}, - Destination: &cfg.Reva.Storages.EOS.GatewaySVC, - }, - } -} diff --git a/storage/pkg/flagset/driverocis.go b/storage/pkg/flagset/driverocis.go deleted file mode 100644 index 08e067cbfbb..00000000000 --- a/storage/pkg/flagset/driverocis.go +++ /dev/null @@ -1,41 +0,0 @@ -package flagset - -import ( - "github.com/owncloud/ocis/ocis-pkg/flags" - "github.com/owncloud/ocis/storage/pkg/config" - "github.com/urfave/cli/v2" -) - -// DriverOCISWithConfig applies cfg to the root flagset -func DriverOCISWithConfig(cfg *config.Config) []cli.Flag { - return []cli.Flag{ - &cli.StringFlag{ - Name: "storage-ocis-root", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.Common.Root, "/var/tmp/ocis/storage/users"), - Usage: "the path to the local storage root", - EnvVars: []string{"STORAGE_DRIVER_OCIS_ROOT"}, - Destination: &cfg.Reva.Storages.Common.Root, - }, - &cli.BoolFlag{ - Name: "storage-ocis-enable-home", - Value: flags.OverrideDefaultBool(cfg.Reva.Storages.Common.EnableHome, false), - Usage: "enable the creation of home storages", - EnvVars: []string{"STORAGE_DRIVER_OCIS_ENABLE_HOME"}, - Destination: &cfg.Reva.Storages.Common.EnableHome, - }, - &cli.StringFlag{ - Name: "storage-ocis-layout", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.Common.UserLayout, "{{.Id.OpaqueId}}"), - Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, - EnvVars: []string{"STORAGE_DRIVER_OCIS_LAYOUT"}, - Destination: &cfg.Reva.Storages.Common.UserLayout, - }, - &cli.StringFlag{ - Name: "service-user-uuid", - Value: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", - Usage: "uuid of the internal service user", - EnvVars: []string{"STORAGE_DRIVER_OCIS_SERVICE_USER_UUID"}, - Destination: &cfg.Reva.Storages.OCIS.ServiceUserUUID, - }, - } -} diff --git a/storage/pkg/flagset/driverowncloud.go b/storage/pkg/flagset/driverowncloud.go deleted file mode 100644 index 20c0071f1d1..00000000000 --- a/storage/pkg/flagset/driverowncloud.go +++ /dev/null @@ -1,62 +0,0 @@ -package flagset - -import ( - "github.com/owncloud/ocis/ocis-pkg/flags" - "github.com/owncloud/ocis/storage/pkg/config" - "github.com/urfave/cli/v2" -) - -// DriverOwnCloudWithConfig applies cfg to the root flagset -func DriverOwnCloudWithConfig(cfg *config.Config) []cli.Flag { - return []cli.Flag{ - &cli.StringFlag{ - Name: "storage-owncloud-datadir", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.OwnCloud.Root, "/var/tmp/ocis/storage/owncloud"), - Usage: "the path to the owncloud data directory", - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUD_DATADIR"}, - Destination: &cfg.Reva.Storages.OwnCloud.Root, - }, - &cli.StringFlag{ - Name: "storage-owncloud-uploadinfo-dir", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.OwnCloud.UploadInfoDir, "/var/tmp/ocis/storage/uploadinfo"), - Usage: "the path to the tus upload info directory", - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUD_UPLOADINFO_DIR"}, - Destination: &cfg.Reva.Storages.OwnCloud.UploadInfoDir, - }, - &cli.StringFlag{ - Name: "storage-owncloud-share-folder", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.OwnCloud.ShareFolder, "/Shares"), - Usage: "name of the shares folder", - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUD_SHARE_FOLDER"}, - Destination: &cfg.Reva.Storages.OwnCloud.ShareFolder, - }, - &cli.BoolFlag{ - Name: "storage-owncloud-scan", - Value: flags.OverrideDefaultBool(cfg.Reva.Storages.OwnCloud.Scan, true), - Usage: "scan files on startup to add fileids", - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUD_SCAN"}, - Destination: &cfg.Reva.Storages.OwnCloud.Scan, - }, - &cli.StringFlag{ - Name: "storage-owncloud-redis", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.OwnCloud.Redis, ":6379"), - Usage: "the address of the redis server", - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUD_REDIS_ADDR"}, - Destination: &cfg.Reva.Storages.OwnCloud.Redis, - }, - &cli.BoolFlag{ - Name: "storage-owncloud-enable-home", - Value: flags.OverrideDefaultBool(cfg.Reva.Storages.OwnCloud.EnableHome, false), - Usage: "enable the creation of home storages", - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUD_ENABLE_HOME"}, - Destination: &cfg.Reva.Storages.OwnCloud.EnableHome, - }, - &cli.StringFlag{ - Name: "storage-owncloud-layout", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.OwnCloud.UserLayout, "{{.Id.OpaqueId}}"), - Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUD_LAYOUT"}, - Destination: &cfg.Reva.Storages.OwnCloud.UserLayout, - }, - } -} diff --git a/storage/pkg/flagset/driverowncloudsql.go b/storage/pkg/flagset/driverowncloudsql.go deleted file mode 100644 index ed42d2df7d6..00000000000 --- a/storage/pkg/flagset/driverowncloudsql.go +++ /dev/null @@ -1,83 +0,0 @@ -package flagset - -import ( - "github.com/owncloud/ocis/ocis-pkg/flags" - "github.com/owncloud/ocis/storage/pkg/config" - "github.com/urfave/cli/v2" -) - -// DriverOwnCloudSQLWithConfig applies cfg to the root flagset -func DriverOwnCloudSQLWithConfig(cfg *config.Config) []cli.Flag { - return []cli.Flag{ - &cli.StringFlag{ - Name: "storage-owncloudsql-datadir", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.OwnCloudSQL.Root, "/var/tmp/ocis/storage/owncloud"), - Usage: "the path to the owncloudsql data directory", - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUDSQL_DATADIR"}, - Destination: &cfg.Reva.Storages.OwnCloudSQL.Root, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-uploadinfo-dir", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.OwnCloudSQL.UploadInfoDir, "/var/tmp/ocis/storage/uploadinfo"), - Usage: "the path to the tus upload info directory", - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR"}, - Destination: &cfg.Reva.Storages.OwnCloudSQL.UploadInfoDir, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-share-folder", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.OwnCloudSQL.ShareFolder, "/Shares"), - Usage: "name of the shares folder", - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUDSQL_SHARE_FOLDER"}, - Destination: &cfg.Reva.Storages.OwnCloudSQL.ShareFolder, - }, - &cli.BoolFlag{ - Name: "storage-owncloudsql-enable-home", - Value: flags.OverrideDefaultBool(cfg.Reva.Storages.OwnCloudSQL.EnableHome, false), - Usage: "enable the creation of home storages", - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUDSQL_ENABLE_HOME"}, - Destination: &cfg.Reva.Storages.OwnCloudSQL.EnableHome, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-layout", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.OwnCloudSQL.UserLayout, "{{.Username}}"), - Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUDSQL_LAYOUT"}, - Destination: &cfg.Reva.Storages.OwnCloudSQL.UserLayout, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-dbusername", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.OwnCloudSQL.DBUsername, "owncloud"), - Usage: `"username for accessing the database" `, - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUDSQL_DBUSERNAME"}, - Destination: &cfg.Reva.Storages.OwnCloudSQL.DBUsername, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-dbpassword", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.OwnCloudSQL.DBPassword, "owncloud"), - Usage: `"password for accessing the database" `, - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUDSQL_DBPASSWORD"}, - Destination: &cfg.Reva.Storages.OwnCloudSQL.DBPassword, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-dbhost", - Value: cfg.Reva.Storages.OwnCloudSQL.DBHost, - Usage: `"the database hostname or IP address" `, - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUDSQL_DBHOST"}, - Destination: &cfg.Reva.Storages.OwnCloudSQL.DBHost, - }, - &cli.IntFlag{ - Name: "storage-owncloudsql-dbport", - Value: flags.OverrideDefaultInt(cfg.Reva.Storages.OwnCloudSQL.DBPort, 3306), - Usage: `"port the database listens on" `, - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUDSQL_DBPORT"}, - Destination: &cfg.Reva.Storages.OwnCloudSQL.DBPort, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-dbname", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.OwnCloudSQL.DBName, "owncloud"), - Usage: `"the database name" `, - EnvVars: []string{"STORAGE_DRIVER_OWNCLOUDSQL_DBNAME"}, - Destination: &cfg.Reva.Storages.OwnCloudSQL.DBName, - }, - } -} diff --git a/storage/pkg/flagset/frontend.go b/storage/pkg/flagset/frontend.go index e7329dba910..1db7120eb53 100644 --- a/storage/pkg/flagset/frontend.go +++ b/storage/pkg/flagset/frontend.go @@ -3,6 +3,7 @@ package flagset import ( "github.com/owncloud/ocis/ocis-pkg/flags" "github.com/owncloud/ocis/storage/pkg/config" + "github.com/owncloud/ocis/storage/pkg/flagset/userdrivers" "github.com/urfave/cli/v2" ) @@ -254,7 +255,7 @@ func FrontendWithConfig(cfg *config.Config) []cli.Flag { flags = append(flags, DebugWithConfig(cfg)...) flags = append(flags, SecretWithConfig(cfg)...) flags = append(flags, SharingSQLWithConfig(cfg)...) - flags = append(flags, DriverEOSWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverEOSWithConfig(cfg)...) return flags } diff --git a/storage/pkg/flagset/metadatadrivers/drivereos.go b/storage/pkg/flagset/metadatadrivers/drivereos.go new file mode 100644 index 00000000000..ec66e4200a2 --- /dev/null +++ b/storage/pkg/flagset/metadatadrivers/drivereos.go @@ -0,0 +1,129 @@ +package metadatadrivers + +import ( + "os" + + "github.com/owncloud/ocis/ocis-pkg/flags" + + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverEOSWithConfig applies cfg to the root flagset +func DriverEOSWithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + + &cli.StringFlag{ + Name: "storage-eos-namespace", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.Root, "/eos/dockertest/reva"), + Usage: "Namespace for metadata operations", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_NAMESPACE"}, + Destination: &cfg.Reva.UserStorage.EOS.Root, + }, + &cli.StringFlag{ + Name: "storage-eos-shadow-namespace", + // Defaults to path.Join(c.Namespace, ".shadow") + Usage: "Shadow namespace where share references are stored", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SHADOW_NAMESPACE"}, + Destination: &cfg.Reva.UserStorage.EOS.ShadowNamespace, + }, + &cli.StringFlag{ + Name: "storage-eos-share-folder", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.ShareFolder, "/Shares"), + Usage: "name of the share folder", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SHARE_FOLDER"}, + Destination: &cfg.Reva.UserStorage.EOS.ShareFolder, + }, + &cli.StringFlag{ + Name: "storage-eos-binary", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.EosBinary, "/usr/bin/eos"), + Usage: "Location of the eos binary", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_BINARY"}, + Destination: &cfg.Reva.UserStorage.EOS.EosBinary, + }, + &cli.StringFlag{ + Name: "storage-eos-xrdcopy-binary", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.XrdcopyBinary, "/usr/bin/xrdcopy"), + Usage: "Location of the xrdcopy binary", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_XRDCOPY_BINARY"}, + Destination: &cfg.Reva.UserStorage.EOS.XrdcopyBinary, + }, + &cli.StringFlag{ + Name: "storage-eos-master-url", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.MasterURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), + Usage: "URL of the Master EOS MGM", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_MASTER_URL"}, + Destination: &cfg.Reva.UserStorage.EOS.MasterURL, + }, + &cli.StringFlag{ + Name: "storage-eos-slave-url", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.SlaveURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), + Usage: "URL of the Slave EOS MGM", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SLAVE_URL"}, + Destination: &cfg.Reva.UserStorage.EOS.SlaveURL, + }, + &cli.StringFlag{ + Name: "storage-eos-cache-directory", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.CacheDirectory, os.TempDir()), + Usage: "Location on the local fs where to store reads", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_CACHE_DIRECTORY"}, + Destination: &cfg.Reva.UserStorage.EOS.CacheDirectory, + }, + &cli.BoolFlag{ + Name: "storage-eos-enable-logging", + Usage: "Enables logging of the commands executed", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_ENABLE_LOGGING"}, + Destination: &cfg.Reva.UserStorage.EOS.EnableLogging, + }, + &cli.BoolFlag{ + Name: "storage-eos-show-hidden-sysfiles", + Usage: "show internal EOS files like .sys.v# and .sys.a# files.", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SHOW_HIDDEN_SYSFILES"}, + Destination: &cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles, + }, + &cli.BoolFlag{ + Name: "storage-eos-force-singleuser-mode", + Usage: "force connections to EOS to use SingleUsername", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_FORCE_SINGLEUSER_MODE"}, + Destination: &cfg.Reva.UserStorage.EOS.ForceSingleUserMode, + }, + &cli.BoolFlag{ + Name: "storage-eos-use-keytab", + Usage: "authenticate requests by using an EOS keytab", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_USE_KEYTAB"}, + Destination: &cfg.Reva.UserStorage.EOS.UseKeytab, + }, + &cli.StringFlag{ + Name: "storage-eos-sec-protocol", + Usage: "the xrootd security protocol to use between the server and EOS", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SEC_PROTOCOL"}, + Destination: &cfg.Reva.UserStorage.EOS.SecProtocol, + }, + &cli.StringFlag{ + Name: "storage-eos-keytab", + Usage: "the location of the keytab to use to authenticate to EOS", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_KEYTAB"}, + Destination: &cfg.Reva.UserStorage.EOS.Keytab, + }, + &cli.StringFlag{ + Name: "storage-eos-single-username", + Usage: "the username to use when SingleUserMode is enabled", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SINGLE_USERNAME"}, + Destination: &cfg.Reva.UserStorage.EOS.SingleUsername, + }, + &cli.StringFlag{ + Name: "storage-eos-layout", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.UserLayout, "{{substr 0 1 .Username}}/{{.Username}}"), + Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.UsernameLower}} and {{.Provider}} also supports prefixing dirs: "{{.UsernamePrefixCount.2}}/{{.UsernameLower}}" will turn "Einstein" into "Ei/Einstein" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_LAYOUT"}, + Destination: &cfg.Reva.UserStorage.EOS.UserLayout, + }, + &cli.StringFlag{ + Name: "storage-eos-reva-gateway-addr", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.GatewaySVC, "127.0.0.1:9142"), + Usage: "Address of REVA gateway endpoint", + EnvVars: []string{"REVA_GATEWAY"}, + Destination: &cfg.Reva.UserStorage.EOS.GatewaySVC, + }, + } +} diff --git a/storage/pkg/flagset/metadatadrivers/driverlocal.go b/storage/pkg/flagset/metadatadrivers/driverlocal.go new file mode 100644 index 00000000000..f1aac4aa3cf --- /dev/null +++ b/storage/pkg/flagset/metadatadrivers/driverlocal.go @@ -0,0 +1,20 @@ +package metadatadrivers + +import ( + "github.com/owncloud/ocis/ocis-pkg/flags" + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverLocalWithConfig applies cfg to the root flagset +func DriverLocalWithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "storage-local-root", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.Local.Root, "/var/tmp/ocis/storage/local"), + Usage: "the path to the local storage root", + EnvVars: []string{"STORAGE_METADATA_DRIVER_LOCAL_ROOT"}, + Destination: &cfg.Reva.MetadataStorage.Local.Root, + }, + } +} diff --git a/storage/pkg/flagset/metadatadrivers/driverocis.go b/storage/pkg/flagset/metadatadrivers/driverocis.go new file mode 100644 index 00000000000..b9c89a36963 --- /dev/null +++ b/storage/pkg/flagset/metadatadrivers/driverocis.go @@ -0,0 +1,34 @@ +package metadatadrivers + +import ( + "github.com/owncloud/ocis/ocis-pkg/flags" + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverOCISWithConfig applies cfg to the root flagset +func DriverOCISWithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "storage-ocis-root", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OCIS.Root, "/var/tmp/ocis/storage/metadata"), + Usage: "the path to the local storage root", + EnvVars: []string{"STORAGE_METADATA_DRIVER_OCIS_ROOT"}, + Destination: &cfg.Reva.MetadataStorage.OCIS.Root, + }, + &cli.StringFlag{ + Name: "storage-ocis-layout", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OCIS.UserLayout, "{{.Id.OpaqueId}}"), + Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_OCIS_LAYOUT"}, + Destination: &cfg.Reva.MetadataStorage.OCIS.UserLayout, + }, + &cli.StringFlag{ + Name: "service-user-uuid", + Value: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", + Usage: "uuid of the internal service user", + EnvVars: []string{"STORAGE_METADATA_DRIVER_OCIS_SERVICE_USER_UUID"}, + Destination: &cfg.Reva.MetadataStorage.OCIS.ServiceUserUUID, + }, + } +} diff --git a/storage/pkg/flagset/metadatadrivers/driverowncloud.go b/storage/pkg/flagset/metadatadrivers/driverowncloud.go new file mode 100644 index 00000000000..cf727526b0c --- /dev/null +++ b/storage/pkg/flagset/metadatadrivers/driverowncloud.go @@ -0,0 +1,55 @@ +package metadatadrivers + +import ( + "github.com/owncloud/ocis/ocis-pkg/flags" + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverOwnCloudWithConfig applies cfg to the root flagset +func DriverOwnCloudWithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "storage-owncloud-datadir", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.Root, "/var/tmp/ocis/storage/owncloud"), + Usage: "the path to the owncloud data directory", + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_DATADIR"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloud.Root, + }, + &cli.StringFlag{ + Name: "storage-owncloud-uploadinfo-dir", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.UploadInfoDir, "/var/tmp/ocis/storage/uploadinfo"), + Usage: "the path to the tus upload info directory", + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_UPLOADINFO_DIR"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloud.UploadInfoDir, + }, + &cli.StringFlag{ + Name: "storage-owncloud-share-folder", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.ShareFolder, "/Shares"), + Usage: "name of the shares folder", + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_SHARE_FOLDER"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloud.ShareFolder, + }, + &cli.BoolFlag{ + Name: "storage-owncloud-scan", + Value: flags.OverrideDefaultBool(cfg.Reva.MetadataStorage.OwnCloud.Scan, true), + Usage: "scan files on startup to add fileids", + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_SCAN"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloud.Scan, + }, + &cli.StringFlag{ + Name: "storage-owncloud-redis", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.Redis, ":6379"), + Usage: "the address of the redis server", + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_REDIS_ADDR"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloud.Redis, + }, + &cli.StringFlag{ + Name: "storage-owncloud-layout", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.UserLayout, "{{.Id.OpaqueId}}"), + Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_LAYOUT"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloud.UserLayout, + }, + } +} diff --git a/storage/pkg/flagset/metadatadrivers/driverowncloudsql.go b/storage/pkg/flagset/metadatadrivers/driverowncloudsql.go new file mode 100644 index 00000000000..ca444348aaf --- /dev/null +++ b/storage/pkg/flagset/metadatadrivers/driverowncloudsql.go @@ -0,0 +1,76 @@ +package metadatadrivers + +import ( + "github.com/owncloud/ocis/ocis-pkg/flags" + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverOwnCloudSQLWithConfig applies cfg to the root flagset +func DriverOwnCloudSQLWithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "storage-owncloudsql-datadir", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.Root, "/var/tmp/ocis/storage/owncloud"), + Usage: "the path to the owncloudsql data directory", + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DATADIR"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.Root, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-uploadinfo-dir", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.UploadInfoDir, "/var/tmp/ocis/storage/uploadinfo"), + Usage: "the path to the tus upload info directory", + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.UploadInfoDir, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-share-folder", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.ShareFolder, "/Shares"), + Usage: "name of the shares folder", + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_SHARE_FOLDER"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.ShareFolder, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-layout", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.UserLayout, "{{.Username}}"), + Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_LAYOUT"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.UserLayout, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-dbusername", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.DBUsername, "owncloud"), + Usage: `"username for accessing the database" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBUSERNAME"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBUsername, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-dbpassword", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.DBPassword, "owncloud"), + Usage: `"password for accessing the database" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBPASSWORD"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBPassword, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-dbhost", + Value: cfg.Reva.MetadataStorage.OwnCloudSQL.DBHost, + Usage: `"the database hostname or IP address" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBHOST"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBHost, + }, + &cli.IntFlag{ + Name: "storage-owncloudsql-dbport", + Value: flags.OverrideDefaultInt(cfg.Reva.MetadataStorage.OwnCloudSQL.DBPort, 3306), + Usage: `"port the database listens on" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBPORT"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBPort, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-dbname", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.DBName, "owncloud"), + Usage: `"the database name" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBNAME"}, + Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBName, + }, + } +} diff --git a/storage/pkg/flagset/drivers3ng.go b/storage/pkg/flagset/metadatadrivers/drivers3ng.go similarity index 51% rename from storage/pkg/flagset/drivers3ng.go rename to storage/pkg/flagset/metadatadrivers/drivers3ng.go index fb1b6e924ae..b94e84a32c3 100644 --- a/storage/pkg/flagset/drivers3ng.go +++ b/storage/pkg/flagset/metadatadrivers/drivers3ng.go @@ -1,4 +1,4 @@ -package flagset +package metadatadrivers import ( "github.com/owncloud/ocis/ocis-pkg/flags" @@ -11,59 +11,52 @@ func DriverS3NGWithConfig(cfg *config.Config) []cli.Flag { return []cli.Flag{ &cli.StringFlag{ Name: "storage-s3ng-root", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.Common.Root, "/var/tmp/ocis/storage/users"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.S3NG.Root, "/var/tmp/ocis/storage/metadata"), Usage: "the path to the local storage root", - EnvVars: []string{"STORAGE_DRIVER_S3NG_ROOT"}, - Destination: &cfg.Reva.Storages.Common.Root, - }, - &cli.BoolFlag{ - Name: "storage-s3ng-enable-home", - Value: flags.OverrideDefaultBool(cfg.Reva.Storages.Common.EnableHome, false), - Usage: "enable the creation of home storages", - EnvVars: []string{"STORAGE_DRIVER_S3NG_ENABLE_HOME"}, - Destination: &cfg.Reva.Storages.Common.EnableHome, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3NG_ROOT"}, + Destination: &cfg.Reva.MetadataStorage.S3NG.Root, }, &cli.StringFlag{ Name: "storage-s3ng-layout", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.Common.UserLayout, "{{.Id.OpaqueId}}"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.S3NG.UserLayout, "{{.Id.OpaqueId}}"), Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, - EnvVars: []string{"STORAGE_DRIVER_S3NG_LAYOUT"}, - Destination: &cfg.Reva.Storages.Common.UserLayout, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3NG_LAYOUT"}, + Destination: &cfg.Reva.MetadataStorage.S3NG.UserLayout, }, &cli.StringFlag{ Name: "storage-s3ng-region", Value: "default", Usage: `"the s3 region" `, - EnvVars: []string{"STORAGE_DRIVER_S3NG_REGION"}, - Destination: &cfg.Reva.Storages.S3NG.Region, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3NG_REGION"}, + Destination: &cfg.Reva.MetadataStorage.S3NG.Region, }, &cli.StringFlag{ Name: "storage-s3ng-accesskey", Value: "", Usage: `"the s3 access key" `, - EnvVars: []string{"STORAGE_DRIVER_S3NG_ACCESS_KEY"}, - Destination: &cfg.Reva.Storages.S3NG.AccessKey, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3NG_ACCESS_KEY"}, + Destination: &cfg.Reva.MetadataStorage.S3NG.AccessKey, }, &cli.StringFlag{ Name: "storage-s3ng-secretkey", Value: "", Usage: `"the secret s3 api key" `, - EnvVars: []string{"STORAGE_DRIVER_S3NG_SECRET_KEY"}, - Destination: &cfg.Reva.Storages.S3NG.SecretKey, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3NG_SECRET_KEY"}, + Destination: &cfg.Reva.MetadataStorage.S3NG.SecretKey, }, &cli.StringFlag{ Name: "storage-s3ng-endpoint", Value: "", Usage: `"s3 compatible API endpoint" `, - EnvVars: []string{"STORAGE_DRIVER_S3NG_ENDPOINT"}, - Destination: &cfg.Reva.Storages.S3NG.Endpoint, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3NG_ENDPOINT"}, + Destination: &cfg.Reva.MetadataStorage.S3NG.Endpoint, }, &cli.StringFlag{ Name: "storage-s3ng-bucket", Value: "", Usage: `"bucket where the data will be stored in`, - EnvVars: []string{"STORAGE_DRIVER_S3NG_BUCKET"}, - Destination: &cfg.Reva.Storages.S3NG.Bucket, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3NG_BUCKET"}, + Destination: &cfg.Reva.MetadataStorage.S3NG.Bucket, }, } } diff --git a/storage/pkg/flagset/storagehome.go b/storage/pkg/flagset/storagehome.go index 03a78137e60..9c989668c0d 100644 --- a/storage/pkg/flagset/storagehome.go +++ b/storage/pkg/flagset/storagehome.go @@ -3,6 +3,7 @@ package flagset import ( "github.com/owncloud/ocis/ocis-pkg/flags" "github.com/owncloud/ocis/storage/pkg/config" + "github.com/owncloud/ocis/storage/pkg/flagset/userdrivers" "github.com/urfave/cli/v2" ) @@ -126,13 +127,6 @@ func StorageHomeWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_HOME_TMP_FOLDER"}, Destination: &cfg.Reva.StorageHome.TempFolder, }, - &cli.BoolFlag{ - Name: "enable-home", - Value: flags.OverrideDefaultBool(cfg.Reva.Storages.Home.EnableHome, true), - Usage: "enable the creation of home directories", - EnvVars: []string{"STORAGE_HOME_ENABLE_HOME"}, - Destination: &cfg.Reva.Storages.Home.EnableHome, - }, // some drivers need to look up users at the gateway @@ -159,12 +153,12 @@ func StorageHomeWithConfig(cfg *config.Config) []cli.Flag { flags = append(flags, TracingWithConfig(cfg)...) flags = append(flags, DebugWithConfig(cfg)...) flags = append(flags, SecretWithConfig(cfg)...) - flags = append(flags, DriverEOSWithConfig(cfg)...) - flags = append(flags, DriverLocalWithConfig(cfg)...) - flags = append(flags, DriverOwnCloudWithConfig(cfg)...) - flags = append(flags, DriverOwnCloudSQLWithConfig(cfg)...) - flags = append(flags, DriverOCISWithConfig(cfg)...) - flags = append(flags, DriverS3NGWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverEOSWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverLocalWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverOwnCloudWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverOwnCloudSQLWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverOCISWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverS3NGWithConfig(cfg)...) return flags } diff --git a/storage/pkg/flagset/storagemetadata.go b/storage/pkg/flagset/storagemetadata.go index 41a717cefd8..644a0042124 100644 --- a/storage/pkg/flagset/storagemetadata.go +++ b/storage/pkg/flagset/storagemetadata.go @@ -3,6 +3,7 @@ package flagset import ( "github.com/owncloud/ocis/ocis-pkg/flags" "github.com/owncloud/ocis/storage/pkg/config" + "github.com/owncloud/ocis/storage/pkg/flagset/metadatadrivers" "github.com/urfave/cli/v2" ) @@ -89,23 +90,14 @@ func StorageMetadata(cfg *config.Config) []cli.Flag { }, } - f = append(f, - &cli.StringFlag{ - Name: "storage-root", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.Common.Root, "/var/tmp/ocis/storage/metadata"), - Usage: "the path to the metadata storage root", - EnvVars: []string{"STORAGE_METADATA_ROOT"}, - Destination: &cfg.Reva.Storages.Common.Root, - }, - ) f = append(f, TracingWithConfig(cfg)...) f = append(f, DebugWithConfig(cfg)...) f = append(f, SecretWithConfig(cfg)...) - f = append(f, DriverEOSWithConfig(cfg)...) - f = append(f, DriverLocalWithConfig(cfg)...) - f = append(f, DriverOwnCloudWithConfig(cfg)...) - f = append(f, DriverOCISWithConfig(cfg)...) - f = append(f, DriverS3NGWithConfig(cfg)...) + f = append(f, metadatadrivers.DriverEOSWithConfig(cfg)...) + f = append(f, metadatadrivers.DriverLocalWithConfig(cfg)...) + f = append(f, metadatadrivers.DriverOwnCloudWithConfig(cfg)...) + f = append(f, metadatadrivers.DriverOCISWithConfig(cfg)...) + f = append(f, metadatadrivers.DriverS3NGWithConfig(cfg)...) return f diff --git a/storage/pkg/flagset/storageusers.go b/storage/pkg/flagset/storageusers.go index accff38dcb1..b5bf0196857 100644 --- a/storage/pkg/flagset/storageusers.go +++ b/storage/pkg/flagset/storageusers.go @@ -3,6 +3,7 @@ package flagset import ( "github.com/owncloud/ocis/ocis-pkg/flags" "github.com/owncloud/ocis/storage/pkg/config" + "github.com/owncloud/ocis/storage/pkg/flagset/userdrivers" "github.com/urfave/cli/v2" ) @@ -149,12 +150,12 @@ func StorageUsersWithConfig(cfg *config.Config) []cli.Flag { flags = append(flags, TracingWithConfig(cfg)...) flags = append(flags, DebugWithConfig(cfg)...) flags = append(flags, SecretWithConfig(cfg)...) - flags = append(flags, DriverEOSWithConfig(cfg)...) - flags = append(flags, DriverLocalWithConfig(cfg)...) - flags = append(flags, DriverOwnCloudWithConfig(cfg)...) - flags = append(flags, DriverOwnCloudSQLWithConfig(cfg)...) - flags = append(flags, DriverOCISWithConfig(cfg)...) - flags = append(flags, DriverS3NGWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverEOSWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverLocalWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverOwnCloudWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverOwnCloudSQLWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverOCISWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverS3NGWithConfig(cfg)...) return flags } diff --git a/storage/pkg/flagset/userdrivers/drivereos.go b/storage/pkg/flagset/userdrivers/drivereos.go new file mode 100644 index 00000000000..78fc81b93b7 --- /dev/null +++ b/storage/pkg/flagset/userdrivers/drivereos.go @@ -0,0 +1,135 @@ +package userdrivers + +import ( + "os" + + "github.com/owncloud/ocis/ocis-pkg/flags" + + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverEOSWithConfig applies cfg to the root flagset +func DriverEOSWithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + + &cli.StringFlag{ + Name: "storage-eos-namespace", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.Root, "/eos/dockertest/reva"), + Usage: "Namespace for metadata operations", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_NAMESPACE"}, + Destination: &cfg.Reva.UserStorage.EOS.Root, + }, + &cli.StringFlag{ + Name: "storage-eos-shadow-namespace", + // Defaults to path.Join(c.Namespace, ".shadow") + Usage: "Shadow namespace where share references are stored", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_SHADOW_NAMESPACE"}, + Destination: &cfg.Reva.UserStorage.EOS.ShadowNamespace, + }, + &cli.StringFlag{ + Name: "storage-eos-share-folder", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.ShareFolder, "/Shares"), + Usage: "name of the share folder", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_SHARE_FOLDER"}, + Destination: &cfg.Reva.UserStorage.EOS.ShareFolder, + }, + &cli.StringFlag{ + Name: "storage-eos-binary", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.EosBinary, "/usr/bin/eos"), + Usage: "Location of the eos binary", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_BINARY"}, + Destination: &cfg.Reva.UserStorage.EOS.EosBinary, + }, + &cli.StringFlag{ + Name: "storage-eos-xrdcopy-binary", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.XrdcopyBinary, "/usr/bin/xrdcopy"), + Usage: "Location of the xrdcopy binary", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_XRDCOPY_BINARY"}, + Destination: &cfg.Reva.UserStorage.EOS.XrdcopyBinary, + }, + &cli.StringFlag{ + Name: "storage-eos-master-url", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.MasterURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), + Usage: "URL of the Master EOS MGM", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_MASTER_URL"}, + Destination: &cfg.Reva.UserStorage.EOS.MasterURL, + }, + &cli.StringFlag{ + Name: "storage-eos-slave-url", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.SlaveURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), + Usage: "URL of the Slave EOS MGM", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_SLAVE_URL"}, + Destination: &cfg.Reva.UserStorage.EOS.SlaveURL, + }, + &cli.StringFlag{ + Name: "storage-eos-cache-directory", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.CacheDirectory, os.TempDir()), + Usage: "Location on the local fs where to store reads", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_CACHE_DIRECTORY"}, + Destination: &cfg.Reva.UserStorage.EOS.CacheDirectory, + }, + &cli.BoolFlag{ + Name: "storage-eos-enable-logging", + Usage: "Enables logging of the commands executed", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_ENABLE_LOGGING"}, + Destination: &cfg.Reva.UserStorage.EOS.EnableLogging, + }, + &cli.BoolFlag{ + Name: "storage-eos-show-hidden-sysfiles", + Usage: "show internal EOS files like .sys.v# and .sys.a# files.", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_SHOW_HIDDEN_SYSFILES"}, + Destination: &cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles, + }, + &cli.BoolFlag{ + Name: "storage-eos-force-singleuser-mode", + Usage: "force connections to EOS to use SingleUsername", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_FORCE_SINGLEUSER_MODE"}, + Destination: &cfg.Reva.UserStorage.EOS.ForceSingleUserMode, + }, + &cli.BoolFlag{ + Name: "storage-eos-use-keytab", + Usage: "authenticate requests by using an EOS keytab", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_USE_KEYTAB"}, + Destination: &cfg.Reva.UserStorage.EOS.UseKeytab, + }, + &cli.BoolFlag{ + Name: "storage-eos-enable-home", + Usage: "enable the creation of home directories", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_ENABLE_HOME"}, + Destination: &cfg.Reva.UserStorage.EOS.EnableHome, + }, + &cli.StringFlag{ + Name: "storage-eos-sec-protocol", + Usage: "the xrootd security protocol to use between the server and EOS", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_SEC_PROTOCOL"}, + Destination: &cfg.Reva.UserStorage.EOS.SecProtocol, + }, + &cli.StringFlag{ + Name: "storage-eos-keytab", + Usage: "the location of the keytab to use to authenticate to EOS", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_KEYTAB"}, + Destination: &cfg.Reva.UserStorage.EOS.Keytab, + }, + &cli.StringFlag{ + Name: "storage-eos-single-username", + Usage: "the username to use when SingleUserMode is enabled", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_SINGLE_USERNAME"}, + Destination: &cfg.Reva.UserStorage.EOS.SingleUsername, + }, + &cli.StringFlag{ + Name: "storage-eos-layout", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.UserLayout, "{{substr 0 1 .Username}}/{{.Username}}"), + Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.UsernameLower}} and {{.Provider}} also supports prefixing dirs: "{{.UsernamePrefixCount.2}}/{{.UsernameLower}}" will turn "Einstein" into "Ei/Einstein" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_LAYOUT"}, + Destination: &cfg.Reva.UserStorage.EOS.UserLayout, + }, + &cli.StringFlag{ + Name: "storage-eos-reva-gateway-addr", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.GatewaySVC, "127.0.0.1:9142"), + Usage: "Address of REVA gateway endpoint", + EnvVars: []string{"REVA_GATEWAY"}, + Destination: &cfg.Reva.UserStorage.EOS.GatewaySVC, + }, + } +} diff --git a/storage/pkg/flagset/driverlocal.go b/storage/pkg/flagset/userdrivers/driverlocal.go similarity index 61% rename from storage/pkg/flagset/driverlocal.go rename to storage/pkg/flagset/userdrivers/driverlocal.go index 6355fcc719c..7241b106497 100644 --- a/storage/pkg/flagset/driverlocal.go +++ b/storage/pkg/flagset/userdrivers/driverlocal.go @@ -1,4 +1,4 @@ -package flagset +package userdrivers import ( "github.com/owncloud/ocis/ocis-pkg/flags" @@ -11,10 +11,10 @@ func DriverLocalWithConfig(cfg *config.Config) []cli.Flag { return []cli.Flag{ &cli.StringFlag{ Name: "storage-local-root", - Value: flags.OverrideDefaultString(cfg.Reva.Storages.Local.Root, "/var/tmp/ocis/storage/local"), + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.Local.Root, "/var/tmp/ocis/storage/local"), Usage: "the path to the local storage root", - EnvVars: []string{"STORAGE_DRIVER_LOCAL_ROOT"}, - Destination: &cfg.Reva.Storages.Local.Root, + EnvVars: []string{"STORAGE_USERS_DRIVER_LOCAL_ROOT"}, + Destination: &cfg.Reva.UserStorage.Local.Root, }, } } diff --git a/storage/pkg/flagset/userdrivers/driverocis.go b/storage/pkg/flagset/userdrivers/driverocis.go new file mode 100644 index 00000000000..3621f8578a3 --- /dev/null +++ b/storage/pkg/flagset/userdrivers/driverocis.go @@ -0,0 +1,48 @@ +package userdrivers + +import ( + "github.com/owncloud/ocis/ocis-pkg/flags" + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverOCISWithConfig applies cfg to the root flagset +func DriverOCISWithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "storage-ocis-root", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OCIS.Root, "/var/tmp/ocis/storage/users"), + Usage: "the path to the local storage root", + EnvVars: []string{"STORAGE_USERS_DRIVER_OCIS_ROOT"}, + Destination: &cfg.Reva.UserStorage.OCIS.Root, + }, + &cli.BoolFlag{ + Name: "storage-ocis-enable-home", + Value: flags.OverrideDefaultBool(cfg.Reva.UserStorage.OCIS.EnableHome, true), + Usage: "enable the creation of home storages", + EnvVars: []string{"STORAGE_USERS_DRIVER_OCIS_ENABLE_HOME"}, + Destination: &cfg.Reva.UserStorage.OCIS.EnableHome, + }, + &cli.StringFlag{ + Name: "storage-ocis-layout", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OCIS.UserLayout, "{{.Id.OpaqueId}}"), + Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_OCIS_LAYOUT"}, + Destination: &cfg.Reva.UserStorage.OCIS.UserLayout, + }, + &cli.StringFlag{ + Name: "storage-ocis-share-folder", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OCIS.ShareFolder, "/Shares"), + Usage: "name of the shares folder", + EnvVars: []string{"STORAGE_USERS_DRIVER_OCIS_SHARE_FOLDER"}, + Destination: &cfg.Reva.UserStorage.OCIS.ShareFolder, + }, + &cli.StringFlag{ + Name: "service-user-uuid", + Value: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", + Usage: "uuid of the internal service user", + EnvVars: []string{"STORAGE_USERS_DRIVER_OCIS_SERVICE_USER_UUID"}, + Destination: &cfg.Reva.UserStorage.OCIS.ServiceUserUUID, + }, + } +} diff --git a/storage/pkg/flagset/userdrivers/driverowncloud.go b/storage/pkg/flagset/userdrivers/driverowncloud.go new file mode 100644 index 00000000000..865b0faff29 --- /dev/null +++ b/storage/pkg/flagset/userdrivers/driverowncloud.go @@ -0,0 +1,62 @@ +package userdrivers + +import ( + "github.com/owncloud/ocis/ocis-pkg/flags" + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverOwnCloudWithConfig applies cfg to the root flagset +func DriverOwnCloudWithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "storage-owncloud-datadir", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloud.Root, "/var/tmp/ocis/storage/owncloud"), + Usage: "the path to the owncloud data directory", + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR"}, + Destination: &cfg.Reva.UserStorage.OwnCloud.Root, + }, + &cli.StringFlag{ + Name: "storage-owncloud-uploadinfo-dir", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloud.UploadInfoDir, "/var/tmp/ocis/storage/uploadinfo"), + Usage: "the path to the tus upload info directory", + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_UPLOADINFO_DIR"}, + Destination: &cfg.Reva.UserStorage.OwnCloud.UploadInfoDir, + }, + &cli.StringFlag{ + Name: "storage-owncloud-share-folder", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloud.ShareFolder, "/Shares"), + Usage: "name of the shares folder", + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_SHARE_FOLDER"}, + Destination: &cfg.Reva.UserStorage.OwnCloud.ShareFolder, + }, + &cli.BoolFlag{ + Name: "storage-owncloud-scan", + Value: flags.OverrideDefaultBool(cfg.Reva.UserStorage.OwnCloud.Scan, true), + Usage: "scan files on startup to add fileids", + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_SCAN"}, + Destination: &cfg.Reva.UserStorage.OwnCloud.Scan, + }, + &cli.StringFlag{ + Name: "storage-owncloud-redis", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloud.Redis, ":6379"), + Usage: "the address of the redis server", + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_REDIS_ADDR"}, + Destination: &cfg.Reva.UserStorage.OwnCloud.Redis, + }, + &cli.BoolFlag{ + Name: "storage-owncloud-enable-home", + Value: flags.OverrideDefaultBool(cfg.Reva.UserStorage.OwnCloud.EnableHome, false), + Usage: "enable the creation of home storages", + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_ENABLE_HOME"}, + Destination: &cfg.Reva.UserStorage.OwnCloud.EnableHome, + }, + &cli.StringFlag{ + Name: "storage-owncloud-layout", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloud.UserLayout, "{{.Id.OpaqueId}}"), + Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_LAYOUT"}, + Destination: &cfg.Reva.UserStorage.OwnCloud.UserLayout, + }, + } +} diff --git a/storage/pkg/flagset/userdrivers/driverowncloudsql.go b/storage/pkg/flagset/userdrivers/driverowncloudsql.go new file mode 100644 index 00000000000..d4e3a4cc9aa --- /dev/null +++ b/storage/pkg/flagset/userdrivers/driverowncloudsql.go @@ -0,0 +1,83 @@ +package userdrivers + +import ( + "github.com/owncloud/ocis/ocis-pkg/flags" + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverOwnCloudSQLWithConfig applies cfg to the root flagset +func DriverOwnCloudSQLWithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "storage-owncloudsql-datadir", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloudSQL.Root, "/var/tmp/ocis/storage/owncloud"), + Usage: "the path to the owncloudsql data directory", + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR"}, + Destination: &cfg.Reva.UserStorage.OwnCloudSQL.Root, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-uploadinfo-dir", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloudSQL.UploadInfoDir, "/var/tmp/ocis/storage/uploadinfo"), + Usage: "the path to the tus upload info directory", + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR"}, + Destination: &cfg.Reva.UserStorage.OwnCloudSQL.UploadInfoDir, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-share-folder", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloudSQL.ShareFolder, "/Shares"), + Usage: "name of the shares folder", + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER"}, + Destination: &cfg.Reva.UserStorage.OwnCloudSQL.ShareFolder, + }, + &cli.BoolFlag{ + Name: "storage-owncloudsql-enable-home", + Value: flags.OverrideDefaultBool(cfg.Reva.UserStorage.OwnCloudSQL.EnableHome, true), + Usage: "enable the creation of home storages", + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_ENABLE_HOME"}, + Destination: &cfg.Reva.UserStorage.OwnCloudSQL.EnableHome, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-layout", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloudSQL.UserLayout, "{{.Username}}"), + Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_LAYOUT"}, + Destination: &cfg.Reva.UserStorage.OwnCloudSQL.UserLayout, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-dbusername", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloudSQL.DBUsername, "owncloud"), + Usage: `"username for accessing the database" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBUSERNAME"}, + Destination: &cfg.Reva.UserStorage.OwnCloudSQL.DBUsername, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-dbpassword", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloudSQL.DBPassword, "owncloud"), + Usage: `"password for accessing the database" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPASSWORD"}, + Destination: &cfg.Reva.UserStorage.OwnCloudSQL.DBPassword, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-dbhost", + Value: cfg.Reva.UserStorage.OwnCloudSQL.DBHost, + Usage: `"the database hostname or IP address" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBHOST"}, + Destination: &cfg.Reva.UserStorage.OwnCloudSQL.DBHost, + }, + &cli.IntFlag{ + Name: "storage-owncloudsql-dbport", + Value: flags.OverrideDefaultInt(cfg.Reva.UserStorage.OwnCloudSQL.DBPort, 3306), + Usage: `"port the database listens on" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPORT"}, + Destination: &cfg.Reva.UserStorage.OwnCloudSQL.DBPort, + }, + &cli.StringFlag{ + Name: "storage-owncloudsql-dbname", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloudSQL.DBName, "owncloud"), + Usage: `"the database name" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBNAME"}, + Destination: &cfg.Reva.UserStorage.OwnCloudSQL.DBName, + }, + } +} diff --git a/storage/pkg/flagset/userdrivers/drivers3ng.go b/storage/pkg/flagset/userdrivers/drivers3ng.go new file mode 100644 index 00000000000..12c61320552 --- /dev/null +++ b/storage/pkg/flagset/userdrivers/drivers3ng.go @@ -0,0 +1,76 @@ +package userdrivers + +import ( + "github.com/owncloud/ocis/ocis-pkg/flags" + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverS3NGWithConfig applies cfg to the root flagset +func DriverS3NGWithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "storage-s3ng-root", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.S3NG.Root, "/var/tmp/ocis/storage/users"), + Usage: "the path to the local storage root", + EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_ROOT"}, + Destination: &cfg.Reva.UserStorage.S3NG.Root, + }, + &cli.BoolFlag{ + Name: "storage-s3ng-enable-home", + Value: flags.OverrideDefaultBool(cfg.Reva.UserStorage.S3NG.EnableHome, true), + Usage: "enable the creation of home storages", + EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_ENABLE_HOME"}, + Destination: &cfg.Reva.UserStorage.S3NG.EnableHome, + }, + &cli.StringFlag{ + Name: "storage-s3ng-layout", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.S3NG.UserLayout, "{{.Id.OpaqueId}}"), + Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_LAYOUT"}, + Destination: &cfg.Reva.UserStorage.S3NG.UserLayout, + }, + &cli.StringFlag{ + Name: "storage-s3ng-share-folder", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.S3NG.ShareFolder, "/Shares"), + Usage: "name of the shares folder", + EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_SHARE_FOLDER"}, + Destination: &cfg.Reva.UserStorage.S3NG.ShareFolder, + }, + &cli.StringFlag{ + Name: "storage-s3ng-region", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.S3NG.Root, "default"), + Usage: `"the s3 region" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_REGION"}, + Destination: &cfg.Reva.UserStorage.S3NG.Region, + }, + &cli.StringFlag{ + Name: "storage-s3ng-accesskey", + Value: "", + Usage: `"the s3 access key" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_ACCESS_KEY"}, + Destination: &cfg.Reva.UserStorage.S3NG.AccessKey, + }, + &cli.StringFlag{ + Name: "storage-s3ng-secretkey", + Value: "", + Usage: `"the secret s3 api key" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_SECRET_KEY"}, + Destination: &cfg.Reva.UserStorage.S3NG.SecretKey, + }, + &cli.StringFlag{ + Name: "storage-s3ng-endpoint", + Value: "", + Usage: `"s3 compatible API endpoint" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_ENDPOINT"}, + Destination: &cfg.Reva.UserStorage.S3NG.Endpoint, + }, + &cli.StringFlag{ + Name: "storage-s3ng-bucket", + Value: "", + Usage: `"bucket where the data will be stored in`, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_BUCKET"}, + Destination: &cfg.Reva.UserStorage.S3NG.Bucket, + }, + } +} diff --git a/tests/acceptance/docker/src/ocis-base.yml b/tests/acceptance/docker/src/ocis-base.yml index 22a2ce076f2..4154bd0d7cb 100644 --- a/tests/acceptance/docker/src/ocis-base.yml +++ b/tests/acceptance/docker/src/ocis-base.yml @@ -4,13 +4,10 @@ services: OCIS_URL: "https://ocis-server:9200" STORAGE_HOME_DRIVER: $STORAGE_DRIVER STORAGE_USERS_DRIVER: $STORAGE_DRIVER - STORAGE_DRIVER_OCIS_ROOT: /srv/app/tmp/ocis/storage/users - STORAGE_DRIVER_LOCAL_ROOT: /srv/app/tmp/ocis/local/root - STORAGE_METADATA_ROOT: /srv/app/tmp/ocis/metadata - STORAGE_DRIVER_OWNCLOUD_DATADIR: /srv/app/tmp/ocis/owncloud/data - STORAGE_DRIVER_OWNCLOUD_REDIS_ADDR: redis:6379 - STORAGE_HOME_DATA_SERVER_URL: http://ocis-server:9155/data - STORAGE_USERS_DATA_SERVER_URL: http://ocis-server:9158/data + STORAGE_USERS_DRIVER_LOCAL_ROOT: /srv/app/tmp/ocis/local/root + STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR: /srv/app/tmp/ocis/owncloud/data + STORAGE_USERS_DRIVER_OCIS_ROOT: /srv/app/tmp/ocis/storage/users + STORAGE_METADATA_DRIVER_OCIS_ROOT: /srv/app/tmp/ocis/storage/metadata STORAGE_SHARING_USER_JSON_FILE: /srv/app/tmp/ocis/shares.json SETTINGS_DATA_PATH: "/srv/app/tmp/ocis/settings" PROXY_ENABLE_BASIC_AUTH: "true" @@ -18,11 +15,11 @@ services: IDP_IDENTIFIER_REGISTRATION_CONF: /drone/src/tests/config/drone/identifier-registration.yml ACCOUNTS_HASH_DIFFICULTY: 4 # s3ng specific settings - STORAGE_DRIVER_S3NG_ENDPOINT: http://ceph:8080 - STORAGE_DRIVER_S3NG_REGION: default - STORAGE_DRIVER_S3NG_ACCESS_KEY: test - STORAGE_DRIVER_S3NG_SECRET_KEY: test - STORAGE_DRIVER_S3NG_BUCKET: test + STORAGE_USERS_DRIVER_S3NG_ENDPOINT: http://ceph:8080 + STORAGE_USERS_DRIVER_S3NG_REGION: default + STORAGE_USERS_DRIVER_S3NG_ACCESS_KEY: test + STORAGE_USERS_DRIVER_S3NG_SECRET_KEY: test + STORAGE_USERS_DRIVER_S3NG_BUCKET: test volumes: - ../../../config:/drone/src/tests/config - oCISownCloud10testsuite:/srv From 6c6cb64364ebdf5da83313b8fb3b963b02af1a6e Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 11 Oct 2021 11:38:34 +0200 Subject: [PATCH 2/2] remove uneeded storages for metadata, add missing flagsets --- .../oc10_ocis_parallel/docker-compose.yml | 1 - storage/pkg/command/storagedrivers/home.go | 11 ++- .../pkg/command/storagedrivers/metadata.go | 50 +----------- .../pkg/flagset/metadatadrivers/drivereos.go | 59 +++++++------- .../flagset/metadatadrivers/driverowncloud.go | 55 -------------- .../metadatadrivers/driverowncloudsql.go | 76 ------------------- .../pkg/flagset/metadatadrivers/drivers3.go | 48 ++++++++++++ storage/pkg/flagset/storagehome.go | 1 + storage/pkg/flagset/storagemetadata.go | 2 +- storage/pkg/flagset/storageusers.go | 1 + storage/pkg/flagset/userdrivers/drivereos.go | 13 ++-- .../pkg/flagset/userdrivers/driverlocal.go | 14 ++++ storage/pkg/flagset/userdrivers/driverocis.go | 7 -- .../pkg/flagset/userdrivers/driverowncloud.go | 7 -- .../flagset/userdrivers/driverowncloudsql.go | 7 -- storage/pkg/flagset/userdrivers/drivers3.go | 48 ++++++++++++ storage/pkg/flagset/userdrivers/drivers3ng.go | 9 +-- 17 files changed, 162 insertions(+), 247 deletions(-) delete mode 100644 storage/pkg/flagset/metadatadrivers/driverowncloud.go delete mode 100644 storage/pkg/flagset/metadatadrivers/driverowncloudsql.go create mode 100644 storage/pkg/flagset/metadatadrivers/drivers3.go create mode 100644 storage/pkg/flagset/userdrivers/drivers3.go diff --git a/deployments/examples/oc10_ocis_parallel/docker-compose.yml b/deployments/examples/oc10_ocis_parallel/docker-compose.yml index 1ec6b5e9578..0440ac42592 100644 --- a/deployments/examples/oc10_ocis_parallel/docker-compose.yml +++ b/deployments/examples/oc10_ocis_parallel/docker-compose.yml @@ -96,7 +96,6 @@ services: STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR: /mnt/data/files STORAGE_USERS_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR: /tmp STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER: "/Shares" - STORAGE_USERS_DRIVER_OWNCLOUDSQL_ENABLE_HOME: "false" STORAGE_USERS_DRIVER_OWNCLOUDSQL_LAYOUT: "{{.Username}}" STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBUSERNAME: owncloud STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPASSWORD: owncloud diff --git a/storage/pkg/command/storagedrivers/home.go b/storage/pkg/command/storagedrivers/home.go index cdd4968a6d4..ccac2f85d1f 100644 --- a/storage/pkg/command/storagedrivers/home.go +++ b/storage/pkg/command/storagedrivers/home.go @@ -63,7 +63,7 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} { "show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles, "force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode, "use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab, - "enable_home": cfg.Reva.UserStorage.EOS.EnableHome, + "enable_home": true, "gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC, }, "local": map[string]interface{}{ @@ -81,7 +81,7 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} { "share_folder": cfg.Reva.UserStorage.OwnCloud.ShareFolder, "user_layout": cfg.Reva.UserStorage.OwnCloud.UserLayout, "redis": cfg.Reva.UserStorage.OwnCloud.Redis, - "enable_home": cfg.Reva.UserStorage.OwnCloud.EnableHome, + "enable_home": true, "scan": cfg.Reva.UserStorage.OwnCloud.Scan, "userprovidersvc": cfg.Reva.Users.Endpoint, }, @@ -90,7 +90,7 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} { "upload_info_dir": cfg.Reva.UserStorage.OwnCloudSQL.UploadInfoDir, "share_folder": cfg.Reva.UserStorage.OwnCloudSQL.ShareFolder, "user_layout": cfg.Reva.UserStorage.OwnCloudSQL.UserLayout, - "enable_home": cfg.Reva.UserStorage.OwnCloudSQL.EnableHome, + "enable_home": true, "dbusername": cfg.Reva.UserStorage.OwnCloudSQL.DBUsername, "dbpassword": cfg.Reva.UserStorage.OwnCloudSQL.DBPassword, "dbhost": cfg.Reva.UserStorage.OwnCloudSQL.DBHost, @@ -100,7 +100,7 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} { }, "ocis": map[string]interface{}{ "root": cfg.Reva.UserStorage.OCIS.Root, - "enable_home": cfg.Reva.UserStorage.OCIS.EnableHome, + "enable_home": true, "user_layout": cfg.Reva.UserStorage.OCIS.UserLayout, "share_folder": cfg.Reva.UserStorage.OCIS.ShareFolder, "treetime_accounting": true, @@ -113,11 +113,10 @@ func HomeDrivers(cfg *config.Config) map[string]interface{} { "secret_key": cfg.Reva.UserStorage.S3.SecretKey, "endpoint": cfg.Reva.UserStorage.S3.Endpoint, "bucket": cfg.Reva.UserStorage.S3.Bucket, - "prefix": cfg.Reva.UserStorage.S3.Root, }, "s3ng": map[string]interface{}{ "root": cfg.Reva.UserStorage.S3NG.Root, - "enable_home": cfg.Reva.UserStorage.S3NG.EnableHome, + "enable_home": true, "user_layout": cfg.Reva.UserStorage.S3NG.UserLayout, "share_folder": cfg.Reva.UserStorage.S3NG.ShareFolder, "s3.region": cfg.Reva.UserStorage.S3NG.Region, diff --git a/storage/pkg/command/storagedrivers/metadata.go b/storage/pkg/command/storagedrivers/metadata.go index 3438c7043e1..578b02fc71f 100644 --- a/storage/pkg/command/storagedrivers/metadata.go +++ b/storage/pkg/command/storagedrivers/metadata.go @@ -23,25 +23,7 @@ func MetadataDrivers(cfg *config.Config) map[string]interface{} { "force_single_user_mode": cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode, "use_keytab": cfg.Reva.MetadataStorage.EOS.UseKeytab, "gatewaysvc": cfg.Reva.MetadataStorage.EOS.GatewaySVC, - }, - "eoshome": map[string]interface{}{ - "namespace": cfg.Reva.MetadataStorage.EOS.Root, - "shadow_namespace": cfg.Reva.MetadataStorage.EOS.ShadowNamespace, - "uploads_namespace": cfg.Reva.MetadataStorage.EOS.UploadsNamespace, - "eos_binary": cfg.Reva.MetadataStorage.EOS.EosBinary, - "xrdcopy_binary": cfg.Reva.MetadataStorage.EOS.XrdcopyBinary, - "master_url": cfg.Reva.MetadataStorage.EOS.MasterURL, - "slave_url": cfg.Reva.MetadataStorage.EOS.SlaveURL, - "cache_directory": cfg.Reva.MetadataStorage.EOS.CacheDirectory, - "sec_protocol": cfg.Reva.MetadataStorage.EOS.SecProtocol, - "keytab": cfg.Reva.MetadataStorage.EOS.Keytab, - "single_username": cfg.Reva.MetadataStorage.EOS.SingleUsername, - "user_layout": cfg.Reva.MetadataStorage.EOS.UserLayout, - "enable_logging": cfg.Reva.MetadataStorage.EOS.EnableLogging, - "show_hidden_sys_files": cfg.Reva.MetadataStorage.EOS.ShowHiddenSysFiles, - "force_single_user_mode": cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode, - "use_keytab": cfg.Reva.MetadataStorage.EOS.UseKeytab, - "gatewaysvc": cfg.Reva.MetadataStorage.EOS.GatewaySVC, + "enable_home": false, }, "eosgrpc": map[string]interface{}{ "namespace": cfg.Reva.MetadataStorage.EOS.Root, @@ -66,37 +48,12 @@ func MetadataDrivers(cfg *config.Config) map[string]interface{} { "local": map[string]interface{}{ "root": cfg.Reva.MetadataStorage.Local.Root, }, - "localhome": map[string]interface{}{ - "root": cfg.Reva.MetadataStorage.Local.Root, - "user_layout": cfg.Reva.MetadataStorage.Local.UserLayout, - }, - "owncloud": map[string]interface{}{ - "datadirectory": cfg.Reva.MetadataStorage.OwnCloud.Root, - "upload_info_dir": cfg.Reva.MetadataStorage.OwnCloud.UploadInfoDir, - "user_layout": cfg.Reva.MetadataStorage.OwnCloud.UserLayout, - "redis": cfg.Reva.MetadataStorage.OwnCloud.Redis, - "enable_home": false, - "scan": cfg.Reva.MetadataStorage.OwnCloud.Scan, - "userprovidersvc": cfg.Reva.Users.Endpoint, - }, - "owncloudsql": map[string]interface{}{ - "datadirectory": cfg.Reva.MetadataStorage.OwnCloudSQL.Root, - "upload_info_dir": cfg.Reva.MetadataStorage.OwnCloudSQL.UploadInfoDir, - "user_layout": cfg.Reva.MetadataStorage.OwnCloudSQL.UserLayout, - "enable_home": false, - "dbusername": cfg.Reva.MetadataStorage.OwnCloudSQL.DBUsername, - "dbpassword": cfg.Reva.MetadataStorage.OwnCloudSQL.DBPassword, - "dbhost": cfg.Reva.MetadataStorage.OwnCloudSQL.DBHost, - "dbport": cfg.Reva.MetadataStorage.OwnCloudSQL.DBPort, - "dbname": cfg.Reva.MetadataStorage.OwnCloudSQL.DBName, - "userprovidersvc": cfg.Reva.Users.Endpoint, - }, "ocis": map[string]interface{}{ "root": cfg.Reva.MetadataStorage.OCIS.Root, "enable_home": false, "user_layout": cfg.Reva.MetadataStorage.OCIS.UserLayout, - "treetime_accounting": true, - "treesize_accounting": true, + "treetime_accounting": false, + "treesize_accounting": false, "owner": cfg.Reva.MetadataStorage.OCIS.ServiceUserUUID, // the accounts service system account uuid }, "s3": map[string]interface{}{ @@ -105,7 +62,6 @@ func MetadataDrivers(cfg *config.Config) map[string]interface{} { "secret_key": cfg.Reva.MetadataStorage.S3.SecretKey, "endpoint": cfg.Reva.MetadataStorage.S3.Endpoint, "bucket": cfg.Reva.MetadataStorage.S3.Bucket, - "prefix": cfg.Reva.MetadataStorage.S3.Root, }, "s3ng": map[string]interface{}{ "root": cfg.Reva.MetadataStorage.S3NG.Root, diff --git a/storage/pkg/flagset/metadatadrivers/drivereos.go b/storage/pkg/flagset/metadatadrivers/drivereos.go index ec66e4200a2..c734e6c1294 100644 --- a/storage/pkg/flagset/metadatadrivers/drivereos.go +++ b/storage/pkg/flagset/metadatadrivers/drivereos.go @@ -15,115 +15,122 @@ func DriverEOSWithConfig(cfg *config.Config) []cli.Flag { &cli.StringFlag{ Name: "storage-eos-namespace", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.Root, "/eos/dockertest/reva"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.Root, "/eos/dockertest/reva"), Usage: "Namespace for metadata operations", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_NAMESPACE"}, - Destination: &cfg.Reva.UserStorage.EOS.Root, + Destination: &cfg.Reva.MetadataStorage.EOS.Root, }, &cli.StringFlag{ Name: "storage-eos-shadow-namespace", // Defaults to path.Join(c.Namespace, ".shadow") Usage: "Shadow namespace where share references are stored", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SHADOW_NAMESPACE"}, - Destination: &cfg.Reva.UserStorage.EOS.ShadowNamespace, + Destination: &cfg.Reva.MetadataStorage.EOS.ShadowNamespace, + }, + &cli.StringFlag{ + Name: "storage-eos-uploads-namespace", + // Defaults to path.Join(c.Namespace, ".uploads") + Usage: "Uploads namespace", + EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_UPLOADS_NAMESPACE"}, + Destination: &cfg.Reva.MetadataStorage.EOS.UploadsNamespace, }, &cli.StringFlag{ Name: "storage-eos-share-folder", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.ShareFolder, "/Shares"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.ShareFolder, "/Shares"), Usage: "name of the share folder", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SHARE_FOLDER"}, - Destination: &cfg.Reva.UserStorage.EOS.ShareFolder, + Destination: &cfg.Reva.MetadataStorage.EOS.ShareFolder, }, &cli.StringFlag{ Name: "storage-eos-binary", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.EosBinary, "/usr/bin/eos"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.EosBinary, "/usr/bin/eos"), Usage: "Location of the eos binary", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_BINARY"}, - Destination: &cfg.Reva.UserStorage.EOS.EosBinary, + Destination: &cfg.Reva.MetadataStorage.EOS.EosBinary, }, &cli.StringFlag{ Name: "storage-eos-xrdcopy-binary", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.XrdcopyBinary, "/usr/bin/xrdcopy"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.XrdcopyBinary, "/usr/bin/xrdcopy"), Usage: "Location of the xrdcopy binary", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_XRDCOPY_BINARY"}, - Destination: &cfg.Reva.UserStorage.EOS.XrdcopyBinary, + Destination: &cfg.Reva.MetadataStorage.EOS.XrdcopyBinary, }, &cli.StringFlag{ Name: "storage-eos-master-url", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.MasterURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.MasterURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), Usage: "URL of the Master EOS MGM", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_MASTER_URL"}, - Destination: &cfg.Reva.UserStorage.EOS.MasterURL, + Destination: &cfg.Reva.MetadataStorage.EOS.MasterURL, }, &cli.StringFlag{ Name: "storage-eos-slave-url", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.SlaveURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.SlaveURL, "root://eos-mgm1.eoscluster.cern.ch:1094"), Usage: "URL of the Slave EOS MGM", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SLAVE_URL"}, - Destination: &cfg.Reva.UserStorage.EOS.SlaveURL, + Destination: &cfg.Reva.MetadataStorage.EOS.SlaveURL, }, &cli.StringFlag{ Name: "storage-eos-cache-directory", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.CacheDirectory, os.TempDir()), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.CacheDirectory, os.TempDir()), Usage: "Location on the local fs where to store reads", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_CACHE_DIRECTORY"}, - Destination: &cfg.Reva.UserStorage.EOS.CacheDirectory, + Destination: &cfg.Reva.MetadataStorage.EOS.CacheDirectory, }, &cli.BoolFlag{ Name: "storage-eos-enable-logging", Usage: "Enables logging of the commands executed", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_ENABLE_LOGGING"}, - Destination: &cfg.Reva.UserStorage.EOS.EnableLogging, + Destination: &cfg.Reva.MetadataStorage.EOS.EnableLogging, }, &cli.BoolFlag{ Name: "storage-eos-show-hidden-sysfiles", Usage: "show internal EOS files like .sys.v# and .sys.a# files.", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SHOW_HIDDEN_SYSFILES"}, - Destination: &cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles, + Destination: &cfg.Reva.MetadataStorage.EOS.ShowHiddenSysFiles, }, &cli.BoolFlag{ Name: "storage-eos-force-singleuser-mode", Usage: "force connections to EOS to use SingleUsername", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_FORCE_SINGLEUSER_MODE"}, - Destination: &cfg.Reva.UserStorage.EOS.ForceSingleUserMode, + Destination: &cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode, }, &cli.BoolFlag{ Name: "storage-eos-use-keytab", Usage: "authenticate requests by using an EOS keytab", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_USE_KEYTAB"}, - Destination: &cfg.Reva.UserStorage.EOS.UseKeytab, + Destination: &cfg.Reva.MetadataStorage.EOS.UseKeytab, }, &cli.StringFlag{ Name: "storage-eos-sec-protocol", Usage: "the xrootd security protocol to use between the server and EOS", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SEC_PROTOCOL"}, - Destination: &cfg.Reva.UserStorage.EOS.SecProtocol, + Destination: &cfg.Reva.MetadataStorage.EOS.SecProtocol, }, &cli.StringFlag{ Name: "storage-eos-keytab", Usage: "the location of the keytab to use to authenticate to EOS", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_KEYTAB"}, - Destination: &cfg.Reva.UserStorage.EOS.Keytab, + Destination: &cfg.Reva.MetadataStorage.EOS.Keytab, }, &cli.StringFlag{ Name: "storage-eos-single-username", Usage: "the username to use when SingleUserMode is enabled", EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_SINGLE_USERNAME"}, - Destination: &cfg.Reva.UserStorage.EOS.SingleUsername, + Destination: &cfg.Reva.MetadataStorage.EOS.SingleUsername, }, &cli.StringFlag{ Name: "storage-eos-layout", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.UserLayout, "{{substr 0 1 .Username}}/{{.Username}}"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.UserLayout, "{{substr 0 1 .Username}}/{{.Username}}"), Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.UsernameLower}} and {{.Provider}} also supports prefixing dirs: "{{.UsernamePrefixCount.2}}/{{.UsernameLower}}" will turn "Einstein" into "Ei/Einstein" `, EnvVars: []string{"STORAGE_METADATA_DRIVER_EOS_LAYOUT"}, - Destination: &cfg.Reva.UserStorage.EOS.UserLayout, + Destination: &cfg.Reva.MetadataStorage.EOS.UserLayout, }, &cli.StringFlag{ Name: "storage-eos-reva-gateway-addr", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.GatewaySVC, "127.0.0.1:9142"), + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.EOS.GatewaySVC, "127.0.0.1:9142"), Usage: "Address of REVA gateway endpoint", EnvVars: []string{"REVA_GATEWAY"}, - Destination: &cfg.Reva.UserStorage.EOS.GatewaySVC, + Destination: &cfg.Reva.MetadataStorage.EOS.GatewaySVC, }, } } diff --git a/storage/pkg/flagset/metadatadrivers/driverowncloud.go b/storage/pkg/flagset/metadatadrivers/driverowncloud.go deleted file mode 100644 index cf727526b0c..00000000000 --- a/storage/pkg/flagset/metadatadrivers/driverowncloud.go +++ /dev/null @@ -1,55 +0,0 @@ -package metadatadrivers - -import ( - "github.com/owncloud/ocis/ocis-pkg/flags" - "github.com/owncloud/ocis/storage/pkg/config" - "github.com/urfave/cli/v2" -) - -// DriverOwnCloudWithConfig applies cfg to the root flagset -func DriverOwnCloudWithConfig(cfg *config.Config) []cli.Flag { - return []cli.Flag{ - &cli.StringFlag{ - Name: "storage-owncloud-datadir", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.Root, "/var/tmp/ocis/storage/owncloud"), - Usage: "the path to the owncloud data directory", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_DATADIR"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloud.Root, - }, - &cli.StringFlag{ - Name: "storage-owncloud-uploadinfo-dir", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.UploadInfoDir, "/var/tmp/ocis/storage/uploadinfo"), - Usage: "the path to the tus upload info directory", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_UPLOADINFO_DIR"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloud.UploadInfoDir, - }, - &cli.StringFlag{ - Name: "storage-owncloud-share-folder", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.ShareFolder, "/Shares"), - Usage: "name of the shares folder", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_SHARE_FOLDER"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloud.ShareFolder, - }, - &cli.BoolFlag{ - Name: "storage-owncloud-scan", - Value: flags.OverrideDefaultBool(cfg.Reva.MetadataStorage.OwnCloud.Scan, true), - Usage: "scan files on startup to add fileids", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_SCAN"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloud.Scan, - }, - &cli.StringFlag{ - Name: "storage-owncloud-redis", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.Redis, ":6379"), - Usage: "the address of the redis server", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_REDIS_ADDR"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloud.Redis, - }, - &cli.StringFlag{ - Name: "storage-owncloud-layout", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloud.UserLayout, "{{.Id.OpaqueId}}"), - Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUD_LAYOUT"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloud.UserLayout, - }, - } -} diff --git a/storage/pkg/flagset/metadatadrivers/driverowncloudsql.go b/storage/pkg/flagset/metadatadrivers/driverowncloudsql.go deleted file mode 100644 index ca444348aaf..00000000000 --- a/storage/pkg/flagset/metadatadrivers/driverowncloudsql.go +++ /dev/null @@ -1,76 +0,0 @@ -package metadatadrivers - -import ( - "github.com/owncloud/ocis/ocis-pkg/flags" - "github.com/owncloud/ocis/storage/pkg/config" - "github.com/urfave/cli/v2" -) - -// DriverOwnCloudSQLWithConfig applies cfg to the root flagset -func DriverOwnCloudSQLWithConfig(cfg *config.Config) []cli.Flag { - return []cli.Flag{ - &cli.StringFlag{ - Name: "storage-owncloudsql-datadir", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.Root, "/var/tmp/ocis/storage/owncloud"), - Usage: "the path to the owncloudsql data directory", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DATADIR"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.Root, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-uploadinfo-dir", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.UploadInfoDir, "/var/tmp/ocis/storage/uploadinfo"), - Usage: "the path to the tus upload info directory", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.UploadInfoDir, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-share-folder", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.ShareFolder, "/Shares"), - Usage: "name of the shares folder", - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_SHARE_FOLDER"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.ShareFolder, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-layout", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.UserLayout, "{{.Username}}"), - Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_LAYOUT"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.UserLayout, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-dbusername", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.DBUsername, "owncloud"), - Usage: `"username for accessing the database" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBUSERNAME"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBUsername, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-dbpassword", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.DBPassword, "owncloud"), - Usage: `"password for accessing the database" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBPASSWORD"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBPassword, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-dbhost", - Value: cfg.Reva.MetadataStorage.OwnCloudSQL.DBHost, - Usage: `"the database hostname or IP address" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBHOST"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBHost, - }, - &cli.IntFlag{ - Name: "storage-owncloudsql-dbport", - Value: flags.OverrideDefaultInt(cfg.Reva.MetadataStorage.OwnCloudSQL.DBPort, 3306), - Usage: `"port the database listens on" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBPORT"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBPort, - }, - &cli.StringFlag{ - Name: "storage-owncloudsql-dbname", - Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.OwnCloudSQL.DBName, "owncloud"), - Usage: `"the database name" `, - EnvVars: []string{"STORAGE_METADATA_DRIVER_OWNCLOUDSQL_DBNAME"}, - Destination: &cfg.Reva.MetadataStorage.OwnCloudSQL.DBName, - }, - } -} diff --git a/storage/pkg/flagset/metadatadrivers/drivers3.go b/storage/pkg/flagset/metadatadrivers/drivers3.go new file mode 100644 index 00000000000..56047756042 --- /dev/null +++ b/storage/pkg/flagset/metadatadrivers/drivers3.go @@ -0,0 +1,48 @@ +package metadatadrivers + +import ( + "github.com/owncloud/ocis/ocis-pkg/flags" + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverS3NGWithConfig applies cfg to the root flagset +func DriverS3WithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "storage-s3-region", + Value: flags.OverrideDefaultString(cfg.Reva.MetadataStorage.S3.Region, "default"), + Usage: `"the s3 region" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3_REGION"}, + Destination: &cfg.Reva.MetadataStorage.S3.Region, + }, + &cli.StringFlag{ + Name: "storage-s3-accesskey", + Value: "", + Usage: `"the s3 access key" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3_ACCESS_KEY"}, + Destination: &cfg.Reva.MetadataStorage.S3.AccessKey, + }, + &cli.StringFlag{ + Name: "storage-s3-secretkey", + Value: "", + Usage: `"the secret s3 api key" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3_SECRET_KEY"}, + Destination: &cfg.Reva.MetadataStorage.S3.SecretKey, + }, + &cli.StringFlag{ + Name: "storage-s3-endpoint", + Value: "", + Usage: `"s3 compatible API endpoint" `, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3_ENDPOINT"}, + Destination: &cfg.Reva.MetadataStorage.S3.Endpoint, + }, + &cli.StringFlag{ + Name: "storage-s3-bucket", + Value: "", + Usage: `"bucket where the data will be stored in`, + EnvVars: []string{"STORAGE_METADATA_DRIVER_S3_BUCKET"}, + Destination: &cfg.Reva.MetadataStorage.S3.Bucket, + }, + } +} diff --git a/storage/pkg/flagset/storagehome.go b/storage/pkg/flagset/storagehome.go index 9c989668c0d..a23c9423409 100644 --- a/storage/pkg/flagset/storagehome.go +++ b/storage/pkg/flagset/storagehome.go @@ -159,6 +159,7 @@ func StorageHomeWithConfig(cfg *config.Config) []cli.Flag { flags = append(flags, userdrivers.DriverOwnCloudSQLWithConfig(cfg)...) flags = append(flags, userdrivers.DriverOCISWithConfig(cfg)...) flags = append(flags, userdrivers.DriverS3NGWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverS3WithConfig(cfg)...) return flags } diff --git a/storage/pkg/flagset/storagemetadata.go b/storage/pkg/flagset/storagemetadata.go index 644a0042124..ce5961c18c2 100644 --- a/storage/pkg/flagset/storagemetadata.go +++ b/storage/pkg/flagset/storagemetadata.go @@ -95,9 +95,9 @@ func StorageMetadata(cfg *config.Config) []cli.Flag { f = append(f, SecretWithConfig(cfg)...) f = append(f, metadatadrivers.DriverEOSWithConfig(cfg)...) f = append(f, metadatadrivers.DriverLocalWithConfig(cfg)...) - f = append(f, metadatadrivers.DriverOwnCloudWithConfig(cfg)...) f = append(f, metadatadrivers.DriverOCISWithConfig(cfg)...) f = append(f, metadatadrivers.DriverS3NGWithConfig(cfg)...) + f = append(f, metadatadrivers.DriverS3WithConfig(cfg)...) return f diff --git a/storage/pkg/flagset/storageusers.go b/storage/pkg/flagset/storageusers.go index b5bf0196857..ef8b4e0fc32 100644 --- a/storage/pkg/flagset/storageusers.go +++ b/storage/pkg/flagset/storageusers.go @@ -156,6 +156,7 @@ func StorageUsersWithConfig(cfg *config.Config) []cli.Flag { flags = append(flags, userdrivers.DriverOwnCloudSQLWithConfig(cfg)...) flags = append(flags, userdrivers.DriverOCISWithConfig(cfg)...) flags = append(flags, userdrivers.DriverS3NGWithConfig(cfg)...) + flags = append(flags, userdrivers.DriverS3WithConfig(cfg)...) return flags } diff --git a/storage/pkg/flagset/userdrivers/drivereos.go b/storage/pkg/flagset/userdrivers/drivereos.go index 78fc81b93b7..a937307ef5d 100644 --- a/storage/pkg/flagset/userdrivers/drivereos.go +++ b/storage/pkg/flagset/userdrivers/drivereos.go @@ -27,6 +27,13 @@ func DriverEOSWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_SHADOW_NAMESPACE"}, Destination: &cfg.Reva.UserStorage.EOS.ShadowNamespace, }, + &cli.StringFlag{ + Name: "storage-eos-uploads-namespace", + // Defaults to path.Join(c.Namespace, ".uploads") + Usage: "Uploads namespace", + EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_UPLOADS_NAMESPACE"}, + Destination: &cfg.Reva.UserStorage.EOS.UploadsNamespace, + }, &cli.StringFlag{ Name: "storage-eos-share-folder", Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.EOS.ShareFolder, "/Shares"), @@ -93,12 +100,6 @@ func DriverEOSWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_USE_KEYTAB"}, Destination: &cfg.Reva.UserStorage.EOS.UseKeytab, }, - &cli.BoolFlag{ - Name: "storage-eos-enable-home", - Usage: "enable the creation of home directories", - EnvVars: []string{"STORAGE_USERS_DRIVER_EOS_ENABLE_HOME"}, - Destination: &cfg.Reva.UserStorage.EOS.EnableHome, - }, &cli.StringFlag{ Name: "storage-eos-sec-protocol", Usage: "the xrootd security protocol to use between the server and EOS", diff --git a/storage/pkg/flagset/userdrivers/driverlocal.go b/storage/pkg/flagset/userdrivers/driverlocal.go index 7241b106497..049178a41e2 100644 --- a/storage/pkg/flagset/userdrivers/driverlocal.go +++ b/storage/pkg/flagset/userdrivers/driverlocal.go @@ -16,5 +16,19 @@ func DriverLocalWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_LOCAL_ROOT"}, Destination: &cfg.Reva.UserStorage.Local.Root, }, + &cli.StringFlag{ + Name: "storage-local-share-folder", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.Local.ShareFolder, "/Shares"), + Usage: "the path to the local share folder", + EnvVars: []string{"STORAGE_USERS_DRIVER_LOCAL_SHARE_FOLDER"}, + Destination: &cfg.Reva.UserStorage.Local.ShareFolder, + }, + &cli.StringFlag{ + Name: "storage-local-user-layout", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.Local.UserLayout, "{{.Username}}"), + Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.UsernameLower}} and {{.Provider}} also supports prefixing dirs: "{{.UsernamePrefixCount.2}}/{{.UsernameLower}}" will turn "Einstein" into "Ei/Einstein" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_LOCAL_USER_LAYOUT"}, + Destination: &cfg.Reva.UserStorage.Local.UserLayout, + }, } } diff --git a/storage/pkg/flagset/userdrivers/driverocis.go b/storage/pkg/flagset/userdrivers/driverocis.go index 3621f8578a3..fd7e0f22605 100644 --- a/storage/pkg/flagset/userdrivers/driverocis.go +++ b/storage/pkg/flagset/userdrivers/driverocis.go @@ -16,13 +16,6 @@ func DriverOCISWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_OCIS_ROOT"}, Destination: &cfg.Reva.UserStorage.OCIS.Root, }, - &cli.BoolFlag{ - Name: "storage-ocis-enable-home", - Value: flags.OverrideDefaultBool(cfg.Reva.UserStorage.OCIS.EnableHome, true), - Usage: "enable the creation of home storages", - EnvVars: []string{"STORAGE_USERS_DRIVER_OCIS_ENABLE_HOME"}, - Destination: &cfg.Reva.UserStorage.OCIS.EnableHome, - }, &cli.StringFlag{ Name: "storage-ocis-layout", Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OCIS.UserLayout, "{{.Id.OpaqueId}}"), diff --git a/storage/pkg/flagset/userdrivers/driverowncloud.go b/storage/pkg/flagset/userdrivers/driverowncloud.go index 865b0faff29..1ba06b2fddf 100644 --- a/storage/pkg/flagset/userdrivers/driverowncloud.go +++ b/storage/pkg/flagset/userdrivers/driverowncloud.go @@ -44,13 +44,6 @@ func DriverOwnCloudWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_REDIS_ADDR"}, Destination: &cfg.Reva.UserStorage.OwnCloud.Redis, }, - &cli.BoolFlag{ - Name: "storage-owncloud-enable-home", - Value: flags.OverrideDefaultBool(cfg.Reva.UserStorage.OwnCloud.EnableHome, false), - Usage: "enable the creation of home storages", - EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_ENABLE_HOME"}, - Destination: &cfg.Reva.UserStorage.OwnCloud.EnableHome, - }, &cli.StringFlag{ Name: "storage-owncloud-layout", Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloud.UserLayout, "{{.Id.OpaqueId}}"), diff --git a/storage/pkg/flagset/userdrivers/driverowncloudsql.go b/storage/pkg/flagset/userdrivers/driverowncloudsql.go index d4e3a4cc9aa..1b965af7d9e 100644 --- a/storage/pkg/flagset/userdrivers/driverowncloudsql.go +++ b/storage/pkg/flagset/userdrivers/driverowncloudsql.go @@ -30,13 +30,6 @@ func DriverOwnCloudSQLWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER"}, Destination: &cfg.Reva.UserStorage.OwnCloudSQL.ShareFolder, }, - &cli.BoolFlag{ - Name: "storage-owncloudsql-enable-home", - Value: flags.OverrideDefaultBool(cfg.Reva.UserStorage.OwnCloudSQL.EnableHome, true), - Usage: "enable the creation of home storages", - EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_ENABLE_HOME"}, - Destination: &cfg.Reva.UserStorage.OwnCloudSQL.EnableHome, - }, &cli.StringFlag{ Name: "storage-owncloudsql-layout", Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.OwnCloudSQL.UserLayout, "{{.Username}}"), diff --git a/storage/pkg/flagset/userdrivers/drivers3.go b/storage/pkg/flagset/userdrivers/drivers3.go new file mode 100644 index 00000000000..72984e684bb --- /dev/null +++ b/storage/pkg/flagset/userdrivers/drivers3.go @@ -0,0 +1,48 @@ +package userdrivers + +import ( + "github.com/owncloud/ocis/ocis-pkg/flags" + "github.com/owncloud/ocis/storage/pkg/config" + "github.com/urfave/cli/v2" +) + +// DriverS3NGWithConfig applies cfg to the root flagset +func DriverS3WithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "storage-s3-region", + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.S3.Region, "default"), + Usage: `"the s3 region" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3_REGION"}, + Destination: &cfg.Reva.UserStorage.S3.Region, + }, + &cli.StringFlag{ + Name: "storage-s3-accesskey", + Value: "", + Usage: `"the s3 access key" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3_ACCESS_KEY"}, + Destination: &cfg.Reva.UserStorage.S3.AccessKey, + }, + &cli.StringFlag{ + Name: "storage-s3-secretkey", + Value: "", + Usage: `"the secret s3 api key" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3_SECRET_KEY"}, + Destination: &cfg.Reva.UserStorage.S3.SecretKey, + }, + &cli.StringFlag{ + Name: "storage-s3-endpoint", + Value: "", + Usage: `"s3 compatible API endpoint" `, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3_ENDPOINT"}, + Destination: &cfg.Reva.UserStorage.S3.Endpoint, + }, + &cli.StringFlag{ + Name: "storage-s3-bucket", + Value: "", + Usage: `"bucket where the data will be stored in`, + EnvVars: []string{"STORAGE_USERS_DRIVER_S3_BUCKET"}, + Destination: &cfg.Reva.UserStorage.S3.Bucket, + }, + } +} diff --git a/storage/pkg/flagset/userdrivers/drivers3ng.go b/storage/pkg/flagset/userdrivers/drivers3ng.go index 12c61320552..434bb1b1e17 100644 --- a/storage/pkg/flagset/userdrivers/drivers3ng.go +++ b/storage/pkg/flagset/userdrivers/drivers3ng.go @@ -16,13 +16,6 @@ func DriverS3NGWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_ROOT"}, Destination: &cfg.Reva.UserStorage.S3NG.Root, }, - &cli.BoolFlag{ - Name: "storage-s3ng-enable-home", - Value: flags.OverrideDefaultBool(cfg.Reva.UserStorage.S3NG.EnableHome, true), - Usage: "enable the creation of home storages", - EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_ENABLE_HOME"}, - Destination: &cfg.Reva.UserStorage.S3NG.EnableHome, - }, &cli.StringFlag{ Name: "storage-s3ng-layout", Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.S3NG.UserLayout, "{{.Id.OpaqueId}}"), @@ -39,7 +32,7 @@ func DriverS3NGWithConfig(cfg *config.Config) []cli.Flag { }, &cli.StringFlag{ Name: "storage-s3ng-region", - Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.S3NG.Root, "default"), + Value: flags.OverrideDefaultString(cfg.Reva.UserStorage.S3NG.Region, "default"), Usage: `"the s3 region" `, EnvVars: []string{"STORAGE_USERS_DRIVER_S3NG_REGION"}, Destination: &cfg.Reva.UserStorage.S3NG.Region,