[full-ci] configure users and metadata storage separately

.drone.star

@@ -1504,17 +1504,13 @@ def notify(ctx):
 
 def ocisServer(storage, accounts_hash_difficulty = 4, volumes = []):
  environment = {
- #'OCIS_LOG_LEVEL': 'debug',
  "OCIS_URL": "https://ocis-server:9200",
  "STORAGE_HOME_DRIVER": "%s" % (storage),
  "STORAGE_USERS_DRIVER": "%s" % (storage),
- "STORAGE_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users",
- "STORAGE_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root",
- "STORAGE_METADATA_ROOT": "/srv/app/tmp/ocis/metadata",
- "STORAGE_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data",
- "STORAGE_DRIVER_OWNCLOUD_REDIS_ADDR": "redis:6379" if storage == "owncloud" else "",
- "STORAGE_HOME_DATA_SERVER_URL": "http://ocis-server:9155/data",
- "STORAGE_USERS_DATA_SERVER_URL": "http://ocis-server:9158/data",
+ "STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root",
+ "STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data",
+ "STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users",
+ "STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata",
  "STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json",
  "PROXY_ENABLE_BASIC_AUTH": True,
  "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",

changelog/unreleased/configure-users-metadata-storage-separately.md

@@ -0,0 +1,9 @@
+Change: Configure users and metadata storage separately
+
+We've fixed the configuration behaviour of the user and metadata service writing in the same
+directory when using oCIS storage.
+
+Therefore we needed to separate the configuration of the users and metadata storage so that they
+now can be configured totally separate.
+
+https://github.com/owncloud/ocis/pull/2598

deployments/examples/oc10_ocis_parallel/docker-compose.yml

@@ -92,17 +92,17 @@ services:
  # ownCloud storage driver
  STORAGE_HOME_DRIVER: owncloudsql
  STORAGE_USERS_DRIVER: owncloudsql
- STORAGE_DRIVER_OWNCLOUDSQL_DATADIR: /mnt/data/files
- STORAGE_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR: /tmp
- STORAGE_DRIVER_OWNCLOUDSQL_SHARE_FOLDER: "/Shares"
- STORAGE_DRIVER_OWNCLOUDSQL_ENABLE_HOME: "false"
- STORAGE_DRIVER_OWNCLOUDSQL_LAYOUT: "{{.Username}}"
- STORAGE_DRIVER_OWNCLOUDSQL_DBUSERNAME: owncloud
- STORAGE_DRIVER_OWNCLOUDSQL_DBPASSWORD: owncloud
- STORAGE_DRIVER_OWNCLOUDSQL_DBHOST: oc10-db
- STORAGE_DRIVER_OWNCLOUDSQL_DBPORT: 3306
- STORAGE_DRIVER_OWNCLOUDSQL_DBNAME: owncloud
- STORAGE_DRIVER_OWNCLOUDSQL_REDIS_ADDR: redis:6379 # TODO: redis is not yet supported
+ STORAGE_METADATA_DRIVER: ocis # keep metadata on ocis storage since this are only small files atm
+ STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR: /mnt/data/files
+ STORAGE_USERS_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR: /tmp
+ STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER: "/Shares"
+ STORAGE_USERS_DRIVER_OWNCLOUDSQL_LAYOUT: "{{.Username}}"
+ STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBUSERNAME: owncloud
+ STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPASSWORD: owncloud
+ STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBHOST: oc10-db
+ STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPORT: 3306
+ STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBNAME: owncloud
+ STORAGE_USERS_DRIVER_OWNCLOUDSQL_REDIS_ADDR: redis:6379 # TODO: redis is not yet supported
  # ownCloud storage readonly
  OCIS_STORAGE_READ_ONLY: "false" # TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303
  # General oCIS config

deployments/examples/ocis_s3/docker-compose.yml

@@ -65,11 +65,11 @@ services:
  STORAGE_USERS_DRIVER: s3ng
  STORAGE_METADATA_DRIVER: ocis # keep metadata on ocis storage since this are only small files atm
  # s3ng specific settings
- STORAGE_DRIVER_S3NG_ENDPOINT: http://minio:9000
- STORAGE_DRIVER_S3NG_REGION: default
- STORAGE_DRIVER_S3NG_ACCESS_KEY: ${MINIO_ACCESS_KEY:-ocis}
- STORAGE_DRIVER_S3NG_SECRET_KEY: ${MINIO_SECRET_KEY:-ocis-secret-key}
- STORAGE_DRIVER_S3NG_BUCKET: ${MINIO_BUCKET:-ocis-bucket}
+ STORAGE_USERS_DRIVER_S3NG_ENDPOINT: http://minio:9000
+ STORAGE_USERS_DRIVER_S3NG_REGION: default
+ STORAGE_USERS_DRIVER_S3NG_ACCESS_KEY: ${MINIO_ACCESS_KEY:-ocis}
+ STORAGE_USERS_DRIVER_S3NG_SECRET_KEY: ${MINIO_SECRET_KEY:-ocis-secret-key}
+ STORAGE_USERS_DRIVER_S3NG_BUCKET: ${MINIO_BUCKET:-ocis-bucket}
  volumes:
  - ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
  - ocis-data:/var/tmp/ocis

storage/pkg/command/frontend.go

@@ -204,7 +204,7 @@ func frontendConfigFromStruct(c *cli.Context, cfg *config.Config, filesCfg map[s
  "db_host": cfg.Reva.Sharing.UserSQLHost,
  "db_port": cfg.Reva.Sharing.UserSQLPort,
  "db_name": cfg.Reva.Sharing.UserSQLName,
- "namespace": cfg.Reva.Storages.EOS.Root,
+ "namespace": cfg.Reva.UserStorage.EOS.Root,
  "gatewaysvc": cfg.Reva.Gateway.Endpoint,
  },
  },

storage/pkg/command/storagedrivers/home.go

@@ -0,0 +1,129 @@
+package storagedrivers
+
+import (
+ "github.com/owncloud/ocis/storage/pkg/config"
+)
+
+func HomeDrivers(cfg *config.Config) map[string]interface{} {
+ return map[string]interface{}{
+ "eos": map[string]interface{}{
+ "namespace": cfg.Reva.UserStorage.EOS.Root,
+ "shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace,
+ "uploads_namespace": cfg.Reva.UserStorage.EOS.UploadsNamespace,
+ "share_folder": cfg.Reva.UserStorage.EOS.ShareFolder,
+ "eos_binary": cfg.Reva.UserStorage.EOS.EosBinary,
+ "xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary,
+ "master_url": cfg.Reva.UserStorage.EOS.MasterURL,
+ "slave_url": cfg.Reva.UserStorage.EOS.SlaveURL,
+ "cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory,
+ "sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol,
+ "keytab": cfg.Reva.UserStorage.EOS.Keytab,
+ "single_username": cfg.Reva.UserStorage.EOS.SingleUsername,
+ "enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging,
+ "show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
+ "force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
+ "use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab,
+ "gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC,
+ },
+ "eoshome": map[string]interface{}{
+ "namespace": cfg.Reva.UserStorage.EOS.Root,
+ "shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace,
+ "uploads_namespace": cfg.Reva.UserStorage.EOS.UploadsNamespace,
+ "share_folder": cfg.Reva.UserStorage.EOS.ShareFolder,
+ "eos_binary": cfg.Reva.UserStorage.EOS.EosBinary,
+ "xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary,
+ "master_url": cfg.Reva.UserStorage.EOS.MasterURL,
+ "slave_url": cfg.Reva.UserStorage.EOS.SlaveURL,
+ "cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory,
+ "sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol,
+ "keytab": cfg.Reva.UserStorage.EOS.Keytab,
+ "single_username": cfg.Reva.UserStorage.EOS.SingleUsername,
+ "user_layout": cfg.Reva.UserStorage.EOS.UserLayout,
+ "enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging,
+ "show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
+ "force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
+ "use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab,
+ "gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC,
+ },
+ "eosgrpc": map[string]interface{}{
+ "namespace": cfg.Reva.UserStorage.EOS.Root,
+ "shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace,
+ "share_folder": cfg.Reva.UserStorage.EOS.ShareFolder,
+ "eos_binary": cfg.Reva.UserStorage.EOS.EosBinary,
+ "xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary,
+ "master_url": cfg.Reva.UserStorage.EOS.MasterURL,
+ "master_grpc_uri": cfg.Reva.UserStorage.EOS.GrpcURI,
+ "slave_url": cfg.Reva.UserStorage.EOS.SlaveURL,
+ "cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory,
+ "sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol,
+ "keytab": cfg.Reva.UserStorage.EOS.Keytab,
+ "single_username": cfg.Reva.UserStorage.EOS.SingleUsername,
+ "user_layout": cfg.Reva.UserStorage.EOS.UserLayout,
+ "enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging,
+ "show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
+ "force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
+ "use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab,
+ "enable_home": true,
+ "gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC,
+ },
+ "local": map[string]interface{}{
+ "root": cfg.Reva.UserStorage.Local.Root,
+ "share_folder": cfg.Reva.UserStorage.Local.ShareFolder,
+ },
+ "localhome": map[string]interface{}{
+ "root": cfg.Reva.UserStorage.Local.Root,
+ "share_folder": cfg.Reva.UserStorage.Local.ShareFolder,
+ "user_layout": cfg.Reva.UserStorage.Local.UserLayout,
+ },
+ "owncloud": map[string]interface{}{
+ "datadirectory": cfg.Reva.UserStorage.OwnCloud.Root,
+ "upload_info_dir": cfg.Reva.UserStorage.OwnCloud.UploadInfoDir,
+ "share_folder": cfg.Reva.UserStorage.OwnCloud.ShareFolder,
+ "user_layout": cfg.Reva.UserStorage.OwnCloud.UserLayout,
+ "redis": cfg.Reva.UserStorage.OwnCloud.Redis,
+ "enable_home": true,
+ "scan": cfg.Reva.UserStorage.OwnCloud.Scan,
+ "userprovidersvc": cfg.Reva.Users.Endpoint,
+ },
+ "owncloudsql": map[string]interface{}{
+ "datadirectory": cfg.Reva.UserStorage.OwnCloudSQL.Root,
+ "upload_info_dir": cfg.Reva.UserStorage.OwnCloudSQL.UploadInfoDir,
+ "share_folder": cfg.Reva.UserStorage.OwnCloudSQL.ShareFolder,
+ "user_layout": cfg.Reva.UserStorage.OwnCloudSQL.UserLayout,
+ "enable_home": true,
+ "dbusername": cfg.Reva.UserStorage.OwnCloudSQL.DBUsername,
+ "dbpassword": cfg.Reva.UserStorage.OwnCloudSQL.DBPassword,
+ "dbhost": cfg.Reva.UserStorage.OwnCloudSQL.DBHost,
+ "dbport": cfg.Reva.UserStorage.OwnCloudSQL.DBPort,
+ "dbname": cfg.Reva.UserStorage.OwnCloudSQL.DBName,
+ "userprovidersvc": cfg.Reva.Users.Endpoint,
+ },
+ "ocis": map[string]interface{}{
+ "root": cfg.Reva.UserStorage.OCIS.Root,
+ "enable_home": true,
+ "user_layout": cfg.Reva.UserStorage.OCIS.UserLayout,
+ "share_folder": cfg.Reva.UserStorage.OCIS.ShareFolder,
+ "treetime_accounting": true,
+ "treesize_accounting": true,
+ "owner": cfg.Reva.UserStorage.OCIS.ServiceUserUUID, // the accounts service system account uuid
+ },
+ "s3": map[string]interface{}{
+ "region": cfg.Reva.UserStorage.S3.Region,
+ "access_key": cfg.Reva.UserStorage.S3.AccessKey,
+ "secret_key": cfg.Reva.UserStorage.S3.SecretKey,
+ "endpoint": cfg.Reva.UserStorage.S3.Endpoint,
+ "bucket": cfg.Reva.UserStorage.S3.Bucket,
+ },
+ "s3ng": map[string]interface{}{
+ "root": cfg.Reva.UserStorage.S3NG.Root,
+ "enable_home": true,
+ "user_layout": cfg.Reva.UserStorage.S3NG.UserLayout,
+ "share_folder": cfg.Reva.UserStorage.S3NG.ShareFolder,
+ "s3.region": cfg.Reva.UserStorage.S3NG.Region,
+ "s3.access_key": cfg.Reva.UserStorage.S3NG.AccessKey,
+ "s3.secret_key": cfg.Reva.UserStorage.S3NG.SecretKey,
+ "s3.endpoint": cfg.Reva.UserStorage.S3NG.Endpoint,
+ "s3.bucket": cfg.Reva.UserStorage.S3NG.Bucket,
+ },
+ }
+}