Skip to content

Commit

Permalink
Merge pull request #2598 from owncloud/fix_metadata_storage
Browse files Browse the repository at this point in the history
[full-ci] configure users and metadata storage separately
  • Loading branch information
wkloucek committed Oct 11, 2021
2 parents 1ee4198 + 6c6cb64 commit 0d4bc30
Show file tree
Hide file tree
Showing 34 changed files with 1,111 additions and 605 deletions.
12 changes: 4 additions & 8 deletions .drone.star
Original file line number Diff line number Diff line change
Expand Up @@ -1504,17 +1504,13 @@ def notify(ctx):

def ocisServer(storage, accounts_hash_difficulty = 4, volumes = []):
environment = {
#'OCIS_LOG_LEVEL': 'debug',
"OCIS_URL": "https://ocis-server:9200",
"STORAGE_HOME_DRIVER": "%s" % (storage),
"STORAGE_USERS_DRIVER": "%s" % (storage),
"STORAGE_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users",
"STORAGE_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root",
"STORAGE_METADATA_ROOT": "/srv/app/tmp/ocis/metadata",
"STORAGE_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data",
"STORAGE_DRIVER_OWNCLOUD_REDIS_ADDR": "redis:6379" if storage == "owncloud" else "",
"STORAGE_HOME_DATA_SERVER_URL": "http://ocis-server:9155/data",
"STORAGE_USERS_DATA_SERVER_URL": "http://ocis-server:9158/data",
"STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root",
"STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data",
"STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users",
"STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata",
"STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json",
"PROXY_ENABLE_BASIC_AUTH": True,
"WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
Change: Configure users and metadata storage separately

We've fixed the configuration behaviour of the user and metadata service writing in the same
directory when using oCIS storage.

Therefore we needed to separate the configuration of the users and metadata storage so that they
now can be configured totally separate.

https://github.com/owncloud/ocis/pull/2598
22 changes: 11 additions & 11 deletions deployments/examples/oc10_ocis_parallel/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,17 +92,17 @@ services:
# ownCloud storage driver
STORAGE_HOME_DRIVER: owncloudsql
STORAGE_USERS_DRIVER: owncloudsql
STORAGE_DRIVER_OWNCLOUDSQL_DATADIR: /mnt/data/files
STORAGE_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR: /tmp
STORAGE_DRIVER_OWNCLOUDSQL_SHARE_FOLDER: "/Shares"
STORAGE_DRIVER_OWNCLOUDSQL_ENABLE_HOME: "false"
STORAGE_DRIVER_OWNCLOUDSQL_LAYOUT: "{{.Username}}"
STORAGE_DRIVER_OWNCLOUDSQL_DBUSERNAME: owncloud
STORAGE_DRIVER_OWNCLOUDSQL_DBPASSWORD: owncloud
STORAGE_DRIVER_OWNCLOUDSQL_DBHOST: oc10-db
STORAGE_DRIVER_OWNCLOUDSQL_DBPORT: 3306
STORAGE_DRIVER_OWNCLOUDSQL_DBNAME: owncloud
STORAGE_DRIVER_OWNCLOUDSQL_REDIS_ADDR: redis:6379 # TODO: redis is not yet supported
STORAGE_METADATA_DRIVER: ocis # keep metadata on ocis storage since this are only small files atm
STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR: /mnt/data/files
STORAGE_USERS_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR: /tmp
STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER: "/Shares"
STORAGE_USERS_DRIVER_OWNCLOUDSQL_LAYOUT: "{{.Username}}"
STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBUSERNAME: owncloud
STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPASSWORD: owncloud
STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBHOST: oc10-db
STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPORT: 3306
STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBNAME: owncloud
STORAGE_USERS_DRIVER_OWNCLOUDSQL_REDIS_ADDR: redis:6379 # TODO: redis is not yet supported
# ownCloud storage readonly
OCIS_STORAGE_READ_ONLY: "false" # TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303
# General oCIS config
Expand Down
10 changes: 5 additions & 5 deletions deployments/examples/ocis_s3/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,11 @@ services:
STORAGE_USERS_DRIVER: s3ng
STORAGE_METADATA_DRIVER: ocis # keep metadata on ocis storage since this are only small files atm
# s3ng specific settings
STORAGE_DRIVER_S3NG_ENDPOINT: http://minio:9000
STORAGE_DRIVER_S3NG_REGION: default
STORAGE_DRIVER_S3NG_ACCESS_KEY: ${MINIO_ACCESS_KEY:-ocis}
STORAGE_DRIVER_S3NG_SECRET_KEY: ${MINIO_SECRET_KEY:-ocis-secret-key}
STORAGE_DRIVER_S3NG_BUCKET: ${MINIO_BUCKET:-ocis-bucket}
STORAGE_USERS_DRIVER_S3NG_ENDPOINT: http://minio:9000
STORAGE_USERS_DRIVER_S3NG_REGION: default
STORAGE_USERS_DRIVER_S3NG_ACCESS_KEY: ${MINIO_ACCESS_KEY:-ocis}
STORAGE_USERS_DRIVER_S3NG_SECRET_KEY: ${MINIO_SECRET_KEY:-ocis-secret-key}
STORAGE_USERS_DRIVER_S3NG_BUCKET: ${MINIO_BUCKET:-ocis-bucket}
volumes:
- ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
- ocis-data:/var/tmp/ocis
Expand Down
128 changes: 0 additions & 128 deletions storage/pkg/command/drivers.go

This file was deleted.

2 changes: 1 addition & 1 deletion storage/pkg/command/frontend.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ func frontendConfigFromStruct(c *cli.Context, cfg *config.Config, filesCfg map[s
"db_host": cfg.Reva.Sharing.UserSQLHost,
"db_port": cfg.Reva.Sharing.UserSQLPort,
"db_name": cfg.Reva.Sharing.UserSQLName,
"namespace": cfg.Reva.Storages.EOS.Root,
"namespace": cfg.Reva.UserStorage.EOS.Root,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
},
},
Expand Down
129 changes: 129 additions & 0 deletions storage/pkg/command/storagedrivers/home.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
package storagedrivers

import (
"github.com/owncloud/ocis/storage/pkg/config"
)

func HomeDrivers(cfg *config.Config) map[string]interface{} {
return map[string]interface{}{
"eos": map[string]interface{}{
"namespace": cfg.Reva.UserStorage.EOS.Root,
"shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace,
"uploads_namespace": cfg.Reva.UserStorage.EOS.UploadsNamespace,
"share_folder": cfg.Reva.UserStorage.EOS.ShareFolder,
"eos_binary": cfg.Reva.UserStorage.EOS.EosBinary,
"xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary,
"master_url": cfg.Reva.UserStorage.EOS.MasterURL,
"slave_url": cfg.Reva.UserStorage.EOS.SlaveURL,
"cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory,
"sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol,
"keytab": cfg.Reva.UserStorage.EOS.Keytab,
"single_username": cfg.Reva.UserStorage.EOS.SingleUsername,
"enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab,
"gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC,
},
"eoshome": map[string]interface{}{
"namespace": cfg.Reva.UserStorage.EOS.Root,
"shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace,
"uploads_namespace": cfg.Reva.UserStorage.EOS.UploadsNamespace,
"share_folder": cfg.Reva.UserStorage.EOS.ShareFolder,
"eos_binary": cfg.Reva.UserStorage.EOS.EosBinary,
"xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary,
"master_url": cfg.Reva.UserStorage.EOS.MasterURL,
"slave_url": cfg.Reva.UserStorage.EOS.SlaveURL,
"cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory,
"sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol,
"keytab": cfg.Reva.UserStorage.EOS.Keytab,
"single_username": cfg.Reva.UserStorage.EOS.SingleUsername,
"user_layout": cfg.Reva.UserStorage.EOS.UserLayout,
"enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab,
"gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC,
},
"eosgrpc": map[string]interface{}{
"namespace": cfg.Reva.UserStorage.EOS.Root,
"shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace,
"share_folder": cfg.Reva.UserStorage.EOS.ShareFolder,
"eos_binary": cfg.Reva.UserStorage.EOS.EosBinary,
"xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary,
"master_url": cfg.Reva.UserStorage.EOS.MasterURL,
"master_grpc_uri": cfg.Reva.UserStorage.EOS.GrpcURI,
"slave_url": cfg.Reva.UserStorage.EOS.SlaveURL,
"cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory,
"sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol,
"keytab": cfg.Reva.UserStorage.EOS.Keytab,
"single_username": cfg.Reva.UserStorage.EOS.SingleUsername,
"user_layout": cfg.Reva.UserStorage.EOS.UserLayout,
"enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab,
"enable_home": true,
"gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC,
},
"local": map[string]interface{}{
"root": cfg.Reva.UserStorage.Local.Root,
"share_folder": cfg.Reva.UserStorage.Local.ShareFolder,
},
"localhome": map[string]interface{}{
"root": cfg.Reva.UserStorage.Local.Root,
"share_folder": cfg.Reva.UserStorage.Local.ShareFolder,
"user_layout": cfg.Reva.UserStorage.Local.UserLayout,
},
"owncloud": map[string]interface{}{
"datadirectory": cfg.Reva.UserStorage.OwnCloud.Root,
"upload_info_dir": cfg.Reva.UserStorage.OwnCloud.UploadInfoDir,
"share_folder": cfg.Reva.UserStorage.OwnCloud.ShareFolder,
"user_layout": cfg.Reva.UserStorage.OwnCloud.UserLayout,
"redis": cfg.Reva.UserStorage.OwnCloud.Redis,
"enable_home": true,
"scan": cfg.Reva.UserStorage.OwnCloud.Scan,
"userprovidersvc": cfg.Reva.Users.Endpoint,
},
"owncloudsql": map[string]interface{}{
"datadirectory": cfg.Reva.UserStorage.OwnCloudSQL.Root,
"upload_info_dir": cfg.Reva.UserStorage.OwnCloudSQL.UploadInfoDir,
"share_folder": cfg.Reva.UserStorage.OwnCloudSQL.ShareFolder,
"user_layout": cfg.Reva.UserStorage.OwnCloudSQL.UserLayout,
"enable_home": true,
"dbusername": cfg.Reva.UserStorage.OwnCloudSQL.DBUsername,
"dbpassword": cfg.Reva.UserStorage.OwnCloudSQL.DBPassword,
"dbhost": cfg.Reva.UserStorage.OwnCloudSQL.DBHost,
"dbport": cfg.Reva.UserStorage.OwnCloudSQL.DBPort,
"dbname": cfg.Reva.UserStorage.OwnCloudSQL.DBName,
"userprovidersvc": cfg.Reva.Users.Endpoint,
},
"ocis": map[string]interface{}{
"root": cfg.Reva.UserStorage.OCIS.Root,
"enable_home": true,
"user_layout": cfg.Reva.UserStorage.OCIS.UserLayout,
"share_folder": cfg.Reva.UserStorage.OCIS.ShareFolder,
"treetime_accounting": true,
"treesize_accounting": true,
"owner": cfg.Reva.UserStorage.OCIS.ServiceUserUUID, // the accounts service system account uuid
},
"s3": map[string]interface{}{
"region": cfg.Reva.UserStorage.S3.Region,
"access_key": cfg.Reva.UserStorage.S3.AccessKey,
"secret_key": cfg.Reva.UserStorage.S3.SecretKey,
"endpoint": cfg.Reva.UserStorage.S3.Endpoint,
"bucket": cfg.Reva.UserStorage.S3.Bucket,
},
"s3ng": map[string]interface{}{
"root": cfg.Reva.UserStorage.S3NG.Root,
"enable_home": true,
"user_layout": cfg.Reva.UserStorage.S3NG.UserLayout,
"share_folder": cfg.Reva.UserStorage.S3NG.ShareFolder,
"s3.region": cfg.Reva.UserStorage.S3NG.Region,
"s3.access_key": cfg.Reva.UserStorage.S3NG.AccessKey,
"s3.secret_key": cfg.Reva.UserStorage.S3NG.SecretKey,
"s3.endpoint": cfg.Reva.UserStorage.S3NG.Endpoint,
"s3.bucket": cfg.Reva.UserStorage.S3NG.Bucket,
},
}
}
Loading

0 comments on commit 0d4bc30

Please sign in to comment.