From cf7074d718bf868484738a22b9938ed59d6098e5 Mon Sep 17 00:00:00 2001 From: Andre Duffeck Date: Fri, 24 Mar 2023 09:51:59 +0000 Subject: [PATCH] Make the number of concurrent goroutines in decomposedfs configurable (#5887) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Make the number of concurrent goroutines in decomposedfs configurable * Update services/storage-users/pkg/config/config.go Co-authored-by: Martin * Update services/storage-users/pkg/config/config.go Co-authored-by: Martin --------- Co-authored-by: Jörn Friedrich Dreyer Co-authored-by: Martin --- .../_includes/adoc/global_configvars.adoc | 52 +++++++++---------- .../adoc/storage-users_configvars.adoc | 18 +++++++ .../storage-users-config-example.yaml | 2 + .../_includes/storage-users_configvars.md | 2 + 4 files changed, 48 insertions(+), 26 deletions(-) diff --git a/services/_includes/adoc/global_configvars.adoc b/services/_includes/adoc/global_configvars.adoc index 9b69fa22f55..96905807399 100644 --- a/services/_includes/adoc/global_configvars.adoc +++ b/services/_includes/adoc/global_configvars.adoc @@ -26,7 +26,7 @@ a| [subs=-attributes] ++false ++ a| [subs=-attributes] -The default role assignments the demo users should be setup. +Flag to enable or disable the creation of the demo users. a| `LDAP_BIND_DN` @@ -41,7 +41,7 @@ a| [subs=-attributes] ++string ++ a| [subs=-attributes] -++uid=idp,ou=sysusers,o=libregraph-idm ++ +++uid=reva,ou=sysusers,o=libregraph-idm ++ a| [subs=-attributes] LDAP DN to use for simple bind authentication with the target LDAP server. @@ -80,7 +80,7 @@ a| [subs=-attributes] ++~/.ocis/idm/ldap.crt ++ a| [subs=-attributes] -Path/File name for the root CA certificate (in PEM format) used to validate TLS server certificates of the LDAP service. If not definied, the root directory derives from $OCIS_BASE_DATA_PATH:/idp. +Path/File name for the root CA certificate (in PEM format) used to validate TLS server certificates of the LDAP service. If not definied, the root directory derives from $OCIS_BASE_DATA_PATH:/idm. a| `LDAP_DISABLED_USERS_GROUP_DN` @@ -163,7 +163,7 @@ a| [subs=-attributes] ++groupOfNames ++ a| [subs=-attributes] -The object class to use for groups in the default group search filter ('groupOfNames'). +The object class to use for groups in the default group search filter like 'groupOfNames'. a| `LDAP_GROUP_SCHEMA_DISPLAYNAME` @@ -213,7 +213,7 @@ a| [subs=-attributes] ++ownclouduuid ++ a| [subs=-attributes] -LDAP Attribute to use as the unique id for groups. This should be a stable globally unique id (e.g. a UUID). +LDAP Attribute to use as the unique ID for groups. This should be a stable globally unique ID like a UUID. a| `LDAP_GROUP_SCHEMA_ID_IS_OCTETSTRING` @@ -229,7 +229,7 @@ a| [subs=-attributes] ++false ++ a| [subs=-attributes] -Set this to true if the defined 'id' attribute for groups is of the 'OCTETSTRING' syntax. This is e.g. required when using the 'objectGUID' attribute of Active Directory for the group IDs. +Set this to true if the defined 'id' attribute for groups is of the 'OCTETSTRING' syntax. This is e.g. required when using the 'objectGUID' attribute of Active Directory for the group ID's. a| `LDAP_GROUP_SCHEMA_MAIL` @@ -314,7 +314,7 @@ a| [subs=-attributes] ++ldaps://localhost:9235 ++ a| [subs=-attributes] -Url of the LDAP service to use as IDP. +URI of the LDAP Server to connect to. Supported URI schemes are 'ldaps://' and 'ldap://' a| `LDAP_USER_BASE_DN` @@ -349,7 +349,7 @@ a| [subs=-attributes] ++ownCloudUserEnabled ++ a| [subs=-attributes] -LDAP Attribute to use as a flag telling if the user is enabled or disabled. +LDAP attribute to use as a flag telling if the user is enabled or disabled. a| `LDAP_USER_FILTER` @@ -385,7 +385,7 @@ a| [subs=-attributes] ++inetOrgPerson ++ a| [subs=-attributes] -LDAP User ObjectClass like 'inetOrgPerson'. +The object class to use for users in the default user search filter like 'inetOrgPerson'. a| `LDAP_USER_SCHEMA_DISPLAYNAME` @@ -416,10 +416,10 @@ a| [subs=-attributes] ++string ++ a| [subs=-attributes] -++uid ++ +++ownclouduuid ++ a| [subs=-attributes] -LDAP User uuid attribute like 'uid'. +LDAP Attribute to use as the unique id for users. This should be a stable globally unique id like a UUID. a| `LDAP_USER_SCHEMA_ID_IS_OCTETSTRING` @@ -435,7 +435,7 @@ a| [subs=-attributes] ++false ++ a| [subs=-attributes] -Set this to true if the defined 'id' attribute for users is of the 'OCTETSTRING' syntax. This is e.g. required when using the 'objectGUID' attribute of Active Directory for the user IDs. +Set this to true if the defined 'id' attribute for users is of the 'OCTETSTRING' syntax. This is e.g. required when using the 'objectGUID' attribute of Active Directory for the user ID's. a| `LDAP_USER_SCHEMA_MAIL` @@ -453,7 +453,7 @@ a| [subs=-attributes] ++mail ++ a| [subs=-attributes] -LDAP User email attribute like 'mail'. +LDAP Attribute to use for the email address of users. a| `LDAP_USER_SCHEMA_USERNAME` @@ -468,10 +468,10 @@ a| [subs=-attributes] ++string ++ a| [subs=-attributes] -++displayName ++ +++uid ++ a| [subs=-attributes] -LDAP User name attribute like 'displayName'. +LDAP Attribute to use for username of users. a| `LDAP_USER_SCOPE` @@ -489,7 +489,7 @@ a| [subs=-attributes] ++sub ++ a| [subs=-attributes] -LDAP search scope to use when looking up users. Supported scopes are 'base', 'one' and 'sub'. +LDAP search scope to use when looking up users. Supported values are 'base', 'one' and 'sub'. a| `OCIS_ADMIN_USER_ID` @@ -524,7 +524,7 @@ a| [subs=-attributes] ++memory ++ a| [subs=-attributes] -Store implementation for the cache. Supported values are 'memory' (default), 'redis', 'redis-sentinel', 'nats-js', and 'etcd'. See the text description for details. +The type of the userinfo cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details. a| `OCIS_CACHE_STORE_ADDRESS` @@ -575,7 +575,7 @@ a| [subs=-attributes] ++[] ++ a| [subs=-attributes] -A comma separated list of nodes to access the configured store. This has no effect when the 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. +A comma separated list of nodes to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. a| `OCIS_CACHE_STORE_SIZE` @@ -663,7 +663,7 @@ a| [subs=-attributes] ++[]string ++ a| [subs=-attributes] -++[Authorization Origin Content-Type Accept X-Requested-With] ++ +++[Origin Accept Content-Type Depth Authorization Ocs-Apirequest If-None-Match If-Match Destination Overwrite X-Request-Id X-Requested-With Tus-Resumable Tus-Checksum-Algorithm Upload-Concat Upload-Length Upload-Metadata Upload-Defer-Length Upload-Expires Upload-Checksum Upload-Offset X-HTTP-Method-Override] ++ a| [subs=-attributes] A comma-separated list of allowed CORS headers. See following chapter for more details: *Access-Control-Request-Headers* at \https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Request-Headers. @@ -682,7 +682,7 @@ a| [subs=-attributes] ++[]string ++ a| [subs=-attributes] -++[GET POST PUT PATCH DELETE OPTIONS] ++ +++[OPTIONS HEAD GET PUT POST DELETE MKCOL PROPFIND PROPPATCH MOVE COPY REPORT SEARCH] ++ a| [subs=-attributes] A comma-separated list of allowed CORS methods. See following chapter for more details: *Access-Control-Request-Method* at \https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Request-Method @@ -744,7 +744,7 @@ a| [subs=-attributes] ++false ++ a| [subs=-attributes] -Enable TLS for the connection to the events broker. The events broker is the ocis service which receives and delivers events between the services.. +Enable TLS for the connection to the events broker. The events broker is the ocis service which receives and delivers events between the services. a| `OCIS_GRPC_CLIENT_TLS_CACERT` @@ -906,7 +906,7 @@ a| [subs=-attributes] ++false ++ a| [subs=-attributes] -Whether to verify the server TLS certificates. +Whether the server should skip the client certificate verification during the TLS handshake. a| `OCIS_JWT_SECRET` @@ -1221,7 +1221,7 @@ a| [subs=-attributes] ++false ++ a| [subs=-attributes] -Set this to true if you want to enforce passwords on Uploader, Editor or Contributor shares. If not using the global OCIS_SHARING_PUBLIC_WRITEABLE_SHARE_MUST_HAVE_PASSWORD, you must define the FRONTEND_OCS_PUBLIC_WRITEABLE_SHARE_MUST_HAVE_PASSWORD in the frontend service. +Set this to true if you want to enforce passwords on Uploader, Editor or Contributor shares. a| `OCIS_SPACES_MAX_QUOTA` @@ -1236,7 +1236,7 @@ a| [subs=-attributes] ++0 ++ a| [subs=-attributes] -Set a global max quota for spaces in bytes. A value of 0 equals unlimited. If not using the global OCIS_SPACES_MAX_QUOTA, you must define the FRONTEND_MAX_QUOTA in the frontend service. +Set the global max quota value in bytes. A value of 0 equals unlimited. The value is provided via capabilities. a| `OCIS_SYSTEM_USER_API_KEY` @@ -1274,7 +1274,7 @@ a| [subs=-attributes] ++ ++ a| [subs=-attributes] -ID of the oCIS STORAGE-SYSTEM system user. Admins need to set the ID for the STORAGE-SYSTEM system user in this config option which is then used to reference the user. Any reasonable long string is possible, preferably this would be an UUIDv4 format. +ID of the oCIS storage-system system user. Admins need to set the ID for the STORAGE-SYSTEM system user in this config option which is then used to reference the user. Any reasonable long string is possible, preferably this would be an UUIDv4 format. a| `OCIS_SYSTEM_USER_IDP` @@ -1483,7 +1483,7 @@ a| [subs=-attributes] ++https://localhost:9200 ++ a| [subs=-attributes] -URL to load themes from. Will be prepended to the theme path. +URL of the OIDC issuer. It defaults to URL of the builtin IDP. a| `REVA_GATEWAY` diff --git a/services/_includes/adoc/storage-users_configvars.adoc b/services/_includes/adoc/storage-users_configvars.adoc index d5d024a6339..a952cd7fe76 100644 --- a/services/_includes/adoc/storage-users_configvars.adoc +++ b/services/_includes/adoc/storage-users_configvars.adoc @@ -315,6 +315,15 @@ a| [subs=-attributes] a| [subs=-attributes] When trying to lock files, ocis will multiply the cycle with this factor and use it as a millisecond timeout. Values of 0 or below will be ignored and the default value of 30 will be used. +a|`STORAGE_USERS_OCIS_MAX_CONCURRENCY` + + +a| [subs=-attributes] +++int ++ +a| [subs=-attributes] +++0 ++ +a| [subs=-attributes] +Maximum number of concurrent go-routines. Higher values can potentially get work done faster but will also cause more load on the system. Values of 0 or below will be ignored and the default value of 100 will be used. + a|`STORAGE_USERS_OCIS_ASYNC_UPLOADS` + a| [subs=-attributes] @@ -461,6 +470,15 @@ a| [subs=-attributes] a| [subs=-attributes] When trying to lock files, ocis will multiply the cycle with this factor and use it as a millisecond timeout. Values of 0 or below will be ignored and the default value of 30 will be used. +a|`STORAGE_USERS_S3NG_MAX_CONCURRENCY` + + +a| [subs=-attributes] +++int ++ +a| [subs=-attributes] +++0 ++ +a| [subs=-attributes] +Maximum number of concurrent go-routines. Higher values can potentially get work done faster but will also cause more load on the system. Values of 0 or below will be ignored and the default value of 100 will be used. + a|`STORAGE_USERS_OWNCLOUDSQL_DATADIR` + a| [subs=-attributes] diff --git a/services/_includes/storage-users-config-example.yaml b/services/_includes/storage-users-config-example.yaml index f751a30d80d..4084f63feab 100644 --- a/services/_includes/storage-users-config-example.yaml +++ b/services/_includes/storage-users-config-example.yaml @@ -44,6 +44,7 @@ drivers: share_folder: /Shares max_acquire_lock_cycles: 20 lock_cycle_duration_factor: 30 + max_concurrency: 0 async_uploads: false max_quota: 0 s3ng: @@ -61,6 +62,7 @@ drivers: share_folder: /Shares max_acquire_lock_cycles: 20 lock_cycle_duration_factor: 30 + max_concurrency: 0 owncloudsql: root: ~/.ocis/storage/owncloud share_folder: /Shares diff --git a/services/_includes/storage-users_configvars.md b/services/_includes/storage-users_configvars.md index 10f354c98ed..00734d8b825 100644 --- a/services/_includes/storage-users_configvars.md +++ b/services/_includes/storage-users_configvars.md @@ -33,6 +33,7 @@ | STORAGE_USERS_OCIS_SHARE_FOLDER | string | /Shares | Name of the folder jailing all shares.| | STORAGE_USERS_OCIS_MAX_ACQUIRE_LOCK_CYCLES | int | 20 | When trying to lock files, ocis will try this amount of times to acquire the lock before failing. After each try it will wait for an increasing amount of time. Values of 0 or below will be ignored and the default value of 20 will be used.| | STORAGE_USERS_OCIS_LOCK_CYCLE_DURATION_FACTOR | int | 30 | When trying to lock files, ocis will multiply the cycle with this factor and use it as a millisecond timeout. Values of 0 or below will be ignored and the default value of 30 will be used.| +| STORAGE_USERS_OCIS_MAX_CONCURRENCY | int | 0 | Maximum number of concurrent go-routines. Higher values can potentially get work done faster but will also cause more load on the system. Values of 0 or below will be ignored and the default value of 100 will be used.| | STORAGE_USERS_OCIS_ASYNC_UPLOADS | bool | false | Enable asynchronous file uploads.| | OCIS_SPACES_MAX_QUOTA
STORAGE_USERS_OCIS_MAX_QUOTA | uint64 | 0 | Set a global max quota for spaces in bytes. A value of 0 equals unlimited. If not using the global OCIS_SPACES_MAX_QUOTA, you must define the FRONTEND_MAX_QUOTA in the frontend service.| | STORAGE_USERS_S3NG_METADATA_BACKEND | string | xattrs | The backend to use for storing metadata. Supported values are 'xattrs' and 'ini'. The setting 'xattrs' uses extended attributes to store file metadata while 'ini' uses a dedicated file to store file metadata. Defaults to 'xattrs'.| @@ -49,6 +50,7 @@ | STORAGE_USERS_S3NG_SHARE_FOLDER | string | /Shares | Name of the folder jailing all shares.| | STORAGE_USERS_S3NG_MAX_ACQUIRE_LOCK_CYCLES | int | 20 | When trying to lock files, ocis will try this amount of times to acquire the lock before failing. After each try it will wait for an increasing amount of time. Values of 0 or below will be ignored and the default value of 20 will be used.| | STORAGE_USERS_S3NG_LOCK_CYCLE_DURATION_FACTOR | int | 30 | When trying to lock files, ocis will multiply the cycle with this factor and use it as a millisecond timeout. Values of 0 or below will be ignored and the default value of 30 will be used.| +| STORAGE_USERS_S3NG_MAX_CONCURRENCY | int | 0 | Maximum number of concurrent go-routines. Higher values can potentially get work done faster but will also cause more load on the system. Values of 0 or below will be ignored and the default value of 100 will be used.| | STORAGE_USERS_OWNCLOUDSQL_DATADIR | string | ~/.ocis/storage/owncloud | The directory where the filesystem storage will store SQL migration data. If not definied, the root directory derives from $OCIS_BASE_DATA_PATH:/storage/owncloud.| | STORAGE_USERS_OWNCLOUDSQL_SHARE_FOLDER | string | /Shares | Name of the folder jailing all shares.| | STORAGE_USERS_OWNCLOUDSQL_LAYOUT | string | {{.Username}} | Path layout to use to navigate into a users folder in an owncloud data directory|