Add more S3-compatible providers

core/build-image.sh

@@ -121,7 +121,7 @@ buildah rm "${container}"
 images+=("${repobase}/${reponame}")
 
 echo "Building the restic/rclone image..."
-container=$(buildah from docker.io/library/alpine:3.18.4)
+container=$(buildah from docker.io/library/alpine:3.20.3)
 reponame="restic"
 buildah add "${container}" restic/ /
 buildah run ${container} sh <<'EOF'

core/imageroot/usr/local/agent/bin/rclone-wrapper

@@ -10,6 +10,14 @@ import agent
 import os
 from urllib.parse import urlparse
 
+def extract_region_code(hostname, position, default=""):
+    """Obtain the region code for S3 backends by looking at the domain
+    parts of the endpoint FQDN."""
+    try:
+        return hostname.split('.')[position]
+    except ValueError:
+        return default
+
 core_env = agent.read_envfile("/etc/nethserver/core.env")
 rclone_image = core_env["RESTIC_IMAGE"]
 
@@ -60,8 +68,22 @@ elif uscheme == "s3":
     if orepo['provider'] == 'aws':
         rclone_env['RCLONE_S3_PROVIDER'] = 'AWS'
         rclone_env['RCLONE_S3_REGION'] = orepo.get("aws_default_region", "")
-    elif orepo['provider'] == 'digitalocean':
+    elif orepo['provider'] == 'generic-s3' and 'digitalocean' in s3_endpoint:
         rclone_env['RCLONE_S3_PROVIDER'] = 'DigitalOcean'
+    elif orepo['provider'] == 'generic-s3' and 'ovh.net' in s3_endpoint:
+        rclone_env['RCLONE_S3_PROVIDER'] = 'Other'
+        rclone_env['RCLONE_S3_REGION'] = orepo.get("aws_default_region", extract_region_code(s3_endpoint, 1))
+        rclone_env['RCLONE_S3_LOCATION_CONSTRAINT'] = rclone_env['RCLONE_S3_REGION']
+    elif orepo['provider'] == 'generic-s3' and 'wasabi' in s3_endpoint:
+        rclone_env['RCLONE_S3_PROVIDER'] = 'Wasabi'
+        rclone_env['RCLONE_S3_REGION'] = orepo.get("aws_default_region", extract_region_code(s3_endpoint, 1))
+    elif orepo['provider'] == 'generic-s3' and 'synology' in s3_endpoint:
+        rclone_env['RCLONE_S3_PROVIDER'] = 'Synology'
+        rclone_env['RCLONE_S3_REGION'] = orepo.get("aws_default_region", extract_region_code(s3_endpoint, 0))
+        rclone_env['RCLONE_S3_NO_CHECK_BUCKET'] = "1"
+    else:
+        rclone_env['RCLONE_S3_PROVIDER'] = 'Other'
+
 elif uscheme == "azure":
     rclone_path = ':azureblob:' + upath.rstrip(":")
     rclone_env = {

core/imageroot/usr/local/agent/pypkg/cluster/backup.py

@@ -29,8 +29,6 @@ def get_default_backup_repository_name(provider, url, rid=""):
             name = "AWS " + url.split('/', 1)[1]
         elif provider == "backblaze":
             name = "Backblaze " + url.split(':', 1)[1]
-        elif provider == "digitalocean":
-            name = "DigitalOcean " + url.split('/', 1)[1]
     except:
         pass
 

core/imageroot/var/lib/nethserver/cluster/actions/add-backup-repository/10validate

@@ -53,6 +53,7 @@ renv['url'] = request['url'] # add url
 renv['provider'] = request['provider'] # add provider
 pvalidate = subprocess.run(['rclone-wrapper', '0', '--low-level-retries', '1', '--contimeout', '10s', 'size', '-q', '--json', 'REMOTE_PATH'], text=True, capture_output=True, env=renv)
 if pvalidate.returncode != 0:
+    print(agent.SD_DEBUG, "The rclone-wrapper probe command has failed:", pvalidate.stderr, file=sys.stderr)
     errors.append({'field':'parameters','parameter':'parameters','value':pvalidate.stderr,'error':'backup_repository_not_accessible'})
 
 if errors:

core/imageroot/var/lib/nethserver/cluster/actions/add-backup-repository/validate-input.json

@@ -25,16 +25,6 @@
                 "aws_secret_access_key": "edfjksof798r7fsdfiougvf7df"
             }
         },
-        {
-            "name": "repository 3",
-            "provider": "digitalocean",
-            "password": "",
-            "url": "s3:ams3.digitaloceanspaces.com/mybucket/mybackup",
-            "parameters": {
-                "aws_access_key_id": "XJEMERNGRIWGN",
-                "aws_secret_access_key": "897wergjkegher987geriugheruiger789"
-            }
-        },
         {
             "name": "repository 4",
             "provider": "azure",
@@ -120,9 +110,12 @@
                 {
                     "title": "S3-based provider schema",
                     "properties": {
+                        "url": {
+                            "pattern": "/"
+                        },
                         "provider": {
                             "title": "S3 providers",
-                            "enum": ["aws", "digitalocean", "generic-s3"]
+                            "enum": ["aws", "generic-s3"]
                         },
                         "parameters": {
                             "$ref": "#/$defs/s3_parameters"

core/imageroot/var/lib/nethserver/cluster/actions/alter-backup-repository/validate-input.json

@@ -72,7 +72,6 @@
                             "title": "S3 providers",
                             "enum": [
                                 "aws",
-                                "digitalocean",
                                 "generic-s3"
                             ]
                         },

core/imageroot/var/lib/nethserver/cluster/actions/read-backup-repositories/50read

@@ -46,8 +46,13 @@ for krepo in rdb.scan_iter('cluster/backup_repository/*'):
     ]
     orepo = rdb.hgetall(krepo)
     repositories[repo_uuid] = orepo
-    proot = subprocess.Popen(rclone_lsjson_cmd, **popen_args)
-    for oroot in json.load(proot.stdout):
+    try:
+        proot = subprocess.Popen(rclone_lsjson_cmd, **popen_args)
+        lsjson_data = json.load(proot.stdout)
+    except Exception as ex:
+        lsjson_data = []
+        print(agent.SD_WARNING, 'Failed invocation of', *rclone_lsjson_cmd, ':', ex, file=sys.stderr)
+    for oroot in lsjson_data:
         restic_prefix, restic_uuid, _ = oroot["Path"].split("/", 2)
         try:
             # Obtain from lsjson the repository creation timestamp

core/ui/public/i18n/ar/translation.json

@@ -322,7 +322,7 @@
         "no_instance_to_restore_description": "It looks like there is no app instance backed up on your repositories",
         "choose_app_instances_to_backup": "Choose app instances to backup.",
         "delete_backup_explanation_1": "<strong>{instanceName}</strong> won't be backed up anymore. If you change your mind, you'll have to schedule a new backup. | <strong>{numInstances} app instances</strong> won't be backed up anymore. If you change your mind, you'll have to schedule a new backup.",
-        "generic_s3": "S3 compatible provider",
+        "generic-s3": "S3 compatible provider",
         "app_instances_not_backed_up": "Apps not backed up",
         "cluster_placeholder": "Choose a node in the cluster",
         "backup_enabled": "Backup enabled",

core/ui/public/i18n/de/translation.json

@@ -896,7 +896,7 @@
         "custom_schedule_tooltip_systemd_time_documentation": "Siehe Systemd Timer Dokumentation",
         "no_instance_to_restore_description": "Es ist keine App-Instanz im Ziel gesichert",
         "delete_backup_explanation_1": "{instanceName} wird nicht mehr gesichert. Soll erneut eine Datensicherung erfolgen so muss eine neue Sicherung geplant werden. | {numInstances} app Instanzen werden nicht mehr gesichert. Soll erneut eine Datensicherung erfolgen so muss eine neue Sicherung geplant werden.",
-        "generic_s3": "S3-kompatibler Anbieter",
+        "generic-s3": "S3-kompatibler Anbieter",
         "no_backup_configured": "Keine Datensicherung konfiguriert",
         "retention": "Aufbewahrung",
         "disabled_backups_description": "Es ist {num} Datensicherung deaktiviert | Es sind {num} Datensicherungen deaktiviert",

core/ui/public/i18n/en/translation.json

@@ -421,7 +421,7 @@
         "aws": "Amazon S3",
         "smb": "Windows File Share (SMB2/3)",
         "cluster": "Local storage",
-        "generic_s3": "S3 compatible provider",
+        "generic-s3": "S3 compatible provider",
         "azure": "Microsoft Azure blob storage",
         "backblaze_short": "B2",
         "aws_short": "S3",
@@ -582,6 +582,7 @@
         "cluster_placeholder": "Choose a node in the cluster",
         "parameters_format": "Incorrect value",
         "url": "URL",
+        "url_pattern": "The S3-compatible hostname and bucket name must be separated by a slash '/'",
         "from_node_name_of_this_cluster": "From node {name} of this cluster",
         "from_node_name_of_different_cluster": "From node {name} of a different cluster",
         "from_this_cluster": "From this cluster",

core/ui/public/i18n/es/translation.json

@@ -328,7 +328,7 @@
         "backblaze": "Backblaze B2",
         "no_instance_to_restore_description": "Parece que no hay una copia de seguridad de ninguna instancia de la aplicación en tus repositorios",
         "choose_app_instances_to_backup": "Elige instancias de aplicaciones para realizar una copia de seguridad.",
-        "generic_s3": "Proveedor compatible con S3",
+        "generic-s3": "Proveedor compatible con S3",
         "cluster_placeholder": "Elige un nodo en el clúster",
         "backup_enabled": "Copia de seguridad habilitada",
         "no_backup_configured": "No hay copias de seguridad configuradas",

core/ui/public/i18n/eu/translation.json

@@ -268,7 +268,7 @@
         "no_instance_to_restore_description": "It looks like there is no app instance backed up on your repositories",
         "choose_app_instances_to_backup": "Choose app instances to backup.",
         "delete_backup_explanation_1": "<strong>{instanceName}</strong> won't be backed up anymore. If you change your mind, you'll have to schedule a new backup. | <strong>{numInstances} app instances</strong> won't be backed up anymore. If you change your mind, you'll have to schedule a new backup.",
-        "generic_s3": "S3 compatible provider",
+        "generic-s3": "S3 compatible provider",
         "cluster_placeholder": "Choose a node in the cluster",
         "backup_enabled": "Backup enabled",
         "no_backup_configured": "No backup configured",

core/ui/public/i18n/it/translation.json

@@ -688,7 +688,7 @@
         "hourly": "Ogni ora",
         "weekly": "Settimanale",
         "monthly": "Mensilmente",
-        "generic_s3": "Provider compatibile con S3",
+        "generic-s3": "Provider compatibile con S3",
         "total_size": "Dimensioni totali",
         "minute": "Minuto",
         "schedule_action": "Pianifica",

core/ui/public/i18n/pt/translation.json

@@ -529,7 +529,7 @@
         "no_instance_to_restore_description": "It looks like there is no app instance backed up on your repositories",
         "choose_app_instances_to_backup": "Choose app instances to backup.",
         "delete_backup_explanation_1": "<strong>{instanceName}</strong> won't be backed up anymore. If you change your mind, you'll have to schedule a new backup. | <strong>{numInstances} app instances</strong> won't be backed up anymore. If you change your mind, you'll have to schedule a new backup.",
-        "generic_s3": "S3 compatible provider",
+        "generic-s3": "S3 compatible provider",
         "app_instances_not_backed_up": "Apps not backed up",
         "cluster_placeholder": "Choose a node in the cluster",
         "backup_enabled": "Backup enabled",

core/ui/public/i18n/pt_BR/translation.json

@@ -942,7 +942,7 @@
         "no_instance_to_restore_description": "It looks like there is no app instance backed up on your repositories",
         "choose_app_instances_to_backup": "Choose app instances to backup.",
         "delete_backup_explanation_1": "<strong>{instanceName}</strong> won't be backed up anymore. If you change your mind, you'll have to schedule a new backup. | <strong>{numInstances} app instances</strong> won't be backed up anymore. If you change your mind, you'll have to schedule a new backup.",
-        "generic_s3": "S3 compatible provider",
+        "generic-s3": "S3 compatible provider",
         "app_instances_not_backed_up": "Apps not backed up",
         "cluster_placeholder": "Choose a node in the cluster",
         "backup_enabled": "Backup enabled",

core/ui/src/components/backup/AddRepositoryModal.vue

@@ -113,7 +113,7 @@
                       />
                     </div>
                     <h6>
-                      {{ $t("backup.generic_s3") }}
+                      {{ $t("backup.generic-s3") }}
                     </h6>
                   </div>
                 </NsTile>
@@ -240,14 +240,6 @@
               ref="url"
             >
             </cv-text-input>
-            <cv-text-input
-              :label="$t('backup.aws_access_key_id')"
-              v-model.trim="aws.aws_access_key_id"
-              :invalid-message="error.aws.aws_access_key_id"
-              :disabled="loading.addBackupRepository"
-              ref="aws_access_key_id"
-            >
-            </cv-text-input>
             <cv-text-input
               :label="$t('backup.aws_default_region')"
               v-model.trim="aws.aws_default_region"
@@ -256,6 +248,14 @@
               ref="aws_default_region"
             >
             </cv-text-input>
+            <cv-text-input
+              :label="$t('backup.aws_access_key_id')"
+              v-model.trim="aws.aws_access_key_id"
+              :invalid-message="error.aws.aws_access_key_id"
+              :disabled="loading.addBackupRepository"
+              ref="aws_access_key_id"
+            >
+            </cv-text-input>
             <cv-text-input
               :label="$t('backup.aws_secret_access_key')"
               type="password"

core/ui/src/components/backup/RepoDetailsModal.vue

@@ -51,15 +51,15 @@
       <!-- amazon s3 -->
       <template v-if="repository.provider == 'aws'">
         <div class="key-value-setting">
-          <span class="label">{{ $t("backup.aws_access_key_id") }}</span>
+          <span class="label">{{ $t("backup.aws_default_region") }}</span>
           <span class="value">{{
-            repository.parameters.aws_access_key_id
+            repository.parameters.aws_default_region
           }}</span>
         </div>
         <div class="key-value-setting">
-          <span class="label">{{ $t("backup.aws_default_region") }}</span>
+          <span class="label">{{ $t("backup.aws_access_key_id") }}</span>
           <span class="value">{{
-            repository.parameters.aws_default_region
+            repository.parameters.aws_access_key_id
           }}</span>
         </div>
         <div class="key-value-setting">