Merge pull request #58 from NethServer/feat-7072

Selective restore of Samba File Server contents

Refs NethServer/dev#7072

build-images.sh

@@ -14,7 +14,7 @@ if ! buildah inspect --type container "${container}" &>/dev/null; then
     buildah run "${container}" -- bash <<'EOF'
 set -e
 apt-get update
-apt-get -y install samba winbind krb5-user iputils-ping bzip2 ldb-tools chrony dnsutils acl smbclient libnss-winbind rsync
+apt-get -y install samba winbind krb5-user iputils-ping bzip2 ldb-tools chrony dnsutils acl smbclient libnss-winbind rsync plocate
 apt-get clean
 find /var/lib/apt/lists/ -type f -delete
 EOF
@@ -84,6 +84,7 @@ buildah add "${container}" ns8-user-manager-${user_manager_version}.tar.gz /imag
 buildah add "${container}" ui/dist /ui
 buildah config \
     --label="org.nethserver.max-per-node=1" \
+    --label="org.nethserver.min-core=3.3.0-0" \
     --label "org.nethserver.images=ghcr.io/nethserver/samba-dc:${IMAGETAG:-latest}" \
     --label 'org.nethserver.authorizations=node:fwadm ldapproxy@node:accountprovider cluster:accountprovider traefik@node:routeadm' \
     --label="org.nethserver.tcp-ports-demand=1" \

imageroot/actions/add-share/validate-input.json

@@ -22,8 +22,8 @@
     "properties": {
         "name": {
             "type": "string",
-            "description": "The name of the share and of the underlying directory",
-            "minLength": 1
+            "pattern": "^[^/\\\\:><\"|?*]+$",
+            "description": "The name of the share and of the underlying directory. Ref https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions"
         },
         "description": {
             "type": "string",

imageroot/actions/list-shares/50list_shares

@@ -70,9 +70,12 @@ list_shares = {
     "shares": []
 }
 
-ocfg = configparser.ConfigParser()
+ocfg = configparser.ConfigParser(delimiters=("="))
 with subprocess.Popen(podman_exec + ["net", "conf", "list"], stdout=subprocess.PIPE, text=True) as hconf:
-    ocfg.read_file(hconf.stdout, 'samba-registry-conf')
+    try:
+        ocfg.read_file(hconf.stdout, 'samba-registry-conf')
+    except Exception as ex:
+        print(agent.SD_ERR + "Share configuration parse error", ex, file=sys.stderr)
 
 psharenames = subprocess.run(podman_exec + ["net", "conf", "listshares"], stdout=subprocess.PIPE, text=True)
 for share_name in filter(None, psharenames.stdout.split("\n")):

imageroot/actions/restore-backup-content/50restore_backup_content

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import json
+import sys
+import os
+import subprocess
+import agent
+import hashlib
+
+request = json.load(sys.stdin)
+
+content_basename = os.path.basename(request['content'])
+content_path = os.path.dirname(request['content'])
+destroot = request.get("destroot", "Restored folder")
+
+pre_script = """
+set -e
+share="${1:?share param missing}"
+destroot="${2:?destroot param missing}"
+content_basename="${3:?content_basename param is missing}"
+# SDDL ACL. World read-only permissions. SYstem and Builtin Admins (BA) have full access:
+destroot_acl='O:LAG:DUD:AI(A;OICIID;0x001f01ff;;;SY)(A;OICIID;0x001f01ff;;;BA)(A;OICIID;0x001301bf;;;WD)'
+cd "${SAMBA_SHARES_DIR:?}/${share}"
+if [ ! -e "${destroot}" ]; then
+    mkdir -vp "${destroot}"
+    chmod -c u+rwx,g+srwx,a+rx "${destroot}"
+    chown -c root:users "${destroot}"
+    samba-tool ntacl set "${destroot_acl}" "${destroot}"
+fi
+# Drop any existing content
+rm -rf "${destroot}/${content_basename}"
+"""
+pre_cmd = ['podman', 'exec', '-i', 'samba-dc', 'bash', '-s', request['share'], destroot, content_basename]
+subprocess.run(pre_cmd, input=pre_script, stdout=sys.stderr, text=True).check_returncode()
+
+podman_args = ["--workdir=/srv"] + agent.agent.get_state_volume_args()
+restic_args = [
+    "restore",
+    "--json",
+    f"{request['snapshot']}:volumes/shares/{request['share']}/{content_path}",
+    f"--include={content_basename}",
+    f"--include={content_basename}/**",
+    f"--target=volumes/shares/{request['share']}/{destroot}"
+]
+
+# Prepare progress callback function that captures non-progress messages too:
+last_restic_message = {}
+def build_restore_progress_callback():
+    restore_progress = agent.get_progress_callback(2, 97)
+    def fprog(omessage):
+        global last_restic_message
+        last_restic_message = omessage
+        if omessage['message_type'] == 'status':
+            fpercent = float(omessage['percent_done'])
+            restore_progress(int(fpercent * 100))
+    return fprog
+# Run the restic restore command capturing the progress status:
+prestore = agent.run_restic(agent.redis_connect(), request["destination"], request["repopath"], podman_args, restic_args, progress_callback=build_restore_progress_callback())
+if prestore.returncode != 0:
+    print(agent.SD_ERR + f"Restic restore command failed with exit code {prestore.returncode}.", file=sys.stderr)
+    sys.exit(1)
+
+json.dump({
+    "request": request,
+    "last_restic_message": last_restic_message,
+}, fp=sys.stdout)

imageroot/actions/restore-backup-content/validate-input.json

@@ -0,0 +1,51 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "restore-backup-content input",
+    "$id": "http://schema.nethserver.org/mail/restore-backup-content-input.json",
+    "description": "Extract content from a backup snapshot",
+    "examples": [
+        {
+            "snapshot": "b9ae143be9a5cf86fccff4bd7907296a3feb9f904457e3d521f215c5445cdac7",
+            "destination": "86d1a8ac-ef89-557a-8e19-8582ab86b7c4",
+            "repopath": "samba/8efb6625-e70f-4a5f-9cb5-2836096d5054",
+            "share": "pub",
+            "content": "Clienti/StudioV"
+        }
+    ],
+    "type": "object",
+    "required": [
+        "destination",
+        "repopath",
+        "snapshot",
+        "share",
+        "content"
+    ],
+    "properties": {
+        "destination": {
+            "type": "string",
+            "description": "The UUID of the backup destination where the Restic repository resides."
+        },
+        "repopath": {
+            "type": "string",
+            "description": "Restic repository path, under the backup destination"
+        },
+        "snapshot": {
+            "type": "string",
+            "description": "Restic snapshot ID to restore"
+        },
+        "share": {
+            "type": "string",
+            "pattern": "^[^/\\\\:><\"|?*]+$",
+            "description": "Seek the paths of this Samba share"
+        },
+        "destroot": {
+            "type": "string",
+            "pattern": "^[^/\\\\:><\"|?*]+$",
+            "description": "Name of a share root-level directory where content is restored. Existing content is removed before restoring the backup."
+        },
+        "content": {
+            "type": "string",
+            "description": "Content path to restore"
+        }
+    }
+}

imageroot/actions/restore-backup-content/validate-output.json

@@ -0,0 +1,36 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "restore-backup-content output",
+    "$id": "http://schema.nethserver.org/mail/restore-backup-content-output.json",
+    "description": "Extract content from a backup snapshot",
+    "examples": [
+        {
+            "request": {
+                "content": "Clienti/StudioV",
+                "destination": "86d1a8ac-ef89-557a-8e19-8582ab86b7c4",
+                "repopath": "samba/8efb6625-e70f-4a5f-9cb5-2836096d5054",
+                "share": "pub",
+                "snapshot": "b9ae143be9a5cf86fccff4bd7907296a3feb9f904457e3d521f215c5445cdac7"
+            },
+            "last_restic_message": {
+                "message_type": "summary",
+                "seconds_elapsed": 14,
+                "total_files": 2165,
+                "files_restored": 2165,
+                "total_bytes": 717166163,
+                "bytes_restored": 717166163
+            }
+        }
+    ],
+    "type": "object",
+    "properties": {
+        "request": {
+            "type": "object",
+            "description": "Original request object"
+        },
+        "last_restic_message": {
+            "type": "object",
+            "description": "Last JSON message from Restic restore"
+        }
+    }
+}

imageroot/actions/seek-snapshot-contents/50seek_snapshot_contents

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import json
+import sys
+import os
+import subprocess
+import agent
+import hashlib
+
+request = json.load(sys.stdin)
+
+limit_reached = False
+contents_limit = request.get("limit", 10)
+cache_seed = "%s-%s" % (request['destination'], request['snapshot'])
+plocate_cache = hashlib.md5(cache_seed.encode(), usedforsecurity=False).hexdigest()
+
+podman_exec = ["podman", "exec", "samba-dc"]
+pcheck = agent.run_helper(*(podman_exec + ["net", "conf", "showshare", request["share"]]), stdout=subprocess.DEVNULL)
+if pcheck.returncode != 0:
+    agent.set_status('validation-failed')
+    json.dump([{'field':'share', 'parameter':'share','value': request['share'], 'error':'share_not_found'}], fp=sys.stdout)
+    sys.exit(2)
+
+def locate_share_content():
+    global limit_reached, contents, contents_limit
+    # Search share paths matching the query:
+    plocate_cmd = ['podman', 'exec', 'samba-dc', 'locate-share-content', '-d', f"/var/lib/samba/plocate/{plocate_cache}", '-s', request['share'], "-q", request.get('query', "")]
+    with subprocess.Popen(plocate_cmd, stdout=subprocess.PIPE, stderr=sys.stderr, text=True, errors='replace') as vproc:
+        contents = []
+        while True:
+            line = vproc.stdout.readline()
+            if not line:
+                break
+            content = line.rstrip()
+            if not content in contents:
+                contents.append(content)
+            if len(contents) >= contents_limit:
+                limit_reached = True
+                break
+    return vproc.wait(timeout=1)
+
+def purge_plocate_cache():
+    # Remove cache dirs older than 8 hours
+    purge_script='echo Removing old plocate databases ; find /var/lib/samba/plocate/ -mindepth 1 -maxdepth 1 -type d -cmin +480 -print0 | xargs -r -0 -- rm -rvf'
+    agent.run_helper('podman', 'exec', 'samba-dc', 'bash', '-c', purge_script)
+
+if locate_share_content() == 3:
+    print(agent.SD_INFO + "DB is not cached, fetch it from the backup snapshot", file=sys.stderr)
+    purge_plocate_cache()
+    # Cache is missing. Extract the .plocate files from the snapshot and
+    # store them under a temporary cache directory:
+    podman_args = ["--workdir=/srv"] + agent.agent.get_state_volume_args()
+    restic_args = [
+        "restore",
+        f"{request['snapshot']}:volumes/data/backup",
+        "--include=*.plocate",
+        f"--target=/srv/volumes/data/plocate/{plocate_cache}"
+    ]
+    agent.run_restic(agent.redis_connect(), request["destination"], request["repopath"], podman_args, restic_args, stdout=sys.stderr).check_returncode()
+    # Repeat the search
+    locate2_returncode = locate_share_content()
+    if not limit_reached and locate2_returncode != 0:
+        print(agent.SD_ERR + f"locate-share-content failed with exit code {locate2_returncode}.", file=sys.stderr)
+        sys.exit(1)
+
+json.dump({
+    "request": request,
+    "contents": contents,
+    "limit_reached": limit_reached,
+}, fp=sys.stdout)

imageroot/actions/seek-snapshot-contents/validate-input.json

@@ -0,0 +1,56 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "seek-snapshot-contents input",
+    "$id": "http://schema.nethserver.org/mail/seek-snapshot-contents-input.json",
+    "description": "Locate a file or directory in a backup snapshot",
+    "examples": [
+        {
+            "snapshot": "b9ae143be9a5cf86fccff4bd7907296a3feb9f904457e3d521f215c5445cdac7",
+            "destination": "86d1a8ac-ef89-557a-8e19-8582ab86b7c4",
+            "repopath": "samba/8efb6625-e70f-4a5f-9cb5-2836096d5054",
+            "share": "Complex Shar€ name"
+        },
+        {
+            "snapshot": "b9ae143be9a5cf86fccff4bd7907296a3feb9f904457e3d521f215c5445cdac7",
+            "destination": "86d1a8ac-ef89-557a-8e19-8582ab86b7c4",
+            "repopath": "samba/8efb6625-e70f-4a5f-9cb5-2836096d5054",
+            "share": "Complex Shar€ name",
+            "limit": 50,
+            "query": "MYFILE.TXT"
+        }
+    ],
+    "type": "object",
+    "required": [
+        "destination",
+        "repopath",
+        "snapshot",
+        "share"
+    ],
+    "properties": {
+        "destination": {
+            "type": "string",
+            "description": "The UUID of the backup destination where the Restic repository resides."
+        },
+        "repopath": {
+            "type": "string",
+            "description": "Restic repository path, under the backup destination"
+        },
+        "snapshot": {
+            "type": "string",
+            "description": "Restic snapshot ID to restore"
+        },
+        "share": {
+            "type": "string",
+            "pattern": "^[^/\\\\:><\"|?*]+$",
+            "description": "Seek the paths of this Samba share"
+        },
+        "query": {
+            "type": "string",
+            "description": "Seek matching paths"
+        },
+        "limit": {
+            "type": "integer",
+            "description": "Limit the number of returned contents"
+        }
+    }
+}

imageroot/actions/seek-snapshot-contents/validate-output.json

@@ -0,0 +1,46 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "seek-snapshot-content output",
+    "$id": "http://schema.nethserver.org/mail/seek-snapshot-content-output.json",
+    "description": "Locate a file or directory in a backup snapshot",
+    "examples": [
+        {
+            "request": {
+                "destination": "14030a59-a4e6-54cc-b8ea-cd5f97fe44c8",
+                "repopath": "mail/4372a5d5-0886-45d3-82e7-68d913716a4c",
+                "snapshot": "latest",
+                "share": "myshare",
+                "query": "*.php",
+                "limit": 100
+            },
+            "contents": [
+                "dir1/file001.php",
+                "dir1/file002.php",
+                "Project/NethServer/Main.php"
+            ],
+            "limit_reached": false
+        }
+    ],
+    "type": "object",
+    "required": [
+        "contents",
+        "limit_reached"
+    ],
+    "properties": {
+        "contents": {
+            "type": "array",
+            "description": "List of absolute share content paths",
+            "items": {
+                "type": "string"
+            }
+        },
+        "limit_reached": {
+            "type": "boolean",
+            "description": "If true, the query matches more contents than the returned items"
+        },
+        "request": {
+            "type": "object",
+            "title": "Original request object"
+        }
+    }
+}

imageroot/bin/module-dump-state

@@ -27,4 +27,9 @@ rm -rvf backup
 samba-tool domain backup offline --targetdir=backup
 cd backup
 mv -v samba-backup-*.tar.bz2 samba-backup.tar.bz2
+echo "Adding indexes of share contents:"
+while read -r share ; do
+    updatedb -o "${share}.plocate" -U "${SAMBA_SHARES_DIR}/${share}"
+    ls -s "${share}.plocate"
+done < <(net conf listshares)
 EOF