diff --git a/build-images.sh b/build-images.sh index 33d4a235..6402e6bb 100644 --- a/build-images.sh +++ b/build-images.sh @@ -14,7 +14,7 @@ if ! buildah inspect --type container "${container}" &>/dev/null; then buildah run "${container}" -- bash <<'EOF' set -e apt-get update -apt-get -y install samba winbind krb5-user iputils-ping bzip2 ldb-tools chrony dnsutils acl smbclient libnss-winbind rsync +apt-get -y install samba winbind krb5-user iputils-ping bzip2 ldb-tools chrony dnsutils acl smbclient libnss-winbind rsync plocate apt-get clean find /var/lib/apt/lists/ -type f -delete EOF @@ -84,6 +84,7 @@ buildah add "${container}" ns8-user-manager-${user_manager_version}.tar.gz /imag buildah add "${container}" ui/dist /ui buildah config \ --label="org.nethserver.max-per-node=1" \ + --label="org.nethserver.min-core=3.3.0-0" \ --label "org.nethserver.images=ghcr.io/nethserver/samba-dc:${IMAGETAG:-latest}" \ --label 'org.nethserver.authorizations=node:fwadm ldapproxy@node:accountprovider cluster:accountprovider traefik@node:routeadm' \ --label="org.nethserver.tcp-ports-demand=1" \ diff --git a/imageroot/actions/add-share/validate-input.json b/imageroot/actions/add-share/validate-input.json index d592d222..8c415116 100644 --- a/imageroot/actions/add-share/validate-input.json +++ b/imageroot/actions/add-share/validate-input.json @@ -22,8 +22,8 @@ "properties": { "name": { "type": "string", - "description": "The name of the share and of the underlying directory", - "minLength": 1 + "pattern": "^[^/\\\\:><\"|?*]+$", + "description": "The name of the share and of the underlying directory. Ref https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions" }, "description": { "type": "string", diff --git a/imageroot/actions/list-shares/50list_shares b/imageroot/actions/list-shares/50list_shares index 92bd75ad..609a1da8 100755 --- a/imageroot/actions/list-shares/50list_shares +++ b/imageroot/actions/list-shares/50list_shares @@ -70,9 +70,12 @@ list_shares = { "shares": [] } -ocfg = configparser.ConfigParser() +ocfg = configparser.ConfigParser(delimiters=("=")) with subprocess.Popen(podman_exec + ["net", "conf", "list"], stdout=subprocess.PIPE, text=True) as hconf: - ocfg.read_file(hconf.stdout, 'samba-registry-conf') + try: + ocfg.read_file(hconf.stdout, 'samba-registry-conf') + except Exception as ex: + print(agent.SD_ERR + "Share configuration parse error", ex, file=sys.stderr) psharenames = subprocess.run(podman_exec + ["net", "conf", "listshares"], stdout=subprocess.PIPE, text=True) for share_name in filter(None, psharenames.stdout.split("\n")): diff --git a/imageroot/actions/restore-backup-content/50restore_backup_content b/imageroot/actions/restore-backup-content/50restore_backup_content new file mode 100755 index 00000000..61f2c6d5 --- /dev/null +++ b/imageroot/actions/restore-backup-content/50restore_backup_content @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import json +import sys +import os +import subprocess +import agent +import hashlib + +request = json.load(sys.stdin) + +content_basename = os.path.basename(request['content']) +content_path = os.path.dirname(request['content']) +destroot = request.get("destroot", "Restored folder") + +pre_script = """ +set -e +share="${1:?share param missing}" +destroot="${2:?destroot param missing}" +content_basename="${3:?content_basename param is missing}" +# SDDL ACL. World read-only permissions. SYstem and Builtin Admins (BA) have full access: +destroot_acl='O:LAG:DUD:AI(A;OICIID;0x001f01ff;;;SY)(A;OICIID;0x001f01ff;;;BA)(A;OICIID;0x001301bf;;;WD)' +cd "${SAMBA_SHARES_DIR:?}/${share}" +if [ ! -e "${destroot}" ]; then + mkdir -vp "${destroot}" + chmod -c u+rwx,g+srwx,a+rx "${destroot}" + chown -c root:users "${destroot}" + samba-tool ntacl set "${destroot_acl}" "${destroot}" +fi +# Drop any existing content +rm -rf "${destroot}/${content_basename}" +""" +pre_cmd = ['podman', 'exec', '-i', 'samba-dc', 'bash', '-s', request['share'], destroot, content_basename] +subprocess.run(pre_cmd, input=pre_script, stdout=sys.stderr, text=True).check_returncode() + +podman_args = ["--workdir=/srv"] + agent.agent.get_state_volume_args() +restic_args = [ + "restore", + "--json", + f"{request['snapshot']}:volumes/shares/{request['share']}/{content_path}", + f"--include={content_basename}", + f"--include={content_basename}/**", + f"--target=volumes/shares/{request['share']}/{destroot}" +] + +# Prepare progress callback function that captures non-progress messages too: +last_restic_message = {} +def build_restore_progress_callback(): + restore_progress = agent.get_progress_callback(2, 97) + def fprog(omessage): + global last_restic_message + last_restic_message = omessage + if omessage['message_type'] == 'status': + fpercent = float(omessage['percent_done']) + restore_progress(int(fpercent * 100)) + return fprog +# Run the restic restore command capturing the progress status: +prestore = agent.run_restic(agent.redis_connect(), request["destination"], request["repopath"], podman_args, restic_args, progress_callback=build_restore_progress_callback()) +if prestore.returncode != 0: + print(agent.SD_ERR + f"Restic restore command failed with exit code {prestore.returncode}.", file=sys.stderr) + sys.exit(1) + +json.dump({ + "request": request, + "last_restic_message": last_restic_message, +}, fp=sys.stdout) diff --git a/imageroot/actions/restore-backup-content/validate-input.json b/imageroot/actions/restore-backup-content/validate-input.json new file mode 100644 index 00000000..714a50a6 --- /dev/null +++ b/imageroot/actions/restore-backup-content/validate-input.json @@ -0,0 +1,51 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "restore-backup-content input", + "$id": "http://schema.nethserver.org/mail/restore-backup-content-input.json", + "description": "Extract content from a backup snapshot", + "examples": [ + { + "snapshot": "b9ae143be9a5cf86fccff4bd7907296a3feb9f904457e3d521f215c5445cdac7", + "destination": "86d1a8ac-ef89-557a-8e19-8582ab86b7c4", + "repopath": "samba/8efb6625-e70f-4a5f-9cb5-2836096d5054", + "share": "pub", + "content": "Clienti/StudioV" + } + ], + "type": "object", + "required": [ + "destination", + "repopath", + "snapshot", + "share", + "content" + ], + "properties": { + "destination": { + "type": "string", + "description": "The UUID of the backup destination where the Restic repository resides." + }, + "repopath": { + "type": "string", + "description": "Restic repository path, under the backup destination" + }, + "snapshot": { + "type": "string", + "description": "Restic snapshot ID to restore" + }, + "share": { + "type": "string", + "pattern": "^[^/\\\\:><\"|?*]+$", + "description": "Seek the paths of this Samba share" + }, + "destroot": { + "type": "string", + "pattern": "^[^/\\\\:><\"|?*]+$", + "description": "Name of a share root-level directory where content is restored. Existing content is removed before restoring the backup." + }, + "content": { + "type": "string", + "description": "Content path to restore" + } + } +} diff --git a/imageroot/actions/restore-backup-content/validate-output.json b/imageroot/actions/restore-backup-content/validate-output.json new file mode 100644 index 00000000..59e43007 --- /dev/null +++ b/imageroot/actions/restore-backup-content/validate-output.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "restore-backup-content output", + "$id": "http://schema.nethserver.org/mail/restore-backup-content-output.json", + "description": "Extract content from a backup snapshot", + "examples": [ + { + "request": { + "content": "Clienti/StudioV", + "destination": "86d1a8ac-ef89-557a-8e19-8582ab86b7c4", + "repopath": "samba/8efb6625-e70f-4a5f-9cb5-2836096d5054", + "share": "pub", + "snapshot": "b9ae143be9a5cf86fccff4bd7907296a3feb9f904457e3d521f215c5445cdac7" + }, + "last_restic_message": { + "message_type": "summary", + "seconds_elapsed": 14, + "total_files": 2165, + "files_restored": 2165, + "total_bytes": 717166163, + "bytes_restored": 717166163 + } + } + ], + "type": "object", + "properties": { + "request": { + "type": "object", + "description": "Original request object" + }, + "last_restic_message": { + "type": "object", + "description": "Last JSON message from Restic restore" + } + } +} diff --git a/imageroot/actions/seek-snapshot-contents/50seek_snapshot_contents b/imageroot/actions/seek-snapshot-contents/50seek_snapshot_contents new file mode 100755 index 00000000..ccfbe25e --- /dev/null +++ b/imageroot/actions/seek-snapshot-contents/50seek_snapshot_contents @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import json +import sys +import os +import subprocess +import agent +import hashlib + +request = json.load(sys.stdin) + +limit_reached = False +contents_limit = request.get("limit", 10) +cache_seed = "%s-%s" % (request['destination'], request['snapshot']) +plocate_cache = hashlib.md5(cache_seed.encode(), usedforsecurity=False).hexdigest() + +podman_exec = ["podman", "exec", "samba-dc"] +pcheck = agent.run_helper(*(podman_exec + ["net", "conf", "showshare", request["share"]]), stdout=subprocess.DEVNULL) +if pcheck.returncode != 0: + agent.set_status('validation-failed') + json.dump([{'field':'share', 'parameter':'share','value': request['share'], 'error':'share_not_found'}], fp=sys.stdout) + sys.exit(2) + +def locate_share_content(): + global limit_reached, contents, contents_limit + # Search share paths matching the query: + plocate_cmd = ['podman', 'exec', 'samba-dc', 'locate-share-content', '-d', f"/var/lib/samba/plocate/{plocate_cache}", '-s', request['share'], "-q", request.get('query', "")] + with subprocess.Popen(plocate_cmd, stdout=subprocess.PIPE, stderr=sys.stderr, text=True, errors='replace') as vproc: + contents = [] + while True: + line = vproc.stdout.readline() + if not line: + break + content = line.rstrip() + if not content in contents: + contents.append(content) + if len(contents) >= contents_limit: + limit_reached = True + break + return vproc.wait(timeout=1) + +def purge_plocate_cache(): + # Remove cache dirs older than 8 hours + purge_script='echo Removing old plocate databases ; find /var/lib/samba/plocate/ -mindepth 1 -maxdepth 1 -type d -cmin +480 -print0 | xargs -r -0 -- rm -rvf' + agent.run_helper('podman', 'exec', 'samba-dc', 'bash', '-c', purge_script) + +if locate_share_content() == 3: + print(agent.SD_INFO + "DB is not cached, fetch it from the backup snapshot", file=sys.stderr) + purge_plocate_cache() + # Cache is missing. Extract the .plocate files from the snapshot and + # store them under a temporary cache directory: + podman_args = ["--workdir=/srv"] + agent.agent.get_state_volume_args() + restic_args = [ + "restore", + f"{request['snapshot']}:volumes/data/backup", + "--include=*.plocate", + f"--target=/srv/volumes/data/plocate/{plocate_cache}" + ] + agent.run_restic(agent.redis_connect(), request["destination"], request["repopath"], podman_args, restic_args, stdout=sys.stderr).check_returncode() + # Repeat the search + locate2_returncode = locate_share_content() + if not limit_reached and locate2_returncode != 0: + print(agent.SD_ERR + f"locate-share-content failed with exit code {locate2_returncode}.", file=sys.stderr) + sys.exit(1) + +json.dump({ + "request": request, + "contents": contents, + "limit_reached": limit_reached, +}, fp=sys.stdout) diff --git a/imageroot/actions/seek-snapshot-contents/validate-input.json b/imageroot/actions/seek-snapshot-contents/validate-input.json new file mode 100644 index 00000000..0737f69d --- /dev/null +++ b/imageroot/actions/seek-snapshot-contents/validate-input.json @@ -0,0 +1,56 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "seek-snapshot-contents input", + "$id": "http://schema.nethserver.org/mail/seek-snapshot-contents-input.json", + "description": "Locate a file or directory in a backup snapshot", + "examples": [ + { + "snapshot": "b9ae143be9a5cf86fccff4bd7907296a3feb9f904457e3d521f215c5445cdac7", + "destination": "86d1a8ac-ef89-557a-8e19-8582ab86b7c4", + "repopath": "samba/8efb6625-e70f-4a5f-9cb5-2836096d5054", + "share": "Complex Shar€ name" + }, + { + "snapshot": "b9ae143be9a5cf86fccff4bd7907296a3feb9f904457e3d521f215c5445cdac7", + "destination": "86d1a8ac-ef89-557a-8e19-8582ab86b7c4", + "repopath": "samba/8efb6625-e70f-4a5f-9cb5-2836096d5054", + "share": "Complex Shar€ name", + "limit": 50, + "query": "MYFILE.TXT" + } + ], + "type": "object", + "required": [ + "destination", + "repopath", + "snapshot", + "share" + ], + "properties": { + "destination": { + "type": "string", + "description": "The UUID of the backup destination where the Restic repository resides." + }, + "repopath": { + "type": "string", + "description": "Restic repository path, under the backup destination" + }, + "snapshot": { + "type": "string", + "description": "Restic snapshot ID to restore" + }, + "share": { + "type": "string", + "pattern": "^[^/\\\\:><\"|?*]+$", + "description": "Seek the paths of this Samba share" + }, + "query": { + "type": "string", + "description": "Seek matching paths" + }, + "limit": { + "type": "integer", + "description": "Limit the number of returned contents" + } + } +} \ No newline at end of file diff --git a/imageroot/actions/seek-snapshot-contents/validate-output.json b/imageroot/actions/seek-snapshot-contents/validate-output.json new file mode 100644 index 00000000..87380a4c --- /dev/null +++ b/imageroot/actions/seek-snapshot-contents/validate-output.json @@ -0,0 +1,46 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "seek-snapshot-content output", + "$id": "http://schema.nethserver.org/mail/seek-snapshot-content-output.json", + "description": "Locate a file or directory in a backup snapshot", + "examples": [ + { + "request": { + "destination": "14030a59-a4e6-54cc-b8ea-cd5f97fe44c8", + "repopath": "mail/4372a5d5-0886-45d3-82e7-68d913716a4c", + "snapshot": "latest", + "share": "myshare", + "query": "*.php", + "limit": 100 + }, + "contents": [ + "dir1/file001.php", + "dir1/file002.php", + "Project/NethServer/Main.php" + ], + "limit_reached": false + } + ], + "type": "object", + "required": [ + "contents", + "limit_reached" + ], + "properties": { + "contents": { + "type": "array", + "description": "List of absolute share content paths", + "items": { + "type": "string" + } + }, + "limit_reached": { + "type": "boolean", + "description": "If true, the query matches more contents than the returned items" + }, + "request": { + "type": "object", + "title": "Original request object" + } + } +} diff --git a/imageroot/bin/module-dump-state b/imageroot/bin/module-dump-state index b03305a4..3c961d7d 100755 --- a/imageroot/bin/module-dump-state +++ b/imageroot/bin/module-dump-state @@ -27,4 +27,9 @@ rm -rvf backup samba-tool domain backup offline --targetdir=backup cd backup mv -v samba-backup-*.tar.bz2 samba-backup.tar.bz2 +echo "Adding indexes of share contents:" +while read -r share ; do + updatedb -o "${share}.plocate" -U "${SAMBA_SHARES_DIR}/${share}" + ls -s "${share}.plocate" +done < <(net conf listshares) EOF diff --git a/samba-dc/usr/local/sbin/locate-share-content b/samba-dc/usr/local/sbin/locate-share-content new file mode 100755 index 00000000..11379ac4 --- /dev/null +++ b/samba-dc/usr/local/sbin/locate-share-content @@ -0,0 +1,76 @@ +#!/bin/bash + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +set -e +set -o pipefail + +usage_error () +{ + exec 1>&2 + [ -n "$1" ] && printf "Error: %s\n" "$1" + printf 'Usage: %s -d DATABASEDIR -s SHARE -q QUERY\n' "$0" + printf ' OPTIONS\n' + printf ' -d DATABASEDIR base directory of plocate databases\n' + printf ' -s SHARE name of the share to search\n' + printf ' -q QUERY query string to search\n' + printf ' -h Print this help\n' + exit 2 +} + +query_regex () +{ + # Ignore non-alphanumeric chars + querymod=$(tr -c -d '[:alnum:]' <<<"$1") + # Build a wide-match regexp with remaining chars + for (( i=0 ; i<${#querymod} ; i++)); do + if (( i == 0 )); then + echo -n "${querymod:${i}:1}" + else + echo -n ".*${querymod:${i}:1}" + fi + done + # The resulting regexp is like "q.*u.*e.*r.*y" + echo +} + +# Parse command-line arguments +while getopts "hd:q:s:" optname ; do + case ${optname} in + q) query="${OPTARG}" ;; + d) databasedir="${OPTARG}" ;; + s) share="${OPTARG}" ;; + h) usage_error "${@}" ;; + *) usage_error "${@}" ;; + esac +done +shift $((OPTIND - 1)) + +[ -z "${databasedir}" ] && usage_error "${@}" +[ -z "${query+x}" ] && usage_error "${@}" + +trap 'rm -f ${basename_results} ${wholename_results} ${regexp_results}' EXIT + +database="${databasedir}/${share}.plocate" +if [ ! -f "${database}" ]; then + echo "Missing plocate database file: ${database}" 1>&2 + exit 3 +fi + +basename_results=$(mktemp) +plocate -i -b -d "${database}" "${query}" >"${basename_results}" & + +if [ -n "${query}" ]; then + wholename_results=$(mktemp) + regexp_results=$(mktemp) + plocate -i -w -d "${database}" "${query}" >"${wholename_results}" & + plocate -i -r -d "${database}" "$(query_regex "${query}")" >"${regexp_results}" & +fi + +wait + +stripprefix="${SAMBA_SHARES_DIR}/${share}/" +cat "${basename_results}" "${wholename_results:-/dev/null}" "${regexp_results:-/dev/null}" | sed "s|^${stripprefix}||" diff --git a/ui/public/i18n/en/translation.json b/ui/public/i18n/en/translation.json index 8c995951..d4033525 100644 --- a/ui/public/i18n/en/translation.json +++ b/ui/public/i18n/en/translation.json @@ -36,6 +36,7 @@ "create_shared_folder": "Create shared folder", "no_shares": "No shared folder", "name": "Name", + "name_pattern": "Forbidden characters are >