From 34d44383ccf54ae2f385d18bd7cb158ab42b9b2b Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 13 Dec 2024 14:30:32 +0100 Subject: [PATCH 1/2] chore: refactor list-modules - Move functions into modules.py to reuse elsewhere. - The new functions are "decorators": they modify their list argument by decorating each element with additional attributes, with no return value. --- .../usr/local/agent/pypkg/cluster/modules.py | 65 +++++++++++++++++ .../cluster/actions/list-modules/50read | 69 ++----------------- 2 files changed, 69 insertions(+), 65 deletions(-) diff --git a/core/imageroot/usr/local/agent/pypkg/cluster/modules.py b/core/imageroot/usr/local/agent/pypkg/cluster/modules.py index 2a7e94686..2eee6c9bb 100644 --- a/core/imageroot/usr/local/agent/pypkg/cluster/modules.py +++ b/core/imageroot/usr/local/agent/pypkg/cluster/modules.py @@ -33,6 +33,7 @@ import datetime import time import cluster.userdomains +import copy _repo_testing_cache = {} def _repo_has_testing_flag(rdb, repo_name): @@ -589,3 +590,67 @@ def _calc_core_update(current, latest): "node_ui_name": instance["node_ui_name"], }) return list(core_modules.values()) + +def decorate_with_install_destinations(rdb, available_modules): + """Decorate each item of available_modules list with + install_destinations info. The available_modules list must be already + decorated with decorate_with_installed(). This procedure has no return + value, it modifies its argument directly.""" + node_core_versions = get_node_core_versions(rdb) + install_destinations = [] + for node_id in set(rdb.hvals("cluster/module_node")): + install_destinations.append({ + "node_id": int(node_id), + "instances": 0, + "eligible": True, + "reject_reason": None, + }) + install_destinations.sort(key=lambda n: n["node_id"]) + for oamodule in available_modules: + oamodule["install_destinations"] = copy.deepcopy(install_destinations) + # Parse labels of the array first element + try: + max_per_node = int(oamodule["versions"][0]["labels"]["org.nethserver.max-per-node"]) + except: + max_per_node = 9999 + try: + min_core = semver.Version.parse(oamodule["versions"][0]["labels"]["org.nethserver.min-core"]) + except: + min_core = semver.Version(0,0,0) + # Find reject reasons in this loop: + for mdest in oamodule["install_destinations"]: + # max-per-node label check: + count_instances = len(list(filter(lambda m: m["node"] == str(mdest["node_id"]), oamodule["installed"]))) + mdest["instances"] = count_instances + if count_instances >= max_per_node: + mdest["eligible"] = False + mdest["reject_reason"] = { + "message": "max_per_node_limit", + "parameter": str(max_per_node), + } + continue + # min-core label check: + snode_id = str(mdest["node_id"]) + if snode_id in node_core_versions and node_core_versions[snode_id] < min_core: + mdest["eligible"] = False + mdest["reject_reason"] = { + "message": "min_core_requirement", + "parameter": str(min_core), + } + continue + +def decorate_with_installed(rdb, available_modules): + """Decorate each item of available_modules list with an + "installed" attribute. This procedure has no return value, it modifies + its argument directly.""" + installed = cluster.modules.list_installed(rdb, skip_core_modules = False) + for oamodule in available_modules: + oamodule["installed"] = installed.get(oamodule["source"], []) + +def decorate_with_updates(rdb, available_modules): + """Decorate each item of available_modules list with an + "updates" attribute. This procedure has no return value, it modifies + its argument directly.""" + updates = cluster.modules.list_updates(rdb, skip_core_modules=True, with_testing_update=True) + for oamodule in available_modules: + oamodule["updates"] = list(filter(lambda mup: mup["source"] == oamodule["source"], updates)) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read index ceb73f4b8..d99cfd5ad 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read @@ -24,77 +24,16 @@ import sys import json import agent import cluster.modules -import copy -import semver - -def get_module(source, modules): - ret = [] - for m in modules: - if m["source"] == source: - ret.append(m) - return ret rdb = agent.redis_connect(privileged=True) -installed = cluster.modules.list_installed(rdb, skip_core_modules = True) available = cluster.modules.list_available(rdb, skip_core_modules = True) -updates = cluster.modules.list_updates(rdb, skip_core_modules=True, with_testing_update=True) -node_core_versions = cluster.modules.get_node_core_versions(rdb) - -install_destinations = [] -for node_id in set(rdb.hvals("cluster/module_node")): - install_destinations.append({ - "node_id": int(node_id), - "instances": 0, - "eligible": True, - "reject_reason": None, - }) -install_destinations.sort(key=lambda n: n["node_id"]) - -def calculate_node_install_destinations(module): - global node_core_versions - module_destinations = copy.deepcopy(install_destinations) - # Parse labels of the array first element - try: - max_per_node = int(module["versions"][0]["labels"]["org.nethserver.max-per-node"]) - except: - max_per_node = 9999 - try: - min_core = semver.Version.parse(module["versions"][0]["labels"]["org.nethserver.min-core"]) - except: - min_core = semver.Version(0,0,0) - # Find reject reasons in this loop: - for mdest in module_destinations: - # max-per-node label check: - count_instances = len(list(filter(lambda m: m["node"] == str(mdest["node_id"]), module["installed"]))) - mdest["instances"] = count_instances - if count_instances >= max_per_node: - mdest["eligible"] = False - mdest["reject_reason"] = { - "message": "max_per_node_limit", - "parameter": str(max_per_node), - } - continue - # min-core label check: - snode_id = str(mdest["node_id"]) - if snode_id in node_core_versions and node_core_versions[snode_id] < min_core: - mdest["eligible"] = False - mdest["reject_reason"] = { - "message": "min_core_requirement", - "parameter": str(min_core), - } - continue - return module_destinations - disabled_updates_reason = cluster.modules.get_disabled_updates_reason(rdb) -# Prepare variables for later use +# Decorate with disabled_updates_reason: for a in available: - a["updates"] = [] a["disabled_updates_reason"] = disabled_updates_reason - a["installed"] = [] - if a["source"] in installed.keys(): - a["installed"] = installed[a["source"]] - a["updates"] = get_module(a["source"], updates) - a["install_destinations"] = calculate_node_install_destinations(a) +cluster.modules.decorate_with_installed(rdb, available) +cluster.modules.decorate_with_updates(rdb, available) +cluster.modules.decorate_with_install_destinations(rdb, available) json.dump(available, fp=sys.stdout) From 3f9e7302c6888fdd295e6500bff1312965bdb1e0 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 13 Dec 2024 14:33:27 +0100 Subject: [PATCH 2/2] feat: implement determine-restore-eligibility This new cluster action inspects the contents of the backup snapshot of a certain application and returns its IMAGE_URL and the list of nodes eligible for restore. --- .../50determine_restore_eligibility | 39 +++++++++ .../validate-input.json | 37 ++++++++ .../validate-output.json | 84 +++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100755 core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/50determine_restore_eligibility create mode 100644 core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/validate-input.json create mode 100644 core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/validate-output.json diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/50determine_restore_eligibility b/core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/50determine_restore_eligibility new file mode 100755 index 000000000..b634f18ae --- /dev/null +++ b/core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/50determine_restore_eligibility @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import agent +import sys +import os +import json +import tempfile +import cluster.modules + +request = json.load(sys.stdin) +repository = request['repository'] +path = request['path'] +snapshot = request.get("snapshot", "latest") +# Write privileges are required for Redis repository metadata caching: +rdb = agent.redis_connect(privileged=True) +# Write the output to a named temporary file, to parse it with the +# existing read_envfile() function +with tempfile.NamedTemporaryFile() as fenv: + agent.run_restic(rdb, repository, path, ["--workdir=/srv"], ["dump", snapshot, "state/environment"], text=True, encoding='utf-8', stdout=fenv, check=True) + fenv.seek(0) + original_environment = agent.read_envfile(fenv.name) +module_source, _ = original_environment['IMAGE_URL'].rsplit(':', 1) +# Reduce the list to one element, matching the original source: +available = list(filter(lambda omod: omod["source"] == module_source, cluster.modules.list_available(rdb, skip_core_modules = False))) +if not available: + agent.set_status('validation-failed') + json.dump([{'field':'none', 'parameter':'none','value': '', 'error':'module_not_available'}], fp=sys.stdout) + sys.exit(2) +cluster.modules.decorate_with_installed(rdb, available) +cluster.modules.decorate_with_install_destinations(rdb, available) +json.dump({ + "image_url": original_environment['IMAGE_URL'], + "install_destinations": available[0]["install_destinations"], +}, fp=sys.stdout) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/validate-input.json b/core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/validate-input.json new file mode 100644 index 000000000..703668fdc --- /dev/null +++ b/core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/validate-input.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "determine-restore-eligibility input", + "$id": "http://schema.nethserver.org/cluster/determine-restore-eligibility-input.json", + "description": "Input schema of the determine-restore-eligibility action", + "examples": [ + { + "repository": "48ce000a-79b7-5fe6-8558-177fd70c27b4", + "path": "dokuwiki/dokuwiki1@f5d24fcd-819c-4b1d-98ad-a1b2ebcee8cf", + "snapshot": "a6b8317eef" + } + ], + "type": "object", + "required": [ + "repository", + "path", + "snapshot" + ], + "properties": { + "repository": { + "title": "Destination ID", + "description": "Backup destination identifier", + "type": "string", + "minLength": 1 + }, + "path": { + "title": "Backup repository path", + "description": "Path of the Restic backup repository in the destination", + "type": "string", + "minLength": 1 + }, + "snapshot": { + "title": "Restic snapshot ID", + "type": "string" + } + } +} diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/validate-output.json new file mode 100644 index 000000000..68b4cfb43 --- /dev/null +++ b/core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/validate-output.json @@ -0,0 +1,84 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "determine-restore-eligibility output", + "$id": "http://schema.nethserver.org/cluster/determine-restore-eligibility-output.json", + "description": "Output schema of the determine-restore-eligibility action", + "examples": [ + { + "image_url": "ghcr.io/nethserver/ejabberd:1.0.1", + "install_destinations": [ + { + "node_id": 1, + "instances": 0, + "eligible": true, + "reject_reason": null + }, + { + "node_id": 2, + "instances": 1, + "eligible": false, + "reject_reason": { + "message": "max_per_node_limit", + "parameter": "1" + } + } + ] + } + ], + "type": "object", + "required": [ + "image_url", + "install_destinations" + ], + "properties": { + "image_url": { + "type":"string" + }, + "install_destinations": { + "description": "Describe for each node of the cluster if the node is eligible or not to install a new module instance. If not, a reject reason is returned.", + "type": "array", + "items": { + "type": "object", + "required": [ + "node_id", + "instances", + "eligible", + "reject_reason" + ], + "properties": { + "node_id": { + "type": "integer", + "description": "Node identifier" + }, + "instances": { + "type": "integer", + "description": "Number of module instances currently installed on the node" + }, + "eligible": { + "type": "boolean", + "description": "True if another instance of the module can be installed on the node" + }, + "reject_reason": { + "type": [ + "object", + "null" + ], + "descripton": "If it is an object, it tells why the node is not eligible to host a module instance", + "properties": { + "message": { + "type": "string" + }, + "parameter": { + "type": "string" + } + }, + "required": [ + "message", + "parameter" + ] + } + } + } + } + } +} \ No newline at end of file