Merge pull request #769 from NethServer/issue-7157-backend

Implement determine-restore-eligibility action

Refs NethServer/dev#7157

core/imageroot/usr/local/agent/pypkg/cluster/modules.py

@@ -33,6 +33,7 @@
 import datetime
 import time
 import cluster.userdomains
+import copy
 
 _repo_testing_cache = {}
 def _repo_has_testing_flag(rdb, repo_name):
@@ -589,3 +590,67 @@ def _calc_core_update(current, latest):
                 "node_ui_name": instance["node_ui_name"],
             })
     return list(core_modules.values())
+
+def decorate_with_install_destinations(rdb, available_modules):
+    """Decorate each item of available_modules list with
+    install_destinations info. The available_modules list must be already
+    decorated with decorate_with_installed(). This procedure has no return
+    value, it modifies its argument directly."""
+    node_core_versions = get_node_core_versions(rdb)
+    install_destinations = []
+    for node_id in set(rdb.hvals("cluster/module_node")):
+        install_destinations.append({
+            "node_id": int(node_id),
+            "instances": 0,
+            "eligible": True,
+            "reject_reason": None,
+        })
+    install_destinations.sort(key=lambda n: n["node_id"])
+    for oamodule in available_modules:
+        oamodule["install_destinations"] = copy.deepcopy(install_destinations)
+        # Parse labels of the array first element
+        try:
+            max_per_node = int(oamodule["versions"][0]["labels"]["org.nethserver.max-per-node"])
+        except:
+            max_per_node = 9999
+        try:
+            min_core = semver.Version.parse(oamodule["versions"][0]["labels"]["org.nethserver.min-core"])
+        except:
+            min_core = semver.Version(0,0,0)
+        # Find reject reasons in this loop:
+        for mdest in oamodule["install_destinations"]:
+            # max-per-node label check:
+            count_instances = len(list(filter(lambda m: m["node"] == str(mdest["node_id"]), oamodule["installed"])))
+            mdest["instances"] = count_instances
+            if count_instances >= max_per_node:
+                mdest["eligible"] = False
+                mdest["reject_reason"] = {
+                    "message": "max_per_node_limit",
+                    "parameter": str(max_per_node),
+                }
+                continue
+            # min-core label check:
+            snode_id = str(mdest["node_id"])
+            if snode_id in node_core_versions and node_core_versions[snode_id] < min_core:
+                mdest["eligible"] = False
+                mdest["reject_reason"] = {
+                    "message": "min_core_requirement",
+                    "parameter": str(min_core),
+                }
+                continue
+
+def decorate_with_installed(rdb, available_modules):
+    """Decorate each item of available_modules list with an
+    "installed" attribute. This procedure has no return value, it modifies
+    its argument directly."""
+    installed = cluster.modules.list_installed(rdb, skip_core_modules = False)
+    for oamodule in available_modules:
+        oamodule["installed"] = installed.get(oamodule["source"], [])
+
+def decorate_with_updates(rdb, available_modules):
+    """Decorate each item of available_modules list with an
+    "updates" attribute. This procedure has no return value, it modifies
+    its argument directly."""
+    updates = cluster.modules.list_updates(rdb, skip_core_modules=True, with_testing_update=True)
+    for oamodule in available_modules:
+        oamodule["updates"] = list(filter(lambda mup: mup["source"] == oamodule["source"], updates))

core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/50determine_restore_eligibility

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import agent
+import sys
+import os
+import json
+import tempfile
+import cluster.modules
+
+request = json.load(sys.stdin)
+repository = request['repository']
+path = request['path']
+snapshot = request.get("snapshot", "latest")
+# Write privileges are required for Redis repository metadata caching:
+rdb = agent.redis_connect(privileged=True)
+# Write the output to a named temporary file, to parse it with the
+# existing read_envfile() function
+with tempfile.NamedTemporaryFile() as fenv:
+    agent.run_restic(rdb, repository, path, ["--workdir=/srv"], ["dump", snapshot, "state/environment"], text=True, encoding='utf-8', stdout=fenv, check=True)
+    fenv.seek(0)
+    original_environment = agent.read_envfile(fenv.name)
+module_source, _ = original_environment['IMAGE_URL'].rsplit(':', 1)
+# Reduce the list to one element, matching the original source:
+available = list(filter(lambda omod: omod["source"] == module_source, cluster.modules.list_available(rdb, skip_core_modules = False)))
+if not available:
+    agent.set_status('validation-failed')
+    json.dump([{'field':'none', 'parameter':'none','value': '', 'error':'module_not_available'}], fp=sys.stdout)
+    sys.exit(2)
+cluster.modules.decorate_with_installed(rdb, available)
+cluster.modules.decorate_with_install_destinations(rdb, available)
+json.dump({
+    "image_url": original_environment['IMAGE_URL'],
+    "install_destinations": available[0]["install_destinations"],
+}, fp=sys.stdout)

core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/validate-input.json

@@ -0,0 +1,37 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "determine-restore-eligibility input",
+    "$id": "http://schema.nethserver.org/cluster/determine-restore-eligibility-input.json",
+    "description": "Input schema of the determine-restore-eligibility action",
+    "examples": [
+        {
+            "repository": "48ce000a-79b7-5fe6-8558-177fd70c27b4",
+            "path": "dokuwiki/dokuwiki1@f5d24fcd-819c-4b1d-98ad-a1b2ebcee8cf",
+            "snapshot": "a6b8317eef"
+        }
+    ],
+    "type": "object",
+    "required": [
+        "repository",
+        "path",
+        "snapshot"
+    ],
+    "properties": {
+        "repository": {
+            "title": "Destination ID",
+            "description": "Backup destination identifier",
+            "type": "string",
+            "minLength": 1
+        },
+        "path": {
+            "title": "Backup repository path",
+            "description": "Path of the Restic backup repository in the destination",
+            "type": "string",
+            "minLength": 1
+        },
+        "snapshot": {
+            "title": "Restic snapshot ID",
+            "type": "string"
+        }
+    }
+}

core/imageroot/var/lib/nethserver/cluster/actions/determine-restore-eligibility/validate-output.json

@@ -0,0 +1,84 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "determine-restore-eligibility output",
+    "$id": "http://schema.nethserver.org/cluster/determine-restore-eligibility-output.json",
+    "description": "Output schema of the determine-restore-eligibility action",
+    "examples": [
+        {
+            "image_url": "ghcr.io/nethserver/ejabberd:1.0.1",
+            "install_destinations": [
+                {
+                    "node_id": 1,
+                    "instances": 0,
+                    "eligible": true,
+                    "reject_reason": null
+                },
+                {
+                    "node_id": 2,
+                    "instances": 1,
+                    "eligible": false,
+                    "reject_reason": {
+                        "message": "max_per_node_limit",
+                        "parameter": "1"
+                    }
+                }
+            ]
+        }
+    ],
+    "type": "object",
+    "required": [
+        "image_url",
+        "install_destinations"
+    ],
+    "properties": {
+        "image_url": {
+            "type":"string"
+        },
+        "install_destinations": {
+            "description": "Describe for each node of the cluster if the node is eligible or not to install a new module instance. If not, a reject reason is returned.",
+            "type": "array",
+            "items": {
+                "type": "object",
+                "required": [
+                    "node_id",
+                    "instances",
+                    "eligible",
+                    "reject_reason"
+                ],
+                "properties": {
+                    "node_id": {
+                        "type": "integer",
+                        "description": "Node identifier"
+                    },
+                    "instances": {
+                        "type": "integer",
+                        "description": "Number of module instances currently installed on the node"
+                    },
+                    "eligible": {
+                        "type": "boolean",
+                        "description": "True if another instance of the module can be installed on the node"
+                    },
+                    "reject_reason": {
+                        "type": [
+                            "object",
+                            "null"
+                        ],
+                        "descripton": "If it is an object, it tells why the node is not eligible to host a module instance",
+                        "properties": {
+                            "message": {
+                                "type": "string"
+                            },
+                            "parameter": {
+                                "type": "string"
+                            }
+                        },
+                        "required": [
+                            "message",
+                            "parameter"
+                        ]
+                    }
+                }
+            }
+        }
+    }
+}

core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read

@@ -24,77 +24,16 @@ import sys
 import json
 import agent
 import cluster.modules
-import copy
-import semver
-
-def get_module(source, modules):
-    ret = []
-    for m in modules:
-        if m["source"] == source:
-            ret.append(m)
-    return ret
 
 rdb = agent.redis_connect(privileged=True)
-installed = cluster.modules.list_installed(rdb, skip_core_modules = True)
 available = cluster.modules.list_available(rdb, skip_core_modules = True)
-updates = cluster.modules.list_updates(rdb, skip_core_modules=True, with_testing_update=True)
-node_core_versions = cluster.modules.get_node_core_versions(rdb)
-
-install_destinations = []
-for node_id in set(rdb.hvals("cluster/module_node")):
-    install_destinations.append({
-        "node_id": int(node_id),
-        "instances": 0,
-        "eligible": True,
-        "reject_reason": None,
-    })
-install_destinations.sort(key=lambda n: n["node_id"])
-
-def calculate_node_install_destinations(module):
-    global node_core_versions
-    module_destinations = copy.deepcopy(install_destinations)
-    # Parse labels of the array first element
-    try:
-        max_per_node = int(module["versions"][0]["labels"]["org.nethserver.max-per-node"])
-    except:
-        max_per_node = 9999
-    try:
-        min_core = semver.Version.parse(module["versions"][0]["labels"]["org.nethserver.min-core"])
-    except:
-        min_core = semver.Version(0,0,0)
-    # Find reject reasons in this loop:
-    for mdest in module_destinations:
-        # max-per-node label check:
-        count_instances = len(list(filter(lambda m: m["node"] == str(mdest["node_id"]), module["installed"])))
-        mdest["instances"] = count_instances
-        if count_instances >= max_per_node:
-            mdest["eligible"] = False
-            mdest["reject_reason"] = {
-                "message": "max_per_node_limit",
-                "parameter": str(max_per_node),
-            }
-            continue
-        # min-core label check:
-        snode_id = str(mdest["node_id"])
-        if snode_id in node_core_versions and node_core_versions[snode_id] < min_core:
-            mdest["eligible"] = False
-            mdest["reject_reason"] = {
-                "message": "min_core_requirement",
-                "parameter": str(min_core),
-            }
-            continue
-    return module_destinations
-
 disabled_updates_reason = cluster.modules.get_disabled_updates_reason(rdb)
 
-# Prepare variables for later use
+# Decorate with disabled_updates_reason:
 for a in available:
-    a["updates"] = []
     a["disabled_updates_reason"] = disabled_updates_reason
-    a["installed"] = []
-    if a["source"] in installed.keys():
-        a["installed"] = installed[a["source"]]
-    a["updates"] = get_module(a["source"], updates)
-    a["install_destinations"] = calculate_node_install_destinations(a)
 
+cluster.modules.decorate_with_installed(rdb, available)
+cluster.modules.decorate_with_updates(rdb, available)
+cluster.modules.decorate_with_install_destinations(rdb, available)
 json.dump(available, fp=sys.stdout)