From aa2f13bfb6151b196c6646c6a9f6e243e9613bc7 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 9 Feb 2024 13:06:52 +0100 Subject: [PATCH 01/26] Update k8s conn to new functionality --- IM/connectors/Kubernetes.py | 75 ++++++++++--------------------------- 1 file changed, 19 insertions(+), 56 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 488664ead..5c81e7a02 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -36,7 +36,6 @@ class KubernetesCloudConnector(CloudConnector): type = "Kubernetes" - _root_password = "Aspecial+0ne" """ Default password to set to the root in the container""" _apiVersions = ["v1", "v1beta3"] """ Supported API versions""" @@ -131,12 +130,14 @@ def concrete_system(self, radl_system, str_url, auth_data): if protocol == 'docker' and url[1]: res_system = radl_system.clone() - res_system.addFeature(Feature("virtual_system_type", "=", "docker"), conflict="other", missing="other") + res_system.addFeature(Feature("virtual_system_type", "=", "kubernetes"), conflict="other", missing="other") res_system.getFeature("cpu.count").operator = "=" res_system.getFeature("memory.size").operator = "=" - res_system.setValue('disk.0.os.credentials.username', 'root') - res_system.setValue('disk.0.os.credentials.password', self._root_password) + # Set it as it is required by the IM but in this case it is not used + username = res_system.getValue('disk.0.os.credentials.username') + if not username: + res_system.setValue('disk.0.os.credentials.username', 'username') return res_system else: @@ -189,17 +190,11 @@ def _create_volumes(self, namespace, system, pod_name, auth_data, persistent=Fal res = [] cont = 1 while (system.getValue("disk." + str(cont) + ".size") and - system.getValue("disk." + str(cont) + ".mount_path") and - system.getValue("disk." + str(cont) + ".device")): + system.getValue("disk." + str(cont) + ".mount_path")): disk_mount_path = system.getValue("disk." + str(cont) + ".mount_path") - # Use the device as volume host path to bind - disk_device = system.getValue("disk." + str(cont) + ".device") disk_size = system.getFeature("disk." + str(cont) + ".size").getValue('B') if not disk_mount_path.startswith('/'): disk_mount_path = '/' + disk_mount_path - if not disk_device.startswith('/'): - disk_device = '/' + disk_device - self.log_info("Binding a volume in %s to %s" % (disk_device, disk_mount_path)) name = "%s-%d" % (pod_name, cont) if persistent: @@ -211,11 +206,11 @@ def _create_volumes(self, namespace, system, pod_name, auth_data, persistent=Fal self.log_debug("Creating PVC: %s/%s" % (namespace, name)) success = self._create_volume_claim(claim_data, auth_data) if success: - res.append((name, disk_device, disk_size, disk_mount_path, persistent)) + res.append((name, disk_size, disk_mount_path, persistent)) else: self.log_error("Error creating PersistentVolumeClaim:" + name) else: - res.append((name, disk_device, disk_size, disk_mount_path, persistent)) + res.append((name, disk_size, disk_mount_path, persistent)) cont += 1 @@ -254,12 +249,12 @@ def _generate_service_data(self, namespace, name, outports): 'labels': {'name': name} } - ports = [{'port': 22, 'targetPort': 22, 'protocol': 'TCP', 'name': 'ssh'}] + ports = [] if outports: for outport in outports: if outport.is_range(): self.log_warn("Port range not allowed in Kubernetes connector. Ignoring.") - elif outport.get_local_port() != 22: + else: ports.append({'port': outport.get_local_port(), 'protocol': outport.get_protocol().upper(), 'targetPort': outport.get_local_port(), 'name': 'port%s' % outport.get_local_port()}) @@ -277,12 +272,12 @@ def _generate_pod_data(self, namespace, name, outports, system, volumes): # The URI has this format: docker://image_name image_name = system.getValue("disk.0.image.url")[9:] - ports = [{'containerPort': 22, 'protocol': 'TCP'}] + ports = [] if outports: for outport in outports: if outport.is_range(): self.log_warn("Port range not allowed in Kubernetes connector. Ignoring.") - elif outport.get_local_port() != 22: + else: ports.append({'containerPort': outport.get_local_port(), 'protocol': outport.get_protocol().upper()}) @@ -292,31 +287,13 @@ def _generate_pod_data(self, namespace, name, outports, system, volumes): 'namespace': namespace, 'labels': {'name': name} } - command = "yum install -y openssh-server python sudo" - command += " ; " - command += "apt-get update && apt-get install -y openssh-server python sudo" - command += " ; " - command += "apk add --no-cache openssh-server python2 sudo" - command += " ; " - command += "mkdir /var/run/sshd" - command += " ; " - command += "sed -i '/PermitRootLogin/c\\PermitRootLogin yes' /etc/ssh/sshd_config" - command += " ; " - command += "ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N ''" - command += " ; " - command += "echo 'root:" + self._root_password + "' | chpasswd" - command += " ; " - command += ("sed 's@session\\s*required\\s*pam_loginuid.so@session" - " optional pam_loginuid.so@g' -i /etc/pam.d/sshd") - command += " ; " - command += " /usr/sbin/sshd -D" containers = [{ 'name': name, 'image': image_name, - 'command': ["/bin/sh", "-c", command], - 'imagePullPolicy': 'IfNotPresent', + 'imagePullPolicy': 'Always', 'ports': ports, - 'resources': {'limits': {'cpu': cpu, 'memory': memory}} + 'resources': {'limits': {'cpu': cpu, 'memory': memory}, + 'requests': {'cpu': cpu, 'memory': memory}} }] if system.getValue("docker.privileged") == 'yes': @@ -324,26 +301,18 @@ def _generate_pod_data(self, namespace, name, outports, system, volumes): if volumes: containers[0]['volumeMounts'] = [] - for (v_name, _, _, v_mount_path, _) in volumes: + for (v_name, _, v_mount_path, _) in volumes: containers[0]['volumeMounts'].append( {'name': v_name, 'mountPath': v_mount_path}) - pod_data['spec'] = {'containers': containers, 'restartPolicy': 'Never'} + pod_data['spec'] = {'containers': containers, 'restartPolicy': 'OnFailure'} if volumes: pod_data['spec']['volumes'] = [] - for (v_name, v_device, _, _, persistent) in volumes: + for (v_name, _, _, persistent) in volumes: if persistent: pod_data['spec']['volumes'].append( {'name': v_name, 'persistentVolumeClaim': {'claimName': v_name}}) - else: - if v_device: - # Use the device as volume host path to bind - pod_data['spec']['volumes'].append( - {'name': v_name, 'hostPath:': {'path': v_device}}) - else: - pod_data['spec']['volumes'].append( - {'name': v_name, 'emptyDir:': {}}) return pod_data @@ -396,8 +365,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): default_domain=Config.DEFAULT_DOMAIN) pod_name = nodename - # Do not use the Persistent volumes yet - volumes = self._create_volumes(namespace, system, pod_name, auth_data) + volumes = self._create_volumes(namespace, system, pod_name, auth_data, True) pod_data = self._generate_pod_data(namespace, pod_name, outports, system, volumes) @@ -417,9 +385,6 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): output = json.loads(resp.text) vm.id = output["metadata"]["name"] - # Set the default user and password to access the container - vm.info.systems[0].setValue('disk.0.os.credentials.username', 'root') - vm.info.systems[0].setValue('disk.0.os.credentials.password', self._root_password) vm.info.systems[0].setValue('instance_id', str(vm.id)) vm.info.systems[0].setValue('instance_name', str(vm.id)) @@ -620,5 +585,3 @@ def alterVM(self, vm, radl, auth_data): self.log_exception( "Error connecting with Kubernetes API server") return (False, "ERROR: " + str(ex)) - - return (False, "Not supported") From 7e3a3564d98e4eb79e9946eab7124341ae1e9c5f Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 9 Feb 2024 13:54:39 +0100 Subject: [PATCH 02/26] Update tosca with new k8s functionality --- IM/tosca/Tosca.py | 89 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 2e564aea9..fa3c10daa 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -201,6 +201,30 @@ def to_radl(self, inf_info=None): cloud_id = self._get_placement_property(oscar_sys.name, "cloud_id") dep = deploy(oscar_sys.name, 1, cloud_id) radl.deploys.append(dep) + elif root_type == "tosca.nodes.Container.Application.Docker": + # if not create a system + k8s_sys, nets = self._gen_k8s_system(node, self.tosca.nodetemplates) + + current_num_instances = self._get_current_num_instances(k8s_sys.name, inf_info) + num_instances = num_instances - current_num_instances + Tosca.logger.debug("User requested %d instances of type %s and there" + " are %s" % (num_instances, k8s_sys.name, current_num_instances)) + + if num_instances < 0: + vm_ids = self._get_vm_ids_to_remove(removal_list, num_instances, inf_info, k8s_sys) + if vm_ids: + all_removal_list.extend(vm_ids) + Tosca.logger.debug("List of K8s pods to delete: %s" % vm_ids) + else: + radl.systems.append(k8s_sys) + radl.networks.extend(nets) + conf = configure(node.name, None) + radl.configures.append(conf) + level = Tosca._get_dependency_level(node) + cont_items.append(contextualize_item(node.name, conf.name, level)) + cloud_id = self._get_placement_property(k8s_sys.name, "cloud_id") + dep = deploy(k8s_sys.name, 1, cloud_id) + radl.deploys.append(dep) else: if root_type == "tosca.nodes.Compute": # Add the system RADL element @@ -2080,3 +2104,68 @@ def _get_oscar_service_json(self, node): Tosca.logger.warn("Property %s not expected. Ignoring." % prop.name) return res + + def _gen_k8s_system(self, node, nodetemplates): + """Generate the system for an K8s container.""" + res = system(node.name) + nets = [] + + artifacts = node.type_definition.get_value('artifacts', node.entity_tpl, True) + if len(artifacts) != 1: + raise Exception("Only one artifact is supported for K8s container.") + + artifact = list(artifacts.values())[0] + image = artifact.get("file", None) + if not image: + raise Exception("No image specified for K8s container.") + if "tosca.artifacts.Deployment.Image.Container.Docker" != artifact.get("type", None): + raise Exception("Only Docker images are supported for K8s container.") + + runtime = self._find_host_compute(node, nodetemplates, base_root_type="tosca.nodes.Container.Runtime") + + property_map = { + 'num_cpus': 'cpu.count', + 'mem_size': 'memory.size' + } + + # Get the properties of the runtime + for prop in runtime.get_capability('host').get_properties_objects(): + value = self._final_function_result(prop.value, node) + if value not in [None, [], {}]: + if prop.name in property_map: + res.setValue(property_map[prop.name], value) + elif prop.name == 'volumes': + cont = 1 + # volume format should be "volume_name:mount_path" + for vol in value: + vol_parts = vol.split(":") + volume = vol_parts[0] + mount_path = None + if len(vol_parts) > 1: + mount_path = vol_parts[1:] + cont += 1 + + # Find the volume BlockStorage node + for node in nodetemplates: + root_type = Tosca._get_root_parent_type(node).type + if root_type == "tosca.nodes.BlockStorage" and node.name == volume: + size = self._final_function_result(node.get_property_value('size'), node) + + if size: + res.setValue('disk.%d.size' % cont, size) + if mount_path: + res.setValue('disk.%d.mount_path' % cont, mount_path) + + elif prop.name == 'publish_ports': + # Asume that publish_ports must be published as NodePort + pub = network("%s_pub" % node.name) + pub.setValue("outbound", "yes") + pub.setValue("outports", self._format_outports(value)) + nets.append(pub) + elif prop.name == 'expose_ports': + # Asume that publish_ports must be published as ClusterIP + priv = network("%s_priv" % node.name) + priv.setValue("outports", self._format_outports(value)) + nets.append(priv) + + return res, nets From 8233c96e92ccd95722feecc5687307e3470a4f55 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 12 Feb 2024 11:00:45 +0100 Subject: [PATCH 03/26] Implment new k8s conn --- IM/connectors/Kubernetes.py | 33 +++++++-- test/unit/connectors/Kubernetes.py | 110 ++++++++++++++++++++++++----- 2 files changed, 120 insertions(+), 23 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 5c81e7a02..11eec69dd 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -255,8 +255,13 @@ def _generate_service_data(self, namespace, name, outports): if outport.is_range(): self.log_warn("Port range not allowed in Kubernetes connector. Ignoring.") else: - ports.append({'port': outport.get_local_port(), 'protocol': outport.get_protocol().upper(), - 'targetPort': outport.get_local_port(), 'name': 'port%s' % outport.get_local_port()}) + port = {'port': outport.get_local_port(), + 'protocol': outport.get_protocol().upper(), + 'targetPort': outport.get_local_port(), + 'name': 'port%s' % outport.get_local_port()} + if outport.get_remote_port(): + port['nodePort'] = outport.get_remote_port() + ports.append(port) service_data['spec'] = { 'type': 'NodePort', @@ -266,7 +271,15 @@ def _generate_service_data(self, namespace, name, outports): return service_data - def _generate_pod_data(self, namespace, name, outports, system, volumes): + @staticmethod + def _get_env_variables(radl_system): + env_vars = [] + for elem in radl_system.getValue("environment.variables", []): + parts = elem.split(":") + env_vars.append({'name': parts[0], 'value': ":".join(parts[1:])}) + return env_vars + + def _generate_pod_data(self, namespace, name, outports, system, volumes, tags): cpu = str(system.getValue('cpu.count')) memory = "%s" % system.getFeature('memory.size').getValue('B') # The URI has this format: docker://image_name @@ -287,6 +300,12 @@ def _generate_pod_data(self, namespace, name, outports, system, volumes): 'namespace': namespace, 'labels': {'name': name} } + + # Add instance tags + if tags: + for k,v in tags.items(): + pod_data['metadata']['labels'][k] = v + containers = [{ 'name': name, 'image': image_name, @@ -296,6 +315,10 @@ def _generate_pod_data(self, namespace, name, outports, system, volumes): 'requests': {'cpu': cpu, 'memory': memory}} }] + env_vars = self._get_env_variables(system) + if env_vars: + containers[0]["env"] = env_vars + if system.getValue("docker.privileged") == 'yes': containers[0]['securityContext'] = {'privileged': True} @@ -367,7 +390,9 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): volumes = self._create_volumes(namespace, system, pod_name, auth_data, True) - pod_data = self._generate_pod_data(namespace, pod_name, outports, system, volumes) + tags = self.get_instance_tags(system, auth_data, inf) + + pod_data = self._generate_pod_data(namespace, pod_name, outports, system, volumes, tags) self.log_debug("Creating POD: %s/%s" % (namespace, pod_name)) uri = self._get_api_url(auth_data, namespace, '/pods') diff --git a/test/unit/connectors/Kubernetes.py b/test/unit/connectors/Kubernetes.py index 8888cb23c..559a098ab 100755 --- a/test/unit/connectors/Kubernetes.py +++ b/test/unit/connectors/Kubernetes.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import json import sys import unittest @@ -53,16 +54,11 @@ def get_kube_cloud(): def test_10_concrete(self): radl_data = """ - network net () system test ( - cpu.arch='x86_64' and cpu.count>=1 and memory.size>=512m and - net_interface.0.connection = 'net' and - net_interface.0.dns_name = 'test' and disk.0.os.name = 'linux' and - disk.0.image.url = 'docker://someimage' and - disk.0.os.credentials.username = 'user' + disk.0.image.url = 'docker://someimage' )""" radl = radl_parse.parse_radl(radl_data) radl_system = radl.systems[0] @@ -86,7 +82,7 @@ def get_response(self, method, url, verify, headers, data): resp.text = '{"versions": "v1"}' elif url.endswith("/pods/1"): resp.status_code = 200 - resp.text = ('{"metadata": {"namespace":"namespace", "name": "name"}, "status": ' + resp.text = ('{"metadata": {"namespace":"infid", "name": "name"}, "status": ' '{"phase":"Running", "hostIP": "158.42.1.1", "podIP": "10.0.0.1"}, ' '"spec": {"volumes": [{"persistentVolumeClaim": {"claimName" : "cname"}}]}}') if url == "/api/v1/namespaces/infid": @@ -94,17 +90,19 @@ def get_response(self, method, url, verify, headers, data): elif method == "POST": if url.endswith("/pods"): resp.status_code = 201 - resp.text = '{"metadata": {"namespace":"namespace", "name": "name"}}' + resp.text = '{"metadata": {"namespace":"infid", "name": "name"}}' elif url.endswith("/services"): resp.status_code = 201 elif url.endswith("/namespaces/"): resp.status_code = 201 + elif url.endswith("/persistentvolumeclaims"): + resp.status_code = 201 elif method == "DELETE": if url.endswith("/pods/1"): resp.status_code = 200 elif url.endswith("/services/1"): resp.status_code = 200 - elif url.endswith("/namespaces/namespace"): + elif url.endswith("/namespaces/infid"): resp.status_code = 200 elif "persistentvolumeclaims" in url: resp.status_code = 200 @@ -118,21 +116,17 @@ def get_response(self, method, url, verify, headers, data): @patch('IM.InfrastructureList.InfrastructureList.save_data') def test_20_launch(self, save_data, requests): radl_data = """ - network net1 (outbound = 'yes' and outports = '8080') - network net2 () + network net (outbound = 'yes' and outports = '38080-8080') system test ( - cpu.arch='x86_64' and cpu.count>=1 and memory.size>=512m and - net_interface.0.connection = 'net1' and + net_interface.0.connection = 'net' and net_interface.0.dns_name = 'test' and - net_interface.1.connection = 'net2' and + environment.variables = ['var:some_val'] and disk.0.os.name = 'linux' and disk.0.image.url = 'docker://someimage' and - disk.0.os.credentials.username = 'user' and - disk.1.size=1GB and - disk.1.device='hdb' and - disk.1.mount_path='/mnt/path' + disk.1.size = 10G and + disk.1.mount_path = '/mnt' )""" radl = radl_parse.parse_radl(radl_data) radl.check() @@ -148,6 +142,76 @@ def test_20_launch(self, save_data, requests): res = kube_cloud.launch(inf, radl, radl, 1, auth) success, _ = res[0] self.assertTrue(success, msg="ERROR: launching a VM.") + + exp_pvc = { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": {"name": "test-1", "namespace": "infid"}, + "spec": { + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": 10737418240}}, + }, + } + self.assertEqual(requests.call_args_list[2][0][1], 'http://server.com:8080/api/v1/namespaces/infid/persistentvolumeclaims') + self.assertEqual(json.loads(requests.call_args_list[2][1]['data']), exp_pvc) + + exp_pod = { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "test", + "namespace": "infid", + "labels": {"name": "test", "IM_INFRA_ID": "infid"}, + }, + "spec": { + "containers": [ + { + "name": "test", + "image": "someimage", + "imagePullPolicy": "Always", + "ports": [{"containerPort": 8080, "protocol": "TCP"}], + "resources": { + "limits": {"cpu": "1", "memory": "536870912"}, + "requests": {"cpu": "1", "memory": "536870912"}, + }, + "env": [{"name": "var", "value": "some_val"}], + "volumeMounts": [{"name": "test-1", "mountPath": "/mnt"}], + } + ], + "restartPolicy": "OnFailure", + "volumes": [ + {"name": "test-1", "persistentVolumeClaim": {"claimName": "test-1"}} + ], + }, + } + self.assertEqual(requests.call_args_list[3][0][1], 'http://server.com:8080/api/v1/namespaces/infid/pods') + self.assertEqual(json.loads(requests.call_args_list[3][1]['data']), exp_pod) + + exp_svc = { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "test", + "namespace": "infid", + "labels": {"name": "test"}, + }, + "spec": { + "type": "NodePort", + "ports": [ + { + "port": 8080, + "protocol": "TCP", + "targetPort": 8080, + "name": "port8080", + "nodePort": 38080, + } + ], + "selector": {"name": "test"}, + }, + } + self.assertEqual(requests.call_args_list[4][0][1], 'http://server.com:8080/api/v1/namespaces/infid/services') + self.assertEqual(json.loads(requests.call_args_list[4][1]['data']), exp_svc) + self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('requests.request') @@ -231,13 +295,21 @@ def test_60_finalize(self, requests): kube_cloud = self.get_kube_cloud() inf = MagicMock() - inf.id = "namespace" + inf.id = "infid" vm = VirtualMachine(inf, "1", kube_cloud.cloud, "", "", kube_cloud, 1) requests.side_effect = self.get_response success, _ = kube_cloud.finalize(vm, True, auth) + self.assertEqual(requests.call_args_list[2][0], ('DELETE', + 'http://server.com:8080/api/v1/namespaces/infid/persistentvolumeclaims/cname')) + self.assertEqual(requests.call_args_list[3][0], ('DELETE', + 'http://server.com:8080/api/v1/namespaces/infid/pods/1')) + self.assertEqual(requests.call_args_list[4][0], ('DELETE', + 'http://server.com:8080/api/v1/namespaces/infid/services/1')) + self.assertEqual(requests.call_args_list[5][0], ('DELETE', + 'http://server.com:8080/api/v1/namespaces/infid')) self.assertTrue(success, msg="ERROR: finalizing VM info.") self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) From be32d165830a378b705b4c0bf3cc5de62608e20d Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 12 Feb 2024 11:01:13 +0100 Subject: [PATCH 04/26] Minor change --- IM/tosca/Tosca.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index fa3c10daa..4b4c33ee3 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -2123,17 +2123,14 @@ def _gen_k8s_system(self, node, nodetemplates): runtime = self._find_host_compute(node, nodetemplates, base_root_type="tosca.nodes.Container.Runtime") - property_map = { - 'num_cpus': 'cpu.count', - 'mem_size': 'memory.size' - } - # Get the properties of the runtime for prop in runtime.get_capability('host').get_properties_objects(): value = self._final_function_result(prop.value, node) if value not in [None, [], {}]: - if prop.name in property_map: - res.setValue(property_map[prop.name], value) + if prop.name == "num_cpus": + res.setValue('cpu.count', value) + elif prop.name == "mem_size": + res.setValue("memory.size", value) elif prop.name == 'volumes': cont = 1 # volume format should be "volume_name:mount_path" From 5e88894063edc0a3aa4a0a87e1bf17eaf3b87105 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 12 Feb 2024 14:02:11 +0100 Subject: [PATCH 05/26] Implment new k8s conn in TOSCA --- IM/tosca/Tosca.py | 136 +++++++++++++++++++++++---------------- test/files/tosca_k8s.yml | 46 +++++++++++++ test/unit/Tosca.py | 11 ++++ 3 files changed, 138 insertions(+), 55 deletions(-) create mode 100644 test/files/tosca_k8s.yml diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 4b4c33ee3..27989ce25 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -201,10 +201,14 @@ def to_radl(self, inf_info=None): cloud_id = self._get_placement_property(oscar_sys.name, "cloud_id") dep = deploy(oscar_sys.name, 1, cloud_id) radl.deploys.append(dep) - elif root_type == "tosca.nodes.Container.Application.Docker": + elif root_type == "tosca.nodes.Container.Application": # if not create a system k8s_sys, nets = self._gen_k8s_system(node, self.tosca.nodetemplates) + min_instances, _, default_instances, count, removal_list = self._get_scalable_properties(node) + num_instances = self._get_num_instances(count, default_instances, min_instances) + if num_instances not in [0, 1]: + raise Exception("Scalability values of %s only allows 0 or 1." % root_type) current_num_instances = self._get_current_num_instances(k8s_sys.name, inf_info) num_instances = num_instances - current_num_instances Tosca.logger.debug("User requested %d instances of type %s and there" @@ -223,7 +227,7 @@ def to_radl(self, inf_info=None): level = Tosca._get_dependency_level(node) cont_items.append(contextualize_item(node.name, conf.name, level)) cloud_id = self._get_placement_property(k8s_sys.name, "cloud_id") - dep = deploy(k8s_sys.name, 1, cloud_id) + dep = deploy(k8s_sys.name, num_instances, cloud_id) radl.deploys.append(dep) else: if root_type == "tosca.nodes.Compute": @@ -417,9 +421,11 @@ def _get_current_num_instances(sys_name, inf_info): return current_num @staticmethod - def _format_outports(ports_dict): + def _format_outports(ports): res = "" - for port in ports_dict.values(): + if isinstance(ports, dict): + ports = list(ports.values()) + for port in ports: protocol = "tcp" source_range = None remote_cidr = "" @@ -736,6 +742,18 @@ def _get_relationships_interfaces(relationships, node): return res + def _get_repository_url(self, repo): + repo_url = None + repositories = self.tosca.tpl.get('repositories') + if repositories: + for repo_name, repo_def in repositories.items(): + if repo_name == repo: + if isinstance(repo_def, dict) and 'url' in repo_def: + repo_url = (repo_def['url']).strip().rstrip("//") + else: + repo_url = repo_def.strip().rstrip("//") + return repo_url + def _get_artifact_full_uri(self, node, artifact_name): artifact_def = artifact_name artifacts = self._get_node_artifacts(node) @@ -748,13 +766,9 @@ def _get_artifact_full_uri(self, node, artifact_name): res = artifact_def['file'] if 'repository' in artifact_def: repo = artifact_def['repository'] - repositories = self.tosca.tpl.get('repositories') - - if repositories: - for repo_name, repo_def in repositories.items(): - if repo_name == repo: - repo_url = ((repo_def['url']).strip()).rstrip("//") - res = repo_url + "/" + artifact_def['file'] + repo_url = self._get_repository_url(repo) + if repo_url: + res = repo_url + "/" + artifact_def['file'] else: res = artifact_def @@ -2120,49 +2134,61 @@ def _gen_k8s_system(self, node, nodetemplates): raise Exception("No image specified for K8s container.") if "tosca.artifacts.Deployment.Image.Container.Docker" != artifact.get("type", None): raise Exception("Only Docker images are supported for K8s container.") - - runtime = self._find_host_compute(node, nodetemplates, base_root_type="tosca.nodes.Container.Runtime") - - # Get the properties of the runtime - for prop in runtime.get_capability('host').get_properties_objects(): - value = self._final_function_result(prop.value, node) - if value not in [None, [], {}]: - if prop.name == "num_cpus": - res.setValue('cpu.count', value) - elif prop.name == "mem_size": - res.setValue("memory.size", value) - elif prop.name == 'volumes': - cont = 1 - # volume format should be "volume_name:mount_path" - for vol in value: - vol_parts = vol.split(":") - volume = vol_parts[0] - mount_path = None - if len(vol_parts) > 1: - mount_path = vol_parts[1:] - cont += 1 - - # Find the volume BlockStorage node - for node in nodetemplates: - root_type = Tosca._get_root_parent_type(node).type - if root_type == "tosca.nodes.BlockStorage" and node.name == volume: - size = self._final_function_result(node.get_property_value('size'), node) - - if size: - res.setValue('disk.%d.size' % cont, size) - if mount_path: - res.setValue('disk.%d.mount_path' % cont, mount_path) - - elif prop.name == 'publish_ports': - # Asume that publish_ports must be published as NodePort - pub = network("%s_pub" % node.name) - pub.setValue("outbound", "yes") - pub.setValue("outports", self._format_outports(value)) - nets.append(pub) - elif prop.name == 'expose_ports': - # Asume that publish_ports must be published as ClusterIP - priv = network("%s_priv" % node.name) - priv.setValue("outports", self._format_outports(value)) - nets.append(priv) + repo = artifact.get("repository", None) + if repo: + repo_url = self._get_repository_url(repo) + if repo_url: + image = repo_url + "/" + image + + runtime = self._find_host_compute(node, nodetemplates, base_root_type="tosca.nodes.SoftwareComponent") + + if runtime: + # Get the properties of the runtime + for prop in runtime.get_capability('host').get_properties_objects(): + value = self._final_function_result(prop.value, node) + if value not in [None, [], {}]: + if prop.name == "num_cpus": + res.setValue('cpu.count', float(value)) + elif prop.name == "mem_size": + if not value.endswith("B"): + value += "B" + value = int(ScalarUnit_Size(value).get_num_from_scalar_unit('B')) + res.setValue("memory.size", value, 'B') + elif prop.name == 'volumes': + cont = 1 + # volume format should be "volume_name:mount_path" + for vol in value: + vol_parts = vol.split(":") + volume = vol_parts[0] + mount_path = None + if len(vol_parts) > 1: + mount_path = vol_parts[1:] + cont += 1 + + # Find the volume BlockStorage node + for node in nodetemplates: + root_type = Tosca._get_root_parent_type(node).type + if root_type == "tosca.nodes.BlockStorage" and node.name == volume: + size = self._final_function_result(node.get_property_value('size'), node) + + if size: + if not size.endswith("B"): + size += "B" + size = int(ScalarUnit_Size(size).get_num_from_scalar_unit('B')) + res.setValue('disk.%d.size' % cont, size, 'B') + if mount_path: + res.setValue('disk.%d.mount_path' % cont, mount_path) + + elif prop.name == 'publish_ports': + # Asume that publish_ports must be published as NodePort + pub = network("%s_pub" % node.name) + pub.setValue("outbound", "yes") + pub.setValue("outports", self._format_outports(value)) + nets.append(pub) + elif prop.name == 'expose_ports': + # Asume that publish_ports must be published as ClusterIP + priv = network("%s_priv" % node.name) + priv.setValue("outports", self._format_outports(value)) + nets.append(priv) return res, nets diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml new file mode 100644 index 000000000..63c3f4710 --- /dev/null +++ b/test/files/tosca_k8s.yml @@ -0,0 +1,46 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +imports: + - grycap_custom_types: https://raw.githubusercontent.com/grycap/tosca/devel/custom_types.yaml + +description: TOSCA test for K8s + +repositories: + + docker_hub: https://docker.io/ + +topology_template: + + node_templates: + + # The MYSQL container based on official MySQL image in Docker hub + mysql_container: + type: tosca.nodes.Container.Application.Docker + requirements: + - host: mysql_runtime + artifacts: + my_image: + file: "mysql:5.7" + type: tosca.artifacts.Deployment.Image.Container.Docker + repository: docker_hub + + # The properties of the runtime to host the container + mysql_runtime: + type: tosca.nodes.Container.Runtime.Docker + capabilities: + host: + properties: + num_cpus: 1 + mem_size: 2 MB + publish_ports: + - protocol: tcp + target: 33306 + source: 3306 + volumes: + - "some_vol:/some/path" + + some_vol: + type: tosca.nodes.BlockStorage + properties: + size: 10 GB + diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index fa905865b..3533c6024 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -425,6 +425,17 @@ def test_tosca_remove(self): remove_list, _ = tosca.to_radl(inf_info) self.assertEqual(remove_list, [2]) + def test_tosca_k8s(self): + """Test TOSCA RADL translation with Containers for K8s""" + tosca_data = read_file_as_string('../files/tosca_k8s.yml') + tosca = Tosca(tosca_data) + _, radl = tosca.to_radl() + radl = parse_radl(str(radl)) + print(radl) + radl.check() + + node = radl.get_system_by_name('mysql_container') + self.assertEqual(node.getValue("disk.0.image.url"), "mysql:5.7") if __name__ == "__main__": unittest.main() From 5f51e8ab9772f2e1bf84545abd2d51d2e375e54f Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 12 Feb 2024 15:53:22 +0100 Subject: [PATCH 06/26] Fix TOSCA k8s --- IM/tosca/Tosca.py | 5 +++-- test/unit/Tosca.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 27989ce25..eede86772 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -2140,6 +2140,7 @@ def _gen_k8s_system(self, node, nodetemplates): if repo_url: image = repo_url + "/" + image + res.setValue("disk.0.image.url", image) runtime = self._find_host_compute(node, nodetemplates, base_root_type="tosca.nodes.SoftwareComponent") if runtime: @@ -2162,8 +2163,7 @@ def _gen_k8s_system(self, node, nodetemplates): volume = vol_parts[0] mount_path = None if len(vol_parts) > 1: - mount_path = vol_parts[1:] - cont += 1 + mount_path = "".join(vol_parts[1:]) # Find the volume BlockStorage node for node in nodetemplates: @@ -2178,6 +2178,7 @@ def _gen_k8s_system(self, node, nodetemplates): res.setValue('disk.%d.size' % cont, size, 'B') if mount_path: res.setValue('disk.%d.mount_path' % cont, mount_path) + cont += 1 elif prop.name == 'publish_ports': # Asume that publish_ports must be published as NodePort diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index 3533c6024..418c82014 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -435,7 +435,7 @@ def test_tosca_k8s(self): radl.check() node = radl.get_system_by_name('mysql_container') - self.assertEqual(node.getValue("disk.0.image.url"), "mysql:5.7") + self.assertEqual(node.getValue("disk.0.image.url"), "https://docker.io/mysql:5.7") if __name__ == "__main__": unittest.main() From 14694b78e425bd2a144d13a7267c88954cb0e764 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 13 Feb 2024 09:23:28 +0100 Subject: [PATCH 07/26] Fix k8s --- IM/connectors/Kubernetes.py | 8 +++++--- IM/tosca/Tosca.py | 6 ++++-- test/files/tosca_k8s.yml | 4 ++-- test/unit/Tosca.py | 12 ++++++++++-- test/unit/connectors/Kubernetes.py | 12 ++---------- 5 files changed, 23 insertions(+), 19 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 11eec69dd..76af5be51 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -127,7 +127,9 @@ def get_api_version(self, auth_data): def concrete_system(self, radl_system, str_url, auth_data): url = urlparse(str_url) protocol = url[0] - if protocol == 'docker' and url[1]: + # it can use the docker protocol or the have a empty protocol and a non empty path + # docker://image:tag or image:tag + if (protocol == 'docker' and url[1]) or (protocol == '' and url[1] == '' and url[2] != ''): res_system = radl_system.clone() res_system.addFeature(Feature("virtual_system_type", "=", "kubernetes"), conflict="other", missing="other") @@ -282,8 +284,8 @@ def _get_env_variables(radl_system): def _generate_pod_data(self, namespace, name, outports, system, volumes, tags): cpu = str(system.getValue('cpu.count')) memory = "%s" % system.getFeature('memory.size').getValue('B') - # The URI has this format: docker://image_name - image_name = system.getValue("disk.0.image.url")[9:] + image_url = urlparse(system.getValue("disk.0.image.url")) + image_name = "".join(image_url[1:]) ports = [] if outports: diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index eede86772..df6015e27 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -2138,9 +2138,11 @@ def _gen_k8s_system(self, node, nodetemplates): if repo: repo_url = self._get_repository_url(repo) if repo_url: - image = repo_url + "/" + image + # Remove the protocol from the URL + repo_url_p = urlparse(repo_url) + image = "".join(repo_url_p[1:]) + "/" + image - res.setValue("disk.0.image.url", image) + res.setValue("disk.0.image.url", "docker://%s" % image) runtime = self._find_host_compute(node, nodetemplates, base_root_type="tosca.nodes.SoftwareComponent") if runtime: diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml index 63c3f4710..a2c9c7af4 100644 --- a/test/files/tosca_k8s.yml +++ b/test/files/tosca_k8s.yml @@ -7,7 +7,7 @@ description: TOSCA test for K8s repositories: - docker_hub: https://docker.io/ + docker_hub: docker.io topology_template: @@ -31,7 +31,7 @@ topology_template: host: properties: num_cpus: 1 - mem_size: 2 MB + mem_size: 2 GB publish_ports: - protocol: tcp target: 33306 diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index 418c82014..50b363ed7 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -431,11 +431,19 @@ def test_tosca_k8s(self): tosca = Tosca(tosca_data) _, radl = tosca.to_radl() radl = parse_radl(str(radl)) - print(radl) radl.check() node = radl.get_system_by_name('mysql_container') - self.assertEqual(node.getValue("disk.0.image.url"), "https://docker.io/mysql:5.7") + self.assertEqual(node.getValue("disk.0.image.url"), "docker://docker.io/mysql:5.7") + self.assertEqual(node.getValue("cpu.count"), 1.0) + self.assertEqual(node.getValue("memory.size"), 2000000000) + self.assertEqual(node.getValue("disk.1.size"), 10000000000) + self.assertEqual(node.getValue("disk.1.mount_path"), '/some/path') + net = radl.get_network_by_id('mysql_container_pub') + self.assertEqual(net.getValue("outports"), '3306/tcp-33306/tcp') + self.assertEqual(net.getValue("outbound"), 'yes') + conf = radl.get_configure_by_name('mysql_container') + self.assertEqual(conf.recipes, None) if __name__ == "__main__": unittest.main() diff --git a/test/unit/connectors/Kubernetes.py b/test/unit/connectors/Kubernetes.py index 559a098ab..28548fc9a 100755 --- a/test/unit/connectors/Kubernetes.py +++ b/test/unit/connectors/Kubernetes.py @@ -219,15 +219,11 @@ def test_30_updateVMInfo(self, requests): radl_data = """ network net (outbound = 'yes') system test ( - cpu.arch='x86_64' and cpu.count=1 and memory.size=512m and net_interface.0.connection = 'net' and net_interface.0.dns_name = 'test' and - disk.0.os.name = 'linux' and - disk.0.image.url = 'docker://someimage' and - disk.0.os.credentials.username = 'user' and - disk.0.os.credentials.password = 'pass' + disk.0.image.url = 'docker://someimage' )""" radl = radl_parse.parse_radl(radl_data) radl.check() @@ -254,15 +250,11 @@ def test_55_alter(self, requests): radl_data = """ network net () system test ( - cpu.arch='x86_64' and cpu.count=1 and memory.size=512m and net_interface.0.connection = 'net' and net_interface.0.dns_name = 'test' and - disk.0.os.name = 'linux' and - disk.0.image.url = 'one://server.com/1' and - disk.0.os.credentials.username = 'user' and - disk.0.os.credentials.password = 'pass' + disk.0.image.url = 'docker://image:tag' )""" radl = radl_parse.parse_radl(radl_data) From b959d00759d11f90daccbb09b800d47ad9d7d752 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 13 Feb 2024 10:40:19 +0100 Subject: [PATCH 08/26] Add env prop to Cont App --- IM/connectors/Kubernetes.py | 2 +- IM/tosca/Tosca.py | 10 ++++++++++ test/files/tosca_k8s.yml | 7 +++++-- test/unit/Tosca.py | 3 ++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 76af5be51..a9f575238 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -386,7 +386,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): vm = VirtualMachine(inf, None, self.cloud, radl, requested_radl, self) vm.destroy = True inf.add_vm(vm) - (nodename, _) = vm.getRequestedName(default_hostname=Config.DEFAULT_VM_NAME, + (nodename, _) = vm.getRequestedName(default_hostname="pod-#N#", default_domain=Config.DEFAULT_DOMAIN) pod_name = nodename diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index df6015e27..86f9a4424 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -2143,6 +2143,16 @@ def _gen_k8s_system(self, node, nodetemplates): image = "".join(repo_url_p[1:]) + "/" + image res.setValue("disk.0.image.url", "docker://%s" % image) + + for prop in node.get_properties_objects(): + value = self._final_function_result(prop.value, node) + if value not in [None, [], {}]: + if prop.name == "environment": + env = [] + for k, v in value.items(): + env.append("%s:%s" % (k, v)) + res.setValue('environment.variables', env) + runtime = self._find_host_compute(node, nodetemplates, base_root_type="tosca.nodes.SoftwareComponent") if runtime: diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml index a2c9c7af4..6e189cf0d 100644 --- a/test/files/tosca_k8s.yml +++ b/test/files/tosca_k8s.yml @@ -16,6 +16,9 @@ topology_template: # The MYSQL container based on official MySQL image in Docker hub mysql_container: type: tosca.nodes.Container.Application.Docker + properties: + environment: + MYSQL_ROOT_PASSWORD: my-secret requirements: - host: mysql_runtime artifacts: @@ -34,8 +37,8 @@ topology_template: mem_size: 2 GB publish_ports: - protocol: tcp - target: 33306 - source: 3306 + target: 3306 + source: 33306 volumes: - "some_vol:/some/path" diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index 50b363ed7..ffe103260 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -439,8 +439,9 @@ def test_tosca_k8s(self): self.assertEqual(node.getValue("memory.size"), 2000000000) self.assertEqual(node.getValue("disk.1.size"), 10000000000) self.assertEqual(node.getValue("disk.1.mount_path"), '/some/path') + self.assertEqual(node.getValue("environment.variables"), ['MYSQL_ROOT_PASSWORD:my-secret']) net = radl.get_network_by_id('mysql_container_pub') - self.assertEqual(net.getValue("outports"), '3306/tcp-33306/tcp') + self.assertEqual(net.getValue("outports"), '33306/tcp-3306/tcp') self.assertEqual(net.getValue("outbound"), 'yes') conf = radl.get_configure_by_name('mysql_container') self.assertEqual(conf.recipes, None) From 06dffcc597d1e81f3ab825f958c80c1ba5b84f0a Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 13 Feb 2024 11:13:59 +0100 Subject: [PATCH 09/26] Style changes --- IM/connectors/Kubernetes.py | 2 +- IM/tosca/Tosca.py | 4 ++-- test/files/tosca_k8s.yml | 2 +- test/unit/Tosca.py | 1 + test/unit/connectors/Kubernetes.py | 23 ++++++++++++++--------- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index a9f575238..12f0d7825 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -305,7 +305,7 @@ def _generate_pod_data(self, namespace, name, outports, system, volumes, tags): # Add instance tags if tags: - for k,v in tags.items(): + for k, v in tags.items(): pod_data['metadata']['labels'][k] = v containers = [{ diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 86f9a4424..d4f2aa7e4 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -212,7 +212,7 @@ def to_radl(self, inf_info=None): current_num_instances = self._get_current_num_instances(k8s_sys.name, inf_info) num_instances = num_instances - current_num_instances Tosca.logger.debug("User requested %d instances of type %s and there" - " are %s" % (num_instances, k8s_sys.name, current_num_instances)) + " are %s" % (num_instances, k8s_sys.name, current_num_instances)) if num_instances < 0: vm_ids = self._get_vm_ids_to_remove(removal_list, num_instances, inf_info, k8s_sys) @@ -2203,5 +2203,5 @@ def _gen_k8s_system(self, node, nodetemplates): priv = network("%s_priv" % node.name) priv.setValue("outports", self._format_outports(value)) nets.append(priv) - + return res, nets diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml index 6e189cf0d..3d84c4636 100644 --- a/test/files/tosca_k8s.yml +++ b/test/files/tosca_k8s.yml @@ -1,7 +1,7 @@ tosca_definitions_version: tosca_simple_yaml_1_0 imports: - - grycap_custom_types: https://raw.githubusercontent.com/grycap/tosca/devel/custom_types.yaml + - grycap_custom_types: https://raw.githubusercontent.com/grycap/tosca/main/custom_types.yaml description: TOSCA test for K8s diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index ffe103260..8c982081b 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -446,5 +446,6 @@ def test_tosca_k8s(self): conf = radl.get_configure_by_name('mysql_container') self.assertEqual(conf.recipes, None) + if __name__ == "__main__": unittest.main() diff --git a/test/unit/connectors/Kubernetes.py b/test/unit/connectors/Kubernetes.py index 28548fc9a..f6cd57f71 100755 --- a/test/unit/connectors/Kubernetes.py +++ b/test/unit/connectors/Kubernetes.py @@ -152,7 +152,8 @@ def test_20_launch(self, save_data, requests): "resources": {"requests": {"storage": 10737418240}}, }, } - self.assertEqual(requests.call_args_list[2][0][1], 'http://server.com:8080/api/v1/namespaces/infid/persistentvolumeclaims') + self.assertEqual(requests.call_args_list[2][0][1], + 'http://server.com:8080/api/v1/namespaces/infid/persistentvolumeclaims') self.assertEqual(json.loads(requests.call_args_list[2][1]['data']), exp_pvc) exp_pod = { @@ -294,14 +295,18 @@ def test_60_finalize(self, requests): success, _ = kube_cloud.finalize(vm, True, auth) - self.assertEqual(requests.call_args_list[2][0], ('DELETE', - 'http://server.com:8080/api/v1/namespaces/infid/persistentvolumeclaims/cname')) - self.assertEqual(requests.call_args_list[3][0], ('DELETE', - 'http://server.com:8080/api/v1/namespaces/infid/pods/1')) - self.assertEqual(requests.call_args_list[4][0], ('DELETE', - 'http://server.com:8080/api/v1/namespaces/infid/services/1')) - self.assertEqual(requests.call_args_list[5][0], ('DELETE', - 'http://server.com:8080/api/v1/namespaces/infid')) + self.assertEqual(requests.call_args_list[2][0], + ('DELETE', + 'http://server.com:8080/api/v1/namespaces/infid/persistentvolumeclaims/cname')) + self.assertEqual(requests.call_args_list[3][0], + ('DELETE', + 'http://server.com:8080/api/v1/namespaces/infid/pods/1')) + self.assertEqual(requests.call_args_list[4][0], + ('DELETE', + 'http://server.com:8080/api/v1/namespaces/infid/services/1')) + self.assertEqual(requests.call_args_list[5][0], + ('DELETE', + 'http://server.com:8080/api/v1/namespaces/infid')) self.assertTrue(success, msg="ERROR: finalizing VM info.") self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) From 518851b097ab33dacc6a700d12e8f8a6921e8a89 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 13 Feb 2024 13:46:28 +0100 Subject: [PATCH 10/26] Fix K8s conn --- IM/connectors/CloudConnector.py | 19 ++++++++++++++ IM/connectors/Kubernetes.py | 42 ++++++++++++++++-------------- IM/connectors/OSCAR.py | 15 +---------- test/unit/connectors/Kubernetes.py | 10 +++---- 4 files changed, 47 insertions(+), 39 deletions(-) diff --git a/IM/connectors/CloudConnector.py b/IM/connectors/CloudConnector.py index 593107f79..3b3bf1307 100644 --- a/IM/connectors/CloudConnector.py +++ b/IM/connectors/CloudConnector.py @@ -19,6 +19,7 @@ import time import yaml import uuid +import re from radl.radl import Feature, outport from IM.config import Config @@ -751,3 +752,21 @@ def manage_dns_entries(self, op, vm, auth_data, extra_args=None): self.error_messages += "Error in %s DNS entries %s.\n" % (op, str(ex)) self.log_exception("Error in %s DNS entries" % op) return False + + @staticmethod + def convert_memory_unit(memory, unit="M"): + unit_dict = {'B': 1, 'K': 1000, 'Ki': 1024, + 'M': 1000000, 'Mi': 1048576, + 'G': 1000000000, 'Gi': 1073741824, + 'T': 1000000000000, 'Ti': 1099511627776} + regex = re.compile(r'([0-9.]+)\s*([a-zA-Z]+)') + result = regex.match(str(memory)).groups() + value = float(result[0]) + orig_unit = result[1] + if len(orig_unit) > 1 and orig_unit[-1] == 'B': + orig_unit = orig_unit[:-1] + + converted = (value * unit_dict[orig_unit] / unit_dict[unit]) + if converted - int(converted) < 0.0000000000001: + converted = int(converted) + return converted diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 12f0d7825..5e500ea86 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -63,7 +63,7 @@ def create_request(self, method, url, auth_data, headers=None, body=None): headers = {} headers.update(auth_header) - if body and isinstance(body, dict): + if body and isinstance(body, (dict, list)): data = json.dumps(body) else: data = body @@ -447,6 +447,14 @@ def updateVMInfo(self, vm, auth_data): output = json.loads(output) vm.state = self.VM_STATE_MAP.get(output["status"]["phase"], VirtualMachine.UNKNOWN) + pod_limits = output['spec']['containers'][0].get('resources', {}).get('limits') + if pod_limits: + vm.info.systems[0].setValue('cpu.count', float(pod_limits['cpu'])) + memory = self.convert_memory_unit(pod_limits['memory'], "B") + vm.info.systems[0].setValue('memory.size', memory) + + vm.info.systems[0].setValue('disk.0.image.url', output['spec']['containers'][0]['image']) + # Update the network info self.setIPs(vm, output) return (True, vm) @@ -564,26 +572,23 @@ def reboot(self, vm, auth_data): def alterVM(self, vm, radl, auth_data): # This function is correctly implemented - # But kubernetes does not permit cpu to be updated yet + # But kubernetes only enable to change the image of the container system = radl.systems[0] try: pod_data = [] - cpu = vm.info.systems[0].getValue('cpu.count') - memory = vm.info.systems[0].getFeature('memory.size').getValue('B') - - new_cpu = system.getValue('cpu.count') - new_memory = system.getFeature('memory.size').getValue('B') + image_url = urlparse(vm.info.systems[0].getValue('disk.0.image.url')) + image = "".join(image_url[1:]) + new_image = system.getValue('disk.0.image.url') + if system.getValue("disk.0.image.url"): + new_image_url = urlparse(system.getValue("disk.0.image.url")) + new_image = "".join(new_image_url[1:]) changed = False - if new_cpu and new_cpu != cpu: - pod_data.append( - {"op": "replace", "path": "/spec/containers/0/resources/limits/cpu", "value": new_cpu}) - changed = True - if new_memory and new_memory != memory: + if new_image and new_image != image: pod_data.append( - {"op": "replace", "path": "/spec/containers/0/resources/limits/memory", "value": new_memory}) + {"op": "replace", "path": "/spec/containers/0/image", "value": new_image}) changed = True if not changed: @@ -598,15 +603,12 @@ def alterVM(self, vm, radl, auth_data): uri = self._get_api_url(auth_data, namespace, "/pods/" + pod_name) resp = self.create_request('PATCH', uri, auth_data, headers, pod_data) - if resp.status_code != 201: + if resp.status_code != 200: return (False, "Error updating the Pod: " + resp.text) else: - if new_cpu: - vm.info.systems[0].setValue('cpu.count', new_cpu) - if new_memory: - vm.info.systems[0].addFeature( - Feature("memory.size", "=", new_memory, 'B'), conflict="other", missing="other") - return (True, self.updateVMInfo(vm, auth_data)) + if new_image: + vm.info.systems[0].setValue('disk.0.image.url', new_image) + return (True, vm) except Exception as ex: self.log_exception( diff --git a/IM/connectors/OSCAR.py b/IM/connectors/OSCAR.py index 4a58be35a..49db9bd4c 100644 --- a/IM/connectors/OSCAR.py +++ b/IM/connectors/OSCAR.py @@ -255,25 +255,12 @@ def finalize(self, vm, last, auth_data): return True, "" - @staticmethod - def _convert_memory_unit(memory, unit="M"): - unit_dict = {'B': 1, 'K': 1000, 'Ki': 1024, - 'M': 1000000, 'Mi': 1048576, - 'G': 1000000000, 'Gi': 1073741824, - 'T': 1000000000000, 'Ti': 1099511627776} - regex = re.compile(r'([0-9.]+)\s*([a-zA-Z]+)') - result = regex.match(str(memory)).groups() - converted = (float(result[0]) * unit_dict[result[1]] / unit_dict[unit]) - if converted - int(converted) < 0.0000000000001: - converted = int(converted) - return converted - def update_system_info_from_service_info(self, system, service_info): if "cpu" in service_info and service_info["cpu"]: system.addFeature(Feature("cpu.count", "=", float(service_info["cpu"])), conflict="other", missing="other") if "memory" in service_info and service_info["memory"]: - memory = self._convert_memory_unit(service_info["memory"], "Mi") + memory = self.convert_memory_unit(service_info["memory"], "Mi") system.addFeature(Feature("memory.size", "=", memory, "M"), conflict="other", missing="other") if "script" in service_info and service_info["script"]: diff --git a/test/unit/connectors/Kubernetes.py b/test/unit/connectors/Kubernetes.py index f6cd57f71..9799c2f72 100755 --- a/test/unit/connectors/Kubernetes.py +++ b/test/unit/connectors/Kubernetes.py @@ -84,7 +84,8 @@ def get_response(self, method, url, verify, headers, data): resp.status_code = 200 resp.text = ('{"metadata": {"namespace":"infid", "name": "name"}, "status": ' '{"phase":"Running", "hostIP": "158.42.1.1", "podIP": "10.0.0.1"}, ' - '"spec": {"volumes": [{"persistentVolumeClaim": {"claimName" : "cname"}}]}}') + '"spec": {"containers": [{"image": "image:1.0"}], ' + '"volumes": [{"persistentVolumeClaim": {"claimName" : "cname"}}]}}') if url == "/api/v1/namespaces/infid": resp.status_code = 200 elif method == "POST": @@ -108,7 +109,7 @@ def get_response(self, method, url, verify, headers, data): resp.status_code = 200 elif method == "PATCH": if url.endswith("/pods/1"): - resp.status_code = 201 + resp.status_code = 200 return resp @@ -255,14 +256,13 @@ def test_55_alter(self, requests): memory.size=512m and net_interface.0.connection = 'net' and net_interface.0.dns_name = 'test' and - disk.0.image.url = 'docker://image:tag' + disk.0.image.url = 'docker://image:1.0' )""" radl = radl_parse.parse_radl(radl_data) new_radl_data = """ system test ( - cpu.count>=2 and - memory.size>=2048m + disk.0.image.url = 'docker://image:2.0' )""" new_radl = radl_parse.parse_radl(new_radl_data) From e54cafa5e01e21cc6b66c65a42179a0a3b6ad829 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 13 Feb 2024 13:50:03 +0100 Subject: [PATCH 11/26] Fix K8s docs --- doc/source/radl.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/radl.rst b/doc/source/radl.rst index a53e983d1..88075d9e6 100644 --- a/doc/source/radl.rst +++ b/doc/source/radl.rst @@ -350,7 +350,7 @@ machine. The supported features are: Set the device name, if it is disk with no source set. It specifies the device where the disk will be located in the system (hdb, hdc, etc.). Depending on the Cloud provider the meaning of this - field may change. In Docker and Kubernetes connectors the device + field may change. In Docker connector the device refers to a path to create a bind in the container, if it starts with character ``/`` or the name of a volume otherwise. @@ -358,10 +358,10 @@ machine. The supported features are: Set the mount point, if it is disk with no source set. It specifies a path to mount the device. In Docker and Kubernetes connectors this path refers to the directory in the container to - bind the host directory specified in ``device``. + mount the or the bind host directory specified in ``device``. ``disk..fstype = `` - Set the mount point, if it is disk with no source set. + Set the filesystem, if it is disk with no source set. It specifies the type of the filesystem of this disk. If specified the contextualization agent will try to format and mount this disk in the path specified in ``mount_path`` field. In case of Docker From f5c423552acf0e1cd4dfed75cf42f27e62ffc7fe Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 13 Feb 2024 13:57:41 +0100 Subject: [PATCH 12/26] Fix K8s docs --- doc/source/radl.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/radl.rst b/doc/source/radl.rst index 88075d9e6..99b6b8b86 100644 --- a/doc/source/radl.rst +++ b/doc/source/radl.rst @@ -358,7 +358,7 @@ machine. The supported features are: Set the mount point, if it is disk with no source set. It specifies a path to mount the device. In Docker and Kubernetes connectors this path refers to the directory in the container to - mount the or the bind host directory specified in ``device``. + mount a PVC or the bind host directory specified in ``device``. ``disk..fstype = `` Set the filesystem, if it is disk with no source set. From a69f10e346f450ba3d67956a972ff431686e1aae Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 14 Feb 2024 09:28:57 +0100 Subject: [PATCH 13/26] Fix ONE incorrect ip in test --- test/unit/connectors/OpenNebula.py | 2 +- test/unit/connectors/files/vm_info.xml | 2 +- test/unit/connectors/files/vm_info_off.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/connectors/OpenNebula.py b/test/unit/connectors/OpenNebula.py index e6e51b585..9c8976516 100755 --- a/test/unit/connectors/OpenNebula.py +++ b/test/unit/connectors/OpenNebula.py @@ -190,7 +190,7 @@ def test_30_updateVMInfo(self, server_proxy): server_proxy.return_value = one_server success, vm = one_cloud.updateVMInfo(vm, auth) - self.assertEqual(vm.info.systems[0].getValue("net_interface.1.ip"), "10.0.0.01") + self.assertEqual(vm.info.systems[0].getValue("net_interface.1.ip"), "10.0.0.1") self.assertEqual(vm.info.systems[0].getValue("net_interface.0.ip"), "158.42.1.1") self.assertTrue(success, msg="ERROR: updating VM info.") diff --git a/test/unit/connectors/files/vm_info.xml b/test/unit/connectors/files/vm_info.xml index b594fbbb4..f232a6308 100644 --- a/test/unit/connectors/files/vm_info.xml +++ b/test/unit/connectors/files/vm_info.xml @@ -68,7 +68,7 @@ - + diff --git a/test/unit/connectors/files/vm_info_off.xml b/test/unit/connectors/files/vm_info_off.xml index 1727c2c07..2a3af7b51 100644 --- a/test/unit/connectors/files/vm_info_off.xml +++ b/test/unit/connectors/files/vm_info_off.xml @@ -68,7 +68,7 @@ - + From 4b0ce5740e53fa68d0a2b8122cd997ca69121f7a Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 14 Feb 2024 09:29:09 +0100 Subject: [PATCH 14/26] Improve code --- IM/tosca/Tosca.py | 52 ++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index d4f2aa7e4..7d89d944d 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -2119,8 +2119,33 @@ def _get_oscar_service_json(self, node): return res + def _gen_k8s_volumes(self, node, nodetemplates, value): + volumes = [] + cont = 1 + # volume format should be "volume_name:mount_path" + for vol in value: + vol_parts = vol.split(":") + volume = vol_parts[0] + mount_path = None + if len(vol_parts) > 1: + mount_path = "".join(vol_parts[1:]) + + # Find the volume BlockStorage node + for node in nodetemplates: + root_type = Tosca._get_root_parent_type(node).type + if root_type == "tosca.nodes.BlockStorage" and node.name == volume: + size = self._final_function_result(node.get_property_value('size'), node) + + if size: + if not size.endswith("B"): + size += "B" + size = int(ScalarUnit_Size(size).get_num_from_scalar_unit('B')) + volumes.append((cont, size, mount_path)) + cont += 1 + return volumes + def _gen_k8s_system(self, node, nodetemplates): - """Generate the system for an K8s container.""" + """Get the volumes attached to an K8s container.""" res = system(node.name) nets = [] @@ -2168,30 +2193,11 @@ def _gen_k8s_system(self, node, nodetemplates): value = int(ScalarUnit_Size(value).get_num_from_scalar_unit('B')) res.setValue("memory.size", value, 'B') elif prop.name == 'volumes': - cont = 1 - # volume format should be "volume_name:mount_path" - for vol in value: - vol_parts = vol.split(":") - volume = vol_parts[0] - mount_path = None - if len(vol_parts) > 1: - mount_path = "".join(vol_parts[1:]) - - # Find the volume BlockStorage node - for node in nodetemplates: - root_type = Tosca._get_root_parent_type(node).type - if root_type == "tosca.nodes.BlockStorage" and node.name == volume: - size = self._final_function_result(node.get_property_value('size'), node) - + for num, size, mount_path in self._gen_k8s_volumes(node, nodetemplates, value): if size: - if not size.endswith("B"): - size += "B" - size = int(ScalarUnit_Size(size).get_num_from_scalar_unit('B')) - res.setValue('disk.%d.size' % cont, size, 'B') + res.setValue('disk.%d.size' % num, size, 'B') if mount_path: - res.setValue('disk.%d.mount_path' % cont, mount_path) - cont += 1 - + res.setValue('disk.%d.mount_path' % num, mount_path) elif prop.name == 'publish_ports': # Asume that publish_ports must be published as NodePort pub = network("%s_pub" % node.name) From a497a7bc2716c964f5af28b27b2fa9947f0864e7 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 14 Feb 2024 12:23:55 +0100 Subject: [PATCH 15/26] Improve code --- IM/tosca/Tosca.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 7d89d944d..4db5d267d 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -143,9 +143,8 @@ def to_radl(self, inf_info=None): if num_instances not in [0, 1]: raise Exception("Scalability values of %s only allows 0 or 1." % root_type) - oscar_host = self._find_host_compute(node, self.tosca.nodetemplates, - "tosca.nodes.SoftwareComponent") - oscar_compute = self._find_host_compute(node, self.tosca.nodetemplates) + oscar_host = self._find_host_node(node, self.tosca.nodetemplates, "tosca.nodes.SoftwareComponent") + oscar_compute = self._find_host_node(node, self.tosca.nodetemplates) # If the function has a host create a recipe if oscar_compute and oscar_host: service_json = self._get_oscar_service_json(node) @@ -259,7 +258,7 @@ def to_radl(self, inf_info=None): compute = node else: # Select the host to host this element - compute = self._find_host_compute(node, self.tosca.nodetemplates) + compute = self._find_host_node(node, self.tosca.nodetemplates) if not compute: Tosca.logger.warn( "Node %s has not compute node to host in." % node.name) @@ -473,7 +472,7 @@ def _get_node_endpoints(self, node, nodetemplates): compute = None if root_type != "tosca.nodes.Compute": # Select the host to host this element - compute = self._find_host_compute(other_node, nodetemplates) + compute = self._find_host_node(other_node, nodetemplates) if compute and compute.name == node.name: node_caps = other_node.get_capabilities() @@ -1150,7 +1149,7 @@ def _get_attribute_result(self, func, node, inf_info): # Get node if node_name == "HOST": - node = self._find_host_compute(node, self.tosca.nodetemplates) + node = self._find_host_node(node, self.tosca.nodetemplates) elif node_name == "SOURCE": node = func.context.source elif node_name == "TARGET": @@ -1206,7 +1205,7 @@ def _get_attribute_result(self, func, node, inf_info): root_type = Tosca._get_root_parent_type(node).type - host_node = self._find_host_compute(node, self.tosca.nodetemplates) + host_node = self._find_host_node(node, self.tosca.nodetemplates) if root_type == "tosca.nodes.aisprint.FaaS.Function" and host_node is None: # in case of FaaS functions without host, the node is the host host_node = node @@ -1292,8 +1291,8 @@ def _get_attribute_result(self, func, node, inf_info): # AWS Lambda function return vm.info.systems[0].get("function.api_url") else: - oscar_host = self._find_host_compute(node, self.tosca.nodetemplates, - "tosca.nodes.SoftwareComponent") + oscar_host = self._find_host_node(node, self.tosca.nodetemplates, + "tosca.nodes.SoftwareComponent") if host_node != node and oscar_host: # OSCAR function deployed in a deployed VM dns_host = self._final_function_result(oscar_host.get_property_value('dns_host'), @@ -1316,8 +1315,8 @@ def _get_attribute_result(self, func, node, inf_info): if vm.getCloudConnector().type == "Lambda": return None else: - oscar_host = self._find_host_compute(node, self.tosca.nodetemplates, - "tosca.nodes.SoftwareComponent") + oscar_host = self._find_host_node(node, self.tosca.nodetemplates, + "tosca.nodes.SoftwareComponent") if host_node != node and oscar_host: # OSCAR function deployed in a deployed VM oscar_pass = self._final_function_result(oscar_host.get_property_value('password'), @@ -1404,8 +1403,8 @@ def _get_attribute_result(self, func, node, inf_info): host_node.name)) elif attribute_name == "endpoint": if root_type == "tosca.nodes.aisprint.FaaS.Function": - oscar_host = self._find_host_compute(node, self.tosca.nodetemplates, - "tosca.nodes.SoftwareComponent") + oscar_host = self._find_host_node(node, self.tosca.nodetemplates, + "tosca.nodes.SoftwareComponent") if host_node != node and oscar_host: # OSCAR function deployed in a deployed VM dns_host = self._final_function_result(oscar_host.get_property_value('dns_host'), oscar_host) @@ -1453,7 +1452,7 @@ def _final_function_result(self, func, node, inf_info=None): # TODO: resolve function values related with run-time values as IM # or ansible variables - def _find_host_compute(self, node, nodetemplates, base_root_type="tosca.nodes.Compute"): + def _find_host_node(self, node, nodetemplates, base_root_type="tosca.nodes.Compute"): """ Select the node to host each node, using the node requirements In most of the cases the are directly specified, otherwise "node_filter" is used @@ -1471,7 +1470,7 @@ def _find_host_compute(self, node, nodetemplates, base_root_type="tosca.nodes.Co if root_type == base_root_type: return n else: - return self._find_host_compute(n, nodetemplates) + return self._find_host_node(n, nodetemplates) # There are no direct HostedOn node # check node_filter requirements @@ -1691,7 +1690,7 @@ def _add_ansible_roles(self, node, nodetemplates, system): compute = other_node else: # Select the host to host this element - compute = self._find_host_compute(other_node, nodetemplates) + compute = self._find_host_node(other_node, nodetemplates) if compute and compute.name == node.name: # Get the artifacts to see if there is a ansible galaxy role @@ -2076,7 +2075,7 @@ def _gen_oscar_system(self, node): deps = [] for r, n in node.relationships.items(): if Tosca._is_derived_from(r, [r.DEPENDSON]): - node_compute = self._find_host_compute(n, self.tosca.nodetemplates) + node_compute = self._find_host_node(n, self.tosca.nodetemplates) deps.append(node_compute.name) if deps: res.setValue('dependencies', deps) @@ -2178,7 +2177,7 @@ def _gen_k8s_system(self, node, nodetemplates): env.append("%s:%s" % (k, v)) res.setValue('environment.variables', env) - runtime = self._find_host_compute(node, nodetemplates, base_root_type="tosca.nodes.SoftwareComponent") + runtime = self._find_host_node(node, nodetemplates, base_root_type="tosca.nodes.SoftwareComponent") if runtime: # Get the properties of the runtime From 9f9de0cbef9f87a8e779acfa8109e2154c6fc457 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 14 Feb 2024 13:24:21 +0100 Subject: [PATCH 16/26] Add volume_id --- IM/connectors/Kubernetes.py | 8 +++++++- IM/tosca/Tosca.py | 9 +++++++-- test/files/tosca_k8s.yml | 2 ++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 5e500ea86..888951b7a 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -191,8 +191,10 @@ def _create_volume_claim(self, claim_data, auth_data): def _create_volumes(self, namespace, system, pod_name, auth_data, persistent=False): res = [] cont = 1 - while (system.getValue("disk." + str(cont) + ".size") and + while ((system.getValue("disk." + str(cont) + ".size") or + system.getValue("disk." + str(cont) + ".image.url")) and system.getValue("disk." + str(cont) + ".mount_path")): + volume_id = system.getValue("disk." + str(cont) + ".image.url") disk_mount_path = system.getValue("disk." + str(cont) + ".mount_path") disk_size = system.getFeature("disk." + str(cont) + ".size").getValue('B') if not disk_mount_path.startswith('/'): @@ -205,6 +207,10 @@ def _create_volumes(self, namespace, system, pod_name, auth_data, persistent=Fal claim_data['spec'] = {'accessModes': ['ReadWriteOnce'], 'resources': { 'requests': {'storage': disk_size}}} + if volume_id: + claim_data['spec']['storageClassName'] = "" + claim_data['spec']['volumeName'] = volume_id + self.log_debug("Creating PVC: %s/%s" % (namespace, name)) success = self._create_volume_claim(claim_data, auth_data) if success: diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 4db5d267d..e61859012 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -2123,6 +2123,8 @@ def _gen_k8s_volumes(self, node, nodetemplates, value): cont = 1 # volume format should be "volume_name:mount_path" for vol in value: + size = None + volume_id = None vol_parts = vol.split(":") volume = vol_parts[0] mount_path = None @@ -2134,12 +2136,13 @@ def _gen_k8s_volumes(self, node, nodetemplates, value): root_type = Tosca._get_root_parent_type(node).type if root_type == "tosca.nodes.BlockStorage" and node.name == volume: size = self._final_function_result(node.get_property_value('size'), node) + volume_id = self._final_function_result(node.get_property_value('volume_id'), node) if size: if not size.endswith("B"): size += "B" size = int(ScalarUnit_Size(size).get_num_from_scalar_unit('B')) - volumes.append((cont, size, mount_path)) + volumes.append((cont, size, mount_path, volume_id)) cont += 1 return volumes @@ -2192,7 +2195,9 @@ def _gen_k8s_system(self, node, nodetemplates): value = int(ScalarUnit_Size(value).get_num_from_scalar_unit('B')) res.setValue("memory.size", value, 'B') elif prop.name == 'volumes': - for num, size, mount_path in self._gen_k8s_volumes(node, nodetemplates, value): + for num, size, mount_path, volume_id in self._gen_k8s_volumes(node, nodetemplates, value): + if volume_id: + res.setValue('disk.%d.image.url' % num, volume_id) if size: res.setValue('disk.%d.size' % num, size, 'B') if mount_path: diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml index 3d84c4636..b7d191f0f 100644 --- a/test/files/tosca_k8s.yml +++ b/test/files/tosca_k8s.yml @@ -46,4 +46,6 @@ topology_template: type: tosca.nodes.BlockStorage properties: size: 10 GB + # Set the PV name in this field + # volume_id: "PV name" From 8ad6abdf281f57cb007ced39f6e2e7c7ab236f15 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 14 Feb 2024 16:27:26 +0100 Subject: [PATCH 17/26] Fix docs --- README.md | 2 +- doc/source/radl.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 42f667b95..ef7470de7 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # IM - Infrastructure Manager [![PyPI](https://img.shields.io/pypi/v/im.svg)](https://pypi.org/project/im) -[![Build Status](https://jenkins.i3m.upv.es/buildStatus/icon?job=grycap/im-unit-master)](https://jenkins.i3m.upv.es/job/grycap/job/im-unit-master/) +[![Tests](https://github.com/grycap/im/actions/workflows/main.yaml/badge.svg)](https://github.com/grycap/im/actions/workflows/main.yaml) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/582a0d6e763f44bdade11133e5191439)](https://www.codacy.com/gh/grycap/im/dashboard?utm_source=github.com&utm_medium=referral&utm_content=grycap/im&utm_campaign=Badge_Grade) [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/582a0d6e763f44bdade11133e5191439)](https://www.codacy.com/gh/grycap/im/dashboard?utm_source=github.com&utm_medium=referral&utm_content=grycap/im&utm_campaign=Badge_Coverage) [![License](https://img.shields.io/badge/license-GPL%20v3.0-brightgreen.svg)](LICENSE) diff --git a/doc/source/radl.rst b/doc/source/radl.rst index a53e983d1..ea057ed4f 100644 --- a/doc/source/radl.rst +++ b/doc/source/radl.rst @@ -43,7 +43,7 @@ An RADL document has the next general structure:: deploy [] The ``description`` optional keyword can only appear once. It is an special keyword -to add some medatada to the RADL. All features on the list are free, except for the ``name`` field. +to add some metadata to the RADL. All features on the list are free, except for the ``name`` field. The ``name`` field is a predefined but optional field used to assign a name to the infrastructure. For instance:: description desc ( From b3a021d6053b2cd4e2e92d70af8a23ea81cff9e6 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 14 Feb 2024 16:27:26 +0100 Subject: [PATCH 18/26] Fix docs --- README.md | 2 +- doc/source/radl.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 42f667b95..ef7470de7 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # IM - Infrastructure Manager [![PyPI](https://img.shields.io/pypi/v/im.svg)](https://pypi.org/project/im) -[![Build Status](https://jenkins.i3m.upv.es/buildStatus/icon?job=grycap/im-unit-master)](https://jenkins.i3m.upv.es/job/grycap/job/im-unit-master/) +[![Tests](https://github.com/grycap/im/actions/workflows/main.yaml/badge.svg)](https://github.com/grycap/im/actions/workflows/main.yaml) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/582a0d6e763f44bdade11133e5191439)](https://www.codacy.com/gh/grycap/im/dashboard?utm_source=github.com&utm_medium=referral&utm_content=grycap/im&utm_campaign=Badge_Grade) [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/582a0d6e763f44bdade11133e5191439)](https://www.codacy.com/gh/grycap/im/dashboard?utm_source=github.com&utm_medium=referral&utm_content=grycap/im&utm_campaign=Badge_Coverage) [![License](https://img.shields.io/badge/license-GPL%20v3.0-brightgreen.svg)](LICENSE) diff --git a/doc/source/radl.rst b/doc/source/radl.rst index 99b6b8b86..f5c06e024 100644 --- a/doc/source/radl.rst +++ b/doc/source/radl.rst @@ -43,7 +43,7 @@ An RADL document has the next general structure:: deploy [] The ``description`` optional keyword can only appear once. It is an special keyword -to add some medatada to the RADL. All features on the list are free, except for the ``name`` field. +to add some metadata to the RADL. All features on the list are free, except for the ``name`` field. The ``name`` field is a predefined but optional field used to assign a name to the infrastructure. For instance:: description desc ( From 28bb316e3be508cb4afdb018cb49af447a711a93 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 14 Feb 2024 16:28:28 +0100 Subject: [PATCH 19/26] Fix style --- IM/connectors/Kubernetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 888951b7a..95c88cf42 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -191,7 +191,7 @@ def _create_volume_claim(self, claim_data, auth_data): def _create_volumes(self, namespace, system, pod_name, auth_data, persistent=False): res = [] cont = 1 - while ((system.getValue("disk." + str(cont) + ".size") or + while ((system.getValue("disk." + str(cont) + ".size") or system.getValue("disk." + str(cont) + ".image.url")) and system.getValue("disk." + str(cont) + ".mount_path")): volume_id = system.getValue("disk." + str(cont) + ".image.url") From 58f39e69fb4bcbd7f17e852e532210cb202463bb Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 14 Feb 2024 16:33:59 +0100 Subject: [PATCH 20/26] remove unused import --- IM/connectors/OSCAR.py | 1 - 1 file changed, 1 deletion(-) diff --git a/IM/connectors/OSCAR.py b/IM/connectors/OSCAR.py index 49db9bd4c..d7dfc3ba8 100644 --- a/IM/connectors/OSCAR.py +++ b/IM/connectors/OSCAR.py @@ -17,7 +17,6 @@ import base64 import json import requests -import re from IM.VirtualMachine import VirtualMachine from .CloudConnector import CloudConnector from radl.radl import Feature From 3a028816ee5f32eba659889c47b8bc0d9be566bd Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 14 Feb 2024 16:27:26 +0100 Subject: [PATCH 21/26] add req --- requirements-tests.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-tests.txt b/requirements-tests.txt index 046c9c96f..b59382bfc 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -32,6 +32,7 @@ pyVmomi hvac psutil scar +nose mock coverage requests-cache From 42f47eeee09d9f214eb4eef4949128b1f1c42389 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 15 Feb 2024 08:25:49 +0100 Subject: [PATCH 22/26] Revert "add req" This reverts commit 3a028816ee5f32eba659889c47b8bc0d9be566bd. --- requirements-tests.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index b59382bfc..046c9c96f 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -32,7 +32,6 @@ pyVmomi hvac psutil scar -nose mock coverage requests-cache From 4a84264fa82ed734e4f2c2856f52c5ba4bc46b0e Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 15 Feb 2024 10:31:22 +0100 Subject: [PATCH 23/26] Improve tests --- .github/workflows/main.yaml | 5 ++--- monitoring/probeim.py | 3 ++- test/functional/test_im.py | 3 ++- test/integration/QuickTestIM.py | 3 ++- test/integration/TestIM.py | 3 ++- test/integration/TestREST.py | 3 ++- test/integration/TestREST_JSON.py | 3 ++- test/unit/AppDBIS.py | 3 ++- test/unit/SSH.py | 3 ++- test/unit/Tosca.py | 3 ++- test/unit/connectors/CloudConn.py | 3 ++- test/unit/openid.py | 3 ++- test/unit/test_im_logic.py | 3 ++- 13 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e9dc16cfa..ca494ad1c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -24,9 +24,8 @@ jobs: - name: Check code style run: pycodestyle --max-line-length=120 --ignore=E402,W504,W605,E722 . --exclude=doc - - name: Test with nose - #run: nosetests test/unit/connectors/*.py test/unit/*.py test/functional/*.py -v --stop --with-xunit --with-coverage --cover-erase --cover-xml --cover-package=IM,contextualization - run: python -m coverage run --source=. -m unittest discover -s test/unit -p '*.py' + - name: Unit tests + run: python -m coverage run --source=. -m unittest discover -v -s test/unit -p '*.py' - name: Generate XML coverage report run: python -m coverage xml diff --git a/monitoring/probeim.py b/monitoring/probeim.py index 8bee3c714..738b8164e 100644 --- a/monitoring/probeim.py +++ b/monitoring/probeim.py @@ -40,7 +40,8 @@ def read_file_as_string(file_name): tests_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(tests_path, file_name) - return open(abs_file_path, 'r').read() + with open(abs_file_path, 'r') as f: + return f.read() class TimeOutExcetion(Exception): diff --git a/test/functional/test_im.py b/test/functional/test_im.py index 5cee2e3a8..7cc3d7817 100755 --- a/test/functional/test_im.py +++ b/test/functional/test_im.py @@ -44,7 +44,8 @@ def read_file_as_string(file_name): tests_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(tests_path, file_name) - return open(abs_file_path, 'r').read() + with open(abs_file_path, 'r') as f: + return f.read() class TestIM(unittest.TestCase): diff --git a/test/integration/QuickTestIM.py b/test/integration/QuickTestIM.py index 483a8b336..4ffd13907 100755 --- a/test/integration/QuickTestIM.py +++ b/test/integration/QuickTestIM.py @@ -44,7 +44,8 @@ def read_file_as_string(file_name): tests_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(tests_path, file_name) - return open(abs_file_path, 'r').read() + with open(abs_file_path, 'r') as f: + return f.read() class QuickTestIM(unittest.TestCase): diff --git a/test/integration/TestIM.py b/test/integration/TestIM.py index ca0b2d2cc..d82cbf048 100755 --- a/test/integration/TestIM.py +++ b/test/integration/TestIM.py @@ -47,7 +47,8 @@ def read_file_as_string(file_name): tests_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(tests_path, file_name) - return open(abs_file_path, 'r').read() + with open(abs_file_path, 'r') as f: + return f.read() class TestIM(unittest.TestCase): diff --git a/test/integration/TestREST.py b/test/integration/TestREST.py index de6f31816..96e1876fb 100755 --- a/test/integration/TestREST.py +++ b/test/integration/TestREST.py @@ -44,7 +44,8 @@ def read_file_as_string(file_name): tests_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(tests_path, file_name) - return open(abs_file_path, 'r').read() + with open(abs_file_path, 'r') as f: + return f.read() class TestIM(unittest.TestCase): diff --git a/test/integration/TestREST_JSON.py b/test/integration/TestREST_JSON.py index 42dbb784e..f8a056a55 100755 --- a/test/integration/TestREST_JSON.py +++ b/test/integration/TestREST_JSON.py @@ -43,7 +43,8 @@ def read_file_as_string(file_name): tests_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(tests_path, file_name) - return open(abs_file_path, 'r').read() + with open(abs_file_path, 'r') as f: + return f.read() class TestIM(unittest.TestCase): diff --git a/test/unit/AppDBIS.py b/test/unit/AppDBIS.py index 60d23d594..12c53995b 100644 --- a/test/unit/AppDBIS.py +++ b/test/unit/AppDBIS.py @@ -13,7 +13,8 @@ def read_file_as_string(file_name): tests_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(tests_path, file_name) - return open(abs_file_path, 'r').read() + with open(abs_file_path, 'r') as f: + return f.read() class TestAppDBIS(unittest.TestCase): diff --git a/test/unit/SSH.py b/test/unit/SSH.py index be742613a..bd4f975a7 100644 --- a/test/unit/SSH.py +++ b/test/unit/SSH.py @@ -26,7 +26,8 @@ def read_file_as_string(file_name): tests_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(tests_path, file_name) - return open(abs_file_path, 'r').read() + with open(abs_file_path, 'r') as f: + return f.read() class TestSSH(unittest.TestCase): diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index 8c982081b..7a8df35b4 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -37,7 +37,8 @@ def read_file_as_string(file_name): tests_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(tests_path, file_name) - return open(abs_file_path, 'r').read() + with open(abs_file_path, 'r') as f: + return f.read() class TestTosca(unittest.TestCase): diff --git a/test/unit/connectors/CloudConn.py b/test/unit/connectors/CloudConn.py index 80b0f71ff..b4683f3a6 100755 --- a/test/unit/connectors/CloudConn.py +++ b/test/unit/connectors/CloudConn.py @@ -58,4 +58,5 @@ def tearDown(self): def read_file_as_string(file_name): tests_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(tests_path, file_name) - return open(abs_file_path, 'r').read() + with open(abs_file_path, 'r') as f: + return f.read() diff --git a/test/unit/openid.py b/test/unit/openid.py index 7822a7a32..65513ce75 100755 --- a/test/unit/openid.py +++ b/test/unit/openid.py @@ -27,7 +27,8 @@ def read_file_as_string(file_name): tests_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(tests_path, file_name) - return open(abs_file_path, 'r').read() + with open(abs_file_path, 'r') as f: + return f.read() class TestOpenIDClient(unittest.TestCase): diff --git a/test/unit/test_im_logic.py b/test/unit/test_im_logic.py index a984c568e..5f21929b8 100644 --- a/test/unit/test_im_logic.py +++ b/test/unit/test_im_logic.py @@ -50,7 +50,8 @@ def read_file_as_string(file_name): tests_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(tests_path, file_name) - return open(abs_file_path, 'r').read() + with open(abs_file_path, 'r') as f: + return f.read() class TestIM(unittest.TestCase): From a3872d8c3168b673a1c29e31602b64f83d60e790 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 15 Feb 2024 10:49:46 +0100 Subject: [PATCH 24/26] Improve code --- IM/ConfManager.py | 4 +-- IM/ansible_utils/ansible_launcher.py | 6 ++-- IM/connectors/OpenNebula.py | 12 ++++---- IM/tosca/Tosca.py | 46 ++++++++++++++-------------- test/unit/REST.py | 3 +- 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/IM/ConfManager.py b/IM/ConfManager.py index 7957c55ad..7bf2de34a 100644 --- a/IM/ConfManager.py +++ b/IM/ConfManager.py @@ -21,7 +21,7 @@ import time import tempfile import shutil -from distutils.version import LooseVersion +from packaging.version import Version try: from StringIO import StringIO @@ -758,7 +758,7 @@ def get_vault_editor(vault_password): """ Get the correct VaultEditor object in different Ansible versions """ - if LooseVersion(ansible_version) >= LooseVersion("2.4.0"): + if Version(ansible_version) >= Version("2.4.0"): # for Ansible version 2.4.0 or higher vault_secrets = [('default', VaultSecret(_bytes=to_bytes(vault_password)))] return VaultEditor(VaultLib(vault_secrets)) diff --git a/IM/ansible_utils/ansible_launcher.py b/IM/ansible_utils/ansible_launcher.py index b69e48ded..9cf6ed9d5 100755 --- a/IM/ansible_utils/ansible_launcher.py +++ b/IM/ansible_utils/ansible_launcher.py @@ -25,7 +25,7 @@ import psutil import signal import logging -from distutils.version import LooseVersion +from packaging.version import Version from collections import namedtuple from ansible import errors from ansible import __version__ as ansible_version @@ -137,7 +137,7 @@ def run(self): self._kill_childs() def get_play_prereqs(self, options): - if LooseVersion(ansible_version) >= LooseVersion("2.4.0"): + if Version(ansible_version) >= Version("2.4.0"): # for Ansible version 2.4.0 or higher return self.get_play_prereqs_2_4(options) else: @@ -208,7 +208,7 @@ def version_info(ansible_version_string): 'revision': ansible_versions[2]} def _gen_options(self): - if LooseVersion(ansible_version) >= LooseVersion("2.8.0"): + if Version(ansible_version) >= Version("2.8.0"): from ansible.module_utils.common.collections import ImmutableDict from ansible import context context.CLIARGS = ImmutableDict(connection='ssh', diff --git a/IM/connectors/OpenNebula.py b/IM/connectors/OpenNebula.py index bd61c2880..9e347fb74 100644 --- a/IM/connectors/OpenNebula.py +++ b/IM/connectors/OpenNebula.py @@ -24,7 +24,7 @@ import os.path import time -from distutils.version import LooseVersion +from packaging.version import Version from IM.xmlobject import XMLObject try: from urlparse import urlparse @@ -430,9 +430,9 @@ def create_security_groups(self, inf, radl, auth_data): session_id = self.getSessionID(auth_data) sgs = {} - one_ver = LooseVersion(self.getONEVersion(auth_data)) + one_ver = Version(self.getONEVersion(auth_data)) # Security Groups appears in version 4.12.0 - if one_ver >= LooseVersion("4.12.0"): + if one_ver >= Version("4.12.0"): sgs = {} i = 0 system = radl.systems[0] @@ -571,7 +571,7 @@ def finalize(self, vm, last, auth_data): if vm.id: one_ver = self.getONEVersion(auth_data) op = 'terminate' - if one_ver <= LooseVersion("4.14.0"): + if one_ver <= Version("4.14.0"): op = 'delete' success, err = server.one.vm.action(session_id, op, int(vm.id))[0:2] else: @@ -587,7 +587,7 @@ def finalize(self, vm, last, auth_data): if last and success: one_ver = self.getONEVersion(auth_data) # Security Groups appears in version 4.12.0 - if one_ver >= LooseVersion("4.12.0"): + if one_ver >= Version("4.12.0"): self.delete_security_groups(vm.inf, auth_data) return (success, err) @@ -1181,7 +1181,7 @@ def create_snapshot(self, vm, disk_num, image_name, auto_delete, auth_data): image_type = "" # Use the default one one_ver = self.getONEVersion(auth_data) - if one_ver >= LooseVersion("5.0"): + if one_ver >= Version("5.0"): success, res_info = server.one.vm.disksaveas(session_id, int(vm.id), disk_num, image_name, image_type, -1)[0:2] else: diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index e61859012..233adbfcb 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -79,7 +79,7 @@ def _get_placement_property(self, sys_name, prop): sys_name)) return policy.properties[prop] else: - Tosca.logger.warn("Policy %s not supported. Ignoring it." % policy.type_definition.type) + Tosca.logger.warning("Policy %s not supported. Ignoring it." % policy.type_definition.type) return None @@ -135,7 +135,7 @@ def to_radl(self, inf_info=None): elif token_type == "private_key": ansible_host.setValue("credentials.private_key", token) else: - Tosca.logger.warn("Unknown tyoe of token %s. Ignoring." % token_type) + Tosca.logger.warning("Unknown tyoe of token %s. Ignoring." % token_type) radl.ansible_hosts = [ansible_host] elif root_type == "tosca.nodes.aisprint.FaaS.Function": min_instances, _, default_instances, count, removal_list = self._get_scalable_properties(node) @@ -260,7 +260,7 @@ def to_radl(self, inf_info=None): # Select the host to host this element compute = self._find_host_node(node, self.tosca.nodetemplates) if not compute: - Tosca.logger.warn( + Tosca.logger.warning( "Node %s has not compute node to host in." % node.name) interfaces = Tosca._get_interfaces(node) @@ -655,7 +655,7 @@ def _add_node_nets(self, node, radl, system, nodetemplates): else: # There are no a private IP, net the provider_id to the priv net if not public_net: - Tosca.logger.warn("Node %s does not require any IP!!" % node.name) + Tosca.logger.warning("Node %s does not require any IP!!" % node.name) if public_net: if pool_name: @@ -928,7 +928,7 @@ def _remove_recipe_header(script_content): try: yamlo = yaml.safe_load(script_content) if not isinstance(yamlo, list): - Tosca.logger.warn("Error parsing YAML: " + script_content + "\n.Do not remove header.") + Tosca.logger.warning("Error parsing YAML: " + script_content + "\n.Do not remove header.") return script_content except Exception: Tosca.logger.exception("Error parsing YAML: " + script_content + "\n.Do not remove header.") @@ -1033,11 +1033,11 @@ def _get_intrinsic_value(self, func, node, inf_info): "Incorrect substring_index in function token.") return None else: - Tosca.logger.warn( + Tosca.logger.warning( "Intrinsic function token must receive 3 parameters.") return None else: - Tosca.logger.warn( + Tosca.logger.warning( "Intrinsic function %s not supported." % func_name) return None @@ -1214,7 +1214,7 @@ def _get_attribute_result(self, func, node, inf_info): vm_list = inf_info.get_vm_list_by_system_name() if host_node.name not in vm_list: - Tosca.logger.warn("There are no VM associated with the name %s." % host_node.name) + Tosca.logger.warning("There are no VM associated with the name %s." % host_node.name) return None else: # As default assume that there will be only one VM per group @@ -1230,14 +1230,14 @@ def _get_attribute_result(self, func, node, inf_info): if node.type == "tosca.nodes.indigo.Compute": return vm.cont_out else: - Tosca.logger.warn("Attribute ctxt_log only supported" + Tosca.logger.warning("Attribute ctxt_log only supported" " in tosca.nodes.indigo.Compute nodes.") return None elif attribute_name == "ansible_output": if node.type == "tosca.nodes.indigo.Compute": return self._get_ansible_output(vm.cont_out, attribute_params) else: - Tosca.logger.warn("Attribute ansible_output only supported" + Tosca.logger.warning("Attribute ansible_output only supported" " in tosca.nodes.indigo.Compute nodes.") return None elif attribute_name == "credential" and capability_name == "endpoint": @@ -1257,7 +1257,7 @@ def _get_attribute_result(self, func, node, inf_info): res = res[index] return res else: - Tosca.logger.warn("Attribute credential of capability endpoint only" + Tosca.logger.warning("Attribute credential of capability endpoint only" " supported in tosca.nodes.indigo.Compute nodes.") return None elif attribute_name == "private_address": @@ -1308,7 +1308,7 @@ def _get_attribute_result(self, func, node, inf_info): # OSCAR function deployed in a pre-deployed cluster or not dns_host set return vm.getCloudConnector().cloud.get_url() - Tosca.logger.warn("Attribute endpoint only supported in tosca.nodes.aisprint.FaaS.Function") + Tosca.logger.warning("Attribute endpoint only supported in tosca.nodes.aisprint.FaaS.Function") return None elif attribute_name == "credential": if root_type == "tosca.nodes.aisprint.FaaS.Function": @@ -1327,7 +1327,7 @@ def _get_attribute_result(self, func, node, inf_info): "token": oscar_pass}, attribute_params) - Tosca.logger.warn("No password defined in tosca.nodes.indigo.OSCAR host node") + Tosca.logger.warning("No password defined in tosca.nodes.indigo.OSCAR host node") return None else: # OSCAR function deployed in a pre-deployed cluster or not dns_host set @@ -1344,16 +1344,16 @@ def _get_attribute_result(self, func, node, inf_info): "token": auth["token"]}, attribute_params) else: - Tosca.logger.warn("No valid auth data in OSCAR connector") + Tosca.logger.warning("No valid auth data in OSCAR connector") return None - Tosca.logger.warn("No auth data in OSCAR connector") + Tosca.logger.warning("No auth data in OSCAR connector") return None - Tosca.logger.warn("Attribute credential only supported in tosca.nodes.aisprint.FaaS.Function") + Tosca.logger.warning("Attribute credential only supported in tosca.nodes.aisprint.FaaS.Function") return None else: - Tosca.logger.warn("Attribute %s not supported." % attribute_name) + Tosca.logger.warning("Attribute %s not supported." % attribute_name) return None else: if attribute_name == "tosca_id": @@ -1411,10 +1411,10 @@ def _get_attribute_result(self, func, node, inf_info): if dns_host.strip("'\""): return "https://%s" % dns_host - Tosca.logger.warn("Attribute endpoint only supported in tosca.nodes.aisprint.FaaS.Function") + Tosca.logger.warning("Attribute endpoint only supported in tosca.nodes.aisprint.FaaS.Function") return None else: - Tosca.logger.warn("Attribute %s not supported." % attribute_name) + Tosca.logger.warning("Attribute %s not supported." % attribute_name) return None def _final_function_result(self, func, node, inf_info=None): @@ -1542,7 +1542,7 @@ def _node_fulfill_filter(self, node, node_filter): elif op == "valid_values": comparation = node_value in filter_value else: - Tosca.logger.warn("Logical operator %s not supported." % op) + Tosca.logger.warning("Logical operator %s not supported." % op) if not comparation: return False @@ -1788,7 +1788,7 @@ def _gen_system(self, node, nodetemplates): feature = Feature("disk.0.os.credentials.public_key", "=", token) res.addFeature(feature) else: - Tosca.logger.warn("Unknown tyoe of token %s. Ignoring." % token_type) + Tosca.logger.warning("Unknown tyoe of token %s. Ignoring." % token_type) if 'user' not in value or not value['user']: raise Exception("User must be specified in the image credentials.") name = "disk.0.os.credentials.username" @@ -2069,7 +2069,7 @@ def _gen_oscar_system(self, node): res.setValue("expose.%s" % elem, value) else: # this should never happen - Tosca.logger.warn("Property %s not expected. Ignoring." % prop.name) + Tosca.logger.warning("Property %s not expected. Ignoring." % prop.name) if node.requirements: deps = [] @@ -2114,7 +2114,7 @@ def _get_oscar_service_json(self, node): res['image_pull_secrets'] = value else: # this should never happen - Tosca.logger.warn("Property %s not expected. Ignoring." % prop.name) + Tosca.logger.warning("Property %s not expected. Ignoring." % prop.name) return res diff --git a/test/unit/REST.py b/test/unit/REST.py index 5d14a0cac..d5354a622 100755 --- a/test/unit/REST.py +++ b/test/unit/REST.py @@ -65,7 +65,8 @@ def read_file_as_bytes(file_name): tests_path = os.path.dirname(os.path.abspath(__file__)) abs_file_path = os.path.join(tests_path, file_name) - return BytesIO(open(abs_file_path, 'r').read().encode()) + with open(abs_file_path, 'r') as f: + return BytesIO(f.read().encode()) class TestREST(unittest.TestCase): From 45a808731d07fa84d618cc338a96ad2152c2a42c Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 15 Feb 2024 10:53:12 +0100 Subject: [PATCH 25/26] Improve code --- IM/tosca/Tosca.py | 6 +++--- test/unit/test_ansible.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 233adbfcb..11d24d122 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -1231,14 +1231,14 @@ def _get_attribute_result(self, func, node, inf_info): return vm.cont_out else: Tosca.logger.warning("Attribute ctxt_log only supported" - " in tosca.nodes.indigo.Compute nodes.") + " in tosca.nodes.indigo.Compute nodes.") return None elif attribute_name == "ansible_output": if node.type == "tosca.nodes.indigo.Compute": return self._get_ansible_output(vm.cont_out, attribute_params) else: Tosca.logger.warning("Attribute ansible_output only supported" - " in tosca.nodes.indigo.Compute nodes.") + " in tosca.nodes.indigo.Compute nodes.") return None elif attribute_name == "credential" and capability_name == "endpoint": if node.type == "tosca.nodes.indigo.Compute": @@ -1258,7 +1258,7 @@ def _get_attribute_result(self, func, node, inf_info): return res else: Tosca.logger.warning("Attribute credential of capability endpoint only" - " supported in tosca.nodes.indigo.Compute nodes.") + " supported in tosca.nodes.indigo.Compute nodes.") return None elif attribute_name == "private_address": if node.type == "tosca.nodes.indigo.Compute": diff --git a/test/unit/test_ansible.py b/test/unit/test_ansible.py index 02cf3d482..6788fc821 100755 --- a/test/unit/test_ansible.py +++ b/test/unit/test_ansible.py @@ -42,7 +42,6 @@ def test_ansible_thread(self): ansible_process.run() _, return_code, output = result.get() - print(output.getvalue()) self.assertEqual(return_code, 0) self.assertIn("failed=0", output.getvalue()) self.assertIn("changed=2", output.getvalue()) From b6a7cf914187432ca3fa4d278016201cd40cc5a3 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 15 Feb 2024 11:07:43 +0100 Subject: [PATCH 26/26] Improve code --- IM/connectors/OpenNebula.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/IM/connectors/OpenNebula.py b/IM/connectors/OpenNebula.py index 9e347fb74..43ce5d4bf 100644 --- a/IM/connectors/OpenNebula.py +++ b/IM/connectors/OpenNebula.py @@ -571,7 +571,7 @@ def finalize(self, vm, last, auth_data): if vm.id: one_ver = self.getONEVersion(auth_data) op = 'terminate' - if one_ver <= Version("4.14.0"): + if Version(one_ver) <= Version("4.14.0"): op = 'delete' success, err = server.one.vm.action(session_id, op, int(vm.id))[0:2] else: @@ -587,7 +587,7 @@ def finalize(self, vm, last, auth_data): if last and success: one_ver = self.getONEVersion(auth_data) # Security Groups appears in version 4.12.0 - if one_ver >= Version("4.12.0"): + if Version(one_ver) >= Version("4.12.0"): self.delete_security_groups(vm.inf, auth_data) return (success, err) @@ -1181,7 +1181,7 @@ def create_snapshot(self, vm, disk_num, image_name, auto_delete, auth_data): image_type = "" # Use the default one one_ver = self.getONEVersion(auth_data) - if one_ver >= Version("5.0"): + if Version(one_ver) >= Version("5.0"): success, res_info = server.one.vm.disksaveas(session_id, int(vm.id), disk_num, image_name, image_type, -1)[0:2] else: