From ebb123a453030b0d7ac607e2139fd899cc391ce4 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 15 Nov 2024 11:17:02 +0100 Subject: [PATCH 01/14] Do not use none in stats --- IM/Stats.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/IM/Stats.py b/IM/Stats.py index f5a53b05..21de2658 100644 --- a/IM/Stats.py +++ b/IM/Stats.py @@ -35,7 +35,7 @@ class Stats(): @staticmethod def _get_data(str_data, init_date=None, end_date=None): dic = json.loads(str_data) - resp = {'creation_date': None} + resp = {'creation_date': ''} if 'creation_date' in dic and dic['creation_date']: creation_date = datetime.datetime.fromtimestamp(float(dic['creation_date'])) resp['creation_date'] = str(creation_date) @@ -44,7 +44,7 @@ def _get_data(str_data, init_date=None, end_date=None): if end_date and creation_date > end_date: return None - resp['tosca_name'] = None + resp['tosca_name'] = '' if 'extra_info' in dic and dic['extra_info'] and "TOSCA" in dic['extra_info']: try: tosca = yaml.safe_load(dic['extra_info']['TOSCA']) @@ -56,8 +56,8 @@ def _get_data(str_data, init_date=None, end_date=None): resp['vm_count'] = 0 resp['cpu_count'] = 0 resp['memory_size'] = 0 - resp['cloud_type'] = None - resp['cloud_host'] = None + resp['cloud_type'] = '' + resp['cloud_host'] = '' resp['hybrid'] = False resp['deleted'] = True if 'deleted' in dic and dic['deleted'] else False for str_vm_data in dic['vm_list']: From a8bc4fd765ffac7f7722406df060fc507d6ee35c Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 15 Nov 2024 11:44:43 +0100 Subject: [PATCH 02/14] User virtualenv --- IM/ConfManager.py | 3 +- contextualization/conf-ansible.yml | 53 ++++++++++++++++++++---------- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/IM/ConfManager.py b/IM/ConfManager.py index 6ce5d057..fd2997b4 100644 --- a/IM/ConfManager.py +++ b/IM/ConfManager.py @@ -397,7 +397,8 @@ def launch_ctxt_agent(self, vm, tasks): vault_password = vm.info.systems[0].getValue("vault.password") if vault_password: vault_export = "export VAULT_PASS='%s' && " % vault_password - (pid, _, _) = ssh.execute("nohup sh -c \"" + vault_export + "python3 " + Config.REMOTE_CONF_DIR + + (pid, _, _) = ssh.execute("nohup sh -c \"" + vault_export + "/var/tmp/.ansible/bin/python3 " + + Config.REMOTE_CONF_DIR + "/" + str(self.inf.id) + "/" + ctxt_agent_command + Config.REMOTE_CONF_DIR + "/" + str(self.inf.id) + "/" + "/general_info.cfg " + remote_dir + "/" + os.path.basename(conf_file) + diff --git a/contextualization/conf-ansible.yml b/contextualization/conf-ansible.yml index de3172eb..d586ac8a 100644 --- a/contextualization/conf-ansible.yml +++ b/contextualization/conf-ansible.yml @@ -6,6 +6,7 @@ vars: # Ansible specific Version or "latest" ANSIBLE_VERSION: 4.10.0 + VENV_PATH: /var/tmp/.ansible tasks: ############## To avoid some issues with cloud-init and unattended upgrades ############### - name: Avoid unattended upgrades @@ -99,15 +100,15 @@ ################### Install Ansible/pip requisites ######################### - name: Debian/Ubuntu install requisites with apt - apt: name=python3-pip,wget,python3-setuptools,python3-psutil,sshpass,openssh-client,unzip install_recommends=no + apt: name=python3-pip,wget,python3-setuptools,sshpass,openssh-client,unzip install_recommends=no when: ansible_os_family == "Debian" - name: Yum install requisites RH 7/8 or Fedora - command: yum install -y python3-pip python3-setuptools python3-psutil sshpass openssh-clients + command: yum install -y python3-pip python3-setuptools sshpass openssh-clients when: ansible_os_family == "RedHat" - name: Zypper install requirements Suse - zypper: name=python3-pip,python3-setuptools,python3-psutil,wget,python3-cryptography state=present + zypper: name=python3-pip,python3-setuptools,wget,python3-cryptography state=present when: ansible_os_family == "Suse" - name: Install python-setuptools @@ -116,11 +117,26 @@ ######################################### Use pip to enable to set the version ############################################# + - name: Set extra_args var + set_fact: + extra_args: --prefer-binary + + - name: Set extra_args var in py3.11 + set_fact: + extra_args: --prefer-binary --break-system-packages + when: ansible_python_version is version('3.11', '>=') + + - name: Install virtualenv with pip + pip: + name: virtualenv + executable: pip3 + extra_args: "{{ extra_args }}" + # Version over 21 does not work with python 3.6 or older - name: Upgrade pip in py3.6- pip: name: pip>18.0,<21.0 - executable: pip3 + virtualenv: "{{ VENV_PATH }}" # in some old distros we need to trust in the pypi to avoid SSL errors extra_args: --trusted-host files.pythonhosted.org --trusted-host pypi.org --trusted-host pypi.python.org when: ansible_python_version is version('3.7', '<') @@ -128,37 +144,38 @@ - name: Upgrade pip in py3.7-py3.8 pip: name: pip>20.0 - executable: pip3 + virtualenv: "{{ VENV_PATH }}" when: ansible_python_version is version('3.7', '>=') and ansible_python_version is version('3.9', '<') - name: Upgrade pip in py3.9-py3.10 pip: name: pip>=22.0 - executable: pip3 + virtualenv: "{{ VENV_PATH }}" when: ansible_python_version is version('3.9', '>=') and ansible_python_version is version('3.11', '<') # Version 66 (#2497) fails - name: Upgrade setuptools with Pip in py3.11- pip: name: setuptools<66.0.0 - executable: pip3 + virtualenv: "{{ VENV_PATH }}" when: ansible_python_version is version('3.11', '<') - name: Set extra_args var set_fact: extra_args: --prefer-binary - - name: Set extra_args var in py3.11 - set_fact: - extra_args: --prefer-binary --break-system-packages - when: ansible_python_version is version('3.11', '>=') + - name: Install psutil + pip: + name: psutil + virtualenv: "{{ VENV_PATH }}" + extra_args: "{{ extra_args }}" - name: Install cryptography & pyOpenSSL in py3.11- pip: name: - cryptography>36.0.0,<39.0.0 - pyOpenSSL>20.0,<22.1.0 - executable: pip3 + virtualenv: "{{ VENV_PATH }}" extra_args: "{{ extra_args }}" when: ansible_python_version is version('3.11', '<') @@ -167,7 +184,7 @@ name: - cryptography>36.0.0 - pyOpenSSL>20.0 - executable: pip3 + virtualenv: "{{ VENV_PATH }}" extra_args: "{{ extra_args }}" when: ansible_python_version is version('3.11', '>=') @@ -178,7 +195,7 @@ - pyyaml - paramiko>=2.9.5 - packaging - executable: pip3 + virtualenv: "{{ VENV_PATH }}" extra_args: "{{ extra_args }}" - name: Set Ansible newer version for python 3.8+ @@ -201,14 +218,14 @@ pip: name: ansible version: "{{ ANSIBLE_VERSION }}" - executable: pip3 + virtualenv: "{{ VENV_PATH }}" extra_args: "{{ extra_args }}" when: ANSIBLE_VERSION != "latest" - name: Install latest ansible version with Pip pip: name: ansible - executable: pip3 + virtualenv: "{{ VENV_PATH }}" extra_args: "{{ extra_args }}" when: ANSIBLE_VERSION == "latest" @@ -219,13 +236,13 @@ name: - jmespath - scp - executable: pip3 + virtualenv: "{{ VENV_PATH }}" extra_args: "{{ extra_args }}" - name: Install pywinrm with Pip pip: name: pywinrm - executable: pip3 + virtualenv: "{{ VENV_PATH }}" extra_args: "{{ extra_args }}" ignore_errors: yes From 0b832dd09b422c2b53ec8e8a53b81c4eae27bd8f Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 15 Nov 2024 12:27:33 +0100 Subject: [PATCH 03/14] Fix --- contextualization/conf-ansible.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/contextualization/conf-ansible.yml b/contextualization/conf-ansible.yml index d586ac8a..0804668b 100644 --- a/contextualization/conf-ansible.yml +++ b/contextualization/conf-ansible.yml @@ -130,7 +130,6 @@ pip: name: virtualenv executable: pip3 - extra_args: "{{ extra_args }}" # Version over 21 does not work with python 3.6 or older - name: Upgrade pip in py3.6- From 1f6b85293ae9f014cbcc1646060b36a88858a878 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 15 Nov 2024 12:45:41 +0100 Subject: [PATCH 04/14] Fix --- contextualization/conf-ansible.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contextualization/conf-ansible.yml b/contextualization/conf-ansible.yml index 0804668b..a8544a95 100644 --- a/contextualization/conf-ansible.yml +++ b/contextualization/conf-ansible.yml @@ -119,17 +119,18 @@ - name: Set extra_args var set_fact: - extra_args: --prefer-binary + extra_args: '' - name: Set extra_args var in py3.11 set_fact: - extra_args: --prefer-binary --break-system-packages + extra_args: --break-system-packages when: ansible_python_version is version('3.11', '>=') - name: Install virtualenv with pip pip: name: virtualenv executable: pip3 + extra_args: "{{ extra_args }}" # Version over 21 does not work with python 3.6 or older - name: Upgrade pip in py3.6- From fe773586d97ae5317cb51458ccfcb10aac2c42a3 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 26 Nov 2024 10:17:22 +0100 Subject: [PATCH 05/14] Fix ansible-galaxy path --- IM/CtxtAgentBase.py | 4 ++-- test/unit/test_ctxt_agent.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/IM/CtxtAgentBase.py b/IM/CtxtAgentBase.py index 15457ccb..486789b1 100644 --- a/IM/CtxtAgentBase.py +++ b/IM/CtxtAgentBase.py @@ -508,7 +508,7 @@ def install_ansible_roles(self, general_conf_data, playbook): task["name"] = "Create YAML file to install the collections with ansible-galaxy" yaml_data[0]['tasks'].append(task) - task = {"command": "ansible-galaxy collection install -c -r %s" % filename} + task = {"command": "/var/tmp/.ansible/bin/ansible-galaxy collection install -c -r %s" % filename} task["name"] = "Install galaxy collections" task["become"] = "yes" task["register"] = "collections_install" @@ -562,7 +562,7 @@ def install_ansible_roles(self, general_conf_data, playbook): task["name"] = "Create YAML file to install the roles with ansible-galaxy" yaml_data[0]['tasks'].append(task) - task = {"command": "ansible-galaxy install -c -r %s" % filename} + task = {"command": "/var/tmp/.ansible/bin/ansible-galaxy install -c -r %s" % filename} task["name"] = "Install galaxy roles" task["become"] = "yes" task["register"] = "roles_install" diff --git a/test/unit/test_ctxt_agent.py b/test/unit/test_ctxt_agent.py index bc4894b2..7354646f 100755 --- a/test/unit/test_ctxt_agent.py +++ b/test/unit/test_ctxt_agent.py @@ -277,7 +277,8 @@ def test_95_install_ansible_roles(self): copy_content = yaml_data[0]['tasks'][1]['copy'][pos + 9:-2] self.assertEqual(copy_content, "[{src: ansible_role}, {name: hadoop, src: " "'git+https://github.com/micafer/ansible-role-hadoop'}]") - self.assertEqual(yaml_data[0]['tasks'][2]['command'][:47], "ansible-galaxy install -c -r /tmp/galaxy_roles_") + self.assertEqual(yaml_data[0]['tasks'][2]['command'][:69], + "/var/tmp/.ansible/bin/ansible-galaxy install -c -r /tmp/galaxy_roles_") os.unlink(res) @@ -300,8 +301,8 @@ def test_99_install_ansible_collections(self): pos = yaml_data[0]['tasks'][0]['copy'].find('content="') copy_content = yaml_data[0]['tasks'][0]['copy'][pos + 9:-2] self.assertEqual(copy_content, "{collections: [{name: ns.collection, version: '1.0'}]}") - self.assertEqual(yaml_data[0]['tasks'][1]['command'][:64], - "ansible-galaxy collection install -c -r /tmp/galaxy_collections_") + self.assertEqual(yaml_data[0]['tasks'][1]['command'][:86], + "/var/tmp/.ansible/bin/ansible-galaxy collection install -c -r /tmp/galaxy_collections_") os.unlink(res) From 48eec8b324f6deb78fdaa9fae6c9b2ffeb3cf7af Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 26 Nov 2024 13:01:45 +0100 Subject: [PATCH 06/14] Fix ctxt dist path --- IM/ConfManager.py | 5 +++-- IM/CtxtAgentBase.py | 5 +++-- contextualization/ctxt_agent_dist.py | 3 ++- test/unit/test_ctxt_agent.py | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/IM/ConfManager.py b/IM/ConfManager.py index fd2997b4..ff3125d1 100644 --- a/IM/ConfManager.py +++ b/IM/ConfManager.py @@ -56,6 +56,7 @@ from IM.recipe import Recipe from IM.config import Config from radl.radl import system, contextualize_item +from IM.CtxtAgentBase import CtxtAgentBase class ConfManager(LoggerMixin, threading.Thread): @@ -397,8 +398,8 @@ def launch_ctxt_agent(self, vm, tasks): vault_password = vm.info.systems[0].getValue("vault.password") if vault_password: vault_export = "export VAULT_PASS='%s' && " % vault_password - (pid, _, _) = ssh.execute("nohup sh -c \"" + vault_export + "/var/tmp/.ansible/bin/python3 " + - Config.REMOTE_CONF_DIR + + (pid, _, _) = ssh.execute("nohup sh -c \"" + vault_export + CtxtAgentBase.VENV_DIR + + "/bin/python3 " + Config.REMOTE_CONF_DIR + "/" + str(self.inf.id) + "/" + ctxt_agent_command + Config.REMOTE_CONF_DIR + "/" + str(self.inf.id) + "/" + "/general_info.cfg " + remote_dir + "/" + os.path.basename(conf_file) + diff --git a/IM/CtxtAgentBase.py b/IM/CtxtAgentBase.py index 486789b1..a88b0953 100644 --- a/IM/CtxtAgentBase.py +++ b/IM/CtxtAgentBase.py @@ -37,6 +37,7 @@ class CtxtAgentBase: # the ConfManager PLAYBOOK_RETRIES = 1 INTERNAL_PLAYBOOK_RETRIES = 1 + VENV_DIR = "/var/tmp/.ansible" def __init__(self, conf_data_filename): self.logger = None @@ -508,7 +509,7 @@ def install_ansible_roles(self, general_conf_data, playbook): task["name"] = "Create YAML file to install the collections with ansible-galaxy" yaml_data[0]['tasks'].append(task) - task = {"command": "/var/tmp/.ansible/bin/ansible-galaxy collection install -c -r %s" % filename} + task = {"command": self.VENV_DIR + "/bin/ansible-galaxy collection install -c -r %s" % filename} task["name"] = "Install galaxy collections" task["become"] = "yes" task["register"] = "collections_install" @@ -562,7 +563,7 @@ def install_ansible_roles(self, general_conf_data, playbook): task["name"] = "Create YAML file to install the roles with ansible-galaxy" yaml_data[0]['tasks'].append(task) - task = {"command": "/var/tmp/.ansible/bin/ansible-galaxy install -c -r %s" % filename} + task = {"command": self.VENV_DIR + "/bin/ansible-galaxy install -c -r %s" % filename} task["name"] = "Install galaxy roles" task["become"] = "yes" task["register"] = "roles_install" diff --git a/contextualization/ctxt_agent_dist.py b/contextualization/ctxt_agent_dist.py index f96a8a01..ac5c1e77 100755 --- a/contextualization/ctxt_agent_dist.py +++ b/contextualization/ctxt_agent_dist.py @@ -162,7 +162,8 @@ def LaunchRemoteAgent(self, vm, vault_pass, pk_file, changed_pass_ok): vm_dir = os.path.abspath(os.path.dirname(self.vm_conf_data_filename)) remote_dir = os.path.abspath(os.path.dirname(self.conf_data_filename)) try: - (pid, _, _) = ssh_client.execute(vault_export + "nohup python3 " + remote_dir + "/ctxt_agent_dist.py " + + (pid, _, _) = ssh_client.execute(vault_export + "nohup " + CtxtAgentBase.VENV_DIR + "/bin/python3 " + + remote_dir + "/ctxt_agent_dist.py " + self.conf_data_filename + " " + self.vm_conf_data_filename + " 1 > " + vm_dir + "/stdout 2> " + vm_dir + "/stderr < /dev/null & echo -n $!") diff --git a/test/unit/test_ctxt_agent.py b/test/unit/test_ctxt_agent.py index 7354646f..3f9dac14 100755 --- a/test/unit/test_ctxt_agent.py +++ b/test/unit/test_ctxt_agent.py @@ -278,7 +278,7 @@ def test_95_install_ansible_roles(self): self.assertEqual(copy_content, "[{src: ansible_role}, {name: hadoop, src: " "'git+https://github.com/micafer/ansible-role-hadoop'}]") self.assertEqual(yaml_data[0]['tasks'][2]['command'][:69], - "/var/tmp/.ansible/bin/ansible-galaxy install -c -r /tmp/galaxy_roles_") + ctxt_agent.VENV_DIR + "/bin/ansible-galaxy install -c -r /tmp/galaxy_roles_") os.unlink(res) @@ -302,7 +302,7 @@ def test_99_install_ansible_collections(self): copy_content = yaml_data[0]['tasks'][0]['copy'][pos + 9:-2] self.assertEqual(copy_content, "{collections: [{name: ns.collection, version: '1.0'}]}") self.assertEqual(yaml_data[0]['tasks'][1]['command'][:86], - "/var/tmp/.ansible/bin/ansible-galaxy collection install -c -r /tmp/galaxy_collections_") + ctxt_agent.VENV_DIR + "/bin/ansible-galaxy collection install -c -r /tmp/galaxy_collections_") os.unlink(res) From b596d223fd8d68c00e94cf2287af20abd23ddaac Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 26 Nov 2024 16:30:51 +0100 Subject: [PATCH 07/14] Fix ctxt dist path --- IM/ConfManager.py | 5 +++-- contextualization/ctxt_agent_dist.py | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/IM/ConfManager.py b/IM/ConfManager.py index ff3125d1..438619e7 100644 --- a/IM/ConfManager.py +++ b/IM/ConfManager.py @@ -391,15 +391,16 @@ def launch_ctxt_agent(self, vm, tasks): if len(self.inf.get_vm_list()) > Config.VM_NUM_USE_CTXT_DIST: self.log_info("Using ctxt_agent_dist") ctxt_agent_command = "/ctxt_agent_dist.py " + python_path = "python3 " else: self.log_info("Using ctxt_agent") ctxt_agent_command = "/ctxt_agent.py " + python_path = CtxtAgentBase.VENV_DIR + "/bin/python3 " vault_export = "" vault_password = vm.info.systems[0].getValue("vault.password") if vault_password: vault_export = "export VAULT_PASS='%s' && " % vault_password - (pid, _, _) = ssh.execute("nohup sh -c \"" + vault_export + CtxtAgentBase.VENV_DIR + - "/bin/python3 " + Config.REMOTE_CONF_DIR + + (pid, _, _) = ssh.execute("nohup sh -c \"" + vault_export + python_path + Config.REMOTE_CONF_DIR + "/" + str(self.inf.id) + "/" + ctxt_agent_command + Config.REMOTE_CONF_DIR + "/" + str(self.inf.id) + "/" + "/general_info.cfg " + remote_dir + "/" + os.path.basename(conf_file) + diff --git a/contextualization/ctxt_agent_dist.py b/contextualization/ctxt_agent_dist.py index ac5c1e77..f96a8a01 100755 --- a/contextualization/ctxt_agent_dist.py +++ b/contextualization/ctxt_agent_dist.py @@ -162,8 +162,7 @@ def LaunchRemoteAgent(self, vm, vault_pass, pk_file, changed_pass_ok): vm_dir = os.path.abspath(os.path.dirname(self.vm_conf_data_filename)) remote_dir = os.path.abspath(os.path.dirname(self.conf_data_filename)) try: - (pid, _, _) = ssh_client.execute(vault_export + "nohup " + CtxtAgentBase.VENV_DIR + "/bin/python3 " + - remote_dir + "/ctxt_agent_dist.py " + + (pid, _, _) = ssh_client.execute(vault_export + "nohup python3 " + remote_dir + "/ctxt_agent_dist.py " + self.conf_data_filename + " " + self.vm_conf_data_filename + " 1 > " + vm_dir + "/stdout 2> " + vm_dir + "/stderr < /dev/null & echo -n $!") From d34b9cb1762af41b9ea123610f8e55084219238e Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 26 Nov 2024 16:53:53 +0100 Subject: [PATCH 08/14] Fix ctxt dist path --- IM/ConfManager.py | 5 ++--- contextualization/ansible_install.sh | 4 ++++ contextualization/ctxt_agent_dist.py | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/IM/ConfManager.py b/IM/ConfManager.py index 438619e7..ff3125d1 100644 --- a/IM/ConfManager.py +++ b/IM/ConfManager.py @@ -391,16 +391,15 @@ def launch_ctxt_agent(self, vm, tasks): if len(self.inf.get_vm_list()) > Config.VM_NUM_USE_CTXT_DIST: self.log_info("Using ctxt_agent_dist") ctxt_agent_command = "/ctxt_agent_dist.py " - python_path = "python3 " else: self.log_info("Using ctxt_agent") ctxt_agent_command = "/ctxt_agent.py " - python_path = CtxtAgentBase.VENV_DIR + "/bin/python3 " vault_export = "" vault_password = vm.info.systems[0].getValue("vault.password") if vault_password: vault_export = "export VAULT_PASS='%s' && " % vault_password - (pid, _, _) = ssh.execute("nohup sh -c \"" + vault_export + python_path + Config.REMOTE_CONF_DIR + + (pid, _, _) = ssh.execute("nohup sh -c \"" + vault_export + CtxtAgentBase.VENV_DIR + + "/bin/python3 " + Config.REMOTE_CONF_DIR + "/" + str(self.inf.id) + "/" + ctxt_agent_command + Config.REMOTE_CONF_DIR + "/" + str(self.inf.id) + "/" + "/general_info.cfg " + remote_dir + "/" + os.path.basename(conf_file) + diff --git a/contextualization/ansible_install.sh b/contextualization/ansible_install.sh index fcdcbb3f..062dd240 100755 --- a/contextualization/ansible_install.sh +++ b/contextualization/ansible_install.sh @@ -39,6 +39,10 @@ distribution_id() { echo ${RETVAL} } +# Create a symbolic link to python3 in case of not venv created +ls /var/tmp/.ansible/bin/ || mkdir -p /var/tmp/.ansible/bin/ +ls /var/tmp/.ansible/bin/python3 || ln -s /usr/bin/python3 /var/tmp/.ansible/bin/python3 + if [ $(which ansible-playbook) ]; then echo "Ansible installed. Do not install." else diff --git a/contextualization/ctxt_agent_dist.py b/contextualization/ctxt_agent_dist.py index f96a8a01..ac5c1e77 100755 --- a/contextualization/ctxt_agent_dist.py +++ b/contextualization/ctxt_agent_dist.py @@ -162,7 +162,8 @@ def LaunchRemoteAgent(self, vm, vault_pass, pk_file, changed_pass_ok): vm_dir = os.path.abspath(os.path.dirname(self.vm_conf_data_filename)) remote_dir = os.path.abspath(os.path.dirname(self.conf_data_filename)) try: - (pid, _, _) = ssh_client.execute(vault_export + "nohup python3 " + remote_dir + "/ctxt_agent_dist.py " + + (pid, _, _) = ssh_client.execute(vault_export + "nohup " + CtxtAgentBase.VENV_DIR + "/bin/python3 " + + remote_dir + "/ctxt_agent_dist.py " + self.conf_data_filename + " " + self.vm_conf_data_filename + " 1 > " + vm_dir + "/stdout 2> " + vm_dir + "/stderr < /dev/null & echo -n $!") From 3fc36d73a21ea7695f7d36e2bdae69132c9f5db2 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 27 Nov 2024 09:46:01 +0100 Subject: [PATCH 09/14] Increase timeout --- test/integration/TestIM.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/TestIM.py b/test/integration/TestIM.py index 2cbba369..3e666a81 100755 --- a/test/integration/TestIM.py +++ b/test/integration/TestIM.py @@ -154,7 +154,7 @@ def test_11_create(self): self.__class__.inf_id = inf_id all_configured = self.wait_inf_state( - inf_id, VirtualMachine.CONFIGURED, 2400) + inf_id, VirtualMachine.CONFIGURED, 2700) self.assertTrue( all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") From 98de1878586c3b8fb6b06ea8342c3aa8180d9260 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 28 Nov 2024 08:19:39 +0100 Subject: [PATCH 10/14] Fix XiB units --- IM/connectors/OSCAR.py | 4 ++-- requirements-tests.txt | 2 +- test/unit/connectors/Fogbow.py | 2 +- test/unit/connectors/GCE.py | 2 +- test/unit/connectors/Kubernetes.py | 6 +++--- test/unit/connectors/OCCI.py | 2 +- test/unit/connectors/OSCAR.py | 6 +++--- test/unit/connectors/OpenNebula.py | 2 +- test/unit/test_im_logic.py | 4 ++-- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/IM/connectors/OSCAR.py b/IM/connectors/OSCAR.py index d7dfc3ba..f5d17603 100644 --- a/IM/connectors/OSCAR.py +++ b/IM/connectors/OSCAR.py @@ -134,7 +134,7 @@ def _get_service_json(radl_system): if radl_system.getValue("name"): service["name"] = radl_system.getValue("name") if radl_system.getValue("memory.size"): - service["memory"] = "%dMi" % radl_system.getFeature('memory.size').getValue('M') + service["memory"] = "%dMi" % radl_system.getFeature('memory.size').getValue('Mi') if radl_system.getValue("cpu.count"): service["cpu"] = "%g" % radl_system.getValue("cpu.count") if radl_system.getValue("gpu.count"): @@ -260,7 +260,7 @@ def update_system_info_from_service_info(self, system, service_info): conflict="other", missing="other") if "memory" in service_info and service_info["memory"]: memory = self.convert_memory_unit(service_info["memory"], "Mi") - system.addFeature(Feature("memory.size", "=", memory, "M"), + system.addFeature(Feature("memory.size", "=", memory, "Mi"), conflict="other", missing="other") if "script" in service_info and service_info["script"]: system.addFeature(Feature("script", "=", service_info["script"]), diff --git a/requirements-tests.txt b/requirements-tests.txt index 456356e3..aaddb89f 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,7 +5,7 @@ PyYAML cheroot boto3 apache-libcloud >= 3.3.1 -RADL >= 1.3.3 +RADL >= 1.3.4 flask werkzeug netaddr diff --git a/test/unit/connectors/Fogbow.py b/test/unit/connectors/Fogbow.py index ad950027..0625d2ea 100755 --- a/test/unit/connectors/Fogbow.py +++ b/test/unit/connectors/Fogbow.py @@ -300,7 +300,7 @@ def test_30_updateVMInfo(self, sleep, requests): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.assertEqual(vm.info.systems[0].getValue("net_interface.1.ip"), "10.0.0.1") self.assertEqual(vm.info.systems[0].getValue("net_interface.0.ip"), "8.8.8.8") - self.assertEqual(vm.info.systems[0].getValue("memory.size"), 1073741824) + self.assertEqual(vm.info.systems[0].getValue("memory.size"), 1024000000) self.assertEqual(vm.info.systems[0].getValue("disk.1.device"), "/dev/sdb") data = json.loads(requests.call_args_list[1][1]["data"]) diff --git a/test/unit/connectors/GCE.py b/test/unit/connectors/GCE.py index fd8524ea..a9d976aa 100755 --- a/test/unit/connectors/GCE.py +++ b/test/unit/connectors/GCE.py @@ -129,7 +129,7 @@ def test_10_concrete(self, get_driver): concrete = gce_cloud.concreteSystem(radl_system, auth) self.assertEqual(len(concrete), 1) - self.assertEqual(concrete[0].getValue("memory.size"), 2147483648) + self.assertEqual(concrete[0].getValue("memory.size"), 2048000000) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('libcloud.compute.drivers.gce.GCENodeDriver') diff --git a/test/unit/connectors/Kubernetes.py b/test/unit/connectors/Kubernetes.py index 3e8667d7..742c7836 100755 --- a/test/unit/connectors/Kubernetes.py +++ b/test/unit/connectors/Kubernetes.py @@ -188,7 +188,7 @@ def test_20_launch(self, save_data, requests): 'labels': {'name': 'test-1'}}, "spec": { "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": 10737418240}}, + "resources": {"requests": {"storage": 10000000000}}, }, } self.assertEqual(requests.call_args_list[1][0][1], @@ -237,8 +237,8 @@ def test_20_launch(self, save_data, requests): "imagePullPolicy": "Always", "ports": [{"containerPort": 8080, "protocol": "TCP"}], "resources": { - "limits": {"cpu": "1", "memory": "536870912"}, - "requests": {"cpu": "1", "memory": "536870912"}, + "limits": {"cpu": "1", "memory": "512000000"}, + "requests": {"cpu": "1", "memory": "512000000"}, }, "env": [{"name": "var", "value": "some_val"}, {"name": "var2", "value": "some,val2"}], diff --git a/test/unit/connectors/OCCI.py b/test/unit/connectors/OCCI.py index 5a967aed..e07e10d3 100755 --- a/test/unit/connectors/OCCI.py +++ b/test/unit/connectors/OCCI.py @@ -327,7 +327,7 @@ def test_30_updateVMInfo(self, get_keystone_uri, requests): self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) memory = vm.info.systems[0].getValue("memory.size") - self.assertEqual(memory, 1824522240) + self.assertEqual(memory, 1740000000) @patch('requests.request') @patch('IM.connectors.OCCI.KeyStoneAuth.get_keystone_uri') diff --git a/test/unit/connectors/OSCAR.py b/test/unit/connectors/OSCAR.py index 9c4a4a83..fa4715a6 100755 --- a/test/unit/connectors/OSCAR.py +++ b/test/unit/connectors/OSCAR.py @@ -124,7 +124,7 @@ def test_20_launch(self, save_data, requests): radl_data = """ system test ( name = 'plants' and - memory.size = 2G and + memory.size = 2GI and cpu.count = 1.0 and cpu.sgx = 1 and gpu.count = 1 and @@ -288,7 +288,7 @@ def test_55_alter(self, requests): new_radl_data = """ system test ( cpu.count>=2 and - memory.size>=4G + memory.size>=4GI )""" new_radl = radl_parse.parse_radl(new_radl_data) @@ -306,7 +306,7 @@ def test_55_alter(self, requests): self.assertTrue(success, msg="ERROR: modifying VM info.") self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) self.assertEqual(new_vm.info.systems[0].getValue("cpu.count"), 2) - self.assertEqual(new_vm.info.systems[0].getFeature("memory.size").getValue("M"), 4096) + self.assertEqual(new_vm.info.systems[0].getFeature("memory.size").getValue("M"), 4295) self.assertEqual(requests.call_args_list[0][0][0], "PUT") self.assertEqual(requests.call_args_list[0][0][1], "http://oscar.com:80/system/services/fname") self.assertEqual(json.loads(requests.call_args_list[0][1]['data']), {'memory': '4096Mi', 'cpu': '2'}) diff --git a/test/unit/connectors/OpenNebula.py b/test/unit/connectors/OpenNebula.py index 9c897651..d73fad01 100755 --- a/test/unit/connectors/OpenNebula.py +++ b/test/unit/connectors/OpenNebula.py @@ -144,7 +144,7 @@ def test_20_launch(self, save_data, getONEVersion, server_proxy): OS = [ ARCH = "x86_64" ] DISK = [ IMAGE_ID = "1" ] - DISK = [ SAVE = no, TYPE = fs , FORMAT = qcow2, SIZE = 1024, TARGET = hdb ] + DISK = [ SAVE = no, TYPE = fs , FORMAT = qcow2, SIZE = 1000, TARGET = hdb ] SCHED_REQUIREMENTS = "CLUSTER_ID=\\"0\\""\n""" diff --git a/test/unit/test_im_logic.py b/test/unit/test_im_logic.py index 2eea95f9..68ce6315 100644 --- a/test/unit/test_im_logic.py +++ b/test/unit/test_im_logic.py @@ -1552,8 +1552,8 @@ def test_estimate_resources(self): 'cloud0': { 'cloudType': 'Dummy', 'cloudEndpoint': 'http://server.com:80/path', - 'compute': [{'cpuCores': 2, 'memoryInMegabytes': 4096, 'diskSizeInGigabytes': 40}, - {'cpuCores': 1, 'memoryInMegabytes': 2048, 'diskSizeInGigabytes': 10}], + 'compute': [{'cpuCores': 2, 'memoryInMegabytes': 4000, 'diskSizeInGigabytes': 40}, + {'cpuCores': 1, 'memoryInMegabytes': 2000, 'diskSizeInGigabytes': 10}], 'storage': [{'sizeInGigabytes': 100}] }}) From 6bf3386792b0f8a502eaf807de777771b79d18b3 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 28 Nov 2024 08:34:53 +0100 Subject: [PATCH 11/14] Fix test --- test/unit/REST.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/REST.py b/test/unit/REST.py index 3b723a9b..f7c8f9ed 100755 --- a/test/unit/REST.py +++ b/test/unit/REST.py @@ -228,8 +228,8 @@ def test_CreateInfrastructure(self, get_infrastructure, CreateInfrastructure): data=read_file_as_bytes("../files/test_simple.json")) self.assertEqual(res.json, {"one": {"cloudType": "OpenNebula", "cloudEndpoint": "http://ramses.i3m.upv.es:2633", - "compute": [{"cpuCores": 1, "memoryInMegabytes": 1024}, - {"cpuCores": 1, "memoryInMegabytes": 1024}], "storage": []}}) + "compute": [{"cpuCores": 1, "memoryInMegabytes": 1074}, + {"cpuCores": 1, "memoryInMegabytes": 1074}], "storage": []}}) headers["Content-Type"] = "application/json" CreateInfrastructure.side_effect = InvaliddUserException() From 5a980938b44b4542b398d946e51869472dd16246 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 28 Nov 2024 09:01:09 +0100 Subject: [PATCH 12/14] Change python ver in tests --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index dfb7eade..b0b202d4 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -16,7 +16,7 @@ jobs: - name: Set up Python 3. uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - name: Install dependencies run: python -m pip install tox From 348badfc57704d7afaf7245a34a11ac1014579c2 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 28 Nov 2024 09:23:02 +0100 Subject: [PATCH 13/14] Change ansible ver in tests --- requirements-tests.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index aaddb89f..2c8c9ae9 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,5 +1,4 @@ -ansible >= 2.4 -ansible-base +ansible == 8.7.0 paramiko >= 1.14 PyYAML cheroot From 3ce577df4250b7cc83586a4e7c1c057840a6ecdc Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 28 Nov 2024 09:26:23 +0100 Subject: [PATCH 14/14] Add nosec --- IM/CtxtAgentBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IM/CtxtAgentBase.py b/IM/CtxtAgentBase.py index a88b0953..a38d4487 100644 --- a/IM/CtxtAgentBase.py +++ b/IM/CtxtAgentBase.py @@ -37,7 +37,7 @@ class CtxtAgentBase: # the ConfManager PLAYBOOK_RETRIES = 1 INTERNAL_PLAYBOOK_RETRIES = 1 - VENV_DIR = "/var/tmp/.ansible" + VENV_DIR = "/var/tmp/.ansible" # nosec def __init__(self, conf_data_filename): self.logger = None