From 414658f806c3a8cb79d7330bef16107a4e17a59f Mon Sep 17 00:00:00 2001 From: Gonzalo Rafuls Date: Thu, 31 Oct 2024 21:17:06 +0100 Subject: [PATCH] fix: lshw, templates, conf var removal * fix lshw not working * fix lshw2meta interface update * fix wiki_url conf removal * fix tests for templates for new url closes: https://github.com/redhat-performance/quads/issues/542 closes: https://github.com/redhat-performance/quads/issues/543 closes: https://github.com/redhat-performance/quads/issues/544 Change-Id: Ia0155091c2ade7de9f86fc47a71cb9ac8a4f1173 --- conf/quads.yml | 5 +- src/quads/quads_api.py | 3 + src/quads/server/blueprints/disks.py | 4 +- src/quads/templates/future_initial_message | 4 +- src/quads/templates/future_message | 6 +- src/quads/templates/initial_message | 6 +- src/quads/templates/message | 6 +- src/quads/tools/lshw.py | 10 +- src/quads/tools/lshw2meta.py | 116 ++++++++++++++------- src/quads/tools/notify.py | 11 +- tests/config.py | 10 +- tests/tools/test_lshw.py | 5 +- 12 files changed, 112 insertions(+), 74 deletions(-) mode change 100644 => 100755 src/quads/tools/lshw.py diff --git a/conf/quads.yml b/conf/quads.yml index 50f4ea1b7..01bf045f9 100644 --- a/conf/quads.yml +++ b/conf/quads.yml @@ -44,9 +44,6 @@ ircbot_channel: #yourchannel webhook_notify: false webhook_url: https://chat.example.com/v1/spaces/AAABBBCCC -# URL for the wiki references -wiki_url: https://wiki.example.com - # (optional ticket system URL) in this case we're using JIRA ticket_url: https://issues.example.com/browse # (optional ticket queue name) this is typically the ticket queue @@ -65,7 +62,7 @@ jira_password: password jira_token: YourJiraTokenHereKeepItSafe # this is used in some of the wiki generate for the links -quads_url: http://quads.scalelab.example.com +quads_url: https://quads.scalelab.example.com # url for a request form if you use one # e.g. http://scalelab.example.com quads_request_url: diff --git a/src/quads/quads_api.py b/src/quads/quads_api.py index 7f108d97e..37a7a7f74 100644 --- a/src/quads/quads_api.py +++ b/src/quads/quads_api.py @@ -346,6 +346,9 @@ def remove_memory(self, memory_id) -> Response: def create_disk(self, hostname, data) -> Response: return self.post(os.path.join("disks", hostname), data) + def update_disk(self, hostname, data) -> Response: + return self.patch(os.path.join("disks", hostname), data) + def remove_disk(self, hostname, disk_id) -> Response: return self.delete(os.path.join("disks", hostname), {"id": disk_id}) diff --git a/src/quads/server/blueprints/disks.py b/src/quads/server/blueprints/disks.py index 25a1d971d..b3ecbfccc 100644 --- a/src/quads/server/blueprints/disks.py +++ b/src/quads/server/blueprints/disks.py @@ -124,7 +124,7 @@ def update_disk(hostname: str) -> Response: if value: update_fields[key] = value - if update_fields.get("count") and update_fields.get("count") <= 0: + if update_fields.get("count") and int(update_fields.get("count")) <= 0: response = { "status_code": 400, "error": "Bad Request", @@ -132,7 +132,7 @@ def update_disk(hostname: str) -> Response: } return make_response(jsonify(response), 400) - if update_fields.get("size_gb") and update_fields.get("size_gb") <= 0: + if update_fields.get("size_gb") and int(update_fields.get("size_gb")) <= 0: response = { "status_code": 400, "error": "Bad Request", diff --git a/src/quads/templates/future_initial_message b/src/quads/templates/future_initial_message index dbd3f6369..87d39c88a 100644 --- a/src/quads/templates/future_initial_message +++ b/src/quads/templates/future_initial_message @@ -23,7 +23,7 @@ are in their final, desired state before being ready to use. For additional information regarding the Scale Lab usage please see the following documentation: -{{ wiki_url }}/faq/ -{{ wiki_url }}/usage/ +{{ quads_url }}/FAQ/ +{{ quads_url }}/Usage/ Perf/Scale DevOps Team diff --git a/src/quads/templates/future_message b/src/quads/templates/future_message index 0c25e5fde..be7992b0a 100644 --- a/src/quads/templates/future_message +++ b/src/quads/templates/future_message @@ -5,7 +5,7 @@ your allocated environment: {{ cloud_info }} (Details) -{{ wiki_url }}/assignments/#{{ cloud }} +{{ quads_url }}/assignments/#{{ cloud }} will change. As host schedules are activated some hosts will automatically be reprovisioned and moved to @@ -22,8 +22,8 @@ hosts: For additional information regarding the Scale Lab usage please see the following documentation: -{{ wiki_url }}/faq/ -{{ wiki_url }}/usage/ +{{ quads_url }}/FAQ/ +{{ quads_url }}/Usage/ Thank you for your attention. diff --git a/src/quads/templates/initial_message b/src/quads/templates/initial_message index ff8099202..b546add94 100644 --- a/src/quads/templates/initial_message +++ b/src/quads/templates/initial_message @@ -6,7 +6,7 @@ You've been allocated a new environment! {{ cloud_info }} (Details) -{{ wiki_url }}/assignments/#{{ cloud }} +{{ quads_url }}/assignments/#{{ cloud }} You can view your machine list, duration and other details above. @@ -21,7 +21,7 @@ Password: {{ password }} For additional information regarding system usage please see the following documentation: -{{ wiki_url }}/faq/ -{{ wiki_url }}/usage/ +{{ quads_url }}/FAQ/ +{{ quads_url }}/Usage/ Perf/Scale DevOps Team diff --git a/src/quads/templates/message b/src/quads/templates/message index af0714e06..a1d4a5bcb 100644 --- a/src/quads/templates/message +++ b/src/quads/templates/message @@ -5,7 +5,7 @@ your allocated environment: {{ cloud_info }} (Details) -{{ wiki_url }}/assignments/#{{ cloud }} +{{ quads_url }}/assignments/#{{ cloud }} will have some or all of the hosts expire. The following hosts will automatically be reprovisioned and returned to @@ -40,8 +40,8 @@ notices until the actual extension is executed. Docs: -{{ wiki_url }}/faq/ -{{ wiki_url }}/usage/ +{{ quads_url }}/FAQ/ +{{ quads_url }}/Usage/ Thank you for your attention. diff --git a/src/quads/tools/lshw.py b/src/quads/tools/lshw.py old mode 100644 new mode 100755 index 867e1e73a..cd1bea136 --- a/src/quads/tools/lshw.py +++ b/src/quads/tools/lshw.py @@ -21,9 +21,9 @@ def run_lshw(hostname: str, file_path: str) -> None: except Exception: print(f"Something went wrong trying to connect to: {hostname}") return - _, output = ssh_helper.run_cmd("lshw -xml") + _, output = ssh_helper.run_cmd("lshw -json") if output: - with open(file_path, "w") as _file: + with open(file_path, "w+") as _file: for line in output: _file.writelines(line) @@ -36,13 +36,13 @@ def main() -> None: cloud = quads.get_cloud("cloud01") hosts = quads.filter_hosts({"cloud": cloud.name, "retired": False, "broken": False}) for host in hosts: - file_name = f"{host.name}.xml" + file_name = f"{host.name}.json" file_path = os.path.join(LSHW_OUTPUT_DIR, file_name) if os.path.exists(file_path): if os.path.getsize(file_path) < 1: run_lshw(host.name, file_path) - else: - run_lshw(host.name, file_path) + else: + run_lshw(host.name, file_path) if __name__ == "__main__": # pragma: no cover diff --git a/src/quads/tools/lshw2meta.py b/src/quads/tools/lshw2meta.py index 1c870b5ee..dbe846de6 100755 --- a/src/quads/tools/lshw2meta.py +++ b/src/quads/tools/lshw2meta.py @@ -28,9 +28,14 @@ def b2g(num, metric=False): # pragma: no cover path, extension = os.path.splitext(filename) if extension == ".json": with open(filename) as _f: - data = json.load(_f) + try: + data = json.load(_f) + except json.JSONDecodeError: + print(f"Error decoding: {filename}") + continue children = parse("$..children[*]").find(data) hostname = parse("$.id").find(data)[0].value + print(f"Processing: {hostname}") host_obj = quads.get_host(hostname) if not host_obj: print(f"Host not found: {hostname}") @@ -50,41 +55,69 @@ def b2g(num, metric=False): # pragma: no cover if speed: speed = int("".join(filter(str.isdigit, speed))) host_interface.speed = speed - host_obj.save() + interface = quads.update_interface( + hostname, host_interface.as_dict() + ) + print( + f" Updated interface: {host_interface.as_dict()}" + ) # disks - for child in [ - child for child in children if child.value["class"] == "disk" - ]: - if child.value.get("size"): + disk_nodes = [ + node.context.value + for node in parse("$..class").find(data) + if node.value == "disk" + ] + disks = {} + for child in disk_nodes: + if child.get("size"): disk_type = None for dt, sub in DISK_TYPES.items(): - if child.value["description"].lower().startswith(sub): + if child.get("description").lower().startswith(sub): disk_type = dt - disk_size = b2g(int(child.value["size"]), True) - filters = { - "name": host_obj.name, - "disks__disk_type": disk_type, - "disks__size_gb": disk_size, + break + disk_size = b2g(int(child.get("size")), True) + disks[f"{disk_type}|{str(disk_size)}"] = ( + disks.get(f"{disk_type}|{str(disk_size)}", 0) + 1 + ) + + for key, count in disks.items(): + disk_type, disk_size = key.split("|") + filters = { + "name": host_obj.name, + "disks.disk_type": disk_type, + "disks.size_gb": disk_size, + } + host = quads.filter_hosts(filters) + if host: + for disk in host[0].disks: + if ( + disk.disk_type == disk_type + and disk.size_gb == int(disk_size) + ): + if disk.count != count: + data = { + "disk_id": disk.id, + "disk_type": disk_type, + "size_gb": disk_size, + "count": count, + } + quads.update_disk(host_obj.name, data) + print(f" Updated disk: {data}") + else: + print(f" Disk already exists: {disk_type, disk_size, count}") + break + else: + data = { + "disk_type": disk_type, + "size_gb": disk_size, + "count": count, } - host = quads.filter_hosts(filters) - if host: - for disk in host[0].disks: - if ( - disk.disk_type == disk_type - and disk.disk_size == disk_size - ): - disk.count += 1 - else: - data = { - "disk_type": disk_type, - "size_gb": disk_size, - "count": 1, - } - disk = quads.create_disk(host_obj.name, data) + disk = quads.create_disk(host_obj.name, data) + print(f" Created disk: {data}") # memory for memory in host_obj.memory: - quads.remove_memory(memory.id) + quads.remove_memory(str(memory.id)) for child in [ child for child in children @@ -104,24 +137,27 @@ def b2g(num, metric=False): # pragma: no cover hostname, data, ) + print(f" Created memory: {data}") # processor for processor in host_obj.processors: - quads.remove_processor(processor.id) + quads.remove_processor(str(processor.id)) for child in [ child for child in children if child.value["class"] == "processor" ]: configuration = child.value.get("configuration") - data = { - "handle": child.value.get("handle"), - "vendor": child.value.get("vendor"), - "product": child.value.get("product"), - "cores": int(configuration.get("cores", 0)), - "threads": int(configuration.get("threads", 0)), - } - processor = quads.create_processor( - hostname, - data, - ) + if configuration.get("cores") and configuration.get("threads"): + data = { + "handle": child.value.get("handle"), + "vendor": child.value.get("vendor"), + "product": child.value.get("product"), + "cores": int(configuration.get("cores")), + "threads": int(configuration.get("threads")), + } + processor = quads.create_processor( + hostname, + data, + ) + print(f" Created processor: {data}") diff --git a/src/quads/tools/notify.py b/src/quads/tools/notify.py index 0a78b1e04..133d39266 100755 --- a/src/quads/tools/notify.py +++ b/src/quads/tools/notify.py @@ -41,7 +41,6 @@ async def create_initial_message(real_owner, cloud, cloud_info, ticket, cc): template = Template(_file.read()) content = template.render( cloud_info=cloud_info, - wiki_url=Config["wiki_url"], cloud=cloud, quads_url=Config["quads_url"], real_owner=real_owner, @@ -63,7 +62,7 @@ async def create_initial_message(real_owner, cloud, cloud_info, ticket, cc): message = "%s QUADS: %s is now active, choo choo! - %s/assignments/#%s - %s %s" % ( irc_bot_channel, cloud_info, - Config["wiki_url"], + Config["quads_url"], cloud, real_owner, Config["report_cc"], @@ -77,7 +76,7 @@ async def create_initial_message(real_owner, cloud, cloud_info, ticket, cc): try: message = "QUADS: %s is now active, choo choo! - %s/assignments/#%s - %s %s" % ( cloud_info, - Config["wiki_url"], + Config["quads_url"], cloud, real_owner, Config["report_cc"], @@ -113,7 +112,7 @@ def create_message( content = template.render( days_to_report=day, cloud_info=cloud_info, - wiki_url=Config["wiki_url"], + quads_url=Config["quads_url"], quads_request_url=quads_request_url, quads_request_deadline_day=Config["quads_request_deadline_day"], quads_notify_until_extended=Config["quads_notify_until_extended"], @@ -139,7 +138,7 @@ def create_future_initial_message(cloud, assignment_obj, cloud_info): template = Template(_file.read()) content = template.render( cloud_info=cloud_info, - wiki_url=Config["wiki_url"], + quads_url=Config["quads_url"], ) postman = Postman( "New QUADS Assignment Defined for the Future: %s - %s" % (cloud, ticket), @@ -167,7 +166,7 @@ def create_future_message( content = template.render( days_to_report=future_days, cloud_info=cloud_info, - wiki_url=Config["wiki_url"], + quads_url=Config["quads_url"], cloud=cloud, hosts=host_list_expire, ) diff --git a/tests/config.py b/tests/config.py index c986d9a67..a0aa82e7e 100644 --- a/tests/config.py +++ b/tests/config.py @@ -632,16 +632,16 @@ "build_start": f"{start_str}T22:00", "build_end": f"{build_end_str}T22:00", } -INITIAL_MESSAGE = "\nGreetings Citizen,\n\nYou've been allocated a new environment!\n\ncloud_info1\n\n(Details)\nhttps://wiki.example.com/assignments/#cloud1\n\nYou can view your machine list, duration and other\ndetails above.\n\nYou can also view/manage your hosts via Foreman:\n\nhttp://foreman.example.com/hosts/\n\nUsername: cloud1\nPassword: rdu2@ticket1\n\nFor additional information regarding system usage\nplease see the following documentation:\n\nhttps://wiki.example.com/faq/\nhttps://wiki.example.com/usage/\n\nPerf/Scale DevOps Team" +INITIAL_MESSAGE = "\nGreetings Citizen,\n\nYou've been allocated a new environment!\n\ncloud_info1\n\n(Details)\nhttps://quads.scalelab.example.com/assignments/#cloud1\n\nYou can view your machine list, duration and other\ndetails above.\n\nYou can also view/manage your hosts via Foreman:\n\nhttp://foreman.example.com/hosts/\n\nUsername: cloud1\nPassword: rdu2@ticket1\n\nFor additional information regarding system usage\nplease see the following documentation:\n\nhttps://quads.scalelab.example.com/FAQ/\nhttps://quads.scalelab.example.com/Usage/\n\nPerf/Scale DevOps Team" INITIAL_SUBJECT = "New QUADS Assignment Allocated - cloud1 ticket1" -FUTURE_INITIAL_MESSAGE = "\nGreetings Citizen,\n\nYou've been allocated a new future environment! The environment\nis not yet ready for use but you are being notified ahead of time\nthat it is being prepared.\n\ncloud_info1\n\nWhen your environment is ready and your hardware/network has\npassed automated validation it will be released to you.\n\nYou'll receive another email with environment-specific\ninformation and additional system details when it's ready.\n\nBefore this happens a future schedule will be entered and\ncommunicated to you via your service ticket/request.\n\nAfterwards, when the time comes a series of automated tests and\nvalidation phases will occur to make sure your systems/network\nare in their final, desired state before being ready to use.\n\nFor additional information regarding the Scale Lab usage\nplease see the following documentation:\n\nhttps://wiki.example.com/faq/\nhttps://wiki.example.com/usage/\n\nPerf/Scale DevOps Team" +FUTURE_INITIAL_MESSAGE = "\nGreetings Citizen,\n\nYou've been allocated a new future environment! The environment\nis not yet ready for use but you are being notified ahead of time\nthat it is being prepared.\n\ncloud_info1\n\nWhen your environment is ready and your hardware/network has\npassed automated validation it will be released to you.\n\nYou'll receive another email with environment-specific\ninformation and additional system details when it's ready.\n\nBefore this happens a future schedule will be entered and\ncommunicated to you via your service ticket/request.\n\nAfterwards, when the time comes a series of automated tests and\nvalidation phases will occur to make sure your systems/network\nare in their final, desired state before being ready to use.\n\nFor additional information regarding the Scale Lab usage\nplease see the following documentation:\n\nhttps://quads.scalelab.example.com/FAQ/\nhttps://quads.scalelab.example.com/Usage/\n\nPerf/Scale DevOps Team" FUTURE_INITIAL_SUBJECT = "New QUADS Assignment Defined for the Future: cloud1 - ticket1" -FUTURE_MESSAGE = "\nThis is a message to alert you that in 1 days\nyour allocated environment:\n\ncloud_info1\n\n(Details)\nhttps://wiki.example.com/assignments/#cloud1\n\nwill change. As host schedules are activated some\nhosts will automatically be reprovisioned and moved to\nyour environment. This could also mean that one or more\nhosts will evacuate your environment ahead of others.\n\nQUADS has detected a change in schedule to the following\nhosts:\n\n\nhost1\n\nhost2\n\n\nFor additional information regarding the Scale Lab usage\nplease see the following documentation:\n\nhttps://wiki.example.com/faq/\nhttps://wiki.example.com/usage/\n\nThank you for your attention.\n\nPerf/Scale DevOps Team" +FUTURE_MESSAGE = "\nThis is a message to alert you that in 1 days\nyour allocated environment:\n\ncloud_info1\n\n(Details)\nhttps://quads.scalelab.example.com/assignments/#cloud1\n\nwill change. As host schedules are activated some\nhosts will automatically be reprovisioned and moved to\nyour environment. This could also mean that one or more\nhosts will evacuate your environment ahead of others.\n\nQUADS has detected a change in schedule to the following\nhosts:\n\n\nhost1\n\nhost2\n\n\nFor additional information regarding the Scale Lab usage\nplease see the following documentation:\n\nhttps://quads.scalelab.example.com/FAQ/\nhttps://quads.scalelab.example.com/Usage/\n\nThank you for your attention.\n\nPerf/Scale DevOps Team" FUTURE_SUBJECT = "QUADS upcoming assignment notification - cloud1 - ticket1" -MESSAGE = "\nThis is a message to alert you that in 1 days\nyour allocated environment:\n\ncloud_info1\n\n(Details)\nhttps://wiki.example.com/assignments/#cloud1\n\nwill have some or all of the hosts expire. The following\nhosts will automatically be reprovisioned and returned to\nthe pool of available hosts.\n\n\nhost1\n\nhost2\n\n\n\n\n\n\n\n\nIf you have already submitted an extension you can disregard\nthis message, our system will continue to send out expiration\nnotices until the actual extension is executed.\n\n\n\nDocs:\n\nhttps://wiki.example.com/faq/\nhttps://wiki.example.com/usage/\n\nThank you for your attention.\n\nPerf/Scale DevOps Team\n" +MESSAGE = "\nThis is a message to alert you that in 1 days\nyour allocated environment:\n\ncloud_info1\n\n(Details)\nhttps://quads.scalelab.example.com/assignments/#cloud1\n\nwill have some or all of the hosts expire. The following\nhosts will automatically be reprovisioned and returned to\nthe pool of available hosts.\n\n\nhost1\n\nhost2\n\n\n\n\n\n\n\n\nIf you have already submitted an extension you can disregard\nthis message, our system will continue to send out expiration\nnotices until the actual extension is executed.\n\n\n\nDocs:\n\nhttps://quads.scalelab.example.com/FAQ/\nhttps://quads.scalelab.example.com/Usage/\n\nThank you for your attention.\n\nPerf/Scale DevOps Team\n" SUBJECT = "QUADS upcoming expiration for cloud1 - ticket1" CC_USERS = [ @@ -651,4 +651,4 @@ "someuser@example.com", ] OWNER = "owner1" -POST_TEXT = "QUADS: cloud_info1 is now active, choo choo! - https://wiki.example.com/assignments/#cloud1 - owner1 someuser@example.com, someuser@example.com, someuser@example.com, someuser@example.com" +POST_TEXT = "QUADS: cloud_info1 is now active, choo choo! - https://quads.scalelab.example.com/assignments/#cloud1 - owner1 someuser@example.com, someuser@example.com, someuser@example.com, someuser@example.com" diff --git a/tests/tools/test_lshw.py b/tests/tools/test_lshw.py index 8d7544c2b..efc1c5311 100644 --- a/tests/tools/test_lshw.py +++ b/tests/tools/test_lshw.py @@ -57,10 +57,13 @@ def test_run_lshw_without_output(self, mock_client, mock_config): run_lshw("host.example.com", temp_filepath) assert not os.path.exists(temp_filepath) + @patch( + "quads.tools.lshw.LSHW_OUTPUT_DIR", new_callable=lambda: os.path.join(os.path.dirname(__file__), "artifacts") + ) @patch("quads.tools.external.ssh_helper.SSHConfig") @patch("quads.tools.external.ssh_helper.SSHClient") @pytest.mark.asyncio - def test_main(self, mock_client, mock_config): + def test_main(self, mock_client, mock_config, mock_dir): mock_config.return_value = MagicMock() mock_client.return_value.exec_command.return_value = [ MagicMock(),