Skip to content

Commit

Permalink
fix: lshw, templates, conf var removal
Browse files Browse the repository at this point in the history 
* fix lshw not working
* fix lshw2meta interface update
* fix wiki_url conf removal
* fix tests for templates for new url

closes: #542
closes: #543
closes: #544

Change-Id: Ia0155091c2ade7de9f86fc47a71cb9ac8a4f1173
  • Loading branch information
@grafuls @sadsfae
grafuls authored and sadsfae committed Nov 1, 2024
1 parent ce5b39e commit 414658f
Show file tree
Hide file tree
Showing 12 changed files with 112 additions and 74 deletions.
5 changes: 1 addition & 4 deletions conf/quads.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,6 @@ ircbot_channel: #yourchannel
webhook_notify: false
webhook_url: https://chat.example.com/v1/spaces/AAABBBCCC

# URL for the wiki references
wiki_url: https://wiki.example.com

# (optional ticket system URL) in this case we're using JIRA
ticket_url: https://issues.example.com/browse
# (optional ticket queue name) this is typically the ticket queue
Expand All @@ -65,7 +62,7 @@ jira_password: password
jira_token: YourJiraTokenHereKeepItSafe

# this is used in some of the wiki generate for the links
quads_url: http://quads.scalelab.example.com
quads_url: https://quads.scalelab.example.com
# url for a request form if you use one
# e.g. http://scalelab.example.com
quads_request_url:
Expand Down
3 changes: 3 additions & 0 deletions src/quads/quads_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,9 @@ def remove_memory(self, memory_id) -> Response:
def create_disk(self, hostname, data) -> Response:
return self.post(os.path.join("disks", hostname), data)

def update_disk(self, hostname, data) -> Response:
return self.patch(os.path.join("disks", hostname), data)

def remove_disk(self, hostname, disk_id) -> Response:
return self.delete(os.path.join("disks", hostname), {"id": disk_id})

Expand Down
4 changes: 2 additions & 2 deletions src/quads/server/blueprints/disks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,15 +124,15 @@ def update_disk(hostname: str) -> Response:
if value:
update_fields[key] = value

if update_fields.get("count") and update_fields.get("count") <= 0:
if update_fields.get("count") and int(update_fields.get("count")) <= 0:
response = {
"status_code": 400,
"error": "Bad Request",
"message": "Argument can't be negative or zero: count",
}
return make_response(jsonify(response), 400)

if update_fields.get("size_gb") and update_fields.get("size_gb") <= 0:
if update_fields.get("size_gb") and int(update_fields.get("size_gb")) <= 0:
response = {
"status_code": 400,
"error": "Bad Request",
Expand Down
4 changes: 2 additions & 2 deletions src/quads/templates/future_initial_message
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ are in their final, desired state before being ready to use.
For additional information regarding the Scale Lab usage
please see the following documentation:

{{ wiki_url }}/faq/
{{ wiki_url }}/usage/
{{ quads_url }}/FAQ/
{{ quads_url }}/Usage/

Perf/Scale DevOps Team
6 changes: 3 additions & 3 deletions src/quads/templates/future_message
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ your allocated environment:
{{ cloud_info }}

(Details)
{{ wiki_url }}/assignments/#{{ cloud }}
{{ quads_url }}/assignments/#{{ cloud }}

will change. As host schedules are activated some
hosts will automatically be reprovisioned and moved to
Expand All @@ -22,8 +22,8 @@ hosts:
For additional information regarding the Scale Lab usage
please see the following documentation:

{{ wiki_url }}/faq/
{{ wiki_url }}/usage/
{{ quads_url }}/FAQ/
{{ quads_url }}/Usage/

Thank you for your attention.

Expand Down
6 changes: 3 additions & 3 deletions src/quads/templates/initial_message
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ You've been allocated a new environment!
{{ cloud_info }}

(Details)
{{ wiki_url }}/assignments/#{{ cloud }}
{{ quads_url }}/assignments/#{{ cloud }}

You can view your machine list, duration and other
details above.
Expand All @@ -21,7 +21,7 @@ Password: {{ password }}
For additional information regarding system usage
please see the following documentation:

{{ wiki_url }}/faq/
{{ wiki_url }}/usage/
{{ quads_url }}/FAQ/
{{ quads_url }}/Usage/

Perf/Scale DevOps Team
6 changes: 3 additions & 3 deletions src/quads/templates/message
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ your allocated environment:
{{ cloud_info }}

(Details)
{{ wiki_url }}/assignments/#{{ cloud }}
{{ quads_url }}/assignments/#{{ cloud }}

will have some or all of the hosts expire. The following
hosts will automatically be reprovisioned and returned to
Expand Down Expand Up @@ -40,8 +40,8 @@ notices until the actual extension is executed.

Docs:

{{ wiki_url }}/faq/
{{ wiki_url }}/usage/
{{ quads_url }}/FAQ/
{{ quads_url }}/Usage/

Thank you for your attention.

Expand Down
10 changes: 5 additions & 5 deletions src/quads/tools/lshw.py
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ def run_lshw(hostname: str, file_path: str) -> None:
except Exception:
print(f"Something went wrong trying to connect to: {hostname}")
return
_, output = ssh_helper.run_cmd("lshw -xml")
_, output = ssh_helper.run_cmd("lshw -json")
if output:
with open(file_path, "w") as _file:
with open(file_path, "w+") as _file:
for line in output:
_file.writelines(line)

Expand All @@ -36,13 +36,13 @@ def main() -> None:
cloud = quads.get_cloud("cloud01")
hosts = quads.filter_hosts({"cloud": cloud.name, "retired": False, "broken": False})
for host in hosts:
file_name = f"{host.name}.xml"
file_name = f"{host.name}.json"
file_path = os.path.join(LSHW_OUTPUT_DIR, file_name)
if os.path.exists(file_path):
if os.path.getsize(file_path) < 1:
run_lshw(host.name, file_path)
else:
run_lshw(host.name, file_path)
else:
run_lshw(host.name, file_path)


if __name__ == "__main__": # pragma: no cover
Expand Down
116 changes: 76 additions & 40 deletions src/quads/tools/lshw2meta.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,14 @@ def b2g(num, metric=False): # pragma: no cover
path, extension = os.path.splitext(filename)
if extension == ".json":
with open(filename) as _f:
data = json.load(_f)
try:
data = json.load(_f)
except json.JSONDecodeError:
print(f"Error decoding: {filename}")
continue
children = parse("$..children[*]").find(data)
hostname = parse("$.id").find(data)[0].value
print(f"Processing: {hostname}")
host_obj = quads.get_host(hostname)
if not host_obj:
print(f"Host not found: {hostname}")
Expand All @@ -50,41 +55,69 @@ def b2g(num, metric=False): # pragma: no cover
if speed:
speed = int("".join(filter(str.isdigit, speed)))
host_interface.speed = speed
host_obj.save()
interface = quads.update_interface(
hostname, host_interface.as_dict()
)
print(
f" Updated interface: {host_interface.as_dict()}"
)
# disks
for child in [
child for child in children if child.value["class"] == "disk"
]:
if child.value.get("size"):
disk_nodes = [
node.context.value
for node in parse("$..class").find(data)
if node.value == "disk"
]
disks = {}
for child in disk_nodes:
if child.get("size"):
disk_type = None
for dt, sub in DISK_TYPES.items():
if child.value["description"].lower().startswith(sub):
if child.get("description").lower().startswith(sub):
disk_type = dt
disk_size = b2g(int(child.value["size"]), True)
filters = {
"name": host_obj.name,
"disks__disk_type": disk_type,
"disks__size_gb": disk_size,
break
disk_size = b2g(int(child.get("size")), True)
disks[f"{disk_type}|{str(disk_size)}"] = (
disks.get(f"{disk_type}|{str(disk_size)}", 0) + 1
)

for key, count in disks.items():
disk_type, disk_size = key.split("|")
filters = {
"name": host_obj.name,
"disks.disk_type": disk_type,
"disks.size_gb": disk_size,
}
host = quads.filter_hosts(filters)
if host:
for disk in host[0].disks:
if (
disk.disk_type == disk_type
and disk.size_gb == int(disk_size)
):
if disk.count != count:
data = {
"disk_id": disk.id,
"disk_type": disk_type,
"size_gb": disk_size,
"count": count,
}
quads.update_disk(host_obj.name, data)
print(f" Updated disk: {data}")
else:
print(f" Disk already exists: {disk_type, disk_size, count}")
break
else:
data = {
"disk_type": disk_type,
"size_gb": disk_size,
"count": count,
}
host = quads.filter_hosts(filters)
if host:
for disk in host[0].disks:
if (
disk.disk_type == disk_type
and disk.disk_size == disk_size
):
disk.count += 1
else:
data = {
"disk_type": disk_type,
"size_gb": disk_size,
"count": 1,
}
disk = quads.create_disk(host_obj.name, data)
disk = quads.create_disk(host_obj.name, data)
print(f" Created disk: {data}")

# memory
for memory in host_obj.memory:
quads.remove_memory(memory.id)
quads.remove_memory(str(memory.id))
for child in [
child
for child in children
Expand All @@ -104,24 +137,27 @@ def b2g(num, metric=False): # pragma: no cover
hostname,
data,
)
print(f" Created memory: {data}")

# processor
for processor in host_obj.processors:
quads.remove_processor(processor.id)
quads.remove_processor(str(processor.id))
for child in [
child
for child in children
if child.value["class"] == "processor"
]:
configuration = child.value.get("configuration")
data = {
"handle": child.value.get("handle"),
"vendor": child.value.get("vendor"),
"product": child.value.get("product"),
"cores": int(configuration.get("cores", 0)),
"threads": int(configuration.get("threads", 0)),
}
processor = quads.create_processor(
hostname,
data,
)
if configuration.get("cores") and configuration.get("threads"):
data = {
"handle": child.value.get("handle"),
"vendor": child.value.get("vendor"),
"product": child.value.get("product"),
"cores": int(configuration.get("cores")),
"threads": int(configuration.get("threads")),
}
processor = quads.create_processor(
hostname,
data,
)
print(f" Created processor: {data}")
11 changes: 5 additions & 6 deletions src/quads/tools/notify.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ async def create_initial_message(real_owner, cloud, cloud_info, ticket, cc):
template = Template(_file.read())
content = template.render(
cloud_info=cloud_info,
wiki_url=Config["wiki_url"],
cloud=cloud,
quads_url=Config["quads_url"],
real_owner=real_owner,
Expand All @@ -63,7 +62,7 @@ async def create_initial_message(real_owner, cloud, cloud_info, ticket, cc):
message = "%s QUADS: %s is now active, choo choo! - %s/assignments/#%s - %s %s" % (
irc_bot_channel,
cloud_info,
Config["wiki_url"],
Config["quads_url"],
cloud,
real_owner,
Config["report_cc"],
Expand All @@ -77,7 +76,7 @@ async def create_initial_message(real_owner, cloud, cloud_info, ticket, cc):
try:
message = "QUADS: %s is now active, choo choo! - %s/assignments/#%s - %s %s" % (
cloud_info,
Config["wiki_url"],
Config["quads_url"],
cloud,
real_owner,
Config["report_cc"],
Expand Down Expand Up @@ -113,7 +112,7 @@ def create_message(
content = template.render(
days_to_report=day,
cloud_info=cloud_info,
wiki_url=Config["wiki_url"],
quads_url=Config["quads_url"],
quads_request_url=quads_request_url,
quads_request_deadline_day=Config["quads_request_deadline_day"],
quads_notify_until_extended=Config["quads_notify_until_extended"],
Expand All @@ -139,7 +138,7 @@ def create_future_initial_message(cloud, assignment_obj, cloud_info):
template = Template(_file.read())
content = template.render(
cloud_info=cloud_info,
wiki_url=Config["wiki_url"],
quads_url=Config["quads_url"],
)
postman = Postman(
"New QUADS Assignment Defined for the Future: %s - %s" % (cloud, ticket),
Expand Down Expand Up @@ -167,7 +166,7 @@ def create_future_message(
content = template.render(
days_to_report=future_days,
cloud_info=cloud_info,
wiki_url=Config["wiki_url"],
quads_url=Config["quads_url"],
cloud=cloud,
hosts=host_list_expire,
)
Expand Down
Loading

0 comments on commit 414658f

Please sign in to comment.