diff --git a/openshift_metrics/invoice.py b/openshift_metrics/invoice.py new file mode 100644 index 0000000..52a2315 --- /dev/null +++ b/openshift_metrics/invoice.py @@ -0,0 +1,196 @@ +import math +from dataclasses import dataclass, field +from collections import namedtuple +from typing import List +from decimal import Decimal, ROUND_HALF_UP + +# GPU types +GPU_A100 = "NVIDIA-A100-40GB" +GPU_A100_SXM4 = "NVIDIA-A100-SXM4-40GB" +GPU_V100 = "Tesla-V100-PCIE-32GB" +GPU_UNKNOWN_TYPE = "GPU_UNKNOWN_TYPE" + +# GPU Resource - MIG Geometries +# A100 Strategies +MIG_1G_5GB = "nvidia.com/mig-1g.5gb" +MIG_2G_10GB = "nvidia.com/mig-2g.10gb" +MIG_3G_20GB = "nvidia.com/mig-3g.20gb" +WHOLE_GPU = "nvidia.com/gpu" + +# SU Types +SU_CPU = "OpenShift CPU" +SU_A100_GPU = "OpenShift GPUA100" +SU_A100_SXM4_GPU = "OpenShift GPUA100SXM4" +SU_V100_GPU = "OpenShift GPUV100" +SU_UNKNOWN_GPU = "OpenShift Unknown GPU" +SU_UNKNOWN_MIG_GPU = "OpenShift Unknown MIG GPU" +SU_UNKNOWN = "Openshift Unknown" + +ServiceUnit = namedtuple("ServiceUnit", ["su_type", "su_count", "determinig_resource"]) + +@dataclass +class Pod: + """Object that represents a pod""" + pod_name: str + namespace: str + start_time: int + duration: int + cpu_request: Decimal + gpu_request: Decimal + memory_request: Decimal + gpu_type: str + gpu_resource: str + node_hostname: str + node_model: str + + @staticmethod + def get_service_unit(cpu_count, memory_count, gpu_count, gpu_type, gpu_resource) -> ServiceUnit: + """ + Returns the type of service unit, the count, and the determining resource + """ + su_type = SU_UNKNOWN + su_count = 0 + + # pods that requested a specific GPU but weren't scheduled may report 0 GPU + if gpu_resource is not None and gpu_count == 0: + return ServiceUnit(SU_UNKNOWN_GPU, 0, "GPU") + + # pods in weird states + if cpu_count == 0 or memory_count == 0: + return ServiceUnit(SU_UNKNOWN, 0, "CPU") + + known_gpu_su = { + GPU_A100: SU_A100_GPU, + GPU_A100_SXM4: SU_A100_SXM4_GPU, + GPU_V100: SU_V100_GPU, + } + + A100_SXM4_MIG = { + MIG_1G_5GB: SU_UNKNOWN_MIG_GPU, + MIG_2G_10GB: SU_UNKNOWN_MIG_GPU, + MIG_3G_20GB: SU_UNKNOWN_MIG_GPU, + } + + # GPU count for some configs is -1 for math reasons, in reality it is 0 + su_config = { + SU_CPU: {"gpu": -1, "cpu": 1, "ram": 4}, + SU_A100_GPU: {"gpu": 1, "cpu": 24, "ram": 74}, + SU_A100_SXM4_GPU: {"gpu": 1, "cpu": 32, "ram": 245}, + SU_V100_GPU: {"gpu": 1, "cpu": 24, "ram": 192}, + SU_UNKNOWN_GPU: {"gpu": 1, "cpu": 8, "ram": 64}, + SU_UNKNOWN_MIG_GPU: {"gpu": 1, "cpu": 8, "ram": 64}, + SU_UNKNOWN: {"gpu": -1, "cpu": 1, "ram": 1}, + } + + if gpu_resource is None and gpu_count == 0: + su_type = SU_CPU + elif gpu_type is not None and gpu_resource == WHOLE_GPU: + su_type = known_gpu_su.get(gpu_type, SU_UNKNOWN_GPU) + elif gpu_type == GPU_A100_SXM4: # for MIG GPU of type A100_SXM4 + su_type = A100_SXM4_MIG.get(gpu_resource, SU_UNKNOWN_MIG_GPU) + else: + return ServiceUnit(SU_UNKNOWN_GPU, 0, "GPU") + + cpu_multiplier = cpu_count / su_config[su_type]["cpu"] + gpu_multiplier = gpu_count / su_config[su_type]["gpu"] + memory_multiplier = memory_count / su_config[su_type]["ram"] + + su_count = max(cpu_multiplier, gpu_multiplier, memory_multiplier) + + # no fractional SUs for GPU SUs + if su_type != SU_CPU: + su_count = math.ceil(su_count) + + if gpu_multiplier >= cpu_multiplier and gpu_multiplier >= memory_multiplier: + determining_resource = "GPU" + elif cpu_multiplier >= gpu_multiplier and cpu_multiplier >= memory_multiplier: + determining_resource = "CPU" + else: + determining_resource = "RAM" + + return ServiceUnit(su_type, su_count, determining_resource) + + def get_runtime(self) -> Decimal: + """Return runtime eligible for billing in hours""" + return Decimal(self.duration) / 3600 + + +@dataclass() +class Rates: + cpu: Decimal + gpu_a100: Decimal + gpu_a100sxm4: Decimal + gpu_v100: Decimal + + +@dataclass +class ProjectInvoce: + """Represents the invoicing data for a project.""" + + invoice_month: str + project: str + project_id: str + pi: str + invoice_email: str + invoice_address: str + intitution: str + institution_specific_code: str + rates: Rates + su_hours: dict = field( + default_factory=lambda: { + SU_CPU: 0, + SU_A100_GPU: 0, + SU_A100_SXM4_GPU: 0, + SU_V100_GPU: 0, + SU_UNKNOWN_GPU: 0, + SU_UNKNOWN_MIG_GPU: 0, + SU_UNKNOWN: 0, + } + ) + + def add_pod(self, pod: Pod) -> None: + """Aggregate a pods data""" + su_type, su_count, _ = Pod.get_service_unit( + cpu_count=pod.cpu_request, + memory_count=pod.memory_request, + gpu_count=pod.gpu_request, + gpu_type=pod.gpu_type, + gpu_resource=pod.gpu_resource, + ) + duration_in_hours = pod.get_runtime() + self.su_hours[su_type] += su_count * duration_in_hours + + def get_rate(self, su_type) -> Decimal: + if su_type == SU_CPU: + return self.rates.cpu + if su_type == SU_A100_GPU: + return self.rates.gpu_a100 + if su_type == SU_A100_SXM4_GPU: + return self.rates.gpu_a100sxm4 + if su_type == SU_V100_GPU: + return self.rates.gpu_v100 + return Decimal(0) + + def generate_invoice_rows(self, report_month) -> List[str]: + rows = [] + for su_type, hours in self.su_hours.items(): + if hours > 0: + hours = math.ceil(hours) + rate = self.get_rate(su_type) + cost = (rate * hours).quantize(Decimal(".01"), rounding=ROUND_HALF_UP) + row = [ + report_month, + self.project, + self.project_id, + self.pi, + self.invoice_email, + self.invoice_address, + self.intitution, + self.institution_specific_code, + hours, + su_type, + rate, + cost, + ] + rows.append(row) + return rows diff --git a/openshift_metrics/tests/test_utils.py b/openshift_metrics/tests/test_utils.py index e8da8ad..a98977e 100644 --- a/openshift_metrics/tests/test_utils.py +++ b/openshift_metrics/tests/test_utils.py @@ -14,7 +14,7 @@ import tempfile from unittest import TestCase, mock -from openshift_metrics import utils +from openshift_metrics import utils, invoice import os class TestGetNamespaceAnnotations(TestCase): @@ -302,116 +302,116 @@ def test_write_metrics_by_namespace_decimal(self, mock_gna): class TestGetServiceUnit(TestCase): def test_cpu_only(self): - su_type, su_count, determining_resource = utils.get_service_unit(4, 16, 0, None, None) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(4, 16, 0, None, None) self.assertEqual(su_type, utils.SU_CPU) self.assertEqual(su_count, 4) self.assertEqual(determining_resource, "CPU") def test_known_gpu(self): - su_type, su_count, determining_resource = utils.get_service_unit(24, 74, 1, utils.GPU_A100, utils.WHOLE_GPU) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(24, 74, 1, utils.GPU_A100, utils.WHOLE_GPU) self.assertEqual(su_type, utils.SU_A100_GPU) self.assertEqual(su_count, 1) self.assertEqual(determining_resource, "GPU") def test_known_gpu_A100_SXM4(self): - su_type, su_count, determining_resource = utils.get_service_unit(32, 245, 1, utils.GPU_A100_SXM4, utils.WHOLE_GPU) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(32, 245, 1, utils.GPU_A100_SXM4, utils.WHOLE_GPU) self.assertEqual(su_type, utils.SU_A100_SXM4_GPU) self.assertEqual(su_count, 1) self.assertEqual(determining_resource, "GPU") def test_known_gpu_high_cpu(self): - su_type, su_count, determining_resource = utils.get_service_unit(50, 96, 1, utils.GPU_A100, utils.WHOLE_GPU) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(50, 96, 1, utils.GPU_A100, utils.WHOLE_GPU) self.assertEqual(su_type, utils.SU_A100_GPU) self.assertEqual(su_count, 3) self.assertEqual(determining_resource, "CPU") def test_known_gpu_high_memory(self): - su_type, su_count, determining_resource = utils.get_service_unit(24, 100, 1, utils.GPU_A100, utils.WHOLE_GPU) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(24, 100, 1, utils.GPU_A100, utils.WHOLE_GPU) self.assertEqual(su_type, utils.SU_A100_GPU) self.assertEqual(su_count, 2) self.assertEqual(determining_resource, "RAM") def test_known_gpu_low_cpu_memory(self): - su_type, su_count, determining_resource = utils.get_service_unit(2, 4, 1, utils.GPU_A100, utils.WHOLE_GPU) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(2, 4, 1, utils.GPU_A100, utils.WHOLE_GPU) self.assertEqual(su_type, utils.SU_A100_GPU) self.assertEqual(su_count, 1) self.assertEqual(determining_resource, "GPU") def test_unknown_gpu(self): - su_type, su_count, determining_resource = utils.get_service_unit(8, 64, 1, "Unknown_GPU_Type", utils.WHOLE_GPU) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(8, 64, 1, "Unknown_GPU_Type", utils.WHOLE_GPU) self.assertEqual(su_type, utils.SU_UNKNOWN_GPU) self.assertEqual(su_count, 1) self.assertEqual(determining_resource, "GPU") def test_known_gpu_zero_count(self): - su_type, su_count, determining_resource = utils.get_service_unit(8, 64, 0, utils.GPU_A100, utils.WHOLE_GPU) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(8, 64, 0, utils.GPU_A100, utils.WHOLE_GPU) self.assertEqual(su_type, utils.SU_UNKNOWN_GPU) self.assertEqual(su_count, 0) self.assertEqual(determining_resource, "GPU") def test_known_mig_gpu(self): - su_type, su_count, determining_resource = utils.get_service_unit(1, 4, 1, utils.GPU_A100_SXM4, utils.MIG_1G_5GB) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(1, 4, 1, utils.GPU_A100_SXM4, utils.MIG_1G_5GB) self.assertEqual(su_type, utils.SU_UNKNOWN_MIG_GPU) self.assertEqual(su_count, 1) self.assertEqual(determining_resource, "GPU") def test_known_gpu_unknown_resource(self): - su_type, su_count, determining_resource = utils.get_service_unit(1, 4, 1, utils.GPU_A100, "nvidia.com/mig_20G_500GB") + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(1, 4, 1, utils.GPU_A100, "nvidia.com/mig_20G_500GB") self.assertEqual(su_type, utils.SU_UNKNOWN_GPU) self.assertEqual(su_count, 0) self.assertEqual(determining_resource, "GPU") def test_unknown_gpu_known_resource(self): - su_type, su_count, determining_resource = utils.get_service_unit(1, 4, 1, "Unknown GPU", utils.MIG_2G_10GB) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(1, 4, 1, "Unknown GPU", utils.MIG_2G_10GB) self.assertEqual(su_type, utils.SU_UNKNOWN_GPU) self.assertEqual(su_count, 0) self.assertEqual(determining_resource, "GPU") def test_zero_memory(self): - su_type, su_count, determining_resource = utils.get_service_unit(1, 0, 0, None, None) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(1, 0, 0, None, None) self.assertEqual(su_type, utils.SU_UNKNOWN) self.assertEqual(su_count, 0) self.assertEqual(determining_resource, "CPU") def test_zero_cpu(self): - su_type, su_count, determining_resource = utils.get_service_unit(0, 1, 0, None, None) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(0, 1, 0, None, None) self.assertEqual(su_type, utils.SU_UNKNOWN) self.assertEqual(su_count, 0) self.assertEqual(determining_resource, "CPU") def test_memory_dominant(self): - su_type, su_count, determining_resource = utils.get_service_unit(8, 64, 0, None, None) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(8, 64, 0, None, None) self.assertEqual(su_type, utils.SU_CPU) self.assertEqual(su_count, 16) self.assertEqual(determining_resource, "RAM") def test_fractional_su_cpu_dominant(self): - su_type, su_count, determining_resource = utils.get_service_unit(0.5, 0.5, 0, None, None) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(0.5, 0.5, 0, None, None) self.assertEqual(su_type, utils.SU_CPU) self.assertEqual(su_count, 0.5) self.assertEqual(determining_resource, "CPU") def test_fractional_su_memory_dominant(self): - su_type, su_count, determining_resource = utils.get_service_unit(0.1, 1, 0, None, None) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(0.1, 1, 0, None, None) self.assertEqual(su_type, utils.SU_CPU) self.assertEqual(su_count, 0.25) self.assertEqual(determining_resource, "RAM") def test_known_gpu_fractional_cpu_memory(self): - su_type, su_count, determining_resource = utils.get_service_unit(0.8, 0.8, 1, utils.GPU_A100, utils.WHOLE_GPU) + su_type, su_count, determining_resource = invoice.Pod.get_service_unit(0.8, 0.8, 1, utils.GPU_A100, utils.WHOLE_GPU) self.assertEqual(su_type, utils.SU_A100_GPU) self.assertEqual(su_count, 1) self.assertEqual(determining_resource, "GPU") def test_decimal_return_type(self): from decimal import Decimal - _, su_count, _ = utils.get_service_unit(Decimal("1"), Decimal("8.1"), Decimal("0"), None, None) + _, su_count, _ = invoice.Pod.get_service_unit(Decimal("1"), Decimal("8.1"), Decimal("0"), None, None) self.assertIsInstance(su_count, Decimal) self.assertEqual(su_count, Decimal('2.025')) def test_not_decimal_return_type_when_gpu_su_type(self): from decimal import Decimal - su_type, su_count, _ = utils.get_service_unit(Decimal("1"), Decimal("76"), Decimal("1"), utils.GPU_A100, utils.WHOLE_GPU) + su_type, su_count, _ = invoice.Pod.get_service_unit(Decimal("1"), Decimal("76"), Decimal("1"), utils.GPU_A100, utils.WHOLE_GPU) # for GPU SUs, we always round up to the nearest integer self.assertIsInstance(su_count, int) self.assertEqual(su_count, 2) diff --git a/openshift_metrics/utils.py b/openshift_metrics/utils.py index 95117b5..02689a4 100755 --- a/openshift_metrics/utils.py +++ b/openshift_metrics/utils.py @@ -21,6 +21,7 @@ import requests import boto3 +from openshift_metrics import invoice from decimal import Decimal import decimal from urllib3.util.retry import Retry @@ -140,73 +141,6 @@ def get_namespace_attributes(): return namespaces_dict -def get_service_unit(cpu_count, memory_count, gpu_count, gpu_type, gpu_resource): - """ - Returns the type of service unit, the count, and the determining resource - """ - su_type = SU_UNKNOWN - su_count = 0 - - # pods that requested a specific GPU but weren't scheduled may report 0 GPU - if gpu_resource is not None and gpu_count == 0: - return SU_UNKNOWN_GPU, 0, "GPU" - - # pods in weird states - if cpu_count == 0 or memory_count == 0: - return SU_UNKNOWN, 0, "CPU" - - known_gpu_su = { - GPU_A100: SU_A100_GPU, - GPU_A100_SXM4: SU_A100_SXM4_GPU, - GPU_V100: SU_V100_GPU, - } - - A100_SXM4_MIG = { - MIG_1G_5GB: SU_UNKNOWN_MIG_GPU, - MIG_2G_10GB: SU_UNKNOWN_MIG_GPU, - MIG_3G_20GB: SU_UNKNOWN_MIG_GPU, - } - - # GPU count for some configs is -1 for math reasons, in reality it is 0 - su_config = { - SU_CPU: {"gpu": -1, "cpu": 1, "ram": 4}, - SU_A100_GPU: {"gpu": 1, "cpu": 24, "ram": 74}, - SU_A100_SXM4_GPU: {"gpu": 1, "cpu": 32, "ram": 245}, - SU_V100_GPU: {"gpu": 1, "cpu": 24, "ram": 192}, - SU_UNKNOWN_GPU: {"gpu": 1, "cpu": 8, "ram": 64}, - SU_UNKNOWN_MIG_GPU: {"gpu": 1, "cpu": 8, "ram": 64}, - SU_UNKNOWN: {"gpu": -1, "cpu": 1, "ram": 1}, - } - - if gpu_resource is None and gpu_count == 0: - su_type = SU_CPU - elif gpu_type is not None and gpu_resource == WHOLE_GPU: - su_type = known_gpu_su.get(gpu_type, SU_UNKNOWN_GPU) - elif gpu_type == GPU_A100_SXM4: # for MIG GPU of type A100_SXM4 - su_type = A100_SXM4_MIG.get(gpu_resource, SU_UNKNOWN_MIG_GPU) - else: - return SU_UNKNOWN_GPU, 0, "GPU" - - cpu_multiplier = cpu_count / su_config[su_type]["cpu"] - gpu_multiplier = gpu_count / su_config[su_type]["gpu"] - memory_multiplier = memory_count / su_config[su_type]["ram"] - - su_count = max(cpu_multiplier, gpu_multiplier, memory_multiplier) - - # no fractional SUs for GPU SUs - if su_type != SU_CPU: - su_count = math.ceil(su_count) - - if gpu_multiplier >= cpu_multiplier and gpu_multiplier >= memory_multiplier: - determining_resource = "GPU" - elif cpu_multiplier >= gpu_multiplier and cpu_multiplier >= memory_multiplier: - determining_resource = "CPU" - else: - determining_resource = "RAM" - - return su_type, su_count, determining_resource - - def csv_writer(rows, file_name): """Writes rows as csv to file_name""" print(f"Writing csv to {file_name}") @@ -215,31 +149,11 @@ def csv_writer(rows, file_name): csvwriter.writerows(rows) -def add_row(rows, report_month, namespace, pi, institution_code, hours, su_type): - - hours = math.ceil(hours) - cost = (RATE.get(su_type) * hours).quantize(Decimal('.01'), rounding=decimal.ROUND_HALF_UP) - row = [ - report_month, - namespace, - namespace, - pi, - "", #Invoice Email - "", #Invoice Address - "", #Institution - institution_code, - hours, - su_type, - RATE.get(su_type), - cost, - ] - rows.append(row) - def write_metrics_by_namespace(condensed_metrics_dict, file_name, report_month): """ Process metrics dictionary to aggregate usage by namespace and then write that to a file """ - metrics_by_namespace = {} + invoices = {} rows = [] namespace_annotations = get_namespace_attributes() headers = [ @@ -259,75 +173,54 @@ def write_metrics_by_namespace(condensed_metrics_dict, file_name, report_month): rows.append(headers) - for namespace, pods in condensed_metrics_dict.items(): + # TODO: the caller will pass in the rates as an argument + rates = invoice.Rates( + cpu = Decimal("0.013"), + gpu_a100 = Decimal("1.803"), + gpu_a100sxm4 = Decimal("2.078"), + gpu_v100 = Decimal("1.214") + ) + for namespace, pods in condensed_metrics_dict.items(): namespace_annotation_dict = namespace_annotations.get(namespace, {}) cf_pi = namespace_annotation_dict.get("cf_pi") - cf_institution_code = namespace_annotation_dict.get("institution_code") - - if namespace not in metrics_by_namespace: - metrics_by_namespace[namespace] = { - "pi": cf_pi, - "cf_institution_code": cf_institution_code, - "_cpu_hours": 0, - "_memory_hours": 0, - "SU_CPU_HOURS": 0, - "SU_A100_GPU_HOURS": 0, - "SU_A100_SXM4_GPU_HOURS": 0, - "SU_V100_GPU_HOURS": 0, - "SU_UNKNOWN_GPU_HOURS": 0, - "total_cost": 0, - } + cf_institution_code = namespace_annotation_dict.get("institution_code", "") + + if namespace not in invoices: + project_invoice = invoice.ProjectInvoce( + invoice_month=report_month, + project=namespace, + project_id=namespace, + pi=cf_pi, + invoice_email="", + invoice_address="", + intitution="", + institution_specific_code=cf_institution_code, + rates=rates + ) + invoices[namespace] = project_invoice + + project_invoice = invoices[namespace] for pod, pod_dict in pods.items(): + for epoch_time, pod_metric_dict in pod_dict["metrics"].items(): + pod_obj = invoice.Pod( + pod_name=pod, + namespace=namespace, + start_time=epoch_time, + duration=pod_metric_dict["duration"], + cpu_request=Decimal(pod_metric_dict.get("cpu_request", 0)), + gpu_request=Decimal(pod_metric_dict.get("gpu_request", 0)), + memory_request=Decimal(pod_metric_dict.get("memory_request", 0)) / 2**30, + gpu_type=pod_metric_dict.get("gpu_type"), + gpu_resource=pod_metric_dict.get("gpu_resource"), + node_hostname=pod_metric_dict.get("node"), + node_model=pod_metric_dict.get("node_model"), + ) + project_invoice.add_pod(pod_obj) - pod_metrics_dict = pod_dict["metrics"] - - for epoch_time, pod_metric_dict in pod_metrics_dict.items(): - duration_in_hours = Decimal(pod_metric_dict["duration"]) / 3600 - cpu_request = Decimal(pod_metric_dict.get("cpu_request", 0)) - gpu_request = Decimal(pod_metric_dict.get("gpu_request", 0)) - gpu_type = pod_metric_dict.get("gpu_type") - gpu_resource = pod_metric_dict.get("gpu_resource") - memory_request = Decimal(pod_metric_dict.get("memory_request", 0)) / 2**30 - - _, su_count, _ = get_service_unit(cpu_request, memory_request, gpu_request, gpu_type, gpu_resource) - - if gpu_type == GPU_A100: - metrics_by_namespace[namespace]["SU_A100_GPU_HOURS"] += su_count * duration_in_hours - elif gpu_type == GPU_A100_SXM4: - metrics_by_namespace[namespace]["SU_A100_SXM4_GPU_HOURS"] += su_count * duration_in_hours - elif gpu_type == GPU_V100: - metrics_by_namespace[namespace]["SU_V100_GPU_HOURS"] += su_count * duration_in_hours - elif gpu_type == GPU_UNKNOWN_TYPE: - metrics_by_namespace[namespace]["SU_UNKNOWN_GPU_HOURS"] += su_count * duration_in_hours - else: - metrics_by_namespace[namespace]["SU_CPU_HOURS"] += su_count * duration_in_hours - - for namespace, metrics in metrics_by_namespace.items(): - - common_args = { - "rows": rows, - "report_month": report_month, - "namespace": namespace, - "pi": metrics["pi"], - "institution_code": metrics["cf_institution_code"] - } - - if metrics["SU_CPU_HOURS"] != 0: - add_row(hours=metrics["SU_CPU_HOURS"], su_type=SU_CPU, **common_args) - - if metrics["SU_A100_GPU_HOURS"] != 0: - add_row(hours=metrics["SU_A100_GPU_HOURS"], su_type=SU_A100_GPU, **common_args) - - if metrics["SU_A100_SXM4_GPU_HOURS"] != 0: - add_row(hours=metrics["SU_A100_SXM4_GPU_HOURS"], su_type=SU_A100_SXM4_GPU, **common_args) - - if metrics["SU_V100_GPU_HOURS"] != 0: - add_row(hours=metrics["SU_V100_GPU_HOURS"], su_type=SU_V100_GPU, **common_args) - - if metrics["SU_UNKNOWN_GPU_HOURS"] != 0: - add_row(hours=metrics["SU_UNKNOWN_GPU_HOURS"], su_type=SU_UNKNOWN_GPU, **common_args) + for project_invoice in invoices.values(): + rows.extend(project_invoice.generate_invoice_rows(report_month)) csv_writer(rows, file_name) @@ -384,7 +277,7 @@ def write_metrics_by_pod(metrics_dict, file_name): node = pod_metric_dict.get("node", "Unknown Node") node_model = pod_metric_dict.get("node_model", "Unknown Model") memory_request = (Decimal(pod_metric_dict.get("memory_request", 0)) / 2**30).quantize(Decimal(".0001"), rounding=decimal.ROUND_HALF_UP) - su_type, su_count, determining_resource = get_service_unit( + su_type, su_count, determining_resource = invoice.Pod.get_service_unit( cpu_request, memory_request, gpu_request, gpu_type, gpu_resource )