From 3bdf2305eb98d9e0dac5917fd539db479f7c2382 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 19 Sep 2024 15:53:48 +0200 Subject: [PATCH 1/6] aws-ce-grafana-backend: total costs data and graph functional --- .../mounted-files/aws.py | 129 +++++++++--------- .../mounted-files/webserver.py | 20 ++- 2 files changed, 77 insertions(+), 72 deletions(-) diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py b/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py index fbd0678010..e8c36d3a33 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py @@ -1,24 +1,22 @@ +""" +Queries to AWS Cost Explorer to get different kinds of cost data. +""" + import boto3 -# AWS client functions most likely: -# -# - get_cost_and_usage -# - get_cost_categories -# - get_tags -# - list_cost_allocation_tags -# aws_ce_client = boto3.client("ce") -def query_total_cost(from_date, to_date): - results = query_aws_cost_explorer(from_date, to_date) +def query_total_cost(cluster_name, from_date, to_date): + results = query_aws_cost_explorer(cluster_name, from_date, to_date) return results -def query_aws_cost_explorer(from_date, to_date): +def query_aws_cost_explorer(cluster_name, from_date, to_date): # ref: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ce/client/get_cost_and_usage.html#get-cost-and-usage response = aws_ce_client.get_cost_and_usage( # Metrics: + # # UnblendedCosts represents costs for an individual AWS account. It is # the default metric in the AWS web console. BlendedCosts represents # the potentially reduced costs stemming from having multiple AWS @@ -37,12 +35,58 @@ def query_aws_cost_explorer(from_date, to_date): "End": to_date, }, Filter={ - "Dimensions": { - # RECORD_TYPE is also called Charge type. By filtering on this - # we avoid results related to credits, tax, etc. - "Key": "RECORD_TYPE", - "Values": ["Usage"], - }, + "And": [ + { + "Dimensions": { + # RECORD_TYPE is also called Charge type. By filtering on this + # we avoid results related to credits, tax, etc. + "Key": "RECORD_TYPE", + "Values": ["Usage"], + }, + }, + { + # ref: https://github.com/2i2c-org/infrastructure/issues/4787#issue-2519110356 + "Or": [ + { + "Tags": { + "Key": "alpha.eksctl.io/cluster-name", + "Values": [cluster_name], + "MatchOptions": ["EQUALS"], + }, + }, + { + "Tags": { + "Key": f"kubernetes.io/cluster/{cluster_name}", + "Values": ["owned"], + "MatchOptions": ["EQUALS"], + }, + }, + { + "Tags": { + "Key": "2i2c.org/cluster-name", + "Values": [cluster_name], + "MatchOptions": ["EQUALS"], + }, + }, + { + "Not": { + "Tags": { + "Key": "2i2c:hub-name", + "MatchOptions": ["ABSENT"], + }, + }, + }, + { + "Not": { + "Tags": { + "Key": "2i2c:node-purpose", + "MatchOptions": ["ABSENT"], + }, + }, + }, + ], + }, + ] }, # FIXME: Add this or something similar back when focusing on something # beyond just the total daily costs. @@ -75,48 +119,11 @@ def query_aws_cost_explorer(from_date, to_date): # # ... # ] # - return response["ResultsByTime"] - - -# Description of Grafana panels requested by Yuvi: -# ref: https://github.com/2i2c-org/infrastructure/issues/4453#issuecomment-2298076415 -# -# Currently our AWS tag 2i2c:hub-name is only capturing a fraction of the costs, -# so initially only the following panels are easy to work on. -# -# - total cost (4) -# - total cost per component (2) -# -# The following panels are dependent on the 2i2c:hub-name tag though. -# -# - total cost per hub (1) -# - total cost per component, repeated per hub (3) -# -# Summarized notes about user facing labels: -# -# - fixed: -# - core nodepool -# - any PV needed for support chart or hub databases -# - Kubernetes master API -# - load balancer services -# - compute: -# - disks -# - networking -# - gpus -# - home storage: -# - backups -# - object storage: -# - tagged buckets -# - not counting requester pays -# - total: -# - all 2i2c managed infra -# -# Working against cost tags directly or cost categories -# -# Cost categories vs Cost allocation tags -# -# - It seems cost categories could be suitable to group misc data under -# categories, and split things like core node pool. -# - I think its worth exploring if we could offload all complexity about user -# facing labels etc by using cost categories to group and label costs. -# + processed_response = [ + { + "date": e["TimePeriod"]["Start"], + "cost": f'{float(e["Total"]["UnblendedCost"]["Amount"]):.2f}', + } + for e in response["ResultsByTime"] + ] + return processed_response diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py index fa72767c42..902cd0d619 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py @@ -6,15 +6,8 @@ app = Flask(__name__) - -@app.route("/") -def hello_world(): - return "

Hello, World!

" - - -@app.route("/health/ready") -def ready(): - return ("", 204) +# Hardcoded, see https://github.com/2i2c-org/infrastructure/issues/4788 +CLUSTER_NAME = "openscapeshub" def parse_from_to_in_query_params(): @@ -47,8 +40,13 @@ def parse_from_to_in_query_params(): return from_date, to_date -@app.route("/aws/total-cost") +@app.route("/health/ready") +def ready(): + return ("", 204) + + +@app.route("/total-cost") def aws_total_cost(): from_date, to_date = parse_from_to_in_query_params() - return query_total_cost(from_date, to_date) + return query_total_cost(CLUSTER_NAME, from_date, to_date) From d2d1674ec72820bad06ea1569173de298f681e35 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 19 Sep 2024 22:50:19 +0200 Subject: [PATCH 2/6] aws-ce-grafana-backend: initial step towards total costs per hub --- .../mounted-files/aws.py | 57 +++++++++++++++---- .../aws-ce-grafana-backend/values.yaml | 7 +++ 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py b/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py index e8c36d3a33..7c2c1638f5 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py @@ -88,18 +88,18 @@ def query_aws_cost_explorer(cluster_name, from_date, to_date): }, ] }, - # FIXME: Add this or something similar back when focusing on something - # beyond just the total daily costs. - # - # GroupBy=[ - # { - # "Type": "DIMENSION", - # "Key": "SERVICE", - # }, - # ], + GroupBy=[ + { + "Type": "TAG", + "Key": "2i2c:hub-name", + }, + ], ) - # response["ResultsByTime"] is a list with entries looking like this... + print(response) + + # response["ResultsByTime"] is a list with entries looking like this if + # GroupBy isn't specified... # # [ # { @@ -119,6 +119,43 @@ def query_aws_cost_explorer(cluster_name, from_date, to_date): # # ... # ] # + # response["ResultsByTime"] is a list with entries looking like this if + # GroupBy is specified... + # + # [ + # { + # "TimePeriod": {"Start": "2024-08-30", "End": "2024-08-31"}, + # "Total": {}, + # "Groups": [ + # { + # "Keys": ["2i2c:hub-name$"], + # "Metrics": { + # "UnblendedCost": {"Amount": "12.1930361882", "Unit": "USD"} + # }, + # }, + # { + # "Keys": ["2i2c:hub-name$prod"], + # "Metrics": { + # "UnblendedCost": {"Amount": "18.662514854", "Unit": "USD"} + # }, + # }, + # { + # "Keys": ["2i2c:hub-name$staging"], + # "Metrics": { + # "UnblendedCost": {"Amount": "0.000760628", "Unit": "USD"} + # }, + # }, + # { + # "Keys": ["2i2c:hub-name$workshop"], + # "Metrics": { + # "UnblendedCost": {"Amount": "0.1969903219", "Unit": "USD"} + # }, + # }, + # ], + # "Estimated": False, + # }, + # ] + # processed_response = [ { "date": e["TimePeriod"]["Start"], diff --git a/helm-charts/aws-ce-grafana-backend/values.yaml b/helm-charts/aws-ce-grafana-backend/values.yaml index 479a18b45c..0fd5c9dae1 100644 --- a/helm-charts/aws-ce-grafana-backend/values.yaml +++ b/helm-charts/aws-ce-grafana-backend/values.yaml @@ -5,6 +5,13 @@ nameOverride: "" fullnameOverride: "" global: {} +# Software configuration +# ----------------------------------------------------------------------------- +# +# FIXME: This chart deploys hardcoded software currently, see +# https://github.com/2i2c-org/infrastructure/issues/4788 about resolving +# it. + # Deployment resource # ----------------------------------------------------------------------------- # From 6b2f42bc4f21274bddbbdc56b4d92a5a285a20b8 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 20 Sep 2024 10:14:25 +0200 Subject: [PATCH 3/6] aws-ce-grafana-backend: let cluster name be configurable via env --- .../aws-ce-grafana-backend/ce-test-config.yaml | 3 +++ .../mounted-files/aws.py | 18 ++++++++++++------ .../mounted-files/webserver.py | 5 +---- .../templates/deployment.yaml | 8 ++++++-- .../aws-ce-grafana-backend/values.schema.yaml | 13 +++++++++++++ helm-charts/aws-ce-grafana-backend/values.yaml | 13 ++++++++++--- 6 files changed, 45 insertions(+), 15 deletions(-) diff --git a/helm-charts/aws-ce-grafana-backend/ce-test-config.yaml b/helm-charts/aws-ce-grafana-backend/ce-test-config.yaml index 8ad2443b47..73ff8e30d9 100644 --- a/helm-charts/aws-ce-grafana-backend/ce-test-config.yaml +++ b/helm-charts/aws-ce-grafana-backend/ce-test-config.yaml @@ -2,3 +2,6 @@ fullnameOverride: ce-test serviceAccount: annotations: eks.amazonaws.com/role-arn: arn:aws:iam::783616723547:role/aws_ce_grafana_backend_iam_role + +envBasedConfig: + clusterName: openscapeshub diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py b/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py index 7c2c1638f5..fa6dd6e32e 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py @@ -2,17 +2,23 @@ Queries to AWS Cost Explorer to get different kinds of cost data. """ +import os + import boto3 +# Environment variables based config isn't great, see fixme comment in +# values.yaml under the software configuration heading +CLUSTER_NAME = os.environ["AWS_CE_GRAFANA_BACKEND__CLUSTER_NAME"] + aws_ce_client = boto3.client("ce") -def query_total_cost(cluster_name, from_date, to_date): - results = query_aws_cost_explorer(cluster_name, from_date, to_date) +def query_total_cost(from_date, to_date): + results = query_aws_cost_explorer(from_date, to_date) return results -def query_aws_cost_explorer(cluster_name, from_date, to_date): +def query_aws_cost_explorer(from_date, to_date): # ref: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ce/client/get_cost_and_usage.html#get-cost-and-usage response = aws_ce_client.get_cost_and_usage( # Metrics: @@ -50,13 +56,13 @@ def query_aws_cost_explorer(cluster_name, from_date, to_date): { "Tags": { "Key": "alpha.eksctl.io/cluster-name", - "Values": [cluster_name], + "Values": [CLUSTER_NAME], "MatchOptions": ["EQUALS"], }, }, { "Tags": { - "Key": f"kubernetes.io/cluster/{cluster_name}", + "Key": f"kubernetes.io/cluster/{CLUSTER_NAME}", "Values": ["owned"], "MatchOptions": ["EQUALS"], }, @@ -64,7 +70,7 @@ def query_aws_cost_explorer(cluster_name, from_date, to_date): { "Tags": { "Key": "2i2c.org/cluster-name", - "Values": [cluster_name], + "Values": [CLUSTER_NAME], "MatchOptions": ["EQUALS"], }, }, diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py index 902cd0d619..b14925afe0 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py @@ -6,9 +6,6 @@ app = Flask(__name__) -# Hardcoded, see https://github.com/2i2c-org/infrastructure/issues/4788 -CLUSTER_NAME = "openscapeshub" - def parse_from_to_in_query_params(): """ @@ -49,4 +46,4 @@ def ready(): def aws_total_cost(): from_date, to_date = parse_from_to_in_query_params() - return query_total_cost(CLUSTER_NAME, from_date, to_date) + return query_total_cost(from_date, to_date) diff --git a/helm-charts/aws-ce-grafana-backend/templates/deployment.yaml b/helm-charts/aws-ce-grafana-backend/templates/deployment.yaml index a9489f8776..7adbf7b524 100644 --- a/helm-charts/aws-ce-grafana-backend/templates/deployment.yaml +++ b/helm-charts/aws-ce-grafana-backend/templates/deployment.yaml @@ -37,10 +37,14 @@ spec: - name: secret mountPath: /srv/aws-ce-grafana-backend readOnly: true - {{- with .Values.extraEnv }} env: + - name: PYTHONUNBUFFERED + value: "1" + - name: AWS_CE_GRAFANA_BACKEND__CLUSTER_NAME + value: "{{ .Values.envBasedConfig.clusterName }}" + {{- with .Values.extraEnv }} {{- tpl (. | toYaml) $ | nindent 12 }} - {{- end }} + {{- end }} resources: {{- .Values.resources | toYaml | nindent 12 }} securityContext: diff --git a/helm-charts/aws-ce-grafana-backend/values.schema.yaml b/helm-charts/aws-ce-grafana-backend/values.schema.yaml index 2d793009bb..70760363ca 100644 --- a/helm-charts/aws-ce-grafana-backend/values.schema.yaml +++ b/helm-charts/aws-ce-grafana-backend/values.schema.yaml @@ -17,6 +17,8 @@ additionalProperties: false required: # General configuration - global + # Software configuration + - envBasedConfig # Deployment resource - image # Other resources @@ -43,6 +45,17 @@ properties: type: object additionalProperties: true + # Software configuration + # --------------------------------------------------------------------------- + # + envBasedConfig: + type: object + additionalProperties: false + required: [clusterName] + properties: + clusterName: + type: string + # Deployment resource # --------------------------------------------------------------------------- # diff --git a/helm-charts/aws-ce-grafana-backend/values.yaml b/helm-charts/aws-ce-grafana-backend/values.yaml index 0fd5c9dae1..25eadf8459 100644 --- a/helm-charts/aws-ce-grafana-backend/values.yaml +++ b/helm-charts/aws-ce-grafana-backend/values.yaml @@ -8,9 +8,16 @@ global: {} # Software configuration # ----------------------------------------------------------------------------- # -# FIXME: This chart deploys hardcoded software currently, see -# https://github.com/2i2c-org/infrastructure/issues/4788 about resolving -# it. +# FIXME: This chart currently configures software via environment variables, and +# it doesn't scale well for a few string values without validation logic. +# +# A better scaling approach is for example to use traitlets and config from the +# chart be mounted via a k8s Secret to be read by the software. That way, +# the chart doesn't have to be updated and maintained to be in parity +# with the software config options. +# +envBasedConfig: + clusterName: "" # Deployment resource # ----------------------------------------------------------------------------- From 44378acfa563a4a5a773c9990d720d2d493d4201 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 20 Sep 2024 11:43:04 +0200 Subject: [PATCH 4/6] aws-ce-grafana-backend: add total costs per hub --- .../mounted-files/README.md | 1 + .../mounted-files/aws.py | 262 +++++++++++------- .../mounted-files/webserver.py | 15 +- 3 files changed, 180 insertions(+), 98 deletions(-) diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/README.md b/helm-charts/aws-ce-grafana-backend/mounted-files/README.md index 73cb11c83b..b2d10626aa 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/README.md +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/README.md @@ -21,6 +21,7 @@ First authenticate yourself against the AWS openscapes account. ```bash cd helm-charts/aws-ce-grafana-backend/mounted-files +export AWS_CE_GRAFANA_BACKEND__CLUSTER_NAME=openscapeshub python -m flask --app=webserver run --port=8080 # visit http://localhost:8080/aws diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py b/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py index fa6dd6e32e..42d0dc1dff 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py @@ -10,102 +10,120 @@ # values.yaml under the software configuration heading CLUSTER_NAME = os.environ["AWS_CE_GRAFANA_BACKEND__CLUSTER_NAME"] -aws_ce_client = boto3.client("ce") +# Metrics: +# +# UnblendedCost represents costs for an individual AWS account. It is +# the default metric in the AWS web console. BlendedCosts represents +# the potentially reduced costs stemming from having multiple AWS +# accounts in an organization the collectively could enter better +# pricing tiers. +# +METRICS_UNBLENDED_COST = "UnblendedCost" +# Granularity: +# +# HOURLY granularity is only available for the last two days, while +# DAILY is available for the last 13 months. +# +GRANULARITY_DAILY = "DAILY" -def query_total_cost(from_date, to_date): - results = query_aws_cost_explorer(from_date, to_date) - return results +FILTER_USAGE_COSTS = { + "Dimensions": { + # RECORD_TYPE is also called Charge type. By filtering on this + # we avoid results related to credits, tax, etc. + "Key": "RECORD_TYPE", + "Values": ["Usage"], + }, +} +FILTER_ATTRIBUTABLE_COSTS = { + # ref: https://github.com/2i2c-org/infrastructure/issues/4787#issue-2519110356 + "Or": [ + { + "Tags": { + "Key": "alpha.eksctl.io/cluster-name", + "Values": [CLUSTER_NAME], + "MatchOptions": ["EQUALS"], + }, + }, + { + "Tags": { + "Key": f"kubernetes.io/cluster/{CLUSTER_NAME}", + "Values": ["owned"], + "MatchOptions": ["EQUALS"], + }, + }, + { + "Tags": { + "Key": "2i2c.org/cluster-name", + "Values": [CLUSTER_NAME], + "MatchOptions": ["EQUALS"], + }, + }, + { + "Not": { + "Tags": { + "Key": "2i2c:hub-name", + "MatchOptions": ["ABSENT"], + }, + }, + }, + { + "Not": { + "Tags": { + "Key": "2i2c:node-purpose", + "MatchOptions": ["ABSENT"], + }, + }, + }, + ] +} -def query_aws_cost_explorer(from_date, to_date): +GROUP_BY_HUB_TAG = { + "Type": "TAG", + "Key": "2i2c:hub-name", +} + + +aws_ce_client = boto3.client("ce") + + +def query_aws_cost_explorer(metrics, granularity, from_date, to_date, filter, group_by): + """ + Function meant to be responsible for making the API call and handling + pagination etc. Currently pagination isn't handled. + """ # ref: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ce/client/get_cost_and_usage.html#get-cost-and-usage response = aws_ce_client.get_cost_and_usage( - # Metrics: - # - # UnblendedCosts represents costs for an individual AWS account. It is - # the default metric in the AWS web console. BlendedCosts represents - # the potentially reduced costs stemming from having multiple AWS - # accounts in an organization the collectively could enter better - # pricing tiers. - # - Metrics=["UnblendedCost"], - # Granularity: - # - # HOURLY granularity is only available for the last two days, while - # DAILY is available for the last 13 months. - # - Granularity="DAILY", - TimePeriod={ - "Start": from_date, - "End": to_date, - }, - Filter={ + Metrics=metrics, + Granularity=granularity, + TimePeriod={"Start": from_date, "End": to_date}, + Filter=filter, + GroupBy=group_by, + ) + return response + + +def query_total_costs(from_date, to_date): + """ + A query with processing of the response tailored query to report hub + independent total costs. + """ + response = query_aws_cost_explorer( + metrics=[METRICS_UNBLENDED_COST], + granularity=GRANULARITY_DAILY, + from_date=from_date, + to_date=to_date, + filter={ "And": [ - { - "Dimensions": { - # RECORD_TYPE is also called Charge type. By filtering on this - # we avoid results related to credits, tax, etc. - "Key": "RECORD_TYPE", - "Values": ["Usage"], - }, - }, - { - # ref: https://github.com/2i2c-org/infrastructure/issues/4787#issue-2519110356 - "Or": [ - { - "Tags": { - "Key": "alpha.eksctl.io/cluster-name", - "Values": [CLUSTER_NAME], - "MatchOptions": ["EQUALS"], - }, - }, - { - "Tags": { - "Key": f"kubernetes.io/cluster/{CLUSTER_NAME}", - "Values": ["owned"], - "MatchOptions": ["EQUALS"], - }, - }, - { - "Tags": { - "Key": "2i2c.org/cluster-name", - "Values": [CLUSTER_NAME], - "MatchOptions": ["EQUALS"], - }, - }, - { - "Not": { - "Tags": { - "Key": "2i2c:hub-name", - "MatchOptions": ["ABSENT"], - }, - }, - }, - { - "Not": { - "Tags": { - "Key": "2i2c:node-purpose", - "MatchOptions": ["ABSENT"], - }, - }, - }, - ], - }, + FILTER_USAGE_COSTS, + FILTER_ATTRIBUTABLE_COSTS, ] }, - GroupBy=[ - { - "Type": "TAG", - "Key": "2i2c:hub-name", - }, - ], + group_by=[], ) - print(response) - - # response["ResultsByTime"] is a list with entries looking like this if - # GroupBy isn't specified... + # response["ResultsByTime"] is a list with entries looking like this... # # [ # { @@ -125,8 +143,48 @@ def query_aws_cost_explorer(from_date, to_date): # # ... # ] # - # response["ResultsByTime"] is a list with entries looking like this if - # GroupBy is specified... + # processed_response is a list with entries looking like this... + # + # [ + # { + # "date":"2024-08-30", + # "cost":"12.19", + # }, + # ] + # + processed_response = [ + { + "date": e["TimePeriod"]["Start"], + "cost": f'{float(e["Total"]["UnblendedCost"]["Amount"]):.2f}', + } + for e in response["ResultsByTime"] + ] + return processed_response + + +def query_total_costs_per_hub(from_date, to_date): + """ + A query with processing of the response tailored query to report total costs + per hub, where costs not attributed to a specific hub is listed under + 'shared'. + """ + response = query_aws_cost_explorer( + metrics=[METRICS_UNBLENDED_COST], + granularity=GRANULARITY_DAILY, + from_date=from_date, + to_date=to_date, + filter={ + "And": [ + FILTER_USAGE_COSTS, + FILTER_ATTRIBUTABLE_COSTS, + ] + }, + group_by=[ + GROUP_BY_HUB_TAG, + ], + ) + + # response["ResultsByTime"] is a list with entries looking like this... # # [ # { @@ -162,11 +220,27 @@ def query_aws_cost_explorer(from_date, to_date): # }, # ] # - processed_response = [ - { - "date": e["TimePeriod"]["Start"], - "cost": f'{float(e["Total"]["UnblendedCost"]["Amount"]):.2f}', - } - for e in response["ResultsByTime"] - ] + # processed_response is a list with entries looking like this... + # + # [ + # { + # "date":"2024-08-30", + # "cost":"12.19", + # "name":"shared", + # }, + # ] + # + processed_response = [] + for e in response["ResultsByTime"]: + processed_response.extend( + [ + { + "date": e["TimePeriod"]["Start"], + "cost": f'{float(g["Metrics"]["UnblendedCost"]["Amount"]):.2f}', + "name": g["Keys"][0].split("$", maxsplit=1)[1] or "shared", + } + for g in e["Groups"] + ] + ) + return processed_response diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py index b14925afe0..a02ada0dba 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py @@ -2,7 +2,7 @@ from flask import Flask, request -from .aws import query_total_cost +from .aws import query_total_costs, query_total_costs_per_hub app = Flask(__name__) @@ -42,8 +42,15 @@ def ready(): return ("", 204) -@app.route("/total-cost") -def aws_total_cost(): +@app.route("/total-costs") +def total_costs(): from_date, to_date = parse_from_to_in_query_params() - return query_total_cost(from_date, to_date) + return query_total_costs(from_date, to_date) + + +@app.route("/total-costs-per-hub") +def total_costs_per_hub(): + from_date, to_date = parse_from_to_in_query_params() + + return query_total_costs_per_hub(from_date, to_date) From 9343bbd1b1aabcd6a603751892e73ac7dd1a1d18 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 20 Sep 2024 11:51:47 +0200 Subject: [PATCH 5/6] aws-ce-grafana-backend: refactor to separate out composable constants --- .../mounted-files/aws.py | 87 ++----------------- .../mounted-files/const.py | 87 +++++++++++++++++++ 2 files changed, 94 insertions(+), 80 deletions(-) create mode 100644 helm-charts/aws-ce-grafana-backend/mounted-files/const.py diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py b/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py index 42d0dc1dff..756ea51f90 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py @@ -2,88 +2,15 @@ Queries to AWS Cost Explorer to get different kinds of cost data. """ -import os - import boto3 -# Environment variables based config isn't great, see fixme comment in -# values.yaml under the software configuration heading -CLUSTER_NAME = os.environ["AWS_CE_GRAFANA_BACKEND__CLUSTER_NAME"] - -# Metrics: -# -# UnblendedCost represents costs for an individual AWS account. It is -# the default metric in the AWS web console. BlendedCosts represents -# the potentially reduced costs stemming from having multiple AWS -# accounts in an organization the collectively could enter better -# pricing tiers. -# -METRICS_UNBLENDED_COST = "UnblendedCost" - -# Granularity: -# -# HOURLY granularity is only available for the last two days, while -# DAILY is available for the last 13 months. -# -GRANULARITY_DAILY = "DAILY" - -FILTER_USAGE_COSTS = { - "Dimensions": { - # RECORD_TYPE is also called Charge type. By filtering on this - # we avoid results related to credits, tax, etc. - "Key": "RECORD_TYPE", - "Values": ["Usage"], - }, -} - -FILTER_ATTRIBUTABLE_COSTS = { - # ref: https://github.com/2i2c-org/infrastructure/issues/4787#issue-2519110356 - "Or": [ - { - "Tags": { - "Key": "alpha.eksctl.io/cluster-name", - "Values": [CLUSTER_NAME], - "MatchOptions": ["EQUALS"], - }, - }, - { - "Tags": { - "Key": f"kubernetes.io/cluster/{CLUSTER_NAME}", - "Values": ["owned"], - "MatchOptions": ["EQUALS"], - }, - }, - { - "Tags": { - "Key": "2i2c.org/cluster-name", - "Values": [CLUSTER_NAME], - "MatchOptions": ["EQUALS"], - }, - }, - { - "Not": { - "Tags": { - "Key": "2i2c:hub-name", - "MatchOptions": ["ABSENT"], - }, - }, - }, - { - "Not": { - "Tags": { - "Key": "2i2c:node-purpose", - "MatchOptions": ["ABSENT"], - }, - }, - }, - ] -} - -GROUP_BY_HUB_TAG = { - "Type": "TAG", - "Key": "2i2c:hub-name", -} - +from .const import ( + FILTER_ATTRIBUTABLE_COSTS, + FILTER_USAGE_COSTS, + GRANULARITY_DAILY, + GROUP_BY_HUB_TAG, + METRICS_UNBLENDED_COST, +) aws_ce_client = boto3.client("ce") diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/const.py b/helm-charts/aws-ce-grafana-backend/mounted-files/const.py new file mode 100644 index 0000000000..e04e69e766 --- /dev/null +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/const.py @@ -0,0 +1,87 @@ +""" +Constants used to compose queries against AWS Cost Explorer API. +""" + +import os + +# Environment variables based config isn't great, see fixme comment in +# values.yaml under the software configuration heading +CLUSTER_NAME = os.environ["AWS_CE_GRAFANA_BACKEND__CLUSTER_NAME"] + +# Metrics: +# +# UnblendedCost represents costs for an individual AWS account. It is +# the default metric in the AWS web console. BlendedCosts represents +# the potentially reduced costs stemming from having multiple AWS +# accounts in an organization the collectively could enter better +# pricing tiers. +# +METRICS_UNBLENDED_COST = "UnblendedCost" + +# Granularity: +# +# HOURLY granularity is only available for the last two days, while +# DAILY is available for the last 13 months. +# +GRANULARITY_DAILY = "DAILY" + +# Filter: +# +# The various filter objects are meant to be combined based on the needs for +# different kinds of queries. +# +FILTER_USAGE_COSTS = { + "Dimensions": { + # RECORD_TYPE is also called Charge type. By filtering on this + # we avoid results related to credits, tax, etc. + "Key": "RECORD_TYPE", + "Values": ["Usage"], + }, +} +FILTER_ATTRIBUTABLE_COSTS = { + # ref: https://github.com/2i2c-org/infrastructure/issues/4787#issue-2519110356 + "Or": [ + { + "Tags": { + "Key": "alpha.eksctl.io/cluster-name", + "Values": [CLUSTER_NAME], + "MatchOptions": ["EQUALS"], + }, + }, + { + "Tags": { + "Key": f"kubernetes.io/cluster/{CLUSTER_NAME}", + "Values": ["owned"], + "MatchOptions": ["EQUALS"], + }, + }, + { + "Tags": { + "Key": "2i2c.org/cluster-name", + "Values": [CLUSTER_NAME], + "MatchOptions": ["EQUALS"], + }, + }, + { + "Not": { + "Tags": { + "Key": "2i2c:hub-name", + "MatchOptions": ["ABSENT"], + }, + }, + }, + { + "Not": { + "Tags": { + "Key": "2i2c:node-purpose", + "MatchOptions": ["ABSENT"], + }, + }, + }, + ] +} + +GROUP_BY_HUB_TAG = { + "Type": "TAG", + "Key": "2i2c:hub-name", +} From 8cb0429232f1143044742094347dda5e2435d077 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 20 Sep 2024 11:53:13 +0200 Subject: [PATCH 6/6] aws-ce-grafana-backend: rename aws.py to query.py --- .../aws-ce-grafana-backend/mounted-files/{aws.py => query.py} | 0 helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename helm-charts/aws-ce-grafana-backend/mounted-files/{aws.py => query.py} (100%) diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py b/helm-charts/aws-ce-grafana-backend/mounted-files/query.py similarity index 100% rename from helm-charts/aws-ce-grafana-backend/mounted-files/aws.py rename to helm-charts/aws-ce-grafana-backend/mounted-files/query.py diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py index a02ada0dba..d8ef70345d 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py @@ -2,7 +2,7 @@ from flask import Flask, request -from .aws import query_total_costs, query_total_costs_per_hub +from .query import query_total_costs, query_total_costs_per_hub app = Flask(__name__)