diff --git a/helm-charts/aws-ce-grafana-backend/ce-test-config.yaml b/helm-charts/aws-ce-grafana-backend/ce-test-config.yaml index 8ad2443b47..73ff8e30d9 100644 --- a/helm-charts/aws-ce-grafana-backend/ce-test-config.yaml +++ b/helm-charts/aws-ce-grafana-backend/ce-test-config.yaml @@ -2,3 +2,6 @@ fullnameOverride: ce-test serviceAccount: annotations: eks.amazonaws.com/role-arn: arn:aws:iam::783616723547:role/aws_ce_grafana_backend_iam_role + +envBasedConfig: + clusterName: openscapeshub diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/README.md b/helm-charts/aws-ce-grafana-backend/mounted-files/README.md index 73cb11c83b..b2d10626aa 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/README.md +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/README.md @@ -21,6 +21,7 @@ First authenticate yourself against the AWS openscapes account. ```bash cd helm-charts/aws-ce-grafana-backend/mounted-files +export AWS_CE_GRAFANA_BACKEND__CLUSTER_NAME=openscapeshub python -m flask --app=webserver run --port=8080 # visit http://localhost:8080/aws diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py b/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py deleted file mode 100644 index fbd0678010..0000000000 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/aws.py +++ /dev/null @@ -1,122 +0,0 @@ -import boto3 - -# AWS client functions most likely: -# -# - get_cost_and_usage -# - get_cost_categories -# - get_tags -# - list_cost_allocation_tags -# -aws_ce_client = boto3.client("ce") - - -def query_total_cost(from_date, to_date): - results = query_aws_cost_explorer(from_date, to_date) - return results - - -def query_aws_cost_explorer(from_date, to_date): - # ref: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ce/client/get_cost_and_usage.html#get-cost-and-usage - response = aws_ce_client.get_cost_and_usage( - # Metrics: - # UnblendedCosts represents costs for an individual AWS account. It is - # the default metric in the AWS web console. BlendedCosts represents - # the potentially reduced costs stemming from having multiple AWS - # accounts in an organization the collectively could enter better - # pricing tiers. - # - Metrics=["UnblendedCost"], - # Granularity: - # - # HOURLY granularity is only available for the last two days, while - # DAILY is available for the last 13 months. - # - Granularity="DAILY", - TimePeriod={ - "Start": from_date, - "End": to_date, - }, - Filter={ - "Dimensions": { - # RECORD_TYPE is also called Charge type. By filtering on this - # we avoid results related to credits, tax, etc. - "Key": "RECORD_TYPE", - "Values": ["Usage"], - }, - }, - # FIXME: Add this or something similar back when focusing on something - # beyond just the total daily costs. - # - # GroupBy=[ - # { - # "Type": "DIMENSION", - # "Key": "SERVICE", - # }, - # ], - ) - - # response["ResultsByTime"] is a list with entries looking like this... - # - # [ - # { - # "Estimated": false, - # "Groups": [], - # "TimePeriod": { - # "End": "2024-07-28", - # "Start": "2024-07-27", - # }, - # "Total": { - # "UnblendedCost": { - # "Amount": "23.3110299724", - # "Unit": "USD", - # }, - # }, - # }, - # # ... - # ] - # - return response["ResultsByTime"] - - -# Description of Grafana panels requested by Yuvi: -# ref: https://github.com/2i2c-org/infrastructure/issues/4453#issuecomment-2298076415 -# -# Currently our AWS tag 2i2c:hub-name is only capturing a fraction of the costs, -# so initially only the following panels are easy to work on. -# -# - total cost (4) -# - total cost per component (2) -# -# The following panels are dependent on the 2i2c:hub-name tag though. -# -# - total cost per hub (1) -# - total cost per component, repeated per hub (3) -# -# Summarized notes about user facing labels: -# -# - fixed: -# - core nodepool -# - any PV needed for support chart or hub databases -# - Kubernetes master API -# - load balancer services -# - compute: -# - disks -# - networking -# - gpus -# - home storage: -# - backups -# - object storage: -# - tagged buckets -# - not counting requester pays -# - total: -# - all 2i2c managed infra -# -# Working against cost tags directly or cost categories -# -# Cost categories vs Cost allocation tags -# -# - It seems cost categories could be suitable to group misc data under -# categories, and split things like core node pool. -# - I think its worth exploring if we could offload all complexity about user -# facing labels etc by using cost categories to group and label costs. -# diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/const.py b/helm-charts/aws-ce-grafana-backend/mounted-files/const.py new file mode 100644 index 0000000000..e04e69e766 --- /dev/null +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/const.py @@ -0,0 +1,87 @@ +""" +Constants used to compose queries against AWS Cost Explorer API. +""" + +import os + +# Environment variables based config isn't great, see fixme comment in +# values.yaml under the software configuration heading +CLUSTER_NAME = os.environ["AWS_CE_GRAFANA_BACKEND__CLUSTER_NAME"] + +# Metrics: +# +# UnblendedCost represents costs for an individual AWS account. It is +# the default metric in the AWS web console. BlendedCosts represents +# the potentially reduced costs stemming from having multiple AWS +# accounts in an organization the collectively could enter better +# pricing tiers. +# +METRICS_UNBLENDED_COST = "UnblendedCost" + +# Granularity: +# +# HOURLY granularity is only available for the last two days, while +# DAILY is available for the last 13 months. +# +GRANULARITY_DAILY = "DAILY" + +# Filter: +# +# The various filter objects are meant to be combined based on the needs for +# different kinds of queries. +# +FILTER_USAGE_COSTS = { + "Dimensions": { + # RECORD_TYPE is also called Charge type. By filtering on this + # we avoid results related to credits, tax, etc. + "Key": "RECORD_TYPE", + "Values": ["Usage"], + }, +} +FILTER_ATTRIBUTABLE_COSTS = { + # ref: https://github.com/2i2c-org/infrastructure/issues/4787#issue-2519110356 + "Or": [ + { + "Tags": { + "Key": "alpha.eksctl.io/cluster-name", + "Values": [CLUSTER_NAME], + "MatchOptions": ["EQUALS"], + }, + }, + { + "Tags": { + "Key": f"kubernetes.io/cluster/{CLUSTER_NAME}", + "Values": ["owned"], + "MatchOptions": ["EQUALS"], + }, + }, + { + "Tags": { + "Key": "2i2c.org/cluster-name", + "Values": [CLUSTER_NAME], + "MatchOptions": ["EQUALS"], + }, + }, + { + "Not": { + "Tags": { + "Key": "2i2c:hub-name", + "MatchOptions": ["ABSENT"], + }, + }, + }, + { + "Not": { + "Tags": { + "Key": "2i2c:node-purpose", + "MatchOptions": ["ABSENT"], + }, + }, + }, + ] +} + +GROUP_BY_HUB_TAG = { + "Type": "TAG", + "Key": "2i2c:hub-name", +} diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/query.py b/helm-charts/aws-ce-grafana-backend/mounted-files/query.py new file mode 100644 index 0000000000..756ea51f90 --- /dev/null +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/query.py @@ -0,0 +1,173 @@ +""" +Queries to AWS Cost Explorer to get different kinds of cost data. +""" + +import boto3 + +from .const import ( + FILTER_ATTRIBUTABLE_COSTS, + FILTER_USAGE_COSTS, + GRANULARITY_DAILY, + GROUP_BY_HUB_TAG, + METRICS_UNBLENDED_COST, +) + +aws_ce_client = boto3.client("ce") + + +def query_aws_cost_explorer(metrics, granularity, from_date, to_date, filter, group_by): + """ + Function meant to be responsible for making the API call and handling + pagination etc. Currently pagination isn't handled. + """ + # ref: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ce/client/get_cost_and_usage.html#get-cost-and-usage + response = aws_ce_client.get_cost_and_usage( + Metrics=metrics, + Granularity=granularity, + TimePeriod={"Start": from_date, "End": to_date}, + Filter=filter, + GroupBy=group_by, + ) + return response + + +def query_total_costs(from_date, to_date): + """ + A query with processing of the response tailored query to report hub + independent total costs. + """ + response = query_aws_cost_explorer( + metrics=[METRICS_UNBLENDED_COST], + granularity=GRANULARITY_DAILY, + from_date=from_date, + to_date=to_date, + filter={ + "And": [ + FILTER_USAGE_COSTS, + FILTER_ATTRIBUTABLE_COSTS, + ] + }, + group_by=[], + ) + + # response["ResultsByTime"] is a list with entries looking like this... + # + # [ + # { + # "Estimated": false, + # "Groups": [], + # "TimePeriod": { + # "End": "2024-07-28", + # "Start": "2024-07-27", + # }, + # "Total": { + # "UnblendedCost": { + # "Amount": "23.3110299724", + # "Unit": "USD", + # }, + # }, + # }, + # # ... + # ] + # + # processed_response is a list with entries looking like this... + # + # [ + # { + # "date":"2024-08-30", + # "cost":"12.19", + # }, + # ] + # + processed_response = [ + { + "date": e["TimePeriod"]["Start"], + "cost": f'{float(e["Total"]["UnblendedCost"]["Amount"]):.2f}', + } + for e in response["ResultsByTime"] + ] + return processed_response + + +def query_total_costs_per_hub(from_date, to_date): + """ + A query with processing of the response tailored query to report total costs + per hub, where costs not attributed to a specific hub is listed under + 'shared'. + """ + response = query_aws_cost_explorer( + metrics=[METRICS_UNBLENDED_COST], + granularity=GRANULARITY_DAILY, + from_date=from_date, + to_date=to_date, + filter={ + "And": [ + FILTER_USAGE_COSTS, + FILTER_ATTRIBUTABLE_COSTS, + ] + }, + group_by=[ + GROUP_BY_HUB_TAG, + ], + ) + + # response["ResultsByTime"] is a list with entries looking like this... + # + # [ + # { + # "TimePeriod": {"Start": "2024-08-30", "End": "2024-08-31"}, + # "Total": {}, + # "Groups": [ + # { + # "Keys": ["2i2c:hub-name$"], + # "Metrics": { + # "UnblendedCost": {"Amount": "12.1930361882", "Unit": "USD"} + # }, + # }, + # { + # "Keys": ["2i2c:hub-name$prod"], + # "Metrics": { + # "UnblendedCost": {"Amount": "18.662514854", "Unit": "USD"} + # }, + # }, + # { + # "Keys": ["2i2c:hub-name$staging"], + # "Metrics": { + # "UnblendedCost": {"Amount": "0.000760628", "Unit": "USD"} + # }, + # }, + # { + # "Keys": ["2i2c:hub-name$workshop"], + # "Metrics": { + # "UnblendedCost": {"Amount": "0.1969903219", "Unit": "USD"} + # }, + # }, + # ], + # "Estimated": False, + # }, + # ] + # + # processed_response is a list with entries looking like this... + # + # [ + # { + # "date":"2024-08-30", + # "cost":"12.19", + # "name":"shared", + # }, + # ] + # + processed_response = [] + for e in response["ResultsByTime"]: + processed_response.extend( + [ + { + "date": e["TimePeriod"]["Start"], + "cost": f'{float(g["Metrics"]["UnblendedCost"]["Amount"]):.2f}', + "name": g["Keys"][0].split("$", maxsplit=1)[1] or "shared", + } + for g in e["Groups"] + ] + ) + + return processed_response diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py index fa72767c42..d8ef70345d 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py @@ -2,21 +2,11 @@ from flask import Flask, request -from .aws import query_total_cost +from .query import query_total_costs, query_total_costs_per_hub app = Flask(__name__) -@app.route("/") -def hello_world(): - return "

Hello, World!

" - - -@app.route("/health/ready") -def ready(): - return ("", 204) - - def parse_from_to_in_query_params(): """ Parse "from" and "to" query parameters, expected to arrive as YYYY-MM-DD @@ -47,8 +37,20 @@ def parse_from_to_in_query_params(): return from_date, to_date -@app.route("/aws/total-cost") -def aws_total_cost(): +@app.route("/health/ready") +def ready(): + return ("", 204) + + +@app.route("/total-costs") +def total_costs(): + from_date, to_date = parse_from_to_in_query_params() + + return query_total_costs(from_date, to_date) + + +@app.route("/total-costs-per-hub") +def total_costs_per_hub(): from_date, to_date = parse_from_to_in_query_params() - return query_total_cost(from_date, to_date) + return query_total_costs_per_hub(from_date, to_date) diff --git a/helm-charts/aws-ce-grafana-backend/templates/deployment.yaml b/helm-charts/aws-ce-grafana-backend/templates/deployment.yaml index a9489f8776..7adbf7b524 100644 --- a/helm-charts/aws-ce-grafana-backend/templates/deployment.yaml +++ b/helm-charts/aws-ce-grafana-backend/templates/deployment.yaml @@ -37,10 +37,14 @@ spec: - name: secret mountPath: /srv/aws-ce-grafana-backend readOnly: true - {{- with .Values.extraEnv }} env: + - name: PYTHONUNBUFFERED + value: "1" + - name: AWS_CE_GRAFANA_BACKEND__CLUSTER_NAME + value: "{{ .Values.envBasedConfig.clusterName }}" + {{- with .Values.extraEnv }} {{- tpl (. | toYaml) $ | nindent 12 }} - {{- end }} + {{- end }} resources: {{- .Values.resources | toYaml | nindent 12 }} securityContext: diff --git a/helm-charts/aws-ce-grafana-backend/values.schema.yaml b/helm-charts/aws-ce-grafana-backend/values.schema.yaml index 2d793009bb..70760363ca 100644 --- a/helm-charts/aws-ce-grafana-backend/values.schema.yaml +++ b/helm-charts/aws-ce-grafana-backend/values.schema.yaml @@ -17,6 +17,8 @@ additionalProperties: false required: # General configuration - global + # Software configuration + - envBasedConfig # Deployment resource - image # Other resources @@ -43,6 +45,17 @@ properties: type: object additionalProperties: true + # Software configuration + # --------------------------------------------------------------------------- + # + envBasedConfig: + type: object + additionalProperties: false + required: [clusterName] + properties: + clusterName: + type: string + # Deployment resource # --------------------------------------------------------------------------- # diff --git a/helm-charts/aws-ce-grafana-backend/values.yaml b/helm-charts/aws-ce-grafana-backend/values.yaml index 479a18b45c..25eadf8459 100644 --- a/helm-charts/aws-ce-grafana-backend/values.yaml +++ b/helm-charts/aws-ce-grafana-backend/values.yaml @@ -5,6 +5,20 @@ nameOverride: "" fullnameOverride: "" global: {} +# Software configuration +# ----------------------------------------------------------------------------- +# +# FIXME: This chart currently configures software via environment variables, and +# it doesn't scale well for a few string values without validation logic. +# +# A better scaling approach is for example to use traitlets and config from the +# chart be mounted via a k8s Secret to be read by the software. That way, +# the chart doesn't have to be updated and maintained to be in parity +# with the software config options. +# +envBasedConfig: + clusterName: "" + # Deployment resource # ----------------------------------------------------------------------------- #