Add commands to generate cost of each GCP project

config/clusters/2i2c-uk/cluster.yaml

@@ -5,6 +5,12 @@ gcp:
   project: two-eye-two-see-uk
   cluster: two-eye-two-see-uk-cluster
   zone: europe-west2
+  billing:
+    paid_by_us: true
+    bigquery:
+      project: two-eye-two-see
+      dataset: cloud_costs
+      billing_id: 0157F7-E3EA8C-25AC3C
 support:
   helm_chart_values_files:
     - support.values.yaml

config/clusters/2i2c/cluster.yaml

@@ -5,6 +5,12 @@ gcp:
   project: two-eye-two-see
   cluster: pilot-hubs-cluster
   zone: us-central1-b
+  billing:
+    paid_by_us: true
+    bigquery:
+      project: two-eye-two-see
+      dataset: cloud_costs
+      billing_id: 0157F7-E3EA8C-25AC3C
 support:
   helm_chart_values_files:
     - support.values.yaml

config/clusters/awi-ciroh/cluster.yaml

@@ -5,6 +5,12 @@ gcp:
   project: awi-ciroh
   cluster: awi-ciroh-cluster
   zone: us-central1
+  billing:
+    paid_by_us: true
+    bigquery:
+      project: two-eye-two-see
+      dataset: cloud_costs
+      billing_id: 0157F7-E3EA8C-25AC3C
 support:
   helm_chart_values_files:
     - support.values.yaml

config/clusters/callysto/cluster.yaml

@@ -5,6 +5,8 @@ gcp:
   project: callysto-202316
   cluster: callysto-cluster
   zone: northamerica-northeast1
+  billing:
+    paid_by_us: false
 support:
   helm_chart_values_files:
     - support.values.yaml

config/clusters/cloudbank/cluster.yaml

@@ -5,6 +5,8 @@ gcp:
   project: cb-1003-1696
   cluster: cb-cluster
   zone: us-central1-b
+  billing:
+    paid_by_us: false
 support:
   helm_chart_values_files:
     - support.values.yaml

config/clusters/leap/cluster.yaml

@@ -5,6 +5,12 @@ gcp:
   project: leap-pangeo
   cluster: leap-cluster
   zone: us-central1
+  billing:
+    paid_by_us: true
+    bigquery:
+      project: leap-pangeo
+      dataset: cloud_costs
+      billing_id: 01A164-923D17-3199D9
 support:
   helm_chart_values_files:
     - support.values.yaml

config/clusters/linked-earth/cluster.yaml

@@ -10,6 +10,8 @@ gcp:
   # Could not find [linked-earth-cluster] in [us-central1-c].
   # Did you mean [linked-earth-cluster] in [us-central1]?
   zone: us-central1
+  billing:
+    paid_by_us: false
 support:
   helm_chart_values_files:
     - support.values.yaml

config/clusters/m2lines/cluster.yaml

@@ -5,6 +5,12 @@ gcp:
   project: m2lines-hub
   cluster: m2lines-cluster
   zone: us-central1
+  billing:
+    paid_by_us: true
+    bigquery:
+      project: two-eye-two-see
+      dataset: cloud_costs
+      billing_id: 0157F7-E3EA8C-25AC3C
 support:
   helm_chart_values_files:
     - support.values.yaml

config/clusters/meom-ige/cluster.yaml

@@ -5,6 +5,8 @@ gcp:
   project: meom-ige-cnrs
   cluster: meom-ige-cluster
   zone: us-central1-b
+  billing:
+    paid_by_us: false
 support:
   helm_chart_values_files:
     - support.values.yaml

config/clusters/pangeo-hubs/cluster.yaml

@@ -6,6 +6,8 @@ gcp:
   project: pangeo-integration-te-3eea
   cluster: pangeo-hubs-cluster
   zone: us-central1-b
+  billing:
+    paid_by_us: false
 support:
   helm_chart_values_files:
     - support.values.yaml

config/clusters/qcl/cluster.yaml

@@ -6,6 +6,12 @@ gcp:
   cluster: qcl-cluster
   # We default to a regional cluster
   zone: europe-west1
+  billing:
+    paid_by_us: true
+    bigquery:
+      project: two-eye-two-see
+      dataset: cloud_costs
+      billing_id: 0157F7-E3EA8C-25AC3C
 support:
   helm_chart_values_files:
     - support.values.yaml

config/secrets/enc-billing-gsheets-writer-key.secret.json

@@ -0,0 +1,30 @@
+{
+    "type": "ENC[AES256_GCM,data:o6wWhUXNbwAvcer0wg9r,iv:FnWvVwIdXL/GbUuohrWIzNVh67kKhRjK53XpRp+5Vks=,tag:zKbBhldajOu2dU+N7roVvw==,type:str]",
+    "project_id": "ENC[AES256_GCM,data:mNgWzl1pzxUClw0sqPC6,iv:AbnzyB5BXlgpJgQ/Ykm9UTZv7PAfBbYhS+Sn+0z2A08=,tag:uCDrzrHt7Rv2w+DtvTmzIw==,type:str]",
+    "private_key_id": "ENC[AES256_GCM,data:i4lJH5jKTl9gxnxTsz9uaXw+f3ZmFFD+1p57vAUHCev4pkljqFQOng==,iv:Putcyq0yq8m88rEVZSIGXPc6LQEBfOYYDtyyudylu5s=,tag:taWqmzPzbFD1jnpz7WzrOg==,type:str]",
+    "private_key": "ENC[AES256_GCM,data:wOMeWridCo/VZp1CVJvSN5llt6TN2qbR7531f/u2k1h8zWInFx9HvummpgIA4Pp2FieDPjTKolc2l1fw7BY4smYjpOays0nrP+zjzfq4QlCEi7PfpnCwtqf6mEVXKWxQrd2zq82FEKdVHw5ETLfvMdZFCb9fQv3sreycXQJzyroCw+PZ+D+32n093Ljk8ECv/I5o1zld/EgDQWjdgU3Ptn54mLn8amQlNcbHCvLnfBkN8Eq/rDE7yGtfJfgbA9peBwbUzLzwgPjBRdkpGTawzDcCj9kU2mJIKgsYTMhLaf3yYlA2H/tETh3FbqQn/2k//9l1t5ipoPrYWbjny53uTD58QWeqIkS5ftPbGqBmJcgu5hmg2Uy8U9tguh7YCpQ3RWzM2QgUNrL3gGvGa442d6KUESI92sLX+l6NuKpdNWjV2E5uOMZWqc1O/4w2xsKQsI7fsQMTt9z5kcoE29tg5iR+Bu/n7WLwXxRUhLD4Z11wOHlOiGcySWMz9KsZDUY5aXg/9A1SUc8O394l4LILjqYNusQI79UVbhulcTazeNqrerLsNk/PfM8A08s6G+KfOEIG8DGhxNsKQdHTM9eMKpRPpSojg6f5VggC6EKWBwDRntjhde7t6xmePnxZHLSNuaWB9l3qxfUnFxlC2zD9uQJc5Lg2mGfucxroen5rKisUEeKkwW1KuHJeY+AYSssIqVggwOKYtr5csKNNlYlDkt71EYYQH91lQ3XJDEoAt5QeKSwS/dBbqxJZH9g4SXTKLQ95yjd1Edmz4X9eBm5CNgK8j6NA0u0YCKwzN3W3tidKOy8XdDlNyOXGJ89IrnrodEmunUMTxiY5WpYg1ZSXDgn7Uy5qHAElkLLdCcIFlaZcI3d19tPZAlklVfRa4OGpo0VAuYKK5ZDkLOHw890fEYT0UQYKDWlJopMoMfZmEDctse7/5mlF0zqVkY9M/cPYKU434/rQfjv60POaE6m+ph4PUsTzgWw3bEeD893VyR6kct1RrAQNLzyvPloESW/4wN1z7ukRiVOlqycCnnRK4Vhgm8dXdm0T+KQinJW9vFp0Bmz1KOwwXmq1rihqkoqEjEcQuOmOlciJkw3I1AMTm54Xb2TOvzqO7EpsNSog1H6mZA611DApupSOW16JtoaRZvThHU1Lsf1NVW2ciUZtQ8GJ21lk2B+fhvg3xdTpo9gUTw1oGUmOfqMbK9eAga14Fu4hMfAHmC5dfGq0j7zhSvhMI/wWS+vcWtkQTMUVPV1nIZ8taGUJ5Fk0nYuwDtncajRYcT6+rFUCNKVCaRPbVcvIAM7oYXlCIOVLG4G1YvD5sV6Gp0/izYEgQ8YlC+JBnAvdxDdzQ7746uFp7sZlv7FtsjJ/4LCG2JfWP4UPzK3DvzKlKI72ISkNXAl/t33uxN9nfd1yhPBrznitqbhm+NpzaPQHCvbJywOwmjIcv5k0UH85JEVLfvLEHdQXskOuoF+B5kM4zDAg3CZpjGxAI6D5EvkJdcEiuiWRXI/T7hC1xt6qdW3WYzSABhojyegSZYkTomRO6Sb/7cw+LiaS7z0+EBxNlLqm0AI+u4EPTYSKGrgDrLtoycxxEkcb+faYqxTjsvF4Ho164ssU3dCbhAvusI7/RQB4hWwZJc3C2P0A/0ob0SbQAW6dfYETxlCDgzTuttARuXeKP2X+ozGMs8OeY+HsLL+0SbAxyKLlY0YwtAeS7VinNHOuG+SwY3B/yxPOSybY7uh0QB2F7fTGMbI38t6DVuHl1r46VfF0VytFJ3xxI+dH4b7tZ6noHjet5T8440IMtLsJ9aPslxMJkIJDkg6eOcLvWDopimIslFjDGTk9nCQJHdajf0LRedBX1nyNHS6QP6c4xxycTy4+DeBglmnHVaQiJdc7A6iG+JSokhW4VF8/LrfMFiKMki42lFsBxKZ0jcTSr6Z+pLSXeR4Q94dR9gRYhy6dj+mfqAeATi2SuAvQ5x5iJT70bsi3VOYioABLsldNCiGcJhJwdYaTB8E/meF1P5JQvd5vbrtwQZZhXbCBGk0Kgv40jroH5yfxKZDNEw9cMJ5IaC+GxljJh8ePoo2BewIO5I48OmjQMcTh7jb/Kv3oSRIhpNAsn+RoGM0AZE3SSPBliFoX3vmsxqgcpZn4QSXHbjJVUzOekpsh+YovlMpGmqOJWPUebMC681Tmwp+eky3/2ZbyxTF8iMn2CQulGsXU3kxKuDLntjNLDBkT9rwgEcfu8Wi/89Wm5vg10p9Rotu/ZCHqfukmJhRVFBdP,iv:AtQVXvwP4s+URfMY4DjzrQW6Khry0ZJDNHcrWSAr4Sc=,tag:+izq8CfFxr1xSONU19flyg==,type:str]",
+    "client_email": "ENC[AES256_GCM,data:3aI9s0Ye+9Dlf64n650QhZdb4OTJAVxAjDfCWhSgSRDHZVj7m9wXvq++a2wH3lbpPFWLk4h0R80kos9iZgAvMixs,iv:NN5vn7u8Q02Po1msbSR9mMYUGGcM+nH3YDTpW7vM1RA=,tag:21PvO2u3kOcZr3HpnbvM3g==,type:str]",
+    "client_id": "ENC[AES256_GCM,data:LHU4wIlM72UwIZsttZ7OodwdHbDK,iv:5CZmEcvW0d1D6aub6nXGJBgOYyZVpMdHCFcOE0pKk9M=,tag:vqkCGrqwjjjREBlbFtthwA==,type:str]",
+    "auth_uri": "ENC[AES256_GCM,data:3ut4UmulEm6pEs31FKj1j2qPmtPz2c1+Dus4x5AxSAbP096TlpONnFg=,iv:b//+zzCEJKk6tZN/l3D/PGsnc6GFJVd4tOylPkAsCoE=,tag:Rk9CrnpRg46viYstbsVdBg==,type:str]",
+    "token_uri": "ENC[AES256_GCM,data:HLAVVZ8+2R+J4W23LKPHmDoi1CO8egFt3Mq3cPIv+ZAqiq4=,iv:A06Z0/UUzFL3IjZ35UhKGs0tqxlOGhdFQ6xFGqGWjVE=,tag:p3nT0C4NZrrQdxFqcjl4xQ==,type:str]",
+    "auth_provider_x509_cert_url": "ENC[AES256_GCM,data:gw6HEhJXLAymNrEVDGzes2qHw6Gna95ySHfgijI4h+7viFp5LPQe5bB8,iv:xyOGIKnDoh9/e+YbHhVUjj17aAGeACmOAm1nW4/WcIk=,tag:Dz613LzWxZ3EaNQ/MRgCzg==,type:str]",
+    "client_x509_cert_url": "ENC[AES256_GCM,data:QwBdXPM6oZtMSyWSPFUwjDTjN7aXkYXHWgcMC+PPtSlTLJKfWXcEDiHxw+f7NVhI76eKSEQx4kjzsi1LPZph1hpEJQHp8a+VZF7nRSxcP4YO/nf/noxgvlS/OQvBrfPzzao5gmo67FTMGq7+KbQs2F6ARabyxQ==,iv:WITwxW28IRmHFI44YdiB2uzGs5FF2t3z8raSV92w8gs=,tag:WocDbaa/bHikxymqs1r0KA==,type:str]",
+    "sops": {
+        "kms": null,
+        "gcp_kms": [
+            {
+                "resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs",
+                "created_at": "2022-12-07T02:39:00Z",
+                "enc": "CiUA4OM7eKKHdNGpKkMDwGl35ZJPB1j7tQkHY0EIIX+uAN+IoOafEkkA+0T9hTGBwve/2QgXgzIisotXc1rmiQOvB4rARVMhAxRSQ41VdXvfXS0K+vy065kF9QAKnyHEFmx0j/Vloc7x74EMt0nNbA1V"
+            }
+        ],
+        "azure_kv": null,
+        "hc_vault": null,
+        "age": null,
+        "lastmodified": "2022-12-07T02:39:01Z",
+        "mac": "ENC[AES256_GCM,data:MU4sihnMn2PbM/3uzD+Qkt1/1dVhA8J6GyelCY1kWZ5qt3s4MuU03PAFbFzSqeO6PGEOWn2+bAm52DLiORYx+g9XrcJsRvIzbDWpmcJyftUU2u/yh/SRaizipYEqx35p5gsNoqUvWh9WGCI1ZrmmbhXKL/quWdkZ9oGLmmtj9RQ=,iv:U587DppVPNPBv9gw3lhtjvxyv/J7mIT86vRNuMY9LSs=,tag:rMKD6qO3NT5fjdh2Uc+gpQ==,type:str]",
+        "pgp": null,
+        "unencrypted_suffix": "_unencrypted",
+        "version": "3.7.3"
+    }
+}

deployer/__main__.py

@@ -1,6 +1,7 @@
 # Import the various subcommands here, they will be automatically
 # registered into the app
 import deployer.auth0_app  # noqa: F401
+import deployer.billing  # noqa: F401
 import deployer.cilogon_app  # noqa: F401
 import deployer.debug  # noqa: F401
 import deployer.deployer  # noqa: F401

deployer/billing.py

@@ -0,0 +1,206 @@
+import re
+from datetime import datetime
+from enum import Enum
+from pathlib import PosixPath
+
+import gspread
+import typer
+from dateutil.relativedelta import relativedelta
+from google.cloud import bigquery
+from rich.console import Console
+from rich.table import Table
+from ruamel.yaml import YAML
+
+from .cli_app import app
+from .file_acquisition import get_decrypted_file
+from .helm_upgrade_decision import get_all_cluster_yaml_files
+
+yaml = YAML(typ="safe")
+
+HERE = PosixPath(__file__).parent.parent
+
+
+def month_validate(month_str: str):
+    """
+    Validate passed string matches YYYY-MM format.
+
+    Returns values in YYYYMM format, which is used by bigquery
+    """
+    match = re.match(r"(\d\d\d\d)-(\d\d)", month_str)
+    if not match:
+        raise typer.BadParameter(
+            f"{month_str} should be formatted as YYYY-MM (eg: 2023-02)"
+        )
+    return f"{match.group(1)}{match.group(2)}"
+
+
+class CostTableOutputFormats(Enum):
+    """
+    Output formats supported by the generate-cost-table command
+    """
+
+    terminal = "terminal"
+    google_sheet = "google-sheet"
+
+
+@app.command()
+def generate_cost_table(
+    start_month: str = typer.Option(
+        (datetime.today() - relativedelta(months=12)).replace(day=1).strftime("%Y-%m"),
+        help="Starting month (as YYYY-MM) to produce cost data for. Defaults to 12 invoicing months ago.",
+        callback=month_validate,
+    ),
+    end_month: str = typer.Option(
+        datetime.utcnow().replace(day=1).strftime("%Y-%m"),
+        help="Ending month (as YYYY-MM) to produce cost data for. Defaults to current invoicing month",
+        callback=month_validate,
+    ),
+    output: CostTableOutputFormats = typer.Option(
+        CostTableOutputFormats.terminal,
+        help="Where to output the cost table to",
+    ),
+    google_sheet_url: str = typer.Option(
+        "https://docs.google.com/spreadsheets/d/1URYCMap-Lxm4e_pAAC3Esxda7tZzRhCS6d85pxUiVQs/edit#gid=0",
+        help="Write to given Google Sheet URL. Used when --output is google-sheet. billing-spreadsheet-writer@two-eye-two-see.iam.gserviceaccount.com should have Editor rights on this spreadsheet.",
+    ),
+):
+    """
+    Generate table with cloud costs for all GCP projects we pass costs through for.
+    """
+
+    cluster_files = get_all_cluster_yaml_files()
+    client = bigquery.Client()
+    rows = []
+
+    for cf in cluster_files:
+        with open(cf) as f:
+            cluster = yaml.load(f)
+        if cluster["provider"] != "gcp":
+            # We only support GCP for now
+            continue
+
+        if not cluster["gcp"]["billing"]["paid_by_us"]:
+            continue
+
+        cluster_project_name = cluster["gcp"]["project"]
+
+        bq = cluster["gcp"]["billing"]["bigquery"]
+
+        # WARN: We are using string interpolation here to construct a sql-like query, which
+        # IS GENERALLY VERY VERY BAD AND NO GOOD AND WE SHOULD NOT DO IT NO EVER.
+        # HOWEVER, I can't seem to find a way to parameterize the *table name* as we must do here,
+        # rather than just query parameters. So we *very* carefully construct the name of the table here,
+        # and use that in the query. In addition, we allow-list the characters available to the table name as
+        # well - and fail hard if something is fishy. This shouldn't really be a problem, as we control the
+        # input to this function (via our YAML file). However, SQL Injections are likely to happen in places
+        # where you least expect them to happen, so the extra layer of protection is nice.
+        table_name = f'{bq["project"]}.{bq["dataset"]}.gcp_billing_export_resource_v1_{bq["billing_id"].replace("-", "_")}'
+        # Make sure the table name only has alphanumeric characters, _ and -
+        assert re.match(r"^[a-zA-Z0-9._-]+$", table_name)
+        query = f"""
+        SELECT
+        invoice.month as month,
+        project.id as project,
+        (SUM(CAST(cost AS NUMERIC))
+            + SUM(IFNULL((SELECT SUM(CAST(c.amount AS NUMERIC))
+                        FROM UNNEST(credits) AS c), 0)))
+            AS total_with_credits
+        FROM `{table_name}`
+        WHERE invoice.month >= @start_month
+              AND invoice.month <= @end_month
+              AND project.id = @project
+        GROUP BY 1, 2
+        ORDER BY invoice.month ASC
+        ;
+
+        """
+
+        job_config = bigquery.QueryJobConfig(
+            query_parameters=[
+                bigquery.ScalarQueryParameter("start_month", "STRING", start_month),
+                bigquery.ScalarQueryParameter("end_month", "STRING", end_month),
+                bigquery.ScalarQueryParameter(
+                    "project", "STRING", cluster_project_name
+                ),
+            ]
+        )
+
+        result = client.query(query, job_config=job_config).result()
+        last_period = None
+        for r in result:
+            if not r.project:
+                # Non-project number is 0$, let's declutter by not showing it
+                continue
+            year = r.month[:4]
+            month = r.month[4:]
+            period = f"{year}-{month}"
+            rows.append(
+                {
+                    "period": period,
+                    "project": r.project,
+                    "total_with_credits": float(r.total_with_credits),
+                }
+            )
+
+    # Sort by period in reverse chronological order
+    rows.sort(key=lambda r: r["period"], reverse=True)
+
+    # TODO: Split these output formats out into their own functions
+    if output == CostTableOutputFormats.google_sheet:
+        # A service account (https://console.cloud.google.com/iam-admin/serviceaccounts/details/113674037014124702779?project=two-eye-two-see)
+        # It is created with no permissions, and the google sheet we want to write to
+        # must give write permissions to the email account for the service account
+        # In this case, it is  billing-spreadsheet-writer@two-eye-two-see.iam.gserviceaccount.com .
+        with get_decrypted_file(
+            "config/secrets/enc-billing-gsheets-writer-key.secret.json"
+        ) as f:
+            gsheets = gspread.service_account(filename=f)
+
+        spreadsheet = gsheets.open_by_url(google_sheet_url)
+        worksheet = spreadsheet.get_worksheet(0)
+        worksheet.clear()
+
+        worksheet.append_row(
+            [
+                "WARNING: Do not manually modify, this sheet is autogenerated by the generate-cost-table subcommand of the deployer"
+            ]
+        )
+        worksheet.append_row([f"Last Updated: {datetime.utcnow().isoformat()}"])
+
+        worksheet.append_row(
+            [
+                "Period",
+                "Project",
+                "Cost (after Credits)",
+            ]
+        )
+
+        worksheet.append_rows(
+            [
+                [
+                    r["period"],
+                    r["project"],
+                    r["total_with_credits"],
+                ]
+                for r in rows
+            ]
+        )
+    else:
+        table = Table(title="Project Costs")
+
+        table.add_column("Period", justify="right", style="cyan", no_wrap=True)
+        table.add_column("Project", style="white")
+        table.add_column("Cost (after credits)", justify="right", style="green")
+
+        for r in rows:
+            if last_period != None and r["period"] != last_period:
+                table.add_section()
+            table.add_row(
+                r["period"],
+                r["project"],
+                str(round(r["total_with_credits"], 2)),
+            )
+            last_period = r["period"]
+
+        console = Console()
+        console.print(table)