WIP: Redpanda Cloud in CI

bin/ci-builder

@@ -241,6 +241,9 @@ case "$cmd" in
             --env GIT_AUTHOR_NAME
             --env GIT_COMMITTER_EMAIL
             --env GIT_COMMITTER_NAME
+            # For cloud canary
+            --env REDPANDA_CLOUD_CLIENT_ID
+            --env REDPANDA_CLOUD_CLIENT_SECRET
         )
 
         if [[ $detach_container == "true" ]]; then

misc/python/materialize/redpanda_cloud.py

@@ -0,0 +1,82 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+import os
+import time
+from typing import Any
+
+import requests
+
+
+class RedpandaCloud:
+    def __init__(self) -> None:
+        client_id = os.environ["REDPANDA_CLOUD_CLIENT_ID"]
+        client_secret = os.environ["REDPANDA_CLOUD_CLIENT_SECRET"]
+
+        response = requests.post(
+            "https://auth.prd.cloud.redpanda.com/oauth/token",
+            json={
+                "client_id": client_id,
+                "client_secret": client_secret,
+                "audience": "cloudv2-production.redpanda.cloud",
+                "grant_type": "client_credentials",
+            },
+        )
+        if response.status_code != 200:
+            print(response.text)
+            raise ValueError(response)
+        result = response.json()
+        assert result["expires_in"] >= 3600, result
+        self.token = result["access_token"]
+        self.url_prefix = "https://api.redpanda.com/v1beta2"
+
+    def wait(self, result: dict[str, Any]) -> str:
+        while True:
+            operation_id = result["operation"]["id"]
+            response = requests.get(
+                f"{self.url_prefix}/operations/{operation_id}",
+                headers={"Authorization": f"Bearer {self.token}"},
+            )
+            if response.status_code not in (200, 202):
+                print(response.text)
+                raise ValueError(response)
+            result = response.json()
+            print(result)
+            if result["operation"]["state"] == "STATE_COMPLETED":
+                return result["operation"].get("resource_id")
+            if result["operation"]["state"] == "STATE_FAILED":
+                raise ValueError(result)
+            if result["operation"]["state"] != "STATE_IN_PROGRESS":
+                raise ValueError(result)
+            time.sleep(10)
+
+    def create(self, object: str, json: dict[str, Any] | None) -> str:
+        response = requests.post(
+            f"{self.url_prefix}/{object}",
+            json=json,
+            headers={"Authorization": f"Bearer {self.token}"},
+        )
+        if response.status_code not in (200, 202):
+            print(response.text)
+            raise ValueError(response)
+        result = response.json()
+        print(result)
+        return result
+
+    def delete(self, object: str, id: str) -> dict[str, Any]:
+        response = requests.delete(
+            f"{self.url_prefix}/{object}/{id}",
+            headers={"Authorization": f"Bearer {self.token}"},
+        )
+        if response.status_code not in (200, 202):
+            print(response.text)
+            raise ValueError(response)
+        result = response.json()
+        print(result)
+        return result

test/cloud-canary/mzcompose.py

@@ -27,8 +27,73 @@
 from materialize.mzcompose.services.materialized import Materialized
 from materialize.mzcompose.services.mz import Mz
 from materialize.mzcompose.services.testdrive import Testdrive
+from materialize.redpanda_cloud import RedpandaCloud
 from materialize.ui import UIError
 
+redpanda_cloud = RedpandaCloud()
+
+REDPANDA_RESOURCE_GROUP_NAME = "ci-resource-group"
+# resource_group_id = redpanda_cloud.create(
+#     "resource-groups", {"name": REDPANDA_RESOURCE_GROUP_NAME}
+# )["resource_group"]["id"]
+resource_group_id = "4beee637-d983-429f-8ab0-7a987ad9ffda"
+
+REDPANDA_NETWORK = "ci-network"
+# result = redpanda_cloud.create(
+#     "networks",
+#     {
+#         "name": REDPANDA_NETWORK,
+#         "region": "us-east-1",
+#         "resource_group_id": resource_group_id,
+#         "cluster_type": "TYPE_DEDICATED",
+#         "cloud_provider": "CLOUD_PROVIDER_AWS",
+#         "cidr_block": "10.0.0.0/20",
+#     },
+# )
+# network_id = redpanda_cloud.wait(result)
+network_id = "cr3nqtkgvegbip4fk0l0"
+
+REDPANDA_CLUSTER = "ci-cluster"
+# result = redpanda_cloud.reate(
+#     "clusters",
+#     {
+#         "name": REDPANDA_CLUSTER,
+#         "cloud_provider": "CLOUD_PROVIDER_AWS",
+#         "connection_type": "CONNECTION_TYPE_PUBLIC",
+#         "resource_group_id": resource_group_id,
+#         "network_id": network_id,
+#         "region": "us-east-1",
+#         "throughput_tier": "tier-1-aws-v2-arm",
+#         "type": "TYPE_DEDICATED",
+#         "zones": ["use1-az2"],
+#     },
+# )
+result = {"operation": {"id": "cr30vcbhoe8001ticndg"}}
+cluster_id = redpanda_cloud.wait(result)
+
+# TODO: Do something interesting inbetween, try first on an existing Sandbox to connect to it using VPC
+# time.sleep(2400)
+
+# result = redpanda_cloud.delete("clusters", cluster_id)
+result = {
+    "operation": {
+        "id": "cr30vfa5ra7001o98s5g",
+        "metadata": {
+            "@type": "type.googleapis.com/redpanda.api.controlplane.v1beta2.DeleteClusterMetadata"
+        },
+        "state": "STATE_IN_PROGRESS",
+        "started_at": "2024-08-22T22:52:59.211Z",
+        "type": "TYPE_DELETE_CLUSTER",
+        "resource_id": "cr3r5flifk5911l1mja0",
+    }
+}
+redpanda_cloud.wait(result)
+
+result = redpanda_cloud.delete("networks", network_id)
+redpanda_cloud.wait(result)
+
+redpanda_cloud.delete("resource-groups", resource_group_id)
+
 REGION = "aws/us-east-1"
 ENVIRONMENT = os.getenv("ENVIRONMENT", "staging")
 USERNAME = os.getenv("NIGHTLY_CANARY_USERNAME", "infra+nightly-canary@materialize.com")