This repository has been archived by the owner on Feb 21, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add GCP infrastructure tracking (#118)
- Loading branch information
1 parent
f22267e
commit fd3a67d
Showing
14 changed files
with
423 additions
and
28 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| from unittest.mock import MagicMock | ||
| import pytest | ||
|
|
||
| from zoo.datacenters import gcp as uut, models | ||
|
|
||
| pytestmark = pytest.mark.django_db | ||
|
|
||
|
|
||
| def test_gcp_map_to_nodes(mocker): | ||
| mocker.patch("zoo.datacenters.utils.gcloud.GCPClient.__init__", return_value=None) | ||
| mocker.patch( | ||
| "zoo.datacenters.utils.gcloud.GCPClient.get_all_projects", | ||
| return_value=[{"projectId": "pid1"}, {"projectId": "pid2"}], | ||
| ) | ||
| mocker.patch( | ||
| "zoo.datacenters.utils.gcloud.GCPClient.get_forwarding_rules", | ||
| return_value=[ | ||
| { | ||
| "id": "test1", | ||
| "loadBalancingScheme": "EXTERNAL", | ||
| "IPAddress": "1.1.1.1", | ||
| "portRange": "443-443", | ||
| }, | ||
| { | ||
| "id": "test2", | ||
| "loadBalancingScheme": "INTERNAL", | ||
| "IPAddress": "2.2.2.2", | ||
| "portRange": "443-443", | ||
| }, | ||
| ], | ||
| ) | ||
| mocker.patch( | ||
| "zoo.datacenters.utils.GCPClient.get_all_clusters", | ||
| return_value=[{"name": "test", "zone": "europe-test"}], | ||
| ) | ||
| mocker.patch( | ||
| "zoo.datacenters.utils.kube.KubernetesClient.__init__", return_value=None | ||
| ) | ||
|
|
||
| workload = MagicMock() | ||
| image1 = MagicMock() | ||
| image2 = MagicMock() | ||
| image1.image = "test/image:0.0.1" | ||
| image2.image = "test/image2:0.0.2" | ||
|
|
||
| workload.metadata.namespace = "namespace-test" | ||
| workload.metadata.name = "resource-test" | ||
| workload.spec.template.spec.containers = [image1, image2] | ||
|
|
||
| mocker.patch( | ||
| "zoo.datacenters.utils.kube.KubernetesClient.iter_workloads", | ||
| return_value={"test-type": [workload]}, | ||
| ) | ||
|
|
||
| uut.map_to_nodes() | ||
|
|
||
| root = models.InfraNode.objects.get(kind=models.NodeKind.GCP_ROOT_PROJ) | ||
|
|
||
| projects = {project.value: project for project in root.targets.all()} | ||
| assert set(projects) == {"pid1", "pid2"} | ||
|
|
||
| ctx = "gke_pid1_europe-test_test" | ||
| clusters = { | ||
| cluster.value: cluster | ||
| for cluster in projects["pid1"].targets.filter( | ||
| kind=models.NodeKind.GCP_CLUSTER_NAME | ||
| ) | ||
| } | ||
| assert set(clusters) == {ctx} | ||
|
|
||
| ip_rules = { | ||
| cluster.value: cluster | ||
| for cluster in projects["pid1"].targets.filter( | ||
| kind=models.NodeKind.GCP_IP_RULE_NAME | ||
| ) | ||
| } | ||
| assert set(ip_rules) == {"test1:1.1.1.1:443-443"} | ||
|
|
||
| workloads = { | ||
| workload.value: workload | ||
| for workload in clusters["gke_pid1_europe-test_test"].targets.all() | ||
| } | ||
| full_name = "test-type:namespace-test/resource-test" | ||
| assert set(workloads) == {f"{ctx}:{full_name}"} | ||
|
|
||
| images = { | ||
| image.value: image for image in workloads[f"{ctx}:{full_name}"].targets.all() | ||
| } | ||
| assert set(images) == {"test/image:0.0.1", "test/image2:0.0.2"} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| from django.db import transaction | ||
| import googleapiclient | ||
|
|
||
| from .models import InfraNode, NodeKind | ||
| from .utils import GCPClient, KubernetesClient | ||
|
|
||
| CLUSTER_IDENTIFIER = "gke_{project_id}_{zone}_{name}" | ||
|
|
||
|
|
||
| def _workload_identifier(cluster, resource_type, resource): | ||
| return f"{cluster}:{resource_type}:{resource.metadata.namespace}/{resource.metadata.name}" | ||
|
|
||
|
|
||
| def _map_workloads(workloads, cluster_ctx, cluster_node): | ||
| for resource_type, resources in workloads.items(): | ||
| for resource in resources: | ||
| workload_node = InfraNode.get_or_create_node( | ||
| kind=NodeKind.GCP_WORKLOAD_NAME, | ||
| value=_workload_identifier(cluster_ctx, resource_type, resource), | ||
| source=cluster_node, | ||
| ) | ||
|
|
||
| if resource_type == "cronjobs": | ||
| containers = resource.spec.job_template.spec.template.spec.containers | ||
| else: | ||
| containers = resource.spec.template.spec.containers | ||
|
|
||
| for container in containers: | ||
| InfraNode.get_or_create_node( | ||
| kind=NodeKind.DOCKER_IMAGE_UUID, | ||
| value=container.image, | ||
| source=workload_node, | ||
| ) | ||
|
|
||
|
|
||
| @transaction.atomic | ||
| def map_to_nodes(): | ||
| """Map GCP projects to GCP services. | ||
| Creates records in the InfraNode table of the following kinds: | ||
| - ``gcp.root.proj`` - root node for all GCP projects | ||
| - ``gcp.proj.id`` - GCP project ID | ||
| - ``gcp.ip_rule.name`` - GCP forwarding rule name | ||
| - ``gcp.cluster.name`` - GCP cluster name | ||
| - ``gcp.workload.name`` - GCP workload name (including the namespace) | ||
| - ``docker.image.uuid`` - Docker image UUID | ||
| """ | ||
| root = InfraNode.get_or_create_node(kind=NodeKind.GCP_ROOT_PROJ, value="*") | ||
| gcloud = GCPClient() | ||
|
|
||
| for project in gcloud.get_all_projects(): | ||
| if project["projectId"].startswith("sys-"): | ||
| # skip "shadow" projects | ||
| # see https://skypicker.slack.com/archives/C1XN8EPAP/p1568641244031000 | ||
| continue | ||
|
|
||
| project_node = InfraNode.get_or_create_node( | ||
| kind=NodeKind.GCP_PROJ_ID, value=project["projectId"], source=root | ||
| ) | ||
|
|
||
| try: | ||
| # skip projects without billing enabled | ||
| ip_rules = list(gcloud.get_forwarding_rules(project["projectId"])) | ||
| except googleapiclient.errors.HttpError: | ||
| continue | ||
|
|
||
| # currently not used anywhere | ||
| for ip_rule in ip_rules: | ||
| port_range = f":{ip_rule['portRange']}" if "portRange" in ip_rule else "" | ||
| if ip_rule["loadBalancingScheme"] == "EXTERNAL": | ||
| InfraNode.get_or_create_node( | ||
| kind=NodeKind.GCP_IP_RULE_NAME, | ||
| value=f"{ip_rule['id']}:{ip_rule['IPAddress']}{port_range}", | ||
| source=project_node, | ||
| ) | ||
|
|
||
| for cluster in gcloud.get_all_clusters(project["projectId"]): | ||
| cluster_ctx = CLUSTER_IDENTIFIER.format( | ||
| project_id=project["projectId"], | ||
| zone=cluster["zone"], | ||
| name=cluster["name"], | ||
| ) | ||
| cluster_node = InfraNode.get_or_create_node( | ||
| kind=NodeKind.GCP_CLUSTER_NAME, value=cluster_ctx, source=project_node | ||
| ) | ||
|
|
||
| kube = KubernetesClient(cluster) | ||
| workloads = kube.iter_workloads() | ||
|
|
||
| _map_workloads(workloads, cluster_ctx, cluster_node) |
Oops, something went wrong.