This repository has been archived by the owner on Feb 21, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add GCP infrastructure tracking (#118)
- Loading branch information
1 parent
0d24bef
commit 1adc27e
Showing
14 changed files
with
382 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| from unittest.mock import MagicMock | ||
| import pytest | ||
|
|
||
| from zoo.datacenters import gcp as uut, models | ||
|
|
||
| pytestmark = pytest.mark.django_db | ||
|
|
||
|
|
||
| def test_gcp_map_to_nodes(mocker): | ||
| mocker.patch("zoo.datacenters.utils.gcloud.GCPClient.__init__", return_value=None) | ||
| mocker.patch( | ||
| "zoo.datacenters.utils.gcloud.GCPClient.get_all_projects", | ||
| return_value=[{"projectId": "pid1"}, {"projectId": "pid2"}], | ||
| ) | ||
| mocker.patch( | ||
| "zoo.datacenters.utils.gcloud.GCPClient.get_forwarding_rules", | ||
| return_value=[ | ||
| { | ||
| "id": "test1", | ||
| "loadBalancingScheme": "EXTERNAL", | ||
| "IPAddress": "1.1.1.1", | ||
| "portRange": "443-443", | ||
| }, | ||
| { | ||
| "id": "test2", | ||
| "loadBalancingScheme": "INTERNAL", | ||
| "IPAddress": "2.2.2.2", | ||
| "portRange": "443-443", | ||
| }, | ||
| ], | ||
| ) | ||
| mocker.patch( | ||
| "zoo.datacenters.utils.GCPClient.get_all_clusters", | ||
| return_value=[{"name": "test", "zone": "europe-test"}], | ||
| ) | ||
| mocker.patch( | ||
| "zoo.datacenters.utils.kube.KubernetesClient.__init__", return_value=None | ||
| ) | ||
|
|
||
| workload = MagicMock() | ||
| image1 = MagicMock() | ||
| image2 = MagicMock() | ||
| image1.image = "test/image:0.0.1" | ||
| image2.image = "test/image2:0.0.2" | ||
|
|
||
| workload.metadata.namespace = "namespace-test" | ||
| workload.metadata.name = "resource-test" | ||
| workload.spec.template.spec.containers = [image1, image2] | ||
|
|
||
| mocker.patch( | ||
| "zoo.datacenters.utils.kube.KubernetesClient.iter_workloads", | ||
| return_value={"test-type": [workload]}, | ||
| ) | ||
|
|
||
| uut.map_to_nodes() | ||
|
|
||
| root = models.InfraNode.objects.get(kind=models.NodeKind.GCP_ROOT_PROJ) | ||
|
|
||
| projects = {project.value: project for project in root.targets.all()} | ||
| assert set(projects) == {"pid1", "pid2"} | ||
|
|
||
| ctx = "gke_pid1_europe-test_test" | ||
| clusters = { | ||
| cluster.value: cluster | ||
| for cluster in projects["pid1"].targets.filter( | ||
| kind=models.NodeKind.GCP_CLUSTER_NAME | ||
| ) | ||
| } | ||
| assert set(clusters) == {ctx} | ||
|
|
||
| ip_rules = { | ||
| cluster.value: cluster | ||
| for cluster in projects["pid1"].targets.filter( | ||
| kind=models.NodeKind.GCP_IP_RULE_NAME | ||
| ) | ||
| } | ||
| assert set(ip_rules) == {"test1:1.1.1.1:443-443"} | ||
|
|
||
| workloads = { | ||
| workload.value: workload | ||
| for workload in clusters["gke_pid1_europe-test_test"].targets.all() | ||
| } | ||
| full_name = "test-type:namespace-test/resource-test" | ||
| assert set(workloads) == {f"{ctx}:{full_name}"} | ||
|
|
||
| images = { | ||
| image.value: image for image in workloads[f"{ctx}:{full_name}"].targets.all() | ||
| } | ||
| assert set(images) == {"test/image:0.0.1", "test/image2:0.0.2"} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| from django.db import transaction | ||
|
|
||
| from .models import InfraNode, NodeKind | ||
| from .utils import GCPClient, KubernetesClient | ||
|
|
||
| CLUSTER_IDENTIFIER = "gke_{project_id}_{zone}_{name}" | ||
|
|
||
|
|
||
| def _workload_identifier(cluster, resource_type, resource): | ||
| return f"{cluster}:{resource_type}:{resource.metadata.namespace}/{resource.metadata.name}" | ||
|
|
||
|
|
||
| @transaction.atomic | ||
| def map_to_nodes(): | ||
| """Map GCP projects to GCP services. | ||
| Creates records in the InfraNode table of the following kinds: | ||
| - ``gcp.root.proj`` - root node for all GCP projects | ||
| - ``gcp.proj.id`` - GCP project ID | ||
| - ``gcp.ip_rule.name`` - GCP forwarding rule name | ||
| - ``gcp.cluster.name`` - GCP cluster name | ||
| - ``gcp.workload.name`` - GCP workload name (including the namespace) | ||
| - ``docker.image.uuid`` - Docker image UUID | ||
| """ | ||
| root = InfraNode.get_or_create_node(kind=NodeKind.GCP_ROOT_PROJ, value="*") | ||
| gcloud = GCPClient() | ||
|
|
||
| for project in gcloud.get_all_projects(): | ||
| project_node = InfraNode.get_or_create_node( | ||
| kind=NodeKind.GCP_PROJ_ID, value=project["projectId"], source=root | ||
| ) | ||
|
|
||
| # currently not used anywhere | ||
| for ip_rule in gcloud.get_forwarding_rules(project["projectId"]): | ||
| if ip_rule["loadBalancingScheme"] == "EXTERNAL": | ||
| InfraNode.get_or_create_node( | ||
| kind=NodeKind.GCP_IP_RULE_NAME, | ||
| value=f"{ip_rule['id']}:{ip_rule['IPAddress']}:{ip_rule['portRange']}", | ||
| source=project_node, | ||
| ) | ||
|
|
||
| for cluster in gcloud.get_all_clusters(project["projectId"]): | ||
| cluster_ctx = CLUSTER_IDENTIFIER.format( | ||
| project_id=project["projectId"], | ||
| zone=cluster["zone"], | ||
| name=cluster["name"], | ||
| ) | ||
| cluster_node = InfraNode.get_or_create_node( | ||
| kind=NodeKind.GCP_CLUSTER_NAME, value=cluster_ctx, source=project_node | ||
| ) | ||
|
|
||
| kube = KubernetesClient(cluster) | ||
| workloads = kube.iter_workloads() | ||
|
|
||
| for resource_type, resources in workloads.items(): | ||
| for resource in resources: | ||
| workload_node = InfraNode.get_or_create_node( | ||
| kind=NodeKind.GCP_WORKLOAD_NAME, | ||
| value=_workload_identifier( | ||
| cluster_ctx, resource_type, resource | ||
| ), | ||
| source=cluster_node, | ||
| ) | ||
|
|
||
| for container in resource.spec.template.spec.containers: | ||
| InfraNode.get_or_create_node( | ||
| kind=NodeKind.DOCKER_IMAGE_UUID, | ||
| value=container.image, | ||
| source=workload_node, | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| from .gcloud import GCPClient | ||
| from .kube import KubernetesClient |
Oops, something went wrong.