Add GCP infrastructure tracking

Dockerfile

@@ -1,8 +1,9 @@
-FROM python:3.7-alpine
+# gcloud doesn't support 3.7
+FROM python:3.6-alpine
 
 ENV ZOO_RDS_PRODUCTION_CERT_PATH=/etc/ssl/certs/rds-combined-ca-bundle.pem \
     DJANGO_SETTINGS_MODULE=zoo.base.settings
-RUN addgroup -S macaque && adduser -H -D -S macaque macaque
+RUN addgroup -S macaque && adduser --h /home/macaque -D -S macaque macaque
 
 WORKDIR /app
 COPY *requirements.txt ./
@@ -14,8 +15,16 @@ RUN apk add --no-cache --virtual=.build-deps curl build-base postgresql-dev && \
     npm install --global yarn && \
     curl -o $ZOO_RDS_PRODUCTION_CERT_PATH https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem && \
     pip install --no-cache-dir -r requirements.txt -r test-requirements.txt && \
+    # gcloud setup
+    curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz && \
+    mkdir -p /usr/local/gcloud && \
+    tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz && \
+    /usr/local/gcloud/google-cloud-sdk/install.sh && \
     apk del .build-deps
 
+ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin
+RUN gcloud components install kubectl
+
 WORKDIR /app/webpack
 COPY webpack/ ./
 RUN yarn install --frozen-lockfile && \

docker-compose.dev.yml

@@ -54,6 +54,7 @@ services:
   worker:
     environment:
       DATABASE_URL: 'postgres://postgres:postgres@postgres/postgres'
+      GCP_SERVICE_KEY:
       ZOO_AUDITING_CHECKS: dummy_standards
       ZOO_DEBUG: '1'
       ZOO_DATADOG_API_KEY:

docker-compose.yml

@@ -4,6 +4,7 @@ services:
     build: .
     environment:
       - DATABASE_URL
+      - GCP_SERVICE_KEY
       - SECRET_KEY
       - SENTRY_DSN
       - SENTRY_PUBLIC_DSN
@@ -44,6 +45,7 @@ services:
     command: [celery, worker, -A, zoo]
     environment:
       - DATABASE_URL
+      - GCP_SERVICE_KEY
       - SECRET_KEY
       - SENTRY_DSN
       - SENTRY_PUBLIC_DSN

test/datacenters/test_gcp.py

@@ -0,0 +1,95 @@
+from faker import Faker
+import pytest
+
+from zoo.datacenters import gcp as uut, models
+
+fake = Faker()
+pytestmark = pytest.mark.django_db
+
+
+def test_gcp_map_to_nodes(mocker):
+    mocker.patch("zoo.datacenters.utils.gcloud.auth", return_value=None)
+    mocker.patch(
+        "zoo.datacenters.utils.gcloud.iter_projects",
+        return_value=[{"projectId": "pid1"}, {"projectId": "pid2"}],
+    )
+    mocker.patch(
+        "zoo.datacenters.utils.gcloud.get_forwarding_rules",
+        return_value=[
+            {
+                "id": "test1",
+                "loadBalancingScheme": "EXTERNAL",
+                "IPAddress": "1.1.1.1",
+                "portRange": "443-443",
+            },
+            {
+                "id": "test2",
+                "loadBalancingScheme": "INTERNAL",
+                "IPAddress": "2.2.2.2",
+                "portRange": "443-443",
+            },
+        ],
+    )
+    mocker.patch(
+        "zoo.datacenters.utils.gcloud.iter_clusters",
+        return_value=[{"name": "test", "zone": "europe-test"}],
+    )
+    mocker.patch(
+        "zoo.datacenters.utils.kubectl.iter_workloads",
+        return_value={
+            "test-type": [
+                {
+                    "metadata": {
+                        "namespace": "namespace-test",
+                        "name": "resource-test",
+                    },
+                    "spec": {
+                        "template": {
+                            "spec": {
+                                "containers": [
+                                    {"image": "test/image:0.0.1"},
+                                    {"image": "test/image2:0.0.2"},
+                                ]
+                            }
+                        }
+                    },
+                }
+            ]
+        },
+    )
+
+    uut.map_to_nodes()
+
+    root = models.InfraNode.objects.get(kind=models.NodeKind.GCP_ROOT_PROJ)
+
+    projects = {project.value: project for project in root.targets.all()}
+    assert set(projects) == {"pid1", "pid2"}
+
+    ctx = "gke_pid1_europe-test_test"
+    clusters = {
+        cluster.value: cluster
+        for cluster in projects["pid1"].targets.filter(
+            kind=models.NodeKind.GCP_CLUSTER_NAME
+        )
+    }
+    assert set(clusters) == {ctx}
+
+    ip_rules = {
+        cluster.value: cluster
+        for cluster in projects["pid1"].targets.filter(
+            kind=models.NodeKind.GCP_IP_RULE_NAME
+        )
+    }
+    assert set(ip_rules) == {"test1:1.1.1.1:443-443"}
+
+    workloads = {
+        workload.value: workload
+        for workload in clusters["gke_pid1_europe-test_test"].targets.all()
+    }
+    full_name = "test-type:namespace-test/resource-test"
+    assert set(workloads) == {f"{ctx}:{full_name}"}
+
+    images = {
+        image.value: image for image in workloads[f"{ctx}:{full_name}"].targets.all()
+    }
+    assert set(images) == {"test/image:0.0.1", "test/image2:0.0.2"}

zoo/base/settings.py

@@ -47,6 +47,8 @@
     RANCHER_API_URL=(str, None),
     RANCHER_ACCESS_KEY=(str, None),
     RANCHER_SECRET_KEY=(str, None),
+    GCP_SERVICE_KEY=(str, None),
+    GCP_SERVICE_KEY_PATH=(str, "/tmp/gcloud-service-key.json"),
 )
 
 SITE_ROOT = str(root)
@@ -246,4 +248,7 @@
 RANCHER_ACCESS_KEY = env("RANCHER_ACCESS_KEY")
 RANCHER_SECRET_KEY = env("RANCHER_SECRET_KEY")
 
+GCP_SERVICE_KEY = env("GCP_SERVICE_KEY")
+GCP_SERVICE_KEY_PATH = env("GCP_SERVICE_KEY_PATH")
+
 logs.configure_structlog(DEBUG)

zoo/datacenters/gcp.py

@@ -0,0 +1,67 @@
+from django.db import transaction
+
+from .models import InfraNode, NodeKind
+from .utils import gcloud, kubectl
+
+
+def _workload_identifier(cluster, resource_type, resource):
+    return f"{cluster}:{resource_type}:{resource['metadata']['namespace']}/{resource['metadata']['name']}"
+
+
+@transaction.atomic
+def map_to_nodes():
+    """Map GCP projects to GCP services.
+
+    Creates records in the InfraNode table of the following kinds:
+
+    - ``gcp.root.proj`` - root node for all GCP projects
+    - ``gcp.proj.id`` - GCP project ID
+    - ``gcp.ip_rule.name`` - GCP forwarding rule name
+    - ``gcp.cluster.name`` - GCP cluster name
+    - ``gcp.workload.name`` - GCP workload name (including the namespace)
+    - ``docker.image.uuid`` - Docker image UUID
+    """
+    gcloud.auth()
+    root = InfraNode.get_or_create_node(kind=NodeKind.GCP_ROOT_PROJ, value="*")
+
+    for project in gcloud.iter_projects():
+        project_id = project["projectId"]
+        project_node = InfraNode.get_or_create_node(
+            kind=NodeKind.GCP_PROJ_ID, value=project_id, source=root
+        )
+
+        # currently not used anywhere
+        for ip_rule in gcloud.get_forwarding_rules(project_id):
+            if ip_rule["loadBalancingScheme"] == "EXTERNAL":
+                InfraNode.get_or_create_node(
+                    kind=NodeKind.GCP_IP_RULE_NAME,
+                    value=f"{ip_rule['id']}:{ip_rule['IPAddress']}:{ip_rule['portRange']}",
+                    source=project_node,
+                )
+
+        clusters = gcloud.iter_clusters(project_id)
+        for cluster in clusters:
+            cluster_ctx = f"gke_{project_id}_{cluster['zone']}_{cluster['name']}"
+            cluster_node = InfraNode.get_or_create_node(
+                kind=NodeKind.GCP_CLUSTER_NAME, value=cluster_ctx, source=project_node
+            )
+            gcloud.get_credentials(cluster["name"], cluster["zone"], project_id)
+
+            workloads = kubectl.iter_workloads(cluster_ctx)
+
+            for resource_type, resources in workloads.items():
+                for resource in resources:
+                    workload_node = InfraNode.get_or_create_node(
+                        kind=NodeKind.GCP_WORKLOAD_NAME,
+                        value=_workload_identifier(
+                            cluster_ctx, resource_type, resource
+                        ),
+                        source=cluster_node,
+                    )
+
+                    for container in resource["spec"]["template"]["spec"]["containers"]:
+                        InfraNode.get_or_create_node(
+                            kind=NodeKind.DOCKER_IMAGE_UUID,
+                            value=container["image"],
+                            source=workload_node,
+                        )

zoo/datacenters/mapping.py

@@ -2,8 +2,9 @@
 
 from django.db import transaction
 
-from . import amazon, models, rancher
+from . import amazon, gcp, models, rancher
 from .models import InfraNode, NodeKind
+from .utils import gcloud, kubectl
 
 
 def url_matches_dns(url, dns_record):
@@ -31,6 +32,7 @@ def map_infra_to_nodes():
     amazon.map_to_nodes()
     rancher.map_to_nodes()
     connect_aws_rancher_nodes()
+    gcp.map_to_nodes()
 
 
 class Mapper:
@@ -55,6 +57,82 @@ def link_image_to_service(self, image_node, service):
         raise NotImplementedError()
 
 
+class GoogleCloudPlatformMapper(Mapper):
+    def _get_ingress_component(self, workload):
+        # we assume that all our services use unique namespaces
+        # and use ingress for routing requests
+        if workload.value in self._components_cache:
+            return self._components_cache[workload.value]
+
+        cluster, _, full_name = workload.value.split(":")
+        namespace, _ = full_name.split("/")
+        hosts = set()
+
+        for ingress in kubectl.get_ingress(namespace, cluster):
+            hosts = hosts.union(
+                {rule["host"] for rule in ingress["spec"]["rules"] if rule.get("host")}
+            )
+
+        result = {"name": namespace, "urls": list(hosts)}
+
+        self._components_cache[workload.value] = result
+        return result
+
+    def _get_project_members(self, project):
+        if project.value not in self._members_cache:
+            self._members_cache[project.value] = gcloud.get_project_owners(
+                project.value
+            )
+        return self._members_cache[project.value]
+
+    def _get_gcp_datacenter(self, cluster, service):
+        _, _, zone, _ = cluster.value.split("_")
+
+        datacenter, _ = models.Datacenter.objects.get_or_create(
+            provider="GCP", region=zone
+        )
+        service_datacenter, _ = models.ServiceDatacenter.objects.get_or_create(
+            service=service, datacenter=datacenter
+        )
+        return service_datacenter
+
+    @transaction.atomic
+    def link_image_to_service(self, image_node, service):
+        gcloud.auth()
+        ingress_components = defaultdict(set)
+
+        for cluster in image_node.find_sources_by_kind(NodeKind.GCP_CLUSTER_NAME):
+            datacenter = self._get_gcp_datacenter(cluster, service)
+
+            for project in cluster.find_sources_by_kind(NodeKind.GCP_PROJ_ID):
+                project_members = self._get_project_members(project)
+
+                for member in project_members:
+                    models.ServiceDatacenterMember.objects.get_or_create(
+                        service_datacenter=datacenter,
+                        name=member,
+                        email=member.split(":", 1)[1],
+                    )
+                models.ServiceDatacenterMember.objects.filter(
+                    service_datacenter=datacenter
+                ).exclude(name__in=project_members).delete()
+
+        for workload in image_node.find_sources_by_kind(NodeKind.GCP_WORKLOAD_NAME):
+            component_data = self._get_ingress_component(workload)
+
+            models.ServiceDatacenterComponent.objects.get_or_create(
+                service_datacenter=datacenter, **component_data
+            )
+            # save datacenter id with its component for later deletion
+            ingress_components[datacenter.id].add(component_data["name"])
+
+        # delete no longer existing cingress in datacenters
+        for datacenter_id in ingress_components:
+            models.ServiceDatacenterComponent.objects.filter(
+                service_datacenter_id=datacenter_id
+            ).exclude(name__in=ingress_components[datacenter_id]).delete()
+
+
 class AmazonRancherMapper(Mapper):
     """Retrieve data from Amazon and Rancher infrastructure and store it."""
 

zoo/datacenters/models.py

@@ -21,6 +21,12 @@ class NodeKind:
     RANCHER_LB_PORTRULE_URI = "rancher.lb.portrule.uri"
     RANCHER_SERVICE_ID = "rancher.service.id"
 
+    GCP_ROOT_PROJ = "gcp.root.proj"
+    GCP_PROJ_ID = "gcp.project.id"
+    GCP_IP_RULE_NAME = "gcp.ip_rule.name"
+    GCP_CLUSTER_NAME = "gcp.cluster.name"
+    GCP_WORKLOAD_NAME = "gcp.workload.name"
+
     DOCKER_IMAGE_UUID = "docker.image.uuid"
 
 

zoo/datacenters/tasks.py

@@ -2,15 +2,17 @@
 from celery import shared_task
 
 from ..services.models import Service
-from .mapping import AmazonRancherMapper, map_infra_to_nodes
+from .mapping import AmazonRancherMapper, GoogleCloudPlatformMapper, map_infra_to_nodes
 from .models import InfraNode
 
 
 @shared_task
 def link_service_to_datacenters(service_id):
     service = Service.objects.get(id=service_id)
-    mapper = AmazonRancherMapper()
-    mapper.link_service_to_datacenters(service)
+    amazon = AmazonRancherMapper()
+    amazon.link_service_to_datacenters(service)
+    gcp = GoogleCloudPlatformMapper()
+    gcp.link_service_to_datacenters(service)
 
 
 @shared_task