Add new app for services infrastructure tracking

- Resolve #87, #89, #90

requirements.in

@@ -1,5 +1,6 @@
 arrow
 attrs
+boto3
 celery
 colorama
 datadog

requirements.txt

@@ -2,16 +2,17 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --upgrade requirements.in
+#    pip-compile --output-file requirements.txt requirements.in
 #
 amqp==2.4.2               # via kombu
 aniso8601==3.0.2          # via graphene
-appnope==0.1.0            # via ipython
 arrow==0.13.1
 attrs==19.1.0
 autopep8==1.4.4           # via django-silk
 backcall==0.1.0           # via ipython
 billiard==3.6.0.0         # via celery
+boto3==1.9.150
+botocore==1.12.150        # via boto3, s3transfer
 celery==4.3.0
 certifi==2019.3.9         # via requests
 chardet==3.0.4            # via requests
@@ -30,6 +31,7 @@ django-stronghold==0.3.0
 django==2.2.1
 djangoql==0.12.6
 dockerfile-parse==0.0.14
+docutils==0.14            # via botocore
 gprof2dot==2017.9.19      # via django-silk
 graphene-django==2.2.0
 graphene==2.1.3
@@ -42,6 +44,7 @@ ipython-genutils==0.2.0   # via traitlets
 ipython==7.5.0
 jedi==0.13.3              # via ipython
 jinja2==2.10.1            # via django-silk
+jmespath==0.9.4           # via boto3, botocore
 kombu==4.5.0              # via celery
 markdown==3.1
 markupsafe==1.1.1         # via jinja2
@@ -60,7 +63,7 @@ pycodestyle==2.5.0        # via autopep8
 pygithub==1.43.7
 pygments==2.3.1           # via django-silk, ipython
 pyjwt==1.7.1              # via pygithub
-python-dateutil==2.8.0    # via arrow, django-silk
+python-dateutil==2.8.0    # via arrow, botocore, django-silk
 python-gitlab==1.8.0
 python3-openid==3.1.0     # via django-allauth
 pytz==2019.1              # via celery, django, django-silk
@@ -72,12 +75,13 @@ requests==2.21.0
 requirements-parser==0.2.0
 retry==0.9.2
 rx==1.6.1                 # via graphql-core
+s3transfer==0.2.0         # via boto3
 singledispatch==3.4.0.3   # via graphene-django
 six==1.12.0               # via django-extensions, dockerfile-parse, graphene, graphene-django, graphql-core, graphql-relay, promise, prompt-toolkit, python-dateutil, python-gitlab, singledispatch, structlog, traitlets
 sqlparse==0.3.0           # via django, django-debug-toolbar, django-silk
 structlog==19.1.0
 traitlets==4.3.2          # via ipython
-urllib3==1.24.3           # via requests
+urllib3==1.24.3           # via botocore, requests
 vine==1.3.0               # via amqp, celery
 wcwidth==0.1.7            # via prompt-toolkit
 whitenoise==4.1.2

zoo/base/apps.py

@@ -15,6 +15,7 @@ def ready(self):
         from ..analytics import tasks as analytics_tasks
         from ..auditing import tasks as auditing_tasks
         from ..objectives import tasks as objective_tasks
+        from ..datacenters import tasks as datacenters_tasks
 
         celery_app.add_periodic_task(timedelta(hours=1), repos_tasks.sync_repos)
         celery_app.add_periodic_task(timedelta(hours=1), repos_tasks.schedule_pulls)
@@ -37,3 +38,6 @@ def ready(self):
         celery_app.add_periodic_task(
             timedelta(days=1), analytics_tasks.check_python_lib_licenses
         )
+        celery_app.add_periodic_task(
+            timedelta(days=1), datacenters_tasks.schedule_infra_mapping
+        )

zoo/base/settings.py

@@ -6,6 +6,7 @@
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/dev/ref/settings/
 """
+import os
 from pathlib import Path
 
 import environ
@@ -36,6 +37,13 @@
     ZOO_AUDITING_DROP_ISSUES=(int, 7),
     ZOO_SONARQUBE_URL=(str, None),
     ZOO_SONARQUBE_TOKEN=(str, None),
+    AWS_CONFIG=(str, None),
+    AWS_CONFIG_FILE=(str, "/tmp/aws/config"),
+    AWS_SHARED_CREDENTIALS=(str, None),
+    AWS_SHARED_CREDENTIALS_FILE=(str, "/tmp/aws/credentials"),
+    RANCHER_API_URL=(str, None),
+    RANCHER_USERNAME=(str, None),
+    RANCHER_PASSWORD=(str, None),
 )
 
 SITE_ROOT = str(root)
@@ -82,6 +90,7 @@
     "zoo.services.apps.ServicesConfig",
     "zoo.analytics.apps.AnalyticsConfig",
     "zoo.objectives.apps.ObjectivesConfig",
+    "zoo.datacenters.apps.DatacentersConfig",
     "zoo.api.apps.ApiConfig",
     "django.contrib.admin",
     "django.contrib.auth",
@@ -219,4 +228,17 @@
 ZOO_AUDITING_CHECKS = env("ZOO_AUDITING_CHECKS")
 ZOO_AUDITING_DROP_ISSUES = env("ZOO_AUDITING_DROP_ISSUES")
 
+AWS_CONFIG = env("AWS_CONFIG")
+AWS_CONFIG_FILE = env("AWS_CONFIG_FILE")
+AWS_CREDENTIALS = env("AWS_SHARED_CREDENTIALS")
+AWS_CREDENTIALS_FILE = env("AWS_SHARED_CREDENTIALS_FILE")
+
+# make boto3 adopt default locations of config files specified in the Zoo
+os.environ.setdefault("AWS_CONFIG_FILE", AWS_CONFIG_FILE)
+os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", AWS_CREDENTIALS_FILE)
+
+RANCHER_API_URL = env("RANCHER_API_URL")
+RANCHER_USERNAME = env("RANCHER_USERNAME")
+RANCHER_PASSWORD = env("RANCHER_PASSWORD")
+
 logs.configure_structlog(DEBUG)

zoo/datacenters/admin.py

@@ -0,0 +1,10 @@
+from django.contrib import admin
+
+from . import models
+
+
+class DatacenterAdmin(admin.ModelAdmin):
+    search_fields = ("provider", "region")
+
+
+admin.site.register(models.Datacenter, DatacenterAdmin)

zoo/datacenters/amazon.py

@@ -0,0 +1,181 @@
+import itertools
+import pathlib
+
+import boto3
+from django.conf import settings
+from django.db import transaction
+
+from .models import InfraNode, NodeKind
+
+config_file = pathlib.Path(settings.AWS_CONFIG_FILE)
+credentials_file = pathlib.Path(settings.AWS_CREDENTIALS_FILE)
+
+if settings.AWS_CONFIG and not config_file.is_file():
+    config_file.parents[0].mkdir(parents=True, exist_ok=True)
+    config_file.write_text(settings.AWS_CONFIG)
+
+if settings.AWS_CREDENTIALS and not credentials_file.is_file():
+    credentials_file.parents[0].mkdir(parents=True, exist_ok=True)
+    credentials_file.write_text(settings.AWS_CREDENTIALS)
+
+
+def _get_url(value):
+    return value.strip().rstrip(".").replace("\\052", "*")
+
+
+def _get_elb_dns(value):
+    if value.startswith("dualstack."):
+        value = value[10:]
+    return _get_url(value)
+
+
+def get_client(service_name):
+    return boto3.client(service_name)
+
+
+def iter_hosted_zones():
+    route53 = get_client("route53")
+    paginator = route53.get_paginator("list_hosted_zones")
+
+    for page in paginator.paginate():
+        yield from page["HostedZones"]
+
+
+def iter_resource_record_sets(hosted_zones_ids):
+    route53 = get_client("route53")
+    for zone_id in hosted_zones_ids:
+        paginator = route53.get_paginator("list_resource_record_sets")
+
+        for page in paginator.paginate(HostedZoneId=zone_id):
+            yield from page["ResourceRecordSets"]
+
+
+def iter_load_balancers(names=None):
+    elb = get_client("elb")
+
+    paginator = elb.get_paginator("describe_load_balancers")
+
+    if names:
+        page_iterator = paginator.paginate(LoadBalancerNames=names)
+    else:
+        page_iterator = paginator.paginate()
+
+    for page in page_iterator:
+        yield from page["LoadBalancerDescriptions"]
+
+
+def iter_load_balancers_v2(names=None):
+    elbv2 = get_client("elbv2")
+
+    paginator = elbv2.get_paginator("describe_load_balancers")
+
+    if names:
+        page_iterator = paginator.paginate(LoadBalancerArns=names)
+    else:
+        page_iterator = paginator.paginate()
+
+    for page in page_iterator:
+        yield from page["LoadBalancers"]
+
+
+def iter_ec2_instances(instance_ids=None):
+    ec2 = get_client("ec2")
+
+    paginator = ec2.get_paginator("describe_instances")
+
+    if instance_ids:
+        page_iterator = paginator.paginate(InstanceIds=instance_ids)
+    else:
+        page_iterator = paginator.paginate()
+
+    for page in page_iterator:
+        for reservations in page["Reservations"]:
+            yield from reservations["Instances"]
+
+
+@transaction.atomic
+def map_dns_records():
+    """Map DNS records (from Amazon Route 53) to the database as a set of InfraNodes.
+
+    Creates records in the InfraNode table of the following kinds:
+
+    - AWS_ROOT_DNS - root of all Amazon DNS records
+    - AWS_HOSTED_ZONE_DNS - corresponds to results from Route 53 > ListHostedZones
+    - AWS_RECORD_SET_DNS - corresponds to results from Route 53 > ListResourceRecordSets
+    """
+    root = InfraNode.get_or_create_node(kind=NodeKind.AWS_ROOT_DNS, value="*")
+
+    for zone in iter_hosted_zones():
+        zone_path = InfraNode.get_or_create_node(
+            kind=NodeKind.AWS_HOSTED_ZONE_DNS, value=_get_url(zone["Name"]), source=root
+        )
+
+        if "AliasTarget" in zone and zone["AliasTarget"].get("DNSName"):
+            InfraNode.get_or_create_node(
+                kind=NodeKind.AWS_RECORD_SET_DNS,
+                value=_get_url(zone["AliasTarget"]["DNSName"]),
+                source=zone_path,
+            )
+
+        record_sets = iter_resource_record_sets([zone["Id"]])
+
+        for record_set in record_sets:
+            record_set_path = InfraNode.get_or_create_node(
+                kind=NodeKind.AWS_RECORD_SET_DNS,
+                value=_get_url(record_set["Name"]),
+                source=zone_path,
+            )
+
+            record_type = record_set.get("Type")
+
+            if record_type == "CNAME":
+                value = record_set["ResourceRecords"][0]["Value"]
+            elif record_type == "A" and "AliasTarget" in record_set:
+                value = record_set["AliasTarget"]["DNSName"]
+            else:
+                continue
+
+            if value:
+                InfraNode.get_or_create_node(
+                    kind=NodeKind.AWS_RECORD_SET_DNS,
+                    value=_get_url(value),
+                    source=record_set_path,
+                )
+
+
+@transaction.atomic
+def map_dns_to_ec2s():
+    """Map all DNS records found in the InfraNode table via ELBs to EC2 instances.
+
+    Creates records in the InfraNode table of the following kinds:
+
+    - AWS_ELB_DNS - Public DNS of an Elastic Load Balancer (ELB and ELBv2)
+    - AWS_EC2_DNS_PRIVATE - Private DNS of an EC2 instance
+    """
+    record_set_nodes = InfraNode.objects.filter(kind=NodeKind.AWS_RECORD_SET_DNS).all()
+
+    load_balancers = {
+        _get_elb_dns(elb["DNSName"]): elb
+        for elb in itertools.chain(iter_load_balancers(), iter_load_balancers_v2())
+    }
+    ec2_instances = {ec2["InstanceId"]: ec2 for ec2 in iter_ec2_instances()}
+
+    for record_set_node in record_set_nodes:
+        elb_info = load_balancers.get(record_set_node.value)
+
+        if elb_info is None:
+            continue
+
+        elb_node = InfraNode.get_or_create_node(
+            kind=NodeKind.AWS_ELB_DNS,
+            value=_get_elb_dns(elb_info["DNSName"]),
+            source=record_set_node,
+        )
+
+        for instance_info in elb_info.get("Instances", []):
+            instance = ec2_instances[instance_info["InstanceId"]]
+            InfraNode.get_or_create_node(
+                kind=NodeKind.AWS_EC2_DNS_PRIVATE,
+                value=instance["PrivateDnsName"],
+                source=elb_node,
+            )

zoo/datacenters/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class DatacentersConfig(AppConfig):
+    name = "zoo.datacenters"