Add new app for services infrastructure tracking

- Resolve #87, #89, #90

requirements.in

@@ -1,5 +1,6 @@
 arrow
 attrs
+boto3
 celery
 celery-redbeat
 colorama

requirements.txt

@@ -12,6 +12,8 @@ attrs==19.1.0
 autopep8==1.4.4           # via django-silk
 backcall==0.1.0           # via ipython
 billiard==3.6.1.0         # via celery
+boto3==1.9.150
+botocore==1.12.150        # via boto3, s3transfer
 celery-redbeat==0.13.0
 celery==4.3.0
 certifi==2019.9.11        # via requests
@@ -32,6 +34,7 @@ django==2.2.5
 djangoql==0.13.0
 dockerfile-parse==0.0.15
 gevent==1.4.0             # via gunicorn
+docutils==0.14            # via botocore
 gprof2dot==2017.9.19      # via django-silk
 graphene-django==2.5.0
 graphene==2.1.8
@@ -46,6 +49,7 @@ ipython-genutils==0.2.0   # via traitlets
 ipython==7.8.0
 jedi==0.15.1              # via ipython
 jinja2==2.10.1            # via django-silk
+jmespath==0.9.4           # via boto3, botocore
 kombu==4.6.4              # via celery
 markdown==3.1.1
 markupsafe==1.1.1         # via jinja2
@@ -65,7 +69,7 @@ pygerduty==0.38.2
 pygithub==1.43.8
 pygments==2.4.2           # via django-silk, ipython
 pyjwt==1.7.1              # via pygithub
-python-dateutil==2.8.0    # via arrow, celery-redbeat, django-silk
+python-dateutil==2.8.0    # via arrow, botocore, celery-redbeat, django-silk
 python-gitlab==1.11.0
 python3-openid==3.1.0     # via django-allauth
 pytz==2019.2              # via celery, django, django-silk
@@ -76,13 +80,14 @@ requests-oauthlib==1.2.0  # via django-allauth
 requests==2.22.0
 requirements-parser==0.2.0
 rx==1.6.1                 # via graphql-core
+s3transfer==0.2.0         # via boto3
 singledispatch==3.4.0.3   # via graphene-django
 six==1.12.0               # via django-extensions, dockerfile-parse, graphene, graphene-django, graphql-core, graphql-relay, promise, prompt-toolkit, pygerduty, python-dateutil, python-gitlab, singledispatch, structlog, tenacity, traitlets
 sqlparse==0.3.0           # via django, django-debug-toolbar, django-silk
 structlog==19.1.0
 tenacity==5.1.1           # via celery-redbeat
 traitlets==4.3.2          # via ipython
-urllib3==1.25.3           # via requests
+urllib3==1.24.3           # via botocore, requests
 vine==1.3.0               # via amqp, celery
 wcwidth==0.1.7            # via prompt-toolkit
 whitenoise==4.1.3

test/conftest.py

@@ -22,6 +22,7 @@
     DependencyFactory,
     DependencyUsageFactory,
     KindFactory,
+    InfraNodeFactory,
 )
 
 
@@ -33,6 +34,7 @@
 register(DependencyFactory)
 register(DependencyUsageFactory)
 register(KindFactory)
+register(InfraNodeFactory)
 
 
 fake = Faker()

test/datacenters/test_amazon.py

@@ -0,0 +1,83 @@
+from faker import Faker
+import pytest
+
+from zoo.datacenters import amazon as uut, models
+
+fake = Faker()
+pytestmark = pytest.mark.django_db
+
+
+def test_amazon_map_to_nodes(mocker):
+    mocker.patch(
+        "zoo.datacenters.amazon.iter_hosted_zones",
+        return_value=[
+            {"Id": "hostedzone/1", "Name": "example.com"},
+            {
+                "Id": "hostedzone/2",
+                "Name": "zoo.example.com",
+                "AliasTarget": {"DNSName": "elb1.example.com"},
+            },
+        ],
+    )
+    mocker.patch(
+        "zoo.datacenters.amazon.iter_resource_record_sets",
+        return_value=[
+            {
+                "Type": "A",
+                "Name": "test.example.com",
+                "AliasTarget": {"DNSName": "elb1.example.com"},
+            },
+            {
+                "Type": "CNAME",
+                "Name": "test2.example.com",
+                "ResourceRecords": [{"Value": "elb2.example.com"}],
+            },
+        ],
+    )
+
+    def get_load_balancers(version=1, **kwargs):
+        result = None
+        if version == 1:
+            result = [
+                {
+                    "DNSName": "elb1.example.com",
+                    "Instances": [{"InstanceId": "i-a"}, {"InstanceId": "i-b"}],
+                }
+            ]
+        elif version == 2:
+            result = [
+                {"DNSName": "elb2.example.com", "Instances": [{"InstanceId": "i-c"}]}
+            ]
+        return result
+
+    mocker.patch("zoo.datacenters.amazon.iter_load_balancers", new=get_load_balancers)
+    mocker.patch(
+        "zoo.datacenters.amazon.iter_ec2_instances",
+        return_value=[
+            {"InstanceId": "i-a", "PrivateDnsName": "ip-1"},
+            {"InstanceId": "i-b", "PrivateDnsName": "ip-2"},
+            {"InstanceId": "i-c", "PrivateDnsName": "ip-3"},
+        ],
+    )
+
+    uut.map_to_nodes(profiles="default")
+
+    root = models.InfraNode.objects.get(kind=models.NodeKind.AWS_ROOT_DNS)
+
+    zones = {zone.value: zone for zone in root.targets.all()}
+    assert set(zones) == {"zoo.example.com", "example.com"}
+
+    record_sets = {rs.value: rs for rs in zones["example.com"].targets.all()}
+    assert set(record_sets) == {"test.example.com", "test2.example.com"}
+
+    [elb1_dns] = record_sets["test.example.com"].targets.all()
+    [elb2_dns] = record_sets["test2.example.com"].targets.all()
+
+    [elb1] = elb1_dns.targets.all()
+    [elb2] = elb2_dns.targets.all()
+
+    assert elb1.value == "elb1.example.com"
+    assert elb2.value == "elb2.example.com"
+
+    assert {instance.value for instance in elb1.targets.all()} == {"ip-1", "ip-2"}
+    assert {instance.value for instance in elb2.targets.all()} == {"ip-3"}

test/datacenters/test_mapping.py

@@ -0,0 +1,11 @@
+from zoo.datacenters import mapping as uut
+
+
+def test_url_matches_dns():
+    assert uut.url_matches_dns("zoo.example.com", "*.example.com")
+    assert uut.url_matches_dns("zoo.example.com", "zoo.*.com")
+    assert uut.url_matches_dns("zoo.example.com", "zoo.example.com")
+
+    assert not uut.url_matches_dns("zoo.example.com", "abc.example.com")
+    assert not uut.url_matches_dns("zoo.example.com", "abc.*.com")
+    assert not uut.url_matches_dns("zoo.example.com", "*.example.cz")

test/datacenters/test_models.py

@@ -0,0 +1,46 @@
+import pytest
+
+from zoo.datacenters import models as uut
+
+pytestmark = pytest.mark.django_db
+
+
+def test_infra_node__get_or_create_node():
+    root = uut.InfraNode.get_or_create_node(kind="kind1", value="value1")
+
+    assert root.value == "value1"
+    assert list(root.sources.all()) == []
+    assert list(root.targets.all()) == []
+
+    node = uut.InfraNode.get_or_create_node(kind="kind2", value="value2", source=root)
+
+    assert list(node.sources.all()) == [root]
+    assert list(root.targets.all()) == [node]
+
+    assert root.id == uut.InfraNode.get_or_create_node(kind="kind1", value="value1").id
+
+
+def test_infra_node__find_sources_by_kind():
+    root = uut.InfraNode.objects.create(kind="root", value="123")
+
+    node_dns = uut.InfraNode.objects.create(kind="dns", value="a1b")
+    node_abc = uut.InfraNode.objects.create(kind="abc", value="a2b")
+
+    root.targets.add(node_dns)
+    root.targets.add(node_abc)
+
+    node_dns2 = uut.InfraNode.objects.create(kind="dns", value="b11")
+    node_dns3 = uut.InfraNode.objects.create(kind="dns", value="b22")
+    node_abc.targets.add(node_dns2)
+    node_dns2.targets.add(node_dns3)
+
+    leaf = uut.InfraNode.objects.create(kind="leaf", value="ccc")
+    node_dns.targets.add(leaf)
+    node_dns3.targets.add(leaf)
+
+    dns_sources = leaf.find_sources_by_kind("dns")
+    dns_nodes_ids = {node_dns.id, node_dns2.id, node_dns3.id}
+    assert {source.id for source in dns_sources} == dns_nodes_ids
+
+    root_sources = leaf.find_sources_by_kind("root")
+    assert list(root_sources) == [root]

test/datacenters/test_rancher.py

@@ -0,0 +1,83 @@
+from faker import Faker
+import pytest
+
+from zoo.datacenters import rancher as uut, models
+
+fake = Faker()
+pytestmark = pytest.mark.django_db
+
+
+def test_rancher_parse_members_from_project():
+    names = [fake.name(), fake.name()]
+    members = uut.parse_members_from_project(
+        {
+            "id": "p2",
+            "members": [
+                {
+                    "type": "projectMember",
+                    "externalId": f"cn={names[0]},ou=People,dc=example,dc=com",
+                    "role": "owner",
+                },
+                {
+                    "type": "projectMember",
+                    "externalId": f"cn={names[1]},ou=People,dc=example,dc=com",
+                    "role": "owner",
+                },
+            ],
+        }
+    )
+    assert members == names
+
+
+def test_rancher_map_to_nodes(mocker):
+    mocker.patch("zoo.datacenters.rancher.iter_projects", return_value=[{"id": "p1"}])
+    mocker.patch(
+        "zoo.datacenters.rancher.iter_services",
+        return_value=[
+            {"id": "s1", "launchConfig": {"imageUuid": "docker:lb"}},
+            {"id": "s2", "launchConfig": {"imageUuid": "docker:zoo"}},
+        ],
+    )
+    mocker.patch(
+        "zoo.datacenters.rancher.iter_load_balancers",
+        return_value=[
+            {
+                "id": "lb1",
+                "lbConfig": {
+                    "portRules": [
+                        {
+                            "hostname": "zoo.example.com",
+                            "sourcePort": 80,
+                            "protocol": "http",
+                            "serviceId": "s2",
+                        }
+                    ]
+                },
+                "publicEndpoints": [{"hostId": "h1"}],
+            }
+        ],
+    )
+    mocker.patch(
+        "zoo.datacenters.rancher.iter_hosts",
+        return_value=[{"id": "h1", "hostname": "ip-127-0-0-1"}],
+    )
+
+    uut.map_to_nodes()
+
+    project = models.InfraNode.objects.get(kind=models.NodeKind.RANCHER_PROJ_ID)
+    assert project.value == "p1"
+
+    hosts = project.targets.all()
+    assert {host.value for host in hosts} == {"ip-127-0-0-1"}
+
+    lbs = hosts[0].targets.all()
+    assert {lb.value for lb in lbs} == {"lb1"}
+
+    portrules = lbs[0].targets.all()
+    assert {portrule.value for portrule in portrules} == {"zoo.example.com"}
+
+    services = portrules[0].targets.all()
+    assert {service.value for service in services} == {"s2"}
+
+    images = services[0].targets.all()
+    assert {image.value for image in images} == {"docker:zoo"}

test/factories.py

@@ -7,6 +7,7 @@
 from zoo.api.models import ApiToken
 from zoo.auditing.models import Issue
 from zoo.auditing.check_discovery import Kind
+from zoo.datacenters.models import InfraNode
 from zoo.repos.models import Repository
 from zoo.services.models import Service
 from zoo.analytics.models import Dependency, DependencyUsage, DependencyType
@@ -94,3 +95,11 @@ class Meta:
     id = Faker("domain_word")
     title = Faker("sentence")
     description = Faker("paragraph")
+
+
+class InfraNodeFactory(Factory):
+    class Meta:
+        model = InfraNode
+
+    kind = Faker("domain_word")
+    value = Faker("slug")

zoo/base/apps.py

@@ -15,6 +15,7 @@ def ready(self):
         from ..analytics import tasks as analytics_tasks
         from ..auditing import tasks as auditing_tasks
         from ..objectives import tasks as objective_tasks
+        from ..datacenters import tasks as datacenters_tasks
 
         celery_app.add_periodic_task(timedelta(hours=1), repos_tasks.sync_repos)
         celery_app.add_periodic_task(timedelta(hours=1), repos_tasks.schedule_pulls)
@@ -37,3 +38,6 @@ def ready(self):
         celery_app.add_periodic_task(
             timedelta(days=1), analytics_tasks.check_python_lib_licenses
         )
+        celery_app.add_periodic_task(
+            timedelta(days=1), datacenters_tasks.schedule_infra_mapping
+        )

zoo/base/settings.py

@@ -6,6 +6,7 @@
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/dev/ref/settings/
 """
+import os
 from pathlib import Path
 
 import environ
@@ -41,6 +42,13 @@
     ZOO_AUDITING_DROP_ISSUES=(int, 7),
     ZOO_SONARQUBE_URL=(str, None),
     ZOO_SONARQUBE_TOKEN=(str, None),
+    AWS_CONFIG=(str, None),
+    AWS_CONFIG_FILE=(str, "/tmp/aws/config"),
+    AWS_SHARED_CREDENTIALS=(str, None),
+    AWS_SHARED_CREDENTIALS_FILE=(str, "/tmp/aws/credentials"),
+    RANCHER_API_URL=(str, None),
+    RANCHER_ACCESS_KEY=(str, None),
+    RANCHER_SECRET_KEY=(str, None),
 )
 
 SITE_ROOT = str(root)
@@ -85,6 +93,7 @@
     "zoo.libraries.apps.LibrariesConfig",
     "zoo.analytics.apps.AnalyticsConfig",
     "zoo.objectives.apps.ObjectivesConfig",
+    "zoo.datacenters.apps.DatacentersConfig",
     "zoo.api.apps.ApiConfig",
     "zoo.pagerduty.apps.PagerdutyConfig",
     "django.contrib.admin",
@@ -225,4 +234,17 @@
 ZOO_AUDITING_CHECKS = env("ZOO_AUDITING_CHECKS")
 ZOO_AUDITING_DROP_ISSUES = env("ZOO_AUDITING_DROP_ISSUES")
 
+AWS_CONFIG = env("AWS_CONFIG")
+AWS_CONFIG_FILE = env("AWS_CONFIG_FILE")
+AWS_CREDENTIALS = env("AWS_SHARED_CREDENTIALS")
+AWS_CREDENTIALS_FILE = env("AWS_SHARED_CREDENTIALS_FILE")
+
+# make boto3 adopt default locations of config files specified in the Zoo
+os.environ.setdefault("AWS_CONFIG_FILE", AWS_CONFIG_FILE)
+os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", AWS_CREDENTIALS_FILE)
+
+RANCHER_API_URL = env("RANCHER_API_URL")
+RANCHER_ACCESS_KEY = env("RANCHER_ACCESS_KEY")
+RANCHER_SECRET_KEY = env("RANCHER_SECRET_KEY")
+
 logs.configure_structlog(DEBUG)

zoo/datacenters/admin.py

@@ -0,0 +1,10 @@
+from django.contrib import admin
+
+from . import models
+
+
+class DatacenterAdmin(admin.ModelAdmin):
+    search_fields = ("provider", "region")
+
+
+admin.site.register(models.Datacenter, DatacenterAdmin)