Merge branch 'master' into master-github

.gitignore

@@ -9,7 +9,7 @@ venv
 *.log
 checkstyle.txt
 .python-version
-.env
+.env*
 .direnv
 .envrc
 .elasticbeanstalk/

Pipfile

@@ -17,6 +17,7 @@ pylint = "*"
 pytest = "*"
 pytest-django = "*"
 django-test-migrations = "*"
+black = "*"
 
 [packages]
 appdirs = "*"

Procfile

@@ -1,2 +1,2 @@
 release: python src/manage.py migrate
-web: gunicorn --bind 0.0.0.0:${PORT:-8000} -w 3 --pythonpath src app.wsgi
+web: gunicorn --bind 0.0.0.0:${PORT:-8000} -w ${GUNICORN_WORKERS:-3} -w ${GUNICORN_THREADS:-2} --pythonpath src app.wsgi

bin/docker

@@ -3,4 +3,4 @@ set -e
 
 .venv/bin/python src/manage.py migrate
 .venv/bin/python src/manage.py collectstatic --no-input
-.venv/bin/python src/manage.py runserver 0.0.0.0:8000
+.venv/bin/gunicorn --bind 0.0.0.0:8000 -w ${GUNICORN_WORKERS:-3} -w ${GUNICORN_THREADS:-2} --pythonpath src app.wsgi

pyproject.toml

@@ -0,0 +1,24 @@
+# NOTE: you have to use single-quoted strings in TOML for regular expressions.
+# It's the equivalent of r-strings in Python.  Multiline strings are treated as
+# verbose regular expressions by Black.  Use [ ] to denote a significant space
+# character.
+
+[tool.black]
+line-length = 88
+target-version = ['py36', 'py37', 'py38']
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+  | migrations
+)/
+'''

readme.md

@@ -24,6 +24,18 @@ simply run the following command from the project root:
 docker-compose up
 ```
 
+## Code Style
+
+We are slowly migrating the code style to use [black](https://github.com/psf/black) as 
+a formatter. Black automatically formats the code for you, you can run the formatter
+by running: 
+
+```
+python -m black path/to/directory/or/file.py
+```
+
+All new code should adhere to black formatting standards. 
+
 ## Databases
 Databases are configured in app/settings/\<env\>.py
 

src/.ebextensions/db-migrate.config

@@ -3,5 +3,4 @@ container_commands:
     command: "django-admin.py migrate"
     leader_only: true
   02_collectstatic:
-    command: "source /opt/python/run/venv/bin/activate && python manage.py collectstatic --noinput"
-    leader_only: true
+    command: "source /opt/python/run/venv/bin/activate && python manage.py collectstatic --noinput"

src/analytics/influxdb_wrapper.py

@@ -1,13 +1,17 @@
 from django.conf import settings
+from influxdb_client import InfluxDBClient
 from influxdb_client import Point
 from influxdb_client.client.write_api import SYNCHRONOUS
-from influxdb_client import InfluxDBClient
 
+url = settings.INFLUXDB_URL
+token = settings.INFLUXDB_TOKEN
+influx_org = settings.INFLUXDB_ORG
+read_bucket = settings.INFLUXDB_BUCKET + "_downsampled_15m"
 
 influxdb_client = InfluxDBClient(
-    url=settings.INFLUXDB_URL,
-    token=settings.INFLUXDB_TOKEN,
-    org=settings.INFLUXDB_ORG
+    url=url,
+    token=token,
+    org=influx_org
 )
 
 
@@ -28,3 +32,30 @@ def _record(self, field_name, field_value, tags):
 
     def write(self):
         self.write_api.write(bucket=settings.INFLUXDB_BUCKET, record=self.record)
+
+
+def get_events_for_organisation(organisation_id):
+    """
+    Query influx db for usage for given organisation id
+
+    :param organisation_id: an id of the organisation to get usage for
+    :return: a number of request counts for organisation
+    """
+    query_api = influxdb_client.query_api()
+    query = ' from(bucket:"%s") \
+                |> range(start: -30d, stop: now()) \
+                |> filter(fn:(r) => r._measurement == "api_call") \
+                |> filter(fn: (r) => r["_field"] == "request_count") \
+                |> filter(fn: (r) => r["organisation_id"] == "%s") \
+                |> drop(columns: ["organisation", "resource",  "project", "project_id"]) \
+                |> sum()' % (read_bucket, organisation_id)
+
+    # we should get only one record back
+    # just in case iterate over and sum them up
+    result = query_api.query(org=influx_org, query=query)
+    total = 0
+    for table in result:
+        for record in table.records:
+            total += record.get_value()
+
+    return total

src/analytics/tests/test_influxdb_wrapper.py

@@ -1,7 +1,10 @@
 from unittest import mock
 
+from django.conf import settings
+
 import analytics
 from analytics.influxdb_wrapper import InfluxDBWrapper
+from analytics.influxdb_wrapper import get_events_for_organisation
 
 
 def test_write(monkeypatch):
@@ -19,3 +22,29 @@ def test_write(monkeypatch):
 
     # Then
     mock_write_api.write.assert_called()
+
+
+def test_influx_db_query_when_get_events_then_query_api_called(monkeypatch):
+    # Given
+    org_id = 123
+    influx_org = settings.INFLUXDB_ORG
+    read_bucket = settings.INFLUXDB_BUCKET + "_downsampled_15m"
+    query = ' from(bucket:"%s") \
+                |> range(start: -30d, stop: now()) \
+                |> filter(fn:(r) => r._measurement == "api_call") \
+                |> filter(fn: (r) => r["_field"] == "request_count") \
+                |> filter(fn: (r) => r["organisation_id"] == "%s") \
+                |> drop(columns: ["organisation", "resource",  "project", "project_id"]) \
+                |> sum()' % (read_bucket, org_id)
+
+    mock_influxdb_client = mock.MagicMock()
+    monkeypatch.setattr(analytics.influxdb_wrapper, "influxdb_client", mock_influxdb_client)
+
+    mock_query_api = mock.MagicMock()
+    mock_influxdb_client.query_api.return_value = mock_query_api
+
+    # When
+    get_events_for_organisation(org_id)
+
+    # Then
+    mock_query_api.query.assert_called_once_with(org=influx_org, query=query)

src/api/urls/deprecated.py

@@ -1,6 +1,7 @@
 from django.conf.urls import url
 
-from environments.views import SDKTraitsDeprecated, SDKIdentitiesDeprecated
+from environments.identities.traits.views import SDKTraitsDeprecated
+from environments.identities.views import SDKIdentitiesDeprecated
 from features.views import SDKFeatureStates
 
 app_name = 'deprecated'

src/api/urls/v1.py

@@ -4,7 +4,8 @@
 from drf_yasg.views import get_schema_view
 from rest_framework import routers, permissions, authentication
 
-from environments.views import SDKIdentities, SDKTraits
+from environments.identities.traits.views import SDKTraits
+from environments.identities.views import SDKIdentities
 from features.views import SDKFeatureStates
 from organisations.views import chargebee_webhook
 from segments.views import SDKSegments

src/app/settings/common.py

@@ -96,7 +96,12 @@
     'users',
     'organisations',
     'projects',
+
     'environments',
+    'environments.permissions',
+    'environments.identities',
+    'environments.identities.traits',
+
     'features',
     'segments',
     'e2etests',
@@ -105,6 +110,7 @@
     'drf_yasg',
     'audit',
     'permissions',
+    'projects.tags',
 
     # 2FA
     'trench',

src/app/settings/master.py

@@ -38,6 +38,3 @@
 }
 
 REST_FRAMEWORK['PAGE_SIZE'] = 999
-
-SECURE_SSL_REDIRECT = True
-SECURE_REDIRECT_EXEMPT = [r'^health$']  # /health is exempt as it's used for EB health checks

src/app/settings/staging.py

@@ -38,6 +38,3 @@
 }
 
 REST_FRAMEWORK['PAGE_SIZE'] = 999
-
-SECURE_SSL_REDIRECT = True
-SECURE_REDIRECT_EXEMPT = [r'^health$']  # /health is exempt as it's used for EB health checks

src/audit/signals.py

@@ -14,7 +14,10 @@
 @receiver(post_save, sender=AuditLog)
 def call_webhooks(sender, instance, **kwargs):
     data = AuditLogSerializer(instance=instance).data
+
     if not (instance.project or instance.environment):
         logger.warning('Audit log without project or environment. Not sending webhook.')
-    organisation = instance.project.organisation or instance.environment.project.organisation
+        return
+
+    organisation = instance.project.organisation if instance.project else instance.environment.project.organisation
     call_organisation_webhooks(organisation, data, WebhookEventType.AUDIT_LOG_CREATED)

src/environments/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'environments.apps.EnvironmentsConfig'

src/environments/admin.py

@@ -3,9 +3,10 @@
 
 from django.conf import settings
 from django.contrib import admin
-from simple_history.admin import SimpleHistoryAdmin
 
-from .models import Identity, Environment, Trait, Webhook
+from .identities.traits.admin import TraitAdmin
+from .models import Environment, Webhook
+from .identities.traits.models import Trait
 
 
 class WebhookInline(admin.TabularInline):
@@ -20,25 +21,3 @@ class EnvironmentAdmin(admin.ModelAdmin):
     list_filter = ('created_date', 'project',)
     search_fields = ('name', 'project__name', 'api_key',)
     inlines = (WebhookInline,)
-
-
-class IdentityAdmin(admin.ModelAdmin):
-    date_hierarchy = 'created_date'
-    list_display = ('__str__', 'created_date', 'environment',)
-    list_filter = ('created_date', 'environment',)
-    search_fields = ('identifier',)
-
-
-class TraitAdmin(SimpleHistoryAdmin):
-    date_hierarchy = 'created_date'
-    list_display = ('__str__', 'value_type', 'boolean_value', 'integer_value', 'string_value',
-                    'created_date', 'identity',)
-    list_filter = ('value_type', 'created_date', 'identity',)
-    raw_id_fields = ('identity',)
-    search_fields = ('string_value', 'trait_key', 'identity__identifier',)
-
-
-if settings.ENV in ('local', 'dev'):
-    # these shouldn't be displayed in production environments but are useful in development environments
-    admin.site.register(Identity, IdentityAdmin)
-    admin.site.register(Trait, TraitAdmin)

src/environments/authentication.py

@@ -1,7 +1,7 @@
 from django.conf import settings
 from django.core.cache import caches
-from rest_framework import exceptions
 from rest_framework.authentication import BaseAuthentication
+from rest_framework.exceptions import AuthenticationFailed
 
 from environments.models import Environment
 
@@ -14,14 +14,18 @@ class EnvironmentKeyAuthentication(BaseAuthentication):
     """
     def authenticate(self, request):
         try:
-            environment = Environment.get_from_cache(request.META.get('HTTP_X_ENVIRONMENT_KEY'))
+            api_key = request.META.get('HTTP_X_ENVIRONMENT_KEY')
+            environment = Environment.get_from_cache(api_key)
         except Environment.DoesNotExist:
-            raise exceptions.AuthenticationFailed('Invalid or missing Environment Key')
+            raise AuthenticationFailed('Invalid or missing Environment Key')
 
         if not self._can_serve_flags(environment):
-            raise exceptions.AuthenticationFailed('Organisation is disabled from serving flags.')
+            raise AuthenticationFailed('Organisation is disabled from serving flags.')
 
         request.environment = environment
 
+        # DRF authentication expects a two tuple to be returned containing User, auth
+        return None, None
+
     def _can_serve_flags(self, environment):
         return not environment.project.organisation.stop_serving_flags

src/environments/exceptions.py

@@ -1,2 +1,4 @@
 class EnvironmentHeaderNotPresentError(Exception):
     pass
+
+

src/environments/identities/__init__.py

@@ -0,0 +1 @@
+default_app_config = "environments.identities.apps.IdentitiesConfig"

src/environments/identities/admin.py

@@ -0,0 +1,16 @@
+from django.conf import settings
+from django.contrib import admin
+
+from .models import Identity
+
+
+class IdentityAdmin(admin.ModelAdmin):
+    date_hierarchy = "created_date"
+    list_display = ("__str__", "created_date", "environment")
+    list_filter = ("created_date", "environment")
+    search_fields = ("identifier",)
+
+
+if settings.ENV in ("local", "dev"):
+    # identities be displayed in prod envs but are useful in development envs
+    admin.site.register(Identity, IdentityAdmin)

src/environments/identities/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class IdentitiesConfig(AppConfig):
+    name = "environments.identities"

src/environments/identities/migrations/0001_initial.py

@@ -0,0 +1,45 @@
+# Generated by Django 2.2.16 on 2020-09-17 10:32
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        # this is just here to ensure that feature models correctly refer to identities
+        # models as identities. instead of environments.
+        # there is still some pain with rolling feature migrations backwards but
+        # hopefully this won't be required
+        ('features', '0023_auto_20200717_1515')
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            state_operations=[
+                migrations.CreateModel(
+                    name='Identity',
+                    fields=[
+                        ('id', models.AutoField(auto_created=True, primary_key=True,
+                                                serialize=False, verbose_name='ID')),
+                        ('identifier', models.CharField(max_length=2000)),
+                        ('created_date', models.DateTimeField(auto_now_add=True,
+                                                              verbose_name='DateCreated')),
+                        ('environment',
+                         models.ForeignKey(on_delete=django.db.models.deletion.CASCADE,
+                                           related_name='identities',
+                                           to='environments.Environment')),
+                    ],
+                    options={
+                        'verbose_name_plural': 'Identities',
+                        'db_table': 'environments_identity',
+                        'ordering': ['id'],
+                        'unique_together': {('environment', 'identifier')},
+                    },
+                ),
+            ],
+            database_operations=[]
+        )
+    ]