Firebreak/data hub test data contacts

.copilot/config.yml

@@ -0,0 +1,4 @@
+repository: data-hub-api
+builder:
+  name: paketobuildpacks/builder-jammy-full
+  version: 0.3.339

.copilot/image_build_run.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Exit early if something goes wrong
+set -e
+
+# Add commands below to run inside the container after all the other buildpacks have been applied
+export ADMIN_OAUTH2_ENABLED="True"
+export ADMIN_OAUTH2_BASE_URL=""
+export ADMIN_OAUTH2_TOKEN_FETCH_PATH="/o/token/"
+export ADMIN_OAUTH2_USER_PROFILE_PATH="/o/v1/user/me/"
+export ADMIN_OAUTH2_AUTH_PATH="/o/authorize/"
+export ADMIN_OAUTH2_CLIENT_ID="client-id"
+export ADMIN_OAUTH2_CLIENT_SECRET="client-secret"
+export ADMIN_OAUTH2_LOGOUT_PATH="/o/logout"
+export ACTIVITY_STREAM_ACCESS_KEY_ID="some-id"
+export ACTIVITY_STREAM_SECRET_ACCESS_KEY="some-secret"
+export DATABASE_URL="postgresql://postgres:datahub@postgres/datahub"
+export DEBUG="True"
+export DJANGO_SECRET_KEY="changeme"
+export DJANGO_SETTINGS_MODULE="config.settings.local"
+export ES_INDEX_PREFIX="test_index"
+export ES5_URL="http://localhost:9200"
+export OPENSEARCH_URL="http://localhost:9200"
+export OPENSEARCH_INDEX_PREFIX="test_index"
+export PAAS_IP_ALLOWLIST="1.2.3.4"
+export AWS_DEFAULT_REGION="eu-west-2"
+export AWS_ACCESS_KEY_ID="foo"
+export AWS_SECRET_ACCESS_KEY="bar"
+export DEFAULT_BUCKET="baz"
+export SSO_ENABLED="True"
+export STAFF_SSO_BASE_URL="http://sso.invalid/"
+export STAFF_SSO_AUTH_TOKEN="sso-token"
+export DIT_EMAIL_DOMAINS="trade.gov.uk,digital.trade.gov.uk"
+export DATA_HUB_FRONTEND_ACCESS_KEY_ID="frontend-key-id"
+export DATA_HUB_FRONTEND_SECRET_ACCESS_KEY="frontend-key"
+export ES_APM_ENABLED="False"
+export ES_APM_SERVICE_NAME="datahub"
+export ES_APM_SECRET_TOKEN=""
+export ES_APM_SERVER_URL="http://localhost:8200"
+export ES_APM_ENVIRONMENT="circleci"
+export REDIS_BASE_URL="redis://localhost:6379"
+
+python manage.py collectstatic  --noinput

.copilot/phases/build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Exit early if something goes wrong
+set -e
+
+# Add commands below to run as part of the build phase

.copilot/phases/install.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Exit early if something goes wrong
+set -e
+
+# Add commands below to run as part of the install phase

.copilot/phases/post_build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Exit early if something goes wrong
+set -e
+
+# Add commands below to run as part of the post_build phase

.copilot/phases/pre_build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Exit early if something goes wrong
+set -e
+
+# Add commands below to run as part of the pre_build phase

config/settings/common_logging.py

@@ -1,6 +1,7 @@
 import sys
 import sentry_sdk
-from django_log_formatter_ecs import ECSFormatter
+from django_log_formatter_asim import ASIMFormatter
+
 from sentry_sdk.integrations.django import DjangoIntegration
 
 from config.settings.common import *
@@ -13,30 +14,30 @@
         'verbose': {
             'format': '%(asctime)s [%(levelname)s] [%(name)s] %(message)s'
         },
-        'ecs_formatter': {
-            '()': ECSFormatter,
+        "asim_formatter": {
+            "()": ASIMFormatter,
         },
     },
     'handlers': {
-        'ecs': {
+        'asim': {
             'class': 'logging.StreamHandler',
-            'formatter': 'ecs_formatter',
+            'formatter': 'asim_formatter',
             'stream': sys.stdout,
         },
     },
     'root': {
         'level': 'INFO',
-        'handlers': ['ecs'],
+        'handlers': ['asim'],
     },
     'loggers': {
         'django': {
             'level': 'INFO',
-            'handlers': ['ecs'],
+            'handlers': ['asim'],
             'propagate': False,
         },
         'django.db.backends': {
             'level': 'ERROR',
-            'handlers': ['ecs'],
+            'handlers': ['asim'],
             'propagate': False,
         },
     },

config/urls.py

@@ -24,7 +24,7 @@
     path('', include('datahub.oauth.admin.urls')),
     *admin_oauth2_urls,
     path('admin/', admin.site.urls),
-    path('ping.xml', ping, name='ping'),
+    path('pingdom/ping.xml', ping, name='ping'),
     path('whoami/', who_am_i, name='who_am_i'),
 ]
 

data_generator.py

@@ -22,14 +22,23 @@
     pre_save,
 )
 
+from datahub.company.models.adviser import Advisor
+from datahub.company.models.company import Company
+from datahub.company.models.contact import Contact
 from datahub.company.test.factories import (
     AdviserFactory,
     ArchivedCompanyFactory,
+    # ArchivedCompanyFactory,
     CompanyFactory,
     CompanyWithAreaFactory,
+    DuplicateCompanyFactory,
+    # CompanyWithAreaFactory,
     ContactFactory,
+    ContactWithOwnAddressFactory,
+    ContactWithOwnAreaFactory,
     SubsidiaryFactory,
 )
+from datahub.metadata.models import Team
 
 
 class DisableSignals:
@@ -68,50 +77,103 @@ def reconnect(self, signal):
 with DisableSignals():
     start_time = time.time()
 
+    # Pre fetch Metadata
+    teams = list(Team.objects.all())
+
+    advisers = Advisor.objects.all()
+
+    contact = Contact.objects.all()
+
     # In February 2024 there were 18,000 advisers, 500,000 companies, and 950,000 contacts.
     # Alter number of adivsers below to create larger or smaller data set.
-    advisers = AdviserFactory.create_batch(200)
-    print(f'Generated {len(advisers)} advisers')  # noqa
+    # Generate Advisers
+    print('Generating advisers')  # noqa
+    for index in range(10):
+        AdviserFactory(dit_team=random.choice(teams))
+        if index % 10 == 0:
+            print('.', end='')  # noqa
+    advisers = Advisor.objects.all()
+
+    print(f'Generated {advisers.count} advisers')  # noqa
+
+    # # Generate base companies
+    print('\nGenerating Companies')  # noqa
     for index, adviser in enumerate(advisers):
-        companies = CompanyFactory.create_batch(
-            random.randint(1, 25),
+        CompanyFactory.create_batch(
+            random.randint(0, 25),
             created_by=adviser,
-            modified_by=adviser,
+            modified_by=random.choice(advisers),
         )
+        if index % 10 == 0:
+            print('.', end='')  # noqa
 
-        # The ratios of the below types of companies do not reflect the live database.
-        companies.extend(
-            SubsidiaryFactory.create_batch(
-                random.randint(1, 5),
-                created_by=adviser,
-                modified_by=adviser,
-            ),
+
+    def generateContacts(advisers, min, max):
+        print('\nGenerating contacts on advisers')
+        for index, adviser in enumerate(advisers):
+            ContactFactory.create_batch(
+                random.randint(min, max),
+                created_by=random.choice(advisers),
+                modified_by=random.choice(advisers),
+            )
+
+        print('\nGenerating contacts on advisers with a different address from company')
+        for index, adviser in enumerate(advisers):
+            ContactWithOwnAddressFactory.create_batch(
+                random.randint(min, max),
+                created_by=random.choice(advisers),
+                modified_by=random.choice(advisers),
+            )
+
+        print('\nGenerating contacts on advisers with a different address from the contact company that includes an '
+              'area')
+        for index, adviser in enumerate(advisers):
+            ContactWithOwnAreaFactory.create_batch(
+                random.randint(min, max),
+                created_by=random.choice(advisers),
+                modified_by=random.choice(advisers),
+            )
+
+    print('\nGenerating Company variations')  # noqa
+    companies = Company.objects.all()
+    # The ratios of the below types of companies do not reflect the live database.
+    # Generate different type of companies
+    for index, adviser in enumerate(advisers):
+        SubsidiaryFactory.create_batch(
+            random.randint(0, 25),
+            created_by=adviser,
+            modified_by=random.choice(advisers),
+            global_headquarters=random.choice(companies),
+        )
+        CompanyWithAreaFactory.create_batch(
+            random.randint(0, 1),
+            created_by=adviser,
+            modified_by=random.choice(advisers),
         )
-        companies.extend(
-            CompanyWithAreaFactory.create_batch(
-                random.randint(0, 1),
-                created_by=adviser,
-                modified_by=adviser,
-            ),
+        ArchivedCompanyFactory.create_batch(
+            random.randint(0, 1),
+            created_by=adviser,
+            modified_by=adviser,
         )
-        companies.extend(
-            ArchivedCompanyFactory.create_batch(
-                random.randint(0, 1),
-                created_by=adviser,
-                modified_by=adviser,
-            ),
+        DuplicateCompanyFactory.create_batch(
+            random.randint(0, 1),
+            created_by=adviser,
+            modified_by=adviser,
+            transferred_by=random.choice(advisers),
+            transferred_to=random.choice(companies),
         )
+
         # Show a sign of life every now and then
         if index % 10 == 0:
             print('.', end='')  # noqa
 
         # The below ratio of contacts to companies does not reflect the live database.
-        for company in companies:
-            ContactFactory.create_batch(
-                random.randint(1, 2),
-                company=company,
-                created_by=adviser,
-            )
+        # for company in companies:
+        #     ContactFactory.create_batch(
+        #         random.randint(1, 2),
+        #         company=company,
+        #         created_by=adviser,
+        # )
 
     elapsed = time.time() - start_time
     print(f'{timedelta(seconds=elapsed)}')  # noqa

datahub/core/management/commands/rq_health_check.py

@@ -0,0 +1,42 @@
+import sys
+
+from functools import reduce
+from logging import getLogger
+from operator import concat
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from redis import Redis
+from rq import Worker
+
+
+logger = getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = 'RQ Health Check'
+
+    def add_arguments(self, parser):
+        """Define extra arguments."""
+        parser.add_argument(
+            '--queue',
+            type=str,
+            help='Name of the queue to perform health check on.',
+        )
+
+    def handle(self, *args, **options):
+        if options['queue']:
+            queue = str(options['queue'])
+            redis = Redis.from_url(settings.REDIS_BASE_URL)
+            workers = Worker.all(connection=redis)
+            queue_names = reduce(concat, [worker.queue_names() for worker in workers], [])
+            missing_queues = set([queue]) - set(queue_names)
+
+            if missing_queues:
+                logger.error(f'RQ queue not running: {missing_queues}')
+                sys.exit(1)
+            logger.info('OK')
+            sys.exit(0)
+
+        logger.error('Nothing checked! Please provide --queue parameter')
+        sys.exit(1)

datahub/core/test/management/commands/test_rq_health_check.py

@@ -0,0 +1,63 @@
+import logging
+from unittest import mock
+from unittest.mock import patch
+
+import pytest
+
+from django.core.management import call_command
+
+
+class MockWorker:
+    """
+    Mock queue names object returned by worker
+    """
+
+    queue_name = ''
+
+    def __init__(self, queue_name, *args, **kwargs):
+        self.queue_name = queue_name
+
+    def queue_names(self):
+        return self.queue_name
+
+
+def test_rq_health_check_ok():
+    logger = logging.getLogger('datahub.core.management.commands.rq_health_check')
+    with patch(
+        'datahub.core.management.commands.rq_health_check.Worker.all',
+        return_value=[MockWorker(['short-running']), MockWorker(['long-running'])],
+    ):
+        with mock.patch.object(logger, 'info') as mock_info:
+            with pytest.raises(SystemExit) as exception_info:
+                call_command('rq_health_check', '--queue=short-running')
+
+            assert exception_info.value.code == 0
+            assert 'OK' in str(mock_info.call_args_list)
+            assert mock_info.call_count == 1
+
+
+def test_rq_health_check_rq_not_running():
+    logger = logging.getLogger('datahub.core.management.commands.rq_health_check')
+    with patch(
+        'datahub.core.management.commands.rq_health_check.Worker.all',
+        return_value=[MockWorker(['long-running'])],
+    ):
+        with mock.patch.object(logger, 'error') as mock_error:
+            with pytest.raises(SystemExit) as exception_info:
+                call_command('rq_health_check', '--queue=short-running')
+
+            assert exception_info.value.code == 1
+            assert "RQ queue not running: {\'short-running\'}" in str(mock_error.call_args_list)
+            assert mock_error.call_count == 1
+
+
+def test_command_called_without_parameter():
+    logger = logging.getLogger('datahub.core.management.commands.rq_health_check')
+    with mock.patch.object(logger, 'error') as mock_error:
+        with pytest.raises(SystemExit) as exception_info:
+            call_command('rq_health_check')
+
+        assert exception_info.value.code == 1
+        assert 'Nothing checked! Please provide --queue parameter' \
+            in str(mock_error.call_args_list)
+        assert mock_error.call_count == 1