Feature/filters

.github/dependabot.yaml

@@ -8,4 +8,7 @@ updates:
  - package-ecosystem: "pip" # See documentation for possible values
  directory: "/" # Location of package manifests
  schedule:
- interval: "weekly"
+ interval: "weekly"
+ ignore:
+ - dependency-name: "*"
+ update-types: ["version-update:semver-major"]

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ exclude: |
 
 repos:
  - repo: https://github.com/psf/black
- rev: 24.2.0
+ rev: 24.3.0
  hooks:
  - id: black
  # args: ["--check"]
@@ -31,6 +31,6 @@ repos:
  - id: flake8
 
  - repo: https://github.com/RobertCraigie/pyright-python
- rev: v1.1.354
+ rev: v1.1.355
  hooks:
  - id: pyright

apps/cap_feed/data_injector/feed.py

@@ -0,0 +1,47 @@
+import json
+import os
+
+from apps.cap_feed.models import Country, Feed, LanguageInfo
+
+module_dir = os.path.dirname(__file__) # get current directory
+
+
+# inject feed configurations if not already present
+def inject_feeds():
+ file_path = os.path.join(
+ os.path.dirname(module_dir),
+ 'feeds.json',
+ )
+ with open(file_path, encoding='utf-8') as file:
+ feeds = json.load(file)
+ print('Injecting feeds...')
+ unique_countries = set()
+ feed_counter = 0
+ for feed_entry in feeds:
+ try:
+ feed = Feed()
+ feed.url = feed_entry['capAlertFeed']
+ feed.country = Country.objects.get(iso3=feed_entry['iso3'])
+ feed_counter += 1
+ unique_countries.add(feed_entry['iso3'])
+ if Feed.objects.filter(url=feed.url).first():
+ continue
+ feed.format = feed_entry['format']
+ feed.polling_interval = 60
+ feed.enable_polling = True
+ feed.enable_rebroadcast = True
+ feed.official = True
+ feed.save()
+
+ language_info = LanguageInfo()
+ language_info.feed = feed
+ language_info.name = feed_entry['name']
+ language_info.language = feed_entry['language']
+ language_info.logo = feed_entry['picUrl']
+ language_info.save()
+
+ except Exception as e:
+ print(feed_entry['name'])
+ print(f'Error injecting feed: {e}')
+
+ print(f'Injected {feed_counter} feeds for {len(unique_countries)} unique countries')

apps/cap_feed/data_injector.py → apps/cap_feed/data_injector/geo.py

@@ -3,7 +3,7 @@
 
 import requests
 
-from .models import Admin1, Continent, Country, Feed, LanguageInfo, Region
+from apps.cap_feed.models import Admin1, Continent, Country, Region
 
 module_dir = os.path.dirname(__file__) # get current directory
 
@@ -43,7 +43,7 @@ def process_continents():
  continent_data = json.loads(response.content)
  process_continents()
  else:
- file_path = os.path.join(module_dir, 'geographical/continents.json')
+ file_path = os.path.join(os.path.dirname(module_dir), 'geographical/continents.json')
  with open(file_path) as file:
  continent_data = json.load(file)
  process_continents()
@@ -68,7 +68,7 @@ def process_regions():
  region_data = json.loads(response.content)
  process_regions()
  else:
- file_path = os.path.join(module_dir, 'geographical/ifrc-regions.json')
+ file_path = os.path.join(os.path.dirname(module_dir), 'geographical/ifrc-regions.json')
  with open(file_path) as file:
  region_data = json.load(file)
  process_regions()
@@ -125,7 +125,7 @@ def process_countries_opendatasoft():
  region_data = json.loads(response.content)
  process_regions()
  else:
- file_path = os.path.join(module_dir, 'geographical/ifrc-regions.json')
+ file_path = os.path.join(os.path.dirname(module_dir), 'geographical/ifrc-regions.json')
  with open(file_path) as file:
  region_data = json.load(file)
  process_regions()
@@ -137,7 +137,7 @@ def process_countries_opendatasoft():
  country_data = json.loads(response.content)
  process_countries_ifrc()
  else:
- file_path = os.path.join(module_dir, 'geographical/ifrc-countries-and-territories.json')
+ file_path = os.path.join(os.path.dirname(module_dir), 'geographical/ifrc-countries-and-territories.json')
  with open(file_path) as file:
  country_data = json.load(file)
  process_countries_ifrc()
@@ -149,7 +149,7 @@ def process_countries_opendatasoft():
  country_data = json.loads(response.content)
  process_countries_opendatasoft()
  else:
- file_path = os.path.join(module_dir, 'geographical/opendatasoft-countries-and-territories.geojson')
+ file_path = os.path.join(os.path.dirname(module_dir), 'geographical/opendatasoft-countries-and-territories.geojson')
  with open(file_path) as file:
  country_data = json.load(file)
  process_countries_opendatasoft()
@@ -184,45 +184,7 @@ def process_admin1s():
  admin1_data = json.loads(response.content)
  process_admin1s()
  else:
- file_path = os.path.join(module_dir, 'geographical/geoBoundariesCGAZ_ADM1.geojson')
+ file_path = os.path.join(os.path.dirname(module_dir), 'geographical/geoBoundariesCGAZ_ADM1.geojson')
  with open(file_path, encoding='utf-8') as f:
  admin1_data = json.load(f)
  process_admin1s()
-
-
-# inject feed configurations if not already present
-def inject_feeds():
- file_path = os.path.join(module_dir, 'feeds.json')
- with open(file_path, encoding='utf-8') as file:
- feeds = json.load(file)
- print('Injecting feeds...')
- unique_countries = set()
- feed_counter = 0
- for feed_entry in feeds:
- try:
- feed = Feed()
- feed.url = feed_entry['capAlertFeed']
- feed.country = Country.objects.get(iso3=feed_entry['iso3'])
- feed_counter += 1
- unique_countries.add(feed_entry['iso3'])
- if Feed.objects.filter(url=feed.url).first():
- continue
- feed.format = feed_entry['format']
- feed.polling_interval = 60
- feed.enable_polling = True
- feed.enable_rebroadcast = True
- feed.official = True
- feed.save()
-
- language_info = LanguageInfo()
- language_info.feed = feed
- language_info.name = feed_entry['name']
- language_info.language = feed_entry['language']
- language_info.logo = feed_entry['picUrl']
- language_info.save()
-
- except Exception as e:
- print(feed_entry['name'])
- print(f'Error injecting feed: {e}')
-
- print(f'Injected {feed_counter} feeds for {len(unique_countries)} unique countries')

apps/cap_feed/dataloaders.py

@@ -9,6 +9,7 @@
 
 from .models import (
  Admin1,
+ Alert,
  AlertAdmin1,
  AlertInfo,
  AlertInfoArea,
@@ -89,6 +90,22 @@ def load_admin1s_by_country(keys: list[int]) -> list[list['Admin1Type']]:
  return [_map[key] for key in keys]
 
 
+def load_info_by_alert(keys: list[int]) -> list[typing.Union['AlertInfoType', None]]:
+ qs = (
+ AlertInfo.objects.filter(alert__in=keys)
+ # TODO: Is this order good enough?
+ .order_by('alert_id', 'id')
+ .distinct('alert_id')
+ .all()
+ )
+
+ _map: dict[int, 'AlertInfoType'] = { # type: ignore[reportGeneralTypeIssues]
+ alert_info.alert_id: alert_info for alert_info in qs
+ }
+
+ return [_map.get(key) for key in keys]
+
+
 def load_infos_by_alert(keys: list[int]) -> list[list['AlertInfoType']]:
  qs = AlertInfo.objects.filter(alert__in=keys).all()
 
@@ -159,6 +176,41 @@ def load_language_info_by_feed(keys: list[int]) -> list[list['LanguageInfoType']
  return [_map[key] for key in keys]
 
 
+def load_alert_count_by_country(keys: list[int]) -> list[int]:
+ qs = (
+ Alert.get_queryset()
+ .filter(country__in=keys)
+ .order_by()
+ .values('country_id')
+ .annotate(
+ count=models.Count('id'),
+ )
+ .values_list('country_id', 'count')
+ )
+
+ _map = {country_id: count for country_id, count in qs}
+
+ return [_map.get(key, 0) for key in keys]
+
+
+def load_alert_count_by_admin1(keys: list[int]) -> list[int]:
+ qs = (
+ Alert.objects
+ # TODO: Add is_expired=False filter
+ .filter(admin1s__in=keys)
+ .order_by()
+ .values('admin1s')
+ .annotate(
+ count=models.Count('id'),
+ )
+ .values_list('admin1s', 'count')
+ )
+
+ _map = {admin1_id: count for admin1_id, count in qs}
+
+ return [_map.get(key, 0) for key in keys]
+
+
 class CapFeedDataloader:
 
  @cached_property
@@ -185,6 +237,10 @@ def load_admin1s_by_alert(self):
  def load_admin1s_by_country(self):
  return DataLoader(load_fn=sync_to_async(load_admin1s_by_country))
 
+ @cached_property
+ def load_info_by_alert(self):
+ return DataLoader(load_fn=sync_to_async(load_info_by_alert))
+
  @cached_property
  def load_infos_by_alert(self):
  return DataLoader(load_fn=sync_to_async(load_infos_by_alert))
@@ -212,3 +268,11 @@ def load_info_area_geocodes_by_info_area(self):
  @cached_property
  def load_language_info_by_feed(self):
  return DataLoader(load_fn=sync_to_async(load_language_info_by_feed))
+
+ @cached_property
+ def load_alert_count_by_country(self):
+ return DataLoader(load_fn=sync_to_async(load_alert_count_by_country))
+
+ @cached_property
+ def load_alert_count_by_admin1(self):
+ return DataLoader(load_fn=sync_to_async(load_alert_count_by_admin1))

apps/cap_feed/filters.py

@@ -1,15 +1,86 @@
 import strawberry
 import strawberry_django
+from django.contrib.postgres.aggregates.general import ArrayAgg
+from django.db import models
 
+from .enums import (
+ AlertInfoCategoryEnum,
+ AlertInfoCertaintyEnum,
+ AlertInfoSeverityEnum,
+ AlertInfoUrgencyEnum,
+)
 from .models import Admin1, Alert, AlertInfo, Country, Feed, Region
 
 
 @strawberry_django.filters.filter(Alert, lookups=True)
 class AlertFilter:
  id: strawberry.auto
- url: strawberry.auto
- sender: strawberry.auto
- admin1s: strawberry.auto
+ country: strawberry.auto
+ sent: strawberry.auto
+
+ @strawberry_django.filter_field
+ def region(
+ self,
+ queryset: models.QuerySet,
+ value: strawberry.ID,
+ prefix: str,
+ ) -> tuple[models.QuerySet, models.Q]:
+ return queryset, models.Q(**{f"{prefix}country__region": value})
+
+ @strawberry_django.filter_field
+ def admin1(
+ self,
+ queryset: models.QuerySet,
+ value: strawberry.ID,
+ prefix: str,
+ ) -> tuple[models.QuerySet, models.Q]:
+ return queryset, models.Q(**{f"{prefix}admin1s": value})
+
+ def _info_enum_fields(self, field, queryset, value, prefix) -> tuple[models.QuerySet, models.Q]:
+ alias_field = f"_infos_{field}_list"
+ queryset = queryset.alias(
+ **{
+ # NOTE: To avoid duplicate alerts when joining infos
+ alias_field: ArrayAgg(f"{prefix}infos__{field}"),
+ }
+ )
+ return queryset, models.Q(**{f"{prefix}{alias_field}__overlap": value})
+
+ @strawberry_django.filter_field
+ def urgency(
+ self,
+ queryset: models.QuerySet,
+ value: list[AlertInfoUrgencyEnum], # type: ignore[reportInvalidTypeForm]
+ prefix: str,
+ ) -> tuple[models.QuerySet, models.Q]:
+ return self._info_enum_fields("urgency", queryset, value, prefix)
+
+ @strawberry_django.filter_field
+ def severity(
+ self,
+ queryset: models.QuerySet,
+ value: list[AlertInfoSeverityEnum], # type: ignore[reportInvalidTypeForm]
+ prefix: str,
+ ) -> tuple[models.QuerySet, models.Q]:
+ return self._info_enum_fields("severity", queryset, value, prefix)
+
+ @strawberry_django.filter_field
+ def certainty(
+ self,
+ queryset: models.QuerySet,
+ value: list[AlertInfoCertaintyEnum], # type: ignore[reportInvalidTypeForm]
+ prefix: str,
+ ) -> tuple[models.QuerySet, models.Q]:
+ return self._info_enum_fields("certainty", queryset, value, prefix)
+
+ @strawberry_django.filter_field
+ def category(
+ self,
+ queryset: models.QuerySet,
+ value: list[AlertInfoCategoryEnum], # type: ignore[reportInvalidTypeForm]
+ prefix: str,
+ ) -> tuple[models.QuerySet, models.Q]:
+ return self._info_enum_fields("category", queryset, value, prefix)
 
 
 @strawberry_django.filters.filter(AlertInfo, lookups=True)
@@ -31,6 +102,17 @@ class CountryFilter:
 class Admin1Filter:
  id: strawberry.auto
 
+ @strawberry_django.filter_field
+ def unknown(
+ self,
+ queryset: models.QuerySet,
+ value: bool,
+ prefix: str,
+ ) -> tuple[models.QuerySet, models.Q]:
+ if value:
+ return queryset, models.Q(**{f"{prefix}id__lt": 0})
+ return queryset, models.Q(**{f"{prefix}id__gte": 0})
+
 
 @strawberry_django.filters.filter(Region, lookups=True)
 class RegionFilter:

apps/cap_feed/models.py

@@ -240,6 +240,11 @@ def __init__(self, *args, **kwargs):
  def __str__(self):
  return self.url
 
+ @classmethod
+ def get_queryset(cls) -> models.QuerySet:
+ # TODO: Add is_expired=False filter
+ return cls.objects.all()
+
  def info_has_been_added(self):
  self.__all_info_added = True