diff --git a/.github/workflows/celery-test_cap-aggregator(celery-test).yml b/.github/workflows/celery-test_cap-aggregator(celery-test).yml
new file mode 100644
index 0000000..ed21ca3
--- /dev/null
+++ b/.github/workflows/celery-test_cap-aggregator(celery-test).yml
@@ -0,0 +1,63 @@
+# Docs for the Azure Web Apps Deploy action: https://github.com/Azure/webapps-deploy
+# More GitHub Actions for Azure: https://github.com/Azure/actions
+# More info on Python, GitHub Actions, and Azure App Service: https://aka.ms/python-webapps-actions
+
+name: Build and deploy Python app to Azure Web App - cap-aggregator
+
+on:
+ push:
+ branches:
+ - celery-test
+ workflow_dispatch:
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Set up Python version
+ uses: actions/setup-python@v1
+ with:
+ python-version: '3.11'
+
+ - name: Create and start virtual environment
+ run: |
+ python -m venv venv
+ source venv/bin/activate
+
+ - name: Install dependencies
+ run: pip install -r requirements.txt
+
+ # Optional: Add step to run tests here (PyTest, Django test suites, etc.)
+
+ - name: Upload artifact for deployment jobs
+ uses: actions/upload-artifact@v2
+ with:
+ name: python-app
+ path: |
+ .
+ !venv/
+
+ deploy:
+ runs-on: ubuntu-latest
+ needs: build
+ environment:
+ name: 'celery-test'
+ url: ${{ steps.deploy-to-webapp.outputs.webapp-url }}
+
+ steps:
+ - name: Download artifact from build job
+ uses: actions/download-artifact@v2
+ with:
+ name: python-app
+ path: .
+
+ - name: 'Deploy to Azure Web App'
+ uses: azure/webapps-deploy@v2
+ id: deploy-to-webapp
+ with:
+ app-name: 'cap-aggregator'
+ slot-name: 'celery-test'
+ publish-profile: ${{ secrets.AZUREAPPSERVICE_PUBLISHPROFILE_0FD4FFF13EAD42BCA6117317784F30E7 }}
diff --git a/.github/workflows/python-test.yaml b/.github/workflows/python-test.yaml
index 747fbb0..b92bf06 100644
--- a/.github/workflows/python-test.yaml
+++ b/.github/workflows/python-test.yaml
@@ -2,9 +2,11 @@ name: Python check
on:
push:
- branches: [ main ]
+ branches:
+ - '*'
pull_request:
- branches: [ main ]
+ branches:
+ - '*'
jobs:
test_package:
diff --git a/.gitignore b/.gitignore
index 9b957b1..f536698 100644
--- a/.gitignore
+++ b/.gitignore
@@ -128,4 +128,7 @@ dmypy.json
# Pyre type checker
.pyre/
-.azure
\ No newline at end of file
+.azure
+
+# editors
+.idea/
\ No newline at end of file
diff --git a/cap_feed/alert_processing.py b/cap_feed/alert_processing.py
new file mode 100644
index 0000000..d56190c
--- /dev/null
+++ b/cap_feed/alert_processing.py
@@ -0,0 +1,163 @@
+import json
+import requests
+
+import xml.etree.ElementTree as ET
+import pytz
+from .models import Alert, Region, Country
+from datetime import datetime
+from django.utils import timezone
+
+
+
+# inject region and country data if not already present
+def inject_unknown_regions():
+ if Region.objects.count() == 0:
+ inject_regions()
+ # inject unknown region for alerts without a defined region
+ unknown_region = Region()
+ unknown_region.id = -1
+ unknown_region.name = "Unknown"
+ unknown_region.save()
+ if Country.objects.count() == 0:
+ inject_countries()
+ # inject unknown country for alerts without a defined country
+ unknown_country = Country()
+ unknown_country.id = -1
+ unknown_country.name = "Unknown"
+ unknown_country.save()
+
+# inject region data
+def inject_regions():
+ with open('cap_feed/region.json') as file:
+ region_data = json.load(file)
+ for region_entry in region_data:
+ region = Region()
+ region.id = region_entry["id"]
+ region.name = region_entry["region_name"]
+ region.centroid = region_entry["centroid"]
+ coordinates = region_entry["bbox"]["coordinates"][0]
+ for coordinate in coordinates:
+ region.polygon += str(coordinate[0]) + "," + str(coordinate[1]) + " "
+ region.save()
+
+# inject country data
+def inject_countries():
+ with open('cap_feed/country.json') as file:
+ country_data = json.load(file)
+ for country_entry in country_data:
+ country = Country()
+ country.id = country_entry["id"]
+ country.name = country_entry["name"]
+ region_id = country_entry["region"]
+ if ("Region" in country.name) or ("Cluster" in country.name):
+ continue
+ if region_id is not None:
+ country.region = Region.objects.get(id=country_entry["region"])
+ if country_entry["iso"] is not None:
+ country.iso = country_entry["iso"]
+ if country_entry["iso3"] is not None:
+ country.iso3 = country_entry["iso3"]
+ if country_entry["bbox"] is not None:
+ coordinates = country_entry["bbox"]["coordinates"][0]
+ for coordinate in coordinates:
+ country.polygon += str(coordinate[0]) + "," + str(coordinate[1]) + " "
+ if country_entry["centroid"] is not None:
+ coordinates = country_entry["centroid"]["coordinates"]
+ country.centroid = str(coordinates[0]) + "," + str(coordinates[1])
+ country.save()
+
+# converts CAP1.2 iso format datetime string to datetime object in UTC timezone
+def convert_datetime(original_datetime):
+ return datetime.fromisoformat(original_datetime).astimezone(pytz.timezone('UTC'))
+
+# gets alerts from sources and processes them different for each source format
+def get_alerts():
+ # list of sources and configurations
+ sources = [
+ ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france", "FRA", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}),
+ ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-belgium", "BEL", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}),
+ ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-austria", "AUT", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}),
+ ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-slovakia", "SVK", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}),
+ ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-slovenia", "SVN", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}),
+ ("https://alert.metservice.gov.jm/capfeed.php", "JAM", "capfeedphp", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}),
+ ]
+
+ for source in sources:
+ url, iso3, format, ns = source
+ match format:
+ case "meteoalarm":
+ get_alert_meteoalarm(url, iso3, ns)
+ case "capfeedphp":
+ get_alert_capfeedphp(url, iso3, ns)
+
+# processing for meteoalarm format, example: https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france
+def get_alert_meteoalarm(url, iso3, ns):
+ response = requests.get(url)
+ root = ET.fromstring(response.content)
+ for entry in root.findall('atom:entry', ns):
+ try:
+ alert = Alert()
+ alert.id = entry.find('atom:id', ns).text
+ alert.identifier = entry.find('cap:identifier', ns).text
+ alert.sender = url
+ alert.sent = convert_datetime(entry.find('cap:sent', ns).text)
+ alert.status = entry.find('cap:status', ns).text
+ alert.msg_type = entry.find('cap:message_type', ns).text
+ alert.scope = entry.find('cap:scope', ns).text
+ alert.urgency = entry.find('cap:urgency', ns).text
+ alert.severity = entry.find('cap:severity', ns).text
+ alert.certainty = entry.find('cap:certainty', ns).text
+ alert.effective = convert_datetime(entry.find('cap:effective', ns).text)
+ alert.expires = convert_datetime(entry.find('cap:expires', ns).text)
+ if alert.expires < timezone.now():
+ continue
+
+ alert.area_desc = entry.find('cap:areaDesc', ns).text
+ alert.event = entry.find('cap:event', ns).text
+
+ geocode = entry.find('cap:geocode', ns)
+ alert.geocode_name = geocode.find('atom:valueName', ns).text
+ alert.geocode_value = geocode.find('atom:value', ns).text
+ alert.country = Country.objects.get(iso3=iso3)
+ alert.save()
+ except:
+ pass
+
+# processing for capfeedphp format, example: https://alert.metservice.gov.jm/capfeed.php
+def get_alert_capfeedphp(url, iso3, ns):
+ response = requests.get(url)
+ root = ET.fromstring(response.content)
+ for entry in root.findall('atom:entry', ns):
+ try:
+ alert = Alert()
+ alert.id = entry.find('atom:id', ns).text
+
+ entry_content = entry.find('atom:content', ns)
+ entry_content_alert = entry_content.find('cap:alert', ns)
+ alert.identifier = entry_content_alert.find('cap:identifier', ns).text
+ alert.sender = entry_content_alert.find('cap:sender', ns).text
+ alert.sent = convert_datetime(entry_content_alert.find('cap:sent', ns).text)
+ alert.status = entry_content_alert.find('cap:status', ns).text
+ alert.msg_type = entry_content_alert.find('cap:msgType', ns).text
+ alert.scope = entry_content_alert.find('cap:scope', ns).text
+
+ entry_content_alert_info = entry_content_alert.find('cap:info', ns)
+ alert.urgency = entry_content_alert_info.find('cap:urgency', ns).text
+ alert.severity = entry_content_alert_info.find('cap:severity', ns).text
+ alert.certainty = entry_content_alert_info.find('cap:certainty', ns).text
+ alert.effective = convert_datetime(entry_content_alert_info.find('cap:effective', ns).text)
+ alert.expires = convert_datetime(entry_content_alert_info.find('cap:expires', ns).text)
+ if alert.expires < timezone.now():
+ continue
+ alert.event = entry_content_alert_info.find('cap:event', ns).text
+
+ entry_content_alert_info_area = entry_content_alert_info.find('cap:area', ns)
+ alert.area_desc = entry_content_alert_info_area.find('cap:areaDesc', ns).text
+ alert.polygon = entry_content_alert_info_area.find('cap:polygon', ns).text
+ alert.country = Country.objects.get(iso3=iso3)
+ alert.save()
+ except:
+ pass
+
+def remove_expired_alerts():
+ Alert.objects.filter(expires__lt=timezone.now()).delete()
\ No newline at end of file
diff --git a/cap_feed/country.json b/cap_feed/country.json
index 92432b3..6fcd59d 100644
--- a/cap_feed/country.json
+++ b/cap_feed/country.json
@@ -68,7 +68,7 @@
"iso": null,
"iso3": null,
"society_url": "",
- "region": null,
+ "region": 2,
"key_priorities": null,
"inform_score": null,
"id": 343,
@@ -282,7 +282,7 @@
"iso": "AI",
"iso3": "AIA",
"society_url": "",
- "region": null,
+ "region": 1,
"key_priorities": null,
"inform_score": null,
"id": 217,
@@ -426,7 +426,7 @@
"iso": "AW",
"iso3": "ABW",
"society_url": "",
- "region": null,
+ "region": 1,
"key_priorities": null,
"inform_score": null,
"id": 231,
@@ -927,7 +927,7 @@
"iso": "BM",
"iso3": "BMU",
"society_url": "",
- "region": null,
+ "region": 1,
"key_priorities": null,
"inform_score": null,
"id": 233,
@@ -992,7 +992,7 @@
"iso": null,
"iso3": null,
"society_url": "",
- "region": null,
+ "region": 0,
"key_priorities": null,
"inform_score": null,
"id": 346,
@@ -3054,7 +3054,7 @@
"iso": "GI",
"iso3": "GIB",
"society_url": "",
- "region": null,
+ "region": 3,
"key_priorities": null,
"inform_score": null,
"id": 241,
@@ -3421,7 +3421,7 @@
"iso": null,
"iso3": null,
"society_url": "",
- "region": null,
+ "region": 0,
"key_priorities": null,
"inform_score": null,
"id": 351,
@@ -3472,7 +3472,7 @@
"iso": "VA",
"iso3": "VAT",
"society_url": "",
- "region": null,
+ "region": 3,
"key_priorities": null,
"inform_score": null,
"id": 274,
@@ -3664,7 +3664,7 @@
"iso": null,
"iso3": null,
"society_url": "",
- "region": null,
+ "region": 0,
"key_priorities": null,
"inform_score": null,
"id": 352,
@@ -4616,7 +4616,7 @@
"iso": "MO",
"iso3": "MAC",
"society_url": "http://www.redcross.org.mo/",
- "region": null,
+ "region": 2,
"key_priorities": null,
"inform_score": null,
"id": 279,
@@ -5311,7 +5311,7 @@
"iso": "NR",
"iso3": "NRU",
"society_url": "",
- "region": null,
+ "region": 2,
"key_priorities": null,
"inform_score": null,
"id": 251,
@@ -5571,7 +5571,7 @@
"iso": "NU",
"iso3": "NIU",
"society_url": "",
- "region": null,
+ "region": 2,
"key_priorities": null,
"inform_score": null,
"id": 253,
@@ -5752,7 +5752,7 @@
"iso": "OM",
"iso3": "OMN",
"society_url": "",
- "region": null,
+ "region": 2,
"key_priorities": null,
"inform_score": null,
"id": 258,
@@ -6060,7 +6060,7 @@
"iso": "PN",
"iso3": "PCN",
"society_url": "",
- "region": null,
+ "region": 2,
"key_priorities": null,
"inform_score": null,
"id": 256,
@@ -6156,7 +6156,7 @@
"iso": "PR",
"iso3": "PRI",
"society_url": "",
- "region": null,
+ "region": 1,
"key_priorities": null,
"inform_score": null,
"id": 257,
@@ -6365,7 +6365,7 @@
"iso": "SH",
"iso3": "SHN",
"society_url": "",
- "region": null,
+ "region": 0,
"key_priorities": null,
"inform_score": null,
"id": 262,
@@ -6616,7 +6616,7 @@
"iso": null,
"iso3": null,
"society_url": "",
- "region": null,
+ "region": 2,
"key_priorities": null,
"inform_score": null,
"id": 358,
@@ -7148,7 +7148,7 @@
"iso": null,
"iso3": null,
"society_url": "",
- "region": null,
+ "region": 2,
"key_priorities": null,
"inform_score": null,
"id": 360,
@@ -8230,7 +8230,7 @@
"iso": "EH",
"iso3": "ESH",
"society_url": "",
- "region": null,
+ "region": 0,
"key_priorities": null,
"inform_score": null,
"id": 269,
diff --git a/cap_feed/migrations/0001_initial.py b/cap_feed/migrations/0001_initial.py
new file mode 100644
index 0000000..e041297
--- /dev/null
+++ b/cap_feed/migrations/0001_initial.py
@@ -0,0 +1,53 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Alert',
+ fields=[
+ ('id', models.CharField(max_length=255, primary_key=True, serialize=False)),
+ ('identifier', models.CharField(max_length=255)),
+ ('sender', models.CharField(max_length=255)),
+ ('sent', models.DateTimeField()),
+ ('status', models.CharField(max_length=255)),
+ ('msg_type', models.CharField(max_length=255)),
+ ('scope', models.CharField(max_length=255)),
+ ('urgency', models.CharField(max_length=255)),
+ ('severity', models.CharField(max_length=255)),
+ ('certainty', models.CharField(max_length=255)),
+ ('expires', models.DateTimeField()),
+ ('area_desc', models.CharField(max_length=255)),
+ ('event', models.CharField(max_length=255)),
+ ('geocode_name', models.CharField(blank=True, default='', max_length=255)),
+ ('geocode_value', models.CharField(blank=True, default='', max_length=255)),
+ ('polygon', models.TextField(blank=True, default='', max_length=16383)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Country',
+ fields=[
+ ('id', models.CharField(max_length=255, primary_key=True, serialize=False)),
+ ('name', models.CharField(max_length=255)),
+ ('iso', models.CharField(blank=True, default='', max_length=255)),
+ ('iso3', models.CharField(blank=True, default='', max_length=255)),
+ ('polygon', models.TextField(blank=True, default='', max_length=16383)),
+ ('centroid', models.CharField(blank=True, default='', max_length=255)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='Region',
+ fields=[
+ ('id', models.CharField(max_length=255, primary_key=True, serialize=False)),
+ ('name', models.CharField(max_length=255)),
+ ('polygon', models.TextField(blank=True, default='', max_length=16383)),
+ ('centroid', models.CharField(blank=True, default='', max_length=255)),
+ ],
+ ),
+ ]
diff --git a/cap_feed/migrations/0002_country_region.py b/cap_feed/migrations/0002_country_region.py
new file mode 100644
index 0000000..b3f2374
--- /dev/null
+++ b/cap_feed/migrations/0002_country_region.py
@@ -0,0 +1,19 @@
+# Generated by Django 4.2.2 on 2023-06-23 11:38
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('cap_feed', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='country',
+ name='region',
+ field=models.ForeignKey(blank=True, default='', on_delete=django.db.models.deletion.SET_DEFAULT, to='cap_feed.region'),
+ ),
+ ]
diff --git a/cap_feed/migrations/0003_alert_country.py b/cap_feed/migrations/0003_alert_country.py
new file mode 100644
index 0000000..06670bc
--- /dev/null
+++ b/cap_feed/migrations/0003_alert_country.py
@@ -0,0 +1,19 @@
+# Generated by Django 4.2.2 on 2023-06-23 13:55
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('cap_feed', '0002_country_region'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='alert',
+ name='country',
+ field=models.ForeignKey(blank=True, default='', on_delete=django.db.models.deletion.SET_DEFAULT, to='cap_feed.country'),
+ ),
+ ]
diff --git a/cap_feed/migrations/0004_alter_alert_country.py b/cap_feed/migrations/0004_alter_alert_country.py
new file mode 100644
index 0000000..4b75327
--- /dev/null
+++ b/cap_feed/migrations/0004_alter_alert_country.py
@@ -0,0 +1,19 @@
+# Generated by Django 4.2.2 on 2023-06-23 13:56
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('cap_feed', '0003_alert_country'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='alert',
+ name='country',
+ field=models.ForeignKey(default='-1', on_delete=django.db.models.deletion.SET_DEFAULT, to='cap_feed.country'),
+ ),
+ ]
diff --git a/cap_feed/migrations/0005_alter_country_region.py b/cap_feed/migrations/0005_alter_country_region.py
new file mode 100644
index 0000000..a74c0a1
--- /dev/null
+++ b/cap_feed/migrations/0005_alter_country_region.py
@@ -0,0 +1,19 @@
+# Generated by Django 4.2.2 on 2023-06-23 14:32
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('cap_feed', '0004_alter_alert_country'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='country',
+ name='region',
+ field=models.ForeignKey(default='-1', on_delete=django.db.models.deletion.SET_DEFAULT, to='cap_feed.region'),
+ ),
+ ]
diff --git a/cap_feed/migrations/0006_alert_effective.py b/cap_feed/migrations/0006_alert_effective.py
new file mode 100644
index 0000000..439a121
--- /dev/null
+++ b/cap_feed/migrations/0006_alert_effective.py
@@ -0,0 +1,20 @@
+# Generated by Django 4.2.2 on 2023-06-24 13:16
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('cap_feed', '0005_alter_country_region'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='alert',
+ name='effective',
+ field=models.DateTimeField(default=django.utils.timezone.now),
+ preserve_default=False,
+ ),
+ ]
diff --git a/cap_feed/models.py b/cap_feed/models.py
index d6bb221..eaff961 100644
--- a/cap_feed/models.py
+++ b/cap_feed/models.py
@@ -2,6 +2,27 @@
from django.db import models
# Create your models here.
+
+class Region(models.Model):
+ id = models.CharField(max_length=255, primary_key=True)
+ name = models.CharField(max_length=255)
+ polygon = models.TextField(max_length=16383, blank=True, default='')
+ centroid = models.CharField(max_length=255, blank=True, default='')
+
+ def __str__(self):
+ return self.name
+
+class Country(models.Model):
+ id = models.CharField(max_length=255, primary_key=True)
+ name = models.CharField(max_length=255)
+ iso = models.CharField(max_length=255, blank=True, default='')
+ iso3 = models.CharField(max_length=255, blank=True, default='')
+ polygon = models.TextField(max_length=16383, blank=True, default='')
+ centroid = models.CharField(max_length=255, blank=True, default='')
+ region = models.ForeignKey(Region, on_delete=models.SET_DEFAULT, default='-1')
+
+ def __str__(self):
+ return self.name
class Alert(models.Model):
id = models.CharField(max_length=255, primary_key=True)
@@ -14,6 +35,7 @@ class Alert(models.Model):
urgency = models.CharField(max_length=255)
severity = models.CharField(max_length=255)
certainty = models.CharField(max_length=255)
+ effective = models.DateTimeField()
expires = models.DateTimeField()
area_desc = models.CharField(max_length=255)
@@ -21,26 +43,7 @@ class Alert(models.Model):
geocode_name = models.CharField(max_length=255, blank=True, default='')
geocode_value = models.CharField(max_length=255, blank=True, default='')
polygon = models.TextField(max_length=16383, blank=True, default='')
-
- def __str__(self):
- return self.id
-
-class Region(models.Model):
- id = models.CharField(max_length=255, primary_key=True)
- name = models.CharField(max_length=255)
- polygon = models.TextField(max_length=16383, blank=True, default='')
- centroid = models.CharField(max_length=255, blank=True, default='')
-
- def __str__(self):
- return self.id
-
-class Country(models.Model):
- id = models.CharField(max_length=255, primary_key=True)
- name = models.CharField(max_length=255)
- iso = models.CharField(max_length=255, blank = True, default='')
- iso3 = models.CharField(max_length=255, blank = True, default='')
- polygon = models.TextField(max_length=16383, blank=True, default='')
- centroid = models.CharField(max_length=255, blank=True, default='')
+ country = models.ForeignKey(Country, on_delete=models.SET_DEFAULT, default='-1')
def __str__(self):
return self.id
\ No newline at end of file
diff --git a/cap_feed/region.json b/cap_feed/region.json
index bf54102..5c6349d 100644
--- a/cap_feed/region.json
+++ b/cap_feed/region.json
@@ -3,6 +3,7 @@
"name": 0,
"id": 0,
"region_name": "Africa",
+ "centroid": "17.458740234362434 -2.677413176352464",
"bbox": {
"type": "Polygon",
"coordinates": [
@@ -21,6 +22,7 @@
"name": 1,
"id": 1,
"region_name": "Americas",
+ "centroid": "-80.83261851536723 -2.6920536197633442",
"bbox": {
"type": "Polygon",
"coordinates": [
@@ -39,6 +41,7 @@
"name": 2,
"id": 2,
"region_name": "Asia Pacific",
+ "centroid": "117.78896429869648 -3.1783208418475954",
"bbox": {
"type": "Polygon",
"coordinates": [
@@ -57,6 +60,7 @@
"name": 3,
"id": 3,
"region_name": "Europe",
+ "centroid": "30.64725652750233 45.572165430308736",
"bbox": {
"type": "Polygon",
"coordinates": [
@@ -75,6 +79,7 @@
"name": 4,
"id": 4,
"region_name": "Middle East & North Africa",
+ "centroid": "21.18749859869599 31.264366696701767",
"bbox": {
"type": "Polygon",
"coordinates": [
diff --git a/cap_feed/tasks.py b/cap_feed/tasks.py
new file mode 100644
index 0000000..3575939
--- /dev/null
+++ b/cap_feed/tasks.py
@@ -0,0 +1,14 @@
+from __future__ import absolute_import, unicode_literals
+from celery import shared_task
+
+import cap_feed.alert_processing as ap
+
+@shared_task(bind=True)
+def get_alerts(self):
+ ap.get_alerts()
+ return "get_alerts DONE"
+
+@shared_task(bind=True)
+def remove_expired_alerts(self):
+ ap.remove_expired_alerts()
+ return "remove_expired_alerts DONE"
\ No newline at end of file
diff --git a/cap_feed/templates/cap_feed/index.html b/cap_feed/templates/cap_feed/index.html
index cf802df..2548f1f 100644
--- a/cap_feed/templates/cap_feed/index.html
+++ b/cap_feed/templates/cap_feed/index.html
@@ -9,9 +9,10 @@ {{ alert.event }}
Severity: {{ alert.severity }}
Urgency: {{ alert.urgency }}
Sent: {{ alert.sent }}
+ Effective: {{ alert.effective }}
+ Expires: {{ alert.expires }}
Sender: {{ alert.sender }}
- Geocode Name: {{ alert.geocode_name }}
- Geocode Value: {{ alert.geocode_value }}
+ Country: {{ alert.country }}
{% endfor %}
diff --git a/cap_feed/tests.py b/cap_feed/tests.py
index f4705fb..e3d5f3e 100644
--- a/cap_feed/tests.py
+++ b/cap_feed/tests.py
@@ -1,7 +1,28 @@
-import datetime
+import pytz
+from datetime import datetime
+import cap_feed.alert_processing as ap
from django.test import TestCase
from django.urls import reverse
+from django.utils import timezone
from .models import Alert
+class AlertModelTests(TestCase):
+ def test_alert_source_datetime_converted_to_utc(self):
+ """
+ Was the iso format cap alert datetime field with timezone offsets processed correctly to utc timezone?
+ """
+ cap_sent = "2023-06-24T22:00:00-05:00"
+ cap_effective = "2023-06-24T22:00:00+00:00"
+ cap_expires = "2023-06-24T22:00:00+05:00"
+ alert = Alert()
+ alert.sent = ap.convert_datetime(cap_sent)
+ alert.effective = ap.convert_datetime(cap_effective)
+ alert.expires = ap.convert_datetime(cap_expires)
+ utc_sent = datetime(2023, 6, 25, 3, 0, 0, 0, pytz.UTC)
+ utc_effective = datetime(2023, 6, 24, 22, 0, 0, 0, pytz.UTC)
+ utc_expires = datetime(2023, 6, 24, 17, 0, 0, 0, pytz.UTC)
+ assert alert.sent == utc_sent
+ assert alert.effective == utc_effective
+ assert alert.expires == utc_expires
\ No newline at end of file
diff --git a/cap_feed/urls.py b/cap_feed/urls.py
index 88a9cac..cefc089 100644
--- a/cap_feed/urls.py
+++ b/cap_feed/urls.py
@@ -4,4 +4,5 @@
urlpatterns = [
path('', views.index, name='index'),
+ path('pollingalerts/', views.polling_alerts, name = 'polling_alerts')
]
diff --git a/cap_feed/views.py b/cap_feed/views.py
index d768e31..0706a96 100644
--- a/cap_feed/views.py
+++ b/cap_feed/views.py
@@ -1,35 +1,15 @@
-import requests
-import xml.etree.ElementTree as ET
+import json
-from django.shortcuts import get_object_or_404, render
from django.http import HttpResponse
from django.template import loader
from .models import Alert, Region, Country
-
-
-import json
-with open('cap_feed/region.json') as file:
- data = json.load(file)
-
-region_centroids = ["17.458740234362434 -2.677413176352464", "-80.83261851536723 -2.6920536197633442", "117.78896429869648 -3.1783208418475954", "30.64725652750233 45.572165430308736", "21.18749859869599 31.264366696701767"]
-
-def saveRegions():
- count = 0
- for region_entry in data:
- region = Region()
- region.id = region_entry["id"]
- region.name = region_entry["region_name"]
- coordinates = region_entry["bbox"]["coordinates"][0]
- for coordinate in coordinates:
- region.polygon += str(coordinates[0]) + "," + str(coordinates[1]) + " "
- region.centroid = region_centroids[count]
- count += 1
- region.save()
+from django_celery_beat.models import IntervalSchedule, PeriodicTask
+import cap_feed.alert_processing as ap
def index(request):
- getAlerts()
+ ap.inject_unknown_regions()
latest_alert_list = Alert.objects.order_by("-sent")[:10]
template = loader.get_template("cap_feed/index.html")
context = {
@@ -37,82 +17,34 @@ def index(request):
}
return HttpResponse(template.render(context, request))
-
-# sources = [
-# ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}),
-# ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-croatia", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}),
-# ]
-
-sources = [
- ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}),
- ("https://alert.metservice.gov.jm/capfeed.php", "capfeedphp", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}),
- ]
-
-
-# ignore non-polygon sources for now
-def getAlerts():
- for source in sources:
- url, format, ns = source
- match format:
- case "meteoalarm":
- #get_alert_meteoalarm(url, ns)
- pass
- case "capfeedphp":
- get_alert_capfeedphp(url, ns)
-
-# Example: https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france
-
-def get_alert_meteoalarm(url, ns):
- response = requests.get(url)
- root = ET.fromstring(response.content)
- for entry in root.findall('atom:entry', ns):
- alert = Alert()
- alert.id = entry.find('atom:id', ns).text
- alert.identifier = entry.find('cap:identifier', ns).text
- alert.sender = url
- alert.sent = entry.find('cap:sent', ns).text
- alert.status = entry.find('cap:status', ns).text
- alert.msg_type = entry.find('cap:message_type', ns).text
- alert.scope = entry.find('cap:scope', ns).text
- alert.urgency = entry.find('cap:urgency', ns).text
- alert.severity = entry.find('cap:severity', ns).text
- alert.certainty = entry.find('cap:certainty', ns).text
- alert.expires = entry.find('cap:expires', ns).text
-
- alert.area_desc = entry.find('cap:areaDesc', ns).text
- alert.event = entry.find('cap:event', ns).text
-
- geocode = entry.find('cap:geocode', ns)
- alert.geocode_name = geocode.find('atom:valueName', ns).text
- alert.geocode_value = geocode.find('atom:value', ns).text
- alert.save()
-
-# Example: https://alert.metservice.gov.jm/capfeed.php
-def get_alert_capfeedphp(url, ns):
- response = requests.get(url)
- root = ET.fromstring(response.content)
- for entry in root.findall('atom:entry', ns):
- alert = Alert()
- alert.id = entry.find('atom:id', ns).text
- alert.expires = entry.find('cap:expires', ns).text
-
- entry_content = entry.find('atom:content', ns)
- entry_content_alert = entry_content.find('cap:alert', ns)
- alert.identifier = entry_content_alert.find('cap:identifier', ns).text
- alert.sender = entry_content_alert.find('cap:sender', ns).text
- alert.sent = entry_content_alert.find('cap:sent', ns).text
- alert.status = entry_content_alert.find('cap:status', ns).text
- alert.msg_type = entry_content_alert.find('cap:msgType', ns).text
- alert.scope = entry_content_alert.find('cap:scope', ns).text
-
- entry_content_alert_info = entry_content_alert.find('cap:info', ns)
- alert.urgency = entry_content_alert_info.find('cap:urgency', ns).text
- alert.severity = entry_content_alert_info.find('cap:severity', ns).text
- alert.certainty = entry_content_alert_info.find('cap:certainty', ns).text
- alert.expires = entry_content_alert_info.find('cap:expires', ns).text
- alert.event = entry_content_alert_info.find('cap:event', ns).text
-
- entry_content_alert_info_area = entry_content_alert_info.find('cap:area', ns)
- alert.area_desc = entry_content_alert_info_area.find('cap:areaDesc', ns).text
- alert.polygon = entry_content_alert_info_area.find('cap:polygon', ns).text
- alert.save()
\ No newline at end of file
+def polling_alerts(request):
+ schedule, created = IntervalSchedule.objects.get_or_create(
+ every=60,
+ period=IntervalSchedule.SECONDS,
+ )
+ PeriodicTask.objects.create(
+ interval=schedule, # we created this above.
+ name='Polls CAP alerts periodically', # simply describes this periodic task.
+ task='cap_feed.tasks.get_alerts', # name of task.
+ args=json.dumps(['arg1', 'arg2']),
+ kwargs=json.dumps({
+ 'be_careful': True,
+ }),
+ )
+ return HttpResponse("DONE")
+
+def removing_alerts(request):
+ schedule, created = IntervalSchedule.objects.get_or_create(
+ every=60,
+ period=IntervalSchedule.SECONDS,
+ )
+ PeriodicTask.objects.create(
+ interval=schedule, # we created this above.
+ name='Removes expired CAP alerts periodically', # simply describes this periodic task.
+ task='cap_feed.tasks.remove_expired_alerts', # name of task.
+ args=json.dumps(['arg1', 'arg2']),
+ kwargs=json.dumps({
+ 'be_careful': True,
+ }),
+ )
+ return HttpResponse("DONE")
diff --git a/capaggregator/celery.py b/capaggregator/celery.py
new file mode 100644
index 0000000..743a43c
--- /dev/null
+++ b/capaggregator/celery.py
@@ -0,0 +1,39 @@
+import os
+from celery import Celery
+from celery.schedules import crontab
+from datetime import timedelta
+from django.conf import settings
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+if 'WEBSITE_HOSTNAME' not in os.environ:
+ load_dotenv(".env")
+ # Set the default Django settings module for the 'celery' program.
+ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'capaggregator.settings')
+else:
+ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'capaggregator.production')
+app = Celery('capaggregator')
+
+app.conf.beat_schedule = {
+ 'poll-cap_alerts-periodically':{
+ 'task': 'cap_feed.tasks.get_alerts',
+ 'schedule': timedelta(minutes=1)
+ },
+ 'remove-expired_cap_alerts-periodically':{
+ 'task': 'cap_feed.tasks.remove_expired_alerts',
+ 'schedule': timedelta(minutes=1)
+ }
+}
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+# should have a `CELERY_` prefix.
+app.config_from_object(settings, namespace='CELERY')
+
+# Load task modules from all registered Django apps.
+app.autodiscover_tasks()
+
+
+@app.task(bind=True, ignore_result=True)
+def debug_task(self):
+ print(f'Request: {self.request!r}')
diff --git a/capaggregator/production.py b/capaggregator/production.py
index 2ff21c3..d442032 100644
--- a/capaggregator/production.py
+++ b/capaggregator/production.py
@@ -37,4 +37,10 @@
'USER': conn_str_params['user'],
'PASSWORD': conn_str_params['password'],
}
-}
\ No newline at end of file
+}
+
+CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL")
+CELERY_ACCEPT_CONTENT = ['application/json']
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_BACKEND = 'django-db'
+CELERY_CACHE_BACKEND = 'django-cache'
\ No newline at end of file
diff --git a/capaggregator/schema.py b/capaggregator/schema.py
index cbc27ce..aec63a5 100644
--- a/capaggregator/schema.py
+++ b/capaggregator/schema.py
@@ -1,12 +1,13 @@
import graphene
from graphene_django import DjangoObjectType #used to change Django object into a format that is readable by GraphQL
+from django.utils import timezone
from cap_feed.models import Alert, Region, Country
class AlertType(DjangoObjectType):
# Describe the data that is to be formatted into GraphQL fields
class Meta:
model = Alert
- field = ("id", "identifier", "sender", "sent", "status", "msg_type", "scope", "urgency", "severity", "certainty", "expires", "area_desc", "event", "geocode_name", "geocode_value", "polygon")
+ field = ("id", "identifier", "sender", "sent", "status", "msg_type", "scope", "urgency", "severity", "certainty", "effective", "expires", "area_desc", "event", "geocode_name", "geocode_value", "polygon")
class RegionType(DjangoObjectType):
# Describe the data that is to be formatted into GraphQL fields
@@ -22,11 +23,20 @@ class Meta:
class Query(graphene.ObjectType):
- #query ContactType to get list of contacts
list_alert=graphene.List(AlertType)
+ list_country=graphene.List(CountryType)
+ list_region=graphene.List(RegionType)
def resolve_list_alert(root, info):
# We can easily optimize query count in the resolve method
- return Alert.objects.order_by("-sent")[:20]
+ return Alert.objects.order_by("-id")
+
+ def resolve_list_country(root, info):
+ # We can easily optimize query count in the resolve method
+ return Country.objects.order_by("-id")
+
+ def resolve_list_region(root, info):
+ # We can easily optimize query count in the resolve method
+ return Region.objects.order_by("-id")
schema = graphene.Schema(query=Query)
\ No newline at end of file
diff --git a/capaggregator/settings.py b/capaggregator/settings.py
index 88e68dc..9d6a083 100644
--- a/capaggregator/settings.py
+++ b/capaggregator/settings.py
@@ -42,6 +42,8 @@
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
+ 'django_celery_results',
+ 'django_celery_beat',
]
MIDDLEWARE = [
@@ -143,3 +145,9 @@
# https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL")
+CELERY_ACCEPT_CONTENT = ['application/json']
+CELERY_TASK_SERIALIZER = 'json'
+
+CELERY_RESULT_BACKEND = 'django-db'
+CELERY_CACHE_BACKEND = 'django-cache'
\ No newline at end of file
diff --git a/celerybeat-schedule.bak b/celerybeat-schedule.bak
new file mode 100644
index 0000000..d24af5c
--- /dev/null
+++ b/celerybeat-schedule.bak
@@ -0,0 +1,4 @@
+'entries', (2048, 622)
+'__version__', (512, 20)
+'tz', (1024, 18)
+'utc_enabled', (1536, 4)
diff --git a/celerybeat-schedule.dat b/celerybeat-schedule.dat
new file mode 100644
index 0000000..4c0d004
Binary files /dev/null and b/celerybeat-schedule.dat differ
diff --git a/celerybeat-schedule.dir b/celerybeat-schedule.dir
new file mode 100644
index 0000000..d24af5c
--- /dev/null
+++ b/celerybeat-schedule.dir
@@ -0,0 +1,4 @@
+'entries', (2048, 622)
+'__version__', (512, 20)
+'tz', (1024, 18)
+'utc_enabled', (1536, 4)
diff --git a/requirements.txt b/requirements.txt
index 6414953..84e0f9a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,20 +1,39 @@
+amqp==5.1.1
aniso8601==9.0.1
asgiref==3.7.2
+billiard==4.1.0
+celery==5.3.1
certifi==2023.5.7
charset-normalizer==3.1.0
+click==8.1.3
+click-didyoumean==0.3.0
+click-plugins==1.1.1
+click-repl==0.3.0
+colorama==0.4.6
+cron-descriptor==1.4.0
Django==4.2.2
+django-celery-beat==2.5.0
+django-celery-results==2.5.1
+django-timezone-field==5.1
graphene==3.2.2
graphene-django==3.1.2
graphql-core==3.2.3
graphql-relay==3.2.0
idna==3.4
+kombu==5.3.1
promise==2.3
+prompt-toolkit==3.0.38
psycopg2-binary==2.9.6
+python-crontab==2.7.1
+python-dateutil==2.8.2
python-dotenv==1.0.0
+pytz==2023.3
requests==2.31.0
six==1.16.0
sqlparse==0.4.4
text-unidecode==1.3
tzdata==2023.3
urllib3==2.0.3
-whitenoise==6.4.0
+vine==5.0.0
+wcwidth==0.2.6
+whitenoise==6.5.0
diff --git a/startup.sh b/startup.sh
index 78a2559..b3340f5 100644
--- a/startup.sh
+++ b/startup.sh
@@ -1,4 +1,4 @@
python manage.py migrate
gunicorn --workers 2 --threads 4 --timeout 60 --access-logfile \
'-' --error-logfile '-' --bind=0.0.0.0:8000 \
- --chdir=/home/site/wwwroot capaggregator.wsgi
\ No newline at end of file
+ --chdir=/home/site/wwwroot capaggregator.wsgi & celery -A capaggregator worker -l info --pool=solo & celery -A capaggregator beat -l info
\ No newline at end of file