From 24163ff9ae3b7b68a71edc0851930d3fabf4c51f Mon Sep 17 00:00:00 2001 From: Alfie Liu <109685890+Alfiejz@users.noreply.github.com> Date: Sat, 24 Jun 2023 18:29:07 +0100 Subject: [PATCH] Celery alert processing automation * Update main_cap-aggregator(dev).yml * added separate configurations for each cap source format * added region data * reset migrations * removed region usage before declaration * fixed variation scope issue * created migration * injecting regions * injecting countries (#7) * matched CAP alerts to country polygons * Integrated Celery and RabbitMQ * Preparation for region data injection (#6) * Update main_cap-aggregator(dev).yml * added separate configurations for each cap source format * added region data * reset migrations * removed region usage before declaration * fixed variation scope issue * Add or update the Azure App Service build and deployment workflow config * add celery tasks --------- Co-authored-by: HangZhou01 * added celery startup commands * changed startup.sh * added filter for unexpired alerts, fixed cors policy * enabled github workflow tests for all branches, whitenoise package version bump * fixed cors policy origin path * disabled celery for deployment * reverted cors policy --------- Co-authored-by: HangZhou01 --- ...elery-test_cap-aggregator(celery-test).yml | 63 +++++++ .github/workflows/python-test.yaml | 6 +- .gitignore | 5 +- cap_feed/alert_processing.py | 163 ++++++++++++++++++ cap_feed/country.json | 38 ++-- cap_feed/migrations/0001_initial.py | 53 ++++++ cap_feed/migrations/0002_country_region.py | 19 ++ cap_feed/migrations/0003_alert_country.py | 19 ++ .../migrations/0004_alter_alert_country.py | 19 ++ .../migrations/0005_alter_country_region.py | 19 ++ cap_feed/migrations/0006_alert_effective.py | 20 +++ cap_feed/models.py | 43 ++--- cap_feed/region.json | 5 + cap_feed/tasks.py | 14 ++ cap_feed/templates/cap_feed/index.html | 5 +- cap_feed/tests.py | 23 ++- cap_feed/urls.py | 1 + cap_feed/views.py | 138 ++++----------- capaggregator/celery.py | 39 +++++ capaggregator/production.py | 8 +- capaggregator/schema.py | 16 +- capaggregator/settings.py | 8 + celerybeat-schedule.bak | 4 + celerybeat-schedule.dat | Bin 0 -> 2670 bytes celerybeat-schedule.dir | 4 + requirements.txt | 21 ++- startup.sh | 2 +- 27 files changed, 601 insertions(+), 154 deletions(-) create mode 100644 .github/workflows/celery-test_cap-aggregator(celery-test).yml create mode 100644 cap_feed/alert_processing.py create mode 100644 cap_feed/migrations/0001_initial.py create mode 100644 cap_feed/migrations/0002_country_region.py create mode 100644 cap_feed/migrations/0003_alert_country.py create mode 100644 cap_feed/migrations/0004_alter_alert_country.py create mode 100644 cap_feed/migrations/0005_alter_country_region.py create mode 100644 cap_feed/migrations/0006_alert_effective.py create mode 100644 cap_feed/tasks.py create mode 100644 capaggregator/celery.py create mode 100644 celerybeat-schedule.bak create mode 100644 celerybeat-schedule.dat create mode 100644 celerybeat-schedule.dir diff --git a/.github/workflows/celery-test_cap-aggregator(celery-test).yml b/.github/workflows/celery-test_cap-aggregator(celery-test).yml new file mode 100644 index 0000000..ed21ca3 --- /dev/null +++ b/.github/workflows/celery-test_cap-aggregator(celery-test).yml @@ -0,0 +1,63 @@ +# Docs for the Azure Web Apps Deploy action: https://github.com/Azure/webapps-deploy +# More GitHub Actions for Azure: https://github.com/Azure/actions +# More info on Python, GitHub Actions, and Azure App Service: https://aka.ms/python-webapps-actions + +name: Build and deploy Python app to Azure Web App - cap-aggregator + +on: + push: + branches: + - celery-test + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python version + uses: actions/setup-python@v1 + with: + python-version: '3.11' + + - name: Create and start virtual environment + run: | + python -m venv venv + source venv/bin/activate + + - name: Install dependencies + run: pip install -r requirements.txt + + # Optional: Add step to run tests here (PyTest, Django test suites, etc.) + + - name: Upload artifact for deployment jobs + uses: actions/upload-artifact@v2 + with: + name: python-app + path: | + . + !venv/ + + deploy: + runs-on: ubuntu-latest + needs: build + environment: + name: 'celery-test' + url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + steps: + - name: Download artifact from build job + uses: actions/download-artifact@v2 + with: + name: python-app + path: . + + - name: 'Deploy to Azure Web App' + uses: azure/webapps-deploy@v2 + id: deploy-to-webapp + with: + app-name: 'cap-aggregator' + slot-name: 'celery-test' + publish-profile: ${{ secrets.AZUREAPPSERVICE_PUBLISHPROFILE_0FD4FFF13EAD42BCA6117317784F30E7 }} diff --git a/.github/workflows/python-test.yaml b/.github/workflows/python-test.yaml index 747fbb0..b92bf06 100644 --- a/.github/workflows/python-test.yaml +++ b/.github/workflows/python-test.yaml @@ -2,9 +2,11 @@ name: Python check on: push: - branches: [ main ] + branches: + - '*' pull_request: - branches: [ main ] + branches: + - '*' jobs: test_package: diff --git a/.gitignore b/.gitignore index 9b957b1..f536698 100644 --- a/.gitignore +++ b/.gitignore @@ -128,4 +128,7 @@ dmypy.json # Pyre type checker .pyre/ -.azure \ No newline at end of file +.azure + +# editors +.idea/ \ No newline at end of file diff --git a/cap_feed/alert_processing.py b/cap_feed/alert_processing.py new file mode 100644 index 0000000..d56190c --- /dev/null +++ b/cap_feed/alert_processing.py @@ -0,0 +1,163 @@ +import json +import requests + +import xml.etree.ElementTree as ET +import pytz +from .models import Alert, Region, Country +from datetime import datetime +from django.utils import timezone + + + +# inject region and country data if not already present +def inject_unknown_regions(): + if Region.objects.count() == 0: + inject_regions() + # inject unknown region for alerts without a defined region + unknown_region = Region() + unknown_region.id = -1 + unknown_region.name = "Unknown" + unknown_region.save() + if Country.objects.count() == 0: + inject_countries() + # inject unknown country for alerts without a defined country + unknown_country = Country() + unknown_country.id = -1 + unknown_country.name = "Unknown" + unknown_country.save() + +# inject region data +def inject_regions(): + with open('cap_feed/region.json') as file: + region_data = json.load(file) + for region_entry in region_data: + region = Region() + region.id = region_entry["id"] + region.name = region_entry["region_name"] + region.centroid = region_entry["centroid"] + coordinates = region_entry["bbox"]["coordinates"][0] + for coordinate in coordinates: + region.polygon += str(coordinate[0]) + "," + str(coordinate[1]) + " " + region.save() + +# inject country data +def inject_countries(): + with open('cap_feed/country.json') as file: + country_data = json.load(file) + for country_entry in country_data: + country = Country() + country.id = country_entry["id"] + country.name = country_entry["name"] + region_id = country_entry["region"] + if ("Region" in country.name) or ("Cluster" in country.name): + continue + if region_id is not None: + country.region = Region.objects.get(id=country_entry["region"]) + if country_entry["iso"] is not None: + country.iso = country_entry["iso"] + if country_entry["iso3"] is not None: + country.iso3 = country_entry["iso3"] + if country_entry["bbox"] is not None: + coordinates = country_entry["bbox"]["coordinates"][0] + for coordinate in coordinates: + country.polygon += str(coordinate[0]) + "," + str(coordinate[1]) + " " + if country_entry["centroid"] is not None: + coordinates = country_entry["centroid"]["coordinates"] + country.centroid = str(coordinates[0]) + "," + str(coordinates[1]) + country.save() + +# converts CAP1.2 iso format datetime string to datetime object in UTC timezone +def convert_datetime(original_datetime): + return datetime.fromisoformat(original_datetime).astimezone(pytz.timezone('UTC')) + +# gets alerts from sources and processes them different for each source format +def get_alerts(): + # list of sources and configurations + sources = [ + ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france", "FRA", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), + ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-belgium", "BEL", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), + ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-austria", "AUT", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), + ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-slovakia", "SVK", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), + ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-slovenia", "SVN", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), + ("https://alert.metservice.gov.jm/capfeed.php", "JAM", "capfeedphp", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), + ] + + for source in sources: + url, iso3, format, ns = source + match format: + case "meteoalarm": + get_alert_meteoalarm(url, iso3, ns) + case "capfeedphp": + get_alert_capfeedphp(url, iso3, ns) + +# processing for meteoalarm format, example: https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france +def get_alert_meteoalarm(url, iso3, ns): + response = requests.get(url) + root = ET.fromstring(response.content) + for entry in root.findall('atom:entry', ns): + try: + alert = Alert() + alert.id = entry.find('atom:id', ns).text + alert.identifier = entry.find('cap:identifier', ns).text + alert.sender = url + alert.sent = convert_datetime(entry.find('cap:sent', ns).text) + alert.status = entry.find('cap:status', ns).text + alert.msg_type = entry.find('cap:message_type', ns).text + alert.scope = entry.find('cap:scope', ns).text + alert.urgency = entry.find('cap:urgency', ns).text + alert.severity = entry.find('cap:severity', ns).text + alert.certainty = entry.find('cap:certainty', ns).text + alert.effective = convert_datetime(entry.find('cap:effective', ns).text) + alert.expires = convert_datetime(entry.find('cap:expires', ns).text) + if alert.expires < timezone.now(): + continue + + alert.area_desc = entry.find('cap:areaDesc', ns).text + alert.event = entry.find('cap:event', ns).text + + geocode = entry.find('cap:geocode', ns) + alert.geocode_name = geocode.find('atom:valueName', ns).text + alert.geocode_value = geocode.find('atom:value', ns).text + alert.country = Country.objects.get(iso3=iso3) + alert.save() + except: + pass + +# processing for capfeedphp format, example: https://alert.metservice.gov.jm/capfeed.php +def get_alert_capfeedphp(url, iso3, ns): + response = requests.get(url) + root = ET.fromstring(response.content) + for entry in root.findall('atom:entry', ns): + try: + alert = Alert() + alert.id = entry.find('atom:id', ns).text + + entry_content = entry.find('atom:content', ns) + entry_content_alert = entry_content.find('cap:alert', ns) + alert.identifier = entry_content_alert.find('cap:identifier', ns).text + alert.sender = entry_content_alert.find('cap:sender', ns).text + alert.sent = convert_datetime(entry_content_alert.find('cap:sent', ns).text) + alert.status = entry_content_alert.find('cap:status', ns).text + alert.msg_type = entry_content_alert.find('cap:msgType', ns).text + alert.scope = entry_content_alert.find('cap:scope', ns).text + + entry_content_alert_info = entry_content_alert.find('cap:info', ns) + alert.urgency = entry_content_alert_info.find('cap:urgency', ns).text + alert.severity = entry_content_alert_info.find('cap:severity', ns).text + alert.certainty = entry_content_alert_info.find('cap:certainty', ns).text + alert.effective = convert_datetime(entry_content_alert_info.find('cap:effective', ns).text) + alert.expires = convert_datetime(entry_content_alert_info.find('cap:expires', ns).text) + if alert.expires < timezone.now(): + continue + alert.event = entry_content_alert_info.find('cap:event', ns).text + + entry_content_alert_info_area = entry_content_alert_info.find('cap:area', ns) + alert.area_desc = entry_content_alert_info_area.find('cap:areaDesc', ns).text + alert.polygon = entry_content_alert_info_area.find('cap:polygon', ns).text + alert.country = Country.objects.get(iso3=iso3) + alert.save() + except: + pass + +def remove_expired_alerts(): + Alert.objects.filter(expires__lt=timezone.now()).delete() \ No newline at end of file diff --git a/cap_feed/country.json b/cap_feed/country.json index 92432b3..6fcd59d 100644 --- a/cap_feed/country.json +++ b/cap_feed/country.json @@ -68,7 +68,7 @@ "iso": null, "iso3": null, "society_url": "", - "region": null, + "region": 2, "key_priorities": null, "inform_score": null, "id": 343, @@ -282,7 +282,7 @@ "iso": "AI", "iso3": "AIA", "society_url": "", - "region": null, + "region": 1, "key_priorities": null, "inform_score": null, "id": 217, @@ -426,7 +426,7 @@ "iso": "AW", "iso3": "ABW", "society_url": "", - "region": null, + "region": 1, "key_priorities": null, "inform_score": null, "id": 231, @@ -927,7 +927,7 @@ "iso": "BM", "iso3": "BMU", "society_url": "", - "region": null, + "region": 1, "key_priorities": null, "inform_score": null, "id": 233, @@ -992,7 +992,7 @@ "iso": null, "iso3": null, "society_url": "", - "region": null, + "region": 0, "key_priorities": null, "inform_score": null, "id": 346, @@ -3054,7 +3054,7 @@ "iso": "GI", "iso3": "GIB", "society_url": "", - "region": null, + "region": 3, "key_priorities": null, "inform_score": null, "id": 241, @@ -3421,7 +3421,7 @@ "iso": null, "iso3": null, "society_url": "", - "region": null, + "region": 0, "key_priorities": null, "inform_score": null, "id": 351, @@ -3472,7 +3472,7 @@ "iso": "VA", "iso3": "VAT", "society_url": "", - "region": null, + "region": 3, "key_priorities": null, "inform_score": null, "id": 274, @@ -3664,7 +3664,7 @@ "iso": null, "iso3": null, "society_url": "", - "region": null, + "region": 0, "key_priorities": null, "inform_score": null, "id": 352, @@ -4616,7 +4616,7 @@ "iso": "MO", "iso3": "MAC", "society_url": "http://www.redcross.org.mo/", - "region": null, + "region": 2, "key_priorities": null, "inform_score": null, "id": 279, @@ -5311,7 +5311,7 @@ "iso": "NR", "iso3": "NRU", "society_url": "", - "region": null, + "region": 2, "key_priorities": null, "inform_score": null, "id": 251, @@ -5571,7 +5571,7 @@ "iso": "NU", "iso3": "NIU", "society_url": "", - "region": null, + "region": 2, "key_priorities": null, "inform_score": null, "id": 253, @@ -5752,7 +5752,7 @@ "iso": "OM", "iso3": "OMN", "society_url": "", - "region": null, + "region": 2, "key_priorities": null, "inform_score": null, "id": 258, @@ -6060,7 +6060,7 @@ "iso": "PN", "iso3": "PCN", "society_url": "", - "region": null, + "region": 2, "key_priorities": null, "inform_score": null, "id": 256, @@ -6156,7 +6156,7 @@ "iso": "PR", "iso3": "PRI", "society_url": "", - "region": null, + "region": 1, "key_priorities": null, "inform_score": null, "id": 257, @@ -6365,7 +6365,7 @@ "iso": "SH", "iso3": "SHN", "society_url": "", - "region": null, + "region": 0, "key_priorities": null, "inform_score": null, "id": 262, @@ -6616,7 +6616,7 @@ "iso": null, "iso3": null, "society_url": "", - "region": null, + "region": 2, "key_priorities": null, "inform_score": null, "id": 358, @@ -7148,7 +7148,7 @@ "iso": null, "iso3": null, "society_url": "", - "region": null, + "region": 2, "key_priorities": null, "inform_score": null, "id": 360, @@ -8230,7 +8230,7 @@ "iso": "EH", "iso3": "ESH", "society_url": "", - "region": null, + "region": 0, "key_priorities": null, "inform_score": null, "id": 269, diff --git a/cap_feed/migrations/0001_initial.py b/cap_feed/migrations/0001_initial.py new file mode 100644 index 0000000..e041297 --- /dev/null +++ b/cap_feed/migrations/0001_initial.py @@ -0,0 +1,53 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Alert', + fields=[ + ('id', models.CharField(max_length=255, primary_key=True, serialize=False)), + ('identifier', models.CharField(max_length=255)), + ('sender', models.CharField(max_length=255)), + ('sent', models.DateTimeField()), + ('status', models.CharField(max_length=255)), + ('msg_type', models.CharField(max_length=255)), + ('scope', models.CharField(max_length=255)), + ('urgency', models.CharField(max_length=255)), + ('severity', models.CharField(max_length=255)), + ('certainty', models.CharField(max_length=255)), + ('expires', models.DateTimeField()), + ('area_desc', models.CharField(max_length=255)), + ('event', models.CharField(max_length=255)), + ('geocode_name', models.CharField(blank=True, default='', max_length=255)), + ('geocode_value', models.CharField(blank=True, default='', max_length=255)), + ('polygon', models.TextField(blank=True, default='', max_length=16383)), + ], + ), + migrations.CreateModel( + name='Country', + fields=[ + ('id', models.CharField(max_length=255, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=255)), + ('iso', models.CharField(blank=True, default='', max_length=255)), + ('iso3', models.CharField(blank=True, default='', max_length=255)), + ('polygon', models.TextField(blank=True, default='', max_length=16383)), + ('centroid', models.CharField(blank=True, default='', max_length=255)), + ], + ), + migrations.CreateModel( + name='Region', + fields=[ + ('id', models.CharField(max_length=255, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=255)), + ('polygon', models.TextField(blank=True, default='', max_length=16383)), + ('centroid', models.CharField(blank=True, default='', max_length=255)), + ], + ), + ] diff --git a/cap_feed/migrations/0002_country_region.py b/cap_feed/migrations/0002_country_region.py new file mode 100644 index 0000000..b3f2374 --- /dev/null +++ b/cap_feed/migrations/0002_country_region.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.2 on 2023-06-23 11:38 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('cap_feed', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='country', + name='region', + field=models.ForeignKey(blank=True, default='', on_delete=django.db.models.deletion.SET_DEFAULT, to='cap_feed.region'), + ), + ] diff --git a/cap_feed/migrations/0003_alert_country.py b/cap_feed/migrations/0003_alert_country.py new file mode 100644 index 0000000..06670bc --- /dev/null +++ b/cap_feed/migrations/0003_alert_country.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.2 on 2023-06-23 13:55 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('cap_feed', '0002_country_region'), + ] + + operations = [ + migrations.AddField( + model_name='alert', + name='country', + field=models.ForeignKey(blank=True, default='', on_delete=django.db.models.deletion.SET_DEFAULT, to='cap_feed.country'), + ), + ] diff --git a/cap_feed/migrations/0004_alter_alert_country.py b/cap_feed/migrations/0004_alter_alert_country.py new file mode 100644 index 0000000..4b75327 --- /dev/null +++ b/cap_feed/migrations/0004_alter_alert_country.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.2 on 2023-06-23 13:56 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('cap_feed', '0003_alert_country'), + ] + + operations = [ + migrations.AlterField( + model_name='alert', + name='country', + field=models.ForeignKey(default='-1', on_delete=django.db.models.deletion.SET_DEFAULT, to='cap_feed.country'), + ), + ] diff --git a/cap_feed/migrations/0005_alter_country_region.py b/cap_feed/migrations/0005_alter_country_region.py new file mode 100644 index 0000000..a74c0a1 --- /dev/null +++ b/cap_feed/migrations/0005_alter_country_region.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.2 on 2023-06-23 14:32 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('cap_feed', '0004_alter_alert_country'), + ] + + operations = [ + migrations.AlterField( + model_name='country', + name='region', + field=models.ForeignKey(default='-1', on_delete=django.db.models.deletion.SET_DEFAULT, to='cap_feed.region'), + ), + ] diff --git a/cap_feed/migrations/0006_alert_effective.py b/cap_feed/migrations/0006_alert_effective.py new file mode 100644 index 0000000..439a121 --- /dev/null +++ b/cap_feed/migrations/0006_alert_effective.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.2 on 2023-06-24 13:16 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('cap_feed', '0005_alter_country_region'), + ] + + operations = [ + migrations.AddField( + model_name='alert', + name='effective', + field=models.DateTimeField(default=django.utils.timezone.now), + preserve_default=False, + ), + ] diff --git a/cap_feed/models.py b/cap_feed/models.py index d6bb221..eaff961 100644 --- a/cap_feed/models.py +++ b/cap_feed/models.py @@ -2,6 +2,27 @@ from django.db import models # Create your models here. + +class Region(models.Model): + id = models.CharField(max_length=255, primary_key=True) + name = models.CharField(max_length=255) + polygon = models.TextField(max_length=16383, blank=True, default='') + centroid = models.CharField(max_length=255, blank=True, default='') + + def __str__(self): + return self.name + +class Country(models.Model): + id = models.CharField(max_length=255, primary_key=True) + name = models.CharField(max_length=255) + iso = models.CharField(max_length=255, blank=True, default='') + iso3 = models.CharField(max_length=255, blank=True, default='') + polygon = models.TextField(max_length=16383, blank=True, default='') + centroid = models.CharField(max_length=255, blank=True, default='') + region = models.ForeignKey(Region, on_delete=models.SET_DEFAULT, default='-1') + + def __str__(self): + return self.name class Alert(models.Model): id = models.CharField(max_length=255, primary_key=True) @@ -14,6 +35,7 @@ class Alert(models.Model): urgency = models.CharField(max_length=255) severity = models.CharField(max_length=255) certainty = models.CharField(max_length=255) + effective = models.DateTimeField() expires = models.DateTimeField() area_desc = models.CharField(max_length=255) @@ -21,26 +43,7 @@ class Alert(models.Model): geocode_name = models.CharField(max_length=255, blank=True, default='') geocode_value = models.CharField(max_length=255, blank=True, default='') polygon = models.TextField(max_length=16383, blank=True, default='') - - def __str__(self): - return self.id - -class Region(models.Model): - id = models.CharField(max_length=255, primary_key=True) - name = models.CharField(max_length=255) - polygon = models.TextField(max_length=16383, blank=True, default='') - centroid = models.CharField(max_length=255, blank=True, default='') - - def __str__(self): - return self.id - -class Country(models.Model): - id = models.CharField(max_length=255, primary_key=True) - name = models.CharField(max_length=255) - iso = models.CharField(max_length=255, blank = True, default='') - iso3 = models.CharField(max_length=255, blank = True, default='') - polygon = models.TextField(max_length=16383, blank=True, default='') - centroid = models.CharField(max_length=255, blank=True, default='') + country = models.ForeignKey(Country, on_delete=models.SET_DEFAULT, default='-1') def __str__(self): return self.id \ No newline at end of file diff --git a/cap_feed/region.json b/cap_feed/region.json index bf54102..5c6349d 100644 --- a/cap_feed/region.json +++ b/cap_feed/region.json @@ -3,6 +3,7 @@ "name": 0, "id": 0, "region_name": "Africa", + "centroid": "17.458740234362434 -2.677413176352464", "bbox": { "type": "Polygon", "coordinates": [ @@ -21,6 +22,7 @@ "name": 1, "id": 1, "region_name": "Americas", + "centroid": "-80.83261851536723 -2.6920536197633442", "bbox": { "type": "Polygon", "coordinates": [ @@ -39,6 +41,7 @@ "name": 2, "id": 2, "region_name": "Asia Pacific", + "centroid": "117.78896429869648 -3.1783208418475954", "bbox": { "type": "Polygon", "coordinates": [ @@ -57,6 +60,7 @@ "name": 3, "id": 3, "region_name": "Europe", + "centroid": "30.64725652750233 45.572165430308736", "bbox": { "type": "Polygon", "coordinates": [ @@ -75,6 +79,7 @@ "name": 4, "id": 4, "region_name": "Middle East & North Africa", + "centroid": "21.18749859869599 31.264366696701767", "bbox": { "type": "Polygon", "coordinates": [ diff --git a/cap_feed/tasks.py b/cap_feed/tasks.py new file mode 100644 index 0000000..3575939 --- /dev/null +++ b/cap_feed/tasks.py @@ -0,0 +1,14 @@ +from __future__ import absolute_import, unicode_literals +from celery import shared_task + +import cap_feed.alert_processing as ap + +@shared_task(bind=True) +def get_alerts(self): + ap.get_alerts() + return "get_alerts DONE" + +@shared_task(bind=True) +def remove_expired_alerts(self): + ap.remove_expired_alerts() + return "remove_expired_alerts DONE" \ No newline at end of file diff --git a/cap_feed/templates/cap_feed/index.html b/cap_feed/templates/cap_feed/index.html index cf802df..2548f1f 100644 --- a/cap_feed/templates/cap_feed/index.html +++ b/cap_feed/templates/cap_feed/index.html @@ -9,9 +9,10 @@

{{ alert.event }}

Severity: {{ alert.severity }}

Urgency: {{ alert.urgency }}

Sent: {{ alert.sent }}

+

Effective: {{ alert.effective }}

+

Expires: {{ alert.expires }}

Sender: {{ alert.sender }}

-

Geocode Name: {{ alert.geocode_name }}

-

Geocode Value: {{ alert.geocode_value }}

+

Country: {{ alert.country }}



{% endfor %} diff --git a/cap_feed/tests.py b/cap_feed/tests.py index f4705fb..e3d5f3e 100644 --- a/cap_feed/tests.py +++ b/cap_feed/tests.py @@ -1,7 +1,28 @@ -import datetime +import pytz +from datetime import datetime +import cap_feed.alert_processing as ap from django.test import TestCase from django.urls import reverse +from django.utils import timezone from .models import Alert +class AlertModelTests(TestCase): + def test_alert_source_datetime_converted_to_utc(self): + """ + Was the iso format cap alert datetime field with timezone offsets processed correctly to utc timezone? + """ + cap_sent = "2023-06-24T22:00:00-05:00" + cap_effective = "2023-06-24T22:00:00+00:00" + cap_expires = "2023-06-24T22:00:00+05:00" + alert = Alert() + alert.sent = ap.convert_datetime(cap_sent) + alert.effective = ap.convert_datetime(cap_effective) + alert.expires = ap.convert_datetime(cap_expires) + utc_sent = datetime(2023, 6, 25, 3, 0, 0, 0, pytz.UTC) + utc_effective = datetime(2023, 6, 24, 22, 0, 0, 0, pytz.UTC) + utc_expires = datetime(2023, 6, 24, 17, 0, 0, 0, pytz.UTC) + assert alert.sent == utc_sent + assert alert.effective == utc_effective + assert alert.expires == utc_expires \ No newline at end of file diff --git a/cap_feed/urls.py b/cap_feed/urls.py index 88a9cac..cefc089 100644 --- a/cap_feed/urls.py +++ b/cap_feed/urls.py @@ -4,4 +4,5 @@ urlpatterns = [ path('', views.index, name='index'), + path('pollingalerts/', views.polling_alerts, name = 'polling_alerts') ] diff --git a/cap_feed/views.py b/cap_feed/views.py index d768e31..0706a96 100644 --- a/cap_feed/views.py +++ b/cap_feed/views.py @@ -1,35 +1,15 @@ -import requests -import xml.etree.ElementTree as ET +import json -from django.shortcuts import get_object_or_404, render from django.http import HttpResponse from django.template import loader from .models import Alert, Region, Country - - -import json -with open('cap_feed/region.json') as file: - data = json.load(file) - -region_centroids = ["17.458740234362434 -2.677413176352464", "-80.83261851536723 -2.6920536197633442", "117.78896429869648 -3.1783208418475954", "30.64725652750233 45.572165430308736", "21.18749859869599 31.264366696701767"] - -def saveRegions(): - count = 0 - for region_entry in data: - region = Region() - region.id = region_entry["id"] - region.name = region_entry["region_name"] - coordinates = region_entry["bbox"]["coordinates"][0] - for coordinate in coordinates: - region.polygon += str(coordinates[0]) + "," + str(coordinates[1]) + " " - region.centroid = region_centroids[count] - count += 1 - region.save() +from django_celery_beat.models import IntervalSchedule, PeriodicTask +import cap_feed.alert_processing as ap def index(request): - getAlerts() + ap.inject_unknown_regions() latest_alert_list = Alert.objects.order_by("-sent")[:10] template = loader.get_template("cap_feed/index.html") context = { @@ -37,82 +17,34 @@ def index(request): } return HttpResponse(template.render(context, request)) - -# sources = [ -# ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), -# ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-croatia", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), -# ] - -sources = [ - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://alert.metservice.gov.jm/capfeed.php", "capfeedphp", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ] - - -# ignore non-polygon sources for now -def getAlerts(): - for source in sources: - url, format, ns = source - match format: - case "meteoalarm": - #get_alert_meteoalarm(url, ns) - pass - case "capfeedphp": - get_alert_capfeedphp(url, ns) - -# Example: https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france - -def get_alert_meteoalarm(url, ns): - response = requests.get(url) - root = ET.fromstring(response.content) - for entry in root.findall('atom:entry', ns): - alert = Alert() - alert.id = entry.find('atom:id', ns).text - alert.identifier = entry.find('cap:identifier', ns).text - alert.sender = url - alert.sent = entry.find('cap:sent', ns).text - alert.status = entry.find('cap:status', ns).text - alert.msg_type = entry.find('cap:message_type', ns).text - alert.scope = entry.find('cap:scope', ns).text - alert.urgency = entry.find('cap:urgency', ns).text - alert.severity = entry.find('cap:severity', ns).text - alert.certainty = entry.find('cap:certainty', ns).text - alert.expires = entry.find('cap:expires', ns).text - - alert.area_desc = entry.find('cap:areaDesc', ns).text - alert.event = entry.find('cap:event', ns).text - - geocode = entry.find('cap:geocode', ns) - alert.geocode_name = geocode.find('atom:valueName', ns).text - alert.geocode_value = geocode.find('atom:value', ns).text - alert.save() - -# Example: https://alert.metservice.gov.jm/capfeed.php -def get_alert_capfeedphp(url, ns): - response = requests.get(url) - root = ET.fromstring(response.content) - for entry in root.findall('atom:entry', ns): - alert = Alert() - alert.id = entry.find('atom:id', ns).text - alert.expires = entry.find('cap:expires', ns).text - - entry_content = entry.find('atom:content', ns) - entry_content_alert = entry_content.find('cap:alert', ns) - alert.identifier = entry_content_alert.find('cap:identifier', ns).text - alert.sender = entry_content_alert.find('cap:sender', ns).text - alert.sent = entry_content_alert.find('cap:sent', ns).text - alert.status = entry_content_alert.find('cap:status', ns).text - alert.msg_type = entry_content_alert.find('cap:msgType', ns).text - alert.scope = entry_content_alert.find('cap:scope', ns).text - - entry_content_alert_info = entry_content_alert.find('cap:info', ns) - alert.urgency = entry_content_alert_info.find('cap:urgency', ns).text - alert.severity = entry_content_alert_info.find('cap:severity', ns).text - alert.certainty = entry_content_alert_info.find('cap:certainty', ns).text - alert.expires = entry_content_alert_info.find('cap:expires', ns).text - alert.event = entry_content_alert_info.find('cap:event', ns).text - - entry_content_alert_info_area = entry_content_alert_info.find('cap:area', ns) - alert.area_desc = entry_content_alert_info_area.find('cap:areaDesc', ns).text - alert.polygon = entry_content_alert_info_area.find('cap:polygon', ns).text - alert.save() \ No newline at end of file +def polling_alerts(request): + schedule, created = IntervalSchedule.objects.get_or_create( + every=60, + period=IntervalSchedule.SECONDS, + ) + PeriodicTask.objects.create( + interval=schedule, # we created this above. + name='Polls CAP alerts periodically', # simply describes this periodic task. + task='cap_feed.tasks.get_alerts', # name of task. + args=json.dumps(['arg1', 'arg2']), + kwargs=json.dumps({ + 'be_careful': True, + }), + ) + return HttpResponse("DONE") + +def removing_alerts(request): + schedule, created = IntervalSchedule.objects.get_or_create( + every=60, + period=IntervalSchedule.SECONDS, + ) + PeriodicTask.objects.create( + interval=schedule, # we created this above. + name='Removes expired CAP alerts periodically', # simply describes this periodic task. + task='cap_feed.tasks.remove_expired_alerts', # name of task. + args=json.dumps(['arg1', 'arg2']), + kwargs=json.dumps({ + 'be_careful': True, + }), + ) + return HttpResponse("DONE") diff --git a/capaggregator/celery.py b/capaggregator/celery.py new file mode 100644 index 0000000..743a43c --- /dev/null +++ b/capaggregator/celery.py @@ -0,0 +1,39 @@ +import os +from celery import Celery +from celery.schedules import crontab +from datetime import timedelta +from django.conf import settings +from dotenv import load_dotenv + +# Load environment variables from .env file +if 'WEBSITE_HOSTNAME' not in os.environ: + load_dotenv(".env") + # Set the default Django settings module for the 'celery' program. + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'capaggregator.settings') +else: + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'capaggregator.production') +app = Celery('capaggregator') + +app.conf.beat_schedule = { + 'poll-cap_alerts-periodically':{ + 'task': 'cap_feed.tasks.get_alerts', + 'schedule': timedelta(minutes=1) + }, + 'remove-expired_cap_alerts-periodically':{ + 'task': 'cap_feed.tasks.remove_expired_alerts', + 'schedule': timedelta(minutes=1) + } +} +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object(settings, namespace='CELERY') + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() + + +@app.task(bind=True, ignore_result=True) +def debug_task(self): + print(f'Request: {self.request!r}') diff --git a/capaggregator/production.py b/capaggregator/production.py index 2ff21c3..d442032 100644 --- a/capaggregator/production.py +++ b/capaggregator/production.py @@ -37,4 +37,10 @@ 'USER': conn_str_params['user'], 'PASSWORD': conn_str_params['password'], } -} \ No newline at end of file +} + +CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL") +CELERY_ACCEPT_CONTENT = ['application/json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_BACKEND = 'django-db' +CELERY_CACHE_BACKEND = 'django-cache' \ No newline at end of file diff --git a/capaggregator/schema.py b/capaggregator/schema.py index cbc27ce..aec63a5 100644 --- a/capaggregator/schema.py +++ b/capaggregator/schema.py @@ -1,12 +1,13 @@ import graphene from graphene_django import DjangoObjectType #used to change Django object into a format that is readable by GraphQL +from django.utils import timezone from cap_feed.models import Alert, Region, Country class AlertType(DjangoObjectType): # Describe the data that is to be formatted into GraphQL fields class Meta: model = Alert - field = ("id", "identifier", "sender", "sent", "status", "msg_type", "scope", "urgency", "severity", "certainty", "expires", "area_desc", "event", "geocode_name", "geocode_value", "polygon") + field = ("id", "identifier", "sender", "sent", "status", "msg_type", "scope", "urgency", "severity", "certainty", "effective", "expires", "area_desc", "event", "geocode_name", "geocode_value", "polygon") class RegionType(DjangoObjectType): # Describe the data that is to be formatted into GraphQL fields @@ -22,11 +23,20 @@ class Meta: class Query(graphene.ObjectType): - #query ContactType to get list of contacts list_alert=graphene.List(AlertType) + list_country=graphene.List(CountryType) + list_region=graphene.List(RegionType) def resolve_list_alert(root, info): # We can easily optimize query count in the resolve method - return Alert.objects.order_by("-sent")[:20] + return Alert.objects.order_by("-id") + + def resolve_list_country(root, info): + # We can easily optimize query count in the resolve method + return Country.objects.order_by("-id") + + def resolve_list_region(root, info): + # We can easily optimize query count in the resolve method + return Region.objects.order_by("-id") schema = graphene.Schema(query=Query) \ No newline at end of file diff --git a/capaggregator/settings.py b/capaggregator/settings.py index 88e68dc..9d6a083 100644 --- a/capaggregator/settings.py +++ b/capaggregator/settings.py @@ -42,6 +42,8 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'django_celery_results', + 'django_celery_beat', ] MIDDLEWARE = [ @@ -143,3 +145,9 @@ # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' +CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL") +CELERY_ACCEPT_CONTENT = ['application/json'] +CELERY_TASK_SERIALIZER = 'json' + +CELERY_RESULT_BACKEND = 'django-db' +CELERY_CACHE_BACKEND = 'django-cache' \ No newline at end of file diff --git a/celerybeat-schedule.bak b/celerybeat-schedule.bak new file mode 100644 index 0000000..d24af5c --- /dev/null +++ b/celerybeat-schedule.bak @@ -0,0 +1,4 @@ +'entries', (2048, 622) +'__version__', (512, 20) +'tz', (1024, 18) +'utc_enabled', (1536, 4) diff --git a/celerybeat-schedule.dat b/celerybeat-schedule.dat new file mode 100644 index 0000000000000000000000000000000000000000..4c0d004351517584f30eb0995b8fa24abfea1bbc GIT binary patch literal 2670 zcmeHJ&1%~~5Vl>nvRVQyB_XEHC6wTnNF>w=q?kepJrvO%`g6&}tfX~z@p>hsRjC68 zpZWuP%j6yU6s5O5N?)V1l59$#sS`rdgC%wuX?MPvosVU{N&Vz|wUR$$XzTSj5`xP^ zhX|Ue)afVD(CJeerVdfgwlm}s^qqj>>d5seQJ|OJ28@Q8pf99KMsPeY%4$I3o`H51 zNe8KWNY(Q^DCkB=6jgjk|1^(b|F-q(*6PyQX3amq6+h!bahZa)(3dEcU~t_yiX`Q- zAE9}G=1XJZf4i5-m-U`}MV%AB|xQ3Jtw7tq%zXREE7kP$IJ!SiQ|Q@U_c>G&gI{c7Uzq@hhyU ztsse{B0i$Gb#)7LwS(g^FqgRoc?^CYMO*Y^%oCczs~_J|vy!<5hv%Lr^%J{v(l>Y9 zC+;>}6zbfu7Y^570p#DB@}FOc1^d@e$`xiXsef58euee^-$LR24{AkHrvGmwbQpc0 zPSIbmbzzS#r9ey*9)&z0LX76n0COWfJf#DQ40_WHis;yi&(s7@TlUsEZn|7eaR2f& zLCXj$=WpdiWuVXAp_`}5gp#LZ24;d{uVwdVg5n-~z}7A%QaY?N!`$7gE&rTLq;y&L HUn1o<{57QT literal 0 HcmV?d00001 diff --git a/celerybeat-schedule.dir b/celerybeat-schedule.dir new file mode 100644 index 0000000..d24af5c --- /dev/null +++ b/celerybeat-schedule.dir @@ -0,0 +1,4 @@ +'entries', (2048, 622) +'__version__', (512, 20) +'tz', (1024, 18) +'utc_enabled', (1536, 4) diff --git a/requirements.txt b/requirements.txt index 6414953..84e0f9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,39 @@ +amqp==5.1.1 aniso8601==9.0.1 asgiref==3.7.2 +billiard==4.1.0 +celery==5.3.1 certifi==2023.5.7 charset-normalizer==3.1.0 +click==8.1.3 +click-didyoumean==0.3.0 +click-plugins==1.1.1 +click-repl==0.3.0 +colorama==0.4.6 +cron-descriptor==1.4.0 Django==4.2.2 +django-celery-beat==2.5.0 +django-celery-results==2.5.1 +django-timezone-field==5.1 graphene==3.2.2 graphene-django==3.1.2 graphql-core==3.2.3 graphql-relay==3.2.0 idna==3.4 +kombu==5.3.1 promise==2.3 +prompt-toolkit==3.0.38 psycopg2-binary==2.9.6 +python-crontab==2.7.1 +python-dateutil==2.8.2 python-dotenv==1.0.0 +pytz==2023.3 requests==2.31.0 six==1.16.0 sqlparse==0.4.4 text-unidecode==1.3 tzdata==2023.3 urllib3==2.0.3 -whitenoise==6.4.0 +vine==5.0.0 +wcwidth==0.2.6 +whitenoise==6.5.0 diff --git a/startup.sh b/startup.sh index 78a2559..b3340f5 100644 --- a/startup.sh +++ b/startup.sh @@ -1,4 +1,4 @@ python manage.py migrate gunicorn --workers 2 --threads 4 --timeout 60 --access-logfile \ '-' --error-logfile '-' --bind=0.0.0.0:8000 \ - --chdir=/home/site/wwwroot capaggregator.wsgi \ No newline at end of file + --chdir=/home/site/wwwroot capaggregator.wsgi & celery -A capaggregator worker -l info --pool=solo & celery -A capaggregator beat -l info \ No newline at end of file