Merge pull request #20 from InspectorIncognito/fix/EDD-3327-speed-dow…

…nload-endpoints-and-historic-calc

Fix/edd 3327 speed download endpoints and historic calc

backend/gtfs_rt/management/commands/get_last_month_avg_speed.py

@@ -1,8 +1,12 @@
 from django.core.management.base import BaseCommand
-from processors.speed.avg_speed import get_last_month_avg_speed
+from processors.speed.avg_speed import get_month_commercial_speed
+from django.utils import timezone
 
 
 class Command(BaseCommand):
     def handle(self, *args, **options):
         print("Calling get_last_month_avg_speed command...")
-        get_last_month_avg_speed()
+        today = timezone.localtime()
+        this_year = today.year
+        this_month = today.month
+        get_month_commercial_speed(year=this_year, month=this_month)

backend/gtfs_rt/tests/test_historic_speeds.py

@@ -0,0 +1,48 @@
+import random
+
+from django.db.models import Sum
+from django.utils import timezone
+from processors.speed.avg_speed import get_month_commercial_speed
+from rest_api.factories import SpeedFactory, SegmentFactory
+from rest_api.models import HistoricSpeed, Speed
+from rest_api.tests.tests_views_base import BaseTestCase
+
+
+class HistoricSpeedTest(BaseTestCase):
+    def setUp(self):
+        day_minutes = 1440
+        temporal_segment_duration = 15
+        self.segments = [SegmentFactory(), SegmentFactory()]
+        self.day_types = ['L', 'S', 'D']
+        self.temporal_segments = int(day_minutes / temporal_segment_duration)
+
+    def test_calculate_monthly_speed(self):
+        for segment in self.segments:
+            for day_type in self.day_types:
+                for temporal_segment in range(self.temporal_segments):
+                    for day in range(4):
+                        distance = random.randint(10, 100)
+                        time_secs = random.randint(1, 5)
+                        SpeedFactory(segment=segment, temporal_segment=temporal_segment, day_type=day_type,
+                                     distance=distance, time_secs=time_secs)
+
+        today = timezone.localtime()
+        this_month = today.month
+        this_year = today.year
+        get_month_commercial_speed(year=this_year, month=this_month)
+
+        historic_speeds = HistoricSpeed.objects.filter(timestamp__year=this_year, timestamp__month=this_month)
+        expected_historic_speed_records = len(self.segments) * len(self.day_types) * self.temporal_segments
+        self.assertEqual(historic_speeds.count(), expected_historic_speed_records)
+
+        for avg_speed in historic_speeds:
+            segment = avg_speed.segment
+            temporal_segment = avg_speed.temporal_segment
+            day_type = avg_speed.day_type
+            expected_speed = avg_speed.speed
+            speeds = Speed.objects.filter(timestamp__year=this_year, timestamp__month=this_month,
+                                          segment=segment, temporal_segment=temporal_segment, day_type=day_type)
+            total_distance = speeds.aggregate(Sum('distance'))['distance__sum']
+            total_time = speeds.aggregate(Sum('time_secs'))['time_secs__sum']
+            actual_speed = round(3.6 * total_distance / total_time, 2)
+            self.assertAlmostEqual(actual_speed, expected_speed, 1)

backend/processors/speed/avg_speed.py

@@ -1,64 +1,40 @@
 from rest_api.models import Speed, HistoricSpeed, Segment
 from datetime import datetime, timedelta
-from django.db.models import Avg, Sum
 from django.utils import timezone
-from django.db.models import Func
-
-
-class Round2(Func):
-    function = "ROUND"
-    template = "%(function)s(%(expressions)s::numeric, 2)"
+from django.db.models import Sum, ExpressionWrapper, FloatField
+from django.db.models.functions import Round
 
 
 def flush_historic_speeds():
     HistoricSpeed.objects.all().delete()
 
 
-def get_last_month_avg_speed():
-    print("Calling get_last_month_avg_speed command...")
-    now = timezone.localtime()
-    yesterday = now - timedelta(days=1)
-    last_month = yesterday.month
-    last_year = yesterday.year
-
-    last_month_speeds = Speed.objects.filter(timestamp__year=last_year, timestamp__month=last_month)
-    avg_speeds = last_month_speeds.values("segment", "day_type", "temporal_segment", "timestamp").annotate(
-        average_segment_speed=Round2(Avg("speed"))).order_by("segment", "day_type", "temporal_segment")
-    for historic_speed in avg_speeds:
-        historic_speed_data = {
-            "segment": Segment.objects.get(pk=historic_speed["segment"]),
-            "speed": historic_speed["average_segment_speed"],
-            "day_type": historic_speed["day_type"],
-            "temporal_segment": historic_speed["temporal_segment"],
-            "timestamp": yesterday
-        }
-        HistoricSpeed.objects.create(**historic_speed_data)
-
-
-def get_avg_speed_by_month(year, month):
-    start_date = datetime(year, month, 1, 0, 0, 0, 0)
-    if month == 12:
-        end_date = datetime(year + 1, 1, 1, 0, 0, 0)
-    else:
-        end_date = datetime(year, month + 1, 1, 0, 0, 0)
-
-    start_date = timezone.make_aware(start_date, timezone.get_current_timezone())
-    end_date = timezone.make_aware(end_date, timezone.get_current_timezone())
-
-    speeds_in_month = Speed.objects.filter(
-        timestamp__gte=start_date,
-        timestamp__lt=end_date,
+def get_month_commercial_speed(year: int, month: int):
+    print("Calling get_month_commercial_speed command...")
+    creation_date = timezone.localtime().replace(year=year, month=month, day=1, hour=0, minute=0, second=0,
+                                                 microsecond=0)
+    last_month_speeds = Speed.objects.filter(timestamp__year=year, timestamp__month=month)
+    avg_speeds = (
+        last_month_speeds
+        .values('segment', 'day_type', 'temporal_segment')
+        .annotate(
+            total_distance=Sum('distance'),
+            total_time=Sum('time_secs'),
+            avg_speed=ExpressionWrapper(
+                Round(3.6 * Sum('distance') / Sum('time_secs'), 2),
+                output_field=FloatField()
+            )
+        )
     )
-
-    avg_speeds = speeds_in_month.values('segment', 'temporal_segment', 'day_type').annotate(
-        avg_speed=Round2(Sum('distance')/Sum('time_secs') * 3.6)
-    ).order_by('segment', 'temporal_segment', 'day_type')
-    #avg_speeds = (speeds_in_month.
-    #              values("segment", "day_type")
-    #              .annotate(avg_speed=Avg("speed"))
-    #              .order_by("segment", "day_type")
-    #              )
-    return avg_speeds
+    for avg_speed in avg_speeds:
+        historic_speed_data = dict(
+            segment=Segment.objects.get(pk=avg_speed['segment']),
+            speed=avg_speed['avg_speed'],
+            day_type=avg_speed['day_type'],
+            temporal_segment=avg_speed['temporal_segment'],
+            timestamp=creation_date
+        )
+        HistoricSpeed.objects.create(**historic_speed_data)
 
 
 def save_historic_avg_speed(avg_speeds, custom_datetime=None):
@@ -67,4 +43,11 @@ def save_historic_avg_speed(avg_speeds, custom_datetime=None):
                                "day_type": avg_speed["day_type"],
                                "speed": avg_speed["avg_speed"],
                                'timestamp': custom_datetime if custom_datetime is not None else None}
-        HistoricSpeed.objects.create(**historic_speed_data)
+
+
+def get_last_month_avg_speed():
+    print("Calling get_last_month_avg_speed command...")
+    today = timezone.localtime() - timedelta(days=1)
+    this_year = today.year
+    this_month = today.month
+    get_month_commercial_speed(year=this_year, month=this_month)

backend/rest_api/factories.py

@@ -1,5 +1,4 @@
 import datetime
-import uuid
 
 import factory
 from rest_api.models import Speed, Shape, Segment, HistoricSpeed, Stop, Alert
@@ -37,6 +36,7 @@ class Meta:
         model = Speed
 
     segment = factory.SubFactory(SegmentFactory)
+    temporal_segment = 1
     distance = 100
     time_secs = 5
     day_type = "L"

backend/rest_api/permissions.py

@@ -0,0 +1,3 @@
+from rest_framework.permissions import BasePermission
+from .models import DownloadToken
+from django.utils import timezone

backend/rest_api/views.py

@@ -2,7 +2,7 @@
 
 from django.utils import timezone
 from rest_framework import viewsets, mixins
-from django.http import JsonResponse, HttpResponse
+from django.http import JsonResponse, HttpResponse, StreamingHttpResponse
 from rest_framework import generics
 from rest_framework.permissions import AllowAny
 from processors.models.shapes import shapes_to_geojson
@@ -100,27 +100,44 @@ def get_queryset(self):
         queryset = queryset.order_by("segment", "temporal_segment")
         return queryset
 
+    @staticmethod
+    def csv_generator(queryset, fieldnames_dict):
+        yield ','.join(list(fieldnames_dict.values())) + '\n'
+        for obj in queryset:
+            fieldnames = list(fieldnames_dict.keys())
+            row = [str(obj[field]) for field in fieldnames]
+            yield ','.join(row) + '\n'
+
 
 class SpeedViewSet(GenericSpeedViewSet):
     serializer_class = SpeedSerializer
     queryset = Speed.objects.all().order_by('-temporal_segment')
 
     def to_csv(self, request, *args, **kwargs):
-        queryset = self.get_queryset()
+        queryset = self.get_queryset().values(
+            'segment__shape',
+            'segment__sequence',
+            'temporal_segment',
+            'day_type',
+            'distance',
+            'time_secs'
+        )
         start_date = request.query_params.get('start_date', None)
         if start_date is not None:
             start_date = datetime.strptime(start_date, '%Y-%d-%mT%H:%M:%S')
             start_date = timezone.make_aware(start_date, timezone.get_current_timezone())
             queryset = queryset.filter(timestamp__gte=start_date)
 
-        response = HttpResponse(content_type='text/csv')
+        fieldnames_dict = dict(
+            segment__shape='shape',
+            segment__sequence='sequence',
+            temporal_segment='temporal_segment',
+            day_type='day_type',
+            distance='distance',
+            time_secs='time_secs'
+        )
+        response = StreamingHttpResponse(self.csv_generator(queryset, fieldnames_dict), content_type='text/csv')
         response['Content-Disposition'] = 'attachment; filename="segment_speeds.csv"'
-        serializer = self.get_serializer(queryset, many=True)
-        fieldnames = ['shape', 'sequence', 'temporal_segment', 'day_type', 'distance', 'time_secs']
-        writer = csv.DictWriter(response, fieldnames=fieldnames)
-        writer.writeheader()
-        for obj in serializer.data:
-            writer.writerow(obj)
 
         return response
 
@@ -130,16 +147,22 @@ class HistoricSpeedViewSet(GenericSpeedViewSet):
     queryset = HistoricSpeed.objects.all().order_by("segment")
 
     def to_csv(self, request, *args, **kwargs):
-        queryset = self.get_queryset()
-        response = HttpResponse(content_type='text/csv')
+        queryset = self.get_queryset().values(
+            'segment__shape',
+            'segment__sequence',
+            'temporal_segment',
+            'day_type',
+            'speed'
+        )
+        fieldnames_dict = dict(
+            segment__shape='shape',
+            segment__sequence='sequence',
+            temporal_segment='temporal_segment',
+            day_type='day_type',
+            speed='speed'
+        )
+        response = StreamingHttpResponse(self.csv_generator(queryset, fieldnames_dict), content_type='text/csv')
         response['Content-Disposition'] = 'attachment; filename="segment_speeds.csv"'
-        serializer = self.get_serializer(queryset, many=True)
-        fieldnames = ['shape', 'sequence', 'temporal_segment', 'day_type', 'speed']
-        writer = csv.DictWriter(response, fieldnames=fieldnames)
-        writer.writeheader()
-        for obj in serializer.data:
-            writer.writerow(obj)
-
         return response