Merge pull request #309 from opencivicdata/feature/hec/metro-event-pa…

…iring

Definitively remove event time constraint from Metro event pairing

lametro/events.py

@@ -132,7 +132,14 @@ def _merge_events(self, events):
                 try:
                     assert event.key not in spanish_events
                 except AssertionError:
-                    raise AssertionError('{0} already exists as a key with a value of {1}'.format(event.key, spanish_events[event.key]))
+                    # Don't allow SAP events to be overwritten in the event
+                    # dictionary. If this error is raised, there is more than
+                    # one SAP event for a meeting body on the same day, i.e.,
+                    # our event pairing criteria are too broad. Consider adding
+                    # back event time as a match constraint. See:
+                    # https://github.com/opencivicdata/scrapers-us-municipal/pull/284 &
+                    # https://github.com/opencivicdata/scrapers-us-municipal/pull/309.
+                    raise ValueError('{0} already exists as a key with a value of {1}'.format(event.key, spanish_events[event.key]))
                 spanish_events[event.key] = (event, web_event)
             else:
                 english_events.append((event, web_event))
@@ -367,15 +374,12 @@ def _partner_name(self):
 
     def is_partner(self, other):
         return (self._partner_name == other['EventBodyName'] and
-                self['EventDate'] == other['EventDate'] and
-                self['EventTime'] == other['EventTime'])
-
+                self['EventDate'] == other['EventDate'])
 
     @property
     def partner_search_string(self):
         search_string = "EventBodyName eq '{}'".format(self._partner_name)
         search_string += " and EventDate eq datetime'{}'".format(self['EventDate'])
-        search_string += " and EventTime eq '{}'".format(self['EventTime'])
 
         return search_string
 

tests/lametro/conftest.py

@@ -60,7 +60,8 @@ def web_event():
                  'Agenda': {'label': 'Agenda', 'url': ''}, 
                  'iCalendar': {'url': ''}, 
                  'Meeting Location': 'One Gateway Plaza, Los Angeles, CA 90012', 
-                 'Recap/Minutes': 'Not\xa0available'}
+                 'Recap/Minutes': 'Not\xa0available',
+                 'Meeting video': 'Not\xa0available'}
 
     return web_event
 

tests/lametro/test_events.py

@@ -5,18 +5,21 @@
 
 from pupa.scrape.event import Event
 
+from lametro.events import LAMetroAPIEvent
+
+
 @pytest.mark.parametrize('api_status_name,scraper_assigned_status', [
-    ('Final', 'passed'), 
+    ('Final', 'passed'),
     ('Final Revised', 'passed'),
     ('Final 2nd Revised', 'passed'),
     ('Draft', 'confirmed'),
     ('Canceled', 'cancelled')
 ])
-def test_status_assignment(event_scraper, 
-                           api_event, 
+def test_status_assignment(event_scraper,
+                           api_event,
                            web_event,
                            api_status_name,
-                           scraper_assigned_status, 
+                           scraper_assigned_status,
                            mocker):
     with requests_mock.Mocker() as m:
         matcher = re.compile('webapi.legistar.com')
@@ -32,12 +35,13 @@ def test_status_assignment(event_scraper,
         for event in event_scraper.scrape():
             assert event.status == scraper_assigned_status
 
+
 @pytest.mark.parametrize('item_sequence,should_error', [
     (12, True),
     (11, False),
 ])
 def test_sequence_duplicate_error(event_scraper,
-                                  api_event, 
+                                  api_event,
                                   web_event,
                                   event_agenda_item,
                                   item_sequence,
@@ -50,9 +54,9 @@ def test_sequence_duplicate_error(event_scraper,
         matcher = re.compile('metro.legistar.com')
         m.get(matcher, json={}, status_code=200)
 
-        api_event['event_details'] = [{'note': 'web', 
+        api_event['event_details'] = [{'note': 'web',
                                       'url': 'https://metro.legistar.com/MeetingDetail.aspx?ID=642118&GUID=F19B2133-928C-4390-9566-C293C61DC89A&Options=info&Search='}]
-        
+
         event_agenda_item_b = event_agenda_item.copy()
         event_agenda_item_b['EventItemAgendaSequence'] = item_sequence
 
@@ -70,3 +74,44 @@ def test_sequence_duplicate_error(event_scraper,
         else:
             for event in event_scraper.scrape():
                 assert len(event.agenda) == 2
+
+
+def test_events_paired(event_scraper, api_event, web_event, mocker):
+    # Create a matching SAP event with a distinct ID
+    sap_api_event = api_event.copy()
+    sap_api_event['EventId'] = 1109
+    sap_api_event['EventBodyName'] = '{} (SAP)'.format(api_event['EventBodyName'])
+
+    # Set a non-matching time to confirm time is not a match constraint
+    sap_api_event['EventTime'] = '12:00 AM'
+
+    # Create a non-matching English event
+    another_api_event = api_event.copy()
+    another_api_event['EventId'] = 41361
+    another_api_event['EventBodyName'] = 'Planning and Programming Committee'
+
+    events = [
+        (LAMetroAPIEvent(api_event), web_event),
+        (LAMetroAPIEvent(sap_api_event), web_event),
+        (LAMetroAPIEvent(another_api_event), web_event)
+    ]
+
+    results = event_scraper._merge_events(events)
+
+    # Assert that the scraper yields two events
+    assert len(results) == 2
+
+    # Assert that the proper English and Spanish events were paired
+    event, web_event = results[0]
+    assert event['EventId'] == api_event['EventId']
+    assert event['SAPEventId'] == sap_api_event['EventId']
+
+    # Add a duplicate SAP event to the event array
+    events.append((LAMetroAPIEvent(sap_api_event), web_event))
+
+    # Assert that duplicate SAP events raise an exception
+    with pytest.raises(ValueError) as excinfo:
+        event_scraper._merge_events(events)
+
+        event_key = LAMetroAPIEvent(sap_api_event).key
+        assert '{} already exists as a key'.format(event_key) in str(excinfo.value)