Merge pull request #7 from olincollege/SAN-54-collect-lamppost

San 54 collect lamppost

docs/source/explanations/datasets.rst

@@ -14,4 +14,9 @@ UMass Amherst has been developing a dataset of all crosswalks in Massachusetts u
 Traffic Dataset
 ***************
 
-Since our project is focused on pedestrian safety at nighttime on crosswalks, we need a dataset that contains information about the volume of traffic. MassDOT provides a convenient dataset that includes average annual daily traffic (AADT) counts for most roads in Massachusetts. The counts will be used to inform the risk of a pedestrian being hit by a car at a given crosswalk. The dataset can be viewed at the `following link <https://www.arcgis.com/apps/mapviewer/index.html?url=https://gis.massdot.state.ma.us/arcgis/rest/services/Roads/VMT/FeatureServer/10&source=sd>`_.
+Since our project is focused on pedestrian safety at nighttime on crosswalks, we need a dataset that contains information about the volume of traffic. MassDOT provides a convenient dataset that includes average annual daily traffic (AADT) counts for most roads in Massachusetts. The counts will be used to inform the risk of a pedestrian being hit by a car at a given crosswalk. The dataset can be viewed at the `following link <https://www.arcgis.com/apps/mapviewer/index.html?url=https://gis.massdot.state.ma.us/arcgis/rest/services/Roads/VMT/FeatureServer/10&source=sd>`_.
+
+Streetlights Dataset
+********************
+
+The main information that the streetlights dataset should contain is the location of the streetlights. Additional information such as the type of bulb, last-replacement year, and wattage, etc. are useful to have as well. After talking to Michael Donaghy, Superintendent of Street Lighting at the City of Boston Public Works Department, we learned that Boston has recently completed a full catalog of their streetlight assets in 2023. We acknowledge that many cities might not have this data available, in which case, `OpenStreetMap features <https://wiki.openstreetmap.org/wiki/Tag:highway%3Dstreet_lamp>`_ could be used to roughly estimate the streetlight locations. The Boston streetlight dataset can be viewed at the `following link <https://sdmaps.maps.arcgis.com/apps/dashboards/84e1553e754b424f9c544ab5079ed99f>`_.

requirements.txt

@@ -13,4 +13,5 @@ requests~=2.32.3
 geopandas~=1.0.1
 folium~=0.18.0
 pytest~=8.3.3
-sphinx~=8.1.3
+sphinx~=8.1.3
+aiohttp~=3.10.10

src/night_light/utils/mapping.py

@@ -32,7 +32,7 @@
 
 
 def create_folium_map(
-    layers: list[folium.GeoJson],
+    layers: list[folium.map.Layer],
     center: list,
     zoom_start: int,
     map_filename: str,
@@ -45,8 +45,8 @@ def create_folium_map(
     an HTML file.
 
     Args:
-        layers (list[folium.GeoJson]): GeoJson layers containing geometries to be added
-        to the map.
+        layers (list[folium.map.Layer]): Layers containing geometries to be added to the
+            map.
         center (list): A list containing the latitude and longitude for the map center
             [latitude, longitude].
         zoom_start (int): The initial zoom level for the map.

tests/conftest.py

@@ -1,9 +1,14 @@
-import pytest
 from night_light.utils import query_geojson
 
+import pytest
+import asyncio
+import aiohttp
+import geopandas as gpd
+
 MA_CROSSWALK_URL = "https://gis.massdot.state.ma.us/arcgis/rest/services/Assets/Crosswalk_Poly/FeatureServer/0/query"
 MA_TRAFFIC_URL = "https://gis.massdot.state.ma.us/arcgis/rest/services/Roads/VMT/FeatureServer/10/query"
 MA_MUNICIPALITIES_URL = "https://arcgisserver.digital.mass.gov/arcgisserver/rest/services/AGOL/Towns_survey_polym/FeatureServer/0/query"
+BOSTON_STREETLIGHTS_URL = "https://services.evari.io/evari/rest/services/Boston/Boston_Read_Only/MapServer/0/query"
 
 MUNICIPALITIES_MA_PARAMS = {
     "where": "1=1",
@@ -33,7 +38,12 @@
     "f": "geojson",
 }
 
-
+STREETLIGHTS_BOSTON_PARAMS = {
+    "where": "1=1",
+    "outFields": "OBJECTID, Final_Fixture_Type, Final_Wattage, Final_Bulb_Type, Final_Luminaires_Per_Pole",
+    "outSR": "4326",
+    "f": "geojson",
+}
 BOSTON_CENTER_COORD = [42.3601, -71.0589]
 
 
@@ -49,3 +59,46 @@ def boston_traffic():
     return query_geojson.fetch_geojson_data(
         url=MA_TRAFFIC_URL, params=TRAFFIC_BOSTON_PARAMS
     )
+
+
+@pytest.fixture
+def boston_streetlights():
+    async def fetch_data(session, url, params):
+        async with session.get(url, params=params) as response:
+            response.raise_for_status()
+            return await response.json()
+
+    async def fetch_all_data():
+        data = []
+        batch_size = 2000
+        max_concurrent_requests = 50
+
+        async with aiohttp.ClientSession() as session:
+            while True:
+                tasks = [
+                    fetch_data(
+                        session,
+                        BOSTON_STREETLIGHTS_URL,
+                        {
+                            **STREETLIGHTS_BOSTON_PARAMS,
+                            "resultOffset": str((i * batch_size)),
+                        },
+                    )
+                    for i in range(max_concurrent_requests)
+                ]
+
+                responses = await asyncio.gather(*tasks)
+
+                for result in responses:
+                    features = result.get("features", [])
+                    data.extend(features)
+                    feature_count = len(features)
+                    if (
+                        not result.get("exceededTransferLimit")
+                        or feature_count < batch_size
+                    ):
+                        return data
+
+    data = asyncio.run(fetch_all_data())
+    gdf = gpd.GeoDataFrame.from_features(data, crs="EPSG:4326")
+    return gdf

tests/test_streetlights_mapping.py

@@ -0,0 +1,43 @@
+import os
+import time
+
+import folium
+from folium.plugins import MarkerCluster
+
+from night_light.utils import create_folium_map
+from night_light.utils.mapping import open_html_file
+from tests.conftest import BOSTON_CENTER_COORD
+
+
+def test_boston_streetlights_map(boston_streetlights):
+    """Test creating a map of the Boston streetlights"""
+    map_filename = "test_boston_streetlights.html"
+    streetlights_marker_cluster = MarkerCluster(
+        name="Streetlights", show=True, control=True, overlay=True
+    )
+    for _, row in boston_streetlights.iterrows():
+        tooltip_text = (
+            f"Streetlight ID: {row['OBJECTID']}<br>"
+            f"Fixture Type: {row['Final_Fixture_Type']}<br>"
+            f"Wattage: {row['Final_Wattage']}<br>"
+            f"Bulb Type: {row['Final_Bulb_Type']}<br>"
+            f"Luminaires Per Pole: {row['Final_Luminaires_Per_Pole']}"
+        )
+
+        folium.Marker(
+            location=[row.geometry.y, row.geometry.x],
+            tooltip=tooltip_text,
+            icon=folium.Icon(color="yellow", icon="lightbulb-o", prefix="fa"),
+        ).add_to(streetlights_marker_cluster)
+
+    create_folium_map(
+        layers=[streetlights_marker_cluster],
+        zoom_start=12,
+        center=BOSTON_CENTER_COORD,
+        map_filename=map_filename,
+    )
+    assert os.path.exists(map_filename)
+    open_html_file(map_filename)
+    time.sleep(1)
+    os.remove(map_filename)
+    assert not os.path.exists(map_filename)

tests/test_streetlights_query.py

@@ -0,0 +1,33 @@
+import os
+from night_light.utils import query_geojson
+
+
+def test_query_boston_streetlights(boston_streetlights):
+    """Test querying the Boston crosswalk"""
+    assert boston_streetlights.shape[0] > 0
+    assert boston_streetlights.crs == "EPSG:4326"
+    assert boston_streetlights.geometry.name == "geometry"
+    assert boston_streetlights.geometry.geom_type.unique()[0] == "Point"
+    assert boston_streetlights.columns.to_list() == [
+        "geometry",
+        "OBJECTID",
+        "Final_Fixture_Type",
+        "Final_Wattage",
+        "Final_Bulb_Type",
+        "Final_Luminaires_Per_Pole",
+    ]
+
+
+def test_save_boston_streetlights_geojson(boston_streetlights):
+    """Test saving the Boston streelights to a GeoJSON file"""
+    query_geojson.save_geojson(boston_streetlights, "test_boston_streetlights.geojson")
+    geojson_filename = "test_boston_streetlights.geojson"
+    saved_gdf = query_geojson.gpd.read_file(geojson_filename)
+
+    assert boston_streetlights.crs == saved_gdf.crs
+    assert set(boston_streetlights.columns) == set(saved_gdf.columns)
+    assert boston_streetlights.index.equals(saved_gdf.index)
+    assert boston_streetlights.shape == saved_gdf.shape
+
+    os.remove(geojson_filename)
+    assert not os.path.exists(geojson_filename)