Adding first street foundation data

data/data-pipeline/data_pipeline/etl/constants.py

@@ -29,6 +29,16 @@
         "module_dir": "mapping_for_ej",
         "class_name": "MappingForEJETL",
     },
+    {
+        "name": "fsf_flood_risk",
+        "module_dir": "fsf_flood_risk",
+        "class_name": "FloodRiskETL",
+    },
+    {
+        "name": "fsf_wildfire_risk",
+        "module_dir": "fsf_wildfire_risk",
+        "class_name": "WildfireRiskETL",
+    },
     {
         "name": "ejscreen",
         "module_dir": "ejscreen",

data/data-pipeline/data_pipeline/etl/score/config/datasets.yml

@@ -129,3 +129,84 @@ datasets:
         include_in_tiles: true
         include_in_downloadable_files: true
 
+  - long_name: "First Street Foundation Flood Risk"
+    short_name: "FSF Flood Risk"
+    module_name: fsf_flood_risk
+    input_geoid_tract_field_name: "GEOID"
+    load_fields:
+      - short_name: "flood_eligible_properties"
+        df_field_name: "COUNT_PROPERTIES"
+        long_name: "Count of properties eligible for flood risk calculation within tract (floor of 250)" 
+        field_type: float
+        include_in_tiles: false
+        include_in_downloadable_files: true
+        create_percentile: false
+      - short_name: "flood_risk_properties_today"
+        df_field_name: "PROPERTIES_AT_RISK_FROM_FLOODING_TODAY"
+        long_name: "Count of properties at risk of flood today" 
+        field_type: float
+        include_in_tiles: false
+        include_in_downloadable_files: true
+        create_percentile: false
+      - short_name: "flood_risk_properties_30yrs"
+        df_field_name: "PROPERTIES_AT_RISK_FROM_FLOODING_IN_30_YEARS"
+        long_name: "Count of properties at risk of flood in 30 years" 
+        field_type: float
+        include_in_tiles: false
+        include_in_downloadable_files: true
+        create_percentile: false
+      - short_name: "flood_risk_share_today"
+        df_field_name: "SHARE_OF_PROPERTIES_AT_RISK_FROM_FLOODING_TODAY"
+        long_name: "Share of properties at risk of flood today" 
+        field_type: float
+        include_in_tiles: false
+        include_in_downloadable_files: true
+        create_percentile: true
+      - short_name: "flood_risk_share_30yrs"
+        df_field_name: "SHARE_OF_PROPERTIES_AT_RISK_FROM_FLOODING_IN_30_YEARS"
+        long_name: "Share of properties at risk of flood in 30 years" 
+        field_type: float
+        include_in_tiles: false
+        include_in_downloadable_files: true
+        create_percentile: true
+
+  - long_name: "First Street Foundation Wildfire Risk"
+    short_name: "FSF Wildfire Risk"
+    module_name: fsf_wildfire_risk
+    input_geoid_tract_field_name: "GEOID"
+    load_fields:
+      - short_name: "fire_eligible_properties"
+        df_field_name: "COUNT_PROPERTIES"
+        long_name: "Count of properties eligible for wildfire risk calculation within tract (floor of 250)" 
+        field_type: float
+        include_in_tiles: false
+        include_in_downloadable_files: true
+        create_percentile: false
+      - short_name: "fire_risk_properties_today"
+        df_field_name: "PROPERTIES_AT_RISK_FROM_FIRE_TODAY"
+        long_name: "Count of properties at risk of wildfire today" 
+        field_type: float
+        include_in_tiles: false
+        include_in_downloadable_files: true
+        create_percentile: false
+      - short_name: "fire_risk_properties_30yrs"
+        df_field_name: "PROPERTIES_AT_RISK_FROM_FIRE_IN_30_YEARS"
+        long_name: "Count of properties at risk of wildfire in 30 years" 
+        field_type: float
+        include_in_tiles: false
+        include_in_downloadable_files: true
+        create_percentile: false
+      - short_name: "fire_risk_share_today"
+        df_field_name: "SHARE_OF_PROPERTIES_AT_RISK_FROM_FIRE_TODAY"
+        long_name: "Share of properties at risk of fire today" 
+        field_type: float
+        include_in_tiles: false
+        include_in_downloadable_files: true
+        create_percentile: true
+      - short_name: "fire_risk_share_30yrs"
+        df_field_name: "SHARE_OF_PROPERTIES_AT_RISK_FROM_FIRE_IN_30_YEARS"
+        long_name: "Share of properties at risk of fire in 30 years" 
+        field_type: float
+        include_in_tiles: false
+        include_in_downloadable_files: true
+        create_percentile: true

data/data-pipeline/data_pipeline/etl/sources/fsf_flood_risk/README.md

@@ -0,0 +1,3 @@
+# FSF flood risk data
+
+Flood risk computed as 1 in 100 year flood zone

data/data-pipeline/data_pipeline/etl/sources/fsf_flood_risk/etl.py

@@ -0,0 +1,93 @@
+# pylint: disable=unsubscriptable-object
+# pylint: disable=unsupported-assignment-operation
+
+import pandas as pd
+from data_pipeline.config import settings
+
+from data_pipeline.etl.base import ExtractTransformLoad, ValidGeoLevel
+from data_pipeline.utils import get_module_logger
+
+logger = get_module_logger(__name__)
+
+
+class FloodRiskETL(ExtractTransformLoad):
+    """ETL class for the First Street Foundation flood risk dataset"""
+
+    NAME = "fsf_flood_risk"
+    SOURCE_URL = settings.AWS_JUSTICE40_DATASOURCES_URL + "/fsf_flood.zip"
+    GEO_LEVEL = ValidGeoLevel.CENSUS_TRACT
+
+    # Output score variables (values set on datasets.yml) for linting purposes
+    COUNT_PROPERTIES: str
+    PROPERTIES_AT_RISK_FROM_FLOODING_TODAY: str
+    PROPERTIES_AT_RISK_FROM_FLOODING_IN_30_YEARS: str
+    SHARE_OF_PROPERTIES_AT_RISK_FROM_FLOODING_TODAY: str
+    SHARE_OF_PROPERTIES_AT_RISK_FROM_FLOODING_IN_30_YEARS: str
+
+    def __init__(self):
+        # define the full path for the input CSV file
+        self.INPUT_CSV = (
+            self.get_tmp_path() / "fsf_flood" / "flood_tract_2010.csv"
+        )
+
+        # this is the main dataframe
+        self.df: pd.DataFrame
+
+        # Start dataset-specific vars here
+        self.COUNT_PROPERTIES_NATIVE_FIELD_NAME = "count_properties"
+        self.COUNT_PROPERTIES_AT_RISK_TODAY = "mid_depth_100_year00"
+        self.COUNT_PROPERTIES_AT_RISK_30_YEARS = "mid_depth_100_year30"
+        self.CLIP_PROPERTIES_COUNT = 250
+
+    def transform(self) -> None:
+        """Reads the unzipped data file into memory and applies the following
+        transformations to prepare it for the load() method:
+
+        - Renames the Census Tract column to match the other datasets
+        - Calculates share of properties at risk, left-clipping number of properties at 250
+        """
+        logger.info("Transforming National Risk Index Data")
+
+        logger.info(self.COLUMNS_TO_KEEP)
+        # read in the unzipped csv data source then rename the
+        # Census Tract column for merging
+        df_fsf_flood_disagg: pd.DataFrame = pd.read_csv(
+            self.INPUT_CSV,
+            dtype={self.INPUT_GEOID_TRACT_FIELD_NAME: str},
+            low_memory=False,
+        )
+
+        df_fsf_flood_disagg[self.GEOID_TRACT_FIELD_NAME] = df_fsf_flood_disagg[
+            self.INPUT_GEOID_TRACT_FIELD_NAME
+        ].str.zfill(11)
+
+        # Because we have some tracts that are listed twice, we aggregate based on
+        # GEOID10_TRACT. Note that I haven't confirmed this with the FSF boys -- to do!
+        df_fsf_flood = (
+            df_fsf_flood_disagg.groupby(self.GEOID_TRACT_FIELD_NAME)
+            .sum()
+            .reset_index()
+        )
+
+        df_fsf_flood[self.COUNT_PROPERTIES] = df_fsf_flood[
+            self.COUNT_PROPERTIES_NATIVE_FIELD_NAME
+        ].clip(lower=self.CLIP_PROPERTIES_COUNT)
+
+        df_fsf_flood[self.SHARE_OF_PROPERTIES_AT_RISK_FROM_FLOODING_TODAY] = (
+            df_fsf_flood[self.COUNT_PROPERTIES_AT_RISK_TODAY]
+            / df_fsf_flood[self.COUNT_PROPERTIES]
+        )
+        df_fsf_flood[
+            self.SHARE_OF_PROPERTIES_AT_RISK_FROM_FLOODING_IN_30_YEARS
+        ] = (
+            df_fsf_flood[self.COUNT_PROPERTIES_AT_RISK_30_YEARS]
+            / df_fsf_flood[self.COUNT_PROPERTIES]
+        )
+
+        # Assign the final df to the class' output_df for the load method with rename
+        self.output_df = df_fsf_flood.rename(
+            columns={
+                self.COUNT_PROPERTIES_AT_RISK_TODAY: self.PROPERTIES_AT_RISK_FROM_FLOODING_TODAY,
+                self.COUNT_PROPERTIES_AT_RISK_30_YEARS: self.PROPERTIES_AT_RISK_FROM_FLOODING_IN_30_YEARS,
+            }
+        )

data/data-pipeline/data_pipeline/etl/sources/fsf_wildfire_risk/README.md

@@ -0,0 +1,3 @@
+# FSF wildfire risk data
+
+Fire risk computed as >= 0.003 burn risk probability

data/data-pipeline/data_pipeline/etl/sources/fsf_wildfire_risk/etl.py

@@ -0,0 +1,91 @@
+# pylint: disable=unsubscriptable-object
+# pylint: disable=unsupported-assignment-operation
+
+import pandas as pd
+from data_pipeline.config import settings
+
+from data_pipeline.etl.base import ExtractTransformLoad, ValidGeoLevel
+from data_pipeline.utils import get_module_logger
+
+logger = get_module_logger(__name__)
+
+
+class WildfireRiskETL(ExtractTransformLoad):
+    """ETL class for the First Street Foundation wildfire risk dataset"""
+
+    NAME = "fsf_wildfire_risk"
+    SOURCE_URL = settings.AWS_JUSTICE40_DATASOURCES_URL + "/fsf_fire.zip"
+    GEO_LEVEL = ValidGeoLevel.CENSUS_TRACT
+
+    # Output score variables (values set on datasets.yml) for linting purposes
+    COUNT_PROPERTIES: str
+    PROPERTIES_AT_RISK_FROM_FIRE_TODAY: str
+    PROPERTIES_AT_RISK_FROM_FIRE_IN_30_YEARS: str
+    SHARE_OF_PROPERTIES_AT_RISK_FROM_FIRE_TODAY: str
+    SHARE_OF_PROPERTIES_AT_RISK_FROM_FIRE_IN_30_YEARS: str
+
+    def __init__(self):
+        # define the full path for the input CSV file
+        self.INPUT_CSV = (
+            self.get_tmp_path() / "fsf_fire" / "fire_tract_2010.csv"
+        )
+
+        # this is the main dataframe
+        self.df: pd.DataFrame
+
+        # Start dataset-specific vars here
+        self.COUNT_PROPERTIES_NATIVE_FIELD_NAME = "count_properties"
+        self.COUNT_PROPERTIES_AT_RISK_TODAY = "burnprob_year00_flag"
+        self.COUNT_PROPERTIES_AT_RISK_30_YEARS = "burnprob_year30_flag"
+        self.CLIP_PROPERTIES_COUNT = 250
+
+    def transform(self) -> None:
+        """Reads the unzipped data file into memory and applies the following
+        transformations to prepare it for the load() method:
+
+        - Renames the Census Tract column to match the other datasets
+        - Calculates share of properties at risk, left-clipping number of properties at 250
+        """
+        logger.info("Transforming National Risk Index Data")
+
+        logger.info(self.COLUMNS_TO_KEEP)
+        # read in the unzipped csv data source then rename the
+        # Census Tract column for merging
+        df_fsf_fire_disagg: pd.DataFrame = pd.read_csv(
+            self.INPUT_CSV,
+            dtype={self.INPUT_GEOID_TRACT_FIELD_NAME: str},
+            low_memory=False,
+        )
+
+        df_fsf_fire_disagg[self.GEOID_TRACT_FIELD_NAME] = df_fsf_fire_disagg[
+            self.INPUT_GEOID_TRACT_FIELD_NAME
+        ].str.zfill(11)
+
+        # Because we have some tracts that are listed twice, we aggregate based on
+        # GEOID10_TRACT. Note that I haven't confirmed this with the FSF boys -- to do!
+        df_fsf_fire = (
+            df_fsf_fire_disagg.groupby(self.GEOID_TRACT_FIELD_NAME)
+            .sum()
+            .reset_index()
+        )
+
+        df_fsf_fire[self.COUNT_PROPERTIES] = df_fsf_fire[
+            self.COUNT_PROPERTIES_NATIVE_FIELD_NAME
+        ].clip(lower=self.CLIP_PROPERTIES_COUNT)
+
+        df_fsf_fire[self.SHARE_OF_PROPERTIES_AT_RISK_FROM_FIRE_TODAY] = (
+            df_fsf_fire[self.COUNT_PROPERTIES_AT_RISK_TODAY]
+            / df_fsf_fire[self.COUNT_PROPERTIES]
+        )
+        df_fsf_fire[self.SHARE_OF_PROPERTIES_AT_RISK_FROM_FIRE_IN_30_YEARS] = (
+            df_fsf_fire[self.COUNT_PROPERTIES_AT_RISK_30_YEARS]
+            / df_fsf_fire[self.COUNT_PROPERTIES]
+        )
+
+        # Assign the final df to the class' output_df for the load method with rename
+        self.output_df = df_fsf_fire.rename(
+            columns={
+                self.COUNT_PROPERTIES_AT_RISK_TODAY: self.PROPERTIES_AT_RISK_FROM_FIRE_TODAY,
+                self.COUNT_PROPERTIES_AT_RISK_30_YEARS: self.PROPERTIES_AT_RISK_FROM_FIRE_IN_30_YEARS,
+            }
+        )