Skip to content

Commit

Permalink
Add tests for HUD housing (#1848)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattbowen-usds
mattbowen-usds committed Sep 12, 2022
1 parent 16c8042 commit bbd78c7
Show file tree
Hide file tree
Showing 7 changed files with 81 additions and 18 deletions.
32 changes: 14 additions & 18 deletions data/data-pipeline/data_pipeline/etl/sources/hud_housing/etl.py
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
import pandas as pd
from data_pipeline.etl.base import ExtractTransformLoad
from data_pipeline.etl.base import ExtractTransformLoad, ValidGeoLevel
from data_pipeline.utils import get_module_logger

logger = get_module_logger(__name__)


class HudHousingETL(ExtractTransformLoad):
NAME = "HudHousingETL"
GEO_LEVEL: ValidGeoLevel = ValidGeoLevel.CENSUS_TRACT

def __init__(self):
self.OUTPUT_PATH = self.DATA_PATH / "dataset" / "hud_housing"
self.GEOID_TRACT_FIELD_NAME = "GEOID10_TRACT"
self.HOUSING_FTP_URL = "https://www.huduser.gov/portal/datasets/cp/2014thru2018-140-csv.zip"
self.HOUSING_ZIP_FILE_DIR = self.get_tmp_path() / "hud_housing"
self.HOUSING_ZIP_FILE_DIR = self.get_tmp_path()

# We measure households earning less than 80% of HUD Area Median Family Income by county
# and paying greater than 30% of their income to housing costs.
Expand All @@ -22,6 +25,14 @@ def __init__(self):
self.NO_KITCHEN_OR_INDOOR_PLUMBING_FIELD_NAME = (
"Share of homes with no kitchen or indoor plumbing (percent)"
)
self.COLUMNS_TO_KEEP = [
self.GEOID_TRACT_FIELD_NAME,
self.HOUSING_BURDEN_NUMERATOR_FIELD_NAME,
self.HOUSING_BURDEN_DENOMINATOR_FIELD_NAME,
self.HOUSING_BURDEN_FIELD_NAME,
self.NO_KITCHEN_OR_INDOOR_PLUMBING_FIELD_NAME,
"DENOM INCL NOT COMPUTED",
]

# Note: some variable definitions.
# HUD-adjusted median family income (HAMFI).
Expand Down Expand Up @@ -234,19 +245,4 @@ def transform(self) -> None:
float
)

def load(self) -> None:
logger.info("Saving HUD Housing Data")

self.OUTPUT_PATH.mkdir(parents=True, exist_ok=True)

# Drop unnecessary fields
self.df[
[
self.GEOID_TRACT_FIELD_NAME,
self.HOUSING_BURDEN_NUMERATOR_FIELD_NAME,
self.HOUSING_BURDEN_DENOMINATOR_FIELD_NAME,
self.HOUSING_BURDEN_FIELD_NAME,
self.NO_KITCHEN_OR_INDOOR_PLUMBING_FIELD_NAME,
"DENOM INCL NOT COMPUTED",
]
].to_csv(path_or_buf=self.OUTPUT_PATH / "usa.csv", index=False)
self.output_df = self.df
Empty file added data/data-pipeline/data_pipeline/tests/sources/hud_housing/__init__.py
View file
Empty file.
Binary file added data/data-pipeline/data_pipeline/tests/sources/hud_housing/data/2014thru2018-140-csv.zip
View file
Binary file not shown.
16 changes: 16 additions & 0 deletions data/data-pipeline/data_pipeline/tests/sources/hud_housing/data/extract.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
source,sumlevel,geoid,name,st,cnty,tract,T3_est1,T3_est2,T3_est3,T3_est4,T3_est5,T3_est6,T3_est7,T3_est8,T3_est9,T3_est10,T3_est11,T3_est12,T3_est13,T3_est14,T3_est15,T3_est16,T3_est17,T3_est18,T3_est19,T3_est20,T3_est21,T3_est22,T3_est23,T3_est24,T3_est25,T3_est26,T3_est27,T3_est28,T3_est29,T3_est30,T3_est31,T3_est32,T3_est33,T3_est34,T3_est35,T3_est36,T3_est37,T3_est38,T3_est39,T3_est40,T3_est41,T3_est42,T3_est43,T3_est44,T3_est45,T3_est46,T3_est47,T3_est48,T3_est49,T3_est50,T3_est51,T3_est52,T3_est53,T3_est54,T3_est55,T3_est56,T3_est57,T3_est58,T3_est59,T3_est60,T3_est61,T3_est62,T3_est63,T3_est64,T3_est65,T3_est66,T3_est67,T3_est68,T3_est69,T3_est70,T3_est71,T3_est72,T3_est73,T3_est74,T3_est75,T3_est76,T3_est77,T3_est78,T3_est79,T3_est80,T3_est81,T3_est82,T3_est83,T3_est84,T3_est85,T3_est86,T3_est87,T3_moe1,T3_moe2,T3_moe3,T3_moe4,T3_moe5,T3_moe6,T3_moe7,T3_moe8,T3_moe9,T3_moe10,T3_moe11,T3_moe12,T3_moe13,T3_moe14,T3_moe15,T3_moe16,T3_moe17,T3_moe18,T3_moe19,T3_moe20,T3_moe21,T3_moe22,T3_moe23,T3_moe24,T3_moe25,T3_moe26,T3_moe27,T3_moe28,T3_moe29,T3_moe30,T3_moe31,T3_moe32,T3_moe33,T3_moe34,T3_moe35,T3_moe36,T3_moe37,T3_moe38,T3_moe39,T3_moe40,T3_moe41,T3_moe42,T3_moe43,T3_moe44,T3_moe45,T3_moe46,T3_moe47,T3_moe48,T3_moe49,T3_moe50,T3_moe51,T3_moe52,T3_moe53,T3_moe54,T3_moe55,T3_moe56,T3_moe57,T3_moe58,T3_moe59,T3_moe60,T3_moe61,T3_moe62,T3_moe63,T3_moe64,T3_moe65,T3_moe66,T3_moe67,T3_moe68,T3_moe69,T3_moe70,T3_moe71,T3_moe72,T3_moe73,T3_moe74,T3_moe75,T3_moe76,T3_moe77,T3_moe78,T3_moe79,T3_moe80,T3_moe81,T3_moe82,T3_moe83,T3_moe84,T3_moe85,T3_moe86,T3_moe87
2014thru2018,140,14000US06027000800,"Census Tract 8, Inyo County, California",6,27,800,1375,800,30,30,0,0,0,0,0,0,0,0,0,0,10,0,0,0,10,0,115,35,30,50,0,4,115,15,35,10,15,35,0,0,0,0,0,0,530,30,65,70,50,320,580,35,25,10,0,0,0,0,0,0,0,0,0,65,0,10,0,55,0,90,70,20,0,0,0,95,10,30,40,0,10,0,0,0,0,0,0,300,10,40,85,0,165,133,101,31,31,12,12,12,12,12,12,12,12,12,12,15,12,12,12,15,12,57,26,32,41,12,5,56,15,34,13,22,35,12,12,12,12,12,12,90,31,34,41,25,79,122,34,30,16,12,12,12,12,12,12,12,12,12,67,12,18,12,64,12,83,79,32,12,12,12,52,20,32,35,12,14,12,12,12,12,12,12,89,14,30,50,12,64
2014thru2018,140,14000US06061021322,"Census Tract 213.22, Placer County, California",6,61,21322,5395,4250,0,0,0,0,0,0,15,0,0,0,0,15,120,0,0,0,55,65,250,70,45,105,35,0,630,30,75,130,105,290,65,65,0,0,0,0,3170,15,45,140,60,2905,1145,0,0,0,0,0,0,50,50,0,0,0,0,45,0,45,0,0,0,240,160,30,50,0,0,270,0,160,85,30,0,15,15,0,0,0,0,520,65,0,135,105,215,179,212,19,19,19,19,19,19,26,19,19,19,19,26,108,19,19,19,65,84,122,57,39,84,39,19,151,37,88,78,67,116,87,87,19,19,19,19,238,20,38,92,56,249,187,19,19,19,19,19,19,81,81,19,19,19,19,73,19,73,19,19,19,100,95,35,52,19,19,115,19,93,90,30,19,26,26,19,19,19,19,170,83,19,105,87,99
2014thru2018,140,14000US06069000802,"Census Tract 8.02, San Benito County, California",6,69,802,885,615,4,4,0,4,0,4,4,0,0,0,0,4,0,0,0,0,0,0,70,4,20,20,4,25,90,4,10,10,10,55,4,4,0,0,0,0,440,10,25,35,35,335,265,4,0,0,4,0,0,4,0,0,0,4,0,20,4,15,0,0,0,25,4,20,0,0,0,25,4,4,15,0,0,4,4,0,0,0,0,190,20,35,20,35,80,69,70,11,5,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,25,12,16,12,1,20,28,3,11,11,12,23,10,10,12,12,12,12,62,10,31,20,19,46,45,4,12,12,4,12,12,11,12,12,12,11,12,19,12,17,12,12,12,17,11,18,12,12,12,14,2,7,14,12,12,3,3,12,12,12,12,43,14,19,14,18,28
2014thru2018,140,14000US15001021010,"Census Tract 210.10, Hawaii County, Hawaii",15,1,21010,3130,2515,230,85,70,4,20,50,55,10,35,0,0,10,110,10,15,50,0,30,245,155,85,4,0,0,275,30,60,135,40,4,55,55,0,0,0,0,1550,170,290,140,190,760,615,40,20,4,0,4,10,65,20,0,0,0,45,50,20,0,0,25,4,55,40,4,0,10,0,100,0,50,45,0,0,70,70,0,0,0,0,235,30,20,40,35,105,176,219,100,44,81,10,32,43,37,13,35,15,15,14,56,23,22,41,15,32,121,112,62,19,15,15,136,38,40,118,34,15,40,40,15,15,15,15,212,104,145,72,107,189,168,26,18,9,15,10,13,77,30,15,15,15,70,40,22,15,15,33,2,35,31,14,15,15,15,55,15,40,44,15,15,86,86,15,15,15,15,111,25,27,36,30,91
2014thru2018,140,14000US15001021101,"Census Tract 211.01, Hawaii County, Hawaii",15,1,21101,1690,1385,125,75,30,0,0,20,35,4,10,10,0,10,4,4,0,0,0,0,255,140,80,35,0,0,125,50,4,20,20,30,30,30,0,0,0,0,810,85,115,115,170,320,300,40,4,0,35,0,0,25,25,0,0,0,0,4,0,0,0,0,4,70,70,0,0,0,0,35,15,10,4,10,0,4,4,0,0,0,0,125,30,10,20,15,50,157,136,73,58,29,11,11,36,29,8,20,15,11,13,3,3,11,11,11,11,78,69,46,34,11,11,40,27,10,14,22,24,23,23,11,11,11,11,117,39,55,61,65,96,98,52,12,11,54,11,11,38,38,11,11,11,11,2,11,11,11,11,2,60,60,11,11,11,11,28,22,13,11,15,11,11,11,11,11,11,11,51,26,13,23,21,36
2014thru2018,140,14000US15001021402,"Census Tract 214.02, Hawaii County, Hawaii",15,1,21402,1340,830,4,0,0,4,0,0,4,0,0,0,0,4,70,0,0,0,10,60,70,30,4,25,4,0,130,10,10,55,0,55,0,0,0,0,0,0,550,10,4,50,30,455,510,30,30,0,0,0,0,10,0,0,10,0,0,15,0,10,0,0,4,115,100,4,4,0,0,85,20,20,15,25,4,0,0,0,0,0,0,250,40,15,35,45,120,103,132,1,11,11,1,11,11,13,11,11,11,11,13,61,11,11,11,19,59,53,46,14,21,2,11,68,16,20,52,11,32,11,11,11,11,11,11,122,17,15,31,30,107,102,50,50,11,11,11,11,13,11,11,13,11,11,21,11,16,11,11,12,74,70,10,13,11,11,51,21,31,22,40,7,11,11,11,11,11,11,92,22,22,33,47,58
2014thru2018,140,14000US15001021800,"Census Tract 218, Hawaii County, Hawaii",15,1,21800,2015,1375,30,25,0,0,0,4,10,0,4,4,4,4,65,0,4,4,4,55,100,35,15,30,0,20,285,25,20,20,15,205,15,15,0,0,0,0,870,15,40,105,95,615,640,55,55,0,4,0,0,15,10,0,0,0,4,25,10,4,0,0,10,55,40,15,0,0,0,100,25,25,35,10,4,0,0,0,0,0,0,395,25,0,50,25,290,201,188,41,40,15,15,15,15,13,15,9,4,5,3,41,15,3,5,1,41,53,34,21,28,15,26,118,25,20,16,20,115,21,21,15,15,15,15,159,16,27,51,40,140,159,80,79,15,15,15,15,17,14,15,15,15,11,24,14,10,15,15,14,37,31,18,15,15,15,53,30,20,42,14,3,15,15,15,15,15,15,162,27,15,34,24,155
2014thru2018,140,14000US15003010201,"Census Tract 102.01, Honolulu County, Hawaii",15,3,10201,1515,785,15,4,0,10,0,0,35,4,0,15,10,4,85,0,4,40,10,30,95,50,10,15,4,10,125,4,15,50,25,35,4,4,0,0,0,0,425,15,4,45,55,305,730,50,10,15,20,0,4,95,25,4,40,15,10,75,0,15,35,4,20,150,55,85,10,0,0,95,10,10,40,25,15,30,30,0,0,0,0,230,4,10,50,45,120,66,73,16,10,15,13,15,15,27,10,15,25,12,9,46,15,9,33,13,22,31,30,13,12,3,17,33,15,13,18,16,15,11,11,15,15,15,15,69,15,11,22,31,52,72,23,11,20,17,15,15,42,21,11,32,14,13,36,15,23,25,15,18,48,28,42,13,15,15,43,14,13,32,22,15,25,25,15,15,15,15,53,10,17,28,23,48
2014thru2018,140,14000US15007040603,"Census Tract 406.03, Kauai County, Hawaii",15,7,40603,1035,595,4,0,0,0,0,4,4,0,0,0,0,4,30,0,0,4,4,25,100,15,20,20,30,20,80,0,15,4,4,55,4,4,0,0,0,0,370,10,20,55,10,275,440,4,0,0,0,0,4,10,10,0,0,0,0,30,4,4,4,10,10,105,55,25,25,0,4,80,4,15,30,15,10,10,10,0,0,0,0,195,15,0,35,35,115,84,70,11,11,11,11,11,11,18,11,11,11,11,18,25,11,11,4,11,26,34,15,16,12,22,13,31,11,14,11,11,29,11,11,11,11,11,11,66,15,16,29,12,55,75,11,11,11,11,11,11,15,15,11,11,11,11,27,3,3,16,13,14,45,26,20,23,11,17,35,15,21,20,13,14,13,13,11,11,11,11,52,16,11,24,25,45
2014thru2018,140,14000US15007040604,"Census Tract 406.04, Kauai County, Hawaii",15,7,40604,1235,655,10,0,0,0,0,10,10,0,0,0,4,4,15,0,4,0,0,10,75,45,4,15,4,0,120,4,15,20,25,55,15,15,0,0,0,0,410,15,30,35,15,315,580,4,4,0,0,0,0,30,0,20,0,0,10,25,4,4,0,4,15,90,15,50,20,0,0,165,35,70,15,20,30,0,0,0,0,0,0,260,30,40,20,25,150,79,96,14,11,11,11,11,14,13,11,11,11,10,1,18,11,10,11,11,13,36,33,10,14,9,11,55,11,22,23,28,32,15,15,11,11,11,11,80,28,30,18,18,74,101,11,11,11,11,11,11,31,11,29,11,11,13,25,7,3,11,10,21,45,18,35,24,11,11,56,28,44,15,23,24,11,11,11,11,11,11,77,27,26,16,22,63
2014thru2018,140,14000US15007040700,"Census Tract 407, Kauai County, Hawaii",15,7,40700,2875,1930,0,0,0,0,0,0,0,0,0,0,0,0,115,0,0,4,0,110,245,80,10,115,15,25,380,80,20,45,70,170,25,25,0,0,0,0,1165,40,20,110,75,920,950,25,0,0,0,4,15,80,0,10,0,10,60,45,0,10,4,0,30,205,90,60,55,0,0,105,20,15,25,0,45,15,15,0,0,0,0,470,25,20,45,100,285,162,199,15,15,15,15,15,15,15,15,15,15,15,15,51,15,15,10,15,52,98,62,17,62,21,30,128,74,29,32,53,83,27,27,15,15,15,15,171,34,17,46,53,179,157,29,15,15,15,13,25,67,15,19,15,21,61,48,15,17,15,15,44,94,64,48,43,15,15,75,23,23,41,15,53,19,19,15,15,15,15,107,18,24,35,72,115
2014thru2018,140,14000US15009030100,"Census Tract 301, Maui County, Hawaii",15,9,30100,500,320,20,10,0,0,0,10,10,0,0,4,0,4,40,4,0,15,0,20,30,25,0,10,0,0,15,0,0,4,0,10,0,0,0,0,0,0,200,4,65,15,20,95,175,4,4,0,0,0,0,10,0,0,0,0,10,20,0,0,0,20,0,20,20,0,0,0,0,4,0,4,0,0,0,0,0,0,0,0,0,120,25,4,10,0,80,71,75,27,18,11,11,11,18,15,11,11,14,11,10,23,11,11,23,11,14,25,19,11,15,11,11,20,11,11,10,11,18,11,11,11,11,11,11,62,11,57,15,23,53,69,11,11,11,11,11,11,16,11,11,11,11,16,25,11,11,11,25,11,24,24,11,11,11,11,9,11,9,11,11,11,11,11,11,11,11,11,62,30,9,12,11,59
2014thru2018,140,14000US15009030201,"Census Tract 302.01, Maui County, Hawaii",15,9,30201,820,605,0,0,0,0,0,0,15,0,0,0,0,15,10,0,0,0,10,0,70,25,35,0,0,10,160,10,10,65,15,60,0,0,0,0,0,0,350,40,40,35,10,225,215,0,0,0,0,0,0,0,0,0,0,0,0,10,0,0,10,0,0,4,0,4,0,0,0,45,0,0,45,0,0,0,0,0,0,0,0,155,0,10,55,0,90,99,113,11,11,11,11,11,11,23,11,11,11,11,23,24,11,11,11,24,11,47,30,31,11,11,17,81,15,18,73,21,31,11,11,11,11,11,11,91,36,34,34,21,80,100,11,11,11,11,11,11,11,11,11,11,11,11,20,11,11,20,11,11,20,11,20,11,11,11,49,11,11,49,11,11,11,11,11,11,11,11,85,11,21,49,11,78
2014thru2018,140,14000US15009030402,"Census Tract 304.02, Maui County, Hawaii",15,9,30402,3140,2205,0,0,0,0,0,0,15,0,0,0,0,15,70,25,0,0,0,45,360,100,85,75,85,20,275,10,0,10,10,245,45,45,0,0,0,0,1435,0,90,150,90,1105,935,0,0,0,0,0,0,0,0,0,0,0,0,75,0,15,15,0,45,95,30,30,40,0,0,220,0,45,90,0,80,0,0,0,0,0,0,540,35,0,85,155,270,167,236,15,15,15,15,15,15,30,15,15,15,15,30,55,39,15,15,15,43,150,74,71,65,62,32,130,19,15,21,21,128,70,70,15,15,15,15,227,15,59,122,67,235,244,15,15,15,15,15,15,15,15,15,15,15,15,83,15,29,36,15,72,67,36,34,48,15,15,141,15,44,98,15,97,15,15,15,15,15,15,182,49,15,97,105,127
2014thru2018,140,14000US15009030800,"Census Tract 308, Maui County, Hawaii",15,9,30800,2250,1810,20,0,0,0,0,20,25,0,4,4,0,20,65,4,0,4,4,60,185,70,30,60,15,15,365,30,15,45,80,195,4,4,0,0,0,0,1140,4,80,50,110,895,445,0,0,0,0,0,0,25,0,10,4,0,10,30,0,0,20,0,10,65,20,15,30,0,0,65,0,10,25,4,25,0,0,0,0,0,0,260,20,35,115,10,85,110,134,27,15,15,15,15,27,19,15,3,8,15,17,38,3,15,4,4,36,70,45,28,36,18,23,120,33,18,31,65,110,11,11,15,15,15,15,166,15,82,41,49,167,132,15,15,15,15,15,15,22,15,17,15,15,15,29,15,15,28,15,17,38,24,19,29,15,15,34,15,17,21,10,32,15,15,15,15,15,15,114,29,31,108,13,40
16 changes: 16 additions & 0 deletions data/data-pipeline/data_pipeline/tests/sources/hud_housing/data/output.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
GEOID10_TRACT,HOUSING_BURDEN_NUMERATOR,HOUSING_BURDEN_DENOMINATOR,Housing burden (percent),Share of homes with no kitchen or indoor plumbing (percent),DENOM INCL NOT COMPUTED
06027000800,370,1376,0.2688953488,0.0471014493,1380
06061021322,985,5265,0.1870845204,0.0000000000,5395
06069000802,136,872,0.1559633028,0.0090909091,880
15001021010,723,2970,0.2434343434,0.0862619808,3130
15001021101,449,1610,0.2788819876,0.0979228487,1685
15001021402,354,1340,0.2641791045,0.0253731343,1340
15001021800,355,2000,0.1775000000,0.0421836228,2015
15003010201,504,1471,0.3426240653,0.0429042904,1515
15007040603,238,1021,0.2331047992,0.0077294686,1035
15007040604,328,1220,0.2688524590,0.0113360324,1235
15007040700,635,2840,0.2235915493,0.0086805556,2880
15009030100,84,491,0.1710794297,0.0484848485,495
15009030201,194,820,0.2365853659,0.0000000000,820
15009030402,555,3095,0.1793214863,0.0000000000,3140
15009030800,385,2251,0.1710350955,0.0088691796,2255
Loading

0 comments on commit bbd78c7

Please sign in to comment.