Adjust for validation (#1848)

usds · Sep 15, 2022 · 68c83b2 · 68c83b2
1 parent 73036b2
commit 68c83b2
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 5 deletions.
diff --git a/data/data-pipeline/data_pipeline/etl/sources/cdc_places/etl.py b/data/data-pipeline/data_pipeline/etl/sources/cdc_places/etl.py
@@ -11,6 +11,7 @@
 class CDCPlacesETL(ExtractTransformLoad):
     NAME = "cdc_places"
     GEO_LEVEL: ValidGeoLevel = ValidGeoLevel.CENSUS_TRACT
+    PUERTO_RICO_EXPECTED_IN_DATA = False
 
     CDC_GEOID_FIELD_NAME = "LocationID"
     CDC_VALUE_FIELD_NAME = "Data_Value"

diff --git a/data/data-pipeline/data_pipeline/etl/sources/historic_redlining/etl.py b/data/data-pipeline/data_pipeline/etl/sources/historic_redlining/etl.py
@@ -10,12 +10,26 @@
 class HistoricRedliningETL(ExtractTransformLoad):
     NAME = "historic_redlining"
     GEO_LEVEL: ValidGeoLevel = ValidGeoLevel.CENSUS_TRACT
+    EXPECTED_MISSING_STATES = [
+        "10",
+        "11",
+        "16",
+        "23",
+        "30",
+        "32",
+        "35",
+        "38",
+        "46",
+        "50",
+        "56",
+    ]
+    PUERTO_RICO_EXPECTED_IN_DATA = False
+    ALASKA_AND_HAWAII_EXPECTED_IN_DATA: bool = False
+    SOURCE_URL = settings.AWS_JUSTICE40_DATASOURCES_URL + "/HRS_2010.zip"
 
     def __init__(self):
         self.CSV_PATH = self.DATA_PATH / "dataset" / "historic_redlining"
-        self.HISTORIC_REDLINING_URL = (
-            settings.AWS_JUSTICE40_DATASOURCES_URL + "/HRS_2010.zip"
-        )
+
         self.HISTORIC_REDLINING_FILE_PATH = (
             self.get_tmp_path() / "HRS_2010.xlsx"
         )

diff --git a/data/data-pipeline/data_pipeline/etl/sources/hud_housing/etl.py b/data/data-pipeline/data_pipeline/etl/sources/hud_housing/etl.py
@@ -6,11 +6,10 @@
 
 
 class HudHousingETL(ExtractTransformLoad):
-    NAME = "HudHousingETL"
+    NAME = "hud_housing"
     GEO_LEVEL: ValidGeoLevel = ValidGeoLevel.CENSUS_TRACT
 
     def __init__(self):
-        self.OUTPUT_PATH = self.DATA_PATH / "dataset" / "hud_housing"
         self.GEOID_TRACT_FIELD_NAME = "GEOID10_TRACT"
         self.HOUSING_FTP_URL = "https://www.huduser.gov/portal/datasets/cp/2014thru2018-140-csv.zip"
         self.HOUSING_ZIP_FILE_DIR = self.get_tmp_path()

diff --git a/data/data-pipeline/data_pipeline/etl/sources/persistent_poverty/etl.py b/data/data-pipeline/data_pipeline/etl/sources/persistent_poverty/etl.py
@@ -21,6 +21,7 @@ class PersistentPovertyETL(ExtractTransformLoad):
 
     NAME = "persistent_poverty"
     GEO_LEVEL: ValidGeoLevel = ValidGeoLevel.CENSUS_TRACT
+    PUERTO_RICO_EXPECTED_IN_DATA = False
 
     def __init__(self):
         self.OUTPUT_PATH = self.DATA_PATH / "dataset" / "persistent_poverty"