Skip to content

Commit

Permalink
Add persistent poverty tests (#1848)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattbowen-usds
mattbowen-usds committed Sep 12, 2022
1 parent 72feee2 commit ebee416
Show file tree
Hide file tree
Showing 7 changed files with 73 additions and 13 deletions.
19 changes: 6 additions & 13 deletions data/data-pipeline/data_pipeline/etl/sources/persistent_poverty/etl.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import pandas as pd

from data_pipeline.config import settings
from data_pipeline.etl.base import ExtractTransformLoad
from data_pipeline.etl.base import ExtractTransformLoad, ValidGeoLevel
from data_pipeline.utils import (
get_module_logger,
unzip_file_from_url,
Expand All @@ -19,6 +19,9 @@ class PersistentPovertyETL(ExtractTransformLoad):
Codebook: `https://s4.ad.brown.edu/Projects/Diversity/Researcher/LTBDDload/Dfiles/codebooks.pdf`.
"""

NAME = "persistent_poverty"
GEO_LEVEL: ValidGeoLevel = ValidGeoLevel.CENSUS_TRACT

def __init__(self):
self.OUTPUT_PATH = self.DATA_PATH / "dataset" / "persistent_poverty"

Expand Down Expand Up @@ -75,7 +78,7 @@ def _join_input_dfs(self, dfs: list) -> pd.DataFrame:
def extract(self) -> None:
logger.info("Starting to download 86MB persistent poverty file.")

unzipped_file_path = self.get_tmp_path() / "persistent_poverty"
unzipped_file_path = self.get_tmp_path()

unzip_file_from_url(
file_url=settings.AWS_JUSTICE40_DATASOURCES_URL
Expand Down Expand Up @@ -155,14 +158,4 @@ def transform(self) -> None:
)
)

self.df = transformed_df

def load(self) -> None:
logger.info("Saving persistent poverty data.")

# mkdir census
self.OUTPUT_PATH.mkdir(parents=True, exist_ok=True)

self.df[self.COLUMNS_TO_KEEP].to_csv(
path_or_buf=self.OUTPUT_PATH / "usa.csv", index=False
)
self.output_df = transformed_df
Empty file added data/data-pipeline/data_pipeline/tests/sources/persistent_poverty/__init__.py
View file
Empty file.
Binary file added ...data-pipeline/data_pipeline/tests/sources/persistent_poverty/data/LTDB_Std_All_Sample.zip
View file
Binary file not shown.
16 changes: 16 additions & 0 deletions data/data-pipeline/data_pipeline/tests/sources/persistent_poverty/data/extract.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
TRTID10,state,county,tract,placefp10,cbsa10,metdiv10,ccflag10,FAMILY90,FHH90,POP90SF3,POP90SF4,RUANC90,ITANC90,GEANC90,IRANC90,SCANC90,RUFB90,ITFB90,GEFB90,IRFB90,SCFB90,FB90,NAT90,N10IMM90,AG5UP90,OLANG90,LEP90,AG25UP90,HS90,COL90,CLF90,UNEMP90,DFLABF90,FLABF90,EMPCLF90,PROF90,MANUF90,SEMP90,AG16CV90,VET90,CNI16U90,DIS90,DPOV90,NPOV90,N65POV90,DFMPOV90,NFMPOV90,DWPOV90,NWPOV90,DBPOV90,NBPOV90,DNAPOV90,NNAPOV90,DHPOV90,NHPOV90,DAPOV90,NAPOV90,INCPC90,HU90SP,H30OLD90,OHU90SP,H10YRS90,DMULTI90,MULTI90,HINC90,HINCW90,HINCB90,HINCH90,HINCA90,HH90,HHW90,HHB90,HHH90,HHA90
6027000800,CA,Inyo County,Census Tract 8,42580,13860,99999,0,947.0000232000,74.0000021900,3473.0000650000,3473.0000650000,37.0000004100,55.0000036300,622.0000106000,362.0000078000,85.0000038300,0.0000000000,2.0000001030,6.0000003420,2.0000000000,6.0000000000,114.0000027000,62.0000010600,34.0000009600,3275.0000600000,283.0000059000,58.0000006200,2628.0000430000,1752.0000340000,226.0000070000,1572.0000260000,128.0000016000,1429.0000340000,692.0000155000,1444.0000240000,269.0000081000,28.0000005800,118.0000038000,2840.0000470000,691.0000141000,2804.0000450000,524.0000085000,3419.0000630000,485.0000113000,114.0000012000,947.0000232000,352.0000117000,3019.0000460000,386.0000059000,15.0000000000,8.0000000000,258.0000152000,87.0000044100,376.0000049000,55.0000017100,34.0000018500,0.0000009240,11282.9890600000,2029.0000420000,765.0000165000,1510.0000340000,960.0000195000,2029.0000420000,164.0000004000,18802.8290100000,19238.2409400000, ,15749.9998600000,15000.0018300000,1520.0000330000,1406.0000270000,0.0000000000,135.0000015000,6.0000008560
6061021322,CA,Placer County,Census Tract 213.22,62938,40900,99999,1,413.8015522000,36.9504473300,1437.7500650000,1437.7500650000,5.7998258990,51.4048475500,323.0300046000,123.0183659000,58.9943210200,0.0000000000,0.0069602750,12.2533182200,0.0000000000,0.0000000000,75.9738778900,37.8148188500,12.5473476100,1364.6021230000,168.7317567000,17.6124934500,929.3841615000,516.7910383000,76.2496196600,700.1886827000,28.5566857300,533.7939340000,287.1322093000,671.6319973000,103.8683825000,80.0058629500,126.2160115000,1089.1046050000,219.7375810000,1081.3416300000,146.6018800000,1429.0499390000,95.1913104400,16.2047431600,413.8015522000,225.9298114000,1319.3738320000,87.5246539400,7.5682855450,0.0000000000,26.4881058700,0.9588502050,148.2951801000,11.1419056600,24.8093601300,6.7078032490,15350.9551100000,532.3409888000,122.7573153000,504.1733706000,330.6022812000,532.3409888000,37.2781012100,37057.6325400000,37560.5137700000,20087.15489,39771.1722300000,42048.1950700000,492.8297963000,457.7224989000,6.7315500680,48.6243697900,0.0646896060
6069000802,CA,San Benito County,Census Tract 8.02,99999,41940,99999,0,566.0214233000,12.1359653500,1951.9486760000,1951.9486760000,9.2233333590,158.2529907000,270.8747559000,162.6219330000,84.4663162200,0.0000000000,4.8543863300,0.0000000000,0.0000000000,2.9126317500,173.3015900000,55.8254394500,83.4954452500,1864.5697250000,333.9817810000,66.0196533200,1427.6749700000,640.2935181000,294.6612549000,935.9256520000,61.6507034300,741.7501831000,351.4575500000,874.2749478000,259.7096558000,121.3596573000,123.3014069000,1577.1900640000,384.4673767000,1498.5490080000,130.0975494000,1866.5114790000,66.9905319200,6.3107018470,566.0214233000,209.2240448000,1679.1321730000,49.5147399900,24.2719306900,0.0000000000,39.3205261200,0.0000000000,256.7970276000,18.9321060200,35.9224586500,0.0000000000,22612.9993700000,791.7503662000,190.2919312000,710.1967163000,505.8270264000,791.7503662000,5.3398246770,43749.9997700000,46405.6612800000, ,32115.3843600000,87500.0022700000,711.6530151000,653.8858032000,0.0000000000,89.8061447100,12.6214036900
15001021010,HI,Hawaii County,Census Tract 210.10,12450,25900,99999,0,771.8966675000,86.6812591600,3130.9272040000,3130.9272040000,26.0043792700,45.0742569000,242.2741241000,138.6900177000,51.1419448900,0.0000000000,0.0000000000,4.3340630530,3.4672505860,0.0000000000,211.9356842000,116.5862961000,49.4083213800,2843.1454120000,355.8265686000,75.4126968400,1971.5653170000,1099.1184080000,332.4226379000,1311.4875030000,83.2140121500,1057.0780030000,588.1323853000,1228.2734910000,259.1769714000,74.5458831800,155.5928650000,2196.9366000000,371.4291992000,2196.9366000000,353.6595459000,3093.6542610000,698.2175903000,46.3744735700,771.8966675000,452.9096069000,1413.3379860000,390.9324951000,15.6026268000,2.1670315270,9.9683456420,7.3679075240,357.9936218000,99.6834487900,1573.2648930000,287.3483887000,10019.0003000000,1394.2680660000,256.5765381000,1128.5900880000,832.1401367000,1394.2680660000,12.1353769300,22281.0001400000,22568.4923400000,15000.00026,22678.5718600000,24593.3011400000,1116.8880620000,606.7688599000,4.3340630530,119.6201401000,455.5100403000
15001021101,HI,Hawaii County,Census Tract 211.01,99999,25900,99999,0,490.9738464000,84.5782699600,2022.9801700000,2022.9801700000,10.4973030100,48.2875900300,143.0632324000,113.6707916000,53.6862030000,0.0000000000,0.0000000000,10.7972259500,2.3993835450,5.3986129760,280.4279480000,180.5536041000,118.4695587000,1845.4257950000,389.8998108000,63.8835830700,1185.8952700000,673.3269653000,161.6584625000,795.6955362000,69.2821960400,712.0170288000,351.8096008000,726.4133391000,128.9668579000,37.7902908300,146.9622345000,1389.5429400000,210.5458984000,1389.5429400000,255.8342590000,2009.1837150000,644.2344360000,38.6900596600,490.9738464000,303.5220032000,909.0664041000,289.4256287000,6.5983042720,0.0000000000,47.9876670800,34.1912155200,234.5397339000,111.5713272000,1009.2406620000,317.3184509000,8204.9999510000,822.6885986000,87.2775726300,681.1250000000,481.6762390000,822.6885986000,24.2937564800,18360.0001000000,18140.0966400000, ,13000.0004000000,19794.8707300000,669.4279785000,370.7047424000,0.0000000000,53.9861259500,273.8296509000
15001021402,HI,Hawaii County,Census Tract 214.02,3850,25900,99999,0,816.2897568000,46.7089718600,2908.3037620000,2908.3037620000,9.3842508490,34.3563535200,85.1050386400,49.6372810000,42.1222216500,0.4366300400,0.0000000000,0.0507011120,0.0000000000,0.0000000000,154.3531404000,35.6412766000,100.6419504000,2718.7213800000,400.5848715000,42.0668509000,2029.9287820000,1120.4883020000,389.2195942000,1513.2753420000,90.2473753700,1164.6149540000,732.8806071000,1423.0279690000,398.0990586000,38.5432898100,148.3798876000,2264.6563310000,400.7801185000,2260.1444870000,204.3723986000,2896.0170940000,164.3064289000,42.8245659800,816.2897568000,369.2554536000,823.5514248000,42.8368821100,9.1951896550,0.2450553770,0.6089137490,0.0000000000,231.1957982000,25.7701943800,2046.6653000000,121.1737876000,15523.0524500000,1168.2462790000,424.7533337000,1042.7740610000,613.2432083000,1168.2462790000,233.6237568000,36432.4851700000,35023.9123700000,2500.00003,19986.2886800000,38377.4267300000,1038.3984520000,350.9512153000,0.0591512960,100.2060835000,672.1332075000
15001021800,HI,Hawaii County,Census Tract 218,99999,25900,99999,0,1029.1142040000,113.6784696000,4267.4235680000,4267.4235680000,6.2395850530,53.9177365400,216.6516714000,128.9012836000,50.9871561900,6.0000000000,0.1063158070,0.1550576070,0.0000000000,5.0000000000,447.7201072000,220.2602014000,108.4859570000,3916.0639380000,732.8903613000,141.4261076000,2614.6461610000,1725.1835680000,326.8660370000,2059.8485030000,80.4418689800,1526.3987780000,994.8898024000,1979.4066340000,257.4833096000,26.3705349100,165.9507309000,3139.4967370000,446.5797471000,3112.4800730000,479.9428353000,4196.1618470000,304.9946221000,35.1246452800,1029.1142040000,599.1987248000,1353.4594670000,88.4248028000,12.0483253700,0.0000000000,29.1300475600,0.0200242130,796.1207250000,80.2921846500,2650.0290710000,198.5235648000,11456.1517100000,1564.6212330000,673.4738505000,1367.9637000000,704.9298707000,1564.6212330000,108.9002704000,31093.0763300000,34891.6831500000, ,36029.4523900000,28993.2036700000,1309.0228300000,454.9163605000,0.0000000000,162.9089448000,789.8354598000
15003010201,HI,Honolulu County,Census Tract 102.01,12400,26180,99999,0,1066.0000000000,123.0000000000,4656.0000160000,4656.0000160000,27.0000000000,106.0000000000,304.0000000000,147.0000000000,83.0000000000,0.0000000000,0.0000000000,20.0000000000,0.0000000000,0.0000000000,309.0000000000,104.0000000000,102.0000000000,4143.0000140000,719.0000000000,44.0000000000,2719.0000090000,1527.0000000000,528.0000000000,2131.0000070000,117.0000000000,1555.0000000000,895.0000000000,2014.0000070000,545.0000000000,67.0000000000,127.0000000000,3303.0000110000,535.0000000000,3267.0000110000,453.0000000000,4587.0000150000,729.0000000000,52.0000000000,1066.0000000000,633.0000000000,1662.0000060000,204.0000000000,42.0000000000,0.0000000000,43.0000000000,28.0000000000,261.0000000000,18.0000000000,2743.0000000000,497.0000000000,14178.0000500000,1876.0000000000,634.0000000000,1494.0000000000,1037.0000000000,1876.0000000000,447.0000000000,35151.0001200000,36678.3218000000,30000.0001,44583.3334800000,36453.4884900000,1462.0000000000,656.0000000000,9.0000000000,23.0000000000,763.0000000000
15007040603,HI,Kauai County,Census Tract 406.03,39200,28180,99999,0,588.9533691000,14.8749074900,2288.4115600000,2288.4115600000,4.1835675240,41.8356781000,141.7764587000,88.7846069300,45.5544052100,0.0000000000,4.1835675240,5.1132493020,0.0000000000,0.0000000000,315.6269531000,189.6550751000,98.0814209000,2138.7328030000,491.8016357000,87.8549194300,1520.0296170000,716.3197632000,270.5373840000,1197.8949000000,18.1287937200,877.6195679000,518.2975464000,1179.7661060000,306.7949829000,46.4840850800,99.0111007700,1745.9422750000,233.3501129000,1714.7979380000,137.1280518000,2260.0562680000,153.8623199000,23.7068843800,588.9533691000,321.6698608000,921.7794281000,104.1243515000,0.0000000000,0.0000000000,5.1132493020,0.0000000000,238.9282074000,29.2849750500,1265.2968750000,33.0037002600,14882.0000400000,836.7135620000,238.4633636000,753.0421753000,469.9541016000,836.7135620000,116.6750565000,38942.0014900000,37622.3762200000, ,34318.1815500000,38219.1778500000,761.4093018000,345.8416138000,0.0000000000,55.3160629300,393.2553711000
15007040604,HI,Kauai County,Census Tract 406.04,57800,28180,99999,0,678.0467656000,17.1250984800,2634.5889580000,2634.5889580000,4.8164320010,48.1643366100,163.2235751000,102.2154066000,52.4456040000,0.0000000000,4.8164319990,5.8867523410,0.0000000000,0.0000000000,363.3731129000,218.3449656000,112.9185931000,2462.2676790000,566.1984698000,101.1450919000,1749.9707180000,824.6804260000,311.4626748000,1379.1053670000,20.8712132900,1010.3806350000,596.7025865000,1358.2341530000,353.2050822000,53.5159359900,113.9889124000,2010.0581130000,268.6499275000,1974.2024500000,157.8719865000,2601.9442500000,177.1377060000,27.2931263100,678.0467656000,370.3301829000,1061.2207750000,119.8756585000,0.0000014800,0.0000009040,5.8867517830,0.0000000055,275.0718744000,33.7150284500,1456.7034330000,37.9963147800,14881.9998700000,963.2866173000,274.5367055000,866.9579964000,541.0460023000,963.2866173000,134.3249464000,38942.0019300000,37622.3794400000,2708.021126,34318.1816500000,38219.1788500000,876.5908071000,398.1584626000,0.0000004130,63.6839504200,452.7447185000
15007040700,HI,Kauai County,Census Tract 407,24950,28180,99999,0,1639.5124120000,84.3661893400,6497.4160920000,6497.4160920000,0.0261385850,179.0821346000,411.3031126000,164.1830575000,112.0709766000,0.0000000000,0.0000000000,19.9999980900,0.0000000000,0.0000000000,803.9005321000,495.4262798000,170.9395295000,6045.6744670000,1284.0984150000,230.2267010000,4202.3582790000,2302.9328280000,716.3603780000,3341.4274500000,85.2877002200,2471.2365200000,1616.8583390000,3256.1397500000,792.3786741000,256.5087146000,160.2696385000,4873.0529120000,677.1858005000,4858.6607240000,465.8659478000,6483.8447060000,315.4761577000,130.1721194000,1639.5124120000,903.8070090000,2542.4494830000,121.2988364000,18.0560093400,10.9999990500,13.2091077300,0.0672135060,809.4964124000,19.4300633300,3747.7267120000,182.9719543000,14847.6424500000,2182.2980030000,656.0313461000,2089.1407680000,1264.6352900000,2182.2980030000,127.4408560000,42756.8743900000,41907.5748200000,2708.021223,33400.6344500000,42210.6869300000,2068.2044520000,930.0281326000,5.0261381080,162.4411862000,1090.9635380000
15009030100,HI,Maui County,Census Tract 301,11350,27980,99999,0,383.3068000000,27.2305791200,1910.5015780000,1910.5015780000,6.2592231110,27.4607778700,91.6621148600,27.1932371400,35.6747865500,0.0000000000,0.0035966930,0.0898007360,0.0000000000,8.1204749640,37.2918099000,6.5858333300,8.6301826290,1675.7861590000,177.6883274000,0.2490721380,1213.4786900000,793.3024273000,173.2832380000,864.0971374000,12.1702029000,576.9324621000,348.0594975000,851.9269343000,157.0642045000,28.2470664600,176.1750701000,1347.1787240000,314.6769609000,1347.1232320000,153.7392875000,1890.3938120000,392.8893720000,49.0661406900,383.3068000000,238.4715474000,807.4789838000,151.7785114000,0.0395477110,0.0000000000,20.0771960800,19.9998645800,82.6743149300,0.0311889900,1052.6142140000,211.1110742000,10415.9855900000,772.7004868000,261.4311255000,595.1029364000,362.3008839000,772.7004868000,23.2643179700,26030.6739600000,26529.6239500000,62499.99972,10011.7479600000,27536.6825200000,634.0477989000,310.2452140000,0.0041105070,14.1661638200,316.7386816000
15009030201,HI,Maui County,Census Tract 302.01,99999,27980,99999,0,370.6344604000,44.2015914900,1578.6282440000,1578.6282440000,14.5508346600,25.8071403500,170.2173004000,68.9105529800,56.8306160000,0.0000000000,0.0000000000,1.9218082430,0.0000000000,0.0000000000,74.4014358500,42.8288688700,32.6707420300,1447.3961910000,143.5865326000,10.1581296900,1008.1257240000,471.1175842000,209.7516479000,834.6138888000,21.9635238600,543.5972290000,359.1036072000,812.6503654000,172.4136658000,95.5413284300,161.4319000000,1135.7887030000,143.0374451000,1135.7887030000,111.7394257000,1568.4701140000,158.1373749000,6.8636012080,370.6344604000,243.5205688000,939.4897130000,125.1920853000,3.0199844840,0.0000000000,20.3162593800,8.5108652110,190.2590179000,18.6689949000,572.1497803000,21.1398906700,13734.0000100000,569.6788940000,110.0921631000,526.5755005000,365.9671936000,569.6788940000,19.2180824300,36271.9983800000,36752.7165200000,29999.99935,38274.6483700000,41103.4484300000,537.2827148000,379.9689636000,1.3727202420,52.9869995100,135.0756683000
15009030402,HI,Maui County,Census Tract 304.02,65900,27980,99999,0,1576.0000000000,169.0000000000,6064.0000020000,6064.0000020000,14.0000000000,21.0000000000,388.0000000000,130.0000000000,116.0000000000,0.0000000000,6.0000000000,12.0000000000,0.0000000000,0.0000000000,384.0000000000,204.0000000000,142.0000000000,5577.0000020000,733.0000000000,120.0000000000,3752.0000010000,1692.0000000000,664.0000000000,3218.0000010000,94.0000000000,2147.0000000000,1431.0000000000,3124.0000010000,752.0000000000,197.0000000000,253.0000000000,4402.0000010000,742.0000000000,4402.0000010000,325.0000000000,6036.0000020000,397.0000000000,22.0000000000,1576.0000000000,970.0000000000,2603.0000010000,182.0000000000,0.0000000000,0.0000000000,65.0000000000,0.0000000000,496.0000000000,63.0000000000,3305.0000000000,215.0000000000,16362.0000100000,1995.0000000000,159.0000000000,1914.0000000000,1194.0000000000,1995.0000000000,22.0000000000,43032.0000100000,41381.5789600000, ,33000.0000100000,43802.6315900000,1919.0000000000,922.0000000000,0.0000000000,152.0000000000,973.0000000000
15009030800,HI,Maui County,Census Tract 308,75950,27980,99999,0,566.0331867000,29.0024671100,2380.1187810000,2380.1187810000,17.9996613400,10.0023785700,58.0117214400,83.0014279100,6.0035655440,0.0000246000,0.0000492000,0.0000246000,0.0000000000,0.0000871000,275.9955389000,160.9963087000,63.0011419100,2200.1173260000,563.9967822000,125.9982522000,1366.1198620000,676.0562812000,278.0319260000,1075.1001440000,14.0013287000,774.0661832000,485.0454395000,1061.0988150000,250.0390396000,61.0059311800,72.0103169500,1604.1293560000,165.0293852000,1604.1262590000,126.0096916000,2373.1152960000,158.9950915000,16.9988727100,566.0331867000,349.0062412000,543.0424034000,75.9953928000,19.9996131500,0.0000000000,0.0005046810,0.0000416000,204.9941585000,6.0003820650,1744.0754940000,82.9987536900,13742.8599100000,725.0676672000,157.0406731000,678.0611339000,390.0359396000,725.0676672000,15.0405894400,43750.2909800000,37947.0558900000,56651.51814,40624.8900800000,47291.6931700000,623.0686622000,167.0220174000,0.0003813110,27.9999225900,438.0467044000
16 changes: 16 additions & 0 deletions data/data-pipeline/data_pipeline/tests/sources/persistent_poverty/data/output.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
GEOID10_TRACT,Individuals in Poverty (percent) (1990),Individuals in Poverty (percent) (2000),Individuals in Poverty (percent) (2010),Persistent Poverty Census Tract
06027000800,0.1418543441,0.1857745998,0.1475304682,False
06061021322,0.0666116053,0.0607935935,0.0947959404,False
06069000802,0.0358907688,0.0632887874,0.0613686534,False
15001021010,0.2256934781,0.2156182598,0.3112706837,True
15001021101,0.3206448625,0.2732381146,0.2703479094,True
15001021402,0.0567353105,0.1106101497,0.0918238994,False
15001021800,0.0726841893,0.1204888825,0.1765701710,False
15003010201,0.1589274030,0.1844549763,0.1866137266,False
15007040603,0.0680789775,0.1103461130,0.0346287033,False
15007040604,0.0680789782,0.1103461036,0.1002400960,False
15007040700,0.0486557239,0.0635433492,0.1525797252,False
15009030100,0.2078346689,0.1729195461,0.0979676915,False
15009030201,0.1008226892,0.1610559432,0.1041162228,False
15009030402,0.0657720344,0.0643312931,0.0270912548,False
15009030800,0.0669984690,0.0322814539,0.0189065700,False
Loading

0 comments on commit ebee416

Please sign in to comment.