Skip to content

Commit

Permalink
com uma URL só funciona
Browse files Browse the repository at this point in the history
  • Loading branch information
endersonmaia committed May 21, 2020
1 parent a64b6c3 commit 6bf5360
Showing 1 changed file with 16 additions and 19 deletions.
35 changes: 16 additions & 19 deletions web/spiders/spider_go.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,45 +13,44 @@ class YMDDateField(rows.fields.DateField):
class Covid19GOSpider(BaseCovid19Spider):
name = "GO"
start_urls = [
"http://datasets.saude.go.gov.br/coronavirus/casos_confirmados.csv",
"http://datasets.saude.go.gov.br/coronavirus/obitos_confirmados.csv"
"http://datasets.saude.go.gov.br/coronavirus/casos_confirmados.csv"
# , "http://datasets.saude.go.gov.br/coronavirus/obitos_confirmados.csv"
]

def __init__(self, report_fobj, case_fobj, *args, **kwargs):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.cases = defaultdict(dict)


# self.total_confirmed = 0
# self.total_deaths = 0
# self.imported_confirmed = 0
# self.imported_deaths = 0

def parse(self, response):
table = rows.import_from_csv(
io.BytesIO(response.body),
encoding=response.encoding,
force_types={"data_notificacao": YMDDateField},
)

table = [row for row in table]

last_date = max(row.data_notificacao for row in table)
self.add_report(date=last_date, url=response.url)

row_key = lambda row: row["codigo_ibge"]
row_key = lambda row: row.municipio
table.sort(key=row_key)

for city_id, city_data in groupby(table, key=row_key):
for city, city_data in groupby(table, key=row_key):
if "casos_confirmados.csv" in response.url:
self.cases[city_id]["confirmed"] = len(list(city_data))
self.cases[city]["confirmed"] = len(list(city_data))
elif "obitos_confirmados.csv" in response.url:
self.cases[city_id]["deaths"] = len(list(city_data))
self.cases[city]["deaths"] = len(list(city_data))

def spider_closed(self):
for city_id, city_data in self.cases.items():
total_confirmed = total_deaths = 0
imported_confirmed = imported_deaths = 0

for city, city_data in self.cases.items():
confirmed = city_data["confirmed"]
deaths = city_data.get("deaths", 0)

try:
city = self.get_city_name_from_id(city_id)
self.get_city_id_from_name(city)
except KeyError:
imported_confirmed += confirmed
imported_deaths += deaths
Expand All @@ -70,6 +69,4 @@ def spider_closed(self):
deaths=imported_deaths,
)

self.add_state_case(confirmed=total_confirmed, deaths=total_deaths)

super().spider_closed()
self.add_state_case(confirmed=total_confirmed, deaths=total_deaths)

0 comments on commit 6bf5360

Please sign in to comment.