-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.html
543 lines (462 loc) · 17.2 KB
/
index.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
<html>
<head>
<!--
Amazon S3 Bucket listing.
Copyright (C) 2008 Francesco Pasqualini
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
-->
<title>COVID-19 Case Data Scraped from NJ.com</title>
<SCRIPT>
//document.write(window.location.search);
function getSpace(s,l){
var ret = "";
while(s.length+ret.length<l){
ret = ret + " ";
}
return ret;
}
location.querystring = (function() {
// The return is a collection of key/value pairs
var result = {};
// Gets the query string with a preceeding '?'
var querystring = location.search;
// document.location.search is empty if a query string is absent
if (!querystring)
return result;
// substring(1) to remove the '?'
var pairs = querystring.substring(1).split("&");
var splitPair;
// Load the key/values of the return collection
for (var i = 0; i < pairs.length; i++) {
splitPair = pairs[i].split("=");
result[splitPair[0]] = splitPair[1];
}
return result;
})();
function createRequestObject(){
var request_o; //declare the variable to hold the object.
var browser = navigator.appName; //find the browser name
if(browser == "Microsoft Internet Explorer"){
/* Create the object using MSIE's method */
request_o = new ActiveXObject("Microsoft.XMLHTTP");
}else{
/* Create the object using other browser's method */
request_o = new XMLHttpRequest();
}
return request_o; //return the object
}
/* You can get more specific with version information by using
parseInt(navigator.appVersion)
Which will extract an integer value containing the version
of the browser being used.
*/
/* The variable http will hold our new XMLHttpRequest object. */
var http = createRequestObject();
function getList(){
http.open('get', location.protocol+'//'+location.hostname);
http.onreadystatechange = handleList;
http.send(null);
}
function handleList(){
/* Make sure that the transaction has finished. The XMLHttpRequest object
has a property called readyState with several states:
0: Uninitialized
1: Loading
2: Loaded
3: Interactive
4: Finished */
if(http.readyState == 4){ //Finished loading the response
/* We have got the response from the server-side script,
let's see just what it was. using the responseText property of
the XMLHttpRequest object. */
var response = http.responseXML;
filex = response.getElementsByTagName('Contents');
res = '';
fileList = new Array();
for(i=0; i<filex.length; i++){
if(filex[i].getElementsByTagName('Key')[0].firstChild.data == 'index.html'){continue;}
fileData =new Array();
fileList[i] = fileData;
size = filex[i].getElementsByTagName('Size')[0].firstChild.data;
name = filex[i].getElementsByTagName('Key')[0].firstChild.data;
lastmod = filex[i].getElementsByTagName('LastModified')[0].firstChild.data;
link = "<A HREF=\""+name+"\">"+name+"</A>";
fileData[0] = name;
fileData[1] = size;
fileData[2] = lastmod;
fileData[3] = link;
}
fileList.sort(getSort());
//document.write(getSort());
for(i=0; i<fileList.length; i++){ //length is the same as count($array)
fileData = fileList[i];
name = fileData[0];
size = fileData[1];
lastmod = fileData[2];
link = fileData[3];
res = res + getSpace(size,15) +size + " B ";
res = res + " "+ getSpace(lastmod,20)+ lastmod + " ";
res = res + " "+ link+ getSpace(name,50) + " ";
res = res + "<BR>";
}
document.getElementById('bucket_list').innerHTML = "<PRE>"+getLink()+"<BR>"+res+"</PRE>" ;
}
}
function getQueryVariable(variable) {
var query = window.location.search.substring(1);
var vars = query.split("&");
for (var i=0;i<vars.length;i++) {
var pair = vars[i].split("=");
if (pair[0] == variable) {
return pair[1];
}
}
return null;
}
function sortSize(a,b) {
if(parseInt(a[1]) > parseInt(b[1])) return 1;
if(parseInt(a[1]) < parseInt(b[1])) return -1;
return 0;
}
function sortSizeDesc(a,b) { return (-sortSize(a,b)); }
function sortLastmod(a,b) {
if(a[2] > b[2]) return 1;
if(a[2] < b[2]) return -1;
return 0;
}
function sortLastmodDesc(a,b) { return (-sortLastmod(a,b)); }
function sortName(a,b) {
if(a[0] > b[0]) return 1;
if(a[0] < b[0]) return -1;
return 0;
}
function sortNameDesc(a,b) { return -sortName(a,b); }
//document.write('http://'+location.hostname);
function getSort(){
var s = getQueryVariable("sort");
var d = getQueryVariable("sortdir");
if(s=='size'){ return d == 'desc' ? sortSizeDesc : sortSize};
if(s=='name'){ return d == 'desc' ? sortNameDesc : sortName};
if(s=='lastmod'){ return d == 'desc' ? sortLastmodDesc : sortLastmod};
return sortName;
}
function getLink(){
return " "+getLinkSize() + " " + getLinkLastmod() + " " + getLinkName() + " " ;
}
function getNextSortDir(sortCol){
if (sortCol == getQueryVariable("sort"))
return getQueryVariable("sortdir") == 'desc' ? 'asc' : 'desc';
return 'asc'
}
function getLinkSize(){
return "<A HREF=\"?sort=size&sortdir=" +getNextSortDir('size') +"\">Size</A>";
}
function getLinkName(){
return "<A HREF=\"?sort=name&sortdir=" +getNextSortDir('name') +"\">Name</A>";
}
function getLinkLastmod(){
return "<A HREF=\"?sort=lastmod&sortdir=" +getNextSortDir('lastmod') +"\">Lastmodified</A>";
}
</SCRIPT>
<style>
@import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,300;0,400;0,700;1,400;1,700&display=swap');
@import url('https://fonts.googleapis.com/css2?family=B612+Mono&display=swap');
body {
font-family: 'Lato', sans-serif;
}
pre,.code {
font-family: 'B612 Mono', monospace;
font-size: smaller;
}
h1 {
margin: -8px -8px 0px;
font-weight: 500;
background-color: lightslategray;
color: white;
padding: 20px
}
h2, h3 {
font-weight: 500;
text-align: center;
}
h3 {
min-width: 200px;
width: 200px !important;
margin: 20px auto 10px;
padding: 0px 30px;
background-color: #ecd6c7;
border: #d3a688 1px solid;
border-radius: 15px;
}
.footer {
margin: 20px 0px;
}
.tableauPlaceholder {
position: relative;
margin: 20px auto;
border: 2px #000 solid;
border-radius: 15px;
}
.bucket_list {
border: 1px #000 solid;
border-radius: 5px;
margin: 10px auto;
padding: 10px;
background-color: rgb(230, 230, 230);
overflow: auto;
}
.table_header {
font-weight: 700;
}
/* Mobile sizes */
h2 {
min-width: 200px;
width: 200px !important;
padding: 10px 30px;
background-color: rgb(199, 215, 236);
border: rgb(176, 196, 222) 1px solid;
border-radius: 30px 280px;
}
.tableauPlaceholder {
max-width: 95%;
}
.text{
max-width: 100%;
margin: 0 auto;
padding: 10px;
}
.bucket_list {
max-width: 100%;
}
@media only screen and (min-width: 1100px) {
/* Desktop sizes */
h2 {
max-width: 90%;
margin: 10px auto;
}
.tableauPlaceholder {
max-width: 70%;
}
.text{
max-width: 80%;
margin: 0 auto;
}
.bucket_list {
max-width: 65%;
}
}
</style>
</head>
<body onLoad="getList();">
<h1><strong>Project NJcomCOVIDData</strong>
<br>
<small>Extracting COVID-19 Case Counts from NJ.com</small>
</h1>
<h2>Functionality</h2>
<div class="text">
<p>
This code scrapes the data from <a href="https://www.nj.com/coronavirus/2020/04/where-is-the-coronavirus-in-nj-latest-map-update-on-county-by-county-cases-april-16-2020.html" target="_blank">NJ.com</a>
on a daily basis (2 PM EST).
The data is parsed to extract the <strong>City, County, Cases of
COVID-19, as well as any reported deaths and recoveries</strong>.
</p>
<p>
In my opinion the data set is
most likely not as robust and accurate as would be ideal. For example, counties have stopped
reporting city level data since this project was created, while other counties have never
provided a break down.
</p>
</div>
<h2>Generated Data</h2>
<div class="text">
<strong>
The current day's data is below. Older files are in the <a href="#archive">Archive</a>.
</strong>
These files can be used as a constant reference to provide data to a Google Sheet with
<span class="code">=IMPORTDATA("https://athenedyne-covid-19.s3.amazonaws.com/Active/current-complete.csv")</span>.
There appears to be no equivalent Excel function.
</div>
<div id="active_list" class="bucket_list">
<table>
<tr class = "table_header">
<td width = 250px>File</td><td>Headers</td>
</tr>
<tr>
<td class = "code"><a href="Active/current-cases.csv">current-cases.csv</a></td>
<td>Zip Code, City, Cases</td>
</tr>
<tr>
<td class = "code"><a href="Active/current-complete.csv">current-complete.csv</a></td>
<td>County, City, Cases, Deaths, Recoveries, Zip Code, Shared ZIPs, Adjusted Cases</td>
</tr>
<tr>
<td class = "code"><a href="Active/current-zips.csv">current-zips.csv</a></td>
<td>Zip Code, Cases</td>
</tr>
</table>
</div>
<h3>Interactive Tableau Map</h3>
<div class="text">
The main dashboard compares four different layouts and has multiple views accessible via the tabs:
<ul>
<li>A heat map</li>
<li>A county summary map</li>
<li>A ZIP map where ZIPs all have the same number of cases if a town is shared. E.g., Newark has 22 ZIP codes
and each ZIP code has the same number of cases. This better illustrates large cities as hot spots but overcounts
the total number of cases and hides other hotspots.
</li>
<li>A ZIP map where ZIPs evenly share number of cases if a town is shared. E.g., Newark has 22 ZIP codes
and each ZIP code has the total reported number of cases for Newark divided by 22.
This better illustrates smaller city hot spots (Lakewood, North Bergen, Passiac, Woodbridge) but does not overcount
the total number of cases.
</li>
</ul>
</div>
<div class='tableauPlaceholder' id='viz1587357055533' style='position: relative'>
<noscript>
<a href='#'>
<img alt=' ' src='https://public.tableau.com/static/images/PP/PPWHHHDCR/1_rss.png' style='border: none' />
</a>
</noscript>
<object class='tableauViz' style='display:none;'>
<param name='host_url' value='https%3A%2F%2Fpublic.tableau.com%2F' />
<param name='embed_code_version' value='3' />
<param name='path' value='shared/PPWHHHDCR' />
<param name='toolbar' value='yes' />
<param name='static_image' value='https://public.tableau.com/static/images/PP/PPWHHHDCR/1.png' />
<param name='animate_transition' value='yes' />
<param name='display_static_image' value='yes' />
<param name='display_spinner' value='yes' />
<param name='display_overlay' value='yes' />
<param name='display_count' value='yes' />
<param name='filter' value='publish=yes' />
</object>
</div>
<script type='text/javascript'>
var divElement = document.getElementById('viz1587357055533');
var vizElement = divElement.getElementsByTagName('object')[0];
if ( divElement.offsetWidth > 800 ) {
vizElement.style.width='100%';
vizElement.style.height=(divElement.offsetWidth*0.75)+'px';
}
else if ( divElement.offsetWidth > 500 ) {
vizElement.style.width='100%';
vizElement.style.height=(divElement.offsetWidth*0.75)+'px';
}
else {
vizElement.style.width='100%';
vizElement.style.minHeight='1650px';
vizElement.style.maxHeight=(divElement.offsetWidth*1.77)+'px';
}
var scriptElement = document.createElement('script');
scriptElement.src = 'https://public.tableau.com/javascripts/api/viz_v1.js';
vizElement.parentNode.insertBefore(scriptElement, vizElement);
</script>
<h2>Data</h2>
<div class="text">
<p>
The extracted data is then formatted and joined with a ZIP list using City and County to
handle cities that share a name (i.e., Franklin is a town in Sussex, Gloucester, Warren,
Hunterdon and Somerset). The data is also aggregated by ZIP to deal with cities that share
ZIP Codes. For example, 08053 zones for Marlton (recommended) and Evesham (recognized).
It also zones for
<ul>
<li>EVESBORO NJ</li>
<li>EVESHAM TWP NJ</li>
<li>KRESSON NJ</li>
<li>MARLTON LAKES NJ</li>
<li>NORTH MARLTON NJ</li>
<li>PINE GROVE NJ</li>
</ul>
</p>
<p>
To explore this further take a look at the
<a href="https://tools.usps.com/zip-code-lookup.htm?citybyzipcode" target="_blank">USPS</a>,
ZIP Code lookup tool.
</p>
<p>
For cities that have more than one ZIP like Newark, Camden, Edison, etc. it appears the
default behavior is to join the cases, deaths, and recoveries data to each of the ZIPs.
<strong>This is a gotcha for aggregation</strong>. The ZIP version of the output drops duplicate (City,
County) tuples and keeps the first. The values are then safely summed to provide a total
per ZIP. Additionally the complete CSVs use the number of ZIPs per (City, County) tuple
and the provide the Adjusted Cases by dividing Cases by Shared ZIPs.
</p>
</div>
<h2>Output</h2>
<div class="text">
The data is then exported as three variants to an
<a href="https://athenedyne-covid-19.s3.amazonaws.com/index.html" target="_blank">AWS S3 Bucket</a>
and they are sorted by Folder to keep each type together and sorted by date.
The variants are:
<ol>
<li><span class="code">MM-DD-YYYY-complete.csv</span> has County, City, Cases, Deaths, Recoveries, Zip Code, Shared
ZIPs, Adjusted Cases. These are in the <strong>Complete folder</strong>.</li>
<li><span class="code">MM-DD-YYYY-cases.csv</span> has Zip Code, City, Cases. These are in the <strong>Cases folder</strong>.</li>
<li><span class="code">MM-DD-YYYY-zips.csv</span> has Zip Code, Cases aggregate. These are in the <strong>ZIPs folder</strong>.</li>
<li><span class="code">MM-DD-YYYY-missing-ZIPs.csv</span> has the empty Zip Code column, City, County. This is
only written if any ZIPs are missing from the master <span class="code">NJzips.csv</span> file. These are in the
<strong>MissingZIPs folder</strong>.</li>
</ol>
<p>
Additionally, the code in the <a href="NJcomCOVIDData_lambda.py" target="_blank">.py</a> creates copies of the
above variants starting with current. This allows bookmarking to the current file or using
the data for a visualization analysis, or other use. These are together in the <strong>Active folder</strong>.
</p>
<p>
Using Google Sheets and the import data function
(<span class="code">=IMPORTDATA("https://athenedyne-covid-19.s3.amazonaws.com/Active/current-complete.csv")</span>), one
can connect to Tableau to create the <a href="https://public.tableau.com/profile/andrew.k.decotiis.mauro#!/vizhome/COVID-19CasesinNJusingNJ_comData/Choropleth-County" target="">interactive viz</a> above that stays up to date:
</p>
</div>
<h2>Automation</h2>
<div class="text">
The Lambda script is set up on AWS Lambda to run at 2 PM EST daily, updating the
S3 Bucket. It was originally scheduled for Noon but the article is not necessarily ready
in time.
</div>
<h2>Archive</h2><a id="archive"></a>
<div class="text">
<strong>
The default organization is by name. Types of documents are contained in the same folder to keep them grouped.
</strong>
</div>
<div id="bucket_list" class="bucket_list">
<!--This is where we'll be displaying the products once they're loaded-->
</div>
<h2>Additional Files</h2>
<div class="text">
The ZIPs were collected into NJzips.csv which is somewhat complete, as it may be missing
ZIPs for towns that share ZIPs. It does contain ZIPs for towns on the page as of 04/25/2020.
The basis of the index.html file (the one you're reading now) is from an answer on the
<a href="https://forums.aws.amazon.com/thread.jspa?threadID=66482">AWS forum</a> by J. Patel on
5/3/2011. The pages uses JS to generate HTML text to list all of the files in the S3 bucket.
I modified it to skip index.html.
</div>
<h2>Additional Notes</h2>
<div class="text">
This repo can be used locally <a href="NJcomCOVIDExtract.ipynb" target="_blank">.ipynb</a> recommended) or as an AWS
Lambda (when zipped with dependencies). When used as a Lambda it's helpful to know that
AWS is running a Linux variant so the pandas and numpy libraries will need to be for a
Linux system. I'm running MacOS X so I needed to source my libraries from PyPi:
<ul>
<li><a href="https://pypi.org/project/pandas/#files" target="_blank">pandas</a> - <span class="code">pandas-1.0.3-cp37-cp37m-manylinux1_x86_64.whl</span></li>
<li><a href="https://pypi.org/project/numpy/#files" target="_blank">numpy</a> - <span class="code">numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl</span></li>
</ul>
This is laid out really well in <a href="https://medium.com/@korniichuk/lambda-with-pandas-fd81aa2ff25e" target="_blank">AWS Lambda with Pandas and NumPy</a>
by <a href="https://medium.com/@korniichuk" target="_blank">Ruslan Korniichuck</a>.
</div>
<div class="footer">
</div>
</body>
</html>