diff --git a/examples/README.md b/examples/README.md index b39311b61..15e153f8d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -54,6 +54,7 @@ - [`loader-airtable`](https://observablehq.observablehq.cloud/framework-example-loader-airtable/) - Loading data from Airtable - [`loader-arrow`](https://observablehq.observablehq.cloud/framework-example-loader-arrow/) - Generating Apache Arrow IPC files +- [`loader-census`](https://observablehq.observablehq.cloud/framework-example-loader-census/) - Loading and transform shapefiles from the U.S. Census Bureau - [`loader-databricks`](https://observablehq.observablehq.cloud/framework-example-loader-databricks/) - Loading data from Databricks - [`loader-duckdb`](https://observablehq.observablehq.cloud/framework-example-loader-duckdb/) - Processing data with DuckDB - [`loader-elasticsearch`](https://observablehq.observablehq.cloud/framework-example-loader-elasticsearch/) - Loading data from Elasticsearch diff --git a/examples/loader-census/.gitignore b/examples/loader-census/.gitignore new file mode 100644 index 000000000..e58735f55 --- /dev/null +++ b/examples/loader-census/.gitignore @@ -0,0 +1,4 @@ +.DS_Store +/dist/ +node_modules/ +yarn-error.log diff --git a/examples/loader-census/README.md b/examples/loader-census/README.md new file mode 100644 index 000000000..ca6976aee --- /dev/null +++ b/examples/loader-census/README.md @@ -0,0 +1,7 @@ +[Framework examples →](../) + +# U.S. Census Bureau data loader + +View live: + +This Observable Framework example demonstrates how to write a bash data loader that downloads high-resolution shapefiles from the US Census Bureau, and converts them into a web-friendly format. \ No newline at end of file diff --git a/examples/loader-census/observablehq.config.js b/examples/loader-census/observablehq.config.js new file mode 100644 index 000000000..fb0f92431 --- /dev/null +++ b/examples/loader-census/observablehq.config.js @@ -0,0 +1,3 @@ +export default { + root: "src" +}; diff --git a/examples/loader-census/package.json b/examples/loader-census/package.json new file mode 100644 index 000000000..94f69b5df --- /dev/null +++ b/examples/loader-census/package.json @@ -0,0 +1,24 @@ +{ + "type": "module", + "private": true, + "scripts": { + "clean": "rimraf src/.observablehq/cache", + "build": "rimraf dist && observable build", + "dev": "observable preview", + "deploy": "observable deploy", + "observable": "observable" + }, + "dependencies": { + "@observablehq/framework": "latest", + "shapefile": "^0.6.6", + "topojson-client": "^3.1.0", + "topojson-server": "^3.0.1", + "topojson-simplify": "^3.0.3" + }, + "devDependencies": { + "rimraf": "^5.0.5" + }, + "engines": { + "node": ">=18" + } +} diff --git a/examples/loader-census/src/.gitignore b/examples/loader-census/src/.gitignore new file mode 100644 index 000000000..1235d15eb --- /dev/null +++ b/examples/loader-census/src/.gitignore @@ -0,0 +1 @@ +/.observablehq/cache/ diff --git a/examples/loader-census/src/data/ca.json.sh b/examples/loader-census/src/data/ca.json.sh new file mode 100755 index 000000000..8d8c728fc --- /dev/null +++ b/examples/loader-census/src/data/ca.json.sh @@ -0,0 +1,14 @@ +# Download the ZIP archive from the Census Bureau (if needed). +if [ ! -f src/.observablehq/cache/cb_2023_06_cousub_500k.zip ]; then + curl -o src/.observablehq/cache/cb_2023_06_cousub_500k.zip 'https://www2.census.gov/geo/tiger/GENZ2023/shp/cb_2023_06_cousub_500k.zip' +fi + +# Unzip the ZIP archive to extract the shapefile. +unzip -oqd src/.observablehq/cache src/.observablehq/cache/cb_2023_06_cousub_500k.zip + +# Convert the shapefile to TopoJSON, simplify, and merge counties. +shp2json --encoding utf-8 -n src/.observablehq/cache/cb_2023_06_cousub_500k.shp > src/.observablehq/cache/cb_2023_06_cousub_500k.ndjson + +geo2topo -q 1e5 -n counties=src/.observablehq/cache/cb_2023_06_cousub_500k.ndjson \ + | toposimplify -f -s 1e-7 \ + | topomerge state=counties diff --git a/examples/loader-census/src/index.md b/examples/loader-census/src/index.md new file mode 100644 index 000000000..60b57cea6 --- /dev/null +++ b/examples/loader-census/src/index.md @@ -0,0 +1,57 @@ +# Census boundaries + +Here’s how to fetch high-resolution shapefiles from the [U.S. Census Bureau](https://www.census.gov/geographies/mapping-files/time-series/geo/cartographic-boundary.html) and convert them into a web-friendly format in a bash data loader. + +First, you’ll need to install a few packages: + +```sh +npm install shapefile topojson-client topojson-server topojson-simplify +``` + +Next, here’s a bash script, `ca.json.sh`: + +```bash +# Download the ZIP archive from the Census Bureau (if needed). +if [ ! -f src/.observablehq/cache/cb_2023_06_cousub_500k.zip ]; then + curl -o src/.observablehq/cache/cb_2023_06_cousub_500k.zip 'https://www2.census.gov/geo/tiger/GENZ2023/shp/cb_2023_06_cousub_500k.zip' +fi + +# Unzip the ZIP archive to extract the shapefile. +unzip -oqd src/.observablehq/cache src/.observablehq/cache/cb_2023_06_cousub_500k.zip + +# Convert the shapefile to GeoJSON, then to TopoJSON, simplify, and merge counties. +shp2json --encoding utf-8 -n src/.observablehq/cache/cb_2023_06_cousub_500k.shp > src/.observablehq/cache/cb_2023_06_cousub_500k.ndjson +geo2topo -q 1e5 -n counties=src/.observablehq/cache/cb_2023_06_cousub_500k.ndjson \ + | toposimplify -f -s 1e-7 \ + | topomerge state=counties +``` + +The census.gov URL comes from the Census Bureau page linked above. Here “06” is the state of California’s FIPS code, and “cousub” means county subdivisions. + +And here’s the result displayed with Plot: + +```js echo +Plot.plot({ + width: 640, + height: 720, + projection: { + type: "conic-conformal", + parallels: [37 + 4 / 60, 38 + 26 / 60], + rotate: [120 + 30 / 60, 0], + domain: castate + }, + marks: [ + Plot.geo(castate), + Plot.geo(cacounties, {strokeOpacity: 0.2}) + ] +}) +``` + +```js echo +const ca = FileAttachment("data/ca.json").json(); +``` + +```js echo +const castate = topojson.feature(ca, ca.objects.state); +const cacounties = topojson.mesh(ca, ca.objects.counties, (a, b) => a !== b); +```