03_geocode.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bed19c7b-1917-4e50-be96-7e72024f8bec",
   "metadata": {},
   "source": [
    "# Geocode\n",
    "\n",
    "Map the standardized dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6acf4f5-da69-4582-ac54-f70c31dad929",
   "metadata": {},
   "source": [
    "Import Python tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "72e90968-d52a-4c26-9577-db8adb8bb5c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import json\n",
    "import pandas as pd\n",
    "from googlegeocoder import GoogleGeocoder\n",
    "from geojson import Feature, Point, FeatureCollection\n",
    "from googlegeocoder import GeocoderResult, AddressComponent, Geometry, Coordinates, Bounds"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6fb5270-6648-4130-bbf6-38ef0a8c72e1",
   "metadata": {},
   "source": [
    "Read in standardized data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1f3b5433-92dd-4d0e-b756-3269f9e9318a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\n",
    "    \"output/trees.csv\",\n",
    "    parse_dates=[\"date\"],\n",
    "    dtype={\"ward\": str}\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25287df5-e819-46d1-8fa0-e501e3802ec5",
   "metadata": {},
   "source": [
    "Pull out unique addresses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4be1e24b-1fd6-471e-b8fa-e914b8fd6456",
   "metadata": {},
   "outputs": [],
   "source": [
    "address_df = pd.DataFrame(df['address'].unique(), columns=[\"address\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "df7e8fee-902e-4e9e-a159-851923385249",
   "metadata": {},
   "outputs": [],
   "source": [
    "address_list = list(address_df.address)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47d511a7-b2be-4fc0-83ca-898bdcb87573",
   "metadata": {},
   "source": [
    "Initialize the Google geocoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "72a8c596-5c93-4b53-9615-2a34bba71ddd",
   "metadata": {},
   "outputs": [],
   "source": [
    "geocoder = GoogleGeocoder(os.getenv(\"GOOGLE_GEOCODER_API_KEY\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2cfdb55-bfde-4c69-ad13-eeeb28ff0c3e",
   "metadata": {},
   "source": [
    "Load any addresses that have been previously geocoded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "39221276-8b6e-49f0-9820-6c6ca002c814",
   "metadata": {},
   "outputs": [],
   "source": [
    "geocoding_cache = json.load(open(\"output/geocoding.json\", \"r\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67f92632-8c03-4a0a-a873-1c4d995ac635",
   "metadata": {},
   "source": [
    "Filter down to addresses that haven't already been geocoded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fba8b1b8-b77e-463e-b351-4f4e659f730a",
   "metadata": {},
   "outputs": [],
   "source": [
    "unmapped_list = [a for a in address_list if a not in geocoding_cache and not pd.isnull(a)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b541b11b-4558-462c-a99a-412fb6ec0de3",
   "metadata": {},
   "source": [
    "Loop through them, geocode them and add them to the cache."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6ff72e78-7a54-4dfe-ba79-c7a7c3d4bd34",
   "metadata": {},
   "outputs": [],
   "source": [
    "for address in unmapped_list:\n",
    "    try:\n",
    "        search = geocoder.get(f\"{address}, Chicago, IL\")\n",
    "    except:\n",
    "        continue\n",
    "    geocoding_cache[address] = search\n",
    "    time.sleep(0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f850c0b0-7d33-49be-aabd-088790ea79a2",
   "metadata": {},
   "source": [
    "Write out the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "af48dbe5-7248-4816-b078-1887b6bfb5cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ComplexEncoder(json.JSONEncoder):\n",
    "    def default(self, obj):\n",
    "        if isinstance(obj, (GeocoderResult,AddressComponent, Geometry, Coordinates, Bounds)):\n",
    "            return obj.__dict__\n",
    "        return json.JSONEncoder.default(self, obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "bb9468b2-26c0-4d26-bfd2-ad595d6dc4e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "json.dump(\n",
    "    geocoding_cache,\n",
    "    open(\"output/geocoding.json\", \"w\"),\n",
    "    cls=ComplexEncoder,\n",
    "    indent=2\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbf9d7aa-092d-413f-8f13-160df23c8d7a",
   "metadata": {},
   "source": [
    "Convert the results to geojson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "368c8544-e73c-463e-a71f-1d2a77865de0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_geojson(address, obj):\n",
    "    \"\"\"\n",
    "    Convert Google Geocoder result to GeoJSON\n",
    "    \"\"\"\n",
    "    geom = obj['geometry']['location']\n",
    "    return Feature(\n",
    "        geometry=Point((geom['lng'], geom['lat'])),\n",
    "        properties={\n",
    "            'address': address,\n",
    "        }\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "43371426-6483-470a-8375-931c76dc674d",
   "metadata": {},
   "outputs": [],
   "source": [
    "geocoding_cache = json.load(open(\"output/geocoding.json\", \"r\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "87c51b4a-485f-4d11-972f-6bcf2947427e",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_list = []\n",
    "for address, obj_list in geocoding_cache.items():\n",
    "    feature_list.append(to_geojson(address, obj_list[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "52b4e10b-4e2c-4316-95a1-2acc280533fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_collex = FeatureCollection(feature_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17b0ab28-155a-4aed-84cd-4d8840df5b3f",
   "metadata": {},
   "source": [
    "Write out the GeoJSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4c9ccf3c-54c1-48ab-b2d3-12394e665f58",
   "metadata": {},
   "outputs": [],
   "source": [
    "json.dump(\n",
    "    feature_collex,\n",
    "    open(\"output/geocoding.geojson\", \"w\"),\n",
    "    indent=2\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}