10_npg_data.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "---\n",
    "description: Provides functions and a pipeline for parsing Nothern Power Grid's primary\n",
    "  operational metering dataset.\n",
    "output-file: npg_data.html\n",
    "title: NPG ETL\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# | default_exp npg.etl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# | export\n",
    "import json\n",
    "import zipfile\n",
    "from typing import Callable\n",
    "\n",
    "import pandas as pd\n",
    "import requests as rq\n",
    "from fastcore.all import *\n",
    "\n",
    "import jupyter_playground.core as core\n",
    "from jupyter_playground.core import DownloadContent, cache\n",
    "\n",
    "if in_jupyter():\n",
    "    from tqdm.notebook import tqdm"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Downloading Dataset metadata and attachments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#12) ['_02_28_zip','_01_31_zip','_04_30_zip','_03_31_zip','_05_31_zip','_06_30_zip','_07_31_zip','_08_31_zip','_09_30_zip','_11_30_zip'...]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# | export\n",
    "HOST = \"northernpowergrid\"\n",
    "DATASET = \"primary-operational-metering\"\n",
    "\n",
    "\n",
    "def _always_true(_):\n",
    "    return True\n",
    "\n",
    "\n",
    "def get_dataset_attachments(host: str, dataset: str, filter: Callable = _always_true):\n",
    "    q = f\"https://{host}.opendatasoft.com/api/v2/catalog/datasets/{dataset}/attachments\"\n",
    "    res = rq.get(q)\n",
    "    res.raise_for_status()\n",
    "\n",
    "    return L(a for a in res.json()[\"attachments\"] if filter(a))\n",
    "\n",
    "\n",
    "files = get_dataset_attachments(\n",
    "    HOST, DATASET, lambda x: x[\"href\"].endswith(\"_zip\")\n",
    ").map(lambda x: x[\"href\"])\n",
    "files.map(lambda x: x[-10:])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For reference:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://northernpowergrid.opendatasoft.com/api/v2/catalog/datasets/primary-operational-metering/attachments\n"
     ]
    }
   ],
   "source": [
    "base = \"https://{host}.opendatasoft.com/api/v2/\".format(host=HOST)\n",
    "suffix = \"catalog/datasets/{dataset_id}/attachments\".format(dataset_id=DATASET)\n",
    "print(base + suffix)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Parser and Extraction Steps\n",
    "> A series of functions to unpack and work with the data.\n",
    "\n",
    "These functions deal with several quirks of the dataset:\n",
    "\n",
    "1. Replacments of bad JSON sections.\n",
    "2. Characters which don't parse correctly as utf-8\n",
    "3. Avoid need to download and extrac multiple large JSON files\n",
    "4. Quickly parse and discard objects and files completed minimise memory usage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IncrementalPipeline(name='npg_etl', _funcs=['attachment_download'])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# | export\n",
    "pipe = core.IncrementalPipeline(\"npg_etl\", funcs=[core.attachment_download])\n",
    "pipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                      \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(#1) [32d8b228a35b2302ed5871f407311777]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# | eval: false\n",
    "\n",
    "files[:1].map(pipe)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting the Zipfile and Serialising the Content"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Steps:\n",
    "\n",
    "1. Check the file is a Zipfile.\n",
    "1. Extract from the Zipfile exactly one JSON file.\n",
    "2. Checking is within a given maximum size unzipped.\n",
    "3. Read as whole file directly as text, fixing a JSON error.\n",
    "4. Finally mask the large string with `DownloadContent`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IncrementalPipeline(name='npg_etl', _funcs=['attachment_download', 'parse_attachment_content'])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# | exports\n",
    "\n",
    "\n",
    "@cache.cache\n",
    "def parse_attachment_content(\n",
    "    Download: DownloadContent, max_file_size: Union[int, float] = 4e9\n",
    ") -> dict:\n",
    "    with zipfile.ZipFile(io.BytesIO(Download.content)) as f:\n",
    "        if f.testzip() is not None:\n",
    "            raise ValueError(\"Content byte string is not a zipfile\")\n",
    "        n = f.infolist()\n",
    "        if len(n) != 1:\n",
    "            raise ValueError(\"Expecting only one file but found the following:\", n)\n",
    "        if n[0].file_size > int(max_file_size):\n",
    "            raise ValueError(f\"{n[0]} exceeds max_file_size ({int(max_file_size)})\")\n",
    "        file_txt = (\n",
    "            zipfile.Path(f, at=n[0].filename)\n",
    "            .read_text(encoding=\"utf-8\", errors=\"ignore\")\n",
    "            .replace(\"[,\", \"[\", 1)  # this is much quicker than regexing\n",
    "        )\n",
    "    # remove newlines and whitespace\n",
    "    return DownloadContent(file_txt)\n",
    "\n",
    "\n",
    "pipe.append_func(parse_attachment_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                      \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(#1) [4baf216cf69414089e228916246be3eb]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# | eval: false \n",
    "\n",
    "files[:1].map(pipe)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parsing JSON and Building a Dataframe"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Parse the JSON and create a dataframe for the values.\n",
    "\n",
    "1. Create a dataframe for each timeseries array and concatenate them.\n",
    "2. Discard the JSON object and initial frames for memory.\n",
    "3. Fix a bad column that failed to decode from utf-8\n",
    "4. `Groupby` effectively on each row and extract each set of values.\n",
    "5. Clear junk index and reset index used in Groupby\n",
    "6. Downcast float, create column for partitioning per file (year and month) and clarify timezone.\n",
    "7. Check neighbouring month haven't slipped into file and discard them.\n",
    "8. Cast non-numeric columns to categories.\n",
    "\n",
    "Resulting arrays are DataFrames approx 300Mb each.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IncrementalPipeline(name='npg_etl', _funcs=['attachment_download', 'parse_attachment_content', 'build_attachment_dataframe'])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# | exports\n",
    "\n",
    "_cat_cols = [\"substation\", \"circuit\", \"unit\", \"description\"]\n",
    "\n",
    "\n",
    "# @cache.cache\n",
    "def build_attachment_dataframe(text: DownloadContent) -> pd.DataFrame:\n",
    "    df = (\n",
    "        pd.concat(  # together the timeseries arrays\n",
    "            [pd.json_normalize(t) for t in json.loads(text.content)[\"timeseries\"]],\n",
    "            axis=0,\n",
    "            ignore_index=True,\n",
    "        )\n",
    "        .rename({\"values\": \"data\"}, axis=1)\n",
    "        .assign(unit=lambda d: d.unit.replace(r\"^\\s*$\", \"deg\", regex=True))\n",
    "        .set_index(_cat_cols)\n",
    "        .groupby(_cat_cols, sort=False)  # no sort for performance\n",
    "        .apply(lambda d: pd.json_normalize(d.iat[0, -1]))  # normalise another tier for each row\n",
    "        .reset_index(-1, drop=True)  # discard index from second parse\n",
    "        .reset_index(drop=False)  # clear rest\n",
    "        .assign(  # fix up types\n",
    "            timestamp=lambda d: pd.to_datetime(d.timestamp),\n",
    "            value=lambda d: pd.to_numeric(d.value, downcast=\"float\"),\n",
    "            yyyy_mm=lambda d: d.timestamp.dt.strftime(\"%Y-%m\"),\n",
    "        )\n",
    "    )\n",
    "    d = df.yyyy_mm.value_counts()  # remove values from neighboring months\n",
    "    if len(d) != 1:\n",
    "        print(text, '\\n' + d.__repr__())  # log these for reference\n",
    "\n",
    "    df[_cat_cols] = df[_cat_cols].astype(\"category\")\n",
    "    return df[d.index[d.argmax()] == df.yyyy_mm]\n",
    "\n",
    "\n",
    "pipe.append_func(build_attachment_dataframe)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# | hide\n",
    "import nbdev; nbdev.nbdev_export()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:13,  7.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16e9515bba143d2ad61d80406e56fbdb \n",
      "2022-12    20136044\n",
      "2023-01           6\n",
      "Name: yyyy_mm, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                       \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>substation</th>\n",
       "      <th>circuit</th>\n",
       "      <th>unit</th>\n",
       "      <th>description</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>value</th>\n",
       "      <th>yyyy_mm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LINTON 132/25kV (ULGHAM CROSSING)</td>\n",
       "      <td>LINTON - TRX1</td>\n",
       "      <td>KW</td>\n",
       "      <td>active power (kw)</td>\n",
       "      <td>2022-12-01 00:00:00+00:00</td>\n",
       "      <td>400.0</td>\n",
       "      <td>2022-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LINTON 132/25kV (ULGHAM CROSSING)</td>\n",
       "      <td>LINTON - TRX1</td>\n",
       "      <td>KW</td>\n",
       "      <td>active power (kw)</td>\n",
       "      <td>2022-12-01 00:30:00+00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2022-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LINTON 132/25kV (ULGHAM CROSSING)</td>\n",
       "      <td>LINTON - TRX1</td>\n",
       "      <td>KW</td>\n",
       "      <td>active power (kw)</td>\n",
       "      <td>2022-12-01 01:00:00+00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2022-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LINTON 132/25kV (ULGHAM CROSSING)</td>\n",
       "      <td>LINTON - TRX1</td>\n",
       "      <td>KW</td>\n",
       "      <td>active power (kw)</td>\n",
       "      <td>2022-12-01 01:30:00+00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2022-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LINTON 132/25kV (ULGHAM CROSSING)</td>\n",
       "      <td>LINTON - TRX1</td>\n",
       "      <td>KW</td>\n",
       "      <td>active power (kw)</td>\n",
       "      <td>2022-12-01 02:00:00+00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2022-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20136045</th>\n",
       "      <td>WAVERLEY BUSINESS PARK</td>\n",
       "      <td>WAVERLEY BUSINESS PARK - WHITTLE WAY</td>\n",
       "      <td>A</td>\n",
       "      <td>current (amp)</td>\n",
       "      <td>2022-12-31 21:30:00+00:00</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2022-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20136046</th>\n",
       "      <td>WAVERLEY BUSINESS PARK</td>\n",
       "      <td>WAVERLEY BUSINESS PARK - WHITTLE WAY</td>\n",
       "      <td>A</td>\n",
       "      <td>current (amp)</td>\n",
       "      <td>2022-12-31 22:00:00+00:00</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2022-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20136047</th>\n",
       "      <td>WAVERLEY BUSINESS PARK</td>\n",
       "      <td>WAVERLEY BUSINESS PARK - WHITTLE WAY</td>\n",
       "      <td>A</td>\n",
       "      <td>current (amp)</td>\n",
       "      <td>2022-12-31 22:30:00+00:00</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2022-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20136048</th>\n",
       "      <td>WAVERLEY BUSINESS PARK</td>\n",
       "      <td>WAVERLEY BUSINESS PARK - WHITTLE WAY</td>\n",
       "      <td>A</td>\n",
       "      <td>current (amp)</td>\n",
       "      <td>2022-12-31 23:00:00+00:00</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2022-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20136049</th>\n",
       "      <td>WAVERLEY BUSINESS PARK</td>\n",
       "      <td>WAVERLEY BUSINESS PARK - WHITTLE WAY</td>\n",
       "      <td>A</td>\n",
       "      <td>current (amp)</td>\n",
       "      <td>2022-12-31 23:30:00+00:00</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2022-12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20136044 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 substation  \\\n",
       "0         LINTON 132/25kV (ULGHAM CROSSING)   \n",
       "1         LINTON 132/25kV (ULGHAM CROSSING)   \n",
       "2         LINTON 132/25kV (ULGHAM CROSSING)   \n",
       "3         LINTON 132/25kV (ULGHAM CROSSING)   \n",
       "4         LINTON 132/25kV (ULGHAM CROSSING)   \n",
       "...                                     ...   \n",
       "20136045             WAVERLEY BUSINESS PARK   \n",
       "20136046             WAVERLEY BUSINESS PARK   \n",
       "20136047             WAVERLEY BUSINESS PARK   \n",
       "20136048             WAVERLEY BUSINESS PARK   \n",
       "20136049             WAVERLEY BUSINESS PARK   \n",
       "\n",
       "                                       circuit unit        description  \\\n",
       "0                                LINTON - TRX1   KW  active power (kw)   \n",
       "1                                LINTON - TRX1   KW  active power (kw)   \n",
       "2                                LINTON - TRX1   KW  active power (kw)   \n",
       "3                                LINTON - TRX1   KW  active power (kw)   \n",
       "4                                LINTON - TRX1   KW  active power (kw)   \n",
       "...                                        ...  ...                ...   \n",
       "20136045  WAVERLEY BUSINESS PARK - WHITTLE WAY    A      current (amp)   \n",
       "20136046  WAVERLEY BUSINESS PARK - WHITTLE WAY    A      current (amp)   \n",
       "20136047  WAVERLEY BUSINESS PARK - WHITTLE WAY    A      current (amp)   \n",
       "20136048  WAVERLEY BUSINESS PARK - WHITTLE WAY    A      current (amp)   \n",
       "20136049  WAVERLEY BUSINESS PARK - WHITTLE WAY    A      current (amp)   \n",
       "\n",
       "                         timestamp  value  yyyy_mm  \n",
       "0        2022-12-01 00:00:00+00:00  400.0  2022-12  \n",
       "1        2022-12-01 00:30:00+00:00    0.0  2022-12  \n",
       "2        2022-12-01 01:00:00+00:00    0.0  2022-12  \n",
       "3        2022-12-01 01:30:00+00:00    0.0  2022-12  \n",
       "4        2022-12-01 02:00:00+00:00    0.0  2022-12  \n",
       "...                            ...    ...      ...  \n",
       "20136045 2022-12-31 21:30:00+00:00    7.0  2022-12  \n",
       "20136046 2022-12-31 22:00:00+00:00    8.0  2022-12  \n",
       "20136047 2022-12-31 22:30:00+00:00    7.0  2022-12  \n",
       "20136048 2022-12-31 23:00:00+00:00    8.0  2022-12  \n",
       "20136049 2022-12-31 23:30:00+00:00    9.0  2022-12  \n",
       "\n",
       "[20136044 rows x 7 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# | eval: false\n",
    "\n",
    "from jupyter_playground.npg.etl import pipe\n",
    "\n",
    "df = files[-1:].map(pipe)[0]\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 20136044 entries, 0 to 20136049\n",
      "Data columns (total 7 columns):\n",
      " #   Column       Dtype              \n",
      "---  ------       -----              \n",
      " 0   substation   category           \n",
      " 1   circuit      category           \n",
      " 2   unit         category           \n",
      " 3   description  category           \n",
      " 4   timestamp    datetime64[ns, UTC]\n",
      " 5   value        float32            \n",
      " 6   yyyy_mm      object             \n",
      "dtypes: category(4), datetime64[ns, UTC](1), float32(1), object(1)\n",
      "memory usage: 653.1+ MB\n"
     ]
    }
   ],
   "source": [
    "# | eval:false\n",
    "df.info()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write the Whole Dataset"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Write out the dataset with some partitioning per data and month."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@pipe.decorate_func\n",
    "def writes_file(df):\n",
    "    df.to_parquet(\"npg.parquet\", partition_cols=[\"description\", \"yyyy_mm\"])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally run against all the files, approx 60-90 minutes assuming already downloaded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ec4f01943d944a3b5c03b53a5e70d21",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16e9515bba143d2ad61d80406e56fbdb \n",
      "2022-12    20136044\n",
      "2023-01           6\n",
      "Name: yyyy_mm, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    }
   ],
   "source": [
    "# |eval: false\n",
    "for f in tqdm(files, leave=False):\n",
    "    pipe(f, tqdm_position=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# | hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jscott/mambaforge/envs/jupyplay/lib/python3.11/site-packages/_distutils_hack/__init__.py:33: UserWarning: Setuptools is replacing distutils.\n",
      "  warnings.warn(\"Setuptools is replacing distutils.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "INSTALLED VERSIONS\n",
      "------------------\n",
      "commit           : 2e218d10984e9919f0296931d92ea851c6a6faf5\n",
      "python           : 3.11.0.final.0\n",
      "python-bits      : 64\n",
      "OS               : Linux\n",
      "OS-release       : 5.15.90.1-microsoft-standard-WSL2\n",
      "Version          : #1 SMP Fri Jan 27 02:56:13 UTC 2023\n",
      "machine          : x86_64\n",
      "processor        : x86_64\n",
      "byteorder        : little\n",
      "LC_ALL           : None\n",
      "LANG             : C.UTF-8\n",
      "LOCALE           : en_US.UTF-8\n",
      "\n",
      "pandas           : 1.5.3\n",
      "numpy            : 1.24.2\n",
      "pytz             : 2022.7.1\n",
      "dateutil         : 2.8.2\n",
      "setuptools       : 67.6.0\n",
      "pip              : 23.0.1\n",
      "Cython           : None\n",
      "pytest           : None\n",
      "hypothesis       : None\n",
      "sphinx           : None\n",
      "blosc            : None\n",
      "feather          : None\n",
      "xlsxwriter       : 3.0.9\n",
      "lxml.etree       : None\n",
      "html5lib         : None\n",
      "pymysql          : None\n",
      "psycopg2         : None\n",
      "jinja2           : 3.1.2\n",
      "IPython          : 8.11.0\n",
      "pandas_datareader: None\n",
      "bs4              : None\n",
      "bottleneck       : None\n",
      "brotli           : \n",
      "fastparquet      : None\n",
      "fsspec           : 2023.3.0\n",
      "gcsfs            : None\n",
      "matplotlib       : 3.7.1\n",
      "numba            : None\n",
      "numexpr          : None\n",
      "odfpy            : None\n",
      "openpyxl         : 3.1.1\n",
      "pandas_gbq       : None\n",
      "pyarrow          : 11.0.0\n",
      "pyreadstat       : None\n",
      "pyxlsb           : None\n",
      "s3fs             : None\n",
      "scipy            : 1.10.1\n",
      "snappy           : None\n",
      "sqlalchemy       : None\n",
      "tables           : None\n",
      "tabulate         : 0.9.0\n",
      "xarray           : None\n",
      "xlrd             : 2.0.1\n",
      "xlwt             : None\n",
      "zstandard        : None\n",
      "tzdata           : None\n"
     ]
    }
   ],
   "source": [
    "pd.show_versions()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}