Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from requests_html import HTMLSession\n",
    "import pandas as pd\n",
    "import time\n",
    "import re\n",
    "session = HTMLSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = \"https://coinmarketcap.com\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scraping the rows"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get every historical link that coinmarketcap provides and put them in a list:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = session.get(base + \"/historical/\")\n",
    "p = re.compile(r'/historical/\\d{8}/')\n",
    "links = [ s for s in r.html.links if p.match(s)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The global variable `state` is used to collect a list of rows for each page indexed by the link of each page. It is initialized with `False` for each page to signify that the page still has to be scraped. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "state = {link: False for link in links}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is how many pages we (still) have to scrape:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([key for (key, value) in state.items() if not value])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This tries to scrape the websites and sleeps 6 seconds after each requests. Moreover it sleeps 30 seconds if it encounters and `IndexError` which happened whenever the request was blocked. After it was blocked 10 times it stops; use the cell above to check whether there are still pages remaining and then rerun."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopped\n"
     ]
    }
   ],
   "source": [
    "for _ in range(10):\n",
    "    try:\n",
    "        for link in (key for (key, value) in state.items() if not value):\n",
    "            print(link)\n",
    "            r = session.get(base + link)\n",
    "            table = r.html.find(\"table\")[2]\n",
    "            table_content = table.lxml[0]\n",
    "            tbody = table_content[1]\n",
    "            state[link] = [[\n",
    "                pd.to_datetime(link[12:-1]),\n",
    "                int(trow[0][0].text), \n",
    "                trow[1][0][1].text,\n",
    "                trow[2][0].text,\n",
    "                int(trow[3][0].text[1:].replace(\",\",\"\")),\n",
    "                float(trow[4][0][0].text[1:].replace(\",\",\"\")),\n",
    "                int(trow[5][0].text.split(\" \")[0].replace(\",\",\"\")),\n",
    "                trow[1][0][0].attrib[\"src\"] if \"src\" in trow[1][0][0].attrib else \"\",\n",
    "            ] for trow in tbody]\n",
    "            time.sleep(6)\n",
    "    except IndexError:\n",
    "        print(\"Index Error\")\n",
    "        time.sleep(30)\n",
    "print(\"Stopped\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sum over the lists of rows in `state`, i.e. concatenate them, and put them into a big dataframe:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>rank</th>\n",
       "      <th>name</th>\n",
       "      <th>symbol</th>\n",
       "      <th>market-cap</th>\n",
       "      <th>price</th>\n",
       "      <th>circulating-supply</th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-03-06</td>\n",
       "      <td>1</td>\n",
       "      <td>Bitcoin</td>\n",
       "      <td>BTC</td>\n",
       "      <td>6232667417</td>\n",
       "      <td>407.710000</td>\n",
       "      <td>15287125</td>\n",
       "      <td>https://s2.coinmarketcap.com/static/img/coins/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-03-06</td>\n",
       "      <td>2</td>\n",
       "      <td>Ethereum</td>\n",
       "      <td>ETH</td>\n",
       "      <td>882886701</td>\n",
       "      <td>11.380000</td>\n",
       "      <td>77562550</td>\n",
       "      <td>https://s2.coinmarketcap.com/static/img/coins/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-03-06</td>\n",
       "      <td>3</td>\n",
       "      <td>XRP</td>\n",
       "      <td>XRP</td>\n",
       "      <td>267938591</td>\n",
       "      <td>0.007860</td>\n",
       "      <td>34090841338</td>\n",
       "      <td>https://s2.coinmarketcap.com/static/img/coins/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016-03-06</td>\n",
       "      <td>4</td>\n",
       "      <td>Litecoin</td>\n",
       "      <td>LTC</td>\n",
       "      <td>144323374</td>\n",
       "      <td>3.220000</td>\n",
       "      <td>44798026</td>\n",
       "      <td>https://s2.coinmarketcap.com/static/img/coins/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-03-06</td>\n",
       "      <td>5</td>\n",
       "      <td>MaidSafeCoin</td>\n",
       "      <td>MAID</td>\n",
       "      <td>47222405</td>\n",
       "      <td>0.104347</td>\n",
       "      <td>452552412</td>\n",
       "      <td>https://s2.coinmarketcap.com/static/img/coins/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73042</th>\n",
       "      <td>2019-05-05</td>\n",
       "      <td>196</td>\n",
       "      <td>Einsteinium</td>\n",
       "      <td>EMC2</td>\n",
       "      <td>18092012</td>\n",
       "      <td>0.082752</td>\n",
       "      <td>218628620</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73043</th>\n",
       "      <td>2019-05-05</td>\n",
       "      <td>197</td>\n",
       "      <td>Gnosis</td>\n",
       "      <td>GNO</td>\n",
       "      <td>18041846</td>\n",
       "      <td>16.330000</td>\n",
       "      <td>1104590</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73044</th>\n",
       "      <td>2019-05-05</td>\n",
       "      <td>198</td>\n",
       "      <td>Ankr</td>\n",
       "      <td>ANKR</td>\n",
       "      <td>17958625</td>\n",
       "      <td>0.008123</td>\n",
       "      <td>2210758610</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73045</th>\n",
       "      <td>2019-05-05</td>\n",
       "      <td>199</td>\n",
       "      <td>Odyssey</td>\n",
       "      <td>OCN</td>\n",
       "      <td>17694306</td>\n",
       "      <td>0.002362</td>\n",
       "      <td>7491360438</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73046</th>\n",
       "      <td>2019-05-05</td>\n",
       "      <td>200</td>\n",
       "      <td>Moeda Loyalty Points</td>\n",
       "      <td>MDA</td>\n",
       "      <td>17636839</td>\n",
       "      <td>0.898514</td>\n",
       "      <td>19628888</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>73047 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            date  rank                  name symbol  market-cap       price  \\\n",
       "0     2016-03-06     1               Bitcoin    BTC  6232667417  407.710000   \n",
       "1     2016-03-06     2              Ethereum    ETH   882886701   11.380000   \n",
       "2     2016-03-06     3                   XRP    XRP   267938591    0.007860   \n",
       "3     2016-03-06     4              Litecoin    LTC   144323374    3.220000   \n",
       "4     2016-03-06     5          MaidSafeCoin   MAID    47222405    0.104347   \n",
       "...          ...   ...                   ...    ...         ...         ...   \n",
       "73042 2019-05-05   196           Einsteinium   EMC2    18092012    0.082752   \n",
       "73043 2019-05-05   197                Gnosis    GNO    18041846   16.330000   \n",
       "73044 2019-05-05   198                  Ankr   ANKR    17958625    0.008123   \n",
       "73045 2019-05-05   199               Odyssey    OCN    17694306    0.002362   \n",
       "73046 2019-05-05   200  Moeda Loyalty Points    MDA    17636839    0.898514   \n",
       "\n",
       "       circulating-supply                                              image  \n",
       "0                15287125  https://s2.coinmarketcap.com/static/img/coins/...  \n",
       "1                77562550  https://s2.coinmarketcap.com/static/img/coins/...  \n",
       "2             34090841338  https://s2.coinmarketcap.com/static/img/coins/...  \n",
       "3                44798026  https://s2.coinmarketcap.com/static/img/coins/...  \n",
       "4               452552412  https://s2.coinmarketcap.com/static/img/coins/...  \n",
       "...                   ...                                                ...  \n",
       "73042           218628620                                                     \n",
       "73043             1104590                                                     \n",
       "73044          2210758610                                                     \n",
       "73045          7491360438                                                     \n",
       "73046            19628888                                                     \n",
       "\n",
       "[73047 rows x 8 columns]"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(sum((value for key, value in state.items() if value), []), \n",
    "    columns=[\n",
    "        \"date\",\n",
    "        \"rank\", \n",
    "        \"name\",\n",
    "        \"symbol\",\n",
    "        \"market-cap\",\n",
    "        \"price\",\n",
    "        \"circulating-supply\",\n",
    "        \"image\"\n",
    "    ])\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A double index can also be used for further analysis:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>symbol</th>\n",
       "      <th>market-cap</th>\n",
       "      <th>price</th>\n",
       "      <th>circulating-supply</th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th>rank</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2016-03-06</th>\n",
       "      <th>1</th>\n",
       "      <td>Bitcoin</td>\n",
       "      <td>BTC</td>\n",
       "      <td>6232667417</td>\n",
       "      <td>407.710000</td>\n",
       "      <td>15287125</td>\n",
       "      <td>https://s2.coinmarketcap.com/static/img/coins/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ethereum</td>\n",
       "      <td>ETH</td>\n",
       "      <td>882886701</td>\n",
       "      <td>11.380000</td>\n",
       "      <td>77562550</td>\n",
       "      <td>https://s2.coinmarketcap.com/static/img/coins/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>XRP</td>\n",
       "      <td>XRP</td>\n",
       "      <td>267938591</td>\n",
       "      <td>0.007860</td>\n",
       "      <td>34090841338</td>\n",
       "      <td>https://s2.coinmarketcap.com/static/img/coins/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Litecoin</td>\n",
       "      <td>LTC</td>\n",
       "      <td>144323374</td>\n",
       "      <td>3.220000</td>\n",
       "      <td>44798026</td>\n",
       "      <td>https://s2.coinmarketcap.com/static/img/coins/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>MaidSafeCoin</td>\n",
       "      <td>MAID</td>\n",
       "      <td>47222405</td>\n",
       "      <td>0.104347</td>\n",
       "      <td>452552412</td>\n",
       "      <td>https://s2.coinmarketcap.com/static/img/coins/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2019-05-05</th>\n",
       "      <th>196</th>\n",
       "      <td>Einsteinium</td>\n",
       "      <td>EMC2</td>\n",
       "      <td>18092012</td>\n",
       "      <td>0.082752</td>\n",
       "      <td>218628620</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>Gnosis</td>\n",
       "      <td>GNO</td>\n",
       "      <td>18041846</td>\n",
       "      <td>16.330000</td>\n",
       "      <td>1104590</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>Ankr</td>\n",
       "      <td>ANKR</td>\n",
       "      <td>17958625</td>\n",
       "      <td>0.008123</td>\n",
       "      <td>2210758610</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>Odyssey</td>\n",
       "      <td>OCN</td>\n",
       "      <td>17694306</td>\n",
       "      <td>0.002362</td>\n",
       "      <td>7491360438</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>Moeda Loyalty Points</td>\n",
       "      <td>MDA</td>\n",
       "      <td>17636839</td>\n",
       "      <td>0.898514</td>\n",
       "      <td>19628888</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>73047 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 name symbol  market-cap       price  \\\n",
       "date       rank                                                        \n",
       "2016-03-06 1                  Bitcoin    BTC  6232667417  407.710000   \n",
       "           2                 Ethereum    ETH   882886701   11.380000   \n",
       "           3                      XRP    XRP   267938591    0.007860   \n",
       "           4                 Litecoin    LTC   144323374    3.220000   \n",
       "           5             MaidSafeCoin   MAID    47222405    0.104347   \n",
       "...                               ...    ...         ...         ...   \n",
       "2019-05-05 196            Einsteinium   EMC2    18092012    0.082752   \n",
       "           197                 Gnosis    GNO    18041846   16.330000   \n",
       "           198                   Ankr   ANKR    17958625    0.008123   \n",
       "           199                Odyssey    OCN    17694306    0.002362   \n",
       "           200   Moeda Loyalty Points    MDA    17636839    0.898514   \n",
       "\n",
       "                 circulating-supply  \\\n",
       "date       rank                       \n",
       "2016-03-06 1               15287125   \n",
       "           2               77562550   \n",
       "           3            34090841338   \n",
       "           4               44798026   \n",
       "           5              452552412   \n",
       "...                             ...   \n",
       "2019-05-05 196            218628620   \n",
       "           197              1104590   \n",
       "           198           2210758610   \n",
       "           199           7491360438   \n",
       "           200             19628888   \n",
       "\n",
       "                                                             image  \n",
       "date       rank                                                     \n",
       "2016-03-06 1     https://s2.coinmarketcap.com/static/img/coins/...  \n",
       "           2     https://s2.coinmarketcap.com/static/img/coins/...  \n",
       "           3     https://s2.coinmarketcap.com/static/img/coins/...  \n",
       "           4     https://s2.coinmarketcap.com/static/img/coins/...  \n",
       "           5     https://s2.coinmarketcap.com/static/img/coins/...  \n",
       "...                                                            ...  \n",
       "2019-05-05 196                                                      \n",
       "           197                                                      \n",
       "           198                                                      \n",
       "           199                                                      \n",
       "           200                                                      \n",
       "\n",
       "[73047 rows x 6 columns]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.set_index([\"date\", \"rank\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the dataframe into a csv:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.sort_values(\"date\").to_csv(\"cryptocurrency-data.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Downloading images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "List containing the image urls that have been donwloaded already:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "downloaded_already = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Iterate over all rows which have a download url and download the ones that aren't yet in `downloaded_already`. Download the 64x64 version instead of the 32x32 version."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "for _, row in df.query(\"image != ''\").iterrows():\n",
    "    if row[\"image\"] not in downloaded_already:\n",
    "        urllib.request.urlretrieve(row[\"image\"].replace(\"32\",\"64\"), f\"assets/{row['name']}.jpg\")\n",
    "        downloaded_already.append(row[\"image\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Replace the remote urls in `df2` with the local paths:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df.copy()\n",
    "for name in df.query(\"image != ''\").name.unique():\n",
    "    df2.loc[df.name == name, \"image\"] = f\"assets/{name}.jpg\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In `df2` we have way more rows with images, because:\n",
    "In `df` a row only had image if it is in the top 10.\n",
    "Now, it has an image if it *ever* was in the top 10."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3997"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.query(\"image != ''\").name.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13900"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.query(\"image != ''\").name.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Top 20 Coins with still no image:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Agoras Tokens', 'YbCoin', 'Zetacoin', 'Bitleu',\n",
       "       'SiliconValleyCoin [old]', 'Peerplays [old]', 'UNUS SED LEO',\n",
       "       'Qtum', 'Zcash', 'OmiseGO', 'HyperCash', 'VeChain', 'Huobi Token',\n",
       "       'USD Coin', 'Iconomi', 'NuBits', 'SuperNET', 'HEX', 'Vertcoin',\n",
       "       'Vechain [Token]', 'FedoraCoin', 'RevolutionVR', 'VeriCoin',\n",
       "       'CloakCoin', 'BBQCoin', 'Ixcoin', 'BitBar', 'GoldCoin', 'Memecoin',\n",
       "       'Maker', 'Asiadigicoin', 'VPNCoin', 'Veritaseum',\n",
       "       'Wrapped Bitcoin', 'BitcoinDark', 'Nxttycoin', 'MonaCoin',\n",
       "       'Acash Coin', 'Cosmos', 'ICON', 'Maxcoin', 'Rimbit', 'MINDOL',\n",
       "       'Siacoin', 'TenX', 'Nano', 'Populous', 'Verge', 'Anoncoin',\n",
       "       'StableCoin', 'GridCoin', 'Rubycoin', 'Gulden', 'Phoenixcoin',\n",
       "       'Fastcoin', 'Ontology', 'Neutrino', 'FuelCoin', 'Gnosis', '999',\n",
       "       'ReddCoin', 'NXTventure', 'AmericanCoin', 'Franko', 'Clams',\n",
       "       'DigiByte', 'Viacoin', 'Decred', 'NetCoin', 'Copperlark',\n",
       "       'Marinecoin', 'Ardor', 'GameCredits', 'NeuCoin', 'SingularDTV',\n",
       "       'Tickets', 'Global Currency Reserve', 'CannabisCoin', 'ARCHcoin',\n",
       "       'Bitgem', 'Round', 'Ethercoin', 'FairCoin', 'BitCrystals',\n",
       "       'ShadowCash', 'Luckycoin', 'InstantDEX'], dtype=object)"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top20noimage = df2.query(\"rank < 20 and image == ''\").name.unique()\n",
    "top20noimage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download images from the currency's main page:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "already_downloaded_images_by_name = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_name(name: str, url_name: str = None):\n",
    "    print(f\"Downloading {name}\")\n",
    "    if not url_name:\n",
    "        url_name = name.lower().replace(\" \", \"-\").replace(\"[\",\"\").replace(\"]\",\"\")\n",
    "    r = session.get(f\"https://coinmarketcap.com/currencies/{url_name}\")\n",
    "    try:\n",
    "        url = r.html.find(\".cmc-static-icon\")[0].lxml[0].attrib[\"src\"]\n",
    "        urllib.request.urlretrieve(url, f\"assets/{name}.jpg\")\n",
    "        already_downloaded_images_by_name.append(name)\n",
    "    except IndexError:\n",
    "        print(f\"Couldn't find {url_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name in top20noimage:\n",
    "    if name not in already_downloaded_images_by_name:\n",
    "        download_name(name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Three ones have a url that can't be generated from their `name` string:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading Peerplays [old]\n",
      "Downloading Vechain [Token]\n",
      "Downloading 999\n"
     ]
    }
   ],
   "source": [
    "download_name(\"Peerplays [old]\", \"peerplays\")\n",
    "download_name(\"Vechain [Token]\", \"vechain\")\n",
    "download_name(\"999\", \"999-coin\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Assign newly downloaded asset paths to `df2`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name in top20noimage:\n",
    "    df2.loc[df2.name == name, \"image\"] = f\"assets/{name}.jpg\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, there are no images ranked below 20 with no image:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([], dtype=object)"
      ]
     },
     "execution_count": 219,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.query(\"rank < 20 and image == ''\").name.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Overwrite the old csv with the new one:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.sort_values(\"date\").to_csv(\"cryptocurrency-data.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}