scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install beautifulsoup4\n",
    "!pip install lxml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from bs4 import BeautifulSoup\n",
    "from random import uniform\n",
    "from requests import get\n",
    "from http import HTTPStatus\n",
    "from lxml import etree\n",
    "from lxml import html\n",
    "from time import sleep\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ProgressBars:\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.progress_bars = {}\n",
    "    \n",
    "    def add_bar(self, bar_name, prog_max, prog=0, num_bars=50):\n",
    "        self.progress_bars[bar_name] = {\n",
    "            'prog': prog,\n",
    "            'prog_max': prog_max,\n",
    "            'num_bars': num_bars\n",
    "        }\n",
    "    \n",
    "    def del_bar(self, bar_name):\n",
    "        del self.progress_bars[bar_name]\n",
    "    \n",
    "    def update_progress(self, bar_name, prog=1):\n",
    "        self.progress_bars[bar_name]['prog'] += prog\n",
    "    \n",
    "    def write_to_screen(self):\n",
    "        output = ''\n",
    "        for key in self.progress_bars.keys():\n",
    "            bar = self.progress_bars[key]\n",
    "            \n",
    "            num_bars = int((bar['prog']/bar['prog_max'])*bar['num_bars'])\n",
    "            num_spaces = bar['num_bars']-num_bars\n",
    "            percent_complete = np.round(bar['prog']/bar['prog_max']*100, 2)\n",
    "            \n",
    "            bar = key+': <'+('|'*num_bars)+(' '*num_spaces)+'> ('+str(percent_complete)+'%)'\n",
    "            if len(self.progress_bars.keys()) > 1:\n",
    "                bar += '\\n'\n",
    "            output += bar\n",
    "        if len(self.progress_bars.keys()) > 1:\n",
    "            clear_output()\n",
    "        print(output, end='\\r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Scraper:\n",
    "    def __init__(self, max_talks=float('inf')):\n",
    "        \"\"\"\n",
    "        max_talks can be used to test the scraper, set it to an arbitrary number to test collecting a small\n",
    "        sample size.\n",
    "        The initializer will automatically collect all the talk urls, so it can take a while to initialize if\n",
    "        you do not set max_talks to a smaller number.\n",
    "        \"\"\"\n",
    "        self.RAND_LONG = lambda: uniform(10, 15)\n",
    "        self.RAND_MED = lambda: uniform(5,8)\n",
    "        self.RAND_SHORT = lambda: uniform(0.5,2)\n",
    "        self.HOMEPAGE = \"https://www.ted.com/talks/\"\n",
    "        self.BASE = \"https://www.ted.com\"\n",
    "        self.SAVE_PATH = \"ted_talk_data.csv\"\n",
    "        self.pages = self.init_page_urls()\n",
    "        self.talks = self.collect_talk_urls(max_talks)\n",
    "    \n",
    "    def make_soup(self, url: str) -> BeautifulSoup:\n",
    "        #takes a url and returns the BeautifulSoup object corresponding to it\n",
    "        r = self.get_(url)\n",
    "        return BeautifulSoup(r.text)\n",
    "    \n",
    "    def make_etree(self, url: str) -> etree._Element:\n",
    "        #similar to makesoup, takes url and returns an lxml element tree\n",
    "        r = self.get_(url)\n",
    "        return etree.HTML(r.text)\n",
    "    \n",
    "    def get_(self, url: str) -> str:\n",
    "        #used to handle http requests and timeout errors\n",
    "        #detects if we are getting a transcript and will correct the headers so that correct\n",
    "        #http response is sent out.\n",
    "        sleep(self.RAND_SHORT())\n",
    "        headers = {\n",
    "            \"user-agent\" :\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36\",\n",
    "            \"accept-language\": \"en-US,en;q=0.9\"\n",
    "        }\n",
    "        if \"/talks/\" in url:\n",
    "            headers.update({\"user-agent\" :\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36\"})\n",
    "        r = get(url, headers=headers)\n",
    "        if r.status_code != 200:\n",
    "            if r.status_code == 429 or r.status_code == 502:\n",
    "                #print(\"Error 429 recieved, sleeping for RAND_LONG, then trying again\")\n",
    "                sleep(self.RAND_LONG())\n",
    "                return self.get_(url)\n",
    "            print(f\"error collecting url: {url} \\nStatus code recieved: {HTTPStatus(r.status_code)}\")\n",
    "            return \"\"\n",
    "        return r\n",
    "    \n",
    "    def get_page_count(self) -> int:\n",
    "        #returns the total amount of pages (each page contains many ted talk links)\n",
    "        dom = self.make_etree(self.HOMEPAGE)\n",
    "        return int(dom.xpath(\"/html/body/div[1]/div[2]/div/div[2]/div[2]/div[2]/div/a[5]\")[0].text)\n",
    "    \n",
    "    def init_page_urls(self) -> dict:\n",
    "        #returns the pages dictionary, where each key is a (str) page url\n",
    "        #and the value is a string which represents that the page has been unscraped\n",
    "        #this value should always either be \"unscraped\", \"scraped\", \"in_progress\", or \"error\".b\n",
    "        page_count = self.get_page_count()\n",
    "        pages = {f\"https://www.ted.com/talks?page={i}\": \"unscraped\" for i in range(1, page_count + 1)}\n",
    "        return pages\n",
    "    \n",
    "    def collect_talk_urls(self, max_talks=float('inf')):\n",
    "        #goes to each page and collects each talk url, assigns them to a list\n",
    "        assert len(self.pages.keys()) != 0, \"pages was never filled out, cannot collect talk urls\"\n",
    "        self.talks = []\n",
    "        errors = 0\n",
    "        prog = ProgressBars()\n",
    "        prog.add_bar(\"Collecting Talk URLs\", len(self.pages.keys()), prog=0, num_bars=50)\n",
    "        for i, page in enumerate(self.pages.keys()):\n",
    "            self.pages[page] = \"in_progress\"\n",
    "            prog.update_progress(\"Collecting Talk URLs\")\n",
    "            prog.write_to_screen()\n",
    "            try:\n",
    "                soup = self.make_soup(page)\n",
    "                thumb_images = soup.find_all(class_=\"thumb thumb--video thumb--crop-top\")\n",
    "                for thumb_image in thumb_images:\n",
    "                    href = thumb_image.parent.get('href')\n",
    "                    self.talks.append(self.BASE+href+\"/transcript\")\n",
    "                    if len(self.talks) >= max_talks:\n",
    "                        return self.talks\n",
    "                self.pages[page] = \"scraped\"\n",
    "            except Exception as e:\n",
    "                self.pages[page] = \"error\"\n",
    "                errors += 1\n",
    "                print(f\"error in collecting page {page}\\n{e}\")\n",
    "        print()\n",
    "        return self.talks\n",
    "    \n",
    "    def collect_talk_data(self, link: str, raw_input=None):\n",
    "        #takes a link (in str format) that points to a specific ted talk.\n",
    "        #will then go to that ted talk and collect relevant information\n",
    "        #returns a dictionary where the key is the name of the data and the value is the data itself\n",
    "        if link:\n",
    "            raw = self.get_(link).text\n",
    "        else:\n",
    "            raw = raw_input\n",
    "            \n",
    "        soup = BeautifulSoup(raw)\n",
    "        \n",
    "        def get_topics() -> dict:\n",
    "            #returns the set representation of the topics, they are transformed into lower case.\n",
    "            topics_raw = soup.find_all(class_=\"inline-block py-1 text-tui-sm capitalize underline\")\n",
    "            topics = set([topic.text.lower() for topic in topics_raw])\n",
    "            return {\"topics\" : topics}\n",
    "        def get_summary() -> dict:\n",
    "            #returns the summary provided by TED.\n",
    "            summary = soup.find_all(class_=\"text-sm mb-6\")[0].text\n",
    "            return {\"summary\": summary}\n",
    "        def get_transcript() -> dict:\n",
    "            #returns the full transcript\n",
    "            start = raw.find('\"transcript\":') + len('\"transcript\":') + 1\n",
    "            end = raw.find('\"embedUrl\":') - 2\n",
    "            transcript = raw[start:end]\n",
    "            transcript = transcript.replace(\"&apos;\", \"'\")\n",
    "            transcript = transcript.replace(\"&quot;\", '\"')\n",
    "            return {\"transcript\": transcript}\n",
    "        def get_title() -> dict:\n",
    "            #returns the title of the talk\n",
    "            title = soup.find_all(class_=\"text-textPrimary-onLight font-light text-tui-2xl leading-tui-sm tracking-tui-tight md:text-tui-3xl md:tracking-tui-tightest lg:text-tui-4xl mr-5\")[0].text\n",
    "            return {\"title\": title}\n",
    "        def get_misc() -> dict:\n",
    "            #returns authors, and event in a dictionary\n",
    "            block = soup.find_all(class_=\"text-sm w-full truncate text-gray-900\")[0].text\n",
    "            separator = block.find(\"•\")\n",
    "            author, event = block[:separator-1], block[separator+2:]\n",
    "            return {\"author\": author, \"event\": event}\n",
    "        out = {\"link\": link}\n",
    "        [out.update(d) for d in [get_topics(), get_summary(), get_transcript(), get_misc(), get_title()]]\n",
    "        return out\n",
    "    \n",
    "    def collect_raw(self, **kwargs):\n",
    "        errs = 0\n",
    "        columns = ['link', 'raw']\n",
    "        dtypes = {'link': str, 'raw': str}\n",
    "        df = pd.DataFrame(columns=columns)\n",
    "        df = df.astype(dtypes)\n",
    "        prog = ProgressBars()\n",
    "        prog.add_bar(\"Scraping Raw Data   \", len(self.talks), prog=0, num_bars=50)\n",
    "        for i, link in enumerate(self.talks):\n",
    "            raw_dat = self.get_(link).text\n",
    "            prog.update_progress(\"Scraping Raw Data   \")\n",
    "            prog.write_to_screen()\n",
    "            df = pd.concat([df, pd.DataFrame([{\"link\":link, \"raw\":raw_dat}])], ignore_index=True)\n",
    "        return df\n",
    "    \n",
    "    def main(self, **kwargs):\n",
    "        #collects data on all talks, has to be run manually\n",
    "        #automatically does not add data for talks with no transcript\n",
    "        \"\"\"\n",
    "        kwargs:\n",
    "            save=True will set function to save a csv file of the collected data.\n",
    "                    automatically saves into ./'SAVE_PATH', which is be default set to \"ted_talk_data.csv\"\n",
    "                    **to change save path, change Scraper.SAVE_PATH to your desired path\n",
    "            overwrite=True will delete the old save file and replace it with the new one\n",
    "                            if an old save file exists in the same directory.\n",
    "        \"\"\"\n",
    "        no_transcripts = 0\n",
    "        columns = ['title', 'topics', 'transcript', 'author', 'event', 'link']\n",
    "        dtypes = {'title': str, 'topics': object, 'transcript': str, 'author': str, 'event': str, 'link': str}\n",
    "        df = pd.DataFrame(columns=columns)\n",
    "        df = df.astype(dtypes)\n",
    "        prog = ProgressBars()\n",
    "        prog.add_bar(\"Scraping Data       \", len(self.talks), prog=0, num_bars=50)\n",
    "        for i, talk in enumerate(self.talks):\n",
    "            dat = self.collect_talk_data(talk)\n",
    "            prog.update_progress(\"Scraping Data       \")\n",
    "            prog.write_to_screen()  \n",
    "            if (not dat['transcript']) or (not dat['topics']):\n",
    "                no_transcripts += 1\n",
    "                continue\n",
    "            df = pd.concat([df, pd.DataFrame([dat])], ignore_index=True)\n",
    "        print(f\"\\n\\nFailed to collect data on {no_transcripts} talks due to a lack of transcript/topics.\")\n",
    "        if \"save\" in kwargs and kwargs[\"save\"] == True:\n",
    "            if \"overwrite\" in kwargs and kwargs[\"overwrite\"] == True:\n",
    "                #delete old file if it exists\n",
    "                if os.path.exists(self.SAVE_PATH):\n",
    "                    os.remove(self.SAVE_PATH)\n",
    "            df.to_csv(self.SAVE_PATH, index_label=False)\n",
    "        return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting Talk URLs: <                                                  > (1.22%)\r"
     ]
    }
   ],
   "source": [
    "scraper = Scraper()\n",
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping Raw Data   : <||||||||||||||||||||||||||||||||||||||||||||||||||> (100.0%)\r"
     ]
    }
   ],
   "source": [
    "test = scraper.collect_raw()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "ename": "ConnectionError",
     "evalue": "HTTPSConnectionPool(host='www.ted.com', port=443): Max retries exceeded with url: /talks/ted_ed_how_to_manage_your_emotions/transcript (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x17ebdb880>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known'))",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mgaierror\u001b[0m                                  Traceback (most recent call last)",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/urllib3/connection.py:174\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    173\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 174\u001b[0m     conn \u001b[38;5;241m=\u001b[39m \u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    175\u001b[0m \u001b[43m        \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mextra_kw\u001b[49m\n\u001b[1;32m    176\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    178\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m SocketTimeout:\n",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/urllib3/util/connection.py:72\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m     68\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m six\u001b[38;5;241m.\u001b[39mraise_from(\n\u001b[1;32m     69\u001b[0m         LocationParseError(\u001b[38;5;124mu\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, label empty or too long\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m host), \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     70\u001b[0m     )\n\u001b[0;32m---> 72\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m res \u001b[38;5;129;01min\u001b[39;00m \u001b[43msocket\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetaddrinfo\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhost\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mport\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfamily\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msocket\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mSOCK_STREAM\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m     73\u001b[0m     af, socktype, proto, canonname, sa \u001b[38;5;241m=\u001b[39m res\n",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/socket.py:955\u001b[0m, in \u001b[0;36mgetaddrinfo\u001b[0;34m(host, port, family, type, proto, flags)\u001b[0m\n\u001b[1;32m    954\u001b[0m addrlist \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m--> 955\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m res \u001b[38;5;129;01min\u001b[39;00m \u001b[43m_socket\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetaddrinfo\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhost\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mport\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfamily\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mproto\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mflags\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m    956\u001b[0m     af, socktype, proto, canonname, sa \u001b[38;5;241m=\u001b[39m res\n",
      "\u001b[0;31mgaierror\u001b[0m: [Errno 8] nodename nor servname provided, or not known",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mNewConnectionError\u001b[0m                        Traceback (most recent call last)",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/urllib3/connectionpool.py:703\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    702\u001b[0m \u001b[38;5;66;03m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    704\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    705\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    706\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    707\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    708\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    709\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    710\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    711\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    713\u001b[0m \u001b[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m    714\u001b[0m \u001b[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m    715\u001b[0m \u001b[38;5;66;03m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m    716\u001b[0m \u001b[38;5;66;03m# mess.\u001b[39;00m\n",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/urllib3/connectionpool.py:386\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    385\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 386\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconn\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    387\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    388\u001b[0m     \u001b[38;5;66;03m# Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.\u001b[39;00m\n",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/urllib3/connectionpool.py:1040\u001b[0m, in \u001b[0;36mHTTPSConnectionPool._validate_conn\u001b[0;34m(self, conn)\u001b[0m\n\u001b[1;32m   1039\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(conn, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msock\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m):  \u001b[38;5;66;03m# AppEngine might not have  `.sock`\u001b[39;00m\n\u001b[0;32m-> 1040\u001b[0m     \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1042\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m conn\u001b[38;5;241m.\u001b[39mis_verified:\n",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/urllib3/connection.py:358\u001b[0m, in \u001b[0;36mHTTPSConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    356\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    357\u001b[0m     \u001b[38;5;66;03m# Add certificate verification\u001b[39;00m\n\u001b[0;32m--> 358\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;241m=\u001b[39m conn \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    359\u001b[0m     hostname \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhost\n",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/urllib3/connection.py:186\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    185\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m SocketError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 186\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[1;32m    187\u001b[0m         \u001b[38;5;28mself\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m e\n\u001b[1;32m    188\u001b[0m     )\n\u001b[1;32m    190\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m conn\n",
      "\u001b[0;31mNewConnectionError\u001b[0m: <urllib3.connection.HTTPSConnection object at 0x17ebdb880>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mMaxRetryError\u001b[0m                             Traceback (most recent call last)",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/requests/adapters.py:489\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    488\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m chunked:\n\u001b[0;32m--> 489\u001b[0m     resp \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    490\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    491\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    492\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    493\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    494\u001b[0m \u001b[43m        \u001b[49m\u001b[43mredirect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    495\u001b[0m \u001b[43m        \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    496\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    497\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    498\u001b[0m \u001b[43m        \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    499\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    500\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    502\u001b[0m \u001b[38;5;66;03m# Send the request.\u001b[39;00m\n\u001b[1;32m    503\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/urllib3/connectionpool.py:785\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    783\u001b[0m     e \u001b[38;5;241m=\u001b[39m ProtocolError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConnection aborted.\u001b[39m\u001b[38;5;124m\"\u001b[39m, e)\n\u001b[0;32m--> 785\u001b[0m retries \u001b[38;5;241m=\u001b[39m \u001b[43mretries\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    786\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msys\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m    787\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    788\u001b[0m retries\u001b[38;5;241m.\u001b[39msleep()\n",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/urllib3/util/retry.py:592\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m    591\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m new_retry\u001b[38;5;241m.\u001b[39mis_exhausted():\n\u001b[0;32m--> 592\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause))\n\u001b[1;32m    594\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIncremented Retry for (url=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, url, new_retry)\n",
      "\u001b[0;31mMaxRetryError\u001b[0m: HTTPSConnectionPool(host='www.ted.com', port=443): Max retries exceeded with url: /talks/ted_ed_how_to_manage_your_emotions/transcript (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x17ebdb880>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known'))",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mConnectionError\u001b[0m                           Traceback (most recent call last)",
      "Input \u001b[0;32mIn [191]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m responses \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mSeries([scraper\u001b[38;5;241m.\u001b[39mcollect_talk_data(test\u001b[38;5;241m.\u001b[39miloc[i][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlink\u001b[39m\u001b[38;5;124m'\u001b[39m]) \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(test\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m])])\n",
      "Input \u001b[0;32mIn [191]\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[0;32m----> 1\u001b[0m responses \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mSeries([\u001b[43mscraper\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect_talk_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43miloc\u001b[49m\u001b[43m[\u001b[49m\u001b[43mi\u001b[49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mlink\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(test\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m])])\n",
      "Input \u001b[0;32mIn [183]\u001b[0m, in \u001b[0;36mScraper.collect_talk_data\u001b[0;34m(self, link, raw_input)\u001b[0m\n\u001b[1;32m     83\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcollect_talk_data\u001b[39m(\u001b[38;5;28mself\u001b[39m, link: \u001b[38;5;28mstr\u001b[39m, raw_input\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m     84\u001b[0m     \u001b[38;5;66;03m#takes a link (in str format) that points to a specific ted talk.\u001b[39;00m\n\u001b[1;32m     85\u001b[0m     \u001b[38;5;66;03m#will then go to that ted talk and collect relevant information\u001b[39;00m\n\u001b[1;32m     86\u001b[0m     \u001b[38;5;66;03m#returns a dictionary where the key is the name of the data and the value is the data itself\u001b[39;00m\n\u001b[1;32m     87\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m link:\n\u001b[0;32m---> 88\u001b[0m         raw \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlink\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mtext\n\u001b[1;32m     89\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     90\u001b[0m         raw \u001b[38;5;241m=\u001b[39m raw_input\n",
      "Input \u001b[0;32mIn [183]\u001b[0m, in \u001b[0;36mScraper.get_\u001b[0;34m(self, url)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/talks/\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m url:\n\u001b[1;32m     32\u001b[0m     headers\u001b[38;5;241m.\u001b[39mupdate({\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124muser-agent\u001b[39m\u001b[38;5;124m\"\u001b[39m :\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36\u001b[39m\u001b[38;5;124m\"\u001b[39m})\n\u001b[0;32m---> 33\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     34\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m r\u001b[38;5;241m.\u001b[39mstatus_code \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m200\u001b[39m:\n\u001b[1;32m     35\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m r\u001b[38;5;241m.\u001b[39mstatus_code \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m429\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m r\u001b[38;5;241m.\u001b[39mstatus_code \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m502\u001b[39m:\n\u001b[1;32m     36\u001b[0m         \u001b[38;5;66;03m#print(\"Error 429 recieved, sleeping for RAND_LONG, then trying again\")\u001b[39;00m\n",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/requests/api.py:73\u001b[0m, in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget\u001b[39m(url, params\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     63\u001b[0m     \u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Sends a GET request.\u001b[39;00m\n\u001b[1;32m     64\u001b[0m \n\u001b[1;32m     65\u001b[0m \u001b[38;5;124;03m    :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     70\u001b[0m \u001b[38;5;124;03m    :rtype: requests.Response\u001b[39;00m\n\u001b[1;32m     71\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m---> 73\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mget\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/requests/api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[1;32m     56\u001b[0m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[1;32m     57\u001b[0m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[1;32m     58\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m sessions\u001b[38;5;241m.\u001b[39mSession() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[0;32m---> 59\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/requests/sessions.py:587\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    582\u001b[0m send_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m    583\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimeout\u001b[39m\u001b[38;5;124m\"\u001b[39m: timeout,\n\u001b[1;32m    584\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m: allow_redirects,\n\u001b[1;32m    585\u001b[0m }\n\u001b[1;32m    586\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 587\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    589\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/requests/sessions.py:701\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    698\u001b[0m start \u001b[38;5;241m=\u001b[39m preferred_clock()\n\u001b[1;32m    700\u001b[0m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[0;32m--> 701\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    703\u001b[0m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m    704\u001b[0m elapsed \u001b[38;5;241m=\u001b[39m preferred_clock() \u001b[38;5;241m-\u001b[39m start\n",
      "File \u001b[0;32m~/miniforge3/envs/classbot/lib/python3.10/site-packages/requests/adapters.py:565\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    561\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e\u001b[38;5;241m.\u001b[39mreason, _SSLError):\n\u001b[1;32m    562\u001b[0m         \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[1;32m    563\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request\u001b[38;5;241m=\u001b[39mrequest)\n\u001b[0;32m--> 565\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request\u001b[38;5;241m=\u001b[39mrequest)\n\u001b[1;32m    567\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    568\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request\u001b[38;5;241m=\u001b[39mrequest)\n",
      "\u001b[0;31mConnectionError\u001b[0m: HTTPSConnectionPool(host='www.ted.com', port=443): Max retries exceeded with url: /talks/ted_ed_how_to_manage_your_emotions/transcript (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x17ebdb880>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known'))"
     ]
    }
   ],
   "source": [
    "responses = pd.Series([scraper.collect_talk_data(test.iloc[i]['link']) for i in range(test.shape[0])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link</th>\n",
       "      <th>raw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.ted.com/talks/megan_alubicki_flick...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.ted.com/talks/kate_kahle_why_autis...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.ted.com/talks/malachy_mchugh_how_s...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.ted.com/talks/alona_fyshe_does_ai_...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.ted.com/talks/dan_shipper_the_unex...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>https://www.ted.com/talks/dan_finkel_this_one_...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>https://www.ted.com/talks/melanie_charles_damb...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>https://www.ted.com/talks/ted_countdown_how_do...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>https://www.ted.com/talks/alexandra_horowitz_w...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>https://www.ted.com/talks/charles_wallace_when...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>https://www.ted.com/talks/john_paget_how_to_ke...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>https://www.ted.com/talks/wendy_suzuki_how_to_...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>https://www.ted.com/talks/birte_kristiansen_an...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>https://www.ted.com/talks/emily_grubert_what_h...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>https://www.ted.com/talks/cristina_junqueira_a...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>https://www.ted.com/talks/jerome_hunter_3_skil...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>https://www.ted.com/talks/antonio_cataldo_what...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>https://www.ted.com/talks/paul_bloom_the_surpr...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>https://www.ted.com/talks/nedra_glover_tawwab_...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>https://www.ted.com/talks/ted_ed_what_happens_...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>https://www.ted.com/talks/ordinary_things_who_...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>https://www.ted.com/talks/catherine_price_3_el...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>https://www.ted.com/talks/nathalia_holt_the_hi...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>https://www.ted.com/talks/phnam_bagley_gourmet...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>https://www.ted.com/talks/william_t_taylor_how...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>https://www.ted.com/talks/kris_alexander_how_v...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>https://www.ted.com/talks/rob_cross_how_to_be_...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>https://www.ted.com/talks/stephanie_honchell_s...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>https://www.ted.com/talks/rebekah_shirley_the_...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>https://www.ted.com/talks/michael_middlebrooks...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>https://www.ted.com/talks/tiffani_ashley_bell_...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>https://www.ted.com/talks/jean_baptiste_p_koeh...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>https://www.ted.com/talks/gala_jackson_how_to_...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>https://www.ted.com/talks/sharif_el_mekki_revi...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>https://www.ted.com/talks/ted_ed_what_if_you_e...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>https://www.ted.com/talks/saad_bhamla_the_fasc...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>https://www.ted.com/talks/jacqueline_dibiasie_...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>https://www.ted.com/talks/kate_ackerman_what_g...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>https://www.ted.com/talks/brendan_rogers_and_j...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>https://www.ted.com/talks/tessa_clarke_the_sur...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>https://www.ted.com/talks/ayelet_fishbach_4_pr...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>https://www.ted.com/talks/mollie_godfrey_one_o...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>https://www.ted.com/talks/jan_wurzbacher_the_m...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>https://www.ted.com/talks/kristen_nguyen_3_ste...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>https://www.ted.com/talks/ted_ed_how_to_manage...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>https://www.ted.com/talks/jade_begay_climate_a...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>https://www.ted.com/talks/mesmin_destin_how_ev...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>https://www.ted.com/talks/shannon_n_tessier_ca...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>https://www.ted.com/talks/ksenia_petrichenko_w...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>https://www.ted.com/talks/ashley_clift_jenning...</td>\n",
       "      <td>&lt;!DOCTYPE html&gt;&lt;html&gt;&lt;head&gt;&lt;link rel=\"preconne...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 link  \\\n",
       "0   https://www.ted.com/talks/megan_alubicki_flick...   \n",
       "1   https://www.ted.com/talks/kate_kahle_why_autis...   \n",
       "2   https://www.ted.com/talks/malachy_mchugh_how_s...   \n",
       "3   https://www.ted.com/talks/alona_fyshe_does_ai_...   \n",
       "4   https://www.ted.com/talks/dan_shipper_the_unex...   \n",
       "5   https://www.ted.com/talks/dan_finkel_this_one_...   \n",
       "6   https://www.ted.com/talks/melanie_charles_damb...   \n",
       "7   https://www.ted.com/talks/ted_countdown_how_do...   \n",
       "8   https://www.ted.com/talks/alexandra_horowitz_w...   \n",
       "9   https://www.ted.com/talks/charles_wallace_when...   \n",
       "10  https://www.ted.com/talks/john_paget_how_to_ke...   \n",
       "11  https://www.ted.com/talks/wendy_suzuki_how_to_...   \n",
       "12  https://www.ted.com/talks/birte_kristiansen_an...   \n",
       "13  https://www.ted.com/talks/emily_grubert_what_h...   \n",
       "14  https://www.ted.com/talks/cristina_junqueira_a...   \n",
       "15  https://www.ted.com/talks/jerome_hunter_3_skil...   \n",
       "16  https://www.ted.com/talks/antonio_cataldo_what...   \n",
       "17  https://www.ted.com/talks/paul_bloom_the_surpr...   \n",
       "18  https://www.ted.com/talks/nedra_glover_tawwab_...   \n",
       "19  https://www.ted.com/talks/ted_ed_what_happens_...   \n",
       "20  https://www.ted.com/talks/ordinary_things_who_...   \n",
       "21  https://www.ted.com/talks/catherine_price_3_el...   \n",
       "22  https://www.ted.com/talks/nathalia_holt_the_hi...   \n",
       "23  https://www.ted.com/talks/phnam_bagley_gourmet...   \n",
       "24  https://www.ted.com/talks/william_t_taylor_how...   \n",
       "25  https://www.ted.com/talks/kris_alexander_how_v...   \n",
       "26  https://www.ted.com/talks/rob_cross_how_to_be_...   \n",
       "27  https://www.ted.com/talks/stephanie_honchell_s...   \n",
       "28  https://www.ted.com/talks/rebekah_shirley_the_...   \n",
       "29  https://www.ted.com/talks/michael_middlebrooks...   \n",
       "30  https://www.ted.com/talks/tiffani_ashley_bell_...   \n",
       "31  https://www.ted.com/talks/jean_baptiste_p_koeh...   \n",
       "32  https://www.ted.com/talks/gala_jackson_how_to_...   \n",
       "33  https://www.ted.com/talks/sharif_el_mekki_revi...   \n",
       "34  https://www.ted.com/talks/ted_ed_what_if_you_e...   \n",
       "35  https://www.ted.com/talks/saad_bhamla_the_fasc...   \n",
       "36  https://www.ted.com/talks/jacqueline_dibiasie_...   \n",
       "37  https://www.ted.com/talks/kate_ackerman_what_g...   \n",
       "38  https://www.ted.com/talks/brendan_rogers_and_j...   \n",
       "39  https://www.ted.com/talks/tessa_clarke_the_sur...   \n",
       "40  https://www.ted.com/talks/ayelet_fishbach_4_pr...   \n",
       "41  https://www.ted.com/talks/mollie_godfrey_one_o...   \n",
       "42  https://www.ted.com/talks/jan_wurzbacher_the_m...   \n",
       "43  https://www.ted.com/talks/kristen_nguyen_3_ste...   \n",
       "44  https://www.ted.com/talks/ted_ed_how_to_manage...   \n",
       "45  https://www.ted.com/talks/jade_begay_climate_a...   \n",
       "46  https://www.ted.com/talks/mesmin_destin_how_ev...   \n",
       "47  https://www.ted.com/talks/shannon_n_tessier_ca...   \n",
       "48  https://www.ted.com/talks/ksenia_petrichenko_w...   \n",
       "49  https://www.ted.com/talks/ashley_clift_jenning...   \n",
       "\n",
       "                                                  raw  \n",
       "0   <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "1   <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "2   <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "3   <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "4   <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "5   <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "6   <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "7   <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "8   <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "9   <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "10  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "11  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "12  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "13  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "14  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "15  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "16  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "17  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "18  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "19  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "20  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "21  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "22  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "23  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "24  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "25  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "26  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "27  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "28  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "29  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "30  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "31  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "32  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "33  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "34  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "35  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "36  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "37  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "38  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "39  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "40  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "41  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "42  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "43  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "44  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "45  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "46  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "47  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "48  <!DOCTYPE html><html><head><link rel=\"preconne...  \n",
       "49  <!DOCTYPE html><html><head><link rel=\"preconne...  "
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "vscode": {
   "interpreter": {
    "hash": "989611b5c64cdf48eb29a06ae705326841fca9fe728dd4a50fa0ef1e755e3e66"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}