web_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import functions as f\n",
    "\n",
    "import requests\n",
    "import re\n",
    "from collections import Counter\n",
    "from bs4 import BeautifulSoup, NavigableString"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Website scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_urls = ['https://www.pitt.edu/~dash/folktexts.html',\n",
    "             'https://www.pitt.edu/~dash/folktexts2.html']\n",
    "base_url = 'https://www.pitt.edu/~dash/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "links = []\n",
    "\n",
    "for url in main_urls:\n",
    "    page = requests.get(url)\n",
    "    soup = BeautifulSoup(page.content, 'html.parser')\n",
    "\n",
    "    for a in soup.find_all('a'):\n",
    "        a_href = a.get('href')\n",
    "        links.append(a_href)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "internal_links = [link.split('#')[0] for link in links if link != None and 'folktexts' not in link and 'http' not in link]\n",
    "internal_links.remove('')\n",
    "internal_links = list(set(internal_links))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checkpoint\n",
    "\n",
    "#save_pickle(links, 'data/links.pickle')\n",
    "#save_pickle(internal_links, 'data/internal_links.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 %\n",
      "5.747126436781609 %\n",
      "11.494252873563218 %\n",
      "17.24137931034483 %\n",
      "22.988505747126435 %\n",
      "28.735632183908045 %\n",
      "34.48275862068966 %\n",
      "40.229885057471265 %\n",
      "45.97701149425287 %\n",
      "51.724137931034484 %\n",
      "57.47126436781609 %\n",
      "63.2183908045977 %\n",
      "68.96551724137932 %\n",
      "74.71264367816092 %\n",
      "80.45977011494253 %\n",
      "86.20689655172414 %\n",
      "91.95402298850574 %\n",
      "97.70114942528735 %\n"
     ]
    }
   ],
   "source": [
    "pages = []\n",
    "total_progress = len(internal_links)\n",
    "p = 0\n",
    "\n",
    "for link in internal_links:\n",
    "    progress = p*100/total_progress\n",
    "    \n",
    "    sub_url = base_url + link\n",
    "    sub_page = requests.get(sub_url)\n",
    "    \n",
    "    pages.append(sub_page)\n",
    "    \n",
    "    if p in range(0, total_progress, 20):\n",
    "        print(progress, '%')\n",
    "    p += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checkpoint\n",
    "\n",
    "#save_pickle(pages, 'data/pages.pickle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pages = f.load_pickle('data/pages.pickle')\n",
    "internal_links = f.load_pickle('data/internal_links.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "soup_list = [BeautifulSoup(page.content, 'html.parser') for page in pages]\n",
    "print(len(internal_links)==len(soup_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "h2_count = {}\n",
    "h2_content = []\n",
    "\n",
    "for i in range(len(internal_links)):\n",
    "    h2_list = soup_list[i].find_all('h2')\n",
    "    h2_content.extend([h2.text.strip().lower() for h2 in h2_list])\n",
    "    h2_count[internal_links[i]] = len(h2_list)\n",
    "\n",
    "h2_content_count = Counter(h2_content).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "h2_black_list = ['contents', 'links to related sites', 'related links', 'links to related tales', 'notes and bibliography', 'links']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pages with h2: 313, Pages without h2: 35\n"
     ]
    }
   ],
   "source": [
    "type1 = [k for k,v in h2_count.items() if v != 0]\n",
    "type2 = [k for k,v in h2_count.items() if v == 0]\n",
    "print('Pages with h2: %d, Pages without h2: %d' % (len(type1), len(type2)))\n",
    "\n",
    "content_dict = {internal_links[i]: soup_list[i] for i in range(len(internal_links))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "type1_content = {}\n",
    "\n",
    "for page in type1:\n",
    "    h2_list = content_dict[page].find_all('h2')\n",
    "    h2_text = [h2.text.strip().lower() for h2 in h2_list]\n",
    "    type1_content[page] = h2_text.count('contents') + h2_text.count('table of contents')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pages with contents section: 262, Pages without contents section: 51\n"
     ]
    }
   ],
   "source": [
    "type1_with_content = [k for k,v in type1_content.items() if v != 0]\n",
    "type1_without_content = [k for k,v in type1_content.items() if v == 0]\n",
    "\n",
    "print('Pages with contents section: %d, Pages without contents section: %d' % (len(type1_with_content), len(type1_without_content)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "page_skip = 0\n",
    "titles = []\n",
    "num_words = []\n",
    "exclude = ['h3']\n",
    "end = soup_list[0].new_tag('hr')\n",
    "\n",
    "for page in type1_with_content:\n",
    "    type1_page_links = content_dict[page].find_all('a', attrs={'name': True})\n",
    "\n",
    "    for a in type1_page_links[1:]:\n",
    "        start = a.find_parent('h2')\n",
    "        if start != None and f.exclude_black_list(start.get_text(), h2_black_list):\n",
    "            title, titles = f.format_file_name(start.get_text(), titles)\n",
    "            text, num_words = f.format_text(start, end, exclude, num_words)\n",
    "            f.save_txt(text, 'data/tales/' + title) if num_words[-1] >= 15 else num_words.pop()\n",
    "        else:\n",
    "            page_skip += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique files with fairy tales: 2314\n",
      "Total number of words in all fairy tales: 1608500\n",
      "Average number of words in a fairy tale: 695\n",
      "Number of words in the shortest story: 1, in the longest story: 11084\n"
     ]
    }
   ],
   "source": [
    "f.words_summary(num_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KAGGLE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "kaggle_file = f.read_txt('data/kaggle_tales.txt')\n",
    "\n",
    "patterns = [re.compile('_The Moral_.*?_Another_.*?\\n\\n\\n', re.DOTALL),\n",
    "            re.compile('\\[Illustration.*?\\]', re.DOTALL),\n",
    "            re.compile(r'_.*?_\\n\\n')]\n",
    "\n",
    "for pattern in patterns:\n",
    "    kaggle_file = re.sub(pattern, '', kaggle_file)\n",
    "tales = kaggle_file.split('\\n\\n\\n\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "kaggle_num_words = []\n",
    "\n",
    "for tale in tales:\n",
    "    tale = tale.replace('\\n\\n', '\\n').strip()\n",
    "    if len(tale)>0:\n",
    "        tale_split = tale.split('\\n')\n",
    "        title, titles = f.format_file_name(tale_split[0], titles)\n",
    "        text = '\\n'.join(tale_split[1:])\n",
    "        kaggle_num_words.append(len(text.split(' ')))\n",
    "        f.save_txt(text, 'data/tales/' + title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique files with fairy tales: 836\n",
      "Total number of words in all fairy tales: 2168464\n",
      "Average number of words in a fairy tale: 2593\n",
      "Number of words in the shortest story: 1, in the longest story: 105959\n"
     ]
    }
   ],
   "source": [
    "f.words_summary(kaggle_num_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}