climate_abbreviation_extraction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "climate_abbreviation_extraction.ipynb",
      "provenance": [],
      "mount_file_id": "1SCnQqmGC8KLTgAMjJQPBAHuKCFFX7evS",
      "authorship_tag": "ABX9TyMCDbchlASXIwWzQzFYYSD/",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ananyas168/petermr/blob/main/climate_abbreviation_extraction.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hqo4TDhn-3mp",
        "outputId": "14ad0aab-f689-4cff-ba5d-03dcd7b20d06"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'pyami'...\n",
            "remote: Enumerating objects: 5735, done.\u001b[K\n",
            "remote: Counting objects: 100% (227/227), done.\u001b[K\n",
            "remote: Compressing objects: 100% (161/161), done.\u001b[K\n",
            "remote: Total 5735 (delta 130), reused 140 (delta 60), pack-reused 5508\u001b[K\n",
            "Receiving objects: 100% (5735/5735), 93.86 MiB | 13.98 MiB/s, done.\n",
            "Resolving deltas: 100% (2434/2434), done.\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/petermr/pyami.git"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install abbreviations"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2KGJREbNHVpU",
        "outputId": "68af43ef-b421-4960-e7cc-5659dc476401"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting abbreviations\n",
            "  Downloading abbreviations-0.2.5-py3-none-any.whl (5.7 kB)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from abbreviations) (2019.12.20)\n",
            "Installing collected packages: abbreviations\n",
            "Successfully installed abbreviations-0.2.5\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/drive/MyDrive/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mMiBsyxpRhUM",
        "outputId": "3188921b-5e58-40ea-ac51-75658078803b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from bs4 import BeautifulSoup\n",
        "from abbreviations import schwartz_hearst\n",
        "import os\n",
        "for i in range(1,300):\n",
        "  path = '/content/'\n",
        "  head_folder = path + f'Pages/page_{i}' +'/'\n",
        "  if not os.path.exists(head_folder):\n",
        "    os.makedirs(head_folder)\n",
        "  print(f'{i}---------------')\n",
        "  with open(f'/content/pyami/temp/html/page.{i}.html', 'r') as f:\n",
        "      html = f.read()\n",
        "      soup = BeautifulSoup(html, features=\"html.parser\")\n",
        "\n",
        "      # kill all script and style elements\n",
        "      for script in soup([\"script\", \"style\"]):\n",
        "          script.extract()    # rip it out\n",
        "\n",
        "      # get text\n",
        "      text = soup.get_text()\n",
        "\n",
        "      # break into lines and remove leading and trailing space on each\n",
        "      lines = (line.strip() for line in text.splitlines())\n",
        "      # break multi-headlines into a line each\n",
        "      chunks = (phrase.strip() for line in lines for phrase in line.split(\"  \"))\n",
        "      # drop blank lines\n",
        "      text = '\\n'.join(chunk for chunk in chunks if chunk)\n",
        "\n",
        "      #print(text)\n",
        "      pairs = schwartz_hearst.extract_abbreviation_definition_pairs(doc_text= text)\n",
        "      #print('pairs',pairs)\n",
        "      \n",
        "      TEXT_ = f'page_{i}_text.txt'\n",
        "      DICTIONARY = f'page_{i}_dictionary.txt'\n",
        "      with open(head_folder + TEXT_, 'w') as file:\n",
        "          file.write(text)\n",
        "      \n",
        "      with open(head_folder + DICTIONARY, 'w') as file:\n",
        "          for key, value in pairs.items():\n",
        "            #print(key, value )\n",
        "            file.write('{'+str(key)+' : '+ str(value)+\"}\")"
      ],
      "metadata": {
        "id": "GRZnZgpw_ju6"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}