scrape_data_final.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Shreya2803/Bengali-Language-Model/blob/main/scrape_data_final.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cUHDfCHb0UTd",
        "outputId": "b631e439-2674-4611-c303-b619c5f35d01"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "cfQl0r8BXjwe",
        "outputId": "860f7e82-a2a8-4908-af52-cd117dc84f20",
        "scrolled": true
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting requests_html\n",
            "  Downloading requests_html-0.10.0-py3-none-any.whl (13 kB)\n",
            "Collecting fake-useragent\n",
            "  Downloading fake_useragent-0.1.14-py3-none-any.whl (13 kB)\n",
            "Requirement already satisfied: bs4 in /usr/local/lib/python3.7/dist-packages (from requests_html) (0.0.1)\n",
            "Collecting w3lib\n",
            "  Downloading w3lib-2.0.1-py3-none-any.whl (20 kB)\n",
            "Collecting pyppeteer>=0.0.14\n",
            "  Downloading pyppeteer-1.0.2-py3-none-any.whl (83 kB)\n",
            "\u001b[K     |████████████████████████████████| 83 kB 1.6 MB/s \n",
            "\u001b[?25hCollecting parse\n",
            "  Downloading parse-1.19.0.tar.gz (30 kB)\n",
            "Collecting pyquery\n",
            "  Downloading pyquery-1.4.3-py3-none-any.whl (22 kB)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from requests_html) (2.23.0)\n",
            "Requirement already satisfied: appdirs<2.0.0,>=1.4.3 in /usr/local/lib/python3.7/dist-packages (from pyppeteer>=0.0.14->requests_html) (1.4.4)\n",
            "Requirement already satisfied: importlib-metadata>=1.4 in /usr/local/lib/python3.7/dist-packages (from pyppeteer>=0.0.14->requests_html) (4.13.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.42.1 in /usr/local/lib/python3.7/dist-packages (from pyppeteer>=0.0.14->requests_html) (4.64.1)\n",
            "Requirement already satisfied: certifi>=2021 in /usr/local/lib/python3.7/dist-packages (from pyppeteer>=0.0.14->requests_html) (2022.9.24)\n",
            "Collecting pyee<9.0.0,>=8.1.0\n",
            "  Downloading pyee-8.2.2-py2.py3-none-any.whl (12 kB)\n",
            "Collecting urllib3<2.0.0,>=1.25.8\n",
            "  Downloading urllib3-1.26.12-py2.py3-none-any.whl (140 kB)\n",
            "\u001b[K     |████████████████████████████████| 140 kB 38.2 MB/s \n",
            "\u001b[?25hCollecting websockets<11.0,>=10.0\n",
            "  Downloading websockets-10.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (106 kB)\n",
            "\u001b[K     |████████████████████████████████| 106 kB 53.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=1.4->pyppeteer>=0.0.14->requests_html) (3.10.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=1.4->pyppeteer>=0.0.14->requests_html) (4.1.1)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (from bs4->requests_html) (4.6.3)\n",
            "Collecting cssselect>0.7.9\n",
            "  Downloading cssselect-1.2.0-py2.py3-none-any.whl (18 kB)\n",
            "Requirement already satisfied: lxml>=2.1 in /usr/local/lib/python3.7/dist-packages (from pyquery->requests_html) (4.9.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->requests_html) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->requests_html) (3.0.4)\n",
            "Collecting urllib3<2.0.0,>=1.25.8\n",
            "  Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 51.9 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: parse\n",
            "  Building wheel for parse (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for parse: filename=parse-1.19.0-py3-none-any.whl size=24590 sha256=b58d188136e7701a81b735d556df2d97a7b84af4653c0654d7fa71f3d4ea2164\n",
            "  Stored in directory: /root/.cache/pip/wheels/9c/aa/cc/f2228050ccb40f22144b073f15a2c84f11204f29fc0dce028e\n",
            "Successfully built parse\n",
            "Installing collected packages: websockets, urllib3, pyee, cssselect, w3lib, pyquery, pyppeteer, parse, fake-useragent, requests-html\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed cssselect-1.2.0 fake-useragent-0.1.14 parse-1.19.0 pyee-8.2.2 pyppeteer-1.0.2 pyquery-1.4.3 requests-html-0.10.0 urllib3-1.25.11 w3lib-2.0.1 websockets-10.4\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "urllib3"
                ]
              }
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: google in /usr/local/lib/python3.7/dist-packages (2.0.3)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (from google) (4.6.3)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting selenium\n",
            "  Downloading selenium-4.6.0-py3-none-any.whl (5.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.2 MB 5.4 MB/s \n",
            "\u001b[?25hCollecting urllib3[socks]~=1.26\n",
            "  Using cached urllib3-1.26.12-py2.py3-none-any.whl (140 kB)\n",
            "Requirement already satisfied: certifi>=2021.10.8 in /usr/local/lib/python3.7/dist-packages (from selenium) (2022.9.24)\n",
            "Collecting trio-websocket~=0.9\n",
            "  Downloading trio_websocket-0.9.2-py3-none-any.whl (16 kB)\n",
            "Collecting trio~=0.17\n",
            "  Downloading trio-0.22.0-py3-none-any.whl (384 kB)\n",
            "\u001b[K     |████████████████████████████████| 384 kB 56.6 MB/s \n",
            "\u001b[?25hCollecting async-generator>=1.9\n",
            "  Downloading async_generator-1.10-py3-none-any.whl (18 kB)\n",
            "Collecting exceptiongroup>=1.0.0rc9\n",
            "  Downloading exceptiongroup-1.0.1-py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.7/dist-packages (from trio~=0.17->selenium) (22.1.0)\n",
            "Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.7/dist-packages (from trio~=0.17->selenium) (2.4.0)\n",
            "Collecting outcome\n",
            "  Downloading outcome-1.2.0-py2.py3-none-any.whl (9.7 kB)\n",
            "Collecting sniffio\n",
            "  Downloading sniffio-1.3.0-py3-none-any.whl (10 kB)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.7/dist-packages (from trio~=0.17->selenium) (2.10)\n",
            "Collecting wsproto>=0.14\n",
            "  Downloading wsproto-1.2.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: PySocks!=1.5.7,<2.0,>=1.5.6 in /usr/local/lib/python3.7/dist-packages (from urllib3[socks]~=1.26->selenium) (1.7.1)\n",
            "Collecting h11<1,>=0.9.0\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[K     |████████████████████████████████| 58 kB 5.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from h11<1,>=0.9.0->wsproto>=0.14->trio-websocket~=0.9->selenium) (4.1.1)\n",
            "Installing collected packages: sniffio, outcome, h11, exceptiongroup, async-generator, wsproto, urllib3, trio, trio-websocket, selenium\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.25.11\n",
            "    Uninstalling urllib3-1.25.11:\n",
            "      Successfully uninstalled urllib3-1.25.11\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "requests 2.23.0 requires urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1, but you have urllib3 1.26.12 which is incompatible.\u001b[0m\n",
            "Successfully installed async-generator-1.10 exceptiongroup-1.0.1 h11-0.14.0 outcome-1.2.0 selenium-4.6.0 sniffio-1.3.0 trio-0.22.0 trio-websocket-0.9.2 urllib3-1.26.12 wsproto-1.2.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "urllib3"
                ]
              }
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/trio/_core/_multierror.py:411: RuntimeWarning: IPython detected, but you already have a custom exception handler installed. I'll skip installing Trio's custom handler, but this means exception groups will not show full tracebacks.\n",
            "  category=RuntimeWarning,\n"
          ]
        }
      ],
      "source": [
        "import requests\n",
        "!pip install requests_html\n",
        "!pip install google\n",
        "!pip install selenium\n",
        "from requests_html import HTMLSession\n",
        "from bs4 import BeautifulSoup\n",
        "from googlesearch import search\n",
        "#import pdfplumber\n",
        "from IPython.display import display, HTML, Javascript\n",
        "import ipywidgets as widgets\n",
        "from selenium import webdriver\n",
        "import webbrowser\n",
        "from IPython.display import clear_output\n",
        "import os, pickle\n",
        "import json\n",
        "import time"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QKxQceYiXjwh"
      },
      "outputs": [],
      "source": [
        "class ScrapeData():\n",
        "    \"\"\"\n",
        "    Python tool to scrap text data from different languages. 3 options available:\n",
        "    \n",
        "    - scrape from link: provided with a link, the tool scrapes through the text body from the webpage \n",
        "      corresponding to the link.\n",
        "      \n",
        "    - scrape from keyword: provided with a keyword, the tool performs a google search and retrieves text from \n",
        "      a priority domain webpage(eg: wikipedia) or top google search result.\n",
        "      \n",
        "    - scrape from document: provided with a pdf document, the tools accesses the text using pdfplumber \n",
        "      python package. Note that some portion of the text may not be returned properly.\n",
        "    \"\"\"\n",
        "    \n",
        "    def __init__(self):\n",
        "        \"\"\"\n",
        "        Initialise parameters for google search and priority website \n",
        "        \"\"\"\n",
        "        \n",
        "        self.parser = 'html.parser'\n",
        "        self.tld = 'co.in'\n",
        "        self.search_num = 10\n",
        "        self.search_stop = 10\n",
        "        self.pause = 2\n",
        "        self.search_priority = 'wikipedia'\n",
        "        self.auto_return_index = 0\n",
        "        \n",
        "    def read_from_link(self, link, replace_list=['\\n']):\n",
        "        \"\"\"\n",
        "        This function accesses the text content from a webpage link using beautiful soup. To clean the text, \n",
        "        provide the list of charecters to be removed in replace_list.\n",
        "        This may not work for js webpages, cloudfare ddos protected pages etc\n",
        "        \"\"\"\n",
        "        #use header if access is denied.\n",
        "        headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) '\\\n",
        "           'AppleWebKit/537.36 (KHTML, like Gecko) '\\\n",
        "           'Chrome/75.0.3770.80 Safari/537.36'}\n",
        "\n",
        "        page = requests.get(link, headers=headers)\n",
        "        page = requests.get(link)\n",
        "        soup = BeautifulSoup(page.content, self.parser)\n",
        "        data = []\n",
        "        for i in range(len(soup.find_all('p'))):\n",
        "            text = soup.find_all('p')[i].get_text()\n",
        "            for j in range(len(replace_list)):\n",
        "                text = text.replace(replace_list[j],'')\n",
        "            if len(text)>0:\n",
        "                data.append(text)\n",
        "        return data\n",
        "\n",
        "    def google_search(self, search_keyword, priority=None):\n",
        "        \"\"\"\n",
        "        This function performs google search on the input keyword. Priority can be provided to a particular \n",
        "        website (Eg: wikipedia) \n",
        "        \"\"\"\n",
        "        search_links = []\n",
        "        for link in search(search_keyword, \n",
        "                           tld=self.tld, \n",
        "                           num=self.search_num, \n",
        "                           stop=self.search_stop, \n",
        "                           pause=self.pause):\n",
        "            search_links.append(link)\n",
        "            \n",
        "            if priority is not None:\n",
        "                if self.search_priority in link:\n",
        "                    return link\n",
        "        return search_links[self.auto_return_index]\n",
        "                \n",
        "    \n",
        "    def read_from_doc(self, document):\n",
        "        \"\"\"\n",
        "        This function extracts text from pdf using pdfplumber tool.\n",
        "        \"\"\"\n",
        "        data = []\n",
        "        with pdfplumber.open(document) as pdf:\n",
        "            for i in range(len(pdf.pages)):\n",
        "                page = pdf.pages[i]\n",
        "                text = page.extract_text()\n",
        "                if text is not None:\n",
        "                    text = text.replace('\\n', ' ')\n",
        "                    data.append(text) \n",
        "        return data\n",
        "    \n",
        "    def read_page(self, search_keyword=None, link=None, document=None):\n",
        "        if search_keyword == link == document == None:\n",
        "            raise Exception('Provide link, keyword or document to scrape from')\n",
        "            \n",
        "        if search_keyword is not None:\n",
        "            keyword_link = self.google_search(search_keyword, priority=self.search_priority)\n",
        "            text = self.read_from_link(keyword_link)\n",
        "            return(text)\n",
        "            \n",
        "        if link:\n",
        "            text = self.read_from_link(link)\n",
        "            return(text)\n",
        "                \n",
        "        if document:\n",
        "            text = self.read_from_doc(document)\n",
        "            print(f'{len(text)} pages found')\n",
        "            return(text)\n",
        "            \n",
        "scrape_tool = ScrapeData()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dym1bZw3Xjwj"
      },
      "source": [
        "<h4>Read from english website</h4>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-IOEmbwlXjwk",
        "outputId": "be2bcbb9-d346-4ef6-c58f-ad5773efed9a",
        "scrolled": true
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['Agriculture or farming is the practice of cultivating plants and livestock.[1] Agriculture was the key development in the rise of sedentary human civilization, whereby farming of domesticated species created food surpluses that enabled people to live in cities. The history of agriculture began thousands of years ago. After gathering wild grains beginning at least 105,000 years ago, nascent farmers began to plant them around 11,500 years ago. Pigs, sheep, and cattle were domesticated over 10,000 years ago. Plants were independently cultivated in at least 11 regions of the world. Industrial agriculture based on large-scale monoculture in the twentieth century came to dominate agricultural output, though about 2 billion people still depended on subsistence agriculture.',\n",
              " \"The major agricultural products can be broadly grouped into foods, fibers, fuels, and raw materials (such as rubber). Food classes include cereals (grains), vegetables, fruits, oils, meat, milk, eggs, and fungi. Over one-third of the world's workers are employed in agriculture, second only to the service sector, although in recent decades, the global trend of a decreasing number of agricultural workers continues, especially in developing countries, where smallholding is being overtaken by industrial agriculture and mechanization that brings an enormous crop yield increase.\",\n",
              " 'Modern agronomy, plant breeding, agrochemicals such as pesticides and fertilizers, and technological developments have sharply increased crop yields, but cause ecological and environmental damage. Selective breeding and modern practices in animal husbandry have similarly increased the output of meat but have raised concerns about animal welfare and environmental damage. Environmental issues include contributions to global warming, depletion of aquifers, deforestation, antibiotic resistance, and other agricultural pollution. Agriculture is both a cause of and sensitive to environmental degradation, such as biodiversity loss, desertification, soil degradation, and global warming, all of which can cause decreases in crop yield. Genetically modified organisms are widely used, although some are banned in certain countries.',\n",
              " 'The word agriculture is a late Middle English adaptation of Latin agricultūra, from ager \\'field\\' and cultūra \\'cultivation\\' or \\'growing\\'.[2] While agriculture usually refers to human activities, certain species of ant,[3][4] termite and beetle have been cultivating crops for up to 60 million years.[5] Agriculture is defined with varying scopes, in its broadest sense using natural resources to \"produce commodities which maintain life, including food, fiber, forest products, horticultural crops, and their related services\".[6] Thus defined, it includes arable farming, horticulture, animal husbandry and forestry, but horticulture and forestry are in practice often excluded.[6]It may also be broadly decomposed into plant agriculture, which concerns the cultivation of useful plants,[7] and animal agriculture, the production of agricultural animals.[8]',\n",
              " 'The development of agriculture enabled the human population to grow many times larger than could be sustained by hunting and gathering.[11] Agriculture began independently in different parts of the globe,[12] and included a diverse range of taxa, in at least 11 separate centers of origin.[9] Wild grains were collected and eaten from at least 105,000 years ago.[13] In the Paleolithic Levant, 23,000 years ago, cereals cultivation of emmer, barley, and oats has been observed near the sea of Galilee.[14][15] Rice was domesticated in China between 11,500 and 6,200 BC with the earliest known cultivation from 5,700 BC,[16] followed by mung, soy and azuki beans. Sheep were domesticated in Mesopotamia between 13,000 and 11,000 years ago.[17] Cattle were domesticated from the wild aurochs in the areas of modern Turkey and Pakistan some 10,500 years ago.[18] Pig production emerged in Eurasia, including Europe, East Asia and Southwest Asia,[19] where wild boar were first domesticated about 10,500 years ago.[20] In the Andes of South America, the potato was domesticated between 10,000 and 7,000 years ago, along with beans, coca, llamas, alpacas, and guinea pigs. Sugarcane and some root vegetables were domesticated in New Guinea around 9,000 years ago. Sorghum was domesticated in the Sahel region of Africa by 7,000 years ago. Cotton was domesticated in Peru by 5,600 years ago,[21] and was independently domesticated in Eurasia. In Mesoamerica, wild teosinte was bred into maize by 6,000 years ago.[22]Scholars have offered multiple hypotheses to explain the historical origins of agriculture. Studies of the transition from hunter-gatherer to agricultural societies indicate an initial period of intensification and increasing sedentism; examples are the Natufian culture in the Levant, and the Early Chinese Neolithic in China. Then, wild stands that had previously been harvested started to be planted, and gradually came to be domesticated.[23][24][25]',\n",
              " 'In Eurasia, the Sumerians started to live in villages from about 8,000 BC, relying on the Tigris and Euphrates rivers and a canal system for irrigation. Ploughs appear in pictographs around 3,000 BC; seed-ploughs around 2,300 BC. Farmers grew wheat, barley, vegetables such as lentils and onions, and fruits including dates, grapes, and figs.[26] Ancient Egyptian agriculture relied on the Nile River and its seasonal flooding. Farming started in the predynastic period at the end of the Paleolithic, after 10,000 BC. Staple food crops were grains such as wheat and barley, alongside industrial crops such as flax and papyrus.[27][28] In India, wheat, barley and jujube were domesticated by 9,000 BC, soon followed by sheep and goats.[29] Cattle, sheep and goats were domesticated in Mehrgarh culture by 8,000–6,000 BC.[30][31][32] Cotton was cultivated by the 5th–4th millennium BC.[33] Archeological evidence indicates an animal-drawn plough from 2,500 BC in the Indus Valley civilisation.[34]',\n",
              " 'In China, from the 5th century BC there was a nationwide granary system and widespread silk farming.[35] Water-powered grain mills were in use by the 1st century BC,[36] followed by irrigation.[37] By the late 2nd century, heavy ploughs had been developed with iron ploughshares and mouldboards.[38][39] These spread westwards across Eurasia.[40] Asian rice was domesticated 8,200–13,500 years ago – depending on the molecular clock estimate that is used[41]– on the Pearl River in southern China with a single genetic origin from the wild rice Oryza rufipogon.[42] In Greece and Rome, the major cereals were wheat, emmer, and barley, alongside vegetables including peas, beans, and olives. Sheep and goats were kept mainly for dairy products.[43][44]',\n",
              " 'In the Americas, crops domesticated in Mesoamerica (apart from teosinte) include squash, beans, and cacao.[45] Cocoa was being domesticated by the Mayo Chinchipe of the upper Amazon around 3,000 BC.[46]The turkey was probably domesticated in Mexico or the American Southwest.[47] The Aztecs developed irrigation systems, formed terraced hillsides, fertilized their soil, and developed chinampas or artificial islands. The Mayas used extensive canal and raised field systems to farm swampland from 400 BC.[48][49][50][51][52] Coca was domesticated in the Andes, as were the peanut, tomato, tobacco, and pineapple.[45] Cotton was domesticated in Peru by 3,600 BC.[53] Animals including llamas, alpacas, and guinea pigs were domesticated there.[54] In North America, the indigenous people of the East domesticated crops such as sunflower, tobacco,[55] squash and Chenopodium.[56][57] Wild foods including wild rice and maple sugar were harvested.[58] The domesticated strawberry is a hybrid of a Chilean and a North American species, developed by breeding in Europe and North America.[59] The indigenous people of the Southwest and the Pacific Northwest practiced forest gardening and fire-stick farming. The natives controlled fire on a regional scale to create a low-intensity fire ecology that sustained a low-density agriculture in loose rotation; a sort of \"wild\" permaculture.[60][61][62][63] A system of companion planting called the Three Sisters was developed in North America. The three crops were winter squash, maize, and climbing beans.[64][65]',\n",
              " 'Indigenous Australians, long supposed to have been nomadic hunter-gatherers, practised systematic burning, possibly to enhance natural productivity in fire-stick farming.[66] Scholars have pointed out that hunter-gatherers need a productive environment to support gathering without cultivation. Because the forests of New Guinea have few food plants, early humans may have used \"selective burning\" to increase the productivity of the wild karuka fruit trees to support the hunter-gatherer way of life.[67]',\n",
              " \"The Gunditjmara and other groups developed eel farming and fish trapping systems from some 5,000 years ago.[68] There is evidence of 'intensification' across the whole continent over that period.[69] In two regions of Australia, the central west coast and eastern central, early farmers cultivated yams, native millet, and bush onions, possibly in permanent settlements.[25][70]\",\n",
              " 'In the Middle Ages, compared to the Roman period, agriculture in Western Europe became more focused on self-sufficiency. The agricultural population under feudalism was typically organized into manors consisting of several hundred or more acres of land presided over by a Lord with a Roman Catholic church and priest.[71]',\n",
              " 'Thanks to the exchange with the Al-Andalus where the Arab agricultural revolution was underway, European agriculture transformed with improved techniques and the diffusion of crop plants, including the introduction of sugar, rice, cotton and fruit trees (such as the orange).[72]',\n",
              " 'After 1492 the Columbian exchange brought New World crops such as maize, potatoes, tomatoes, sweet potatoes and manioc to Europe, and Old World crops such as wheat, barley, rice and turnips, and livestock (including horses, cattle, sheep and goats) to the Americas.[73]',\n",
              " 'Irrigation, crop rotation, and fertilizers advanced from the 17th century with the British Agricultural Revolution, allowing global population to rise significantly. Since 1900 agriculture in developed nations, and to a lesser extent in the developing world, has seen large rises in productivity as mechanization replaces human labor, and assisted by synthetic fertilizers, pesticides, and selective breeding. The Haber-Bosch method allowed the synthesis of ammonium nitrate fertilizer on an industrial scale, greatly increasing crop yields and sustaining a further increase in global population.[74][75] Modern agriculture has raised or encountered ecological, political, and economic issues including water pollution, biofuels, genetically modified organisms, tariffs and farm subsidies, leading to alternative approaches such as the organic movement.[76][77]In the 1930, there was a Dust Bowl in the United States with tragic consequences.[78]',\n",
              " 'Pastoralism involves managing domesticated animals. In nomadic pastoralism, herds of livestock are moved from place to place in search of pasture, fodder, and water. This type of farming is practised in arid and semi-arid regions of Sahara, Central Asia and some parts of India.[79]',\n",
              " 'In shifting cultivation, a small area of forest is cleared by cutting and burning the trees. The cleared land is used for growing crops for a few years until the soil becomes too infertile, and the area is abandoned. Another patch of land is selected and the process is repeated. This type of farming is practiced mainly in areas with abundant rainfall where the forest regenerates quickly. This practice is used in Northeast India, Southeast Asia, and the Amazon Basin.[80]',\n",
              " \"Subsistence farming is practiced to satisfy family or local needs alone, with little left over for transport elsewhere. It is intensively practiced in Monsoon Asia and South-East Asia.[81] An estimated 2.5 billion subsistence farmers worked in 2018, cultivating about 60% of the earth's arable land.[82]\",\n",
              " 'Intensive farming is cultivation to maximise productivity, with a low fallow ratio and a high use of inputs (water, fertilizer, pesticide and automation). It is practiced mainly in developed countries.[83][84]',\n",
              " \"From the twentieth century, intensive agriculture increased productivity of crops. It substituted synthetic fertilizers and pesticides for labour, but caused increased water pollution, and often involved farm subsidies. In recent years there has been a backlash against the environmental effects of conventional agriculture, resulting in the organic, regenerative, and sustainable agriculture movements.[76][86] One of the major forces behind this movement has been the European Union, which first certified organic food in 1991 and began reform of its Common Agricultural Policy (CAP) in 2005 to phase out commodity-linked farm subsidies,[87] also known as decoupling. The growth of organic farming has renewed research in alternative technologies such as integrated pest management, selective breeding,[88] and controlled-environment agriculture.[89][90] Recent mainstream technological developments include genetically modified food.[91] Demand for non-food biofuel crops,[92] development of former farm lands, rising transportation costs, climate change, growing consumer demand in China and India, and population growth,[93] are threatening food security in many parts of the world.[94][95][96][97][98] The International Fund for Agricultural Development posits that an increase in smallholder agriculture may be part of the solution to concerns about food prices and overall food security, given the favorable experience of Vietnam.[99] Soil degradation and diseases such as stem rust are major concerns globally;[100] approximately 40% of the world's agricultural land is seriously degraded.[101][102] By 2015, the agricultural output of China was the largest in the world, followed by the European Union, India and the United States.[85] Economists measure the total factor productivity of agriculture and by this measure agriculture in the United States is roughly 1.7 times more productive than it was in 1948.[103]\",\n",
              " 'Following the three-sector theory, the number of people employed in agriculture and other primary activities (such as fishing) can be more than 80% in the least developed countries, and less than 2% in the most highly developed countries.[104] Since the Industrial Revolution, many countries have made the transition to developed economies, and the proportion of people working in agriculture has steadily fallen. During the 16th century in Europe, for example, between 55 and 75% of the population was engaged in agriculture; by the 19th century, this had dropped to between 35 and 65%.[105] In the same countries today, the figure is less than 10%.[104]At the start of the 21st century, some one billion people, or over 1/3 of the available work force, were employed in agriculture. It constitutes approximately 70% of the global employment of children, and in many countries employs the largest percentage of women of any industry.[106] The service sector overtook the agricultural sector as the largest global employer in 2007.[107]',\n",
              " 'Agriculture, specifically farming, remains a hazardous industry, and farmers worldwide remain at high risk of work-related injuries, lung disease, noise-induced hearing loss, skin diseases, as well as certain cancers related to chemical use and prolonged sun exposure. On industrialized farms, injuries frequently involve the use of agricultural machinery, and a common cause of fatal agricultural injuries in developed countries is tractor rollovers.[108] Pesticides and other chemicals used in farming can be hazardous to worker health, and workers exposed to pesticides may experience illness or have children with birth defects.[109] As an industry in which families commonly share in work and live on the farm itself, entire families can be at risk for injuries, illness, and death.[110] Ages 0–6 May be an especially vulnerable population in agriculture;[111] common causes of fatal injuries among young farm workers include drowning, machinery and motor accidents, including with all-terrain vehicles.[110][111][112]',\n",
              " 'The International Labour Organization considers agriculture \"one of the most hazardous of all economic sectors\".[106] It estimates that the annual work-related death toll among agricultural employees is at least 170,000, twice the average rate of other jobs. In addition, incidences of death, injury and illness related to agricultural activities often go unreported.[113] The organization has developed the Safety and Health in Agriculture Convention, 2001, which covers the range of risks in the agriculture occupation, the prevention of these risks and the role that individuals and organizations engaged in agriculture should play.[106]',\n",
              " 'In the United States, agriculture has been identified by the National Institute for Occupational Safety and Health as a priority industry sector in the National Occupational Research Agenda to identify and provide intervention strategies for occupational health and safety issues.[114][115]In the European Union, the European Agency for Safety and Health at Work has issued guidelines on implementing health and safety directives in agriculture, livestock farming, horticulture, and forestry.[116] The Agricultural Safety and Health Council of America (ASHCA) also holds a yearly summit to discuss safety.[117]',\n",
              " 'Overall production varies by country as listed.',\n",
              " 'The twenty largest countries by agricultural output (in nominal terms) at peak level as of 2018, according to the IMF and CIA World Factbook.',\n",
              " 'Cropping systems vary among farms depending on the available resources and constraints; geography and climate of the farm; government policy; economic, social and political pressures; and the philosophy and culture of the farmer.[119][120]',\n",
              " 'Shifting cultivation (or slash and burn) is a system in which forests are burnt, releasing nutrients to support cultivation of annual and then perennial crops for a period of several years.[121] Then the plot is left fallow to regrow forest, and the farmer moves to a new plot, returning after many more years (10–20). This fallow period is shortened if population density grows, requiring the input of nutrients (fertilizer or manure) and some manual pest control. Annual cultivation is the next phase of intensity in which there is no fallow period. This requires even greater nutrient and pest control inputs.[121]',\n",
              " 'Further industrialization led to the use of monocultures, when one cultivar is planted on a large acreage. Because of the low biodiversity, nutrient use is uniform and pests tend to build up, necessitating the greater use of pesticides and fertilizers.[120] Multiple cropping, in which several crops are grown sequentially in one year, and intercropping, when several crops are grown at the same time, are other kinds of annual cropping systems known as polycultures.[121]',\n",
              " 'In subtropical and arid environments, the timing and extent of agriculture may be limited by rainfall, either not allowing multiple annual crops in a year, or requiring irrigation. In all of these environments perennial crops are grown (coffee, chocolate) and systems are practiced such as agroforestry. In temperate environments, where ecosystems were predominantly grassland or prairie, highly productive annual farming is the dominant agricultural system.[121]',\n",
              " 'Important categories of food crops include cereals, legumes, forage, fruits and vegetables.[122] Natural fibers include cotton, wool, hemp, silk and flax.[123] Specific crops are cultivated in distinct growing regions throughout the world. Production is listed in millions of metric tons, based on FAO estimates.[122]',\n",
              " 'Animal husbandry is the breeding and raising of animals for meat, milk, eggs, or wool, and for work and transport.[124] Working animals, including horses, mules, oxen, water buffalo, camels, llamas, alpacas, donkeys, and dogs, have for centuries been used to help cultivate fields, harvest crops, wrangle other animals, and transport farm products to buyers.[125]',\n",
              " \"Livestock production systems can be defined based on feed source, as grassland-based, mixed, and landless.[126] As of 2010[update], 30% of Earth's ice- and water-free area was used for producing livestock, with the sector employing approximately 1.3 billion people. Between the 1960s and the 2000s, there was a significant increase in livestock production, both by numbers and by carcass weight, especially among beef, pigs and chickens, the latter of which had production increased by almost a factor of 10. Non-meat animals, such as milk cows and egg-producing chickens, also showed significant production increases. Global cattle, sheep and goat populations are expected to continue to increase sharply through 2050.[127] Aquaculture or fish farming, the production of fish for human consumption in confined operations, is one of the fastest growing sectors of food production, growing at an average of 9% a year between 1975 and 2007.[128]\",\n",
              " 'During the second half of the 20th century, producers using selective breeding focused on creating livestock breeds and crossbreeds that increased production, while mostly disregarding the need to preserve genetic diversity. This trend has led to a significant decrease in genetic diversity and resources among livestock breeds, leading to a corresponding decrease in disease resistance and local adaptations previously found among traditional breeds.[129]',\n",
              " 'Grassland based livestock production relies upon plant material such as shrubland, rangeland, and pastures for feeding ruminant animals. Outside nutrient inputs may be used, however manure is returned directly to the grassland as a major nutrient source. This system is particularly important in areas where crop production is not feasible because of climate or soil, representing 30–40 million pastoralists.[121] Mixed production systems use grassland, fodder crops and grain feed crops as feed for ruminant and monogastric (one stomach; mainly chickens and pigs) livestock. Manure is typically recycled in mixed systems as a fertilizer for crops.[126]',\n",
              " 'Landless systems rely upon feed from outside the farm, representing the de-linking of crop and livestock production found more prevalently in Organisation for Economic Co-operation and Development member countries. Synthetic fertilizers are more heavily relied upon for crop production and manure use becomes a challenge as well as a source for pollution.[126] Industrialized countries use these operations to produce much of the global supplies of poultry and pork. Scientists estimate that 75% of the growth in livestock production between 2003 and 2030 will be in confined animal feeding operations, sometimes called factory farming. Much of this growth is happening in developing countries in Asia, with much smaller amounts of growth in Africa.[127] Some of the practices used in commercial livestock production, including the usage of growth hormones, are controversial.[130]',\n",
              " 'Tillage is the practice of breaking up the soil with tools such as the plow or harrow to prepare for planting, for nutrient incorporation, or for pest control. Tillage varies in intensity from conventional to no-till. It may improve productivity by warming the soil, incorporating fertilizer and controlling weeds, but also renders soil more prone to erosion, triggers the decomposition of organic matter releasing CO2, and reduces the abundance and diversity of soil organisms.[131][132]',\n",
              " 'Pest control includes the management of weeds, insects, mites, and diseases. Chemical (pesticides), biological (biocontrol), mechanical (tillage), and cultural practices are used. Cultural practices include crop rotation, culling, cover crops, intercropping, composting, avoidance, and resistance. Integrated pest management attempts to use all of these methods to keep pest populations below the number which would cause economic loss, and recommends pesticides as a last resort.[133]',\n",
              " 'Nutrient management includes both the source of nutrient inputs for crop and livestock production, and the method of use of manure produced by livestock. Nutrient inputs can be chemical inorganic fertilizers, manure, green manure, compost and minerals.[134] Crop nutrient use may also be managed using cultural techniques such as crop rotation or a fallow period. Manure is used either by holding livestock where the feed crop is growing, such as in managed intensive rotational grazing, or by spreading either dry or liquid formulations of manure on cropland or pastures.[131][135]',\n",
              " 'Water management is needed where rainfall is insufficient or variable, which occurs to some degree in most regions of the world.[121] Some farmers use irrigation to supplement rainfall. In other areas such as the Great Plains in the U.S. and Canada, farmers use a fallow year to conserve soil moisture to use for growing a crop in the following year.[136] Agriculture represents 70% of freshwater use worldwide.[137]',\n",
              " 'According to a report by the International Food Policy Research Institute, agricultural technologies will have the greatest impact on food production if adopted in combination with each other; using a model that assessed how eleven technologies could impact agricultural productivity, food security and trade by 2050, the International Food Policy Research Institute found that the number of people at risk from hunger could be reduced by as much as 40% and food prices could be reduced by almost half.[138]',\n",
              " 'Payment for ecosystem services is a method of providing additional incentives to encourage farmers to conserve some aspects of the environment. Measures might include paying for reforestation upstream of a city, to improve the supply of fresh water.[139]',\n",
              " 'Climate change and agriculture are interrelated on a global scale. Global warming affects agriculture through changes in average temperatures, rainfall, and weather extremes (like storms and heat waves); changes in pests and diseases; changes in atmospheric carbon dioxide and ground-level ozone concentrations; changes in the nutritional quality of some foods;[140] and changes in sea level.[141] Global warming is already affecting agriculture, with effects unevenly distributed across the world.[142] Future climate change will probably negatively affect crop production in low latitude countries, while effects in northern latitudes may be positive or negative.[142] Global warming will probably increase the risk of food insecurity for some vulnerable groups, such as the poor.[143]',\n",
              " 'Crop alteration has been practiced by humankind for thousands of years, since the beginning of civilization. Altering crops through breeding practices changes the genetic make-up of a plant to develop crops with more beneficial characteristics for humans, for example, larger fruits or seeds, drought-tolerance, or resistance to pests. Significant advances in plant breeding ensued after the work of geneticist Gregor Mendel. His work on dominant and recessive alleles, although initially largely ignored for almost 50 years, gave plant breeders a better understanding of genetics and breeding techniques. Crop breeding includes techniques such as plant selection with desirable traits, self-pollination and cross-pollination, and molecular techniques that genetically modify the organism.[144]',\n",
              " 'Domestication of plants has, over the centuries increased yield, improved disease resistance and drought tolerance, eased harvest and improved the taste and nutritional value of crop plants. Careful selection and breeding have had enormous effects on the characteristics of crop plants. Plant selection and breeding in the 1920s and 1930s improved pasture (grasses and clover) in New Zealand. Extensive X-ray and ultraviolet induced mutagenesis efforts (i.e. primitive genetic engineering) during the 1950s produced the modern commercial varieties of grains such as wheat, corn (maize) and barley.[145][146]',\n",
              " 'The Green Revolution popularized the use of conventional hybridization to sharply increase yield by creating \"high-yielding varieties\". For example, average yields of corn (maize) in the US have increased from around 2.5 tons per hectare (t/ha) (40 bushels per acre) in 1900 to about 9.4 t/ha (150 bushels per acre) in 2001. Similarly, worldwide average wheat yields have increased from less than 1 t/ha in 1900 to more than 2.5 t/ha in 1990. South American average wheat yields are around 2 t/ha, African under 1 t/ha, and Egypt and Arabia up to 3.5 to 4 t/ha with irrigation. In contrast, the average wheat yield in countries such as France is over 8 t/ha. Variations in yields are due mainly to variation in climate, genetics, and the level of intensive farming techniques (use of fertilizers, chemical pest control, growth control to avoid lodging).[147][148][149]',\n",
              " 'Genetically modified organisms (GMO) are organisms whose genetic material has been altered by genetic engineering techniques generally known as recombinant DNA technology. Genetic engineering has expanded the genes available to breeders to use in creating desired germlines for new crops. Increased durability, nutritional content, insect and virus resistance and herbicide tolerance are a few of the attributes bred into crops through genetic engineering.[150] For some, GMO crops cause food safety and food labeling concerns. Numerous countries have placed restrictions on the production, import or use of GMO foods and crops.[151] Currently a global treaty, the Biosafety Protocol, regulates the trade of GMOs. There is ongoing discussion regarding the labeling of foods made from GMOs, and while the EU currently requires all GMO foods to be labeled, the US does not.[152]',\n",
              " 'Herbicide-resistant seed has a gene implanted into its genome that allows the plants to tolerate exposure to herbicides, including glyphosate. These seeds allow the farmer to grow a crop that can be sprayed with herbicides to control weeds without harming the resistant crop. Herbicide-tolerant crops are used by farmers worldwide.[153] With the increasing use of herbicide-tolerant crops, comes an increase in the use of glyphosate-based herbicide sprays. In some areas glyphosate resistant weeds have developed, causing farmers to switch to other herbicides.[154][155] Some studies also link widespread glyphosate usage to iron deficiencies in some crops, which is both a crop production and a nutritional quality concern, with potential economic and health implications.[156]',\n",
              " 'Other GMO crops used by growers include insect-resistant crops, which have a gene from the soil bacterium Bacillus thuringiensis (Bt), which produces a toxin specific to insects. These crops resist damage by insects.[157] Some believe that similar or better pest-resistance traits can be acquired through traditional breeding practices, and resistance to various pests can be gained through hybridization or cross-pollination with wild species. In some cases, wild species are the primary source of resistance traits; some tomato cultivars that have gained resistance to at least 19 diseases did so through crossing with wild populations of tomatoes.[158]',\n",
              " 'Agriculture is both a cause of and sensitive to environmental degradation, such as biodiversity loss, desertification, soil degradation and global warming, which cause decrease in crop yield.[159] Agriculture is one of the most important drivers of environmental pressures, particularly habitat change, climate change, water use and toxic emissions. Agriculture is the main source of toxins released into the environment, including insecticides, especially those used on cotton.[160][161][page\\xa0needed] The 2011 UNEP Green Economy report stated that agricultural operations produced some 13 per cent of anthropogenic global greenhouse gas emissions. This includes gases from the use of inorganic fertilizers, agro-chemical pesticides, and herbicides, as well as fossil fuel-energy inputs.[162]',\n",
              " 'Agriculture imposes multiple external costs upon society through effects such as pesticide damage to nature (especially herbicides and insecticides), nutrient runoff, excessive water usage, and loss of natural environment. A 2000 assessment of agriculture in the UK determined total external costs for 1996 of £2,343 million, or £208 per hectare.[163] A 2005 analysis of these costs in the US concluded that cropland imposes approximately $5 to $16 billion ($30 to $96 per hectare), while livestock production imposes $714 million.[164] Both studies, which focused solely on the fiscal impacts, concluded that more should be done to internalize external costs. Neither included subsidies in their analysis, but they noted that subsidies also influence the cost of agriculture to society.[163][164]',\n",
              " 'Agriculture seeks to increase yield and to reduce costs. Yield increases with inputs such as fertilisers and removal of pathogens, predators, and competitors (such as weeds). Costs decrease with increasing scale of farm units, such as making fields larger; this means removing hedges, ditches and other areas of habitat. Pesticides kill insects, plants and fungi. These and other measures have cut biodiversity to very low levels on intensively farmed land.[165] Effective yields fall with on-farm losses, which may be caused by poor production practices during harvesting, handling, and storage.[166]',\n",
              " 'A senior UN official, Henning Steinfeld, said that \"Livestock are one of the most significant contributors to today\\'s most serious environmental problems\".[167] Livestock production occupies 70% of all land used for agriculture, or 30% of the land surface of the planet. It is one of the largest sources of greenhouse gases, responsible for 18% of the world\\'s greenhouse gas emissions as measured in CO2 equivalents. By comparison, all transportation emits 13.5% of the CO2. It produces 65% of human-related nitrous oxide (which has 296 times the global warming potential of CO2) and 37% of all human-induced methane (which is 23 times as warming as CO2.) It also generates 64% of the ammonia emission. Livestock expansion is cited as a key factor driving deforestation; in the Amazon basin 70% of previously forested area is now occupied by pastures and the remainder used for feed crops.[168] Through deforestation and land degradation, livestock is also driving reductions in biodiversity. Furthermore, the UNEP states that \"methane emissions from global livestock are projected to increase by 60 per cent by 2030 under current practices and consumption patterns.\"[162]',\n",
              " \"Land transformation, the use of land to yield goods and services, is the most substantial way humans alter the Earth's ecosystems, and is the driving force causing biodiversity loss. Estimates of the amount of land transformed by humans vary from 39 to 50%.[169] Land degradation, the long-term decline in ecosystem function and productivity, is estimated to be occurring on 24% of land worldwide, with cropland overrepresented.[170] Land management is the driving factor behind degradation; 1.5 billion people rely upon the degrading land. Degradation can be through deforestation, desertification, soil erosion, mineral depletion, acidification, or salinization.[121]\",\n",
              " \"Eutrophication, excessive nutrient enrichment in aquatic ecosystems resulting in algal blooms and anoxia, leads to fish kills, loss of biodiversity, and renders water unfit for drinking and other industrial uses. Excessive fertilization and manure application to cropland, as well as high livestock stocking densities cause nutrient (mainly nitrogen and phosphorus) runoff and leaching from agricultural land. These nutrients are major nonpoint pollutants contributing to eutrophication of aquatic ecosystems and pollution of groundwater, with harmful effects on human populations.[171] Fertilisers also reduce terrestrial biodiversity by increasing competition for light, favouring those species that are able to benefit from the added nutrients.[172]Agriculture accounts for 70 percent of withdrawals of freshwater resources.[173][174] Agriculture is a major draw on water from aquifers, and currently draws from those underground water sources at an unsustainable rate. It is long known that aquifers in areas as diverse as northern China, the Upper Ganges and the western US are being depleted, and new research extends these problems to aquifers in Iran, Mexico and Saudi Arabia.[175] Increasing pressure is being placed on water resources by industry and urban areas, meaning that water scarcity is increasing and agriculture is facing the challenge of producing more food for the world's growing population with reduced water resources.[176] Agricultural water usage can also cause major environmental problems, including the destruction of natural wetlands, the spread of water-borne diseases, and land degradation through salinization and waterlogging, when irrigation is performed incorrectly.[177]\",\n",
              " 'Pesticide use has increased since 1950 to 2.5 million short tons annually worldwide, yet crop loss from pests has remained relatively constant.[178] The World Health Organization estimated in 1992 that three million pesticide poisonings occur annually, causing 220,000 deaths.[179] Pesticides select for pesticide resistance in the pest population, leading to a condition termed the \"pesticide treadmill\" in which pest resistance warrants the development of a new pesticide.[180]',\n",
              " 'An alternative argument is that the way to \"save the environment\" and prevent famine is by using pesticides and intensive high yield farming, a view exemplified by a quote heading the Center for Global Food Issues website: \\'Growing more per acre leaves more land for nature\\'.[181][182] However, critics argue that a trade-off between the environment and a need for food is not inevitable,[183] and that pesticides simply replace good agronomic practices such as crop rotation.[180] The Push–pull agricultural pest management technique involves intercropping, using plant aromas to repel pests from crops (push) and to lure them to a place from which they can then be removed (pull).[184]',\n",
              " \"Agriculture, and in particular animal husbandry, is responsible for greenhouse gas emissions of CO2 and a percentage of the world's methane, and future land infertility, and the displacement of wildlife. Agriculture contributes to climate change by anthropogenic emissions of greenhouse gases, and by the conversion of non-agricultural land such as forest for agricultural use.[185] Agriculture, forestry and land-use change contributed around 20 to 25% to global annual emissions in 2010.[186] A range of policies can reduce the risk of negative climate change impacts on agriculture,[187][188] and greenhouse gas emissions from the agriculture sector.[189][190][191]\",\n",
              " 'Current farming methods have resulted in over-stretched water resources, high levels of erosion and reduced soil fertility. There is not enough water to continue farming using current practices; therefore how critical water, land, and ecosystem resources are used to boost crop yields must be reconsidered. A solution would be to give value to ecosystems, recognizing environmental and livelihood tradeoffs, and balancing the rights of a variety of users and interests.[192] Inequities that result when such measures are adopted would need to be addressed, such as the reallocation of water from poor to rich, the clearing of land to make way for more productive farmland, or the preservation of a wetland system that limits fishing rights.[193]',\n",
              " 'Technological advancements help provide farmers with tools and resources to make farming more sustainable.[194] Technology permits innovations like conservation tillage, a farming process which helps prevent land loss to erosion, reduces water pollution, and enhances carbon sequestration.[195] Other potential practices include conservation agriculture, agroforestry, improved grazing, avoided grassland conversion, and biochar.[196][197] Current mono-crop farming practices in the United States preclude widespread adoption of sustainable practices, such as 2-3 crop rotations that incorporate grass or hay with annual crops, unless negative emission goals such as soil carbon sequestration become policy.[198]',\n",
              " \"The International Food Policy Research Institute states that agricultural technologies will have the greatest impact on food production if adopted in combination with each other; using a model that assessed how eleven technologies could impact agricultural productivity, food security and trade by 2050, it found that the number of people at risk from hunger could be reduced by as much as 40% and food prices could be reduced by almost half.[138] The food demand of Earth's projected population, with current climate change predictions, could be satisfied by improvement of agricultural methods, expansion of agricultural areas, and a sustainability-oriented consumer mindset.[199]\",\n",
              " 'Since the 1940s, agricultural productivity has increased dramatically, due largely to the increased use of energy-intensive mechanization, fertilizers and pesticides. The vast majority of this energy input comes from fossil fuel sources.[200] Between the 1960s and the 1980s, the Green Revolution transformed agriculture around the globe, with world grain production increasing significantly (between 70% and 390% for wheat and 60% to 150% for rice, depending on geographic area)[201] as world population doubled. Heavy reliance on petrochemicals has raised concerns that oil shortages could increase costs and reduce agricultural output.[202]',\n",
              " 'Industrialized agriculture depends on fossil fuels in two fundamental ways: direct consumption on the farm and manufacture of inputs used on the farm. Direct consumption includes the use of lubricants and fuels to operate farm vehicles and machinery.[202]',\n",
              " \"Indirect consumption includes the manufacture of fertilizers, pesticides, and farm machinery.[202] In particular, the production of nitrogen fertilizer can account for over half of agricultural energy usage.[206] Together, direct and indirect consumption by US farms accounts for about 2% of the nation's energy use. Direct and indirect energy consumption by U.S. farms peaked in 1979, and has since gradually declined.[202] Food systems encompass not just agriculture but off-farm processing, packaging, transporting, marketing, consumption, and disposal of food and food-related items. Agriculture accounts for less than one-fifth of food system energy use in the US.[204][207]\",\n",
              " 'Plastic products are used extensively in agriculture, for example to increase crop yield and improve the efficiency of water and agrichemical use. \"Agriplastic\" products include films to cover greenhouses and tunnels, mulch to cover soil (e.g. to suppress weeds, conserve water, increase soil temperature and aid fertilizer application), shade cloth, pesticide containers, seedling trays, protective mesh and irrigation tubing. The polymers most commonly used in these products are low- density polyethylene (LPDE), linear low-density polyethylene (LLDPE), polypropylene (PP) and polyvinyl chloride (PVC).[208]',\n",
              " 'The total amount of plastics used in agriculture is difficult to quantify. A 2012 study reported that almost 6.5 million tonnes per year were consumed globally while a later study estimated that global demand in 2015 was between 7.3 million and 9 million tonnes. Widespread use of plastic mulch and lack of systematic collection and management have led to the generation of large amounts of mulch residue. Weathering and degradation eventually cause the mulch to fragment. These fragments and larger pieces of plastic accumulate in soil. Mulch residue has been measured at levels of 50 to 260\\xa0kg per hectare in topsoil in areas where the mulch has been used for more than 10 years, which confirms that mulching is a major source of both microplastic and macroplastic contamination of soil.[208]',\n",
              " 'Agricultural plastics, especially plastic films, are not easy to recycle because of high contamination levels (up to 40- 50% by weight contamination by pesticides, fertilizers, soil and debris, moist vegetation, silage juice water, and UV stabilizers) and collection difficulties . Therefore, they are often buried or abandoned in fields and watercourses or burned. These disposal practices lead to soil degradation and can result in contamination of soils and leakage of microplastics into the marine environment as a result of precipitation run-off and tidal washing. In addition, additives in residual plastic film (such as UV and thermal stabilizers) may have deleterious effects on crop growth, soil structure, nutrient transport and salt levels. There is a risk that plastic mulch will deteriorate soil quality, deplete soil organic matter stocks, increase soil water repellence and emit greenhouse gases. Microplastics released through fragmentation of agricultural plastics can absorb and concentrate contaminants capable of being passed up the trophic chain.[208]',\n",
              " 'Agricultural economics is economics as it relates to the \"production, distribution and consumption of [agricultural] goods and services\".[210] Combining agricultural production with general theories of marketing and business as a discipline of study began in the late 1800s, and grew significantly through the 20th century.[211] Although the study of agricultural economics is relatively recent, major trends in agriculture have significantly affected national and international economies throughout history, ranging from tenant farmers and sharecropping in the post-American Civil War Southern United States[212] to the European feudal system of manorialism.[213] In the United States, and elsewhere, food costs attributed to food processing, distribution, and agricultural marketing, sometimes referred to as the value chain, have risen while the costs attributed to farming have declined. This is related to the greater efficiency of farming, combined with the increased level of value addition (e.g. more highly processed products) provided by the supply chain. Market concentration has increased in the sector as well, and although the total effect of the increased market concentration is likely increased efficiency, the changes redistribute economic surplus from producers (farmers) and consumers, and may have negative implications for rural communities.[214]',\n",
              " 'National government policies can significantly change the economic marketplace for agricultural products, in the form of taxation, subsidies, tariffs and other measures.[215] Since at least the 1960s, a combination of trade restrictions, exchange rate policies and subsidies have affected farmers in both the developing and the developed world. In the 1980s, non-subsidized farmers in developing countries experienced adverse effects from national policies that created artificially low global prices for farm products. Between the mid-1980s and the early 2000s, several international agreements limited agricultural tariffs, subsidies and other trade restrictions.[216]',\n",
              " 'However, as of 2009[update], there was still a significant amount of policy-driven distortion in global agricultural product prices. The three agricultural products with the most trade distortion were sugar, milk and rice, mainly due to taxation. Among the oilseeds, sesame had the most taxation, but overall, feed grains and oilseeds had much lower levels of taxation than livestock products. Since the 1980s, policy-driven distortions have seen a greater decrease among livestock products than crops during the worldwide reforms in agricultural policy.[215] Despite this progress, certain crops, such as cotton, still see subsidies in developed countries artificially deflating global prices, causing hardship in developing countries with non-subsidized farmers.[217] Unprocessed commodities such as corn, soybeans, and cattle are generally graded to indicate quality, affecting the price the producer receives. Commodities are generally reported by production quantities, such as volume, number or weight.[218]',\n",
              " 'Agricultural science is a broad multidisciplinary field of biology that encompasses the parts of exact, natural, economic and social sciences used in the practice and understanding of agriculture. It covers topics such as agronomy, plant breeding and genetics, plant pathology, crop modelling, soil science, entomology, production techniques and improvement, study of pests and their management, and study of adverse environmental effects such as soil degradation, waste management, and bioremediation.[219][220]',\n",
              " 'The scientific study of agriculture began in the 18th century, when Johann Friedrich Mayer conducted experiments on the use of gypsum (hydrated calcium sulphate) as a fertilizer.[221] Research became more systematic when in 1843, John Lawes and Henry Gilbert began a set of long-term agronomy field experiments at Rothamsted Research Station in England; some of them, such as the Park Grass Experiment, are still running.[222][223] In America, the Hatch Act of 1887 provided funding for what it was the first to call \"agricultural science\", driven by farmers\\' interest in fertilizers.[224] In agricultural entomology, the USDA began to research biological control in 1881; it instituted its first large program in 1905, searching Europe and Japan for natural enemies of the gypsy moth and brown-tail moth, establishing parasitoids (such as solitary wasps) and predators of both pests in the USA.[225][226][227]',\n",
              " \"Agricultural policy is the set of government decisions and actions relating to domestic agriculture and imports of foreign agricultural products. Governments usually implement agricultural policies with the goal of achieving a specific outcome in the domestic agricultural product markets. Some overarching themes include risk management and adjustment (including policies related to climate change, food safety and natural disasters), economic stability (including policies related to taxes), natural resources and environmental sustainability (especially water policy), research and development, and market access for domestic commodities (including relations with global organizations and agreements with other countries).[229] Agricultural policy can also touch on food quality, ensuring that the food supply is of a consistent and known quality, food security, ensuring that the food supply meets the population's needs, and conservation. Policy programs can range from financial programs, such as subsidies, to encouraging producers to enroll in voluntary quality assurance programs.[230]\",\n",
              " \"There are many influences on the creation of agricultural policy, including consumers, agribusiness, trade lobbies and other groups. Agribusiness interests hold a large amount of influence over policy making, in the form of lobbying and campaign contributions. Political action groups, including those interested in environmental issues and labor unions, also provide influence, as do lobbying organizations representing individual agricultural commodities.[231] The Food and Agriculture Organization of the United Nations (FAO) leads international efforts to defeat hunger and provides a forum for the negotiation of global agricultural regulations and agreements. Samuel Jutzi, director of FAO's animal production and health division, states that lobbying by large corporations has stopped reforms that would improve human health and the environment. For example, proposals in 2010 for a voluntary code of conduct for the livestock industry that would have provided incentives for improving standards for health, and environmental regulations, such as the number of animals an area of land can support without long-term damage, were successfully defeated due to large food company pressure.[232]\",\n",
              " '\\xa0This article incorporates text from a free content work.  Licensed under CC BY-SA 3.0 IGO License statement/permission. Text taken from Drowning in Plastics – Marine Litter and Plastic Waste Vital Graphics,   United Nations Environment Programme. To learn how to add open license text to Wikipedia articles, please see this how-to page. For information on reusing text from Wikipedia, please see the terms of use.']"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "scrape_tool.read_page(link='https://en.wikipedia.org/wiki/Agriculture')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XoVyqXI0Xjwl"
      },
      "source": [
        "<h4>Read from english keyword</h4>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Sz9BBIhiXjwm"
      },
      "outputs": [],
      "source": [
        "scrape_tool.read_page(search_keyword='agriculture')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PD4F1ORyXjwn"
      },
      "source": [
        "<h4>Read from english pdf document</h4>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b5Lf3JooXjwn"
      },
      "outputs": [],
      "source": [
        "scrape_tool.read_page(document='document_name.pdf')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TN_hNhJuXjwo"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YkfHepCAXjwo"
      },
      "source": [
        "<h3>Read from list of links with GUI</h3>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZSC2e2x_Xjwp"
      },
      "outputs": [],
      "source": [
        "class ActiveScrappingGUI():\n",
        "    def __init__(self, links):\n",
        "        if not isinstance(links, list):\n",
        "            raise Exception('link input shoudl be present as a python list')\n",
        "        self.linkIdx = 0\n",
        "        self.links = links\n",
        "        self.approvedList = {}\n",
        "        self.storedText = {}\n",
        "        self.defaultColor  = '#EEEEEE'\n",
        "        self.approvedColor = 'lightgreen'\n",
        "        self.rejectedColor = '#FF4500'\n",
        "        self.nextButton = widgets.Button(description = 'Next')\n",
        "        self.approve = widgets.Button(description = 'Approve', disabled=True)\n",
        "        self.reject = widgets.Button(description = 'Reject', disabled=True)\n",
        "        self.nextButton.add_class(\"red_label\")\n",
        "        self.approve.add_class(\"red_label\")\n",
        "        self.reject.add_class(\"red_label\")\n",
        "        self.output = widgets.Output()\n",
        "        display(widgets.HBox((self.nextButton, self.approve,self.reject )), self.output, \n",
        "        HTML(\"<style>.red_label { font-weight: bold}</style>\"),\n",
        "        HTML(\"<style>.red_label { font-family:calibri}</style>\"),\n",
        "        HTML(\"<style>.red_label { font-size:16px}</style>\"))\n",
        "        \n",
        "    def on_button_clicked_approve(self, b):\n",
        "        with self.output:\n",
        "            self.nextButton.disabled = False\n",
        "            self.approve.style.button_color = self.approvedColor\n",
        "            self.reject.style.button_color = self.defaultColor\n",
        "            self.approvedList[self.links[self.linkIdx-1]] = True\n",
        "                \n",
        "    def on_button_clicked_reject(self, b):\n",
        "        with self.output:\n",
        "            self.nextButton.disabled = False\n",
        "            self.approve.style.button_color = self.defaultColor\n",
        "            self.reject.style.button_color = self.rejectedColor\n",
        "            self.approvedList[self.links[self.linkIdx-1]] = False\n",
        "                \n",
        "    def on_button_clicked(self, b):\n",
        "        with self.output:\n",
        "            if  self.linkIdx == len(self.links):\n",
        "                clear_output()\n",
        "                print('All links visited. Approved link can be accessed with \"gui.approvedList\"')\n",
        "                self.reject.disabled = True\n",
        "                self.approve.disabled = True\n",
        "                self.nextButton.disabled = True\n",
        "                self.reject.style.button_color = self.defaultColor\n",
        "                self.approve.style.button_color = self.defaultColor\n",
        "            else:\n",
        "                clear_output()\n",
        "                self.reject.disabled = True\n",
        "                self.approve.disabled = True\n",
        "                self.nextButton.disabled = True\n",
        "                self.reject.style.button_color = self.defaultColor\n",
        "                self.approve.style.button_color = self.defaultColor\n",
        "                print('Extracting..')\n",
        "                text = scrape_tool.read_page(link=self.links[self.linkIdx])\n",
        "                clear_output()\n",
        "                self.storedText[self.links[self.linkIdx]] = text\n",
        "                for para in text:\n",
        "                    print(para, '\\n')\n",
        "                webbrowser.open(self.links[self.linkIdx])\n",
        "                self.reject.disabled = False\n",
        "                self.approve.disabled = False\n",
        "                self.linkIdx += 1\n",
        "            \n",
        "    def start(self):\n",
        "        self.approve.on_click(self.on_button_clicked_approve)\n",
        "        self.reject.on_click(self.on_button_clicked_reject)\n",
        "        self.nextButton.on_click(self.on_button_clicked)\n",
        "    \n",
        "    def save(self, path, topic_name):\n",
        "        existingFiles = set(os.listdir(path))\n",
        "        for key, item in self.approvedList.items():\n",
        "            if item == True:                \n",
        "                    name = os.path.join(path, '{topic_name}_'+str(time.time())+'.json')\n",
        "                    if name in existingFiles:\n",
        "                        raise Exception('A file already exists in the save path for the current file. '\\\n",
        "                                        'Change saving procedure for your setup')\n",
        "                    with open(name, 'w') as f: \n",
        "                        json.dump({'link':key,\n",
        "                                  'text':self.storedText[key]}, f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "referenced_widgets": [
            "047a4815b9b344e8af3879b51ac11054",
            "1be92b1493044c9186817f1b3fc0955e"
          ]
        },
        "id": "XZ9DsLaZXjwq",
        "outputId": "fcb3b020-df89-42a6-bc16-e1a9be4c051e",
        "scrolled": false
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "047a4815b9b344e8af3879b51ac11054",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(Button(description='Next', style=ButtonStyle(), _dom_classes=('red_label',)), Button(descriptio…"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1be92b1493044c9186817f1b3fc0955e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Output()"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<style>.red_label { font-weight: bold}</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<style>.red_label { font-family:calibri}</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<style>.red_label { font-size:16px}</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "links = set(['https://en.wikipedia.org/wiki/Agriculture', 'https://www.bbc.com/hindi/india-56901831',\n",
        "        'https://en.wikipedia.org/wiki/Main_Page'])\n",
        "topic_name = []\n",
        "# existingFiles = set(os.listdir(path))\n",
        "# existing_links = set()\n",
        "# for filename in existingFiles:\n",
        "#     with open(os.path.join(path, filename), \"rb\") as read_file:\n",
        "#         data = json.load(read_file)\n",
        "#     link = data['link']\n",
        "#     existing_links.add(link)\n",
        "# existing_links  \n",
        "# for link in links:\n",
        "#     if link in existing_links:\n",
        "#         links.pop(link)\n",
        "#load links from csv into a list\n",
        "gui = ActiveScrappingGUI(links)\n",
        "gui.start()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hbsMO2n9Xjwq",
        "outputId": "9e239b98-0e1d-4051-c992-5c1478a6be3b",
        "scrolled": false
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'https://en.wikipedia.org/wiki/Agriculture': True,\n",
              " 'https://www.bbc.com/hindi/india-56901831': False,\n",
              " 'https://en.wikipedia.org/wiki/Main_Page': True}"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "gui.approvedList"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VPRRmoHfXjwr"
      },
      "outputs": [],
      "source": [
        "#store this as .pickle if rejected links are processed at a later time.\n",
        "linkStatusPath = 'status.pickle'\n",
        "with open(linkStatusPath, 'wb') as handle:\n",
        "    pickle.dump(gui.approvedList, handle, protocol=pickle.HIGHEST_PROTOCOL)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fAXt_XotXjwr"
      },
      "source": [
        "<b> Save approved text </b>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "birO3XpCXjwr"
      },
      "outputs": [],
      "source": [
        "#provide your folder path\n",
        "# gui.save(path='saved')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LZ3rV6IbXjws"
      },
      "outputs": [],
      "source": [
        "# f = json.load('saved/extracted_file_1.json')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "25nKnCs2Xjws"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UdbsgiTrXjwt"
      },
      "source": [
        "<b>Example on how to proceed with rejected links (Work In Progress)<b>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 324
        },
        "id": "MEUqJaJEXjwt",
        "outputId": "a75ec26e-815b-42ef-d300-7516ebe33340",
        "scrolled": true
      },
      "outputs": [
        {
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-4-10f4bd73666d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    107\u001b[0m \u001b[0mlinkStatusPath\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'status.pickle'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 108\u001b[0;31m \u001b[0mhandle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mHandleRejects\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstoredGUIStatusDict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlinkStatusPath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-4-10f4bd73666d>\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, storedGUIStatusDict, openWebPage)\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrefresh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"layout\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdiscard\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"layout\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m         \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlinkStatusPath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatusDct\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresolvedLinks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'status.pickle'"
          ]
        }
      ],
      "source": [
        "class HandleRejects():\n",
        "    def __init__(self, storedGUIStatusDict, openWebPage = True, ):\n",
        "        self.approve = widgets.Button(description = 'Approve Changes')\n",
        "        self.refresh = widgets.Button(description = 'Refresh Changes')\n",
        "        self.discard = widgets.Button(description = 'Unable to process')\n",
        "        self.output = widgets.Output()\n",
        "        self.openWebPage = openWebPage\n",
        "        self.defaultColor  = '#EEEEEE'\n",
        "        self.approve.disabled = True\n",
        "        self.discard.disabled = True\n",
        "        self.approve.add_class(\"layout\")\n",
        "        self.refresh.add_class(\"layout\")\n",
        "        self.discard.add_class(\"layout\")\n",
        "        with open(linkStatusPath, 'rb') as handle:\n",
        "            self.statusDct = pickle.load(handle)\n",
        "        self.resolvedLinks = {}\n",
        "        self.discardedLinks = {}\n",
        "        self.initalVal = None\n",
        "        self.initalVal = self.reportStatus()\n",
        "        self.linkIdx = 0\n",
        "        self.visitSet = set()\n",
        "        self.linkFromStatus = [key for key, item in self.statusDct.items() if item == False]\n",
        "        \n",
        "    def reportStatus(self):\n",
        "        if self.initalVal == None:\n",
        "            toBeResolved = sum([1 if item == False else 0 for key, item in self.statusDct.items()])\n",
        "            return toBeResolved\n",
        "        else:\n",
        "            self.remainingCount = self.initalVal - len(self.resolvedLinks) - len(self.discardedLinks)\n",
        "            print(f'{self.remainingCount}/{self.initalVal} links left to be resolved')\n",
        "    \n",
        "    def on_button_clicked_approve(self, b):\n",
        "        with self.output:\n",
        "            self.approve.style.button_color = self.defaultColor\n",
        "            self.resolvedLinks[self.currentLink] = self.currentText\n",
        "            self.linkIdx += 1 \n",
        "            clear_output()\n",
        "            print(f'Text extracted from {self.currentLink} approved')\n",
        "            self.reportStatus()\n",
        "            self.discard.disabled = True\n",
        "            self.approve.disabled = True\n",
        "    \n",
        "    def on_button_clicked_discard(self, b):\n",
        "        with self.output:\n",
        "            self.discard.style.button_color = self.defaultColor\n",
        "            self.discardedLinks[self.currentLink] = self.currentText\n",
        "            self.linkIdx += 1 \n",
        "            clear_output()\n",
        "            print(f'Text extracted from {self.currentLink} needs manual inspection. Unable to proceed with web scraping code')\n",
        "            self.reportStatus()\n",
        "            self.approve.disabled = True\n",
        "            self.discard.disabled = True\n",
        "            \n",
        "    def on_button_clicked_refresh(self, b):\n",
        "        with self.output:\n",
        "            if len(self.linkFromStatus) == 0:\n",
        "                raise Exception('No sentences dound in dict')\n",
        "            self.approve.disabled = False\n",
        "            self.discard.disabled = False\n",
        "            self.reportStatus()\n",
        "            print('Extracting..')\n",
        "            if self.linkIdx >= len(self.linkFromStatus):\n",
        "                clear_output()\n",
        "                self.discard.disabled = True\n",
        "                self.approve.disabled = True\n",
        "                self.refresh.disabled = True\n",
        "                raise Exception('All links visited. : )')\n",
        "            self.currentLink = self.linkFromStatus[self.linkIdx]\n",
        "            if self.currentLink not in self.resolvedLinks.keys():\n",
        "                if self.currentLink not in self.visitSet:\n",
        "                    self.visitSet.add(self.currentLink)\n",
        "#                     webbrowser.open(self.currentLink)\n",
        "                    print(self.currentLink)\n",
        "                self.currentText = self.readFn(self.currentLink)\n",
        "                clear_output()\n",
        "                for line in self.currentText:\n",
        "                    print(line, '\\n')\n",
        "    \n",
        "    def start(self, readFn):\n",
        "        display(widgets.HBox((self.approve, self.refresh, self.discard)), self.output, \n",
        "        HTML(\"<style>.layout { font-weight: bold}</style>\"),\n",
        "        HTML(\"<style>.layout { font-family:calibri}</style>\"),\n",
        "        HTML(\"<style>.layout { font-size:16px}</style>\"))\n",
        "        self.readFn = readFn\n",
        "        self.refresh.on_click(self.on_button_clicked_refresh)\n",
        "        self.approve.on_click(self.on_button_clicked_approve)\n",
        "        self.discard.on_click(self.on_button_clicked_discard)\n",
        "    \n",
        "    def save(self, path):\n",
        "        existingFiles = set(os.listdir(path))\n",
        "        for key, item in self.resolvedLinks.items():\n",
        "            if item == True:                \n",
        "                    name = os.path.join(path, 'extracted_file_'+str(time.time())+'.json')\n",
        "                    if name in existingFiles:\n",
        "                        raise Exception('A file already exists in the save path for the current file. '\\\n",
        "                                        'Change saving procedure for your setup')\n",
        "                    with open(name, 'w') as f: \n",
        "                        json.dump({'link':key,\n",
        "                                  'text':self.storedText[key]}, f)\n",
        "   \n",
        "    def saveDiscarded(self, path):\n",
        "        with open(path, 'w') as f:\n",
        "            data = list(self.discardedLinks.keys())\n",
        "            for line in data:\n",
        "                f.write(line+'\\n')\n",
        "                \n",
        "linkStatusPath = 'status.pickle'\n",
        "handle = HandleRejects(storedGUIStatusDict=linkStatusPath);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "referenced_widgets": [
            "efd79860ea034c95be5f891ffc15a98e",
            "e13b39a8bf244b1fb21c03d2ffe312a2"
          ]
        },
        "id": "hYijnQx4Xjwt",
        "outputId": "a9663ace-2507-4205-ebe3-aa0a99b31c67"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "efd79860ea034c95be5f891ffc15a98e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(Button(description='Approve Changes', disabled=True, style=ButtonStyle(), _dom_classes=('layout…"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e13b39a8bf244b1fb21c03d2ffe312a2",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Output()"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<style>.layout { font-weight: bold}</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<style>.layout { font-family:calibri}</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<style>.layout { font-size:16px}</style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "def read_from_link_custom(link, replace_list=['\\n']):  \n",
        "    page = requests.get(link)\n",
        "    soup = BeautifulSoup(page.content, 'html.parser')\n",
        "    data = []\n",
        "    for i in range(len(soup.find_all('p'))):\n",
        "        text = soup.find_all('p')[i].get_text()\n",
        "        for j in range(len(replace_list)):\n",
        "            text = text.replace(replace_list[j],'')\n",
        "        if len(text)>0:\n",
        "            data.append(text)\n",
        "    return data  \n",
        "\n",
        "handle.start(readFn=read_from_link_custom)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vPQCKStfXjwu",
        "outputId": "36e20d50-ed06-4989-dfcd-641502436f54",
        "scrolled": true
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "dict_keys(['https://en.wikipedia.org/wiki/Agriculture'])"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#resolved links with extracted text present here\n",
        "handle.resolvedLinks.keys()\n",
        "#links which need manual inspection present here\n",
        "# list(handle.discardedLinks.keys())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MvYQkwzgXjwu"
      },
      "outputs": [],
      "source": [
        "handle.save('saveResolved')\n",
        "handle.saveDiscarded('furtherInspection/day1links.txt')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZbxagYeLXjwu"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mYtVBCUjXjwu"
      },
      "source": [
        "<b>Read from hindi webpage</b>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BqS9eKFeXjwv"
      },
      "outputs": [],
      "source": [
        "# scrape_tool.read_page(link=\"https://hi.wikipedia.org/wiki/%E0%A4%95%E0%A5%83%E0%A4%B7%E0%A4%BF\")\n",
        "scrape_tool.read_page(link=\"https://www.bbc.com/hindi/india-56901831\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BLKN-jhAXjwv"
      },
      "source": [
        "<b>Read from hindi document</b>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S_KddD7yXjwv"
      },
      "outputs": [],
      "source": [
        "scrape_tool.read_page(document='RedRidingHood-H-2mb.pdf')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gmZ5C2CkXjwv"
      },
      "source": [
        "<b>Read from kannada webpage</b>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "agLl3dZ5Xjwv"
      },
      "outputs": [],
      "source": [
        "# scrape_tool.read_page(link=\"https://kn.wikipedia.org/wiki/%E0%B2%B8%E0%B2%BE%E0%B2%B5%E0%B2%AF%E0%B2%B5_%E0%B2%AC%E0%B3%87%E0%B2%B8%E0%B2%BE%E0%B2%AF\")\n",
        "scrape_tool.read_page(link=\"https://kannada.asianetnews.com/karnataka-districts/bjp-mla-g-somashekara-reddy-talks-lockdown-in-karnataka-grg-qs9n0r\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LxSz96YpHGQ-"
      },
      "outputs": [],
      "source": [
        "class ScrapeData():\n",
        "    \"\"\"\n",
        "    Python tool to scrap text data from different languages. 3 options available:\n",
        "    \n",
        "    - scrape from link: provided with a link, the tool scrapes through the text body from the webpage \n",
        "      corresponding to the link.\n",
        "      \n",
        "    - scrape from keyword: provided with a keyword, the tool performs a google search and retrieves text from \n",
        "      a priority domain webpage(eg: wikipedia) or top google search result.\n",
        "      \n",
        "    - scrape from document: provided with a pdf document, the tools accesses the text using pdfplumber \n",
        "      python package. Note that some portion of the text may not be returned properly.\n",
        "    \"\"\"\n",
        "    \n",
        "    def __init__(self):\n",
        "        \"\"\"\n",
        "        Initialise parameters for google search and priority website \n",
        "        \"\"\"\n",
        "        \n",
        "        self.parser = 'html.parser'\n",
        "        self.tld = 'co.in'\n",
        "        self.search_num = 10\n",
        "        self.search_stop = 10\n",
        "        self.pause = 2\n",
        "        self.search_priority = 'wikipedia'\n",
        "        self.auto_return_index = 0\n",
        "        \n",
        "    def read_from_link(self, link, replace_list=['\\n']):\n",
        "        \"\"\"\n",
        "        This function accesses the text content from a webpage link using beautiful soup. To clean the text, \n",
        "        provide the list of charecters to be removed in replace_list.\n",
        "        This may not work for js webpages, cloudfare ddos protected pages etc\n",
        "        \"\"\"\n",
        "        #use header if access is denied.\n",
        "        headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) '\\\n",
        "           'AppleWebKit/537.36 (KHTML, like Gecko) '\\\n",
        "           'Chrome/75.0.3770.80 Safari/537.36'}\n",
        "\n",
        "        page = requests.get(link, headers=headers)\n",
        "        page = requests.get(link)\n",
        "        soup = BeautifulSoup(page.content, self.parser)\n",
        "        data = []\n",
        "        # for i in range(len(soup.find_all(\"div\", {\"id\":\"div993.1.3.2.2541663\"}))):\n",
        "        #     text = soup.find_all(\"div\", {\"id\":\"div993.1.3.2.2541663\"})[i].get_text()\n",
        "        # for i in range(len(soup.find_all(\"div\", {\"class\": \"description\"}))):\n",
        "        #     text = soup.find_all(\"div\", {\"class\": \"description\"})[i].get_text()\n",
        "        # for i in range(len(soup.find_all(\"div\", {\"style\": \"text-align: justify;\"}))):\n",
        "        #     text = soup.find_all(\"div\", {\"style\": \"text-align: justify;\"})[i].get_text()\n",
        "        # for i in range(len(soup.find_all(['h2','h3','p','li']))):\n",
        "        #     text = soup.find_all(['h2','h3','p','li'])[i].get_text() \n",
        "        # for i in range(len(soup.find_all('span'))):\n",
        "        #     text = soup.find_all('span')[i].get_text()\n",
        "        for i in range(len(soup.find_all('p'))):\n",
        "            text = soup.find_all('p')[i].get_text()\n",
        "            for j in range(len(replace_list)):\n",
        "                text = text.replace(replace_list[j],'')\n",
        "                text = text.replace('\\t', '')\n",
        "                text = text.replace('\\xa0', '')\n",
        "                text = text.replace('\\r', '')\n",
        "                text = text.replace('\\u200c', '')      \n",
        "                text = text.replace('\\u200b\\u200b\\u200b\\u200b\\u200b\\u200b\\u200b', '')  \n",
        "                text = text.replace('&apos', '')\n",
        "                text = text.replace('\\uf0b4', '')\n",
        "                \n",
        "            if len(text)>0:\n",
        "                data.append(text)\n",
        "        return data\n",
        "    \n",
        "\n",
        "    def google_search(self, search_keyword, priority=None):\n",
        "        \"\"\"\n",
        "        This function performs google search on the input keyword. Priority can be provided to a particular \n",
        "        website (Eg: wikipedia) \n",
        "        \"\"\"\n",
        "        search_links = []\n",
        "        for link in search(search_keyword, \n",
        "                           tld=self.tld, \n",
        "                           num=self.search_num, \n",
        "                           stop=self.search_stop, \n",
        "                           pause=self.pause):\n",
        "            search_links.append(link)\n",
        "            \n",
        "            if priority is not None:\n",
        "                if self.search_priority in link:\n",
        "                    return link\n",
        "        return search_links[self.auto_return_index]\n",
        "                \n",
        "    \n",
        "    def read_from_doc(self, document):\n",
        "        \"\"\"\n",
        "        This function extracts text from pdf using pdfplumber tool.\n",
        "        \"\"\"\n",
        "        data = []\n",
        "        with pdfplumber.open(document) as pdf:\n",
        "            for i in range(len(pdf.pages)):\n",
        "                page = pdf.pages[i]\n",
        "                text = page.extract_text()\n",
        "                if text is not None:\n",
        "                    text = text.replace('\\n', ' ')\n",
        "                    data.append(text) \n",
        "        return data\n",
        "    \n",
        "    def read_page(self, search_keyword=None, link=None, document=None):\n",
        "        if search_keyword == link == document == None:\n",
        "            raise Exception('Provide link, keyword or document to scrape from')\n",
        "            \n",
        "        if search_keyword is not None:\n",
        "            keyword_link = self.google_search(search_keyword, priority=self.search_priority)\n",
        "            text = self.read_from_link(keyword_link)\n",
        "            return(text)\n",
        "            \n",
        "        if link:\n",
        "            text = self.read_from_link(link)\n",
        "            return(text)\n",
        "                \n",
        "        if document:\n",
        "            text = self.read_from_doc(document)\n",
        "            print(f'{len(text)} pages found')\n",
        "            return(text)\n",
        "            \n",
        "scrape_tool = ScrapeData()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Create a *txt* file to store all data of each domain**"
      ],
      "metadata": {
        "id": "H7Wjf1Z2wFIw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "import string\n",
        "file = '/content/drive/MyDrive/web_scraper/General/Transport_30.json'\n",
        "with open('/content/drive/MyDrive/web_scraper/General/Transport_30.json', \"rb\") as read_file:\n",
        "  data = json.load(read_file)\n",
        "  #deletes the first key (which is a link) from the json data \n",
        "  del data['link']\n",
        "  vals = list( data.values() )\n",
        "  keys = list( data.keys() )\n",
        "  count = 0\n",
        "  # create an empty string\n",
        "  text=\"\"\n",
        "  for val in vals:\n",
        "    if len(val)>1:\n",
        "      for part in val:\n",
        "        text = text +\" \" +(part)+\"\\n\"\n",
        "    else:\n",
        "      text = text + val[0]+\"\\n\" \n",
        "      count += 1\n",
        "    #deletes punctuation(Ex : , ! ?)\n",
        "    text=text.translate(text.maketrans('','',string.punctuation))\n",
        "    print(text)\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data/General Raw Data.txt\", \"a\") as write_file:\n",
        "   write_file.write(text)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VMmWA0ZRwFYX",
        "outputId": "239fa8ac-aee1-4e58-ecf6-141dae0179c8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " বিকেল হতেই ধর্মতলার বাস টার্মিনাসে দেখা গেল শ’য়ে শ’য়ে অন্য জেলার যাত্রীদের ভিড়। \n",
            " হাতে আর মাত্র কয়েক ঘণ্টা। তার মধ্যেই ফিরে যেতে হবে বাড়ি। পরিবারপরিজনের কাছে। না হলেই আটকে পড়তে হবে। রবিবার থেকে রাজ্যে কার্যত লকডাউনের ঘোষণা হওয়ার পর থেকেই ভিড় জমতে শুরু করেছিল শহরের বাস টার্মিনাসগুলোতে। যাঁরা ভিন্জেলা থেকে কর্মসূত্রে কলকাতা শহরে এসেছিলেন শনিবার বিকেল থেকেই তাঁদের মধ্যে বাড়ি ফেরার তাড়াহুড়ো লক্ষ করা গেল।\n",
            " বিকেল হতেই ধর্মতলার বাস টার্মিনাসে দেখা গেল শ’য়ে শ’য়ে অন্য জেলার যাত্রীদের ভিড়। তাঁদের হাতে বোঁচকা ব্যাগ লটবহর। অনেকের মুখেই মাস্ক নেই। কারও মুখে মাস্কের পরিবর্তে গামছা জড়ানো। কোভিড পরিস্থিতির মধ্যেও তাঁদের ঠাসাঠাসি করেই উঠতে হচ্ছে বাসে। দূরত্ব বিধির তোয়াক্কা না করে কোনও রকমে চেপেচুপে বাসের মাথাতে জায়গা করে নিচ্ছেন তাঁরা।\n",
            " কোভিড সংক্রমণে রাশ টানতে রবিবার থেকে রাজ্যে কার্যত লকডাউন ঘোষণা করেছে সরকার। জারি থাকবে আগামী ১৫ দিন পর্যন্ত। সরকারের তরফে স্পষ্ট জানিয়ে দেওয়া হয়েছে জরুরি পরিষেবা বাদে বন্ধ থাকবে সমস্ত সরকারি এবং বেসরকারি অফিস দফতর। আগেই রাজ্যে লোকাল ট্রেন বন্ধ করার সিদ্ধান্ত নিয়েছিল রাজ্য সরকার। শনিবারের ঘোষণায় জানিয়ে দেওয়া হল বাস মেট্রো ফেরি পরিষেবাও বন্ধ থাকবে। জরুরি প্রয়োজন ছাড়া চলাচল করতে পারবে না ট্যাক্সি কিংবা অটো। যার ফলে দূর থেকে যাঁরা এসেছিলেন বাড়ি ফেরার জন্য তাঁদের হাতে সময় বলতে রয়েছে শনিবার বিকেল থেকে রাত পর্যন্তই। ট্রেন আগে থেকেই বন্ধ থাকায় এখন ফিরে যাওয়ার একমাত্র উপায় বাস।\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Remove characters (|, ' ,>>)from the text file and save it in a different file**"
      ],
      "metadata": {
        "id": "2u08dFhh1gqd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with open('/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data/General Raw Data.txt', 'r') as read_file, open('/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data Final/General Raw Data Final.txt', 'w') as outfile:\n",
        "    data = read_file.read()\n",
        "    remove_char='-♦©।‘‘‧—…‘’“”～®→•●'\n",
        "    translation_table = str.maketrans('', '', remove_char)\n",
        "    data = data.translate(translation_table)\n",
        "    # temp = re.sub('[-♦©।‘‘‧’র—…’s‘’»ো“”～®→•]', ' ', temp)\n",
        "    # temp = infile.read().replace(\"-\", \" \")\n",
        "    outfile.write(data)"
      ],
      "metadata": {
        "id": "2e3ff2vJ1cgb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        " **Count the total number of Lines  from the extracted Json files**"
      ],
      "metadata": {
        "id": "87l-3eJMXp0y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "file = '/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data Final/General Raw Data Final.txt'\n",
        "with open('/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data Final/General Raw Data Final.txt', \"rb\") as read_file:\n",
        "    data =len(read_file.readlines())\n",
        "    print('Total Number of lines:', data)\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Word Frequency/Frequency/General Word Frequency.txt\", \"w\") as write_file:\n",
        "  write_file.write(\"Total number of lines: \"+str(data) +\"\\n\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ct2IsfuJXqGT",
        "outputId": "2f8bdce1-15fa-463c-e5f2-0fb7c62a0842"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total Number of lines: 34963\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Word Frequency**"
      ],
      "metadata": {
        "id": "i5dx--Wzx9aU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import operator\n",
        "text = open(\"/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data Final/General Raw Data Final.txt\", \"r\")\n",
        "\n",
        "# Create an empty dictionary\n",
        "d = dict()\n",
        "\n",
        "# Loop through each line of the file\n",
        "for line in text:\n",
        "\t# Remove the leading spaces and newline character\n",
        "\tline = line.strip()\n",
        "\n",
        "\t# Convert the characters in line to\n",
        "\t# lowercase to avoid case mismatch\n",
        "\tline = line.lower()\n",
        "\n",
        "\t# Split the line into words\n",
        "\twords = line.split(\" \")\n",
        "\t\t\t\t\t\t\n",
        "\n",
        "\t# Iterate over each word in line\n",
        "\tfor word in words:\n",
        "\t\t# Check if the word is already in dictionary\n",
        "\t\tif word in d:\n",
        "\t\t\t# Increment count of word by 1\n",
        "\t\t\td[word] = d[word] + 1\n",
        "\t\telse:\n",
        "\t\t\t# Add the word to dictionary with count 1\n",
        "\t\t\td[word] = 1\n",
        "sorted_d = dict( sorted(d.items(), key=operator.itemgetter(1),reverse=True))\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Word Frequency/Frequency/General Word Frequency.txt\", \"a\") as write_file:\n",
        "  # Print the contents of dictionary\n",
        "  for key in list(sorted_d.keys()):\n",
        "    write_file.write(key+ \":\"+str(sorted_d[key]) +\"\\n\")\n",
        "# \tprint(key, \":\", d[key])\n"
      ],
      "metadata": {
        "id": "8rKRTJeOyCjZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Character Frequency(Doesn't  include punctuations)**"
      ],
      "metadata": {
        "id": "ePEln2sNTgJ5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import operator\n",
        "data =  open(\"/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data/General Raw Data (1).txt\", \"r\")\n",
        "input_string = data.read()\n",
        "mySet = set(input_string)\n",
        "countOfChars = dict()\n",
        "for element in mySet:\n",
        "    countOfChar = 0\n",
        "    for character in input_string:\n",
        "        if character == element:\n",
        "            countOfChar += 1\n",
        "    countOfChars[element] = countOfChar\n",
        "sorted_freq = dict( sorted(countOfChars.items(), key=operator.itemgetter(1),reverse=True))\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Dictionary/General Char Frequency.txt\", \"w\") as write_file:\n",
        "  for key in list(sorted_freq.keys()):\n",
        "    write_file.write(key+ \":\"+str(sorted_freq[key]) +\"\\n\")\n",
        "    #write_file.write(key+ \":\"+str(sorted_freq[key]) +\"\\n\")\n",
        "# \tprint(key, \":\", d[key])\n"
      ],
      "metadata": {
        "id": "cJDHdU0NTkEQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Find words having frequency <=50**"
      ],
      "metadata": {
        "id": "8Hy7liuoTvtf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n",
        "# Open the input file for reading\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Final/Agriculture Char Frequency Final.txt\", \"r\") as input_file:\n",
        "    # Read the contents of the file into a string\n",
        "    input_str = input_file.read()\n",
        "\n",
        "# Convert the string to a dictionary\n",
        "dictionary = eval(input_str)\n",
        "\n",
        "# empty list to store the keys\n",
        "keys = []\n",
        "\n",
        "# Loop over the items in the dictionary\n",
        "for key, value in dictionary.items():\n",
        "    # If the value is less than 50, add the key to the list\n",
        "    if value < 50:\n",
        "        keys.append(key)\n",
        "\n",
        "# Open the output file for writing\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Word Frequency/50_frequency\", \"w\") as output_file:\n",
        "    # Write the list of keys to the file, one per line\n",
        "    for key in keys:\n",
        "        output_file.write(key + \"\\n\")\n"
      ],
      "metadata": {
        "id": "yuvxU926Tyds"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "AvFE3KgxT2L7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "original_file = \"/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data/Agriculture Raw Data.txt\"\n",
        "final_file = \"/content/drive/MyDrive/web_scraper/Data Clean/Agriculture.txt\"\n",
        "\n",
        "word = ['文', 'க','本']\n",
        "with open(original_file, \"r\") as input:\n",
        "    with open(final_file, \"w\") as output:\n",
        "        for line in input:\n",
        "          for w in word:\n",
        "            if w not in line.strip(\"\\n\"):\n",
        "              output.write(line)\n",
        "              break\n",
        "                \n",
        "            "
      ],
      "metadata": {
        "id": "S_cEFYTzT7Ms"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Creating Excel File"
      ],
      "metadata": {
        "id": "qBHBUjxCT8In"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "dict1 =  open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Dictionary/Agriculture Char Frequency.txt\", \"r\")\n",
        "df = pd.DataFrame(data=dict1, index=[0])\n",
        "df = (df.T)\n",
        "#print (df)\n",
        "df.to_excel('/content/drive/MyDrive/web_scraper/Character Frequency/Excel Sheet/Agriculture.xlsx')"
      ],
      "metadata": {
        "id": "IFAOzmZpUCPl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Save Dictionary data to Excel Sheet**"
      ],
      "metadata": {
        "id": "nTtmk5yiU2aa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "dict1 =  open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Dictionary/Agriculture Char Frequency.txt\", \"r\")\n",
        "df = pd.DataFrame(data=dict1, index=[0])\n",
        "df = (df.T)\n",
        "#print (df)\n",
        "df.to_excel('/content/drive/MyDrive/web_scraper/Character Frequency/Excel Sheet/Agriculture.xlsx')"
      ],
      "metadata": {
        "id": "vvJ1YrSWU6QR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Save Dictionary data to Excel Sheet**"
      ],
      "metadata": {
        "id": "C4hNHfKpiuJq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "#creating data frames\n",
        "import pandas as pd\n",
        "dict = ('/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data Final/General Raw Data Final.txt')\n",
        "data_frame=pd.DataFrame(eval(dict))\n",
        "\n",
        "#path=r\"/content/drive/MyDrive/web_scraper/Excel Sheet/Agriculture.xlsx\"\n",
        "with pd.ExcelWriter('/content/drive/MyDrive/web_scraper/Excel Sheet/Agriculture.xlsx\") as engine:\n",
        "  data_frame.to_excel(excel_writer=engine,index=false)"
      ],
      "metadata": {
        "id": "k2GEJkdbiugU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0eMAu1zM8XTa"
      },
      "source": [
        "**Converts the list into dictionary where links are saved as key and content is saved as value and then the file is saved in Directory**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ixizlfik5m3Z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4da97b02-6c6f-49ca-fd83-67d2638b3fcd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{\n",
            "    \"link\": \"https://bn.how-what-finance.com/12738748-how-to-choose-a-credit-counseling-agency-14-steps-with-pictures\",\n",
            "    \"content\": [\n",
            "        \"You don't have permission to access /12738748-how-to-choose-a-credit-counseling-agency-14-steps-with-pictureson this server.\"\n",
            "    ]\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "import json\n",
        "link=\"https://bn.how-what-finance.com/12738748-how-to-choose-a-credit-counseling-agency-14-steps-with-pictures\"\n",
        "content = scrape_tool.read_page(link=\"https://bn.how-what-finance.com/12738748-how-to-choose-a-credit-counseling-agency-14-steps-with-pictures\")\n",
        "#delete words from the extracted data from the last\n",
        "# del content[10:]\n",
        "# del content[-10:]\n",
        "# del content[:2]\n",
        "# content.remove(\"Ready to Invest?Talk to our investment specialistDisclaimer\")\n",
        "# content.remove(\"প্রতীকী ছবি।   \")\n",
        "# content.remove(\" \")\n",
        "# content.remove(\"জমি বা শস্যগুলিতে জলের জলের সরবরাহ সাধারণত বৃদ্ধির পক্ষে সহায়তা করে চ্যানেলগুলির মাধ্যমে।জল বা medicationষধের অবিচ্ছিন্ন প্রবাহের সাথে কোনও অঙ্গ বা ক্ষত ধুয়ে ফেলার প্রক্রিয়া।গর্ত ইত্যাদি দিয়ে শুকনো জমি সরবরাহ করা(medicineষধ) জলে বা medicষধিযুক্ত দ্রবণ দিয়ে ফ্লাশ করে বা ধৌত করে ক্ষত বা দেহের অঙ্গ পরিষ্কার করে\")\n",
        "# # del content[11:]\n",
        "# Data to be written\n",
        "dictionary ={\n",
        "    \"link\" :link,\n",
        "    \"content\" :content\n",
        "}\n",
        "json_object = json.dumps(dictionary, ensure_ascii=False,indent = 4) \n",
        "print(json_object)    \n",
        "with open(\"/content/drive/MyDrive/web_scraper/Finance/Credit counseling_2.json\", \"w\") as json_file:\n",
        "    json.dump(dictionary, json_file ,ensure_ascii=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Unicode list of Bengali Characters(Source: https://unicode.org/charts/PDF/U0980.pdf)"
      ],
      "metadata": {
        "id": "AscgJ3PWjDkQ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aLlpgDFtMsDN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e75e3606-1276-4210-e904-96853b5530c9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['ঀ', 'ঁ', 'ং', 'ঃ', 'অ', 'আ', 'ই', 'ঈ', 'উ', 'ঐ', 'ও', 'ঔ', 'ক', 'খ', 'গ', 'ঘ', 'ঙ', 'ঊ', 'ঋ', 'ঌ', 'এ', 'ঐ', 'ও', 'ঔ', 'ক', 'খ', 'গ', 'ঘ', 'ঙ', 'চ', 'ছ', 'জ', 'ঝ', 'ঞ', 'ট', 'ঠ', 'ড', 'ঢ', 'ণ', 'ত', 'থ', 'দ', 'ধ', 'ন', 'প', 'ফ', 'ব', 'ভ', 'ম', 'য', 'র', 'ল', 'শ', 'ষ', 'স', 'হ', '়', 'ঽ', 'া', 'ি', 'ী', 'ু', 'ূ', 'ৃ', 'ৄ', 'ে', 'ৈ', 'ো', 'ৌ', '্', 'ৎ', 'ৗ', 'ড়', 'ঢ়', 'য়', 'ৠ', 'ৡ', 'ৢ', 'ৣ', '০', '১', '২', '৩', '৪', '৫', '৬', '৭', '৮', '৯', 'ৰ', 'ৱ', '৲', '৳', '৴', '৵', '৶', '৷', '৸', '৹', '৺', '৻', 'ৼ', '৽']\n"
          ]
        }
      ],
      "source": [
        "\n",
        "unicode_list = [chr(i) for i in range(0x0980, 0x0983 + 1)]+[chr(i) for i in range(0x0985, 0x0989 + 1)]+[chr(i) for i in range(0x0990, 0x0990 + 1)]+[chr(i) for i in range(0x0993, 0x0999 + 1)]+[chr(i) for i in range(0x098A, 0x098C + 1)]+[chr(i) for i in range(0x098F, 0x098F + 1)]+[chr(i) for i in range(0x0990, 0x0990 + 1)]+[chr(i) for i in range(0x0993, 0x0999 + 1)]+[chr(i) for i in range(0x099A, 0x099F + 1)]+[chr(i) for i in range(0x09A0, 0x09A8 + 1)]+[chr(i) for i in range(0x09AA, 0x09AF + 1)]+[chr(i) for i in range(0x09B0, 0x09B0 + 1)]+[chr(i) for i in range(0x09B2, 0x09B2 + 1)]+[chr(i) for i in range(0x09B6, 0x09B9 + 1)]+[chr(i) for i in range(0x09BC, 0x09BF + 1)]+[chr(i) for i in range(0x09C0, 0x09C4 + 1)]+[chr(i) for i in range(0x09C7, 0x09C8 + 1)]+[chr(i) for i in range(0x09CB, 0x09CE + 1)]+[chr(i) for i in range(0x09D7, 0x09D7 + 1)]+[chr(i) for i in range(0x09DC, 0x09DD + 1)]+[chr(i) for i in range(0x09DF, 0x09DF + 1)]+[chr(i) for i in range(0x09E0, 0x09E3 + 1)]+[chr(i) for i in range(0x09E6, 0x09E9 + 1)]+[chr(i) for i in range(0x09EA, 0x09EF + 1)]+[chr(i) for i in range(0x09F0, 0x09F9 + 1)]+[chr(i) for i in range(0x09FA, 0x09FD + 1)]\n",
        "print(unicode_list)\n",
        "# with open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Unicode List/Unicode_list.txt\", \"w\") as write_file:\n",
        "#    write_file.write(str(unicode_list))"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Read The Excel file of Character Frequency (General)**\n"
      ],
      "metadata": {
        "id": "7CNgrBwLjH1E"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "SHEET_ID = '14jOMbRVREUd-jszbXnb9P5IO6PveN42c'\n",
        "SHEET_NAME = 'scrape_data'\n",
        "url = f'https://docs.google.com/spreadsheets/d/{SHEET_ID}/gviz/tq?tqx=out:csv&sheet={SHEET_NAME}'\n",
        "df = pd.read_csv(url)\n",
        "print(df.head(5))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dAuzZgqdjMl6",
        "outputId": "aca71975-41b1-4531-b78a-1b86a7fed054"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Unnamed: 0  4209824\n",
            "0          া  1150503\n",
            "1          e  1084987\n",
            "2          র  1045440\n",
            "3          ্   936717\n",
            "4          ে   892214\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Converting the General Character Column in list**"
      ],
      "metadata": {
        "id": "yVFRRt27jPZC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "new_column=df.iloc[:,0]\n",
        "data_list_general = new_column.values.tolist()\n",
        "\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Excel Sheet to List Format/General Character List.txt\", \"w\") as write_file:\n",
        "   write_file.write(str(data_list_general))"
      ],
      "metadata": {
        "id": "CgpXRC9YjUUj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Comparing the Unicode and Character Frequency List**"
      ],
      "metadata": {
        "id": "frffWIK6jad9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# data_list_general\n",
        "# unicode_list\n",
        "\n",
        "'''\n",
        "(ABSENT CHARACTER)\n",
        "(PRESENT CHARACTERS)\n",
        "absent characters define the characters of unicode_list not present in the General Character Frequency list\n",
        "'''\n",
        "absent_characters=[]\n",
        "present_characters={}\n",
        "for i, element in enumerate(data_list_general):\n",
        "        if element in unicode_list:\n",
        "            present_characters[element] = i\n",
        "        else:\n",
        "            absent_characters.append(element)\n",
        "\n",
        "      \n",
        "print(\"Absent Characters  : \",absent_characters)\n",
        "print(\"Present Characters  : \",present_characters)        \n",
        "with open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Absent Characters/General Absent Character.txt\", \"w\") as write_file:\n",
        "   write_file.write(str(absent_characters))\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Present Characters/General Present-Character Index.txt\", \"w\") as write_file:\n",
        "   write_file.write(str(present_characters))\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vfigry8UjbNq",
        "outputId": "8a730057-8ba2-4ee3-f615-deece73873bb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Absent Characters  :  ['e', 'a', 't', 'i', 'o', 'n', 's', 'r', 'l', 'h', 'c', 'd', 'u', 'm', 'g', 'p', 'f', 'y', '।', 'w', nan, 'b', 'v', '1', '0', '2', 'k', '9', '3', 'T', 'A', '5', 'S', '4', '8', 'I', 'C', 'x', 'P', 'E', 'M', 'B', 'N', 'R', 'D', 'F', 'W', 'L', 'G', 'O', 'H', '’', '6', 'j', '7', 'q', 'U', 'z', 'J', 'K', '–', 'V', '‘', '“', 'Y', '”', '—', 'Q', 'Z', '•', 'а', 'ا', 'X', 'и', '…', '🎯', '\\u200e', 'с', '©', 'к', 'о', 'َ', 'ل', 'р', 'н', '°', 'é', 'е', '↑', 'ر', 'و', 'т', '·', 'ن', 'ा', 'µ', 'л', '\\u200d', 'م', 'ِ', 'ُ', '्', 'ي', '→', 'í', 'у', 'र', 'ç', 'ā', 'ə', 'ۡ', 'ी', 'ی', 'й', 'ت', 'े', 'ب', 'न', 'в', 'ि', 'د', 'क', 'â', 'س', 'ة', 'á', 'ْ', 'ü', 'ó', 'ف', 'ع', 'д', 'г', 'з', 'п', 'Р', 'А', 'म', 'Б', '்', 'α', 'λ', 'я', 'ّ', 'š', 'ج', 'त', 'ो', 'स', 'ك', 'ш', '\\u200b', 'ă', 'Δ', 'ک', 'ح', 'ν', 'ö', 'ι', 'č', 'ह', 'М', 'ल', 'द', 'ز', 'м', 'ο', '−', 'Т', 'х', 'ь', 'Н', 'प', 'Е', 'Â', 'ه', '\\x92', 'ч', 'ã', '®', 'ं', 'ி', 'τ', 'य', 'ص', '×', '\\ufeff', 'θ', 'μ', 'κ', '»', '語', 'ы', 'ٰ', 'ä', 'ñ', 'æ', 'ρ', 'ம', 'ե', 'ज', 'ˈ', 'י', 'ц', 'η', 'И', 'ာ', 'У', 'à', '₹', 'ê', 'б', '†', 'த', 'ú', 'ɪ', 'რ', 'ː', 'ա', 'ა', 'γ', 'व', 'С', 'è', 'ு', 'ō', 'К', 'ς', 'О', '👉', 'ά', 'ր', 'ท', 'ನ', 'भ', '국', 'ʃ', 'ế', '€', 'ہ', '어', 'π', '한', 'უ', 'ר', 'ლ', 'ი', 'ї', 'ु', 'မ', 'ย', 'ъ', '中', '❑', '文', '日', 'க', '本', 'þ', '←', 'П', '್', 'ழ', 'ë', 'ء', 'ు', 'ð', 'ट', 'ไ', 'ת', 'ش', 'ệ', 'ע', 'ै', 'Č', 'ய', 'ҷ', 'å', 'ב', 'і', '\\x80', 'σ', 'ാ', 'ն', 'ग', '‐', 'ţ', 'ŋ', 'ब', 'ല', 'յ', 'യ', 'ള', 'ം', 'Ε', 'മ', 'א', 'э', 'ʒ', 'ε', 'پ', 'ड', 'ū', 'ø', 'Л', '💡', 'В', 'თ', 'च', 'ē', 'ી', 'ਾ', 'ქ', '✓', '̪', 'õ', '\\u2061', 'إ', '±', 'ق', 'ж', 'ठ', '్', 'έ', '∼', '►', 'ల', 'Հ', 'ಡ', 'ṇ', '⋅', 'Г', 'ɒ', 'ร', 'ɡ', '🙂', 'қ', '²', 'श', 'ś', '🎓', 'ή', 'ț', 'इ', 'గ', '粵', '➤', 'ʊ', 'غ', 'थ', 'મ', 'เ', 'ً', 'န', '်', 'သ', 'آ', 'ြ', 'ဘ', '\\xad', 'Қ', 'Ś', '🏻', 'Ҷ', 'Д', '⟩', 'ș', '持', 'ž', 'ī', 'δ', '长', 'أ', 'Ъ', 'ć', 'ಕ', 'ַ', 'Α', 'Ү', 'ҳ', 'એ', 'अ', 'φ', '⟨', 'ί', 'Ц', 'Я', '增', 'ئ', 'ட', 'ų', 'î', 'ɔ', 'త', 'ى', 'ɜ', 'ੰ', 'É', 'ۆ', 'ુ', 'ல', 'ர', 'ό', 'ɾ', 'ط', '☺', 'Í', 'ı', 'ؕ', '′', 'ె', nan, 'ख', 'ô', '½', 'ʻ', 'ˑ', 'ष', 'β', 'ذ', '👨', 'ऑ', 'Ч', 'ɑ', '§', 'ල', 'ි', 'ӣ', '⇒', 'ਪ', 'ӑ', 'හ', 'Å', 'Á', 'வ', 'ф', 'ස', 'է', 'ப', '≈', 'ώ', 'خ', 'ў', 'ெ', 'ං', 'ồ', 'ध', 'მ', 'ҡ', 'ਬ', 'ள', 'ਜ', 'გ', 'ੀ', 'ण', '语', 'ؤ', 'υ', 'ި', 'ض', 'ύ', '̯', '¤', '吴', 'ะ', '💻', 'उ', 'आ', 'ל', 'ɛ', 'ศ', 'ை', 'ṣ', 'మ', '‒', 'ర', 'מ', 'ښ', 'ٌ', 'ற', '️', '‚', 'Π', '总', 'Ἰ', '😍', '«', 'ิ', 'ม', 'ڌ', '人', 'ป', '์', 'ು', 'ė', 'χ', 'ջ', '📖', '\\x99', '\\x98', 'ګ', 'ו', '速', 'ù', 'Š', '″', 'ې', 'ړ', '據', 'ា', 'ॉ', 'լ', '口', 'ט', 'Ž', 'ય', 'બ', '統', 'อ', '計', 'ೆ', 'உ', 'Ӏ', '度', 'Ó', '，', '量', '（', '數', '续', 'З', 'ד', 'ո', 'ଆ', 'խ', 'ଓ', '保', '手', 'ټ', 'ି', '🤔', 'ò', '⊕', 'Ú', 'Ć', '❏', '缓', 'ต', '\\uf0b0', 'ẽ', 'Κ', 'ટ', '庫', '଼', '放', nan, 'ા', 'ଡ', 'ɚ', '）', 'վ', '😃', 'ਗ', 'ש', 'Գ', '\\uf0a7', 'Ю', '⁄', 'ా', '£', 'ʰ', 'ה', '∴', '√', 'ಿ', 'ಗ', 'ई', 'ગ', 'ર', 'ಮ', '¯', '🇰', '\\uf005', 'ன', '✍', '🇳', 'Æ', 'જ', 'ா', 'ן', '空', 'ರ', 'ṭ', 'Ā', 'ʔ', '͡', '🇵', 'அ', 'פ', '👇', '🇮', 'մ', 'ત', '😌', 'ۤ', '॥', 'ए', 'ث', '👍', 'ृ', '›', '♦', 'ω', 'đ', '🌹', 'ೇ', 'ސ', '´', '▼', 'ظ', 'ீ', '😎', 'ަ', 'տ', 'ʁ', '🚂', 'ވ', 'ू', 'ễ', '♻', '≤', 'ಲ', '⚲', 'Τ', 'ಯ', 'ದ', '‧', 'ހ', 'औ', 'ދ', 'Х', 'ի', 'ް', 'ວ', 'ٍ', 'ṛ', '¢', 'າ', 'ξ', 'ބ', 'ެ', 'ʼ', '航', 'Γ', 'ລ', 'Ñ', 'ీ', '﴿', 'น', '\\x9c', 'फ', '़', 'ँ', 'ವ', 'ಸ', 'ి', 'డ', 'ק', 'گ', '\\x9d', 'ů', 'û', '👫', 'ִ', '📝', '✈', 'ָ', '柔', '∝', 'ਿ', '្', 'ĭ', 'ٹ', 'ఆ', 'հ', '۬', 'ச', 'ਧ', '😔', 'ห', 'Э', 'ੱ', '🚩', 'ւ', '術', '\\u202f', 'ʱ', '●', 'ខ', '〉', 'ᴀ', '¡', '员', 'ែ', '≥', 'घ', 'រ', 'ੜ', 'న', 'ಾ', '🏼', 'ŏ', 'ʌ', '💯', 'ண', 'ɖ', 'ĕ', '💵', '😟', 'ـ', 'Ա', 'ബ', '国', '💰', 'Ü', 'ស', 'ಆ', '天', 'ភ', 'ਭ', '🜨', 'ម', 'ก', '家', '民', '마', 'ż', '장', 'ɴ', '合', '🙏', 'వ', 'ï', 'ว', '사', 'Đ', 'ۃ', 'ʷ', '출', 'ప', '道', 'ḗ', 'ั', '🗞', 'ɐ', '🤩', '举', 'ὑ', '̥', '字', 'ʲ', 'ｘ', 'ン', 'か', 'ట', 'ಎ', '̚', 'య', '重', '🔥', 'ᱟ', 'ך', '🎭', '💥', '联', '⚖', 'ǐ', 'ˌ', 'ʍ', '🏆', 'ì', 'ǒ', 'ే', 'ą', 'ト', '🆓', '函', 'ܐ', 'ौ', '漢', '⇌', 'ę', '⊙', '太', 'ఎ', 'ん', '眠', 'ּ', 'ۚ', 'ळ', 'じ', 'ɽ', '球', 'ʈ', 'ħ', '♄', '土', 'ɫ', 'எ', '¬', '😋', '÷', 'ὴ', 'œ', 'า', '™', 'ŭ', '青', '会', 'ǫ', '🌸', '💼', 'ਲ', 'ƿ', 'ಹ', nan, 'ḕ', 'ٔ', 'ß', '💐', 'ײ', 'ֿ', '🎬', '😊', '⠰', '📚', '📰', '組', 'ז', 'ἄ', '⊂', 'ῆ', '✝', '¦', 'స', '际', 'ἀ', 'ÿ', 'క', 'Ō', 'ỹ', 'ᴇ', 'ộ', '∠', '도', '∎', 'జ', '۪', 'ἔ', 'ھ', '客', '지', 'נ', 'ٖ', 'ɣ', 'ὐ', '́', 'ᴄ', 'ۢ', 'ం', '連', 'ő', '제', 'ో', 'ố', 'ʟ', 'ġ', '象', '💎', 'ི', 'Ծ', 'ஜ', '理', 'ě', '琵', '🌺', '籍', 'ஆ', '۽', 'ầ', 'ۭ', '🛒', 'フ', 'ോ', 'ֹ', '宇', 'Ά', '👈', 'ṅ', '̤', '乓', 'ᱲ', '연', 'ᱱ', '‑', 'ਕ', 'ओ', '맹', '際', '体', 'ᱤ', '琶', 'ಪ', 'ɩ', '时', '方', 'ả', '具', 'リ', 'ạ', 'ொ', 'ğ', '오', 'ೀ', 'զ', 'ɯ', 'ә', 'ڑ', 'ィ', 'ོ', '💺', 'ڈ', 'ル', '🎖', '་', '⚡', '۫', '🖥', '💪', '🆘', 'イ', '🔋', 'ಜ', '古', '隧', '്', 'グ', 'ⲣ', 'ദ', '🛬', 'テ', 'झ', '戶', '―', 'ʀ', '🇲', '和', '討', 'ಳ', 'ส', 'ᴛ', 'ܡ', 'ɦ', 'ై', 'ೈ', 'ำ', 'ತ', 'ԥ', '³', 'ག', '❤', '🌍', 'ஞ', 'ց', '̄', '🛫', 'ウ', 'ٓ', 'ജ', 'Β', '布', '↓', '트', 'భ', 'ű', '🥺', '🥇', 'צ', '★', 'ܪ', '力', 'ി', 'ਨ', 'ṳ', 'ṓ', 'ɹ', '角', 'ϕ', 'བ', '안', 'ǔ', '卓', '라', 'દ', '📱', 'ᴅ', 'ᾶ', '💫', 'അ', '👮', '打', '欧', '州', '🏄', 'ஏ', 'ё', 'ઝ', 'ਚ', '之', 'இ', '이', '发', 'գ', 'ད', '流', 'ǝ', '趙', '📞', '乒', '💴', 'ᱛ', 'ۙ', '□', 'ద', '间', 'พ', 'ந', 'ಭ', '廻', 'ネ', '🤳', 'ਵ', 'ே', '💸', '唐', 'ɢ', '小', '腰', '🇷', '∗', '魔', '昱', '😉', '่', 'ữ', 'ཡ', '：', 'ܝ', 'エ', '棋', 'ᴏ', 'ụ', '足', '주', '盟', 'ｃ', 'ூ', '้', 'ἡ', 'ു', 'ં', 'ὠ', '📲', 'ธ', '역', '︾', 'դ', '裡', 'ᱥ', 'Ο', 'ง', '一', 'ֆ', '捕', 'օ', '🌐', '隊', '\\uf06e', '카', 'ᴋ', 'ῶ', 'ક', 'ﺚ', 'ొ', 'ロ', 'ʙ', '미', 'ℕ', '부', '̩', 'శ', '화', '바', 'ℚ', '천', 'ẖ', 'ધ', '🥰', '🇬', '➡', 'ヨ', 'Ö', '☎', '해', 'ー', 'ł', '경', '의', 'ચ', '\\u200f', '🇧', 'ψ', 'ો', '배', 'ோ', '샤', '름', '파', '通', 'ම', 'ᴜ', 'ừ', '\\x93', '먹', '성', 'ッ', 'ℤ', '،', 'ń', '♣', 'ᴡ', 'چ', 'ٗ', 'ද', 'ἱ', 'ὤ', '歐', '٪', 'ℂ', '̃', '¼', 'パ', '리', '포', 'ʏ', '那', '無', 'ⓠ', '👌', '¥', '구', 'ℝ', '路', '탱', 'ු', 'ặ', '警', 'ὀ', 'હ', '交', '剎', '察', '젖', '외', 'ř', '國', '高', '̱', 'ः', '℅', '인', 'ન', '튀', '洲', '정', '벳', 'Ὀ', 'ุ', 'સ', '흐', '당', 'ں', '추', 'ษ']\n",
            "Present Characters  :  {'া': 0, 'র': 2, '্': 3, 'ে': 4, 'ি': 7, 'ন': 12, 'ক': 14, 'ব': 15, 'ত': 16, 'য': 17, 'স': 19, 'প': 22, 'ল': 24, 'ম': 25, 'দ': 28, 'ু': 29, '়': 32, 'ট': 34, 'শ': 35, 'হ': 36, 'জ': 37, 'গ': 38, 'এ': 39, 'ো': 40, 'ই': 44, 'ী': 45, 'ষ': 48, 'চ': 52, 'থ': 53, 'ভ': 54, 'ধ': 55, 'ছ': 56, 'আ': 57, 'অ': 59, 'ং': 60, 'য়': 61, 'ও': 62, 'ড': 63, 'ণ': 64, 'খ': 65, 'উ': 66, 'ফ': 73, 'ূ': 75, 'ৃ': 77, 'ঠ': 81, 'ঞ': 85, 'ঙ': 86, 'ৰ': 91, 'ৈ': 92, 'ঘ': 95, 'ঁ': 96, 'ৌ': 101, 'ৎ': 107, 'ড়': 111, 'ঝ': 115, 'ঃ': 121, 'ঢ': 123, 'ঐ': 124, '৷': 126, 'ৱ': 128, 'ঋ': 143, 'ঈ': 152, 'ঔ': 159, 'ঊ': 163, 'ঢ়': 257, 'ৗ': 306, '৳': 417, 'ৡ': 1051, 'ঌ': 1063, 'ৠ': 1123}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Convert Bengali Paragraph into New Line seperated Text Paragraph.**"
      ],
      "metadata": {
        "id": "rohcRkqxjhDy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "split_character = \"|\"\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data/Random Bengali.txt\", \"r\") as input_file:\n",
        "    contents = input_file.read()\n",
        "\n",
        "\n",
        "split_contents = contents.split(split_character)\n",
        "\n",
        "\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data/ Raw data.txt\", \"w\") as output_file:\n",
        "    for split_string in split_contents:\n",
        "        output_file.write(split_string.strip() + \"\\n\")\n",
        "\n"
      ],
      "metadata": {
        "id": "nZ-DaZOpjkyA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Count Lines That Can be removed  (In English Text)**"
      ],
      "metadata": {
        "id": "tt65kTPJjxaa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data/Random.txt\", \"r\") as file:\n",
        "    lines = file.readlines()\n",
        "    characters_to_remove = [\"a\", \"r\", \"p\"]\n",
        "    num_removed = 0\n",
        "    for i, line in enumerate(lines):\n",
        "        if any(char in line for char in characters_to_remove):\n",
        "            lines[i] = \"\"\n",
        "            num_removed += 1\n",
        "\n",
        "\n",
        "print(f\"Removed {num_removed} lines from the file.\")\n",
        "#Formatted Text\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data/Removable Lines.txt\", \"w\") as write_file:\n",
        "   write_file.write(str(lines))"
      ],
      "metadata": {
        "id": "Is9ZXrByjyJB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Add new lines in the Raw Data**"
      ],
      "metadata": {
        "id": "UIhExjB7oJD3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the character to split on\n",
        "split_character = \"|\"\n",
        "\n",
        "# Open the input file and read its contents\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data/General Raw Data.txt\", \"r\", encoding=\"utf-8\") as input_file:\n",
        "    contents = input_file.read()\n",
        "\n",
        "# Split the contents by the split character\n",
        "split_contents = contents.split(split_character)\n",
        "\n",
        "# Open the output file for writing\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data with New Lines/General.txt\", \"w\", encoding=\"utf-8\") as output_file:\n",
        "    # Write each split string to the output file\n",
        "    for split_string in split_contents:\n",
        "        output_file.write(split_string.strip() + \"\\n\")\n"
      ],
      "metadata": {
        "id": "jKq3Vh-OoMZf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Extract Character List from the Google Sheet (General) that need to be removed**"
      ],
      "metadata": {
        "id": "AHtMhtP8mGcP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "row_data=df.iloc[-809:, 0]\n",
        "row_data_final=row_data.values.tolist()\n",
        "print(row_data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aDmTpvuwmKgH",
        "outputId": "9fc63e51-4abf-4a07-a0e7-c6022d2a02b1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "455     ئ\n",
            "456     ட\n",
            "457     ų\n",
            "458     î\n",
            "459     ɔ\n",
            "       ..\n",
            "1259    흐\n",
            "1260    당\n",
            "1261    ں\n",
            "1262    추\n",
            "1263    ษ\n",
            "Name: Unnamed: 0, Length: 809, dtype: object\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Make new Lines in The Raw data**"
      ],
      "metadata": {
        "id": "CGaFXxeONAH-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text = '''Insert Your Text Here'''\n",
        "\n",
        "text = text.split(\"।\")\n",
        "\n",
        "new_text = \"\"\n",
        "for line in text:\n",
        "    new_text += line.strip() + \"।\\n\"\n",
        "\n",
        "print(new_text)\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data with New Lines/General.txt\", \"a\", encoding=\"utf-8\") as output_file:\n",
        "  output_file.write(new_text)\n",
        "\n"
      ],
      "metadata": {
        "id": "OMfEwaBcNAqV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Count Lines that can be removed in **Bengali Text(General)**"
      ],
      "metadata": {
        "id": "9RY1-vF5VaEc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#  Read the Bengali text file\n",
        "with open('/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data with New Lines/General.txt', 'r', encoding='utf-8') as f:\n",
        "    text = f.read()\n",
        "\n",
        "#  Convert the text into a set of unique characters\n",
        "text_chars = set(text)\n",
        "\n",
        "#  Read the list of characters to check\n",
        "check_chars = row_data\n",
        "\n",
        "#  Find the intersection of the two sets\n",
        "common_chars = text_chars.intersection(set(check_chars))\n",
        "\n",
        "#  Split the text into lines and iterate through each line\n",
        "lines = text.split('\\n')\n",
        "removed_count = 0\n",
        "for i, line in enumerate(lines):\n",
        "    #  Check if any of the characters from the intersection set are present in the line\n",
        "    if any(char in common_chars for char in line):\n",
        "        lines[i] = ''\n",
        "        removed_count += 1\n",
        "\n",
        "print(f\"Total lines removed: {removed_count}\")\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Remove Lines /Count Removed Lines/General.txt\", \"w\") as write_file:\n",
        "  write_file.write(f\"Removed {removed_count} lines from the file.\")\n",
        "\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Text After Removing/General.txt', 'w', encoding='utf-8') as f:\n",
        "    for line in lines:\n",
        "        if line:\n",
        "            f.write(line + '\\n')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JxDT6Bu9OaU8",
        "outputId": "1664d365-1323-4d50-c351-ce2f473321b7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total lines removed: 1453\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Count The total Removable Lines(English Sentences) and save after Removal**"
      ],
      "metadata": {
        "id": "vPV4x4aoBfjy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "bengali_pattern = re.compile('[\\u0980-\\u09FF]+')\n",
        "\n",
        "\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Text After Removing/General.txt', 'r', encoding='utf-8') as f:\n",
        "    content = f.readlines()\n",
        "\n",
        "# remove lines that contain only English words\n",
        "filtered_content = [line for line in content if not all(re.search(r'\\w', word) and not bengali_pattern.search(word) for word in line.split())]\n",
        "\n",
        "\n",
        "num_removable_lines = len(content) - len(filtered_content)\n",
        "print(f\"Removable {num_removable_lines} English lines from the file.\")\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Remove Lines /Count Removable English Line/General.txt\", \"w\") as write_file:\n",
        "  write_file.write(f\"Removable {num_removable_lines} English lines from the file.\")\n",
        "\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Text without English Lines/General.txt', 'w', encoding='utf-8') as f:\n",
        "    f.writelines(filtered_content)\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3l5cxI80BneS",
        "outputId": "7ae49f46-ee9b-4294-be48-290b76e32274"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Removable 45377 English lines from the file.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Count No of Lines mixed with english and Bengali words**"
      ],
      "metadata": {
        "id": "-kYBkca_7T5k"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n",
        "import re\n",
        "\n",
        "input_file = '/content/drive/MyDrive/web_scraper/Remove Lines /English Line Manual removing/General.txt'\n",
        "output_file = '/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Bengali Lines/output_General.txt'\n",
        "removed_lines = []\n",
        "\n",
        "\n",
        "with open(input_file, 'r', encoding='utf-8') as f_in, open(output_file, 'w', encoding='utf-8') as f_out:\n",
        "    for line in f_in:\n",
        "        # Check if the line contains both English and Bengali words\n",
        "        if re.search(r'[a-zA-Z]+', line) and re.search(r'[\\u0980-\\u09FF]+', line):\n",
        "            removed_lines.append(line)\n",
        "        else:\n",
        "            f_out.write(line)\n",
        "\n",
        "# Count the number of removed lines\n",
        "num_removed_lines = len(removed_lines)\n",
        "print(f'Number of removed lines: {num_removed_lines}')\n",
        "\n",
        "# Write the removed lines to a separate file\n",
        "removed_file = '/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Removed English and Bengali Lines/removed_lines_General.txt'\n",
        "with open(removed_file, 'w', encoding='utf-8') as f_removed:\n",
        "    f_removed.writelines(removed_lines)\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TZKKzkLj7ULw",
        "outputId": "ddc5d1f1-676a-43f3-d504-6ed0a7426961"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of removed lines: 11543\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "char_list=['👉','↑',' »','•',''] \n",
        "dict_char={}\n",
        "removed_char_lines=[]\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Bengali Lines/output_General.txt','r',encoding='utf-8') as file_input, open('/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Removed English and Bengali Lines,', 'w', encoding='utf-8') as file_out :\n",
        "  for line in file_input:\n",
        "    if char_list in line :\n",
        "      dict_char[char_list]=count\n",
        " \n",
        "\n",
        "    else:\n",
        "      file_output.write(line)\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Bengali Lines/output.txt','w',encoding='utf-8') as file_write:\n",
        "  file_write.writelines(removed_char_lines)\n",
        "\n"
      ],
      "metadata": {
        "id": "0ybeu-VKAOiO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Remove extra space and lines having word count< 5 (GENERAL)**"
      ],
      "metadata": {
        "id": "Vea-RaTRUhHB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Bengali Lines/output_General.txt', 'r') as input_file:\n",
        "    lines = input_file.readlines()\n",
        "\n",
        "output_lines = []\n",
        "for line in lines:\n",
        "    # Remove extra spaces\n",
        "    cleaned_line = ' '.join(line.strip().split())\n",
        "    #  word count < 5\n",
        "    if len(cleaned_line.split()) >= 5:\n",
        "        output_lines.append(cleaned_line)\n",
        "\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Removal after Less than five Words/General.txt', 'w') as output_file:\n",
        "    output_file.write('\\n'.join(output_lines))"
      ],
      "metadata": {
        "id": "DJtX0bvS-7cZ"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Read The Excel file of Character Frequency (Agriculture)**"
      ],
      "metadata": {
        "id": "O32o_GPilyWI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "SHEET_ID = '1Kwhj8wjzZDIsREq0NXYcbQ1EpgOwPcUD-_rtKzGeugk'\n",
        "SHEET_NAME = 'scrape_data'\n",
        "url = f'https://docs.google.com/spreadsheets/d/{SHEET_ID}/gviz/tq?tqx=out:csv&sheet={SHEET_NAME}'\n",
        "df = pd.read_csv(url)\n",
        "print(df.head())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kNsqP6oWl1Ob",
        "outputId": "9b99bcec-308d-407f-f481-35d4dd6fe898"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Unnamed: 0  1353668\n",
            "0          া   621095\n",
            "1          র   526306\n",
            "2          ে   448130\n",
            "3          ্   412542\n",
            "4          ি   345158\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Converting the Agriculture Character Column in list**"
      ],
      "metadata": {
        "id": "ezG9fAL7lv8Q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "new_column=df.iloc[:,0]\n",
        "data_list_agriculture = new_column.values.tolist()\n",
        "\n",
        "print(data_list_agriculture)\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Excel Sheet to List Format/Agriculture Character List.txt\", \"a\") as write_file:\n",
        "   write_file.write(str(data_list_agriculture))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5nCEI67Xlwhw",
        "outputId": "17f34a27-51f0-4e67-8c41-bdc485df6192"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['া', 'র', 'ে', '্', 'ি', 'ক', 'ন', 'ব', 'ত', 'য', 'প', 'স', 'ম', 'ল', 'দ', 'ু', 'ট', 'জ', 'হ', '়', 'গ', '।', 'ো', 'শ', 'এ', 'ষ', 'চ', 'e', 'য়', 'ী', 'ভ', 'ধ', 'থ', 'ছ', 'ই', 'a', 'আ', 'ও', 'ং', 'ণ', nan, 't', 'অ', 'উ', 'o', 'i', 'খ', 'r', 'n', 'ড', 's', 'ফ', 'ৃ', '0', 'l', '1', 'd', '2', 'ূ', 'c', 'h', 'u', 'm', 'ঁ', 'p', 'g', '5', '3', 'ঠ', 'ড়', 'f', 'ৈ', 'ৎ', '4', 'b', 'y', 'ঙ', 'w', 'ঞ', '9', '6', 'ঘ', '8', '7', 'v', 'ৌ', 'ঝ', 'k', 'A', 'S', 'C', 'P', 'T', 'ঃ', '’', 'B', 'I', 'M', 'N', 'L', '‘', 'G', 'D', 'E', 'R', 'F', 'H', 'O', 'ঢ', '৷', '–', 'x', 'W', 'U', 'z', 'J', 'j', 'Y', 'ঋ', 'K', 'q', 'V', '•', '”', 'ঐ', '©', '—', 'Z', '“', 'Q', 'ঔ', '●', '·', 'X', '°', '»', '⇣', '…', 'ঈ', '×', 'ৰ', 'ঢ়', '☞', 'ा', '\\u200d', 'ঊ', 'क', 'к', 'े', 'र', '्', '\\u200b', 'ि', '\\xad', 'स', '→', 'с', 'न', 'а', '👉', nan, 'и', 'ी', 'ह', 'ो', 'ৗ', '″', 'ا', '‚', 'р', 'ə', 'त', 'Δ', 'ˈ', 'ल', 'ं', 'व', '⁄', 'द', 'य', '®', 'ر', 'č', 'प', '₹', 'е', 'म', 'י', 'ᵒ', 'ट', '♦', 'ū', 'о', 'æ', 'ம', 'α', 'ி', 'ు', 'л', 'ó', '⇒', '−', 'ü', 'ė', 'н', 'œ', '§', 'λ', 'ु', 'β', 'າ', 'á', '£', '❌', '்', '➖', 'ग', 'ل', 'ज', '¤', 'ɪ', 'ː', 'ழ', 'ž', '™', 'ي', 'µ', 'ب', 'ै', 'Ð', 'ä', '\\ue603', '«', 'Ñ', '中', 'ե', 'μ', 'º', '文', 'ā', 'ા', '\\x81', 'у', 'த', 'š', 'à', 'ʊ', 'ý', '√', 'ड', 'κ', 'ç', 'С', '̃', 'ن', 'ನ', 'و', 'й', 'ש', 'గ', '↑', 'ల', 'ز', 'ι', '\\u200e', 'ν', '´', 'ब', 'च', 'س', 'д', 'ર', 'ی', '‧', '€', '🙂', 'ь', 'फ', '॥', 'ف', 'త', 'د', '\\x90', 'г', '್', '►', 'જ', 'ત', '어', 'å', 'ख', 'Р', '국', 'ె', 'भ', 'ع', 'ৱ', 'â', '²', '―', 'ગ', 'ñ', 'ô', 'я', 'з', '한', 'Б', 'ท', 'ಸ', 'ா', 'ò', '日', 'ệ', '¯', 'ة', 'ঌ', 'ਰ', 'η', 'б', 'ऑ', 'க', '本', 'ї', 'У', 'ી', '\\ue604', 'ê', 'ά', 'ெ', 'ુ', '¡', 'ष', 'რ', 'ร', '\\ufeff', 'പ', 'ა', 'ย', '📖', 'ი', 'յ', '⏩', 'ಗ', 'आ', 'უ', 'і', 'თ', 'ր', 'ա', 'ų', 'ն', 'ಪ', 'п', 'Ε', 'ö', 'ไ', '¦', 'é', 'ქ', 'ლ', 'ಡ', 'ວ', 'ॉ', 'ែ', 'ִ', 'ר', 'ສ', 'ב', 'ɑ', '繁', 'м', 'រ', 'ø', 'ữ', 'ू', 'ה', 'า', '৳', 'ʼ', '្', '体', 'ລ', 'ד', '體', 'ο', 'ј', 'פ', 'ພ', 'ю', 'ע', 'ת', 'ш', 'ध', '简', 'հ', 'ρ', 'श', 'ម', 'ខ', 'ъ', 'ᴀ', 'δ', 'ಕ', 'थ', 'ɴ', '⃣', 'ത', 'ς', '്', 'ಿ', 'ಾ', 'ு', '▼', '‒', 'ँ', '\\uf005', 'ه', 'ਾ', 'इ', 'ό', 'ō', 'ج', '○', 'ೆ', '℃', 'ت', 'บ', 'ύ', 'ക', 'ി', 'ค', '：', 'đ', '\\x8f', 'ậ', 'М', 'ా', 'ë', 'अ', 'ാ', '油', 'ன', 'ī', '˚', 'ப', '′', 'ല', 'ச', 'മ', 'ି', 'ण', '무', '\\ue620', 'ശ', 'ळ', '\\ue6d5', '☎', 'ー', 'ǐ', 'ം', '😉', '\\x88', 'ß', 'ɡ', 'ᴄ', '‡', '़', 'т', 'π', 'ɛ', 'ʟ', 'ᴇ', '語', 'อ', 'స', 'Ø', 'ă', 'ੀ', 'ठ', 'ை', 'в', '豉', 'ん', 'ч', 'ï', '➡', '茶', 'ல', 'ั', 'ɒ', 'ള', 'ē', 'ư', 'τ', 'ர', '№', 'ł', '¾', 'ੂ', 'ɢ', 'Т', '\\ue618', '～', 'ˌ', '›', 'ข', '‹', 'ಜ', 'ੱ', 'ച', '☐', 'औ', '러', 'ế', 'ರ', 'ì', 'ಬ', 'ਬ', 'Í', 'ᴅ', 'ᴏ', '¿', 'ɜ', 'ʀ', 'ɬ', 'ಳ', 'യ', 'ओ', 'қ', 'ƒ', 'í', 'ك', 'ᴛ', 'ள', 'ම', 'ା', 'ω', 'ു', 'ِ', 'ơ', 'Қ', 'ε', '️', 'ತ', 'ര', 'ಲ', 'َ', '醤', '一', 'è', 'ක', '\\ue821', 'අ', '조', 'उ', 'ś', 'แ', '平', '🌹', 'ӣ', 'జ', '嫘', 'ಯ', 'ದ', 'ജ', 'మ', 'Ч', '\\ue6eb', 'ర', 'ਲ', 'પ', 'な', 'ର', '↓', 'હ', 'Հ', nan, 'ᴋ', 'ᴡ', '等', 'ҡ', 'ક', '්', 'ҷ', 'Ⓒ', '\\ue64a', '🔰', 'വ', 'ี', '★', 'ి', 'ズ', 'ú', 'ಹ', 'ද', '๋', '花', 'ು', '造', 'ढ', '↴', '½', 'ロ', 'छ', nan, 'ʙ', 'ତ', 'ప', 'Þ', '\\ue84e', '드', 'ෝ', 'ξ', '🌲', '🖋', 'െ', 'Э', 'ί', '\\ue60d', '🇧', 'ා', 'ೊ', 'ස', 'ʃ', 'ц', '⇓', 'ǔ', 'ἔ', 'ଶ', 'х', 'أ', 'ਦ', '\\u09e4', 'į', 'ط', '潢', 'さ', 'Ü', '🍀', '營', 'ර', '\\u202f', '¹', '祖', '材', 'ଳ', '朱', 'झ', 'ଜ', 'ි', '🌵', '부', 'ೋ', 'ਕ', '\\ue694', 'σ', '莉', '粵', 'ਪ', 'ș', 'ਿ', '外', 'ラ', 'ප', 'ح', '청', '¶', 'ਗ', 'ਹ', 'ᴜ', '\\ue650', 'ੰ', '辊', '蘗', '\\ue608', '式', 'ૂ', '̧', '롤', '《', 'ک', 'શ', 'ต', '滑', 'ぴ', 'ὠ', 'ถ', 'ವ', '\\ue6bb', 'ृ', 'ὰ', 'ഴ', 'ʏ', 'ӑ', '₄', '》', 'É', '\\ue617', '선', '⏬', 'χ', 'ග', 'વ', 'ス', 'િ', '槿', 'ଉ', '\\ue9d2', 'γ', '🌻', 'સ', 'ච', 'ị', 'ौ', '法', 'ણ', '±', 'ム', 'ய', '운', 'ಮ', 'φ', 'ۆ', '🇬', 'Н', 'υ', '͡', 'ए', 'உ', 'ਸ', '茉', 'น', '🌳', 'வ', 'ପ', 'ං']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Comparing the Unicode and Character Frequency List(Agriculture)**"
      ],
      "metadata": {
        "id": "-KXT1VNjlj1M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# data_list_general\n",
        "# unicode_list\n",
        "\n",
        "'''\n",
        "(ABSENT CHARACTER)\n",
        "(PRESENT CHARACTERS)\n",
        "absent characters define the characters of unicode_list not present in the Agriculture Character Frequency list\n",
        "'''\n",
        "absent_characters_agriculture=[]\n",
        "present_characters_agriculture={}\n",
        "for i, element in enumerate(data_list_agriculture):\n",
        "        if element in unicode_list:\n",
        "            present_characters_agriculture[element] = i\n",
        "        else:\n",
        "            absent_characters_agriculture.append(element)\n",
        "\n",
        "      \n",
        "print(\"Absent Characters  : \",absent_characters_agriculture)\n",
        "print(\"Present Characters  : \",present_characters_agriculture)        \n",
        "with open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Absent Characters/Agriculture Absent Character.txt\", \"w\") as write_file:\n",
        "   write_file.write(str(absent_characters_agriculture))\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Present Characters/Agriculture Present-Character Index.txt\", \"w\") as write_file:\n",
        "   write_file.write(str(present_characters_agriculture))\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c3SzF3ixloNS",
        "outputId": "30b2cd82-915f-4cba-8e21-4b0b1407ea14"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Absent Characters  :  ['।', 'e', 'a', nan, 't', 'o', 'i', 'r', 'n', 's', '0', 'l', '1', 'd', '2', 'c', 'h', 'u', 'm', 'p', 'g', '5', '3', 'f', '4', 'b', 'y', 'w', '9', '6', '8', '7', 'v', 'k', 'A', 'S', 'C', 'P', 'T', '’', 'B', 'I', 'M', 'N', 'L', '‘', 'G', 'D', 'E', 'R', 'F', 'H', 'O', '–', 'x', 'W', 'U', 'z', 'J', 'j', 'Y', 'K', 'q', 'V', '•', '”', '©', '—', 'Z', '“', 'Q', '●', '·', 'X', '°', '»', '⇣', '…', '×', '☞', 'ा', '\\u200d', 'क', 'к', 'े', 'र', '्', '\\u200b', 'ि', '\\xad', 'स', '→', 'с', 'न', 'а', '👉', nan, 'и', 'ी', 'ह', 'ो', '″', 'ا', '‚', 'р', 'ə', 'त', 'Δ', 'ˈ', 'ल', 'ं', 'व', '⁄', 'द', 'य', '®', 'ر', 'č', 'प', '₹', 'е', 'म', 'י', 'ᵒ', 'ट', '♦', 'ū', 'о', 'æ', 'ம', 'α', 'ி', 'ు', 'л', 'ó', '⇒', '−', 'ü', 'ė', 'н', 'œ', '§', 'λ', 'ु', 'β', 'າ', 'á', '£', '❌', '்', '➖', 'ग', 'ل', 'ज', '¤', 'ɪ', 'ː', 'ழ', 'ž', '™', 'ي', 'µ', 'ب', 'ै', 'Ð', 'ä', '\\ue603', '«', 'Ñ', '中', 'ե', 'μ', 'º', '文', 'ā', 'ા', '\\x81', 'у', 'த', 'š', 'à', 'ʊ', 'ý', '√', 'ड', 'κ', 'ç', 'С', '̃', 'ن', 'ನ', 'و', 'й', 'ש', 'గ', '↑', 'ల', 'ز', 'ι', '\\u200e', 'ν', '´', 'ब', 'च', 'س', 'д', 'ર', 'ی', '‧', '€', '🙂', 'ь', 'फ', '॥', 'ف', 'త', 'د', '\\x90', 'г', '್', '►', 'જ', 'ત', '어', 'å', 'ख', 'Р', '국', 'ె', 'भ', 'ع', 'â', '²', '―', 'ગ', 'ñ', 'ô', 'я', 'з', '한', 'Б', 'ท', 'ಸ', 'ா', 'ò', '日', 'ệ', '¯', 'ة', 'ਰ', 'η', 'б', 'ऑ', 'க', '本', 'ї', 'У', 'ી', '\\ue604', 'ê', 'ά', 'ெ', 'ુ', '¡', 'ष', 'რ', 'ร', '\\ufeff', 'പ', 'ა', 'ย', '📖', 'ი', 'յ', '⏩', 'ಗ', 'आ', 'უ', 'і', 'თ', 'ր', 'ա', 'ų', 'ն', 'ಪ', 'п', 'Ε', 'ö', 'ไ', '¦', 'é', 'ქ', 'ლ', 'ಡ', 'ວ', 'ॉ', 'ែ', 'ִ', 'ר', 'ສ', 'ב', 'ɑ', '繁', 'м', 'រ', 'ø', 'ữ', 'ू', 'ה', 'า', 'ʼ', '្', '体', 'ລ', 'ד', '體', 'ο', 'ј', 'פ', 'ພ', 'ю', 'ע', 'ת', 'ш', 'ध', '简', 'հ', 'ρ', 'श', 'ម', 'ខ', 'ъ', 'ᴀ', 'δ', 'ಕ', 'थ', 'ɴ', '⃣', 'ത', 'ς', '്', 'ಿ', 'ಾ', 'ு', '▼', '‒', 'ँ', '\\uf005', 'ه', 'ਾ', 'इ', 'ό', 'ō', 'ج', '○', 'ೆ', '℃', 'ت', 'บ', 'ύ', 'ക', 'ി', 'ค', '：', 'đ', '\\x8f', 'ậ', 'М', 'ా', 'ë', 'अ', 'ാ', '油', 'ன', 'ī', '˚', 'ப', '′', 'ല', 'ச', 'മ', 'ି', 'ण', '무', '\\ue620', 'ശ', 'ळ', '\\ue6d5', '☎', 'ー', 'ǐ', 'ം', '😉', '\\x88', 'ß', 'ɡ', 'ᴄ', '‡', '़', 'т', 'π', 'ɛ', 'ʟ', 'ᴇ', '語', 'อ', 'స', 'Ø', 'ă', 'ੀ', 'ठ', 'ை', 'в', '豉', 'ん', 'ч', 'ï', '➡', '茶', 'ல', 'ั', 'ɒ', 'ള', 'ē', 'ư', 'τ', 'ர', '№', 'ł', '¾', 'ੂ', 'ɢ', 'Т', '\\ue618', '～', 'ˌ', '›', 'ข', '‹', 'ಜ', 'ੱ', 'ച', '☐', 'औ', '러', 'ế', 'ರ', 'ì', 'ಬ', 'ਬ', 'Í', 'ᴅ', 'ᴏ', '¿', 'ɜ', 'ʀ', 'ɬ', 'ಳ', 'യ', 'ओ', 'қ', 'ƒ', 'í', 'ك', 'ᴛ', 'ள', 'ම', 'ା', 'ω', 'ു', 'ِ', 'ơ', 'Қ', 'ε', '️', 'ತ', 'ര', 'ಲ', 'َ', '醤', '一', 'è', 'ක', '\\ue821', 'අ', '조', 'उ', 'ś', 'แ', '平', '🌹', 'ӣ', 'జ', '嫘', 'ಯ', 'ದ', 'ജ', 'మ', 'Ч', '\\ue6eb', 'ర', 'ਲ', 'પ', 'な', 'ର', '↓', 'હ', 'Հ', nan, 'ᴋ', 'ᴡ', '等', 'ҡ', 'ક', '්', 'ҷ', 'Ⓒ', '\\ue64a', '🔰', 'വ', 'ี', '★', 'ి', 'ズ', 'ú', 'ಹ', 'ද', '๋', '花', 'ು', '造', 'ढ', '↴', '½', 'ロ', 'छ', nan, 'ʙ', 'ତ', 'ప', 'Þ', '\\ue84e', '드', 'ෝ', 'ξ', '🌲', '🖋', 'െ', 'Э', 'ί', '\\ue60d', '🇧', 'ා', 'ೊ', 'ස', 'ʃ', 'ц', '⇓', 'ǔ', 'ἔ', 'ଶ', 'х', 'أ', 'ਦ', '\\u09e4', 'į', 'ط', '潢', 'さ', 'Ü', '🍀', '營', 'ර', '\\u202f', '¹', '祖', '材', 'ଳ', '朱', 'झ', 'ଜ', 'ි', '🌵', '부', 'ೋ', 'ਕ', '\\ue694', 'σ', '莉', '粵', 'ਪ', 'ș', 'ਿ', '外', 'ラ', 'ප', 'ح', '청', '¶', 'ਗ', 'ਹ', 'ᴜ', '\\ue650', 'ੰ', '辊', '蘗', '\\ue608', '式', 'ૂ', '̧', '롤', '《', 'ک', 'શ', 'ต', '滑', 'ぴ', 'ὠ', 'ถ', 'ವ', '\\ue6bb', 'ृ', 'ὰ', 'ഴ', 'ʏ', 'ӑ', '₄', '》', 'É', '\\ue617', '선', '⏬', 'χ', 'ග', 'વ', 'ス', 'િ', '槿', 'ଉ', '\\ue9d2', 'γ', '🌻', 'સ', 'ච', 'ị', 'ौ', '法', 'ણ', '±', 'ム', 'ய', '운', 'ಮ', 'φ', 'ۆ', '🇬', 'Н', 'υ', '͡', 'ए', 'உ', 'ਸ', '茉', 'น', '🌳', 'வ', 'ପ', 'ං']\n",
            "Present Characters  :  {'া': 0, 'র': 1, 'ে': 2, '্': 3, 'ি': 4, 'ক': 5, 'ন': 6, 'ব': 7, 'ত': 8, 'য': 9, 'প': 10, 'স': 11, 'ম': 12, 'ল': 13, 'দ': 14, 'ু': 15, 'ট': 16, 'জ': 17, 'হ': 18, '়': 19, 'গ': 20, 'ো': 22, 'শ': 23, 'এ': 24, 'ষ': 25, 'চ': 26, 'য়': 28, 'ী': 29, 'ভ': 30, 'ধ': 31, 'থ': 32, 'ছ': 33, 'ই': 34, 'আ': 36, 'ও': 37, 'ং': 38, 'ণ': 39, 'অ': 42, 'উ': 43, 'খ': 46, 'ড': 49, 'ফ': 51, 'ৃ': 52, 'ূ': 58, 'ঁ': 63, 'ঠ': 68, 'ড়': 69, 'ৈ': 71, 'ৎ': 72, 'ঙ': 76, 'ঞ': 78, 'ঘ': 81, 'ৌ': 85, 'ঝ': 86, 'ঃ': 93, 'ঢ': 108, '৷': 109, 'ঋ': 118, 'ঐ': 124, 'ঔ': 130, 'ঈ': 138, 'ৰ': 140, 'ঢ়': 141, 'ঊ': 145, 'ৗ': 165, 'ৱ': 296, 'ঌ': 315, '৳': 377}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Extract Character List from the Google Sheet (Agriculture) that need to be removed**"
      ],
      "metadata": {
        "id": "Yfart6ylXtP_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "row_data=df.iloc[-546:, 0]\n",
        "row_data_final=row_data.values.tolist()\n",
        "print(row_data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xmgmsIS5XkbG",
        "outputId": "71ee8d63-a3ad-4eaa-8d26-a1a3da70c127"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "190    ♦\n",
            "191    ū\n",
            "192    о\n",
            "193    æ\n",
            "194    ம\n",
            "      ..\n",
            "731    น\n",
            "732    🌳\n",
            "733    வ\n",
            "734    ପ\n",
            "735    ං\n",
            "Name: Unnamed: 0, Length: 546, dtype: object\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Count Lines that can be removed in **Bengali Text(Agriculture)**"
      ],
      "metadata": {
        "id": "NgdTj5RYysv4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#  Read the Bengali text file\n",
        "with open('/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data with New Lines/Agriculture.txt', 'r', encoding='utf-8') as f:\n",
        "    text = f.read()\n",
        "\n",
        "#  Convert the text into a set of unique characters\n",
        "text_chars = set(text)\n",
        "\n",
        "#  Read the list of characters to check\n",
        "check_chars = row_data\n",
        "\n",
        "#  Find the intersection of the two sets\n",
        "common_chars = text_chars.intersection(set(check_chars))\n",
        "\n",
        "#  Split the text into lines and iterate through each line\n",
        "lines = text.split('\\n')\n",
        "removed_count = 0\n",
        "for i, line in enumerate(lines):\n",
        "    #  Check if any of the characters from the intersection set are present in the line\n",
        "    if any(char in common_chars for char in line):\n",
        "        lines[i] = ''\n",
        "        removed_count += 1\n",
        "\n",
        "print(f\"Total lines removed: {removed_count}\")\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Remove Lines /Count Removed Lines/Agriculture.txt\", \"w\") as write_file:\n",
        "  write_file.write(f\"Removed {removed_count} lines from the file.\")\n",
        "\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Text After Removing/Agriculture.txt', 'w', encoding='utf-8') as f:\n",
        "    for line in lines:\n",
        "        if line:\n",
        "            f.write(line + '\\n')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CcUw-WJCywZX",
        "outputId": "1115ea5e-80f2-4d37-d160-772fdf25bcf0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total lines removed: 858\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Count The total Removable Lines(English Sentences) and save after Removal -- Agriculture**"
      ],
      "metadata": {
        "id": "j-ShrzfZ3pXb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "bengali_pattern = re.compile('[\\u0980-\\u09FF]+')\n",
        "\n",
        "\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Text After Removing/Agriculture.txt', 'r', encoding='utf-8') as f:\n",
        "    content = f.readlines()\n",
        "\n",
        "# remove lines that contain only English words\n",
        "filtered_content = [line for line in content if not all(re.search(r'\\w', word) and not bengali_pattern.search(word) for word in line.split())]\n",
        "\n",
        "\n",
        "num_removable_lines = len(content) - len(filtered_content)\n",
        "print(f\"Removable {num_removable_lines} English lines from the file.\")\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Remove Lines /Count Removable English Line/Agriculture.txt\", \"w\") as write_file:\n",
        "  write_file.write(f\"Removable {num_removable_lines} English lines from the file.\")\n",
        "\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Text without English Lines/Agriculture.txt', 'w', encoding='utf-8') as f:\n",
        "    f.writelines(filtered_content)\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "As6Bxw3S3ptA",
        "outputId": "adea495f-c7e9-4a57-d26d-fc4144c13243"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Removable 6503 English lines from the file.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Count No of Lines mixed with english and Bengali words**"
      ],
      "metadata": {
        "id": "yVRtJAzKzjAA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import re\n",
        "\n",
        "input_file = '/content/drive/MyDrive/web_scraper/Remove Lines /English Line Manual removing/Agriculture.txt'\n",
        "output_file = '/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Bengali Lines/output_Agriculture.txt'\n",
        "removed_lines = []\n",
        "\n",
        "\n",
        "with open(input_file, 'r', encoding='utf-8') as f_in, open(output_file, 'w', encoding='utf-8') as f_out:\n",
        "    for line in f_in:\n",
        "        # Check if the line contains both English and Bengali words\n",
        "        if re.search(r'[a-zA-Z]+', line) and re.search(r'[\\u0980-\\u09FF]+', line):\n",
        "            removed_lines.append(line)\n",
        "        else:\n",
        "            f_out.write(line)\n",
        "\n",
        "# Count the number of removed lines\n",
        "num_removed_lines = len(removed_lines)\n",
        "print(f'Number of removed lines: {num_removed_lines}')\n",
        "\n",
        "# Write the removed lines to a separate file\n",
        "removed_file = '/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Removed English and Bengali Lines/removed_lines_Agriculture.txt'\n",
        "with open(removed_file, 'w', encoding='utf-8') as f_removed:\n",
        "    f_removed.writelines(removed_lines)\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ofCW9uBTzlnf",
        "outputId": "d8e8a5ba-393e-4a91-ac1f-d53986cbb691"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of removed lines: 5954\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Remove extra space and lines having word count< 5 **(Agriculture)**"
      ],
      "metadata": {
        "id": "grmiMNRRhitF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Bengali Lines/output_Agriculture.txt', 'r') as input_file:\n",
        "    lines = input_file.readlines()\n",
        "\n",
        "output_lines = []\n",
        "for line in lines:\n",
        "    # Remove extra spaces\n",
        "    cleaned_line = ' '.join(line.strip().split())\n",
        "    #  word count < 5\n",
        "    if len(cleaned_line.split()) >= 5:\n",
        "        output_lines.append(cleaned_line)\n",
        "\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Removal after Less than five Words/Agriculture.txt', 'w') as output_file:\n",
        "    output_file.write('\\n'.join(output_lines))"
      ],
      "metadata": {
        "id": "0dPuY9nuhkaH"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Read The Excel file of Character Frequency (Finance)**\n",
        "\n",
        "---\n",
        "\n"
      ],
      "metadata": {
        "id": "8L6UjlI8lYxg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "SHEET_ID = '16ApIp9ocb4gJBsxfAtC0RRnsHWgX4yfOE5YNOdDMCPk'\n",
        "SHEET_NAME = 'scrape_data'\n",
        "url = f'https://docs.google.com/spreadsheets/d/{SHEET_ID}/gviz/tq?tqx=out:csv&sheet={SHEET_NAME}'\n",
        "df = pd.read_csv(url)\n",
        "print(df.head())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Lz5XYeYalZVN",
        "outputId": "464aa6eb-c912-4ebc-d398-0127ffc894f3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Unnamed: 0  1097424\n",
            "0          া   439387\n",
            "1          র   404193\n",
            "2          ে   354776\n",
            "3          ্   346469\n",
            "4          ি   267117\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Converting the Finance Character Column in list**"
      ],
      "metadata": {
        "id": "eIwFiT54lRBl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "new_column=df.iloc[:,0]\n",
        "data_list_finance = new_column.values.tolist()\n",
        "\n",
        "print(data_list_finance)\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Excel Sheet to List Format/Finanace Character List.txt\", \"w\") as write_file:\n",
        "   write_file.write(str(data_list_finance))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zMahsRinlULW",
        "outputId": "fdf56094-ed61-4546-e37d-2afedbe32188"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['া', 'র', 'ে', '্', 'ি', 'ক', 'ন', 'ব', 'ত', 'য', 'স', 'প', 'ম', 'ল', 'দ', 'ট', 'ু', 'য়', 'হ', 'এ', 'জ', 'ো', 'শ', '।', 'e', 'গ', 'ই', '়', 'ী', 't', 'a', 'থ', 'ষ', 'n', 'ছ', 'আ', 'অ', 'i', 'o', 'ং', nan, 'ধ', 'চ', 'ভ', 'ও', 'ণ', 'r', 'ড', 's', '0', '2', '1', 'খ', 'উ', 'l', 'd', 'c', 'ফ', 'u', 'h', 'ূ', 'm', 'p', 'g', 'ৃ', 'ঙ', '5', 'f', '3', 'y', 'b', 'ঁ', 'ঠ', '4', '9', 'v', 'w', '8', '6', 'ড়', 'S', '7', 'A', 'ঋ', 'ঘ', 'I', 'k', 'ঞ', 'ৈ', 'C', 'B', 'ৎ', 'T', 'P', '’', 'R', 'M', 'ঝ', 'F', 'D', '‘', 'G', 'L', 'ৌ', 'E', 'N', 'x', 'O', 'Y', 'ঃ', 'H', '₹', 'q', 'U', '–', 'z', '©', '৷', 'W', 'ঢ', 'V', '·', 'j', 'ৰ', 'Z', '•', '“', '—', 'K', '”', 'J', '↑', 'ঐ', '\\x80', 'Q', 'Â', 'ঊ', '»', 'â', '↓', '☆', 'ঈ', 'X', 'ঔ', 'ৱ', '…', 'а', 'ঢ়', '👉', '›', 'ৗ', 'к', 'с', 'р', '‚', 'ı', 'Δ', '¤', 'ी', 'и', 'о', 'æ', '×', 'ç', 'л', 'ر', 'ś', '◾', 'स', 'ü', '्', 'н', 'ṃ', '£', 'ا', 'त', '⁄', 'у', 'ा', 'ल', 'ह', 'ि', '●', 'र', 'म', 'क', 'е', '☞', 'ե', 'ø', 'े', 'т', '♦', 'й', 'š', 'é', 'द', 'प', '¡', 'ğ', 'в', 'י', 'न', '®', 'Í', 'Ð', 'و', '語', 'د', 'г', 'Б', 'ं', 'à', 'ч', 'ы', '➡', 'რ', 'ی', '★', 'ა', 'ь', 'С', 'ր', 'ñ', 'ա', 'д', 'ŋ', 'ö', '§', 'ш', 'ъ', 'λ', 'ो', 'ु', 'ာ', 'ള', 'ò', 'ˈ', 'Р', '日', 'ം', 'м', 'Т', 'ი', 'ة', 'յ', 'भ', 'უ', 'ع', 'х', '→', '॥', 'ل', 'М', 'യ', '本', 'í', 'ف', 'У', 'ാ', 'ն', 'č', 'ä', 'ê', 'п', 'я', '文', 'ല', 'ლ', 'മ', 'م', 'ā', 'Ӏ', '어', '🚀', 'ɔ', 'မ', 'ា', '♥', 'å', 'ி', 'ي', 'Ñ', '−', 'ר', 'Հ', '்', 'ι', 'ख', '°', '국', 'ב', '中', 'ड', 'ệ', 'ம', 'ು', 'ు', 'κ', 'Ⱄ', 'आ', 'ب', 'Л', 'फ', 'ν', 'ế', 'ː', 'η', 'თ', 'ै', '౼', 'ನ', '💫', '‑', '़', 'ṇ', 'œ', '🐕', 'ɛ', 'ї', 'К', 'ë', 'ழ', 'த', 'Č', 'س', 'ી', 'ک', 'ृ', '️', 'Ε', 'ά', 'ج', 'з', 'ע', 'ת', 'ქ', '한', 'ت', 'è', 'Ꮳ', 'थ', 'જ', 'ó', 'ି', 'उ', 'ण', 'ત', 'ི', 'ӣ', 'ಡ', 'Ч', 'გ', 'Ա', 'ວ', 'ॉ', 'ص', 'ల', 'ែ', '଼', 'ә', 'ɪ', 'ြ', 'ɒ', 'ဘ', 'Ⰲ', 'ִ', 'ල', 'ز', 'ོ', '€', 'տ', 'ҡ', 'ҷ', '☎', 'Ⓒ', 'န', 'າ', 'ા', '\\ufeff', '言', '་', '√', 'ú', 'व', 'ɑ', 'А', '🏻', 'រ', '¯', 'ย', '📖', 'හ', 'ى', 'ॅ', 'á', 'Ꮃ', 'ଡ', 'ె', 'ښ', 'Ⰽ', 'ৠ', 'བ', 'ස', '်', 'ཡ', 'ц', 'ɡ', 'ւ', 'Ⰾ', 'է', 'ن', 'Ž', 'і', 'ė', 'б', 'သ', 'Ü', '²', 'ි', 'پ', 'ہ', 'ə', 'ગ', 'ļ', 'ಳ', '粵', 'ў', 'ą', 'ज', '្', 'მ', 'ស', 'ه', 'ų', 'త', 'Ⱐ', 'इ', 'ש', 'ລ', 'ד', '贛', 'î', 'ភ', 'ɩ', 'ଆ', 'గ', 'ِ', 'ô', 'ʻ', 'ù', 'ӑ', 'ō', 'ર', 'ढ़', 'Ⱏ', 'Ꭹ', 'ತ', 'ไ', 'χ', 'õ', 'ԥ', 'ག', 'Ⱁ', 'Ś', 'િ', 'Ⱑ', 'ད', 'ѣ', 'ẽ', 'ă', 'ट', '್', 'ଓ', 'հ', 'И', 'ۆ', 'ម', 'Ⱀ', 'Н', 'ठ', 'ខ', 'ท', 'մ', 'ę', 'Г', 'य', 'ů', 'ુ', 'ಕ', 'ං']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Comparing the Unicode and Character Frequency List(FINANCE)**"
      ],
      "metadata": {
        "id": "XTvpSTuXlLNr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "(ABSENT CHARACTER)\n",
        "(PRESENT CHARACTERS)\n",
        "absent characters define the characters of unicode_list not present in the Finance Character Frequency list\n",
        "'''\n",
        "absent_characters_finance=[]\n",
        "present_characters_finance={}\n",
        "for i, element in enumerate(data_list_finance):\n",
        "        if element in unicode_list:\n",
        "            present_characters_finance[element] = i\n",
        "        else:\n",
        "            absent_characters_finance.append(element)\n",
        "\n",
        "      \n",
        "print(\"Absent Characters  : \",absent_characters_finance)\n",
        "print(\"Present Characters  : \",present_characters_finance)        \n",
        "with open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Absent Characters/Finance Absent Character.txt\", \"a\") as write_file:\n",
        "   write_file.write(str(absent_characters_finance))\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Character Frequency/Present Characters/Finance Present-Character Index.txt\", \"a\") as write_file:\n",
        "   write_file.write(str(present_characters_finance))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sWwUvXxYlL6_",
        "outputId": "6fd8ac94-33f6-4925-e3cf-0ae3151d93d2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Absent Characters  :  ['।', 'e', 't', 'a', 'n', 'i', 'o', nan, 'r', 's', '0', '2', '1', 'l', 'd', 'c', 'u', 'h', 'm', 'p', 'g', '5', 'f', '3', 'y', 'b', '4', '9', 'v', 'w', '8', '6', 'S', '7', 'A', 'I', 'k', 'C', 'B', 'T', 'P', '’', 'R', 'M', 'F', 'D', '‘', 'G', 'L', 'E', 'N', 'x', 'O', 'Y', 'H', '₹', 'q', 'U', '–', 'z', '©', 'W', 'V', '·', 'j', 'Z', '•', '“', '—', 'K', '”', 'J', '↑', '\\x80', 'Q', 'Â', '»', 'â', '↓', '☆', 'X', '…', 'а', '👉', '›', 'к', 'с', 'р', '‚', 'ı', 'Δ', '¤', 'ी', 'и', 'о', 'æ', '×', 'ç', 'л', 'ر', 'ś', '◾', 'स', 'ü', '्', 'н', 'ṃ', '£', 'ا', 'त', '⁄', 'у', 'ा', 'ल', 'ह', 'ि', '●', 'र', 'म', 'क', 'е', '☞', 'ե', 'ø', 'े', 'т', '♦', 'й', 'š', 'é', 'द', 'प', '¡', 'ğ', 'в', 'י', 'न', '®', 'Í', 'Ð', 'و', '語', 'د', 'г', 'Б', 'ं', 'à', 'ч', 'ы', '➡', 'რ', 'ی', '★', 'ა', 'ь', 'С', 'ր', 'ñ', 'ա', 'д', 'ŋ', 'ö', '§', 'ш', 'ъ', 'λ', 'ो', 'ु', 'ာ', 'ള', 'ò', 'ˈ', 'Р', '日', 'ം', 'м', 'Т', 'ი', 'ة', 'յ', 'भ', 'უ', 'ع', 'х', '→', '॥', 'ل', 'М', 'യ', '本', 'í', 'ف', 'У', 'ാ', 'ն', 'č', 'ä', 'ê', 'п', 'я', '文', 'ല', 'ლ', 'മ', 'م', 'ā', 'Ӏ', '어', '🚀', 'ɔ', 'မ', 'ា', '♥', 'å', 'ி', 'ي', 'Ñ', '−', 'ר', 'Հ', '்', 'ι', 'ख', '°', '국', 'ב', '中', 'ड', 'ệ', 'ம', 'ು', 'ు', 'κ', 'Ⱄ', 'आ', 'ب', 'Л', 'फ', 'ν', 'ế', 'ː', 'η', 'თ', 'ै', '౼', 'ನ', '💫', '‑', '़', 'ṇ', 'œ', '🐕', 'ɛ', 'ї', 'К', 'ë', 'ழ', 'த', 'Č', 'س', 'ી', 'ک', 'ृ', '️', 'Ε', 'ά', 'ج', 'з', 'ע', 'ת', 'ქ', '한', 'ت', 'è', 'Ꮳ', 'थ', 'જ', 'ó', 'ି', 'उ', 'ण', 'ત', 'ི', 'ӣ', 'ಡ', 'Ч', 'გ', 'Ա', 'ວ', 'ॉ', 'ص', 'ల', 'ែ', '଼', 'ә', 'ɪ', 'ြ', 'ɒ', 'ဘ', 'Ⰲ', 'ִ', 'ල', 'ز', 'ོ', '€', 'տ', 'ҡ', 'ҷ', '☎', 'Ⓒ', 'န', 'າ', 'ા', '\\ufeff', '言', '་', '√', 'ú', 'व', 'ɑ', 'А', '🏻', 'រ', '¯', 'ย', '📖', 'හ', 'ى', 'ॅ', 'á', 'Ꮃ', 'ଡ', 'ె', 'ښ', 'Ⰽ', 'བ', 'ස', '်', 'ཡ', 'ц', 'ɡ', 'ւ', 'Ⰾ', 'է', 'ن', 'Ž', 'і', 'ė', 'б', 'သ', 'Ü', '²', 'ි', 'پ', 'ہ', 'ə', 'ગ', 'ļ', 'ಳ', '粵', 'ў', 'ą', 'ज', '្', 'მ', 'ស', 'ه', 'ų', 'త', 'Ⱐ', 'इ', 'ש', 'ລ', 'ד', '贛', 'î', 'ភ', 'ɩ', 'ଆ', 'గ', 'ِ', 'ô', 'ʻ', 'ù', 'ӑ', 'ō', 'ર', 'ढ़', 'Ⱏ', 'Ꭹ', 'ತ', 'ไ', 'χ', 'õ', 'ԥ', 'ག', 'Ⱁ', 'Ś', 'િ', 'Ⱑ', 'ད', 'ѣ', 'ẽ', 'ă', 'ट', '್', 'ଓ', 'հ', 'И', 'ۆ', 'ម', 'Ⱀ', 'Н', 'ठ', 'ខ', 'ท', 'մ', 'ę', 'Г', 'य', 'ů', 'ુ', 'ಕ', 'ං']\n",
            "Present Characters  :  {'া': 0, 'র': 1, 'ে': 2, '্': 3, 'ি': 4, 'ক': 5, 'ন': 6, 'ব': 7, 'ত': 8, 'য': 9, 'স': 10, 'প': 11, 'ম': 12, 'ল': 13, 'দ': 14, 'ট': 15, 'ু': 16, 'য়': 17, 'হ': 18, 'এ': 19, 'জ': 20, 'ো': 21, 'শ': 22, 'গ': 25, 'ই': 26, '়': 27, 'ী': 28, 'থ': 31, 'ষ': 32, 'ছ': 34, 'আ': 35, 'অ': 36, 'ং': 39, 'ধ': 41, 'চ': 42, 'ভ': 43, 'ও': 44, 'ণ': 45, 'ড': 47, 'খ': 52, 'উ': 53, 'ফ': 57, 'ূ': 60, 'ৃ': 64, 'ঙ': 65, 'ঁ': 71, 'ঠ': 72, 'ড়': 79, 'ঋ': 83, 'ঘ': 84, 'ঞ': 87, 'ৈ': 88, 'ৎ': 91, 'ঝ': 97, 'ৌ': 103, 'ঃ': 109, '৷': 117, 'ঢ': 119, 'ৰ': 123, 'ঐ': 132, 'ঊ': 136, 'ঈ': 141, 'ঔ': 143, 'ৱ': 144, 'ঢ়': 147, 'ৗ': 150, 'ৠ': 401}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Extract Character List from the Google Sheet (Agriculture) that need to be removed**"
      ],
      "metadata": {
        "id": "gzU_Buylh5u2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "row_data=df.iloc[-340:, 0]\n",
        "row_data_final=row_data.values.tolist()\n",
        "print(row_data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NuK4cN4Sh32k",
        "outputId": "e0c9ea0e-b1b5-4e8c-ce8e-17ed4379fea1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "151    к\n",
            "152    с\n",
            "153    р\n",
            "154    ‚\n",
            "155    ı\n",
            "      ..\n",
            "486    य\n",
            "487    ů\n",
            "488    ુ\n",
            "489    ಕ\n",
            "490    ං\n",
            "Name: Unnamed: 0, Length: 340, dtype: object\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**RemoveLines Frequency < 23 in Finance**"
      ],
      "metadata": {
        "id": "nLFSTf8lh6zY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#  Read the Bengali text file\n",
        "with open('/content/drive/MyDrive/web_scraper/Word Frequency/Raw Data with New Lines/Finance.txt', 'r', encoding='utf-8') as f:\n",
        "    text = f.read()\n",
        "\n",
        "#  Convert the text into a set of unique characters\n",
        "text_chars = set(text)\n",
        "\n",
        "#  Read the list of characters to check\n",
        "check_chars = row_data\n",
        "\n",
        "#  Find the intersection of the two sets\n",
        "common_chars = text_chars.intersection(set(check_chars))\n",
        "\n",
        "#  Split the text into lines and iterate through each line\n",
        "lines = text.split('\\n')\n",
        "removed_count = 0\n",
        "for i, line in enumerate(lines):\n",
        "    #  Check if any of the characters from the intersection set are present in the line\n",
        "    if any(char in common_chars for char in line):\n",
        "        lines[i] = ''\n",
        "        removed_count += 1\n",
        "\n",
        "print(f\"Total lines removed: {removed_count}\")\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Remove Lines /Count Removed Lines/Finance.txt\", \"w\") as write_file:\n",
        "  write_file.write(f\"Removed {removed_count} lines from the file.\")\n",
        "\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Text After Removing/Finance.txt', 'w', encoding='utf-8') as f:\n",
        "    for line in lines:\n",
        "        if line:\n",
        "            f.write(line + '\\n')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N7kswTaRh4q3",
        "outputId": "d2366981-a4c9-4989-c204-5b878c0550a1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total lines removed: 315\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Count The total Removable Lines(English Sentences) and save after Removal --Finance**"
      ],
      "metadata": {
        "id": "vVee5ZlG5NHD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "bengali_pattern = re.compile('[\\u0980-\\u09FF]+')\n",
        "\n",
        "\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Text After Removing/Finance.txt', 'r', encoding='utf-8') as f:\n",
        "    content = f.readlines()\n",
        "\n",
        "# remove lines that contain only English words\n",
        "filtered_content = [line for line in content if not all(re.search(r'\\w', word) and not bengali_pattern.search(word) for word in line.split())]\n",
        "\n",
        "\n",
        "num_removable_lines = len(content) - len(filtered_content)\n",
        "print(f\"Removable {num_removable_lines} English lines from the file.\")\n",
        "with open(\"/content/drive/MyDrive/web_scraper/Remove Lines /Count Removable English Line/Finance.txt\", \"w\") as write_file:\n",
        "  write_file.write(f\"Removable {num_removable_lines} English lines from the file.\")\n",
        "\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Text without English Lines/Finance.txt', 'w', encoding='utf-8') as f:\n",
        "    f.writelines(filtered_content)\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KAtqwNW84qwb",
        "outputId": "3a864ef4-4432-4b2a-e282-f6f35deb7db4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Removable 5088 English lines from the file.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Count No of Lines mixed with english and Bengali words**"
      ],
      "metadata": {
        "id": "-xipgka10JNz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n",
        "import re\n",
        "\n",
        "input_file = '/content/drive/MyDrive/web_scraper/Remove Lines /English Line Manual removing/Finance.txt'\n",
        "output_file = '/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Bengali Lines/output_Finance.txt'\n",
        "removed_lines = []\n",
        "\n",
        "\n",
        "with open(input_file, 'r', encoding='utf-8') as f_in, open(output_file, 'w', encoding='utf-8') as f_out:\n",
        "    for line in f_in:\n",
        "        # Check if the line contains both English and Bengali words\n",
        "        if re.search(r'[a-zA-Z]+', line) and re.search(r'[\\u0980-\\u09FF]+', line):\n",
        "            removed_lines.append(line)\n",
        "        else:\n",
        "            f_out.write(line)\n",
        "\n",
        "# Count the number of removed lines\n",
        "num_removed_lines = len(removed_lines)\n",
        "print(f'Number of removed lines: {num_removed_lines}')\n",
        "\n",
        "# Write the removed lines to a separate file\n",
        "removed_file = '/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Removed English and Bengali Lines/removed_lines_Finance.txt'\n",
        "with open(removed_file, 'w', encoding='utf-8') as f_removed:\n",
        "    f_removed.writelines(removed_lines)\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ELrE4LhC0Kl1",
        "outputId": "522df683-7245-4de2-d37e-90cac8d1ef63"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of removed lines: 4820\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Remove extra space and line having word count < 5(FINANCE)**"
      ],
      "metadata": {
        "id": "D1ff496ijmwh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Bengali Lines/output_Finance.txt', 'r') as input_file:\n",
        "    lines = input_file.readlines()\n",
        "\n",
        "output_lines = []\n",
        "for line in lines:\n",
        "    # Remove extra spaces\n",
        "    cleaned_line = ' '.join(line.strip().split())\n",
        "    #  word count < 5\n",
        "    if len(cleaned_line.split()) >= 5:\n",
        "        output_lines.append(cleaned_line)\n",
        "\n",
        "with open('/content/drive/MyDrive/web_scraper/Remove Lines /Bengali Lines /Removal after Less than five Words/Finance.txt', 'w') as output_file:\n",
        "    output_file.write('\\n'.join(output_lines))"
      ],
      "metadata": {
        "id": "0ucjVXKXjjOU"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7tlqM7k1degJ"
      },
      "source": [
        "**Read from bengali webpage**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ch7KluwudlZU",
        "outputId": "b86fb7c1-7493-489c-8276-5c02aaebc9df"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# scrape_tool.read_page(link=\"https://www.anandabazar.com/rabibashoriyo/%E0%A6%97-%E0%A6%9C-%E0%A6%9A%E0%A6%B0%E0%A6%B8-%E0%A6%B9-%E0%A6%B6-%E0%A6%B6-1.313437\")\n",
        "scrape_tool.read_page(link=\"https://bn.wikipedia.org/wiki/%E0%A6%B8%E0%A7%8D%E0%A6%AC%E0%A6%BE%E0%A6%B8%E0%A7%8D%E0%A6%A5%E0%A7%8D%E0%A6%AF%E0%A7%87%E0%A6%B0_%E0%A6%89%E0%A6%AA%E0%A6%B0_%E0%A6%A4%E0%A6%BE%E0%A6%AE%E0%A6%BE%E0%A6%95%E0%A7%87%E0%A6%B0_%E0%A6%AA%E0%A7%8D%E0%A6%B0%E0%A6\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JAj7rnMhB60h"
      },
      "source": [
        "**Read the saved Json File**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1nlgKrr6Xjww",
        "outputId": "a88b8cda-1f50-49d2-d9ea-f5120834e52d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'https://bengali.krishijagran.com/education': ['নবোদয় বিদ্যালয়ে ভর্তি হতে চান, তাহলে এইভাবে আবেদন করুন',\n",
              "  'দেশের সকল অভিভাবকদের জন্য সুখবর রয়েছে যারা তাদের সন্তানদের নবোদয় বিদ্যালয়ে পাঠাতে চান। প্রকৃতপক্ষে, নবোদয় বিদ্যালয় সমিতি',\n",
              "  ' ‘৮৯ হাজার সংখ্যাটা নেহাত কম নয়’,চাকরি প্রার্থীদের আশ্বস্ত করলেন মমতা ',\n",
              "  'নিয়োগ দুর্নীতি নিয়ে গোটা বাংলা যখন তোলপার ঠিক তখনই শিক্ষাক্ষেত্রে বিপুল নিয়োগের কথা জানালেন মুখ্যমন্ত্রী মমতা বন্দ্যোপাধ্যায়।',\n",
              "  'ভারতীয় সেনাবাহিনীতে প্রচুর নিয়োগ ,জারি হল বিজ্ঞপ্তি ',\n",
              "  'ভারতীয় সেনাবাহিনীতে এনসিসি স্পেশাল এন্ট্রি স্কিমের আওতায় নিয়োগের বিজ্ঞপ্তি জারি হয়েছে। উপযুক্ত শিক্ষাগত যোগ্যতা থাকলে আগ্রহী প্রার্থীরা এই',\n",
              "  'IBPS RRB Admit Card 2022: আইবিপিএস এর অ্যাডমিট কার্ড ডাউনলোড করুন,জানুন পদ্ধতি',\n",
              "  'মোট আট হাজারেরও বেশি শূন্য পদে যোগ্য প্রার্থী বাছাই করা হবে। প্রাথমিক ও মূল পরীক্ষার ভিত্তিতে প্রার্থী বাছাই করা হবে। কোনো ইন্টারভিউ নেওয়া হবে না।',\n",
              "  'প্রাইমারি দুর্নীতি নিয়ে সিবিআই কে সিট গঠনের নির্দেশ দিল কলকাতা হাইকোর্ট ',\n",
              "  'প্রাইমারিতে নিয়োগ দুর্নীতি মামলায় সিবিআই তদন্তে সন্তুষ্ট নয় কলকাতা হাইকোর্ট। সিবিআই তদন্তের নির্দেশের পর এবার এই মামলায় আরও কড়া পদক্ষেপ নিল হাইকোর্ট।',\n",
              "  'স্নাতক, স্নাতকোত্তর শিক্ষার্থীরা 9 লাখ 46 হাজার 522 টাকার বৃত্তি পাবেন, শীঘ্রই আবেদন করুন',\n",
              "  'তফশিলি উপজাতির ছাত্রদের পড়াশোনায় সাহায্য করার জন্য সরকার বিভিন্ন পরিকল্পনা শুরু করেছে।',\n",
              "  'ভর্তি হবে না কেন্দ্রীয় ভাবে,জানিয়ে দিল শিক্ষা দফতর',\n",
              "  'কেন্দ্রীয় অনলাইনে ভর্তি নিয়ে পিছু হঠল রাজ্য সরকার।কলেজ-বিশ্ববিদ্যালয়ে কেন্দ্রীয়ভাবে অনলাইনে ভর্তি নিয়ে সিদ্ধান্ত বদল রাজ্য সরকারের।',\n",
              "  'জামিয়ায় স্নাতক, স্নাতকোত্তর, ডিপ্লোমা ভর্তি শুরু হয়েছে, বিস্তারিত জানুন',\n",
              "  'জামিয়া বিশ্ববিদ্যালয় 12 তম পাস আউট শিক্ষার্থীদের জন্য ভর্তি প্রক্রিয়া শুরু করেছে, যার শেষ তারিখ 14 আগস্ট।',\n",
              "  'ভর্তি 2022: কৃষি বিশ্ববিদ্যালয়ে ভর্তি শুরু, জেনে নিন পুরো প্রক্রিয়া',\n",
              "  'চৌধুরী চরণ সিং হরিয়ানা কৃষি বিশ্ববিদ্যালয়ে নতুন সেশন 2022-2023-এর জন্য কৃষি কোর্সে ভর্তির প্রক্রিয়া শুরু হয়েছে।',\n",
              "  'AIIMS-এ নিয়োগ: 30 জুনের মধ্যে আবেদন করার বিজ্ঞপ্তি!',\n",
              "  'অল ইন্ডিয়া ইনস্টিটিউট অফ মেডিক্যাল সায়েন্সেস, নিউ দিল্লির পদের জন্য দরখাস্ত আহ্বান করা হচ্ছে।',\n",
              "  'গ্রামীণ এলাকা এবং কৃষি বিভাগে এই পদগুলির জন্য নিয়োগ, রইল বিস্তারিত ',\n",
              "  'আজকে আমাদের এই একটি নিবন্ধে আমরা বিভিন্ন বিভাগে নিয়োগ সম্পর্কে বিস্তারিত বলব।',\n",
              "  'অগ্নিপথ যোজনা: সেনাবাহিনীতে আগ্রহী যুবকদের জন্য সুবর্ণ সুযোগ দিচ্ছে সরকার ',\n",
              "  'ভারত সরকার প্রতিরক্ষায় আগ্রহী তরুণদের জন্য একটি উপহার নিয়ে এসেছে।',\n",
              "  'NABARD নিয়োগ 2022: কৃষি বিশেষজ্ঞ সহ বিভিন্ন পদের জন্য আমন্ত্রণ',\n",
              "  'NABARD কনসালটেন্সি সার্ভিসেস প্রাইভেট লিমিটেড (NABCONS), NABARD-এর সম্পূর্ণ মালিকানাধীন একটি সহযোগী প্রতিষ্ঠান এবং একটি নেতৃস্থানীয় কৃষি পরামর্শক সংস্থা, বিভিন্ন পদের জন্য প্রার্থীদের কাছ থেকে অনলাইনে',\n",
              "  'আয়ুষ মন্ত্রণালয়ে নিয়োগ: প্রতি মাসে ₹75000 বেতন ',\n",
              "  'আপনি যদি চাকরি খুঁজছেন তবে আপনার জন্য এখানে একটি দুর্দান্ত সুযোগ রয়েছে। আয়ুষ মন্ত্রক প্রতি মাসে 75,000 টাকা বেতনে নিয়োগ করেছে ।',\n",
              "  'নিয়োগ: বিশ্বের বৃহত্তম দুধ সমবায়ে স্নাতকদের জন্য কর্মসংস্থানের সুযোগ',\n",
              "  'আমুল টেরিটরি সেলস নিয়োগের জন্য প্রার্থীদের কাছ থেকে দরখাস্ত আহ্বান করছে। যারা চাকরির জন্য আবেদন করতে আগ্রহী তারা আরও বিস্তারিত জানার জন্য অফিসিয়াল সাইটে যেতে',\n",
              "  'ICAR Recruitment 2022:  স্নাতক, স্নাতকোত্তর এবং পিএইচডি লোকেদের জন্য ICAR-তে নিয়োগ',\n",
              "  'আপনি যদি কৃষি সেক্টরে চাকরি করতে আগ্রহী হন , তাহলে আমরা আপনাকে বলি যে ইন্ডিয়ান কাউন্সিল অফ এগ্রিকালচারাল রিসার্চ (ICAR) আইটি প্রফেশনাল পদের জন্য আবেদনের',\n",
              "  'Amul Recruitment 2022: বিশ্বের বৃহত্তম দুধ সমবায়ে যাওয়ার সুবর্ণ সুযোগ',\n",
              "  'আমুল নিয়োগ 2022: বিশ্বের বৃহত্তম দুধ সমবায়ে প্রবেশের একটি সুবর্ণ সুযোগ। শিল্পের সেরা বেতনপ্রাপ্ত আমুল অ্যাকাউন্ট সহকারী নিয়োগের বিজ্ঞপ্তি দিয়েছে।',\n",
              "  'স্কুল থেকে শিক্ষা নিয়ে বাড়িতে সব্জি চাষ ! পড়ুয়ারা জোগাচ্ছে পড়ার খরচ',\n",
              "  'স্কুলেই দেওয়া হচ্ছে কৃষি বিষয়ে বিশেষ শিক্ষা। আর সেখান থেকেই শিক্ষা নিয়ে পড়ুয়ারা ঘরের বাগানে সব্জি চাষ করে উপার্জন করছে টাকা।',\n",
              "  'Rubber Board Recruitment 2022: এগ্রি গ্র্যাজুয়েটদের জন্য সুবর্ণ সুযোগ; ফিল্ড অফিসার পদের জন্য আবেদন করুন',\n",
              "  'রাবার বোর্ড বর্তমানে ফিল্ড অফিসার পদের জন্য প্রার্থীদের নিয়োগ করছে। 2 মে, 2022 এর মধ্যে, আগ্রহী ব্যক্তিরা চাকরি বিভাগের অধীনে অফিসিয়াল ওয়েবসাইটে রাবার বোর্ড নিয়োগ',\n",
              "  'RBI নিয়োগ 2022: RBI-তে অফিসার পদের জন্য  নিয়োগ! কোথায় আবেদন করতে হবে তা জেনে নিন',\n",
              "  'আপনি যদি সরকারি চাকরি খুঁজছেন, তাহলে এই নিবন্ধটি আপনার জন্য খুবই উপযোগী। প্রকৃতপক্ষে, রিজার্ভ ব্যাঙ্ক অফ ইন্ডিয়া (আরবিআই) সম্প্রতি বিভিন্ন পদে নিয়োগের জন্য একটি বিজ্ঞপ্তি']}"
            ]
          },
          "execution_count": 43,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import json\n",
        "\n",
        "file = '/content/drive/MyDrive/web_scraper/Agriculture/Farmer's_bill_india_(new)_1.json'\n",
        "with open('/content/drive/MyDrive/web_scraper/Agriculture/Farmer's_bill_india_(new)_1.json', \"rb\") as read_file:\n",
        "    data = json.load(read_file)\n",
        "data"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}