Python_Web_Scrapper.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Python Web Scrapper.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyNbNMAlZ1Eg7w4bn3TIgMPS",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dhruvalgupta2003/0xSCAN/blob/master/Python_Web_Scrapper.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 935
        },
        "id": "yKz2iF1wEXfm",
        "outputId": "1fd3666f-4e35-4d3c-b79a-cda73bc4944d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1\n",
            "2\n",
            "3\n",
            "4\n",
            "5\n",
            "6\n",
            "7\n",
            "8\n",
            "9\n",
            "10\n",
            "11\n",
            "12\n",
            "13\n",
            "14\n",
            "15\n",
            "16\n",
            "17\n",
            "18\n",
            "19\n",
            "20\n",
            "21\n",
            "22\n",
            "23\n",
            "24\n",
            "25\n",
            "26\n",
            "27\n",
            "28\n",
            "29\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/core/internals/construction.py\u001b[0m in \u001b[0;36m_finalize_columns_and_data\u001b[0;34m(content, columns, dtype)\u001b[0m\n\u001b[1;32m    905\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 906\u001b[0;31m         \u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_validate_or_indexify_columns\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontents\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    907\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mAssertionError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/core/internals/construction.py\u001b[0m in \u001b[0;36m_validate_or_indexify_columns\u001b[0;34m(content, columns)\u001b[0m\n\u001b[1;32m    954\u001b[0m             raise AssertionError(\n\u001b[0;32m--> 955\u001b[0;31m                 \u001b[0;34mf\"{len(columns)} columns passed, passed data had \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    956\u001b[0m                 \u001b[0;34mf\"{len(content)} columns\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAssertionError\u001b[0m: 6 columns passed, passed data had 5 columns",
            "\nThe above exception was the direct cause of the following exception:\n",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-10-c7c1c3d725ae>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     52\u001b[0m \u001b[0mheaders\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'title'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'degrees'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'funds'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'date'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'location'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'link'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;31m#create dataframe from scrapped data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m \u001b[0mschoolarships\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, index, columns, dtype, copy)\u001b[0m\n\u001b[1;32m    698\u001b[0m                         \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    699\u001b[0m                         \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;31m# type: ignore[arg-type]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 700\u001b[0;31m                         \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    701\u001b[0m                     )\n\u001b[1;32m    702\u001b[0m                     mgr = arrays_to_mgr(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/core/internals/construction.py\u001b[0m in \u001b[0;36mnested_data_to_arrays\u001b[0;34m(data, columns, index, dtype)\u001b[0m\n\u001b[1;32m    481\u001b[0m         \u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mensure_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    482\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 483\u001b[0;31m     \u001b[0marrays\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    484\u001b[0m     \u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mensure_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    485\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/core/internals/construction.py\u001b[0m in \u001b[0;36mto_arrays\u001b[0;34m(data, columns, dtype)\u001b[0m\n\u001b[1;32m    805\u001b[0m         \u001b[0marr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_list_to_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    806\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 807\u001b[0;31m     \u001b[0mcontent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_finalize_columns_and_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    808\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mcontent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    809\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/core/internals/construction.py\u001b[0m in \u001b[0;36m_finalize_columns_and_data\u001b[0;34m(content, columns, dtype)\u001b[0m\n\u001b[1;32m    907\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mAssertionError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    908\u001b[0m         \u001b[0;31m# GH#26429 do not raise user-facing AssertionError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 909\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    910\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    911\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontents\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mcontents\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobject_\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: 6 columns passed, passed data had 5 columns"
          ]
        }
      ],
      "source": [
        "#import some libraries\n",
        "import pandas as pd\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import time\n",
        "#create an empty list to append it later\n",
        "list = []\n",
        "#create a list of countries which are presented on website\n",
        "locations = ['united-states','united-kingdom','canada','europe','south-africa','nigeria','india','australia','brazil','germany','hong-kong','united-aram-emirates','korea','japan','china']\n",
        "#first loop - scraping all the pages from website\n",
        "for n in range(1,30):\n",
        "    print(n)\n",
        "    #sleep, in order not to be baned\n",
        "    time.sleep(1)\n",
        "    #second loop - scraping suggestions from different countries\n",
        "    for location in locations:\n",
        "    \n",
        "        url = f'https://www.scholarshipportal.com/scholarships/{location}?page={n}'\n",
        "\n",
        "        r = requests.get(url)\n",
        "        soup = BeautifulSoup(r.text,'lxml')\n",
        "\n",
        "        articles = soup.findAll('a', class_='flex flex-col p-8 text-black no-underline bg-white rounded shadow md:flex-row hover:text-black group mb-4')\n",
        "        #third loop - scraping all the necessary information for each suggextion\n",
        "        for article in articles:\n",
        "            #also, add except in order to get error in empty 'find'\n",
        "            try:\n",
        "                title = article.find('h3', class_='font-bold text-xl group-hover:underline group-hover:text-primary-500').text\n",
        "            except:\n",
        "                title = None\n",
        "            try:\n",
        "                degrees = article.findAll('span', class_='bg-gray-100 inline-block mr-2 py-1 px-3 text-sm text-gray-600 rounded mb-1')[0].text\n",
        "            except:\n",
        "                degrees = None\n",
        "            try:\n",
        "                funds = article.findAll('span', class_='bg-gray-100 inline-block mr-2 py-1 px-3 text-sm text-gray-600 rounded mb-1')[1].text\n",
        "            except:\n",
        "                funds = None\n",
        "            try:\n",
        "                date = article.findAll('span', class_='bg-gray-100 inline-block mr-2 py-1 px-3 text-sm text-gray-600 rounded')[0].text\n",
        "            except:\n",
        "                date = None\n",
        "            try:\n",
        "                link = article.findAll('href', class_='flex flex-col p-8 text-black no-underline bg-white rounded shadow md:flex-row hover:text-black group mb-4').text\n",
        "            except:\n",
        "                link = None\n",
        "            country = location\n",
        "            \n",
        "            #append the empty list\n",
        "            list.append([title, degrees, funds, date, location])\n",
        "#create titles for future table\n",
        "headers = ['title', 'degrees', 'funds', 'date', 'location','link']\n",
        "#create dataframe from scrapped data\n",
        "schoolarships = pd.DataFrame(data = list, columns = headers)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "schoolarships.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bFL7voXBJMHp",
        "outputId": "ddeba6ec-5182-43db-bbbb-87d45ed78d43"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 5310 entries, 0 to 5309\n",
            "Data columns (total 5 columns):\n",
            " #   Column    Non-Null Count  Dtype \n",
            "---  ------    --------------  ----- \n",
            " 0   title     5310 non-null   object\n",
            " 1   degrees   5310 non-null   object\n",
            " 2   funds     3380 non-null   object\n",
            " 3   date      1867 non-null   object\n",
            " 4   location  5310 non-null   object\n",
            "dtypes: object(5)\n",
            "memory usage: 207.5+ KB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "\n",
        "schoolarships.to_csv('output2.csv', encoding = 'utf-8-sig') \n",
        "files.download('output2.csv')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "xGaBdlchKEem",
        "outputId": "4c5d3e2e-4ec7-42f5-957e-9faa65e824c9"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_6f36523d-2d5a-4b46-bc53-752161db62e9\", \"output1.csv\", 486089)"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}