WEB-SCRAPING-ASSIGNMENT-1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## WEB-SCRAPING-ASSIGNMENT-1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Qustion 1 :-\n",
    "\n",
    "## Write a python program to display all the header tags from  ‘en.wikipedia.org/wiki/Main_Page’."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "List all the header tags :\n",
      "\n",
      "<h1 class=\"firstHeading\" id=\"firstHeading\" lang=\"en\">Main Page</h1>\n",
      "\n",
      "<h2 class=\"mp-h2\" id=\"mp-tfa-h2\"><span id=\"From_today.27s_featured_article\"></span><span class=\"mw-headline\" id=\"From_today's_featured_article\">From today's featured article</span></h2>\n",
      "\n",
      "<h2 class=\"mp-h2\" id=\"mp-dyk-h2\"><span class=\"mw-headline\" id=\"Did_you_know_...\">Did you know ...</span></h2>\n",
      "\n",
      "<h2 class=\"mp-h2\" id=\"mp-itn-h2\"><span class=\"mw-headline\" id=\"In_the_news\">In the news</span></h2>\n",
      "\n",
      "<h2 class=\"mp-h2\" id=\"mp-otd-h2\"><span class=\"mw-headline\" id=\"On_this_day\">On this day</span></h2>\n",
      "\n",
      "<h2 class=\"mp-h2\" id=\"mp-tfl-h2\"><span id=\"From_today.27s_featured_list\"></span><span class=\"mw-headline\" id=\"From_today's_featured_list\">From today's featured list</span></h2>\n",
      "\n",
      "<h2 class=\"mp-h2\" id=\"mp-tfp-h2\"><span id=\"Today.27s_featured_picture\"></span><span class=\"mw-headline\" id=\"Today's_featured_picture\">Today's featured picture</span></h2>\n",
      "\n",
      "<h2 class=\"mp-h2\" id=\"mp-other\"><span class=\"mw-headline\" id=\"Other_areas_of_Wikipedia\">Other areas of Wikipedia</span></h2>\n",
      "\n",
      "<h2 class=\"mp-h2\" id=\"mp-sister\"><span id=\"Wikipedia.27s_sister_projects\"></span><span class=\"mw-headline\" id=\"Wikipedia's_sister_projects\">Wikipedia's sister projects</span></h2>\n",
      "\n",
      "<h2 class=\"mp-h2\" id=\"mp-lang\"><span class=\"mw-headline\" id=\"Wikipedia_languages\">Wikipedia languages</span></h2>\n",
      "\n",
      "<h2>Navigation menu</h2>\n",
      "\n",
      "<h3 id=\"p-personal-label\">\n",
      "<span>Personal tools</span>\n",
      "</h3>\n",
      "\n",
      "<h3 id=\"p-namespaces-label\">\n",
      "<span>Namespaces</span>\n",
      "</h3>\n",
      "\n",
      "<h3 id=\"p-variants-label\">\n",
      "<span>Variants</span>\n",
      "</h3>\n",
      "\n",
      "<h3 id=\"p-views-label\">\n",
      "<span>Views</span>\n",
      "</h3>\n",
      "\n",
      "<h3 id=\"p-cactions-label\">\n",
      "<span>More</span>\n",
      "</h3>\n",
      "\n",
      "<h3>\n",
      "<label for=\"searchInput\">Search</label>\n",
      "</h3>\n",
      "\n",
      "<h3 id=\"p-navigation-label\">\n",
      "<span>Navigation</span>\n",
      "</h3>\n",
      "\n",
      "<h3 id=\"p-interaction-label\">\n",
      "<span>Contribute</span>\n",
      "</h3>\n",
      "\n",
      "<h3 id=\"p-tb-label\">\n",
      "<span>Tools</span>\n",
      "</h3>\n",
      "\n",
      "<h3 id=\"p-coll-print_export-label\">\n",
      "<span>Print/export</span>\n",
      "</h3>\n",
      "\n",
      "<h3 id=\"p-wikibase-otherprojects-label\">\n",
      "<span>In other projects</span>\n",
      "</h3>\n",
      "\n",
      "<h3 id=\"p-lang-label\">\n",
      "<span>Languages</span>\n",
      "</h3>\n"
     ]
    }
   ],
   "source": [
    "##  import the requried  libraries for web -scraping\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "## Lode the source code \n",
    "html = urlopen('https://en.wikipedia.org/wiki/Main_Page')\n",
    "\n",
    "## pares the source\n",
    "bs = BeautifulSoup(html, \"html.parser\")\n",
    "\n",
    "## find the all headers tages\n",
    "titles = bs.find_all(['h1', 'h2','h3','h4','h5','h6'])\n",
    "print('List all the header tags :', *titles, sep='\\n\\n')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Question 2:- \n",
    "\n",
    "## Write a python program to display IMDB’s Top rated 100 movies’ data (i.e. Name, IMDB rating, Year of release) and save it in form of a CSV file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  import the requried  libraries for web -scraping\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup as Soup\n",
    "import requests as rs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Lode the source code\n",
    "page = rs.get(\"https://www.imdb.com/list/ls091520106/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = Soup(page.content , \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create 3 empty list(\"Movie_Name\", \"IMDB_rating\",\"Release_Year\")\n",
    "\n",
    "Movie_Name = []\n",
    "IMDB_rating = []\n",
    "Release_Year = []\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<h3 class=\"lister-item-header\">\n",
       " <span class=\"lister-item-index unbold text-primary\">1.</span>\n",
       " <a href=\"/title/tt0111161/\">The Shawshank Redemption</a>\n",
       " <span class=\"lister-item-year text-muted unbold\">(1994)</span>\n",
       " </h3>,\n",
       " <h3 class=\"lister-item-header\">\n",
       " <span class=\"lister-item-index unbold text-primary\">2.</span>\n",
       " <a href=\"/title/tt0068646/\">The Godfather</a>\n",
       " <span class=\"lister-item-year text-muted unbold\">(1972)</span>\n",
       " </h3>,\n",
       " <h3 class=\"lister-item-header\">\n",
       " <span class=\"lister-item-index unbold text-primary\">3.</span>\n",
       " <a href=\"/title/tt0071562/\">The Godfather: Part II</a>\n",
       " <span class=\"lister-item-year text-muted unbold\">(1974)</span>\n",
       " </h3>,\n",
       " <h3 class=\"lister-item-header\">\n",
       " <span class=\"lister-item-index unbold text-primary\">4.</span>\n",
       " <a href=\"/title/tt0468569/\">The Dark Knight</a>\n",
       " <span class=\"lister-item-year text-muted unbold\">(2008)</span>\n",
       " </h3>]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Movie_Name .\n",
    "\n",
    "Name = soup.find_all(\"h3\", class_ =\"lister-item-header\")\n",
    "Name[0:4]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' 1. The Shawshank Redemption (1994) ',\n",
       " ' 2. The Godfather (1972) ',\n",
       " ' 3. The Godfather: Part II (1974) ',\n",
       " ' 4. The Dark Knight (2008) ']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now we extact the text form the these tegs by using for loop over the tags \n",
    "\n",
    "for i in Name :\n",
    "    Movie_Name.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Movie_Name[0:4]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<div class=\"ipl-rating-star small\">\n",
       " <span class=\"ipl-rating-star__star\">\n",
       " <svg class=\"ipl-icon ipl-star-icon\" fill=\"#000000\" height=\"24\" viewbox=\"0 0 24 24\" width=\"24\" xmlns=\"http://www.w3.org/2000/svg\">\n",
       " <path d=\"M0 0h24v24H0z\" fill=\"none\"></path>\n",
       " <path d=\"M12 17.27L18.18 21l-1.64-7.03L22 9.24l-7.19-.61L12 2 9.19 8.63 2 9.24l5.46 4.73L5.82 21z\"></path>\n",
       " <path d=\"M0 0h24v24H0z\" fill=\"none\"></path>\n",
       " </svg>\n",
       " </span>\n",
       " <span class=\"ipl-rating-star__rating\">9.3</span>\n",
       " </div>,\n",
       " <div class=\"ipl-rating-star small\">\n",
       " <span class=\"ipl-rating-star__star\">\n",
       " <svg class=\"ipl-icon ipl-star-icon\" fill=\"#000000\" height=\"24\" viewbox=\"0 0 24 24\" width=\"24\" xmlns=\"http://www.w3.org/2000/svg\">\n",
       " <path d=\"M0 0h24v24H0z\" fill=\"none\"></path>\n",
       " <path d=\"M12 17.27L18.18 21l-1.64-7.03L22 9.24l-7.19-.61L12 2 9.19 8.63 2 9.24l5.46 4.73L5.82 21z\"></path>\n",
       " <path d=\"M0 0h24v24H0z\" fill=\"none\"></path>\n",
       " </svg>\n",
       " </span>\n",
       " <span class=\"ipl-rating-star__rating\">9.2</span>\n",
       " </div>,\n",
       " <div class=\"ipl-rating-star small\">\n",
       " <span class=\"ipl-rating-star__star\">\n",
       " <svg class=\"ipl-icon ipl-star-icon\" fill=\"#000000\" height=\"24\" viewbox=\"0 0 24 24\" width=\"24\" xmlns=\"http://www.w3.org/2000/svg\">\n",
       " <path d=\"M0 0h24v24H0z\" fill=\"none\"></path>\n",
       " <path d=\"M12 17.27L18.18 21l-1.64-7.03L22 9.24l-7.19-.61L12 2 9.19 8.63 2 9.24l5.46 4.73L5.82 21z\"></path>\n",
       " <path d=\"M0 0h24v24H0z\" fill=\"none\"></path>\n",
       " </svg>\n",
       " </span>\n",
       " <span class=\"ipl-rating-star__rating\">9</span>\n",
       " </div>,\n",
       " <div class=\"ipl-rating-star small\">\n",
       " <span class=\"ipl-rating-star__star\">\n",
       " <svg class=\"ipl-icon ipl-star-icon\" fill=\"#000000\" height=\"24\" viewbox=\"0 0 24 24\" width=\"24\" xmlns=\"http://www.w3.org/2000/svg\">\n",
       " <path d=\"M0 0h24v24H0z\" fill=\"none\"></path>\n",
       " <path d=\"M12 17.27L18.18 21l-1.64-7.03L22 9.24l-7.19-.61L12 2 9.19 8.63 2 9.24l5.46 4.73L5.82 21z\"></path>\n",
       " <path d=\"M0 0h24v24H0z\" fill=\"none\"></path>\n",
       " </svg>\n",
       " </span>\n",
       " <span class=\"ipl-rating-star__rating\">9</span>\n",
       " </div>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Ratings .\n",
    "\n",
    "Ratings = soup.find_all(\"div\", class_ =\"ipl-rating-star small\")\n",
    "Ratings[0:4]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['        9.3 ', '        9.2 ', '        9 ', '        9 ']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in Ratings :\n",
    "    IMDB_rating.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "IMDB_rating[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<span class=\"lister-item-year text-muted unbold\">(1994)</span>,\n",
       " <span class=\"lister-item-year text-muted unbold\">(1972)</span>,\n",
       " <span class=\"lister-item-year text-muted unbold\">(1974)</span>,\n",
       " <span class=\"lister-item-year text-muted unbold\">(2008)</span>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Years .\n",
    "\n",
    "Years = soup.find_all(\"span\", class_ =\"lister-item-year text-muted unbold\")\n",
    "Years[0:4]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['(1994)', '(1972)', '(1974)', '(2008)']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#now we extact the text form the these tegs by using for loop over the tags \n",
    "\n",
    "for i in Years :\n",
    "    Release_Year.append(i.get_text().replace(\"\\n\",\"\"))\n",
    "\n",
    "Release_Year [0:4]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 100 100\n"
     ]
    }
   ],
   "source": [
    "# check the lenghth of the all 3 columns\n",
    "print(len(Movie_Name),len(IMDB_rating),len(Release_Year))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### Create the dataframe \n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "Movie_data = pd.DataFrame({})\n",
    "Movie_data [\"Movie_Name\"] = Movie_Name\n",
    "Movie_data[\"IMDB_rating\"] = IMDB_rating\n",
    "Movie_data[\"Release_Year\"] = Release_Year\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Movie_Name</th>\n",
       "      <th>IMDB_rating</th>\n",
       "      <th>Release_Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1. The Shawshank Redemption (1994)</td>\n",
       "      <td>9.3</td>\n",
       "      <td>(1994)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2. The Godfather (1972)</td>\n",
       "      <td>9.2</td>\n",
       "      <td>(1972)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3. The Godfather: Part II (1974)</td>\n",
       "      <td>9</td>\n",
       "      <td>(1974)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4. The Dark Knight (2008)</td>\n",
       "      <td>9</td>\n",
       "      <td>(2008)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5. 12 Angry Men (1957)</td>\n",
       "      <td>9</td>\n",
       "      <td>(1957)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>96. North by Northwest (1959)</td>\n",
       "      <td>8.3</td>\n",
       "      <td>(1959)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>97. A Clockwork Orange (1971)</td>\n",
       "      <td>8.3</td>\n",
       "      <td>(1971)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>98. Snatch (2000)</td>\n",
       "      <td>8.3</td>\n",
       "      <td>(2000)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>99. Amélie (2001)</td>\n",
       "      <td>8.3</td>\n",
       "      <td>(2001)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>100. The Kid (1921)</td>\n",
       "      <td>8.3</td>\n",
       "      <td>(1921)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Movie_Name   IMDB_rating Release_Year\n",
       "0    1. The Shawshank Redemption (1994)           9.3        (1994)\n",
       "1               2. The Godfather (1972)           9.2        (1972)\n",
       "2      3. The Godfather: Part II (1974)             9        (1974)\n",
       "3             4. The Dark Knight (2008)             9        (2008)\n",
       "4                5. 12 Angry Men (1957)             9        (1957)\n",
       "..                                   ...           ...          ...\n",
       "95        96. North by Northwest (1959)           8.3        (1959)\n",
       "96        97. A Clockwork Orange (1971)           8.3        (1971)\n",
       "97                    98. Snatch (2000)           8.3        (2000)\n",
       "98                    99. Amélie (2001)           8.3        (2001)\n",
       "99                  100. The Kid (1921)           8.3        (1921)\n",
       "\n",
       "[100 rows x 3 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Movie_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "import re\n",
    "warnings.filterwarnings('ignore')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean(text):\n",
    "    text = ''.join([i for i in text if not i.isdigit()])\n",
    "    text =  text.replace(\"()\",\"\").replace(\".\",\"\")\n",
    "    return text\n",
    "\n",
    "Movie_data[\"Movie_Name\"]=Movie_data[\"Movie_Name\"].apply(clean)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Movie_Name</th>\n",
       "      <th>IMDB_rating</th>\n",
       "      <th>Release_Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>9.3</td>\n",
       "      <td>(1994)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Godfather</td>\n",
       "      <td>9.2</td>\n",
       "      <td>(1972)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Godfather: Part II</td>\n",
       "      <td>9</td>\n",
       "      <td>(1974)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The Dark Knight</td>\n",
       "      <td>9</td>\n",
       "      <td>(2008)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Angry Men</td>\n",
       "      <td>9</td>\n",
       "      <td>(1957)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Movie_Name   IMDB_rating Release_Year\n",
       "0    The Shawshank Redemption            9.3        (1994)\n",
       "1               The Godfather            9.2        (1972)\n",
       "2      The Godfather: Part II              9        (1974)\n",
       "3             The Dark Knight              9        (2008)\n",
       "4                   Angry Men              9        (1957)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Movie_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Qustion 3:-\n",
    "\n",
    "## Write a python program to display IMDB’s Top rated 100 Indian movies’ data (i.e. Name, IMDB rating, Year of release) and save it in form of a CSV file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "### ##  import the requried  libraries for web -scraping\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup as Soup\n",
    "import requests as rs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Lode the source code\n",
    "page_source = rs.get(\"https://www.imdb.com/india/top-rated-indian-movies/\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Now parse the source code bu BeautifulSoup and the parser used here is html parse as source code in html\n",
    "\n",
    "M_soup = Soup(page_source.content , \"html.parser\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create 3 empty list(\"IND_Movie_Name\", \"IND_IMDB_rating\",\"IND_Release_Year\")\n",
    "\n",
    "IND_Movie_Name = []\n",
    "IND_IMDB_rating = []\n",
    "IND_Release_Year = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['       1.       Pather Panchali (1955) ',\n",
       " '       2.       Ratsasan (2018) ',\n",
       " '       3.       Gol Maal (1979) ',\n",
       " '       4.       Nayakan (1987) ']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Movie_Name .\n",
    "\n",
    "Name1 = M_soup.find_all(\"td\", class_ =\"titleColumn\")\n",
    "\n",
    "# now we extact the text form the these tegs by using for loop over the tags \n",
    "\n",
    "for i in Name1 :\n",
    "    IND_Movie_Name.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "IND_Movie_Name[0:4]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' 8.5 ', ' 8.5 ', ' 8.5 ', ' 8.5 ']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have IND_IMDB_rating .\n",
    "\n",
    "Ratings1 = M_soup.find_all(\"td\", class_=\"ratingColumn imdbRating\")\n",
    "Ratings1[0:4]\n",
    "\n",
    "# now we extact the text form the these tegs by using for loop over the tags \n",
    "\n",
    "for i in Ratings1 :\n",
    "    IND_IMDB_rating.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "IND_IMDB_rating[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['(1955)', '(2018)', '(1979)', '(1987)']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have IND_Release_Year .\n",
    "\n",
    "YEAR = M_soup.find_all(\"span\", class_=\"secondaryInfo\")\n",
    "YEAR[0:4]\n",
    "\n",
    "# now we extact the text form the these tegs by using for loop over the tags \n",
    "\n",
    "for i in YEAR:\n",
    "    IND_Release_Year.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "IND_Release_Year[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "250 250 250\n"
     ]
    }
   ],
   "source": [
    "# check the lenghth of the all 3 columns\n",
    "print(len(IND_Movie_Name),len(IND_IMDB_rating),len(IND_Release_Year))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IND_Movie_Name</th>\n",
       "      <th>IND_IMDB_rating</th>\n",
       "      <th>IND_Release_Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.       Pather Panchali (1955)</td>\n",
       "      <td>8.5</td>\n",
       "      <td>(1955)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.       Ratsasan (2018)</td>\n",
       "      <td>8.5</td>\n",
       "      <td>(2018)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.       Gol Maal (1979)</td>\n",
       "      <td>8.5</td>\n",
       "      <td>(1979)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.       Nayakan (1987)</td>\n",
       "      <td>8.5</td>\n",
       "      <td>(1987)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.       Anbe Sivam (2003)</td>\n",
       "      <td>8.5</td>\n",
       "      <td>(2003)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            IND_Movie_Name IND_IMDB_rating IND_Release_Year\n",
       "0         1.       Pather Panchali (1955)             8.5            (1955)\n",
       "1                2.       Ratsasan (2018)             8.5            (2018)\n",
       "2                3.       Gol Maal (1979)             8.5            (1979)\n",
       "3                 4.       Nayakan (1987)             8.5            (1987)\n",
       "4              5.       Anbe Sivam (2003)             8.5            (2003)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##### Create the dataframe \n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "IND_Movies_data = pd.DataFrame({})\n",
    "IND_Movies_data [\"IND_Movie_Name\"] = IND_Movie_Name\n",
    "IND_Movies_data[\"IND_IMDB_rating\"] = IND_IMDB_rating\n",
    "IND_Movies_data[\"IND_Release_Year\"] = IND_Release_Year\n",
    "\n",
    "\n",
    "IND_Movies_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "## clean the IND_Movie_Name Column\n",
    "\n",
    "## create the function to clean the data \n",
    "\n",
    "def clean(text):\n",
    "    text = ''.join([i for i in text if not i.isdigit()])\n",
    "    text =  text.replace(\"()\",\"\").replace(\".\",\"\")\n",
    "    return text\n",
    "IND_Movies_data[\"IND_Movie_Name\"]=IND_Movies_data[\"IND_Movie_Name\"].apply(clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IND_Movie_Name</th>\n",
       "      <th>IND_IMDB_rating</th>\n",
       "      <th>IND_Release_Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Pather Panchali</td>\n",
       "      <td>8.5</td>\n",
       "      <td>(1955)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ratsasan</td>\n",
       "      <td>8.5</td>\n",
       "      <td>(2018)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Gol Maal</td>\n",
       "      <td>8.5</td>\n",
       "      <td>(1979)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Nayakan</td>\n",
       "      <td>8.5</td>\n",
       "      <td>(1987)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Anbe Sivam</td>\n",
       "      <td>8.5</td>\n",
       "      <td>(2003)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    IND_Movie_Name IND_IMDB_rating IND_Release_Year\n",
       "0                Pather Panchali              8.5            (1955)\n",
       "1                       Ratsasan              8.5            (2018)\n",
       "2                       Gol Maal              8.5            (1979)\n",
       "3                        Nayakan              8.5            (1987)\n",
       "4                     Anbe Sivam              8.5            (2003)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "IND_Movies_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Question 4:-\n",
    "\n",
    "## Write a python program to scrap book name, author name, genre and book review of any 5 books from\n",
    "‘www.bookpage.com’"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "### ##  import the requried  libraries for web -scraping\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup as Soup\n",
    "import requests as rs\n",
    "\n",
    "## Lode the source code\n",
    "page_Book = rs.get(\"https://bookpage.com/reviews?book_genre=fiction&page=1\")\n",
    "\n",
    "## Now parse the source code bu BeautifulSoup and the parser used here is html parse as source code in html\n",
    "\n",
    "Book_soup = Soup(page_Book.content , \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create the emplty list \n",
    "\n",
    "books_name=[]\n",
    "author_name=[]\n",
    "genre=[]\n",
    "books_review=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' Summerwater ',\n",
       " \" The Children's Train \",\n",
       " ' What Could Be Saved ',\n",
       " '  ★ Black Buck ']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have books_name .\n",
    "\n",
    "Book = Book_soup.find_all(\"h4\", class_=\"italic\")\n",
    "Book[0:4]\n",
    "\n",
    "# now we extact the text form the these tegs by using for loop over the tags \n",
    "\n",
    "for i in Book :\n",
    "    books_name.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "books_name[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' Sarah Moss ',\n",
       " ' Viola Ardone, Clarissa Botsford ',\n",
       " \" Liese O'Halloran Schwarz \",\n",
       " ' Mateo Askaripour ']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Author .\n",
    "\n",
    "Author = Book_soup.find_all(\"p\", class_=\"sans bold\")\n",
    "Author[0:4]\n",
    "\n",
    "# now we extact the text form the these tegs by using for loop over the tags \n",
    "\n",
    "for i in Author :\n",
    "    author_name.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "author_name[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' Fiction  /  Literary Fiction ',\n",
       " ' Fiction  /  Historical Fiction ',\n",
       " ' Fiction  /  Family Saga ',\n",
       " ' Fiction  /  Satirical Fiction ']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have genre .\n",
    "\n",
    "genre1 = Book_soup.find_all(\"p\", class_=\"genre-links hidden-phone\")\n",
    "genre1[0:4]\n",
    "\n",
    "# now we extact the text form the these tegs by using for loop over the tags \n",
    "\n",
    "for i in genre1 :\n",
    "    genre.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "genre[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' Read the Review  ',\n",
       " ' Read the Review  ',\n",
       " ' Read the Review  ',\n",
       " ' Read the Review  ']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Review .\n",
    "\n",
    "Review = Book_soup.find_all(\"div\", class_=\"read-full\")\n",
    "Review[0:4]\n",
    "\n",
    "# now we extact the text form the these tegs by using for loop over the tags \n",
    "\n",
    "for i in Review :\n",
    "    books_review.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "books_review[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>books_name</th>\n",
       "      <th>author_name</th>\n",
       "      <th>genre</th>\n",
       "      <th>books_review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Summerwater</td>\n",
       "      <td>Sarah Moss</td>\n",
       "      <td>Fiction  /  Literary Fiction</td>\n",
       "      <td>Read the Review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Children's Train</td>\n",
       "      <td>Viola Ardone, Clarissa Botsford</td>\n",
       "      <td>Fiction  /  Historical Fiction</td>\n",
       "      <td>Read the Review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>What Could Be Saved</td>\n",
       "      <td>Liese O'Halloran Schwarz</td>\n",
       "      <td>Fiction  /  Family Saga</td>\n",
       "      <td>Read the Review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>★ Black Buck</td>\n",
       "      <td>Mateo Askaripour</td>\n",
       "      <td>Fiction  /  Satirical Fiction</td>\n",
       "      <td>Read the Review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Lost Manuscript</td>\n",
       "      <td>Cathy Bonidan, Emma Ramadan</td>\n",
       "      <td>Fiction  /  Popular Fiction</td>\n",
       "      <td>Read the Review</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               books_name                        author_name  \\\n",
       "0            Summerwater                         Sarah Moss    \n",
       "1   The Children's Train    Viola Ardone, Clarissa Botsford    \n",
       "2    What Could Be Saved           Liese O'Halloran Schwarz    \n",
       "3           ★ Black Buck                   Mateo Askaripour    \n",
       "4    The Lost Manuscript        Cathy Bonidan, Emma Ramadan    \n",
       "\n",
       "                              genre        books_review  \n",
       "0     Fiction  /  Literary Fiction    Read the Review    \n",
       "1   Fiction  /  Historical Fiction    Read the Review    \n",
       "2          Fiction  /  Family Saga    Read the Review    \n",
       "3    Fiction  /  Satirical Fiction    Read the Review    \n",
       "4      Fiction  /  Popular Fiction    Read the Review    "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##### Create the dataframe \n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "Book_data = pd.DataFrame({})\n",
    "Book_data [\"books_name\"] = books_name\n",
    "Book_data [\"author_name\"] = author_name\n",
    "Book_data[\"genre\"] = genre\n",
    "Book_data[\"books_review\"] = books_review\n",
    "\n",
    "\n",
    "Book_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Question 5:- \n",
    " \n",
    " ## Write a python program to scrape cricket rankings from ‘www.icc-cricket.com’. You have to scrape    \n",
    "\n",
    "\n",
    "## 1)-Top 10 ODI teams in men’s cricket along with the records for matches, points and rating."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  import the requried  libraries for web -scraping\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup as Soup\n",
    "import requests as rs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Lode the source code\n",
    "page_ODI = rs.get(\"https://www.icc-cricket.com/rankings/mens/team-rankings/odi\")\n",
    "page_ODI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Now parse the source code bu BeautifulSoup and the parser used here is html parse as source code in html\n",
    "\n",
    "ODI_soup = Soup(page_ODI.content , \"html.parser\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create 4 empty list\n",
    "\n",
    "Team = []\n",
    "Match = []\n",
    "Points = []\n",
    "Rating =  []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<span class=\"u-hide-phablet\">England</span>,\n",
       " <span class=\"u-hide-phablet\">India</span>,\n",
       " <span class=\"u-hide-phablet\">New Zealand</span>,\n",
       " <span class=\"u-hide-phablet\">Australia</span>]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Team name.\n",
    "\n",
    "M_Team = ODI_soup.find_all(\"span\", class_=\"u-hide-phablet\")\n",
    "M_Team[0:4]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    " \n",
    "for i in M_Team:\n",
    "    Team.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['England', 'India', 'New Zealand', 'Australia', 'South Africa']"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Team=Team[0:20]\n",
    "Team[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['52', '6,102', '32', '3,716']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Matchs .\n",
    "\n",
    "ODI_Match = ODI_soup.find_all(\"td\", class_=\"table-body__cell u-center-text\")\n",
    "ODI_Match[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    " \n",
    "for i in ODI_Match:\n",
    "    Match.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Match[0:4]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['52', '32', '39', '31', '35', '34', '39', '43', '28', '24', '5', '27', '12', '16', '9', '15', '9', '14', '14']\n"
     ]
    }
   ],
   "source": [
    "## we get both the data in on class so we clean the data / extect the (nb of match )  and Points\n",
    "Total_Match=[]\n",
    "i=0\n",
    "while i < len(Match):\n",
    "    Total_Match.append(Match[i])\n",
    "    i += 2\n",
    "        \n",
    "print(Total_Match)       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['6,102', '3,716', '4,344', '3,345', '3,490', '2,989', '3,297', '3,285', '1,549', '1,256', '222', '1,121', '479', '419', '161', '259', '152', '185', '0']\n"
     ]
    }
   ],
   "source": [
    "#extect the Points for the Match\n",
    "\n",
    "\n",
    "Points=[]\n",
    "i=1\n",
    "while i < len(Match):\n",
    "    Points.append(Match[i])\n",
    "    i += 2\n",
    "        \n",
    "print(Points) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<td class=\"table-body__cell u-text-right rating\">117</td>,\n",
       " <td class=\"table-body__cell u-text-right rating\">116</td>,\n",
       " <td class=\"table-body__cell u-text-right rating\">111</td>,\n",
       " <td class=\"table-body__cell u-text-right rating\">108</td>]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have jobs titles .\n",
    "\n",
    "ODI_Rating = ODI_soup.find_all(\"td\", class_=\"table-body__cell u-text-right rating\")\n",
    "ODI_Rating[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    " \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['117', '116', '111', '108']"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in ODI_Rating:\n",
    "    Rating.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Rating[0:4] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 19 19 19\n"
     ]
    }
   ],
   "source": [
    "## check the length of all 4 columns \n",
    "\n",
    "print(len(Team), len(Total_Match),len(Points),len(Rating))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "<class 'list'>\n",
      "<class 'list'>\n",
      "<class 'list'>\n"
     ]
    }
   ],
   "source": [
    "## check the Type of all 4 columns \n",
    "\n",
    "print(type(Team))\n",
    "print(type(Total_Match))\n",
    "print(type(Points))\n",
    "print(type(Rating))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "## now add the Englend Team Data manualy\n",
    "Total_Match.insert(0, '44')\n",
    "Points.insert(0, '5405')\n",
    "Rating.insert(0, '123')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 20 20 20\n"
     ]
    }
   ],
   "source": [
    "## check the again check the length of all 4 columns \n",
    "\n",
    "print(len(Team), len(Total_Match),len(Points),len(Rating))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "## creat a Dataframe \n",
    "import pandas as pd\n",
    "\n",
    "Top_10_Man = pd.DataFrame({})\n",
    "Top_10_Man[\"Team_name\"] = Team\n",
    "Top_10_Man[\"Total_Match\"] = Total_Match\n",
    "Top_10_Man[\"Points\"] = Points\n",
    "Top_10_Man[\"Rating\"] = Rating\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Team_name</th>\n",
       "      <th>Total_Match</th>\n",
       "      <th>Points</th>\n",
       "      <th>Rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>England</td>\n",
       "      <td>44</td>\n",
       "      <td>5405</td>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>India</td>\n",
       "      <td>52</td>\n",
       "      <td>6,102</td>\n",
       "      <td>117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>New Zealand</td>\n",
       "      <td>32</td>\n",
       "      <td>3,716</td>\n",
       "      <td>116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Australia</td>\n",
       "      <td>39</td>\n",
       "      <td>4,344</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>South Africa</td>\n",
       "      <td>31</td>\n",
       "      <td>3,345</td>\n",
       "      <td>108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Pakistan</td>\n",
       "      <td>35</td>\n",
       "      <td>3,490</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Bangladesh</td>\n",
       "      <td>34</td>\n",
       "      <td>2,989</td>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Sri Lanka</td>\n",
       "      <td>39</td>\n",
       "      <td>3,297</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>West Indies</td>\n",
       "      <td>43</td>\n",
       "      <td>3,285</td>\n",
       "      <td>76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>28</td>\n",
       "      <td>1,549</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Ireland</td>\n",
       "      <td>24</td>\n",
       "      <td>1,256</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Team_name Total_Match Points Rating\n",
       "0        England          44   5405    123\n",
       "1          India          52  6,102    117\n",
       "2    New Zealand          32  3,716    116\n",
       "3      Australia          39  4,344    111\n",
       "4   South Africa          31  3,345    108\n",
       "5       Pakistan          35  3,490    100\n",
       "6     Bangladesh          34  2,989     88\n",
       "7      Sri Lanka          39  3,297     85\n",
       "8    West Indies          43  3,285     76\n",
       "9    Afghanistan          28  1,549     55\n",
       "10       Ireland          24  1,256     52"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Top_10_Man.head(11)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Question 5:- \n",
    "\n",
    "## 2)-- Top 10 TEST Batsmen in men along with the records of their team and rating."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  import the requried  libraries for web -scraping\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup as Soup\n",
    "import requests as rs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Lode the source code\n",
    "page_ODI_Bat = rs.get(\"https://www.icc-cricket.com/rankings/mens/player-rankings/test/batting\")\n",
    "page_ODI_Bat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Now parse the source code bu BeautifulSoup and the parser used here is html parse as source code in html\n",
    "\n",
    "ODI_soup_Bat = Soup(page_ODI_Bat.content , \"html.parser\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create 3 empty list\n",
    "Batsmen_Name = []\n",
    "Team = []\n",
    "Rating =  []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<td class=\"table-body__cell rankings-table__name name\">\n",
       " <a href=\"/rankings/mens/player-rankings/271\">Steve Smith</a>\n",
       " </td>,\n",
       " <td class=\"table-body__cell rankings-table__name name\">\n",
       " <a href=\"/rankings/mens/player-rankings/164\">Virat Kohli</a>\n",
       " </td>,\n",
       " <td class=\"table-body__cell rankings-table__name name\">\n",
       " <a href=\"/rankings/mens/player-rankings/4029\">Marnus Labuschagne</a>\n",
       " </td>,\n",
       " <td class=\"table-body__cell rankings-table__name name\">\n",
       " <a href=\"/rankings/mens/player-rankings/2759\">Babar Azam</a>\n",
       " </td>]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Now extect all the tags where we have Batsmen_Name.\n",
    "\n",
    "Batsmen = ODI_soup_Bat.find_all(\"td\", class_ =\"table-body__cell rankings-table__name name\")\n",
    "Batsmen[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' Steve Smith ', ' Virat Kohli ', ' Marnus Labuschagne ']"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Extect the text for the tags \n",
    "\n",
    "for i in Batsmen:\n",
    "    Batsmen_Name.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "    \n",
    "Batsmen_Name[0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['AUS', 'IND', 'AUS', 'PAK']"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Team_Name .\n",
    "\n",
    "Team_Name = ODI_soup_Bat.find_all(\"span\", class_=\"table-body__logo-text\")\n",
    "Team_Name[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    " \n",
    "for i in Team_Name:\n",
    "    Team.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Team[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<td class=\"table-body__cell rating\">900</td>,\n",
       " <td class=\"table-body__cell rating\">870</td>,\n",
       " <td class=\"table-body__cell rating\">866</td>,\n",
       " <td class=\"table-body__cell rating\">781</td>]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Rating .\n",
    "\n",
    "Rating_Bat = ODI_soup_Bat.find_all(\"td\", class_=\"table-body__cell rating\")\n",
    "Rating_Bat[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['900', '870', '866', '781']"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in Rating_Bat:\n",
    "    Rating.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Rating[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99 99 99\n"
     ]
    }
   ],
   "source": [
    "## c check the length of all 3 columns \n",
    "\n",
    "print(len(Team), len(Batsmen_Name),len(Rating))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Add the top players records in the lists \n",
    "Batsmen_Name.insert(0 , 'Kane Williamson')\n",
    "Team.insert(0 ,'NZ')\n",
    "Rating.insert(0,'919')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 100 100\n"
     ]
    }
   ],
   "source": [
    "# now  again check the length of all columns \n",
    "\n",
    "print(len(Team), len(Batsmen_Name),len(Rating))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create the DataFrame \n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "Top_Batsmen_ODI = pd.DataFrame({})\n",
    "Top_Batsmen_ODI[\"Batsmen_Names\"]=Batsmen_Name\n",
    "Top_Batsmen_ODI[\"Team_Names\"]=Team\n",
    "Top_Batsmen_ODI[\"Rating\"]=Rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Batsmen_Names</th>\n",
       "      <th>Team_Names</th>\n",
       "      <th>Rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Kane Williamson</td>\n",
       "      <td>NZ</td>\n",
       "      <td>919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Steve Smith</td>\n",
       "      <td>AUS</td>\n",
       "      <td>900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Virat Kohli</td>\n",
       "      <td>IND</td>\n",
       "      <td>870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Marnus Labuschagne</td>\n",
       "      <td>AUS</td>\n",
       "      <td>866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Babar Azam</td>\n",
       "      <td>PAK</td>\n",
       "      <td>781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Ben Stokes</td>\n",
       "      <td>ENG</td>\n",
       "      <td>760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Ajinkya Rahane</td>\n",
       "      <td>IND</td>\n",
       "      <td>756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Cheteshwar Pujara</td>\n",
       "      <td>IND</td>\n",
       "      <td>753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Henry Nicholls</td>\n",
       "      <td>NZ</td>\n",
       "      <td>747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>David Warner</td>\n",
       "      <td>AUS</td>\n",
       "      <td>745</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Batsmen_Names Team_Names Rating\n",
       "0       Kane Williamson         NZ    919\n",
       "1          Steve Smith         AUS    900\n",
       "2          Virat Kohli         IND    870\n",
       "3   Marnus Labuschagne         AUS    866\n",
       "4           Babar Azam         PAK    781\n",
       "5           Ben Stokes         ENG    760\n",
       "6       Ajinkya Rahane         IND    756\n",
       "7    Cheteshwar Pujara         IND    753\n",
       "8       Henry Nicholls          NZ    747\n",
       "9         David Warner         AUS    745"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Top_Batsmen_ODI.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Question 5:-\n",
    "\n",
    " ## 3) Top 10 ODI bowlers along with the records of their team and rating.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  import the requried  libraries for web -scraping\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup as Soup\n",
    "import requests as rs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Lode the source code\n",
    "page_ODI_Bow = rs.get(\"https://www.icc-cricket.com/rankings/mens/player-rankings/odi/bowling\")\n",
    "page_ODI_Bow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Now parse the source code bu BeautifulSoup and the parser used here is html parse as source code in html\n",
    "\n",
    "ODI_soup_Bow = Soup(page_ODI_Bow.content , \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create 3 empty list\n",
    "Bowler_Name = []\n",
    "Teams = []\n",
    "Ratings =  []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' Mujeeb Ur Rahman ', ' Jasprit Bumrah ', ' Chris Woakes ']"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Now extect all the tags where we have Bowler_Name.\n",
    "\n",
    "Bowler = ODI_soup_Bow.find_all(\"td\", class_ =\"table-body__cell rankings-table__name name\")\n",
    "Bowler[0:4]\n",
    "### Extect the text for the tags \n",
    "\n",
    "for i in Bowler:\n",
    "    Bowler_Name.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "    \n",
    "Bowler_Name[0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['AFG', 'IND', 'ENG', 'SA']"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Team_Name .\n",
    "\n",
    "Team_Name_B = ODI_soup_Bow.find_all(\"span\", class_=\"table-body__logo-text\")\n",
    "Team_Name_B[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    " \n",
    "for i in Team_Name_B:\n",
    "    Teams.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Teams[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['701', '700', '675', '665']"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Rating .\n",
    "\n",
    "Rating_Bows = ODI_soup_Bow.find_all(\"td\", class_=\"table-body__cell rating\")\n",
    "Rating_Bows[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    "for i in Rating_Bows:\n",
    "    Ratings.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Ratings[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99 99 99\n"
     ]
    }
   ],
   "source": [
    "## c check the length of all 3 columns \n",
    "\n",
    "print(len(Teams), len(Bowler_Name),len(Ratings))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Add the top players records in the lists \n",
    "Bowler_Name.insert(0 , 'Trent Boult')\n",
    "Teams.insert(0 ,'NZ')\n",
    "Ratings.insert(0,'722')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 100 100\n"
     ]
    }
   ],
   "source": [
    "# now  again check the length of all columns \n",
    "\n",
    "print(len(Teams), len(Bowler_Name),len(Ratings))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Batsmen_Names</th>\n",
       "      <th>Team_Names</th>\n",
       "      <th>Rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Trent Boult</td>\n",
       "      <td>NZ</td>\n",
       "      <td>722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Mujeeb Ur Rahman</td>\n",
       "      <td>AFG</td>\n",
       "      <td>701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Jasprit Bumrah</td>\n",
       "      <td>IND</td>\n",
       "      <td>700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Chris Woakes</td>\n",
       "      <td>ENG</td>\n",
       "      <td>675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Kagiso Rabada</td>\n",
       "      <td>SA</td>\n",
       "      <td>665</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Batsmen_Names Team_Names Rating\n",
       "0         Trent Boult         NZ    722\n",
       "1   Mujeeb Ur Rahman         AFG    701\n",
       "2     Jasprit Bumrah         IND    700\n",
       "3       Chris Woakes         ENG    675\n",
       "4      Kagiso Rabada          SA    665"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## create the DataFrame \n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "Top_Bowler_ODI = pd.DataFrame({})\n",
    "Top_Bowler_ODI[\"Batsmen_Names\"]=Bowler_Name\n",
    "Top_Bowler_ODI[\"Team_Names\"]=Teams\n",
    "Top_Bowler_ODI[\"Rating\"]=Ratings\n",
    "\n",
    "Top_Bowler_ODI.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Question :-6\n",
    "\n",
    "## Write a python program to scrape cricket rankings from ‘www.icc-cricket.com’. You have to scrape:\n",
    "\n",
    "   ## 1) Top 10 ODI teams in women’s cricket along with the records for matches, points and rating."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  import the requried  libraries for web -scraping\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup as Soup\n",
    "import requests as rs\n",
    "\n",
    "## Lode the source code\n",
    "page_ODI_W = rs.get(\"https://www.icc-cricket.com/rankings/womens/team-rankings/odi\")\n",
    "\n",
    "\n",
    "## Now parse the source code bu BeautifulSoup and the parser used here is html parse as source code in html\n",
    "\n",
    "ODI_soup_W = Soup(page_ODI_W.content , \"html.parser\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create 4 empty list\n",
    "\n",
    "Team = []\n",
    "Match = []\n",
    "Points = []\n",
    "Rating =  []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Now extect all the tags where we have Team name.\n",
    "\n",
    "W_Team = ODI_soup_W.find_all(\"span\", class_=\"u-hide-phablet\")\n",
    "W_Team[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    " \n",
    "for i in W_Team:\n",
    "    Team.append(i.get_text().replace(\"\\n\",\" \"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Australia',\n",
       " 'India',\n",
       " 'England',\n",
       " 'South Africa',\n",
       " 'New Zealand',\n",
       " 'West Indies',\n",
       " 'Pakistan',\n",
       " 'Bangladesh',\n",
       " 'Sri Lanka',\n",
       " 'Ireland']"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## take only top 10 teams\n",
    "\n",
    "Team=Team[0:10]\n",
    "Team[0:11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['15', '1,812', '14', '1,670']"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Matchs .\n",
    "\n",
    "ODI_Match_W = ODI_soup_W.find_all(\"td\", class_=\"table-body__cell u-center-text\")\n",
    "ODI_Match_W[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    " \n",
    "for i in ODI_Match_W:\n",
    "    Match.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Match[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['15', '14', '16', '15', '12', '12', '5', '11', '2']\n"
     ]
    }
   ],
   "source": [
    "## we get both the data in on class so we clean the data / extect the (nb of match )  and Points\n",
    "Total_Match=[]\n",
    "i=0\n",
    "while i < len(Match):\n",
    "    Total_Match.append(Match[i])\n",
    "    i += 2\n",
    "        \n",
    "print(Total_Match) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['1,812', '1,670', '1,713', '1,384', '1,025', '927', '306', '519', '25']\n"
     ]
    }
   ],
   "source": [
    "#extect the Points for the Match\n",
    "\n",
    "Points=[]\n",
    "i=1\n",
    "while i < len(Match):\n",
    "    Points.append(Match[i])\n",
    "    i += 2\n",
    "        \n",
    "print(Points) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['121', '119', '107', '92']"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Ratings titles .\n",
    "\n",
    "ODI_Rating_W = ODI_soup_W.find_all(\"td\", class_=\"table-body__cell u-text-right rating\")\n",
    "ODI_Rating_W[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    " \n",
    "for i in ODI_Rating_W:\n",
    "    Rating.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Rating[0:4] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 9 9 9\n"
     ]
    }
   ],
   "source": [
    "## check the again check the length of all 4 columns \n",
    "\n",
    "print(len(Team), len(Total_Match),len(Points),len(Rating))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "## now add the Englend Team Data manualy\n",
    "Total_Match.insert(0, '15')\n",
    "Points.insert(0, '2436')\n",
    "Rating.insert(0, '163')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "## creat a Dataframe \n",
    "import pandas as pd\n",
    "\n",
    "Top_10_Woman = pd.DataFrame({})\n",
    "Top_10_Woman[\"Team_name\"] = Team\n",
    "Top_10_Woman[\"Total_Match\"] = Total_Match\n",
    "Top_10_Woman[\"Points\"] = Points\n",
    "Top_10_Woman[\"Rating\"] = Rating\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Team_name</th>\n",
       "      <th>Total_Match</th>\n",
       "      <th>Points</th>\n",
       "      <th>Rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Australia</td>\n",
       "      <td>15</td>\n",
       "      <td>2436</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>India</td>\n",
       "      <td>15</td>\n",
       "      <td>1,812</td>\n",
       "      <td>121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>England</td>\n",
       "      <td>14</td>\n",
       "      <td>1,670</td>\n",
       "      <td>119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>South Africa</td>\n",
       "      <td>16</td>\n",
       "      <td>1,713</td>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>New Zealand</td>\n",
       "      <td>15</td>\n",
       "      <td>1,384</td>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>West Indies</td>\n",
       "      <td>12</td>\n",
       "      <td>1,025</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Pakistan</td>\n",
       "      <td>12</td>\n",
       "      <td>927</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Bangladesh</td>\n",
       "      <td>5</td>\n",
       "      <td>306</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Sri Lanka</td>\n",
       "      <td>11</td>\n",
       "      <td>519</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Ireland</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Team_name Total_Match Points Rating\n",
       "0     Australia          15   2436    163\n",
       "1         India          15  1,812    121\n",
       "2       England          14  1,670    119\n",
       "3  South Africa          16  1,713    107\n",
       "4   New Zealand          15  1,384     92\n",
       "5   West Indies          12  1,025     85\n",
       "6      Pakistan          12    927     77\n",
       "7    Bangladesh           5    306     61\n",
       "8     Sri Lanka          11    519     47\n",
       "9       Ireland           2     25     13"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Top_10_Woman"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Question :-6\n",
    "\n",
    "## Write a python program to scrape cricket rankings from ‘www.icc-cricket.com’. You have to scrape:\n",
    "\n",
    "  ## 2) Top 10 women’s ODI players along with the records of their team and rating."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  import the requried  libraries for web -scraping\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup as Soup\n",
    "import requests as rs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Lode the source code\n",
    "page_ODI_Bat_w = rs.get(\"https://www.icc-cricket.com/rankings/womens/player-rankings/odi/batting\")\n",
    "page_ODI_Bat_w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Now parse the source code bu BeautifulSoup and the parser used here is html parse as source code in html\n",
    "\n",
    "ODI_soup_Bat_w = Soup(page_ODI_Bat_w.content , \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create 3 empty list\n",
    "Player_Name = []\n",
    "Team_W = []\n",
    "Ratings_W =  []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' Stafanie Taylor ', ' Alyssa Healy ', ' Smriti Mandhana ']"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Now extect all the tags where we have Batsmen_Name.\n",
    "\n",
    "player_W = ODI_soup_Bat_w.find_all(\"td\", class_ =\"table-body__cell rankings-table__name name\")\n",
    "player_W[0:4]\n",
    "### Extect the text for the tags \n",
    "\n",
    "for i in player_W:\n",
    "    Player_Name.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "    \n",
    "Player_Name[0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['WI', 'AUS', 'IND', 'NZ']"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Team_Name .\n",
    "\n",
    "Team_Name_W = ODI_soup_Bat_w.find_all(\"span\", class_=\"table-body__logo-text\")\n",
    "Team_Name_W[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    " \n",
    "for i in Team_Name_W:\n",
    "    Team_W.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Team_W[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['746', '741', '732', '723']"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Rating .\n",
    "\n",
    "Rating_Bat_w = ODI_soup_Bat_w.find_all(\"td\", class_=\"table-body__cell rating\")\n",
    "Rating_Bat_w[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    "for i in Rating_Bat_w:\n",
    "    Ratings_W.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Ratings_W[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99 99 99\n"
     ]
    }
   ],
   "source": [
    "##  check the length of all 3 columns \n",
    "\n",
    "print(len(Team_W), len(Player_Name),len(Ratings_W))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Add the top players records in the lists \n",
    "Player_Name.insert(0 , 'Meg Lanning')\n",
    "Team_W.insert(0 ,'AUS')\n",
    "Ratings_W.insert(0,'749')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 100 100\n"
     ]
    }
   ],
   "source": [
    "##  again check the length of all 3 columns \n",
    "\n",
    "print(len(Team_W), len(Player_Name),len(Ratings_W))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Batsmen_Names</th>\n",
       "      <th>Team_Names</th>\n",
       "      <th>Rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Meg Lanning</td>\n",
       "      <td>AUS</td>\n",
       "      <td>749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Stafanie Taylor</td>\n",
       "      <td>WI</td>\n",
       "      <td>746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Alyssa Healy</td>\n",
       "      <td>AUS</td>\n",
       "      <td>741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Smriti Mandhana</td>\n",
       "      <td>IND</td>\n",
       "      <td>732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Amy Satterthwaite</td>\n",
       "      <td>NZ</td>\n",
       "      <td>723</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Batsmen_Names Team_Names Rating\n",
       "0          Meg Lanning        AUS    749\n",
       "1     Stafanie Taylor          WI    746\n",
       "2        Alyssa Healy         AUS    741\n",
       "3     Smriti Mandhana         IND    732\n",
       "4   Amy Satterthwaite          NZ    723"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## create the DataFrame \n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "Top_player_ODI_W = pd.DataFrame({})\n",
    "Top_player_ODI_W[\"Batsmen_Names\"]=Player_Name\n",
    "Top_player_ODI_W[\"Team_Names\"]=Team_W\n",
    "Top_player_ODI_W[\"Rating\"]=Ratings_W\n",
    "\n",
    "Top_player_ODI_W.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Qustion 6:- \n",
    "\n",
    "## 3) Top 10 women’s ODI all-rounder along with the records of their team and rating."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  import the requried  libraries for web -scraping\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup as Soup\n",
    "import requests as rs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Lode the source code\n",
    "page_ODI_ALL = rs.get(\"https://www.icc-cricket.com/rankings/womens/player-rankings/odi/all-rounder\")\n",
    "page_ODI_ALL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Now parse the source code bu BeautifulSoup and the parser used here is html parse as source code in html\n",
    "\n",
    "ODI_soup_ALL = Soup(page_ODI_ALL.content , \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create 3 empty list\n",
    "All_Rounder_Name = []\n",
    "Team_All = []\n",
    "Rating_All =  []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' Stafanie Taylor ', ' Marizanne Kapp ', ' Deepti Sharma ']"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Now extect all the tags where we have All_Rounder_Name.\n",
    "\n",
    "All_Rounder = ODI_soup_ALL.find_all(\"td\", class_ =\"table-body__cell rankings-table__name name\")\n",
    "All_Rounder[0:4]\n",
    "### Extect the text for the tags \n",
    "\n",
    "for i in All_Rounder:\n",
    "    All_Rounder_Name.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "    \n",
    "All_Rounder_Name[0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['WI', 'SA', 'IND', 'SA']"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Team_Name .\n",
    "\n",
    "Team_Name_All = ODI_soup_ALL.find_all(\"span\", class_=\"table-body__logo-text\")\n",
    "Team_Name_All[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    " \n",
    "for i in Team_Name_All:\n",
    "    Team_All.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Team_All[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['WI', 'SA', 'IND', 'SA']"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Team_Name .\n",
    "\n",
    "Team_Name_All = ODI_soup_ALL.find_all(\"span\", class_=\"table-body__logo-text\")\n",
    "Team_Name_All[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    " \n",
    "for i in Team_Name_All:\n",
    "    Team_All.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Team_All[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['410', '389', '359', '335']"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Now extect all the tags where we have Rating .\n",
    "\n",
    "Rating_All_W = ODI_soup_ALL.find_all(\"td\", class_=\"table-body__cell rating\")\n",
    "Rating_All_W[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    "for i in Rating_All_W:\n",
    "    Rating_All.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Rating_All[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38 19 19\n"
     ]
    }
   ],
   "source": [
    "##  check the length of all 3 columns \n",
    "\n",
    "print(len(Team_All), len(All_Rounder_Name),len(Rating_All))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Add the top players records in the lists \n",
    "All_Rounder_Name.insert(0 , 'Ellyse Perry')\n",
    "Team_All.insert(0 ,'AUS')\n",
    "Rating_All.insert(0,'460')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39 20 20\n"
     ]
    }
   ],
   "source": [
    "## again check the length of all 3 columns \n",
    "\n",
    "print(len(Team_All), len(All_Rounder_Name),len(Rating_All))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>All_Rounder_Name</th>\n",
       "      <th>Team_Names</th>\n",
       "      <th>Rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ellyse Perry</td>\n",
       "      <td>AUS</td>\n",
       "      <td>460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Stafanie Taylor</td>\n",
       "      <td>WI</td>\n",
       "      <td>410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Marizanne Kapp</td>\n",
       "      <td>SA</td>\n",
       "      <td>389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Deepti Sharma</td>\n",
       "      <td>IND</td>\n",
       "      <td>359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Dane van Niekerk</td>\n",
       "      <td>SA</td>\n",
       "      <td>335</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     All_Rounder_Name Team_Names Rating\n",
       "0        Ellyse Perry        AUS    460\n",
       "1    Stafanie Taylor          WI    410\n",
       "2     Marizanne Kapp          SA    389\n",
       "3      Deepti Sharma         IND    359\n",
       "4   Dane van Niekerk          SA    335"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## create the DataFrame \n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "All_ODI_W = pd.DataFrame({})\n",
    "All_ODI_W[\"All_Rounder_Name\"]=All_Rounder_Name\n",
    "All_ODI_W[\"Team_Names\"]=Team_All[0:20]\n",
    "All_ODI_W[\"Rating\"]=Rating_All\n",
    "\n",
    "All_ODI_W.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Question 7:-\n",
    "\n",
    "## Write a python program to scrape details of all the mobile phones under Rs. 20,000 listed on Amazon.in. The scraped data should include Product Name, Price, Image URL and Average Rating."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  import the requried  libraries for web -scraping\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup as Soup\n",
    "import requests as rs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "HEADERS = ({'User-Agent':\n",
    "            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36',\n",
    "            'Accept-Language': 'en-US, en;q=0.5'})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n"
     ]
    }
   ],
   "source": [
    "## Lode the source code\n",
    "url = \"https://www.amazon.in/s?k=mobile+under+20000&rh=p_36%3A500000-2000000&crid=3KJ0WEF6R63UF&qid=1610700723&rnid=1318502031&sprefix=mobi%2Caps%2C486&ref=sr_nr_p_36_4\"\n",
    "page_Amz = rs.get(url , headers=HEADERS)\n",
    "print(page_Amz)\n",
    "## Now parse the source code bu BeautifulSoup and the parser used here is html parse as source code in html\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html class=\"a-no-js\" data-19ax5a9jf=\"dingo\" lang=\"en-in\"><!-- sp:feature:head-start -->\n",
      "<head><script>var aPageStart = (new Date()).getTime();</script><meta charset=\"utf-8\"/>\n",
      "<script type=\"text/javascript\">var ue_t0=ue_t0||+new Date();</script>\n",
      "<!-- sp:feature:cs-optimization -->\n",
      "<meta content=\"on\" http-equiv=\"x-dns-prefetch-control\"/>\n",
      "<link href=\"https://images-eu.ssl-images-amazon.com\" rel=\"dns-prefetch\"/>\n",
      "<link href=\"https://m.media-amazon.com\" rel=\"dns-prefetch\"/>\n",
      "<link href=\"https://completion.amazon.com\" rel=\"dns-prefetch\"/>\n",
      "<script type=\"text/javascript\">\n",
      "window.ue_ihb = (window.ue_ihb || window.ueinit || 0) + 1;\n",
      "if (window.ue_ihb === 1) {\n",
      "\n",
      "var ue_csm = window,\n",
      "    ue_hob = +new Date();\n",
      "(function(d){var e=d.ue=d.ue||{},f=Date.now||function(){return+new Date};e.d=function(b){return f()-(b?0:d.ue_t0)};e.stub=function(b,a){if(!b[a]){var c=[];b[a]=function(){c.push([c.slice.call(arguments),e.d(),d.ue_id])};b[a].replay=function(b){for(var a;a=c.shift();)b(a[0],a[1],a[2])};b[a].isStub=1}};e.exec=function(b,a){return function(){try{return b.apply(this,arguments)}catch(c){ueLogError(c,{attribution:a||\"undefined\",logLevel:\"WARN\"})}}}})(ue_csm);\n",
      "\n",
      "\n",
      "    var ue_err_chan = 'jserr-rw';\n",
      "(function(d,e){function h(f,b){if(!(a.ec>a.mxe)&&f){a.ter.push(f);b=b||{};var c=f.logLevel||b.logLevel;c&&c!==k&&c!==m&&c!==n&&c!==p||a.ec++;c&&c!=k||a.ecf++;b.pageURL=\"\"+(e.location?e.location.href:\"\");b.logLevel=c;b.attribution=f.attribution||b.attribution;a.erl.push({ex:f,info:b})}}function l(a,b,c,e,g){d.ueLogError({m:a,f:b,l:c,c:\"\"+e,err:g,fromOnError:1,args:arguments},g?{attribution:g.attribution,logLevel:g.logLevel}:void 0);return!1}var k=\"FATAL\",m=\"ERROR\",n=\"WARN\",p=\"DOWNGRADED\",a={ec:0,ecf:0,\n",
      "pec:0,ts:0,erl:[],ter:[],mxe:50,startTimer:function(){a.ts++;setInterval(function(){d.ue&&a.pec<a.ec&&d.uex(\"at\");a.pec=a.ec},1E4)}};l.skipTrace=1;h.skipTrace=1;h.isStub=1;d.ueLogError=h;d.ue_err=a;e.onerror=l})(ue_csm,window);\n",
      "\n",
      "\n",
      "var ue_id = 'ZR1E8BCAN3KR351B6FBC',\n",
      "    ue_url = '/rd/uedata',\n",
      "    ue_navtiming = 1,\n",
      "    ue_mid = 'A21TJRUUN4KGV',\n",
      "    ue_sid = '258-8217098-9409765',\n",
      "    ue_sn = 'www.amazon.in',\n",
      "    ue_furl = 'fls-eu.amazon.in',\n",
      "    ue_surl = 'https://unagi-eu.amazon.com/1/events/com.amazon.csm.nexusclient.prod',\n",
      "    ue_int = 0,\n",
      "    ue_fcsn = 1,\n",
      "    ue_urt = 3,\n",
      "    ue_rpl_ns = 'cel-rpl',\n",
      "    ue_ddq = 1,\n",
      "    ue_fpf = '//fls-eu.amazon.in/1/batch/1/OP/A21TJRUUN4KGV:258-8217098-9409765:ZR1E8BCAN3KR351B6FBC$uedata=s:',\n",
      "    ue_sbuimp = 1,\n",
      "    ue_bfd = 400,\n",
      "    ue_fnt = 0,\n",
      "\n",
      "    ue_swi = 1;\n",
      "var ue_viz=function(){(function(c,e,a){function k(b){if(c.ue.viz.length<p&&!l){var a=b.type;b=b.originalEvent;/^focus./.test(a)&&b&&(b.toElement||b.fromElement||b.relatedTarget)||(a=e[m]||(\"blur\"==a||\"focusout\"==a?\"hidden\":\"visible\"),c.ue.viz.push(a+\":\"+(+new Date-c.ue.t0)),\"visible\"==a&&(ue.isl&&uex(\"at\"),l=1))}}for(var l=0,f,g,m,n=[\"\",\"webkit\",\"o\",\"ms\",\"moz\"],d=0,p=20,h=0;h<n.length&&!d;h++)if(a=n[h],f=(a?a+\"H\":\"h\")+\"idden\",d=\"boolean\"==typeof e[f])g=a+\"visibilitychange\",m=(a?a+\"V\":\"v\")+\"isibilityState\";\n",
      "k({});d&&e.addEventListener(g,k,0);c.ue&&d&&(c.ue.pageViz={event:g,propHid:f})})(ue_csm,document,window)};\n",
      "\n",
      "(function(d,k,L){function F(a){return a&&a.replace&&a.replace(/^\\s+|\\s+$/g,\"\")}function t(a){return\"undefined\"===typeof a}function B(a,b){for(var c in b)b[u](c)&&(a[c]=b[c])}function G(a){try{var b=L.cookie.match(RegExp(\"(^| )\"+a+\"=([^;]+)\"));if(b)return b[2].trim()}catch(c){}}function M(n,b,c){var e=(v||{}).type;2!==e&&1!==e&&(n&&(d.ue_id=a.id=a.rid=n,w=w.replace(/((.*?:){2})(\\w+)/,function(a,b){return b+n})),b&&(w=w.replace(/(.*?:)(\\w|-)+/,function(a,c){return c+b}),d.ue_sid=b),c&&a.tag(\"page-source:\"+\n",
      "c),d.ue_fpf=w)}function N(){var a={};return function(b){b&&(a[b]=1);b=[];for(var c in a)a[u](c)&&b.push(c);return b}}function x(d,b,c,e){e=e||+new C;var h,p;if(b||t(c)){if(d)for(p in h=b?g(\"t\",b)||g(\"t\",b,{}):a.t,h[d]=e,c)c[u](p)&&g(p,b,c[p]);return e}}function g(d,b,c){var e=b&&b!=a.id?a.sc[b]:a;e||(e=a.sc[b]={});\"id\"===d&&c&&(O=1);return e[d]=c||e[d]}function P(d,b,c,e,h){c=\"on\"+c;var g=b[c];\"function\"===typeof g?d&&(a.h[d]=g):g=function(){};b[c]=function(a){h?(e(a),g(a)):(g(a),e(a))};b[c]&&(b[c].isUeh=\n",
      "1)}function Q(n,b,c,e){function r(b,c){var d=[b],e=0,f={},h,k;c?(d.push(\"m=1\"),f[c]=1):f=a.sc;for(k in f)if(f[u](k)){var r=g(\"wb\",k),l=g(\"t\",k)||{},p=g(\"t0\",k)||a.t0,m;if(c||2==r){r=r?e++:\"\";d.push(\"sc\"+r+\"=\"+k);for(m in l)3>=m.length&&!t(l[m])&&null!==l[m]&&d.push(m+r+\"=\"+(l[m]-p));d.push(\"t\"+r+\"=\"+l[n]);if(g(\"ctb\",k)||g(\"wb\",k))h=1}}!H&&h&&d.push(\"ctb=1\");return d.join(\"&\")}function p(b,c,f,e){if(b){var g=d.ue_err;d.ue_url&&!e&&b&&0<b.length&&(e=new Image,a.iel.push(e),e.src=b,a.count&&a.count(\"postbackImageSize\",\n",
      "b.length));if(w){var h=k.encodeURIComponent;h&&b&&(e=new Image,b=\"\"+d.ue_fpf+h(b)+\":\"+(+new C-d.ue_t0),a.iel.push(e),e.src=b)}else a.log&&(a.log(b,\"uedata\",{n:1}),a.ielf.push(b));g&&!g.ts&&g.startTimer();a.b&&(g=a.b,a.b=\"\",p(g,c,f,1))}}function z(b){var c=v?v.type:D,d=2==c||a.isBFonMshop,c=c&&!d,e=a.bfini;O||(e&&1<e&&(b+=\"&bfform=1\",c||(a.isBFT=e-1)),d&&(b+=\"&bfnt=1\",a.isBFT=a.isBFT||1),a.ssw&&a.isBFT&&(a.isBFonMshop&&(a.isNRBF=0),t(a.isNRBF)&&(d=a.ssw(a.oid),d.e||t(d.val)||(a.isNRBF=1<d.val?0:1)),\n",
      "t(a.isNRBF)||(b+=\"&nrbf=\"+a.isNRBF)),a.isBFT&&!a.isNRBF&&(b+=\"&bft=\"+a.isBFT));return b}if(!a.paused&&(b||t(c))){for(var m in c)c[u](m)&&g(m,b,c[m]);a.isBFonMshop||x(\"pc\",b,c);m=g(\"id\",b)||a.id;var s=g(\"id2\",b),f=a.url+\"?\"+n+\"&v=\"+a.v+\"&id=\"+m,H=g(\"ctb\",b)||g(\"wb\",b),y;H&&(f+=\"&ctb=\"+H);s&&(f+=\"&id2=\"+s);1<d.ueinit&&(f+=\"&ic=\"+d.ueinit);if(!(\"ld\"!=n&&\"ul\"!=n||b&&b!=m)){if(\"ld\"==n){try{k[I]&&k[I].isUeh&&(k[I]=null)}catch(G){}if(k.chrome)for(s=0;s<J.length;s++)R(E,J[s]);(s=L.ue_backdetect)&&s.ue_back&&\n",
      "s.ue_back.value++;d._uess&&(y=d._uess());a.isl=1}a._bf&&(f+=\"&bf=\"+a._bf());d.ue_navtiming&&h&&(g(\"ctb\",m,\"1\"),a.isBFonMshop||x(\"tc\",D,D,K));!A||a.isBFonMshop||S||(h&&B(a.t,{na_:h.navigationStart,ul_:h.unloadEventStart,_ul:h.unloadEventEnd,rd_:h.redirectStart,_rd:h.redirectEnd,fe_:h.fetchStart,lk_:h.domainLookupStart,_lk:h.domainLookupEnd,co_:h.connectStart,_co:h.connectEnd,sc_:h.secureConnectionStart,rq_:h.requestStart,rs_:h.responseStart,_rs:h.responseEnd,dl_:h.domLoading,di_:h.domInteractive,de_:h.domContentLoadedEventStart,\n",
      "_de:h.domContentLoadedEventEnd,_dc:h.domComplete,ld_:h.loadEventStart,_ld:h.loadEventEnd,ntd:(\"function\"!==typeof A.now||t(K)?0:new C(K+A.now())-new C)+a.t0}),v&&B(a.t,{ty:v.type+a.t0,rc:v.redirectCount+a.t0}),S=1);a.isBFonMshop||B(a.t,{hob:d.ue_hob,hoe:d.ue_hoe});a.ifr&&(f+=\"&ifr=1\")}x(n,b,c,e);c=\"ld\"==n&&b&&g(\"wb\",b);var q,l;c||b&&b!==m||$(b);c||m==a.oid||aa(m,(g(\"t\",b)||{}).tc||+g(\"t0\",b),+g(\"t0\",b));(e=d.ue_mbl)&&e.cnt&&!c&&(f+=e.cnt());c?g(\"wb\",b,2):\"ld\"==n&&(a.lid=F(m));for(q in a.sc)if(1==\n",
      "g(\"wb\",q))break;if(c){if(a.s)return;f=r(f,null)}else e=r(f,null),e!=f&&(e=z(e),a.b=e),y&&(f+=y),f=r(f,b||a.id);f=z(f);if(a.b||c)for(q in a.sc)2==g(\"wb\",q)&&delete a.sc[q];y=0;a._rt&&(f+=\"&rt=\"+a._rt());e=k.csa;if(!c&&e)for(l in q=g(\"t\",b)||{},e=e(\"PageTiming\"),q)q[u](l)&&e(\"mark\",ba[l]||l,q[l]);c||(a.s=0,(l=d.ue_err)&&0<l.ec&&l.pec<l.ec&&(l.pec=l.ec,f+=\"&ec=\"+l.ec+\"&ecf=\"+l.ecf),y=g(\"ctb\",b),\"ld\"!==n||b||a.markers||(a.markers={},B(a.markers,g(\"t\",b))),g(\"t\",b,{}));a.tag&&a.tag().length&&(f+=\"&csmtags=\"+\n",
      "a.tag().join(\"|\"),a.tag=N());l=a.viz||[];(q=l.length)&&(f+=\"&viz=\"+l.splice(0,q).join(\"|\"));t(d.ue_pty)||(f+=\"&pty=\"+d.ue_pty+\"&spty=\"+d.ue_spty+\"&pti=\"+d.ue_pti);a.tabid&&(f+=\"&tid=\"+a.tabid);a.aftb&&(f+=\"&aftb=1\");!a._ui||b&&b!=m||(f+=a._ui());a.a=f;p(f,n,y,c)}}function $(a){var b=k.ue_csm_markers||{},c;for(c in b)b[u](c)&&x(c,a,D,b[c])}function z(a,b,c){c=c||k;if(c[T])c[T](a,b,!1);else if(c[U])c[U](\"on\"+a,b)}function R(a,b,c){c=c||k;if(c[V])c[V](a,b,!1);else if(c[W])c[W](\"on\"+a,b)}function X(){function a(){d.onUl()}\n",
      "function b(a){return function(){c[a]||(c[a]=1,Q(a))}}var c={},e,g;d.onLd=b(\"ld\");d.onLdEnd=b(\"ld\");d.onUl=b(\"ul\");e={stop:b(\"os\")};k.chrome?(z(E,a),J.push(a)):e[E]=d.onUl;for(g in e)e[u](g)&&P(0,k,g,e[g]);d.ue_viz&&ue_viz();z(\"load\",d.onLd);x(\"ue\")}function aa(g,b,c){var e=d.ue_mbl,h=k.csa,p=h&&h(\"SPA\"),h=h&&h(\"PageTiming\");e&&e.ajax&&e.ajax(b,c);p&&h&&(p(\"newPage\",{requestId:g,transitionType:\"soft\"}),h(\"mark\",\"transitionStart\",b));a.tag(\"ajax-transition\")}d.ueinit=(d.ueinit||0)+1;var a=d.ue=d.ue||\n",
      "{};a.t0=k.aPageStart||d.ue_t0;a.id=d.ue_id;a.url=d.ue_url;a.rid=d.ue_id;a.a=\"\";a.b=\"\";a.h={};a.s=1;a.t={};a.sc={};a.iel=[];a.ielf=[];a.viz=[];a.v=\"0.215226.0\";a.paused=!1;var u=\"hasOwnProperty\",E=\"beforeunload\",I=\"on\"+E,T=\"addEventListener\",V=\"removeEventListener\",U=\"attachEvent\",W=\"detachEvent\",ba={cf:\"criticalFeature\",af:\"aboveTheFold\",fn:\"functional\",fp:\"firstPaint\",fcp:\"firstContentfulPaint\",bb:\"bodyBegin\",be:\"bodyEnd\",ld:\"loaded\"},C=k.Date,A=k.performance||k.webkitPerformance,h=(A||{}).timing,\n",
      "v=(A||{}).navigation,K=(h||{}).navigationStart,w=d.ue_fpf,O=0,S=0,J=[],D;a.oid=F(a.id);a.lid=F(a.id);a._t0=a.t0;a.tag=N();a.ifr=k.top!==k.self||k.frameElement?1:0;a.markers=null;a.attach=z;a.detach=R;if(\"000-0000000-8675309\"===d.ue_sid){var Y=G(\"cdn-rid\"),Z=G(\"session-id\");Y&&Z&&M(Y,Z,\"cdn\")}d.uei=X;d.ueh=P;d.ues=g;d.uet=x;d.uex=Q;a.reset=M;a.pause=function(d){a.paused=d};X()})(ue_csm,window,ue_csm.document);\n",
      "\n",
      "\n",
      "ue.stub(ue,\"event\");ue.stub(ue,\"onSushiUnload\");ue.stub(ue,\"onSushiFlush\");\n",
      "\n",
      "ue.stub(ue,\"log\");ue.stub(ue,\"onunload\");ue.stub(ue,\"onflush\");\n",
      "(function(c){var a=c.ue;a.cv={};a.cv.scopes={};a.count=function(d,c,b){var e={},f=a.cv,g=b&&0===b.c;e.counter=d;e.value=c;e.t=a.d();b&&b.scope&&(f=a.cv.scopes[b.scope]=a.cv.scopes[b.scope]||{},e.scope=b.scope);if(void 0===c)return f[d];f[d]=c;d=0;b&&b.bf&&(d=1);ue_csm.ue_sclog||!a.clog||0!==d||g?a.log&&a.log(e,\"csmcount\",{c:1,bf:d}):a.clog(e,\"csmcount\",{bf:d})};a.count(\"baselineCounter2\",1);a&&a.event&&(a.event({requestId:c.ue_id||\"rid\",server:c.ue_sn||\"sn\",obfuscatedMarketplaceId:c.ue_mid||\"mid\"},\n",
      "\"csm\",\"csm.CSMBaselineEvent.4\"),a.count(\"nexusBaselineCounter\",1,{bf:1}))})(ue_csm);\n",
      "\n",
      "\n",
      "\n",
      "var ue_hoe = +new Date();\n",
      "}\n",
      "window.ueinit = window.ue_ihb;\n",
      "</script>\n",
      "<!-- 3"
     ]
    },
    {
     "data": {
      "text/html": [
       "<b>limit_output extension: Maximum message size of 10000 exceeded with 626372 characters</b>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Now parse the source code bu BeautifulSoup and the parser\n",
    "Amz_soup = Soup(page_Amz.content , \"html.parser\" )\n",
    "print(Amz_soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create 3 empty list\n",
    "Moblie_Name = []\n",
    "Price = []\n",
    "Avg_Rating =  []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Vivo Y51 (Titanium Sapphire, 8GB RAM, 128GB ROM) with No Cost EMI/Additional Exchange Offers',\n",
       " 'R Realme Narzo 20 (Victory Blue, 4GB+128GB)',\n",
       " 'Samsung Galaxy M31s (Mirage Blue, 6GB RAM, 128GB Storage)',\n",
       " 'Samsung Galaxy M31 (Ocean Blue, 6GB RAM, 128GB Storage)']"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Outer Tag Object\n",
    "Mobile = Amz_soup.find_all(\"span\", attrs={\"class\":\"a-size-medium a-color-base a-text-normal\",\"dir\":\"auto\"})\n",
    "Mobile[0:4]\n",
    "\n",
    "#now we extact the text form the these tegs by using for loop over the tags \n",
    "for i in Mobile:\n",
    "    Moblie_Name.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Moblie_Name[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Moblie_Name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['17,990', '12,699', '17,999', '15,999']"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find the price of the phone :-\n",
    "Money = Amz_soup.find_all(\"span\", attrs={\"class\":\"a-price-whole\"})\n",
    "Money[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    "for i in Money:\n",
    "    Price.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Price[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['4.2 out of 5 stars',\n",
       " '4.2 out of 5 stars',\n",
       " '4.0 out of 5 stars',\n",
       " '4.2 out of 5 stars']"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find the Average rating  of the phone :-\n",
    "\n",
    "A_rating = Amz_soup.find_all(\"i\", attrs={\"class\":\"a-icon a-icon-star-small a-star-small-4 aok-align-bottom\"})\n",
    "A_rating[0:4]\n",
    "\n",
    "##now we extact the text form the these tegs by using for loop over the tags \n",
    "for i in A_rating:\n",
    "    Avg_Rating.append(i.get_text().replace(\"\\n\",\" \"))\n",
    "\n",
    "Avg_Rating[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Avg_Rating)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://m.media-amazon.com/images/I/61EJv4xde4L._AC_UY218_.jpg',\n",
       " 'https://m.media-amazon.com/images/I/51Rl+pkTVUL._AC_UY218_.jpg',\n",
       " 'https://m.media-amazon.com/images/I/61d-phh4GfL._AC_UY218_.jpg',\n",
       " 'https://m.media-amazon.com/images/I/71-Su4Wr0HL._AC_UY218_.jpg']"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find the Img_Url of the phone :-\n",
    "\n",
    "\n",
    "# Fetch links as List of Tag Objects\n",
    "links_img = Amz_soup.find_all(\"img\", attrs={'class':'s-image'})\n",
    "\n",
    "# Store the links\n",
    "links_list = []\n",
    " \n",
    "# Loop for extracting links from Tag Objects\n",
    "for link in links_img:\n",
    "    if(link.get(\"src\") != \"#\"):\n",
    "        links_list.append(link.get('src'))\n",
    "        \n",
    "links_list[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22 22 26 14\n"
     ]
    }
   ],
   "source": [
    "##  check the length of all 4 columns \n",
    "\n",
    "print(len(Moblie_Name), len(Price),len(links_list),len(Avg_Rating))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "Moblie_Name=Moblie_Name[0:13]\n",
    "Price=Price[0:13]\n",
    "links_list=links_list[0:13]\n",
    "Avg_Rating=Avg_Rating[0:13]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13 13 13 13\n"
     ]
    }
   ],
   "source": [
    "##  check the length of all 4 columns \n",
    "\n",
    "print(len(Moblie_Name), len(Price),len(links_list),len(Avg_Rating))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create the DataFrame \n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "Amz_mobile_data = pd.DataFrame({})\n",
    "Amz_mobile_data[\"Moblie_Name\"]=Moblie_Name\n",
    "Amz_mobile_data[\"Price\"]=Price\n",
    "Amz_mobile_data[\"Avg_Rating\"]=Avg_Rating\n",
    "Amz_mobile_data[\"Link\"]=links_list\n",
    "\n",
    "\n",
    "Amz_mobile_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Question 8:-\n",
    "\n",
    "### Write a python program to extract information about the local weather from the National Weather Service\n",
    "### website of USA, https://www.weather.gov/ for the city, San Francisco. You need to extract data about 7 day\n",
    "### extended forecast display for the city. The data should include period, short description, temperature and description."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  import the requried  libraries for web -scraping\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup as Soup\n",
    "import requests as rs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Lode the source code\n",
    "page_weather = rs.get(\"https://forecast.weather.gov/MapClick.php?lat=37.777120000000025&lon=-122.41963999999996#.YAKtTsUzYUs\")\n",
    "page_weather\n",
    "## Now parse the source code bu BeautifulSoup and the parser\n",
    "weather_soup = Soup(page_weather.content , \"html.parser\" )\n",
    "print(weather_soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create the Emplty list\n",
    "\n",
    "Days=[]\n",
    "Short_Description=[]\n",
    "Temperature=[]\n",
    "Description=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "day=weather_soup.find_all('p',class_='period-name')\n",
    "\n",
    "for i in day:\n",
    "    Days.append(i.get_text().replace(\"\\n\",\"\"))\n",
    "Days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "short_descrition=weather_soup.find_all('p',class_='short-desc')\n",
    "\n",
    "for i in short_descrition:\n",
    "    Short_Description.append(i.get_text().replace(\"\\n\",\"\"))\n",
    "    \n",
    "Short_Description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Temp=weather_soup.find_all('p',class_='temp temp-low')\n",
    "\n",
    "for i in Temp:\n",
    "    Temperature.append(i.get_text().replace(\"\\n\",\"\"))\n",
    "    \n",
    "Temperature\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "descrition=weather_soup.find_all('div',class_='col-sm-10 forecast-text')\n",
    "\n",
    "for i in descrition:\n",
    "    Description.append(i.get_text().replace(\"\\n\",\"\"))\n",
    "Description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Weather=pd.DataFrame({})\n",
    "\n",
    "Weather['Period']=Days[0:9]\n",
    "Weather['SHORT DESCRIPTION']=Short_Description[0:9]\n",
    "Weather['DESCRIPTION']=Description[0:9]\n",
    "\n",
    "Weather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}