Scraping google.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "735e0ad6",
   "metadata": {},
   "source": [
    "## Importing the Necessary Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "c9cbfd29",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from urllib.parse import quote\n",
    "from bs4 import BeautifulSoup\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.chrome.service import Service as ChromeService\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from time import sleep\n",
    "from rich import print"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc197871",
   "metadata": {},
   "source": [
    "## Link creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "65688bd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "Job = 'Dental Jobs'\n",
    "search_params = [\n",
    "    'Boston, MA. 100miles',\n",
    "    'Houston TX 100 miles',\n",
    "    'Greensboro, NC. 50 miles',\n",
    "    'High point NC 50 miles', \n",
    "    'Wintson-Salem NC  50 miles',\n",
    "    'Los Angeles, CA 100 miles',\n",
    "    'Cleveland OH 100miles',\n",
    "]\n",
    "\n",
    "full_search_prompt = [(Job+' '+ search_param) for search_param in search_params]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "c4b05803",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_google_search_url(query):\n",
    "    \"\"\"\n",
    "    Constructs a Google search URL based on the provided search query.\n",
    "\n",
    "    Args:\n",
    "      query: The search query string.\n",
    "\n",
    "    Returns:\n",
    "      A string containing the formatted Google search URL.\n",
    "    \"\"\"\n",
    "    # Encode special characters in the query for safe URL inclusion\n",
    "    encoded_query = quote(query)\n",
    "    # Base URL for Google Search\n",
    "    google_search_url = \"https://www.google.com/search?q=\"\n",
    "\n",
    "    # Combine the base URL and encoded query\n",
    "    return google_search_url + encoded_query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "dbb51594",
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = [build_google_search_url(prompt) for prompt in full_search_prompt]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35c14231",
   "metadata": {},
   "source": [
    "## Get the Webdrivers Ready to work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "dbc78f3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set chrome to headless\n",
    "chrome_options = webdriver.ChromeOptions()\n",
    "# chrome_options.add_argument('--headless')\n",
    "chrome_options.add_argument('--no-sandbox')\n",
    "chrome_options.add_argument('--disable-dev-shm-usage')\n",
    "\n",
    "# Add the driver to be able to load the pages\n",
    "browser = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options = chrome_options)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6849ef6b",
   "metadata": {},
   "source": [
    "## Get the Google search Page and the html of the page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "eb44fb13",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_job_details(job_div):\n",
    "    \"\"\"\n",
    "    Extracts job title, location (if available), and link from a job posting div.\n",
    "\n",
    "    Args:\n",
    "      job_div: A string containing the HTML content of the job posting div.\n",
    "\n",
    "    Returns:\n",
    "      A dictionary containing extracted job details, or None if no details are found.\n",
    "    \"\"\"\n",
    "\n",
    "    # Find the job title element (assuming h3 with specific class)\n",
    "    job_title_element = job_div.find('h3', class_='LC20lb MBeuO DKV0Md')\n",
    "\n",
    "    if job_title_element:\n",
    "        # Extract job title\n",
    "        job_title = job_title_element.text.strip()\n",
    "\n",
    "        # Find location information (might be within the same element or separate)\n",
    "        location_element = job_title_element\n",
    "        location_text = location_element.text.strip()\n",
    "        location_parts = [part.strip() for part in location_text.split(',') if part.strip()]  # Split and clean location\n",
    "\n",
    "        # Find the job link\n",
    "        job_link_element = job_div.find('a')\n",
    "        if job_link_element:\n",
    "            job_link = job_link_element['href']\n",
    "        else:\n",
    "            job_link = None\n",
    "\n",
    "        # Create dictionary with extracted details\n",
    "        job_details = {\n",
    "            'Title': job_title,\n",
    "            'Location': ', '.join(location_parts) if location_parts else None,\n",
    "            'Job Link': job_link\n",
    "        }\n",
    "        return job_details\n",
    "    else:\n",
    "        return None  # No details found"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "3c8743ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_jobs = []\n",
    "for url in urls:\n",
    "    browser.get(url)\n",
    "    browser.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "\n",
    "    content = browser.page_source\n",
    "    soup = BeautifulSoup(content, 'html.parser')\n",
    "\n",
    "    link_cards = soup.find_all('div', class_='g Ww4FFb vt6azd tF2Cxc asEBEc')\n",
    "\n",
    "    all_job = [extract_job_details(link_cards[i]) for i in range(len(link_cards))]\n",
    "    all_jobs.extend(all_job)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "cee89c74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Converting to Dataframe\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Converting to Dataframe\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Data saved to indeeds_job_data.csv\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Data saved to indeeds_job_data.csv\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Convert list of records into a DataFrame\n",
    "print('Converting to Dataframe')\n",
    "df = pd.DataFrame(all_jobs)\n",
    "# Save DataFrame to a CSV file\n",
    "df.to_csv('google_jobs_data.csv', index=False)\n",
    "\n",
    "print(\"Data saved to indeeds_job_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "739dbe81",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Location</th>\n",
       "      <th>Job Link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>169 General Dentist jobs in Boston, Massachuse...</td>\n",
       "      <td>169 General Dentist jobs in Boston, Massachuse...</td>\n",
       "      <td>https://www.linkedin.com/jobs/general-dentist-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>68 Dentist Jobs in Boston, MA</td>\n",
       "      <td>68 Dentist Jobs in Boston, MA</td>\n",
       "      <td>https://www.ziprecruiter.com/Jobs/Dentist/-in-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>276 Dentist jobs in Boston, Massachusetts, Uni...</td>\n",
       "      <td>276 Dentist jobs in Boston, Massachusetts, Uni...</td>\n",
       "      <td>https://www.linkedin.com/jobs/dentist-jobs-bos...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Dentist Boston Jobs, Employment in Boston, MA</td>\n",
       "      <td>Dentist Boston Jobs, Employment in Boston, MA</td>\n",
       "      <td>https://www.indeed.com/q-dentist-boston-l-bost...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>$18-$27/hr Dental Assistant Jobs in Boston, MA...</td>\n",
       "      <td>$18-$27/hr Dental Assistant Jobs in Boston, MA...</td>\n",
       "      <td>https://www.ziprecruiter.com/Jobs/Dental-Assis...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>Dentist jobs in Cleveland, OH</td>\n",
       "      <td>Dentist jobs in Cleveland, OH</td>\n",
       "      <td>https://www.indeed.com/q-dentist-l-cleveland,-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>624 Dentist jobs in Cleveland, Ohio, United St...</td>\n",
       "      <td>624 Dentist jobs in Cleveland, Ohio, United St...</td>\n",
       "      <td>https://www.linkedin.com/jobs/dentist-jobs-cle...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>Jobs | Heartland Dental and Supported Offices ...</td>\n",
       "      <td>Jobs | Heartland Dental and Supported Offices ...</td>\n",
       "      <td>https://jobs.heartland.com/jobs/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>14000+ Dental jobs in Cleveland, Ohio, United ...</td>\n",
       "      <td>14000+ Dental jobs in Cleveland, Ohio, United ...</td>\n",
       "      <td>https://www.linkedin.com/jobs/dental-jobs-clev...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>Heartland Dental</td>\n",
       "      <td>Heartland Dental</td>\n",
       "      <td>https://jobs.heartland.com/</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>70 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Title  \\\n",
       "0   169 General Dentist jobs in Boston, Massachuse...   \n",
       "1                       68 Dentist Jobs in Boston, MA   \n",
       "2   276 Dentist jobs in Boston, Massachusetts, Uni...   \n",
       "3       Dentist Boston Jobs, Employment in Boston, MA   \n",
       "4   $18-$27/hr Dental Assistant Jobs in Boston, MA...   \n",
       "..                                                ...   \n",
       "65                      Dentist jobs in Cleveland, OH   \n",
       "66  624 Dentist jobs in Cleveland, Ohio, United St...   \n",
       "67  Jobs | Heartland Dental and Supported Offices ...   \n",
       "68  14000+ Dental jobs in Cleveland, Ohio, United ...   \n",
       "69                                   Heartland Dental   \n",
       "\n",
       "                                             Location  \\\n",
       "0   169 General Dentist jobs in Boston, Massachuse...   \n",
       "1                       68 Dentist Jobs in Boston, MA   \n",
       "2   276 Dentist jobs in Boston, Massachusetts, Uni...   \n",
       "3       Dentist Boston Jobs, Employment in Boston, MA   \n",
       "4   $18-$27/hr Dental Assistant Jobs in Boston, MA...   \n",
       "..                                                ...   \n",
       "65                      Dentist jobs in Cleveland, OH   \n",
       "66  624 Dentist jobs in Cleveland, Ohio, United St...   \n",
       "67  Jobs | Heartland Dental and Supported Offices ...   \n",
       "68  14000+ Dental jobs in Cleveland, Ohio, United ...   \n",
       "69                                   Heartland Dental   \n",
       "\n",
       "                                             Job Link  \n",
       "0   https://www.linkedin.com/jobs/general-dentist-...  \n",
       "1   https://www.ziprecruiter.com/Jobs/Dentist/-in-...  \n",
       "2   https://www.linkedin.com/jobs/dentist-jobs-bos...  \n",
       "3   https://www.indeed.com/q-dentist-boston-l-bost...  \n",
       "4   https://www.ziprecruiter.com/Jobs/Dental-Assis...  \n",
       "..                                                ...  \n",
       "65  https://www.indeed.com/q-dentist-l-cleveland,-...  \n",
       "66  https://www.linkedin.com/jobs/dentist-jobs-cle...  \n",
       "67                   https://jobs.heartland.com/jobs/  \n",
       "68  https://www.linkedin.com/jobs/dental-jobs-clev...  \n",
       "69                        https://jobs.heartland.com/  \n",
       "\n",
       "[70 rows x 3 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfx = pd.read_csv('google_jobs_data.csv')\n",
    "dfx"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "web-scraping",
   "language": "python",
   "name": "web-scraping"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}