Notice Board Automator.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2617301e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: selenium in c:\\users\\achin\\anaconda3\\lib\\site-packages (4.12.0)\n",
      "Requirement already satisfied: certifi>=2021.10.8 in c:\\users\\achin\\anaconda3\\lib\\site-packages (from selenium) (2021.10.8)\n",
      "Requirement already satisfied: trio-websocket~=0.9 in c:\\users\\achin\\anaconda3\\lib\\site-packages (from selenium) (0.10.4)\n",
      "Requirement already satisfied: urllib3[socks]<3,>=1.26 in c:\\users\\achin\\anaconda3\\lib\\site-packages (from selenium) (1.26.9)\n",
      "Requirement already satisfied: trio~=0.17 in c:\\users\\achin\\anaconda3\\lib\\site-packages (from selenium) (0.22.2)\n",
      "Requirement already satisfied: idna in c:\\users\\achin\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (3.3)\n",
      "Note: you may need to restart the kernel to use updated packages.Requirement already satisfied: outcome in c:\\users\\achin\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (1.2.0)\n",
      "Requirement already satisfied: exceptiongroup>=1.0.0rc9 in c:\\users\\achin\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (1.1.3)\n",
      "Requirement already satisfied: sniffio in c:\\users\\achin\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (1.2.0)\n",
      "Requirement already satisfied: attrs>=20.1.0 in c:\\users\\achin\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (21.4.0)\n",
      "Requirement already satisfied: cffi>=1.14 in c:\\users\\achin\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (1.15.0)\n",
      "Requirement already satisfied: sortedcontainers in c:\\users\\achin\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (2.4.0)\n",
      "Requirement already satisfied: pycparser in c:\\users\\achin\\anaconda3\\lib\\site-packages (from cffi>=1.14->trio~=0.17->selenium) (2.21)\n",
      "Requirement already satisfied: wsproto>=0.14 in c:\\users\\achin\\anaconda3\\lib\\site-packages (from trio-websocket~=0.9->selenium) (1.2.0)\n",
      "Requirement already satisfied: PySocks!=1.5.7,<2.0,>=1.5.6 in c:\\users\\achin\\anaconda3\\lib\\site-packages (from urllib3[socks]<3,>=1.26->selenium) (1.7.1)\n",
      "Requirement already satisfied: h11<1,>=0.9.0 in c:\\users\\achin\\anaconda3\\lib\\site-packages (from wsproto>=0.14->trio-websocket~=0.9->selenium) (0.14.0)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pip install selenium\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "114ecd5d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current URL: https://onboard.bits-pilani.ac.in/index.php?sid=88f7e44f02181de25849c8653b7c4b6a\n",
      "Notices examples:\n",
      "Title: The SunPedal Ride\" by Mr. Sushil Reddy, an IITB alumnus\n",
      "Link: ./viewtopic.php?f=26&t=7178\n",
      "Last Post: Last post by onboard\n",
      "\n",
      "\n",
      "\n",
      " Tue Sep 26, 2023 10:35 am\n",
      "Views: 2 Views\n",
      "---------------------------------------\n",
      "Notices examples:\n",
      "Title: G20 University Connect Event on 26th September 2023, New Delhi\n",
      "Link: ./viewtopic.php?f=26&t=7175\n",
      "Last Post: Last post by director\n",
      "\n",
      "\n",
      "\n",
      " Tue Sep 26, 2023 10:28 am\n",
      "Views: 3 Views\n",
      "---------------------------------------\n",
      "Notices examples:\n",
      "Title: The SunPedal Ride\n",
      "Link: ./viewtopic.php?f=26&t=7157\n",
      "Last Post: Last post by chiefwarden\n",
      "\n",
      "\n",
      "\n",
      " Mon Sep 25, 2023 4:45 pm\n",
      "Views: 76 Views\n",
      "---------------------------------------\n",
      "Notices examples:\n",
      "Title: The SunPedal Ride\n",
      "Link: ./viewtopic.php?f=30&t=7160\n",
      "Last Post: Last post by chiefwarden\n",
      "\n",
      "\n",
      "\n",
      " Mon Sep 25, 2023 4:45 pm\n",
      "Views: 7 Views\n",
      "---------------------------------------\n",
      "Notices examples:\n",
      "Title: Holiday on the first half of 27th September 2023 on account of the visit of Hon'ble Vice President of India\n",
      "Link: ./viewtopic.php?f=26&t=7152\n",
      "Last Post: Last post by onboard\n",
      "\n",
      "\n",
      "\n",
      " Mon Sep 25, 2023 4:23 pm\n",
      "Views: 113 Views\n",
      "---------------------------------------\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "\n",
    "def get_credentials_from_file(filename):\n",
    "#     Extracts email and password from the given file. Write the file in the same directory as follows:\n",
    "#     bits mail\n",
    "#     mail password (google)\n",
    "#     smtp mail user name\n",
    "#     smtp key\n",
    "#     smtp email-id   \n",
    "    with open(filename, 'r') as file:\n",
    "\n",
    "\n",
    "        email = file.readline().strip()\n",
    "        password = file.readline().strip()\n",
    "    return email, password\n",
    "\n",
    "browser = webdriver.Chrome()\n",
    "\n",
    "browser.get('https://onboard.bits-pilani.ac.in/ucp.php?mode=login&redirect=index.php')\n",
    "\n",
    "# Click on Google login button using the provided HTML info\n",
    "google_login_btn = browser.find_element(By.XPATH, '//a[@title=\"BITS Email  Login\"]')\n",
    "google_login_btn.click()\n",
    "\n",
    "# Retrieve credentials from file\n",
    "email, password = get_credentials_from_file('sensitive.txt')\n",
    "\n",
    "# Find email field and fill it in\n",
    "email_field = browser.find_element(By.ID, 'identifierId')\n",
    "email_field.send_keys(email)\n",
    "next_btn = browser.find_element(By.ID, 'identifierNext')\n",
    "next_btn.click()\n",
    "\n",
    "\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "\n",
    "# Wait until password field is available and visible, then select it\n",
    "password_field = WebDriverWait(browser, 10).until(\n",
    "    EC.visibility_of_element_located((By.NAME, 'Passwd'))\n",
    ")\n",
    "\n",
    "\n",
    "# Assuming a delay, fill in the password\n",
    "password_field.send_keys(password)\n",
    "\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.common.exceptions import StaleElementReferenceException, TimeoutException\n",
    "import time\n",
    "\n",
    "MAX_RETRIES = 3\n",
    "retry_count = 0\n",
    "\n",
    "while retry_count < MAX_RETRIES:\n",
    "    try:\n",
    "        # Wait until the button is present and clickable.\n",
    "        wait = WebDriverWait(browser, 10)\n",
    "        signin_btn = wait.until(EC.element_to_be_clickable((By.ID, 'passwordNext')))\n",
    "        \n",
    "        signin_btn.click()\n",
    "        break\n",
    "    except StaleElementReferenceException:\n",
    "        # In case of stale element, retry after a short delay\n",
    "        time.sleep(1)\n",
    "        retry_count += 1\n",
    "        continue\n",
    "    except TimeoutException:\n",
    "        retry_count += 1\n",
    "        continue\n",
    "\n",
    "if retry_count == MAX_RETRIES:\n",
    "    print(\"Failed to click the button after maximum retries.\")\n",
    "\n",
    "\n",
    "try:\n",
    "    element_present = EC.presence_of_element_located((By.CSS_SELECTOR, 'span.username'))\n",
    "    WebDriverWait(browser, 10).until(element_present)\n",
    "\n",
    "    # Get the current URL after signing in.\n",
    "    current_url = browser.current_url\n",
    "    print(\"Current URL:\", current_url)\n",
    "\n",
    "except TimeoutException:\n",
    "    print(\"Timed out waiting for the new page to load after sign-in.\")\n",
    "\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "BASE_URL = f'{current_url}&recent_topics_start='\n",
    "MAX_PAGES = 10  # Adjust according to your needs\n",
    "\n",
    "def get_notices_from_page(browser, url):\n",
    "    notices = []\n",
    "    \n",
    "    browser.get(url)\n",
    "    soup = BeautifulSoup(browser.page_source, 'html.parser')\n",
    "    topics = soup.select('ul.topiclist.topics.collapsible li')\n",
    "    \n",
    "    for topic in topics:\n",
    "        topic_element = topic.select_one('.topictitle')\n",
    "        topic_title = topic_element.text.strip()\n",
    "        topic_link = topic_element['href'].strip()  # Extracting the href attribute for the link\n",
    "        last_post = topic.select_one('.lastpost span').text.strip()\n",
    "        views = topic.select_one('.views').text.strip()\n",
    "        \n",
    "        notice_info = {\n",
    "            'Title': topic_title,\n",
    "            'Link': topic_link,  # Adding the link to the notice info dictionary\n",
    "            'Last Post': last_post,\n",
    "            'Views': views\n",
    "        }\n",
    "        \n",
    "        notices.append(notice_info)\n",
    "    \n",
    "    return notices\n",
    "\n",
    "\n",
    "all_notices = []\n",
    "for i in range(0, MAX_PAGES * 5, 5):\n",
    "    url = BASE_URL + str(i)\n",
    "    all_notices.extend(get_notices_from_page(browser, url))\n",
    "\n",
    "# Printing all notices\n",
    "print(\"Notices examples:\")\n",
    "\n",
    "for notice in all_notices[:5]:\n",
    "    print(\"Title:\", notice['Title'])\n",
    "    print(\"Link:\", notice['Link']) \n",
    "    print(\"Last Post:\", notice['Last Post'])\n",
    "    print(\"Views:\", notice['Views'])\n",
    "    print('---------------------------------------')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "36e3a5d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: filetype in c:\\users\\achin\\anaconda3\\lib\\site-packages (1.2.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install filetype\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0d8a97e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from PIL import Image\n",
    "# from io import BytesIO\n",
    "# import os\n",
    "# import requests\n",
    "# from bs4 import BeautifulSoup\n",
    "# import filetype\n",
    "\n",
    "# BASE_URL = \"https://onboard.bits-pilani.ac.in/\"\n",
    "\n",
    "# def guess_extension_from_content(content):\n",
    "#     print(content)\n",
    "#     kind = filetype.guess(content)\n",
    "#     if kind is None:\n",
    "#         return ''\n",
    "#     return kind.extension\n",
    "\n",
    "# def sanitize_filename(filename):\n",
    "#     invalid_chars = ['<', '>', ':', '\"', '/', '\\\\', '|', '?', '*']\n",
    "#     for char in invalid_chars:\n",
    "#         filename = filename.replace(char, '_')\n",
    "#     return filename\n",
    "\n",
    "# def scrape_topic_content(url):\n",
    "#     full_url = BASE_URL + url.lstrip('./')\n",
    "    \n",
    "#     browser.get(full_url)\n",
    "    \n",
    "#     soup = BeautifulSoup(browser.page_source, 'html.parser')\n",
    "\n",
    "#     heading = soup.select_one('.topic-title').text.strip()\n",
    "\n",
    "#     author = soup.select_one('.username-coloured').text.strip()\n",
    "\n",
    "#     post_content = soup.select_one('.postbody .content')\n",
    "#     content_text = post_content.text.strip()\n",
    "\n",
    "#     attachments = []\n",
    "\n",
    "#     for attach_div in soup.select('.attachbox, .inline-attachment'):\n",
    "#         for attach in attach_div.select('.file'):\n",
    "#             img_tag = attach.select_one('img.postimage')\n",
    "#             link = img_tag['src'] if img_tag else None\n",
    "#             filename = attach.select_one('dd').text.split()[0]  # Assumes filename is the first word\n",
    "#             filesize = ' '.join(attach.select_one('dd').text.split()[1:4])  # Assumes next three words are filesize details\n",
    "\n",
    "#             attachment = {\n",
    "#                 \"filename\": filename,\n",
    "#                 \"filesize\": filesize,\n",
    "#                 \"link\": link\n",
    "#             }\n",
    "#             attachments.append(attachment)\n",
    "#     from selenium.webdriver.common.action_chains import ActionChains\n",
    "#     from selenium.webdriver.common.by import By\n",
    "#     from selenium.webdriver.common.keys import Keys\n",
    "#     import time\n",
    "\n",
    "\n",
    "#     download_dir = \"downloaded_attachments\"\n",
    "#     os.makedirs(download_dir, exist_ok=True)\n",
    "#     downloaded_files = []\n",
    "\n",
    "#     for attach in attachments:\n",
    "#         browser.get(BASE_URL + attach[\"link\"].lstrip('./'))\n",
    "\n",
    "#         time.sleep(2)\n",
    "\n",
    "#         if len(browser.find_elements(By.TAG_NAME, 'img')) > 0:\n",
    "#             image = browser.find_element(By.TAG_NAME, 'img')\n",
    "#             action = ActionChains(browser)\n",
    "#             action.context_click(image).send_keys(Keys.ARROW_DOWN).send_keys(Keys.RETURN).perform()\n",
    "\n",
    "#         elif len(browser.find_elements(By.CLASS_NAME, 'downloadButton')) > 0:\n",
    "#             download_button = browser.find_element(By.CLASS_NAME, 'downloadButton')\n",
    "#             download_button.click()\n",
    "\n",
    "#         time.sleep(5)\n",
    "\n",
    "#         safe_filename = sanitize_filename(attach[\"filename\"])\n",
    "#         filepath = os.path.join(download_dir, safe_filename)\n",
    "#         downloaded_files.append(filepath)\n",
    "\n",
    "#     return {\n",
    "#         'heading': heading,\n",
    "#         'author': author,\n",
    "#         'content': content_text,\n",
    "#         'attachments': downloaded_files\n",
    "#     }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "a0cb85aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "from io import BytesIO\n",
    "import os\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import filetype\n",
    "\n",
    "BASE_URL = \"https://onboard.bits-pilani.ac.in/\"\n",
    "\n",
    "def scrape_topic_content(url):\n",
    "    full_url = BASE_URL + url.lstrip('./')\n",
    "    \n",
    "    browser.get(full_url)\n",
    "    \n",
    "    soup = BeautifulSoup(browser.page_source, 'html.parser')\n",
    "\n",
    "    postbody_div = soup.select_one('.postbody')\n",
    "\n",
    "    content_html = str(postbody_div)\n",
    "\n",
    "    heading = soup.select_one('.topic-title').text.strip()\n",
    "\n",
    "    author = soup.select_one('.username-coloured').text.strip()\n",
    "    \n",
    "    return   {'heading': heading,\n",
    "        'author': author,\n",
    "        'content': content_html}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "c6521875",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import os\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "\n",
    "def download_images(html_content, base_url):\n",
    "    soup = BeautifulSoup(html_content, 'html.parser')\n",
    "    img_tags = soup.find_all('img')\n",
    "    \n",
    "    if not os.path.exists('downloaded_images'):\n",
    "        os.makedirs('downloaded_images')\n",
    "    \n",
    "    saved_image_data = []\n",
    "\n",
    "    for img_tag in img_tags:\n",
    "        img_url = img_tag['src']\n",
    "        if img_url.startswith(\"./\"):\n",
    "            img_url = base_url + img_url[1:]\n",
    "\n",
    "        browser.get(img_url)\n",
    "        \n",
    "        img_element = browser.find_element(By.TAG_NAME, \"img\")\n",
    "        img_content = img_element.screenshot_as_png\n",
    "        \n",
    "        width = img_element.size[\"width\"]\n",
    "        height = img_element.size[\"height\"]\n",
    "        \n",
    "        img_id = img_url.split('=')[-1]\n",
    "        img_filename = os.path.join('downloaded_images', f\"image_{img_id}.png\")\n",
    "        \n",
    "        with open(img_filename, 'wb') as img_file:\n",
    "            img_file.write(img_content)\n",
    "            \n",
    "        saved_image_data.append({\"path\": img_filename, \"width\": width, \"height\": height})\n",
    "\n",
    "    return saved_image_data\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "ae4ae68d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import os\n",
    "\n",
    "def replace_img_src_with_cid(html_content, image_data_list):\n",
    "    soup = BeautifulSoup(html_content, 'html.parser')\n",
    "    img_tags = soup.find_all('img')\n",
    "    \n",
    "    for img_tag, image_data in zip(img_tags, image_data_list):\n",
    "        img_name = os.path.basename(image_data[\"path\"])\n",
    "        img_tag[\"src\"] = \"cid:{}\".format(img_name)\n",
    "        img_tag[\"width\"] = str(image_data[\"width\"])\n",
    "        img_tag[\"height\"] = str(image_data[\"height\"])\n",
    "        \n",
    "    return str(soup)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "90298cc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import smtplib\n",
    "from email.mime.multipart import MIMEMultipart\n",
    "from email.mime.text import MIMEText\n",
    "from email.mime.base import MIMEBase\n",
    "from email import encoders\n",
    "\n",
    "def send_email(email_content, recipient_email, subject_prefix=\"Automated scraped email\"):\n",
    "    # Download images and replace image URLs with Content-ID references\n",
    "    image_data_list = download_images(email_content['content'], BASE_URL)\n",
    "    email_content['content'] = replace_img_src_with_cid(email_content['content'], image_data_list)\n",
    "    \n",
    "    with open(\"sensitive.txt\", 'r') as file:\n",
    "        lines = file.readlines()\n",
    "        SMTP_SERVER = \"smtp.gmail.com\"\n",
    "        SMTP_PORT = 465\n",
    "        SMTP_USERNAME = lines[2].strip()\n",
    "        SMTP_PASSWORD = lines[3].strip()\n",
    "        FROM_EMAIL = lines[4].strip()\n",
    "\n",
    "    msg = MIMEMultipart()\n",
    "    msg['From'] = FROM_EMAIL\n",
    "    msg['To'] = recipient_email\n",
    "    msg['Subject'] = f\"{subject_prefix} - {email_content['heading']}\"\n",
    "    \n",
    "    body = MIMEText(email_content['content'], 'html', 'utf-8')\n",
    "    msg.attach(body)\n",
    "\n",
    "    for img_data in image_data_list:\n",
    "        img_path = img_data[\"path\"]\n",
    "        with open(img_path, \"rb\") as img_file:\n",
    "            img_name = os.path.basename(img_path)\n",
    "            img_mime = MIMEBase('image', 'png')\n",
    "            img_mime.set_payload(img_file.read())\n",
    "            encoders.encode_base64(img_mime)\n",
    "            img_mime.add_header('Content-Disposition', f'inline; filename={img_name}')\n",
    "            img_mime.add_header('Content-ID', f'<{img_name}>')\n",
    "            msg.attach(img_mime)\n",
    "\n",
    "    with smtplib.SMTP_SSL(SMTP_SERVER, SMTP_PORT) as server:\n",
    "        server.login(FROM_EMAIL, SMTP_PASSWORD)\n",
    "        server.sendmail(FROM_EMAIL, recipient_email, msg.as_string())\n",
    "\n",
    "    print(\"Sent mail..\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "1ad8b82c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sent mail..\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for notice in all_notices:\n",
    "    scraped_content = scrape_topic_content(notice['Link'])\n",
    "    send_email(scraped_content, \"f20211457@pilani.bits-pilani.ac.in\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f5e8de5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}