main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pages(page):\n",
    "    reqUrl = f\"https://github.com/topics?page={page}\"\n",
    "    return reqUrl\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_page_content(page_url):\n",
    "    try:\n",
    "        response = requests.get(page_url)\n",
    "        response.raise_for_status()\n",
    "    except requests.exceptions.RequestException as e:\n",
    "        print(f'An error occurred while requesting the page: {e}')\n",
    "        return None\n",
    "\n",
    "    page_doc = BeautifulSoup(response.text, 'html.parser')\n",
    "    return page_doc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_topic_title(doc):\n",
    "    topic_title_tag = doc.find_all(\n",
    "        'p', {'class': 'f3 lh-condensed mb-0 mt-1 Link--primary'})\n",
    "    if not topic_title_tag:\n",
    "        raise Exception('Could not find topic title tags')\n",
    "    topic_title = []\n",
    "    for item in topic_title_tag:\n",
    "        topic_title.append(item.text.strip())\n",
    "    return topic_title\n",
    "\n",
    "\n",
    "def get_topic_desc(doc):\n",
    "    topic_desc_tag = doc.find_all(\n",
    "        'p', {'class': 'f5 color-fg-muted mb-0 mt-1'})\n",
    "    if not topic_desc_tag:\n",
    "        raise Exception('Could not find topic description tags')\n",
    "    topic_desc = []\n",
    "    for item in topic_desc_tag:\n",
    "        topic_desc.append(item.text.strip())\n",
    "    return topic_desc\n",
    "\n",
    "\n",
    "def get_topic_url(doc):\n",
    "    topic_url_tag = doc.find_all(\n",
    "        'a', {'class': 'no-underline flex-1 d-flex flex-column'})\n",
    "    if not topic_url_tag:\n",
    "        raise Exception('Could not find topic URL tags')\n",
    "    base_url = 'https://github.com'\n",
    "    topic_url = []\n",
    "    for item in topic_url_tag:\n",
    "        topic_url.append(base_url + item['href'])\n",
    "    return topic_url\n",
    "\n",
    "\n",
    "def scrape_topics_info(page_doc):\n",
    "    try:\n",
    "        topics_dict = {\n",
    "            'topic_title': get_topic_title(page_doc),\n",
    "            'topic_desc': get_topic_desc(page_doc),\n",
    "            'topic_url': get_topic_url(page_doc)\n",
    "        }\n",
    "    except Exception as e:\n",
    "        print(f'An error occurred while extracting topic information: {e}')\n",
    "        return None\n",
    "\n",
    "    return pd.DataFrame(topics_dict)\n",
    "\n",
    "\n",
    "def import_to_csv(dataFrame):\n",
    "    try:\n",
    "        dataFrame.to_csv('./topics/topics.csv', index=None)\n",
    "    except Exception as e:\n",
    "        print(f'An error occurred while writing to the CSV file: {e}')\n",
    "\n",
    "\n",
    "def update_and_filter_data(new_df):\n",
    "    try:\n",
    "        re_index = [*range(100, len(new_df))]\n",
    "        update_df = new_df.drop(re_index)\n",
    "    except Exception as e:\n",
    "        print(f'An error occurred while filtering data: {e}')\n",
    "        return\n",
    "\n",
    "    import_to_csv(update_df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def access_topic_page_url(df, topic_num):\n",
    "    try:\n",
    "        return df['topic_url'][topic_num]\n",
    "    except KeyError:\n",
    "        print(\n",
    "            f\"Error: The topic_num '{topic_num}' is not found in the dataframe.\")\n",
    "        return None\n",
    "\n",
    "\n",
    "def parse_star_count(stars_str):\n",
    "    try:\n",
    "        stars_str = stars_str.strip()\n",
    "        if stars_str[-1] == 'k':\n",
    "            return int(float(stars_str[:-1])*1000)\n",
    "        else:\n",
    "            return int(stars_str)\n",
    "    except ValueError:\n",
    "        print(f\"Error: Unable to parse the string '{stars_str}' as a number.\")\n",
    "        return None\n",
    "\n",
    "\n",
    "def get_repo_info(repo_tag, star_tag):\n",
    "    base_url = 'https://github.com'\n",
    "    try:\n",
    "        a_tag = repo_tag.find_all('a')\n",
    "        username = a_tag[0].text.strip()\n",
    "        repo_name = a_tag[1].text.strip()\n",
    "        stars = parse_star_count(star_tag.text.strip())\n",
    "        repo_url = base_url + a_tag[1]['href']\n",
    "        return username, repo_name, stars, repo_url\n",
    "    except (AttributeError, IndexError) as e:\n",
    "        print(f\"Error: {str(e)}\")\n",
    "        return None, None, None, None\n",
    "\n",
    "\n",
    "def get_topic_repos(topic_doc):\n",
    "    try:\n",
    "        repo_tag = topic_doc.find_all(\n",
    "            'h3', {'class': 'f3 color-fg-muted text-normal lh-condensed'})\n",
    "        star_tag = topic_doc.find_all(\n",
    "            'span', {'class': 'Counter js-social-count'})\n",
    "    except AttributeError as e:\n",
    "        print(f\"Error: {str(e)}\")\n",
    "        return None\n",
    "\n",
    "    topic_repos_dict = {\n",
    "        'username': [],\n",
    "        'repo_name': [],\n",
    "        'stars': [],\n",
    "        'repo_url': []\n",
    "    }\n",
    "\n",
    "    for i in range(len(repo_tag)):\n",
    "        repo_info = get_repo_info(repo_tag[i], star_tag[i])\n",
    "        topic_repos_dict['username'].append(repo_info[0])\n",
    "        topic_repos_dict['repo_name'].append(repo_info[1])\n",
    "        topic_repos_dict['stars'].append(repo_info[2])\n",
    "        topic_repos_dict['repo_url'].append(repo_info[3])\n",
    "    return pd.DataFrame(topic_repos_dict)\n",
    "\n",
    "\n",
    "def import_topic_repos_to_csv(df, topic):\n",
    "    try:\n",
    "        df.to_csv(f'./topics/repos/{topic}.csv', index=None)\n",
    "    except Exception as e:\n",
    "        print(f\"Error: Unable to write to file. {str(e)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_df = pd.DataFrame()\n",
    "for page in range(1, 6):\n",
    "    if len(temp_df) < 100:\n",
    "        try:\n",
    "            url = get_pages(page)\n",
    "            page_doc = get_page_content(url)\n",
    "            df = scrape_topics_info(page_doc)\n",
    "            temp_df = temp_df.append(df)\n",
    "        except Exception as e:\n",
    "            print(f\"Error in page {page}: {e}\")\n",
    "            continue\n",
    "    else:\n",
    "        break\n",
    "\n",
    "try:\n",
    "    import_to_csv(temp_df)\n",
    "    new_df = pd.read_csv('./topics/topics.csv')\n",
    "    update_and_filter_data(new_df)\n",
    "except Exception as e:\n",
    "    print(f\"Error importing to CSV: {e}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    new_df = pd.read_csv('./topics/topics.csv')\n",
    "    for page in range(len(new_df)):\n",
    "        try:\n",
    "            url = access_topic_page_url(new_df, page)\n",
    "            doc = get_page_content(url)\n",
    "            df = get_topic_repos(doc)\n",
    "            import_topic_repos_to_csv(df, new_df['topic_title'][page])\n",
    "        except Exception as e:\n",
    "            print(f\"Error in page {page}: {e}\")\n",
    "            continue\n",
    "except Exception as e:\n",
    "    print(f\"Error reading from CSV: {e}\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.8 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9 (main, Dec 15 2022, 17:11:09) [Clang 14.0.0 (clang-1400.0.29.202)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}