x0-parsing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This notebook helps in parsing different types of lists available online, and creates a massive `conglom-labeled.csv` list at the end. The logic is complete here, but more lists could be added in the future. This work is done as of 5 APRIL 2024."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dotenv import load_dotenv\n",
    "import os\n",
    "import requests\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cluster import KMeans\n",
    "import random\n",
    "import plotly.express as px\n",
    "import matplotlib.pyplot as plt\n",
    "from urllib.parse import urlparse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First parsing yoyo.csv from https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./datasets/yoyo.csv')\n",
    "df.columns = ['URL']\n",
    "df['URL'] = df['URL'].str.replace('127.0.0.1 ', '', regex=False)\n",
    "df['type'] = 'ads'\n",
    "df = df.drop_duplicates()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now add \"ads\" column for later merging with dataset and dropping duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3757 entries, 0 to 3756\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   URL     3757 non-null   object\n",
      " 1   type    3757 non-null   object\n",
      "dtypes: object(2)\n",
      "memory usage: 58.8+ KB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>URL</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3757</td>\n",
       "      <td>3757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>3757</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>101com.com</td>\n",
       "      <td>ads</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>3757</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               URL  type\n",
       "count         3757  3757\n",
       "unique        3757     1\n",
       "top     101com.com   ads\n",
       "freq             1  3757"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()\n",
    "df.info()\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('./datasets/yoyo-labeled.csv', index=False) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now EasyList https://easylist.to/easylist/easyprivacy.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            URL      type\n",
      "0      00px.net  tracking\n",
      "1     1cros.net  tracking\n",
      "2      2cnt.net  tracking\n",
      "3  2l6ddsmnm.de  tracking\n",
      "4  2smt6mfgo.de  tracking\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>URL</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>39177</td>\n",
       "      <td>39177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>39177</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>00px.net</td>\n",
       "      <td>tracking</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>39177</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             URL      type\n",
       "count      39177     39177\n",
       "unique     39177         1\n",
       "top     00px.net  tracking\n",
       "freq           1     39177"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Step 1: Read the file line by line and filter based on your criteria\n",
    "filtered_urls = []\n",
    "with open('./datasets/easylist-tracking.csv', 'r') as file:\n",
    "    for line in file:\n",
    "        # Check if the line matches the desired pattern\n",
    "        if line.startswith('||') and line.endswith('^\\n'):\n",
    "            # Extract the URL part and remove the trailing newline character\n",
    "            url = line[2:-2]\n",
    "            filtered_urls.append(url)\n",
    "\n",
    "# Step 2: Convert the list of URLs to a DataFrame\n",
    "import pandas as pd\n",
    "df = pd.DataFrame(filtered_urls, columns=['URL'])\n",
    "df['type'] = 'tracking'\n",
    "df = df.drop_duplicates()\n",
    "# Now 'df' contains only the URLs extracted from the file\n",
    "print(df.head())\n",
    "\n",
    "df.to_csv('./datasets/easylist-tracking-labeled.csv', index=False)\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ok now easylist ads https://ublockorigin.pages.dev/thirdparties/easylist.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              URL type\n",
      "0  0008d6ba2e.com  ads\n",
      "1  0024ad98dd.com  ads\n",
      "2  0083334e84.com  ads\n",
      "3  00d3ed994e.com  ads\n",
      "4  00d84987c0.com  ads\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>URL</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>38732</td>\n",
       "      <td>38732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>38732</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>0008d6ba2e.com</td>\n",
       "      <td>ads</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>38732</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   URL   type\n",
       "count            38732  38732\n",
       "unique           38732      1\n",
       "top     0008d6ba2e.com    ads\n",
       "freq                 1  38732"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_urls = []\n",
    "with open('./datasets/easylist-ads.csv', 'r') as file:\n",
    "    for line in file:\n",
    "        # pattern of ||example.com^\n",
    "        if line.startswith('||') and line.endswith('^\\n'):\n",
    "            # extract URL\n",
    "            url = line[2:-2]\n",
    "            filtered_urls.append(url)\n",
    "\n",
    "# convert to df\n",
    "import pandas as pd\n",
    "df = pd.DataFrame(filtered_urls, columns=['URL'])\n",
    "df['type'] = 'ads'\n",
    "df = df.drop_duplicates()\n",
    "# Now 'df' contains only the URLs extracted from the file\n",
    "print(df.head())\n",
    "\n",
    "df.to_csv('./datasets/easylist-ads-labeled.csv', index=False)\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ok now the malware list from https://malware-filter.gitlab.io/urlhaus-filter/urlhaus-filter-ag-online.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                          URL     type\n",
      "0                             1ecosolution.it  malware\n",
      "1  988skins.com/admin/view/stylesheet/50k.png  malware\n",
      "2                            a.alie3ksgee.com  malware\n",
      "3                            a0920080.xsph.ru  malware\n",
      "4                   abissnet.net/se12y5vm.zip  malware\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>URL</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>701</td>\n",
       "      <td>701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>701</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>1ecosolution.it</td>\n",
       "      <td>malware</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>701</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    URL     type\n",
       "count               701      701\n",
       "unique              701        1\n",
       "top     1ecosolution.it  malware\n",
       "freq                  1      701"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_urls = []\n",
    "with open('./datasets/malware.csv', 'r') as file:\n",
    "    for line in file:\n",
    "        # pattern of ||example.com^\n",
    "        if line.startswith('||') and line.endswith('$all\\n'):\n",
    "            # extract URL\n",
    "            url = line[2:-5]\n",
    "            filtered_urls.append(url)\n",
    "\n",
    "# convert to df\n",
    "import pandas as pd\n",
    "df = pd.DataFrame(filtered_urls, columns=['URL'])\n",
    "df['type'] = 'malware'\n",
    "df = df.drop_duplicates()\n",
    "print(df.head())\n",
    "\n",
    "df.to_csv('./datasets/malware-labeled.csv', index=False)\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              URL type\n",
      "0  0008d6ba2e.com  ads\n",
      "1  0024ad98dd.com  ads\n",
      "2  0083334e84.com  ads\n",
      "3  00d3ed994e.com  ads\n",
      "4  00d84987c0.com  ads\n"
     ]
    }
   ],
   "source": [
    "dfs = []\n",
    "for filename in ['./datasets/easylist-ads-labeled.csv', #read each csv into df\n",
    "                 './datasets/easylist-tracking-labeled.csv', \n",
    "                 './datasets/malicious-phish.csv', \n",
    "                 './datasets/malware-labeled.csv', \n",
    "                 './datasets/yoyo-labeled.csv']:\n",
    "    df = pd.read_csv(filename, usecols=['URL', 'type'])  # ensure only URL and type are read\n",
    "    dfs.append(df)\n",
    "\n",
    "combined_df = pd.concat(dfs, ignore_index=True)\n",
    "combined_df.to_csv('./datasets/conglom-labeled.csv', index=False)\n",
    "df_conglom = pd.read_csv('./datasets/conglom-labeled.csv')\n",
    "print(df_conglom.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 733558 entries, 0 to 733557\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count   Dtype \n",
      "---  ------  --------------   ----- \n",
      " 0   URL     733558 non-null  object\n",
      " 1   type    733558 non-null  object\n",
      "dtypes: object(2)\n",
      "memory usage: 11.2+ MB\n"
     ]
    }
   ],
   "source": [
    "df_conglom.describe()\n",
    "df_conglom.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                      URL    type\n",
      "count                                              733558  733558\n",
      "unique                                             722296       6\n",
      "top     http://style.org.hc360.com/css/detail/mysite/s...  benign\n",
      "freq                                                  180  428103\n"
     ]
    }
   ],
   "source": [
    "print(df_conglom.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "type\n",
       "benign        428103\n",
       "defacement     96457\n",
       "phishing       94111\n",
       "ads            42489\n",
       "tracking       39177\n",
       "malware        33221\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_conglom.value_counts('type')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml-class2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}