Data_Exploration.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "duZ8OXEVAlJr"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import numpy as np\n",
        "\n",
        "filename = \"raw_data/train.json\"\n",
        "\n",
        "# Load JSON data\n",
        "with open(filename, \"r\") as file:\n",
        "    data = json.load(file)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CGkkmcCvJk_Z"
      },
      "outputs": [],
      "source": [
        "# Initialize lists to store PII data, positions, and spacing\n",
        "pii_data = []\n",
        "pii_positions = []\n",
        "spacing = []\n",
        "pii_ratio = []\n",
        "\n",
        "# Iterate through the elements\n",
        "for item in data:\n",
        "    tokens = item[\"tokens\"]\n",
        "    labels = item[\"labels\"]\n",
        "\n",
        "    # Extract PII data and their positions\n",
        "    current_pii = []\n",
        "    current_pii_positions = []\n",
        "    item_pii_data = []\n",
        "\n",
        "    for i, (token, label) in enumerate(zip(tokens, labels)):\n",
        "        if label != \"O\":  # Check if the token is labeled as PII\n",
        "            current_pii.append((token, label[2:]))\n",
        "            current_pii_positions.append([label[2:], i, len(tokens)])\n",
        "        else:\n",
        "            if current_pii:\n",
        "                item_pii_data.extend(current_pii)\n",
        "                pii_positions.append([item[\"document\"]] + current_pii_positions[0])\n",
        "                current_pii = []\n",
        "                current_pii_positions = []\n",
        "\n",
        "    # Calculate ratio of PII Tokens / 10000 Tokens\n",
        "    pii_ratio.append((len(item_pii_data) / len(labels)) * 10000)\n",
        "    pii_data.extend(item_pii_data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 333
        },
        "id": "clOhNmmZZaFc",
        "outputId": "1247750c-afff-4880-bf7b-b23e2a553bab"
      },
      "outputs": [],
      "source": [
        "# Plot the PII Ratio across documents\n",
        "plt.figure(figsize=(4, 3))\n",
        "sns.histplot(x=pii_ratio, bins=12, color=\"skyblue\")\n",
        "plt.title(\"Ratio of PII Tokens in each Document\")\n",
        "plt.xlabel(\"Number of PII Tokens / 10,000 Tokens\")\n",
        "plt.ylabel(\"Logarithmic Frequency\")\n",
        "plt.yscale(\"log\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A8WljlCxjrSV"
      },
      "outputs": [],
      "source": [
        "# Label frequency\n",
        "pii_df = pd.DataFrame(pii_data, columns=[\"Token\", \"Label\"])\n",
        "pii_count = pii_df[\"Label\"].value_counts().reset_index()\n",
        "pii_count.columns = [\"Label\", \"Frequency\"]\n",
        "pii_count = pii_count.sort_values(by=\"Frequency\", ascending=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 338
        },
        "id": "RG0d7hl-MvJK",
        "outputId": "94aa190b-ab8f-4b2a-b35f-04c15587756e"
      },
      "outputs": [],
      "source": [
        "# Plot the PII labels and their frequencies\n",
        "plt.figure(figsize=(4, 3))\n",
        "sns.barplot(x=\"Frequency\", y=\"Label\", data=pii_count, palette=\"pastel\", hue=\"Label\")\n",
        "plt.xlabel(\"Logarithmic Frequency\")\n",
        "plt.ylabel(\"PII Label\")\n",
        "plt.title(\"Frequency of PII Labels\")\n",
        "plt.xticks(rotation=0)\n",
        "plt.xscale(\"log\")\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_B5Ng58Bj3J7"
      },
      "outputs": [],
      "source": [
        "# PII Location\n",
        "pii_position_normalized = [\n",
        "    (label, pos / total) for _, label, pos, total in pii_positions\n",
        "]\n",
        "pii_position_df = pd.DataFrame(pii_position_normalized, columns=[\"Label\", \"Position\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 487
        },
        "id": "0WbbyvTLGZz2",
        "outputId": "a90527e1-31fd-4e3f-d576-7600eed19d77"
      },
      "outputs": [],
      "source": [
        "# Plot the location of PIIs in the documents\n",
        "plt.figure(figsize=(7, 5))\n",
        "sns.histplot(\n",
        "    pii_position_df,\n",
        "    x=\"Position\",\n",
        "    bins=20,\n",
        "    palette=\"pastel\",\n",
        "    hue=\"Label\",\n",
        "    multiple=\"stack\",\n",
        ")\n",
        "plt.title(\"Position of PII Token in a Document\")\n",
        "plt.xlabel(\"Location\")\n",
        "plt.ylabel(\"Frequency\")\n",
        "plt.yscale(\"log\")\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rYIW9_SQjxMb",
        "outputId": "ddb5eb63-ad05-4273-fd1a-e005858c8f0d"
      },
      "outputs": [],
      "source": [
        "# PII Label Token Distance\n",
        "labels = set([pii[1] for pii in pii_positions])\n",
        "label_distance = {la1: {la2: [] for la2 in labels} for la1 in labels}\n",
        "\n",
        "# Calculate distances between PII tokens in the same document\n",
        "for doc1, label1, pos1, _ in pii_positions:\n",
        "    for doc2, label2, pos2, _ in pii_positions:\n",
        "        if doc1 == doc2 and pos1 != pos2:\n",
        "            label_distance[label1][label2].append(abs(pos1 - pos2))\n",
        "\n",
        "\n",
        "label_distance_df = pd.DataFrame(label_distance)\n",
        "label_avg_distance = label_distance_df.map(lambda x: np.mean(x))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 503
        },
        "id": "drOvomqPMtq9",
        "outputId": "bb154a8e-dd3a-4778-c2c4-95e86af0f289"
      },
      "outputs": [],
      "source": [
        "# Plot the spacing between PII data\n",
        "plt.figure(figsize=(5, 4))\n",
        "sns.heatmap(label_avg_distance, annot=True, fmt=\".0f\", linewidth=0.5, cmap=\"crest\")\n",
        "plt.title(\"Average Distance (Number of Tokens) Between PII Labels\")\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 367
        },
        "id": "KqwPYMkRWvbP",
        "outputId": "72111671-71d9-4340-d5f2-63bc40d856ce"
      },
      "outputs": [],
      "source": [
        "from wordcloud import WordCloud\n",
        "\n",
        "# Create a word cloud of PII data\n",
        "wordcloud = WordCloud(width=900, height=600, background_color=\"white\").generate(\n",
        "    \" \".join(pii_df[\"Token\"])\n",
        ")\n",
        "# plt.figure(figsize=(7, 6))\n",
        "plt.imshow(wordcloud, interpolation=\"bilinear\")\n",
        "plt.axis(\"off\")\n",
        "# plt.title('Word Cloud of PII Data')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 333
        },
        "id": "gQsiRJ6tXFmH",
        "outputId": "eb64141c-eb8d-41d4-efc2-ac1da0d16cb7"
      },
      "outputs": [],
      "source": [
        "pii_df[\"Token Length\"] = pii_df[\"Token\"].apply(len)\n",
        "plt.figure(figsize=(4, 3))\n",
        "sns.histplot(pii_df[\"Token Length\"], bins=15, color=\"skyblue\")\n",
        "plt.xlabel(\"Token Length (in Alphabetic Characters)\")\n",
        "plt.ylabel(\"Frequency\")\n",
        "plt.yscale(\"log\")\n",
        "plt.title(\"Distribution of PII Token Lengths\")\n",
        "plt.show()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.7"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}