LogisticRegression(sci_kit).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LogisticRegression(sci kit).ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "xVg-lho6_ezp",
        "colab_type": "code",
        "outputId": "4d12cab7-2385-4386-fd9d-107ca697bc9f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!pip install zeugma"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting zeugma\n",
            "  Downloading https://files.pythonhosted.org/packages/37/06/987f0c591e4f46fc31446d541d17d63981394f099c455955680dfe7bd980/zeugma-0.46.tar.gz\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from zeugma) (1.18.4)\n",
            "Requirement already satisfied: Cython>=0.27.3 in /usr/local/lib/python3.6/dist-packages (from zeugma) (0.29.18)\n",
            "Requirement already satisfied: pandas>=0.20.3 in /usr/local/lib/python3.6/dist-packages (from zeugma) (1.0.3)\n",
            "Requirement already satisfied: gensim>=3.5.0 in /usr/local/lib/python3.6/dist-packages (from zeugma) (3.6.0)\n",
            "Requirement already satisfied: scikit_learn>=0.19.1 in /usr/local/lib/python3.6/dist-packages (from zeugma) (0.22.2.post1)\n",
            "Requirement already satisfied: tensorflow>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from zeugma) (2.2.0)\n",
            "Requirement already satisfied: keras>=2.1.3 in /usr/local/lib/python3.6/dist-packages (from zeugma) (2.3.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.20.3->zeugma) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.20.3->zeugma) (2.8.1)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.5.0->zeugma) (1.4.1)\n",
            "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.5.0->zeugma) (2.0.0)\n",
            "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.5.0->zeugma) (1.12.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit_learn>=0.19.1->zeugma) (0.15.1)\n",
            "Requirement already satisfied: tensorboard<2.3.0,>=2.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (2.2.1)\n",
            "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (2.10.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (1.29.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (0.2.0)\n",
            "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (1.6.3)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (3.2.1)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (0.9.0)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (3.10.0)\n",
            "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (0.34.2)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (1.1.2)\n",
            "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (0.3.3)\n",
            "Requirement already satisfied: tensorflow-estimator<2.3.0,>=2.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (2.2.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (1.12.1)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->zeugma) (1.1.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras>=2.1.3->zeugma) (3.13)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from keras>=2.1.3->zeugma) (1.0.8)\n",
            "Requirement already satisfied: boto in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim>=3.5.0->zeugma) (2.49.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim>=3.5.0->zeugma) (2.23.0)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim>=3.5.0->zeugma) (1.13.13)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (46.3.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (0.4.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (3.2.2)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (1.7.2)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (1.0.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (1.6.0.post3)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim>=3.5.0->zeugma) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim>=3.5.0->zeugma) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim>=3.5.0->zeugma) (2.9)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim>=3.5.0->zeugma) (2020.4.5.1)\n",
            "Requirement already satisfied: botocore<1.17.0,>=1.16.13 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim>=3.5.0->zeugma) (1.16.13)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim>=3.5.0->zeugma) (0.10.0)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim>=3.5.0->zeugma) (0.3.3)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (1.6.0)\n",
            "Requirement already satisfied: cachetools<3.2,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (3.1.1)\n",
            "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (4.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (0.2.8)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.17.0,>=1.16.13->boto3->smart-open>=1.2.1->gensim>=3.5.0->zeugma) (0.15.2)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (3.1.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (3.1.0)\n",
            "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.6/dist-packages (from rsa<4.1,>=3.1.4->google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow>=1.5.0->zeugma) (0.4.8)\n",
            "Building wheels for collected packages: zeugma\n",
            "  Building wheel for zeugma (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for zeugma: filename=zeugma-0.46-cp36-none-any.whl size=8612 sha256=49def7ae26ad658c4c79be9d58f4b0e74277f953db6e93f1a9a7422ec69bbf03\n",
            "  Stored in directory: /root/.cache/pip/wheels/49/ce/d3/22bc15de9112558b220d9dba3bfcd7d9ad0d8cc4d44d3e7813\n",
            "Successfully built zeugma\n",
            "Installing collected packages: zeugma\n",
            "Successfully installed zeugma-0.46\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wq9KjbkYANQu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from zeugma.embeddings import EmbeddingTransformer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pr9xqX_kAoUa",
        "colab_type": "code",
        "outputId": "894b1d56-43d5-4a92-ac4f-6ad55e1e58d8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "def preprocess(data):\n",
        "    '''                                                                         \n",
        "    Credit goes to https://www.kaggle.com/gpreda/jigsaw-fast-compact-solution   \n",
        "    '''\n",
        "    punct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~`\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°\\\n",
        "£€\\×™√²—–&'\n",
        "    def clean_special_chars(text, punct):\n",
        "        for p in punct:\n",
        "            text = text.replace(p, ' ')\n",
        "        return text\n",
        "\n",
        "    data = clean_special_chars(str(data), punct)\n",
        "    data = data.split()\n",
        "    stop_words = set(stopwords.words('english'))\n",
        "    cleaned = [word for word in data if word not in stop_words]\n",
        "    return \" \".join(cleaned)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NvwqWCFeBFzM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_file = \"/content/drive/My Drive/MachineLearning/toxic/train.csv\"\n",
        "df = pd.read_csv(train_file)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CF6HxUxHBmRi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Clean data\n",
        "df['clean_text'] = df['comment_text'].apply(lambda x: preprocess(x))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jjvu0EcMBJfp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Split into training and test data\n",
        "train, test = train_test_split(df, test_size=0.2)\n",
        "x_train = train['clean_text']\n",
        "x_test = test['clean_text']\n",
        "y_train = np.where(train['target'] >= 0.5, 1, 0)\n",
        "y_test = np.where(test['target'] >= 0.5, 1, 0)\n",
        "#y_cat_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]\n",
        "#y_cat_test = test[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SXepTNFyRwnw",
        "colab_type": "code",
        "outputId": "c3182c30-5bc5-4bd6-deee-010f915c95e4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "y_cat_train.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target</th>\n",
              "      <th>severe_toxicity</th>\n",
              "      <th>obscene</th>\n",
              "      <th>identity_attack</th>\n",
              "      <th>insult</th>\n",
              "      <th>threat</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1491557</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1744262</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>213320</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>751548</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>344134</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         target  severe_toxicity  obscene  identity_attack  insult  threat\n",
              "1491557     0.0              0.0      0.0              0.0     0.0     0.0\n",
              "1744262     0.0              0.0      0.0              0.0     0.0     0.0\n",
              "213320      0.0              0.0      0.0              0.0     0.0     0.0\n",
              "751548      0.0              0.0      0.0              0.0     0.0     0.0\n",
              "344134      0.0              0.0      0.0              0.0     0.0     0.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1kutgCjg_iEi",
        "colab_type": "code",
        "outputId": "46af5d1f-5410-4152-fd92-9a16ac821727",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        }
      },
      "source": [
        "#Encode training data with glove vectors\n",
        "glove = EmbeddingTransformer('glove')\n",
        "x_train = glove.transform(x_train)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[=================================-----------------] 66.7% 69.9/104.8MB downloaded\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:253: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JalLk6buDP_E",
        "colab_type": "code",
        "outputId": "ca5a63da-1d7c-4c95-df14-bb0449f3279d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "#Fit LR model\n",
        "target_model = LogisticRegression(C=5, random_state=42, solver='sag', max_iter=1000, n_jobs=-1)\n",
        "target_model.fit(x_train, y_train)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LogisticRegression(C=5, class_weight=None, dual=False, fit_intercept=True,\n",
              "                   intercept_scaling=1, l1_ratio=None, max_iter=1000,\n",
              "                   multi_class='auto', n_jobs=-1, penalty='l2', random_state=42,\n",
              "                   solver='sag', tol=0.0001, verbose=0, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-jwqxK6IEvsY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Validate model on test data\n",
        "x_test_glove = glove.transform(x_test)\n",
        "predictions = target_model.predict_proba(x_test_glove)[:,1]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4YGdlJNEtA8h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "submission = pd.DataFrame.from_dict({\n",
        "    'id': test['id'],\n",
        "    'prediction': predictions\n",
        "})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uT1DYHjhtPr5",
        "colab_type": "code",
        "outputId": "a7202629-559e-4a31-9b8a-25cbe3b38d4f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        }
      },
      "source": [
        "submission"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>prediction</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>380532</th>\n",
              "      <td>708537</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1089178</th>\n",
              "      <td>5447745</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>955143</th>\n",
              "      <td>5287055</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1099517</th>\n",
              "      <td>5460127</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1068116</th>\n",
              "      <td>5421929</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1759729</th>\n",
              "      <td>6278512</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15882</th>\n",
              "      <td>261688</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>161388</th>\n",
              "      <td>439259</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>398668</th>\n",
              "      <td>730690</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>290427</th>\n",
              "      <td>597620</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>360975 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "              id  prediction\n",
              "380532    708537           0\n",
              "1089178  5447745           0\n",
              "955143   5287055           0\n",
              "1099517  5460127           0\n",
              "1068116  5421929           0\n",
              "...          ...         ...\n",
              "1759729  6278512           0\n",
              "15882     261688           0\n",
              "161388    439259           0\n",
              "398668    730690           0\n",
              "290427    597620           0\n",
              "\n",
              "[360975 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F8WIJGqdrOEu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# From baseline kernel\n",
        "from sklearn import metrics\n",
        "def calculate_overall_auc(df, model_name):\n",
        "    true_labels = df[TOXICITY_COLUMN]>0.5\n",
        "    predicted_labels = df[model_name]\n",
        "    return metrics.roc_auc_score(true_labels, predicted_labels)\n",
        "\n",
        "def power_mean(series, p):\n",
        "    total = sum(np.power(series, p))\n",
        "    return np.power(total / len(series), 1 / p)\n",
        "\n",
        "def get_final_metric(bias_df, overall_auc, POWER=-5, OVERALL_MODEL_WEIGHT=0.25):\n",
        "    bias_score = np.average([\n",
        "        power_mean(bias_df[SUBGROUP_AUC], POWER),\n",
        "        power_mean(bias_df[BPSN_AUC], POWER),\n",
        "        power_mean(bias_df[BNSP_AUC], POWER)\n",
        "    ])\n",
        "    return (OVERALL_MODEL_WEIGHT * overall_auc) + ((1 - OVERALL_MODEL_WEIGHT) * bias_score)\n",
        "\n",
        "\n",
        "\n",
        "SUBGROUP_AUC = 'subgroup_auc'\n",
        "BPSN_AUC = 'bpsn_auc'  # stands for background positive, subgroup negative\n",
        "BNSP_AUC = 'bnsp_auc'  # stands for background negative, subgroup positive\n",
        "\n",
        "def compute_auc(y_true, y_pred):\n",
        "    try:\n",
        "        return metrics.roc_auc_score(y_true, y_pred)\n",
        "    except ValueError:\n",
        "        return np.nan\n",
        "\n",
        "def compute_subgroup_auc(df, subgroup, label, model_name):\n",
        "    subgroup_examples = df[df[subgroup]>0.5]\n",
        "    return compute_auc((subgroup_examples[label]>0.5), subgroup_examples[model_name])\n",
        "\n",
        "def compute_bpsn_auc(df, subgroup, label, model_name):\n",
        "    \"\"\"Computes the AUC of the within-subgroup negative examples and the background positive examples.\"\"\"\n",
        "    subgroup_negative_examples = df[(df[subgroup]>0.5) & (df[label]<=0.5)]\n",
        "    non_subgroup_positive_examples = df[(df[subgroup]<=0.5) & (df[label]>0.5)]\n",
        "    examples = subgroup_negative_examples.append(non_subgroup_positive_examples)\n",
        "    return compute_auc(examples[label]>0.5, examples[model_name])\n",
        "\n",
        "def compute_bnsp_auc(df, subgroup, label, model_name):\n",
        "    \"\"\"Computes the AUC of the within-subgroup positive examples and the background negative examples.\"\"\"\n",
        "    subgroup_positive_examples = df[(df[subgroup]>0.5) & (df[label]>0.5)]\n",
        "    non_subgroup_negative_examples = df[(df[subgroup]<=0.5) & (df[label]<=0.5)]\n",
        "    examples = subgroup_positive_examples.append(non_subgroup_negative_examples)\n",
        "    return compute_auc(examples[label]>0.5, examples[model_name])\n",
        "\n",
        "def compute_bias_metrics_for_model(dataset,\n",
        "                                   subgroups,\n",
        "                                   model,\n",
        "                                   label_col,\n",
        "                                   include_asegs=False):\n",
        "    \"\"\"Computes per-subgroup metrics for all subgroups and one model.\"\"\"\n",
        "    records = []\n",
        "    for subgroup in subgroups:\n",
        "        record = {\n",
        "            'subgroup': subgroup,\n",
        "            'subgroup_size': len(dataset[dataset[subgroup]>0.5])\n",
        "        }\n",
        "        record[SUBGROUP_AUC] = compute_subgroup_auc(dataset, subgroup, label_col, model)\n",
        "        record[BPSN_AUC] = compute_bpsn_auc(dataset, subgroup, label_col, model)\n",
        "        record[BNSP_AUC] = compute_bnsp_auc(dataset, subgroup, label_col, model)\n",
        "        records.append(record)\n",
        "    return pd.DataFrame(records).sort_values('subgroup_auc', ascending=True)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9l9SLFFMrQ2C",
        "colab_type": "code",
        "outputId": "748b03ef-2d93-45e8-ffc7-c0d05ac431cd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        }
      },
      "source": [
        "identity_columns = [\n",
        "    'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',\n",
        "    'muslim', 'black', 'white', 'psychiatric_or_mental_illness']\n",
        "MODEL_NAME = 'model1'\n",
        "test[MODEL_NAME]= submission[\"prediction\"]\n",
        "TOXICITY_COLUMN = 'target'\n",
        "bias_metrics_df = compute_bias_metrics_for_model(test, identity_columns, MODEL_NAME, 'target')\n",
        "bias_metrics_df\n",
        "get_final_metric(bias_metrics_df, calculate_overall_auc(test, MODEL_NAME))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7612232051135233"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    }
  ]
}