Class 7 - Text Classifiers - Part 2 - Solution .ipynb

{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":23611,"status":"ok","timestamp":1716365889882,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"},"user_tz":-120},"id":"dcDm6T3HaRYG","outputId":"00a18a83-7a1a-4173-8908-783bd39e7843"},"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}],"source":["#Import drive\n","from google.colab import drive\n","#Mount Google Drive\n","ROOT=\"/content/drive\"\n","drive.mount(ROOT, force_remount=True)"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":36},"id":"6Zxub8uiRF7M","outputId":"02dac40f-15bb-476a-f19f-15d5ff63ca50","executionInfo":{"status":"ok","timestamp":1716365893381,"user_tz":-120,"elapsed":438,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"outputs":[{"output_type":"execute_result","data":{"text/plain":["'/content'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":2}],"source":["%pwd"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ajyqZyUQRLdL","outputId":"4d6eb6b6-f916-40fe-9cd9-3c41696a43c3","executionInfo":{"status":"ok","timestamp":1716365901045,"user_tz":-120,"elapsed":237,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["/content/drive/MyDrive/dlss24\n"]}],"source":["%cd /content/drive/MyDrive/dlss24"]},{"cell_type":"code","execution_count":4,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":36},"id":"LTf-v1gWRNv2","outputId":"c9d548f6-bad2-467f-fb1e-6059b042eff3","executionInfo":{"status":"ok","timestamp":1716365909794,"user_tz":-120,"elapsed":268,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"outputs":[{"output_type":"execute_result","data":{"text/plain":["'/content/drive/MyDrive/dlss24'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":4}],"source":["%pwd"]},{"cell_type":"markdown","metadata":{"id":"4lnmzacMRV9e"},"source":["# Data\n","\n","Download the corpora data from: https://codeocean.com/capsule/0078777/tree/v1\n","\n","Uploaded to a location outside your gitHub (to do not be tracked)\n","\n","You pwd should have the files:\n","\n","\n","1.   source_corpus.csv\n","\n"]},{"cell_type":"markdown","metadata":{"id":"BE3M-5mHbRdU"},"source":["\n","# Module IV:  - Class 7 - Text Classifiers\n","\n"]},{"cell_type":"markdown","metadata":{"id":"T5LxOrR6bjFY"},"source":["# Reading data"]},{"cell_type":"code","execution_count":5,"metadata":{"id":"TmnDoKifpFZ7","executionInfo":{"status":"ok","timestamp":1716365990294,"user_tz":-120,"elapsed":845,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"outputs":[],"source":["import pandas as pd\n"]},{"cell_type":"code","execution_count":6,"metadata":{"id":"eOaGLl-Lbxjf","executionInfo":{"status":"ok","timestamp":1716365997513,"user_tz":-120,"elapsed":1217,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"outputs":[],"source":["df_source_corpus=pd.read_csv('/content/drive/MyDrive/dlss24/source_corpus.csv')"]},{"cell_type":"code","source":["df_source_corpus = df_source_corpus.dropna(subset=['text'])"],"metadata":{"id":"V1zJsjYHxx77","executionInfo":{"status":"ok","timestamp":1716366016063,"user_tz":-120,"elapsed":210,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"execution_count":7,"outputs":[]},{"cell_type":"code","execution_count":8,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"executionInfo":{"elapsed":9,"status":"ok","timestamp":1716366018317,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"},"user_tz":-120},"id":"N1JiJSClKR2M","outputId":"e6a44b9c-f649-49c0-cf61-236ebd8dabc0"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["                   topic_44                topic_8  \\\n","0                 democracy  freedom and democracy   \n","1  freedom and human rights  freedom and democracy   \n","2             law and order      fabric of society   \n","3                  no topic               no topic   \n","4          civic mindedness      fabric of society   \n","\n","                                                text  \n","0  Two hundred summers ago, this Democratic Party...  \n","1  In 1992, the party Thomas Jefferson founded in...  \n","2  Our land reverberates with a battle cry of fru...  \n","3                     America is on the wrong track.  \n","4                   The American people are hurting.  "],"text/html":["\n","  <div id=\"df-74f41475-4b31-4609-92f4-b6699d09f4af\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>topic_44</th>\n","      <th>topic_8</th>\n","      <th>text</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>democracy</td>\n","      <td>freedom and democracy</td>\n","      <td>Two hundred summers ago, this Democratic Party...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>freedom and human rights</td>\n","      <td>freedom and democracy</td>\n","      <td>In 1992, the party Thomas Jefferson founded in...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>law and order</td>\n","      <td>fabric of society</td>\n","      <td>Our land reverberates with a battle cry of fru...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>no topic</td>\n","      <td>no topic</td>\n","      <td>America is on the wrong track.</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>civic mindedness</td>\n","      <td>fabric of society</td>\n","      <td>The American people are hurting.</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-74f41475-4b31-4609-92f4-b6699d09f4af')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-74f41475-4b31-4609-92f4-b6699d09f4af button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-74f41475-4b31-4609-92f4-b6699d09f4af');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-fd9929e5-9cab-4f12-b158-36045c085b94\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-fd9929e5-9cab-4f12-b158-36045c085b94')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-fd9929e5-9cab-4f12-b158-36045c085b94 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df_source_corpus"}},"metadata":{},"execution_count":8}],"source":["df_source_corpus.head()"]},{"cell_type":"code","source":["\n"],"metadata":{"id":"PQKMyozRLKTz"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Word2vec\n","\n","[Image of word2vec 1-hidden layer NN](https://becominghuman.ai/mathematical-introduction-to-glove-word-embedding-60f24154e54c)\n","\n","Word2vec creates vectors that represent the context of words, while GloVe creates vectors that represent the co-occurrence of words.\n","\n","Word2vec uses a shallow neural network to create vectors, while GloVe uses a global matrix factorization technique\n","\n","\n","Word2vec requires a large amount of training data, while GloVe can be trained on smaller datasets. This makes GloVe more suitable for smaller tasks, while Word2vec is better suited for larger applications."],"metadata":{"id":"jgYz_Kq2sd8x"}},{"cell_type":"code","source":["# word2vec requires sentences as input\n","import nltk\n","nltk.download('stopwords')\n","nltk.download('punkt')\n","from string import punctuation\n","translator = str.maketrans('','',punctuation)\n","from nltk.corpus import stopwords\n","stoplist = set(stopwords.words('english'))\n","from nltk.stem import SnowballStemmer\n","stemmer = SnowballStemmer('english')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"x_LiEdoZshp7","executionInfo":{"status":"ok","timestamp":1716366181781,"user_tz":-120,"elapsed":2845,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"961f291e-74c3-4340-f0cb-024565611ba1"},"execution_count":9,"outputs":[{"output_type":"stream","name":"stderr","text":["[nltk_data] Downloading package stopwords to /root/nltk_data...\n","[nltk_data]   Unzipping corpora/stopwords.zip.\n","[nltk_data] Downloading package punkt to /root/nltk_data...\n","[nltk_data]   Unzipping tokenizers/punkt.zip.\n"]}]},{"cell_type":"code","source":["#from previous class\n","def normalize_text(doc):\n","    \"Input doc and return clean list of tokens\"\n","    doc = doc.replace('\\r', ' ').replace('\\n', ' ')\n","    lower = doc.lower() # all lower case\n","    nopunc = lower.translate(translator) # remove punctuation\n","    words = nopunc.split() # split into tokens\n","    nostop = [w for w in words if w not in stoplist] # remove stopwords\n","    no_numbers = [w if not w.isdigit() else '#' for w in nostop] # normalize numbers\n","    stemmed = [stemmer.stem(w) for w in no_numbers] # stem each word\n","    return stemmed"],"metadata":{"id":"McKq2yIqslqN","executionInfo":{"status":"ok","timestamp":1716366929751,"user_tz":-120,"elapsed":252,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"execution_count":11,"outputs":[]},{"cell_type":"markdown","source":["\n","\n","> 1. Apply the normalize_text function to the data (df_source_corpus)\n","2. Train word2vec on all of the data (All that is required is that the input yields one sentence (list of utf8 words) after another.)\n","\n"],"metadata":{"id":"tOQZ7jdpyECO"}},{"cell_type":"code","source":["from gensim.models import Word2Vec"],"metadata":{"id":"b7j35xacaCU5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"gcy5pHwOaCal"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"TM70IC9AaCzt"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["sentences_normalized=[normalize_text(row) for row in df_source_corpus['text']]\n"],"metadata":{"id":"VuFXHf68wZWi","executionInfo":{"status":"ok","timestamp":1716366962334,"user_tz":-120,"elapsed":29161,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"execution_count":12,"outputs":[]},{"cell_type":"code","source":["df_source_corpus['text'][0]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":53},"id":"oaV0kDzUcafL","executionInfo":{"status":"ok","timestamp":1716366986952,"user_tz":-120,"elapsed":5,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"bf4e050f-ec3e-440d-a80d-95f73f3d6000"},"execution_count":14,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'Two hundred summers ago, this Democratic Party was founded by the man whose burning pen fired the spirit of the American Revolution - who once argued we should overthrow our own government every 20 years to renew our freedom and keep pace with a changing world.'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":14}]},{"cell_type":"code","source":["sentences_normalized[0]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Mi0pAgk2cQ4K","executionInfo":{"status":"ok","timestamp":1716366990052,"user_tz":-120,"elapsed":220,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"ae46d14c-04bd-4ecd-e262-d17a69bef454"},"execution_count":15,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['two',\n"," 'hundr',\n"," 'summer',\n"," 'ago',\n"," 'democrat',\n"," 'parti',\n"," 'found',\n"," 'man',\n"," 'whose',\n"," 'burn',\n"," 'pen',\n"," 'fire',\n"," 'spirit',\n"," 'american',\n"," 'revolut',\n"," 'argu',\n"," 'overthrow',\n"," 'govern',\n"," 'everi',\n"," '#',\n"," 'year',\n"," 'renew',\n"," 'freedom',\n"," 'keep',\n"," 'pace',\n"," 'chang',\n"," 'world']"]},"metadata":{},"execution_count":15}]},{"cell_type":"code","source":["sentences_normalized[44]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Na-mqMFMx7kd","executionInfo":{"status":"ok","timestamp":1716350874400,"user_tz":-120,"elapsed":6,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"c29634d1-2a49-409e-d3a4-26cf8c2c75fb"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['reject',\n"," 'donoth',\n"," 'govern',\n"," 'last',\n"," 'twelv',\n"," 'year',\n"," 'big',\n"," 'govern',\n"," 'theori',\n"," 'say',\n"," 'hamstr',\n"," 'busi',\n"," 'tax',\n"," 'spend',\n"," 'way',\n"," 'prosper']"]},"metadata":{},"execution_count":13}]},{"cell_type":"code","source":["# train the model\n","from gensim.models import Word2Vec\n","w2v = Word2Vec(sentences_normalized,  # list of tokenized sentences\n","               workers = 8, # Number of threads to run in parallel\n","               vector_size=300,  # Word vector dimensionality\n","               min_count =  25, # Minimum word count\n","               window = 5, # Context window size\n","               sample = 1e-3, # Downsample setting for frequent words\n","               )\n","\n","\n","#w2v.save('w2v-vectors.pkl')\n","\n"],"metadata":{"id":"qZoFbC0fsxBX","executionInfo":{"status":"ok","timestamp":1716367102542,"user_tz":-120,"elapsed":15417,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"execution_count":16,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"QsteGqaCdrrV","executionInfo":{"status":"ok","timestamp":1716367313665,"user_tz":-120,"elapsed":256,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"e9d95943-9878-4e54-8bd2-a012de9b1b0d"},"execution_count":17,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<gensim.models.word2vec.Word2Vec at 0x7c7a828f3ca0>"]},"metadata":{},"execution_count":17}]},{"cell_type":"markdown","source":["\n","\n","> 1. Look for the word vectors for immigration, economy, security and rights\n","2. See which of these (economy, security and rights) is closer to immigration\n","\n","Try w2v.wv.\n","\n"],"metadata":{"id":"pAi5kOetjPYT"}},{"cell_type":"code","source":[" len(w2v.wv.index_to_key)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"OTFoU-hFanjb","executionInfo":{"status":"ok","timestamp":1716367960370,"user_tz":-120,"elapsed":5,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"16bf095a-dee4-49d2-e302-68d284d9e32c"},"execution_count":19,"outputs":[{"output_type":"execute_result","data":{"text/plain":["3557"]},"metadata":{},"execution_count":19}]},{"cell_type":"code","source":["w2v.wv.index_to_key[0]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":36},"id":"5zuEbw1tanfy","executionInfo":{"status":"ok","timestamp":1716368112179,"user_tz":-120,"elapsed":6,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"b350b4c1-bd30-4e83-f359-da87f509d3a0"},"execution_count":20,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'#'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":20}]},{"cell_type":"code","source":["w2v.wv.index_to_key['immigr']"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":142},"id":"sk1CtZtbanch","executionInfo":{"status":"error","timestamp":1716368119686,"user_tz":-120,"elapsed":238,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"69ccda9c-8127-41d1-c770-dc454fcd0648"},"execution_count":21,"outputs":[{"output_type":"error","ename":"TypeError","evalue":"list indices must be integers or slices, not str","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-21-09302e88eb6f>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mw2v\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex_to_key\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'immigr'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mTypeError\u001b[0m: list indices must be integers or slices, not str"]}]},{"cell_type":"code","source":[],"metadata":{"id":"yAUOM3oQanZY"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["#immigration\n","[i for i in w2v.wv.index_to_key if i[0:2]=='im']"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"z6mgEzGM07TL","executionInfo":{"status":"ok","timestamp":1716368236626,"user_tz":-120,"elapsed":228,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"70064ccf-1c89-4d83-e4df-5606ec47c637"},"execution_count":36,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['improv',\n"," 'implement',\n"," 'import',\n"," 'impact',\n"," 'immigr',\n"," 'immedi',\n"," 'impos',\n"," 'immunis',\n"," 'imagin',\n"," 'imprison',\n"," 'immens',\n"," 'implic',\n"," 'imposs',\n"," 'imag',\n"," 'imper',\n"," 'impedi',\n"," 'impair',\n"," 'imbal',\n"," 'imposit',\n"," 'immun',\n"," 'imparti',\n"," 'imped',\n"," 'immers']"]},"metadata":{},"execution_count":36}]},{"cell_type":"code","source":["#economy\n","[i for i in w2v.wv.index_to_key if i[0:2]=='ec']"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bm4ErYwh1vOt","executionInfo":{"status":"ok","timestamp":1716368220070,"user_tz":-120,"elapsed":290,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"cd5ae57d-289c-432b-e92a-f2ba4673c883"},"execution_count":33,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['econom', 'economi', 'ece', 'ecolog', 'ecosystem', 'economist']"]},"metadata":{},"execution_count":33}]},{"cell_type":"code","source":["#security\n"," [i for i in w2v.wv.index_to_key if i[0:4]=='secu']"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2U6h1AsM163F","executionInfo":{"status":"ok","timestamp":1716350891138,"user_tz":-120,"elapsed":43,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"647e3061-3c33-4a10-810b-24e92281dcf4"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['secur']"]},"metadata":{},"execution_count":17}]},{"cell_type":"code","source":["#rights\n"," [i for i in w2v.wv.index_to_key if i[0:4]=='righ']"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"wFYDx7ER2D2j","executionInfo":{"status":"ok","timestamp":1716350891138,"user_tz":-120,"elapsed":37,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"4bfaaa9e-daf8-4a2f-e362-fabb92945763"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['right']"]},"metadata":{},"execution_count":18}]},{"cell_type":"code","source":["w2v.wv.similarity('immigr','econom') # similarity"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"LGQkQwOlz-iP","executionInfo":{"status":"ok","timestamp":1716350891138,"user_tz":-120,"elapsed":24,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"e0c8d801-ce1f-4c39-e674-2f0c38726e31"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.06183112"]},"metadata":{},"execution_count":21}]},{"cell_type":"code","source":["w2v.wv.similarity('immigr','secur') # similarity"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4AjygM_K12pE","executionInfo":{"status":"ok","timestamp":1716350891138,"user_tz":-120,"elapsed":20,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"8c7920a2-0e8b-4573-bb72-d514881f5702"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.1503622"]},"metadata":{},"execution_count":22}]},{"cell_type":"code","source":["w2v.wv.similarity('immigr','right') # similarity"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"rGr7nTnH2HL-","executionInfo":{"status":"ok","timestamp":1716350891138,"user_tz":-120,"elapsed":16,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"ba3966a5-1a2b-4a26-bc3c-21d995fd3035"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.20254493"]},"metadata":{},"execution_count":23}]},{"cell_type":"code","source":["w2v.wv.most_similar('immigr') # most similar words"],"metadata":{"id":"qLbrPKxbjzx-","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1716368286604,"user_tz":-120,"elapsed":233,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"0f5fef93-433c-4b53-b6d2-b09aeff8853a"},"execution_count":37,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[('asylum', 0.6509559750556946),\n"," ('deter', 0.58147132396698),\n"," ('refuge', 0.5697773098945618),\n"," ('seeker', 0.5590358972549438),\n"," ('deport', 0.5226292014122009),\n"," ('coven', 0.5181108117103577),\n"," ('visa', 0.5159613490104675),\n"," ('grievanc', 0.5148912072181702),\n"," ('illeg', 0.513752818107605),\n"," ('copyright', 0.5129243731498718)]"]},"metadata":{},"execution_count":37}]},{"cell_type":"markdown","source":["\n","\n","> Analogies with the dataset:\n","1. Scientist is to man as __ is to woman\n","2. Scientist is to woman as __ is to man\n","\n"],"metadata":{"id":"pFmF2Otwj_ku"}},{"cell_type":"code","source":["# analogies: economy is to man as __ is to woman\n","w2v.wv.most_similar(positive=['scientist','man'],\n","                 negative=['woman'])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"YIroc3te0qlh","executionInfo":{"status":"ok","timestamp":1716368338482,"user_tz":-120,"elapsed":240,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"fe94e654-c9bd-4f26-f869-465ca2c4bb6c"},"execution_count":38,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[('leader', 0.6559566855430603),\n"," ('format', 0.6077491641044617),\n"," ('bid', 0.6055317521095276),\n"," ('elit', 0.5831114053726196),\n"," ('brightest', 0.5734731554985046),\n"," ('philanthropi', 0.5641298294067383),\n"," ('discuss', 0.556861162185669),\n"," ('globalis', 0.555897057056427),\n"," ('stakehold', 0.5556820631027222),\n"," ('distinct', 0.5549700856208801)]"]},"metadata":{},"execution_count":38}]},{"cell_type":"code","source":["w2v.wv.most_similar(positive=['scientist','woman'],\n","                 negative=['man'])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"-_DPntSh2VFc","executionInfo":{"status":"ok","timestamp":1716350891139,"user_tz":-120,"elapsed":9,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"dde90f01-c5fd-49c1-f4f0-c8c37cad4894"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[('clinician', 0.652355968952179),\n"," ('bilingu', 0.5782425403594971),\n"," ('fás', 0.567655622959137),\n"," ('autism', 0.5564168691635132),\n"," ('sandf', 0.5482902526855469),\n"," ('leisur', 0.546715259552002),\n"," ('advisor', 0.5436047911643982),\n"," ('distinct', 0.53558748960495),\n"," ('athlet', 0.5323788523674011),\n"," ('horizon', 0.5315958261489868)]"]},"metadata":{},"execution_count":25}]},{"cell_type":"markdown","source":["# GLove\n","\n"],"metadata":{"id":"GcaseSp126K2"}},{"cell_type":"markdown","source":["\n","\n","> Train Glove on your dataset and check most similar words to environment\n","\n"],"metadata":{"id":"qLaY8FBqpTqW"}},{"cell_type":"code","source":["!pip install glove-python3"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"RqCdsLVb3UV9","executionInfo":{"status":"ok","timestamp":1716369420018,"user_tz":-120,"elapsed":28452,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"dd9dbe45-0be4-4173-faf1-1702c19f7795"},"execution_count":39,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting glove-python3\n","  Downloading glove_python3-0.1.0.tar.gz (326 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m327.0/327.0 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from glove-python3) (1.25.2)\n","Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from glove-python3) (1.11.4)\n","Building wheels for collected packages: glove-python3\n","  Building wheel for glove-python3 (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for glove-python3: filename=glove_python3-0.1.0-cp310-cp310-linux_x86_64.whl size=1065513 sha256=64db91b479179434533dadb02685018cf0a8eac0f6bcb5b2b7b79863c328e792\n","  Stored in directory: /root/.cache/pip/wheels/fe/2f/79/34314d44a0907e90e323c8c182ec23f126eb460829e02d98cf\n","Successfully built glove-python3\n","Installing collected packages: glove-python3\n","Successfully installed glove-python3-0.1.0\n"]}]},{"cell_type":"code","source":["import itertools\n","from glove import Corpus, Glove"],"metadata":{"id":"CAB9XU2c3Bpc","executionInfo":{"status":"ok","timestamp":1716369427584,"user_tz":-120,"elapsed":352,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"execution_count":40,"outputs":[]},{"cell_type":"code","source":["corpus = Corpus()\n","corpus.fit(sentences_normalized, window=10)\n","glove = Glove(no_components=100, learning_rate=0.05)"],"metadata":{"id":"f3Jox1Tc273w","executionInfo":{"status":"ok","timestamp":1716369432677,"user_tz":-120,"elapsed":366,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"execution_count":41,"outputs":[]},{"cell_type":"code","source":["glove.fit(corpus.matrix, epochs=30, no_threads=4, verbose=True)\n","glove.add_dictionary(corpus.dictionary)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6tB7T1gO3z4r","executionInfo":{"status":"ok","timestamp":1716369537988,"user_tz":-120,"elapsed":104493,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"a1947a74-9ce9-4179-e0bd-90e8f39a6a63"},"execution_count":42,"outputs":[{"output_type":"stream","name":"stdout","text":["Performing 30 training epochs with 4 threads\n","Epoch 0\n","Epoch 1\n","Epoch 2\n","Epoch 3\n","Epoch 4\n","Epoch 5\n","Epoch 6\n","Epoch 7\n","Epoch 8\n","Epoch 9\n","Epoch 10\n","Epoch 11\n","Epoch 12\n","Epoch 13\n","Epoch 14\n","Epoch 15\n","Epoch 16\n","Epoch 17\n","Epoch 18\n","Epoch 19\n","Epoch 20\n","Epoch 21\n","Epoch 22\n","Epoch 23\n","Epoch 24\n","Epoch 25\n","Epoch 26\n","Epoch 27\n","Epoch 28\n","Epoch 29\n"]}]},{"cell_type":"code","source":["glove.word_vectors[glove.dictionary['environment']].shape"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"CXBikr0kpgKB","executionInfo":{"status":"ok","timestamp":1716353633842,"user_tz":-120,"elapsed":6,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"ac255437-9df4-4dc3-94da-51cbff877dba"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(100,)"]},"metadata":{},"execution_count":69}]},{"cell_type":"code","source":["glove.word_vectors[glove.dictionary['environment']]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"aqkLH5Xy3uZy","executionInfo":{"status":"ok","timestamp":1716353612340,"user_tz":-120,"elapsed":403,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"14fa11f1-708a-4c58-94e8-a927b8e17567"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([ 1.72157389e-01,  5.24760726e-01,  4.79366011e-01,  2.44504808e-01,\n","        8.62694774e-02, -4.30167892e-01,  3.50070104e-01,  1.96383570e-01,\n","       -3.74036851e-02,  9.22049750e-02,  4.17906902e-02,  8.54516579e-02,\n","       -6.95356852e-02,  1.20410553e-01,  5.61525833e-01, -7.81944678e-02,\n","       -7.11222706e-02, -1.69982273e-02,  8.98567312e-02, -2.08754969e-01,\n","        6.95545429e-02, -3.52967573e-01, -4.62215007e-01,  4.16381245e-01,\n","        2.05167332e-01, -2.51081403e-01, -3.56390643e-01, -2.34925635e-01,\n","       -4.79705582e-02,  2.10012633e-02, -3.81413661e-01,  6.28390035e-02,\n","       -1.52516746e-01,  3.80466818e-03, -1.57986222e-01, -1.79531348e-01,\n","       -3.48323882e-01, -1.68420041e-01, -5.08048309e-02, -1.87594650e-01,\n","       -1.18914746e-01,  1.26961687e-01, -2.63438630e-01, -3.27405734e-02,\n","       -5.47137856e-01,  2.11711152e-01,  9.77844705e-04,  6.26988483e-02,\n","        1.60322921e-01,  1.39169804e-01, -2.14607942e-01, -2.22307912e-01,\n","       -3.98466476e-02,  2.29446894e-01, -2.03487201e-01,  5.38351409e-01,\n","        1.42821257e-01, -3.00252419e-01,  2.00629157e-01, -4.76804917e-01,\n","       -5.29979059e-04,  3.79415318e-01,  1.50596725e-02, -2.28650932e-01,\n","       -4.03868403e-02, -2.49690234e-01,  2.69359965e-01,  2.62913214e-01,\n","       -7.02906499e-02,  2.58123750e-01,  2.40819012e-01, -1.87984646e-01,\n","        5.87917704e-01,  1.59506879e-01,  1.22422068e-01, -2.73113142e-01,\n","       -3.80111200e-02,  1.66644900e-01,  1.15632072e-01, -1.36522238e-01,\n","       -1.16171042e-01, -2.91918665e-02,  3.87510441e-01,  3.37447876e-02,\n","       -2.63258820e-01, -1.93821393e-01, -1.04598770e-01, -1.04507307e-01,\n","       -2.41641568e-01,  5.07268937e-01,  1.67304382e-01, -3.68263965e-01,\n","        1.14900731e-01,  1.08578241e-01, -2.24172011e-02,  2.67141533e-01,\n","       -2.19312415e-01, -9.74360277e-02,  1.56691351e-04, -3.28983545e-01])"]},"metadata":{},"execution_count":67}]},{"cell_type":"code","source":["glove.most_similar('environment')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"EeriekgC3yP8","executionInfo":{"status":"ok","timestamp":1716353621550,"user_tz":-120,"elapsed":329,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"0dedb216-ad43-42b6-b1fe-cc4492722bb8"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[('impact', 0.6321464721438153),\n"," ('stewardship', 0.6001896002229782),\n"," ('sustain', 0.5874697464152396),\n"," ('ecolog', 0.575077843653751)]"]},"metadata":{},"execution_count":68}]},{"cell_type":"markdown","source":["\n","\n","# 2017: Attention is all you need\n","\n"],"metadata":{"id":"eHP6ewXACXpC"}},{"cell_type":"code","source":["# Attention Basically\n","import numpy as np\n","\n","class Node:\n","\n","  def __init__(self):\n","\n","    # the vector stored at this node (what is learned, the output of the layer)\n","    self.data=np.random.randn(20)\n","\n","    # weights governing how this node interacts with other nodes\n","    self.wkey = np.random.randn(20, 20)\n","    self.wquery = np.random.randn(20, 20)\n","    self.wvalue = np.random.randn(20, 20)\n","\n","  def key(self):\n","    # what do I have?\n","    return self.wkey @ self.data\n","\n","  def query(self):\n","    # what am I looking for? (next work is the text shift by 1, classification is the class)\n","    return self.wquery @ self.data\n","\n","  def value(self):\n","    # what do I oublicly reveal to others?\n","    return self.wvalue @ self.data\n"],"metadata":{"id":"l6oXzqVbCXRT","executionInfo":{"status":"ok","timestamp":1716371049627,"user_tz":-120,"elapsed":370,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"execution_count":43,"outputs":[]},{"cell_type":"code","source":["class Graph:\n","\n","  def __init__(self):\n","    # make 10 nodes\n","    self.nodes=[Node() for _ in range(10)]\n","    #make 40 edges\n","    randi=lambda: np.random.randint(len(self.nodes))\n","    self.edges=[[randi(),randi()] for _ in range(40)]\n","\n","  def run(self):\n","\n","    updates=[]\n","    for i,n in enumerate(self.nodes):\n","\n","      #what is the node looking for?\n","      q=n.query()\n","\n","      #find all edges that are input to this node\n","      inputs = [self.nodes[ifrom] for (ifrom, ito) in self.edges if ito==i]\n","      if len(inputs)==0:\n","        continue #ignore, next in for loop\n","      # gather keys, i.e what they hold\n","      keys=[m.key() for m in inputs]\n","      #calculate compatibilities: dot product of  key with query\n","      scores=[k.dot(q) for k in keys]\n","      #softmax them so they sum to 1\n","      scores=np.exp(scores)\n","      scores=scores/np.sum(scores)\n","      # gather the appropriate values with weighted sum\n","      values=[m.value() for m in inputs]\n","      update=sum([s*v for s,v in zip(scores, values)])\n","      updates.append(update)\n","\n","    for n,u in zip(self.nodes, updates):\n","      n.data=n.data +u #residual connection"],"metadata":{"id":"LK4Nz5KYEa_L","executionInfo":{"status":"ok","timestamp":1716371049985,"user_tz":-120,"elapsed":3,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"execution_count":44,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"xk4SNQfuIbn0"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["\n","\n","> 1. Create a Graph\n","2. Print the key, value and query for the 4th node\n","3. Print the data before running and after running\n","\n"],"metadata":{"id":"Q118hX6lIrPl"}},{"cell_type":"code","source":[],"metadata":{"id":"c4yTzX1IQhQQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"k-jzsaiBQhNQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"neHQKNaiQhKO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"w0X1vgSDQil6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"5j6CN8N8Qii5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"XlkrPiw2Qifr"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["g=Graph()"],"metadata":{"id":"qbtWDTjKG1jY","executionInfo":{"status":"ok","timestamp":1716371054079,"user_tz":-120,"elapsed":250,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"execution_count":45,"outputs":[]},{"cell_type":"code","source":["g.nodes[0].query()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"M6HNttY5r_Lb","executionInfo":{"status":"ok","timestamp":1716371120370,"user_tz":-120,"elapsed":6,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"5bacbc48-4bf0-4c71-dc57-6c2ccf75ae5b"},"execution_count":50,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([ -2.26009653,  -3.86570119, -10.68306983,   2.10093105,\n","         8.24468375,   0.32323647,   1.03318534,  -2.5471518 ,\n","        -3.84820401,  -2.54643845,   2.94953266,   2.28183714,\n","        -2.9826758 ,   2.54974613,  -6.74749392,   0.87090985,\n","         6.94793187,   6.238638  ,   4.26249733,  -8.57029551])"]},"metadata":{},"execution_count":50}]},{"cell_type":"code","source":["g.nodes[0].key()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"3XEelWGiHA7R","executionInfo":{"status":"ok","timestamp":1716361466724,"user_tz":-120,"elapsed":346,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"adf80d7b-f639-4a23-94d3-0eb1dec98ab5"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([ 6.80817114, -0.16310615,  0.09909053,  6.74913728, -3.27475085,\n","        1.68078406, -0.59136218, -5.39002545,  8.12771051,  0.04429295,\n","       -8.38573856, -8.08030379, -5.17804022,  2.67510497,  7.03893079,\n","       -3.71964299,  4.1559444 ,  4.44817306, -1.81178428,  3.76416691])"]},"metadata":{},"execution_count":86}]},{"cell_type":"code","source":["g.nodes[0].value()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"h4vSGwzpHhCJ","executionInfo":{"status":"ok","timestamp":1716371139995,"user_tz":-120,"elapsed":235,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"e58d07de-e3c9-4554-aea2-2619b09ffcb5"},"execution_count":51,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([-0.16796905,  3.38603153,  8.87232844,  2.85645858,  3.83613868,\n","       -8.92490701,  2.09601661,  5.98362151,  0.93092894, -1.53902913,\n","       -3.77749424, -0.52533548, -1.75477106, -0.49328547, -2.74778709,\n","       -0.01105748,  5.67891244,  8.33002298,  0.42194456, -0.41449461])"]},"metadata":{},"execution_count":51}]},{"cell_type":"code","source":["g.nodes[0].query()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"coFFfkEvHigK","executionInfo":{"status":"ok","timestamp":1716361512299,"user_tz":-120,"elapsed":6,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"10bb39a0-4c96-4dc2-9a65-1db687377ee4"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([11.5282681 , -0.40117409, -3.86519441,  3.856318  ,  2.24188279,\n","        3.19754436, -1.51368747, -3.3892236 , -5.78609625, -0.05760908,\n","       -3.41750032,  3.31154274,  0.71510075, -1.1500739 , -7.68822842,\n","       -2.91063493,  3.46468184,  5.48590603,  1.22081941, -4.19792393])"]},"metadata":{},"execution_count":88}]},{"cell_type":"code","source":["g.nodes[0].data"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"-fZi_UmJIdS9","executionInfo":{"status":"ok","timestamp":1716371150463,"user_tz":-120,"elapsed":339,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"0dbbf33d-5669-4c6b-d90a-f628cc22c9a7"},"execution_count":52,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([-1.14056506, -1.23900008,  0.53904634,  0.94699827,  1.76198684,\n","       -0.10946224, -0.04302294,  0.81154406, -0.01547008, -0.2008717 ,\n","       -0.00457421,  1.98391545,  1.25505927, -0.627131  ,  0.34393967,\n","        0.13177731,  2.24757624, -0.44489159, -0.82046772, -0.28209497])"]},"metadata":{},"execution_count":52}]},{"cell_type":"code","source":["g.run()"],"metadata":{"id":"zmoXasPosZ6X","executionInfo":{"status":"ok","timestamp":1716371179438,"user_tz":-120,"elapsed":8,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}}},"execution_count":53,"outputs":[]},{"cell_type":"code","source":["g.nodes[0].data"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"eubJznw6sdct","executionInfo":{"status":"ok","timestamp":1716371188358,"user_tz":-120,"elapsed":9,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"5a98e5bd-fe8d-4158-fe26-6afeae2dcb2f"},"execution_count":54,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([ 2.97638451, -2.69937069, -0.65788764, -0.31446105,  0.69543204,\n","        6.45238175,  5.86212886, -1.25776177,  2.4952229 , -0.24442091,\n","        1.63603448,  0.10129498, -1.95933993, -0.83745152, -2.49197691,\n","       -6.64497786, -3.26629945, -3.4498158 , -1.6729622 , -4.9820016 ])"]},"metadata":{},"execution_count":54}]},{"cell_type":"markdown","source":["# Transformers from scratch\n","\n","[Documentation](https://uvadlc-notebooks.readthedocs.io/en/latest/tutorial_notebooks/tutorial6/Transformers_and_MHAttention.html)"],"metadata":{"id":"eNN6CACNJHPr"}},{"cell_type":"code","source":["import torch\n","import torch.nn as nn\n","import torch.nn.functional as F\n","import math"],"metadata":{"id":"7aMFni29OkMm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Define the scaled dot-product function\n","def scaled_dot_product(q, k, v, mask=None):\n","    d_k = q.size()[-1]\n","    attn_logits = torch.matmul(q, k.transpose(-2, -1))\n","    attn_logits = attn_logits / math.sqrt(d_k)\n","    if mask is not None:\n","        attn_logits = attn_logits.masked_fill(mask == 0, -9e15)\n","    attention = F.softmax(attn_logits, dim=-1)\n","    values = torch.matmul(attention, v)\n","    return values, attention\n","\n","# Define the expand mask function\n","def expand_mask(mask):\n","    assert mask.ndim >= 2, \"Mask must be at least 2-dimensional with seq_length x seq_length\"\n","    if mask.ndim == 3:\n","        mask = mask.unsqueeze(1)\n","    while mask.ndim < 4:\n","        mask = mask.unsqueeze(0)\n","    return mask"],"metadata":{"id":"zUxmTfWCOpH7"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Define the MultiheadAttention class\n","class MultiheadAttention(nn.Module):\n","    def __init__(self, input_dim, embed_dim, num_heads):\n","        super().__init__()\n","        assert embed_dim % num_heads == 0, \"Embedding dimension must be 0 modulo number of heads.\"\n","        self.embed_dim = embed_dim\n","        self.num_heads = num_heads\n","        self.head_dim = embed_dim // num_heads\n","        self.qkv_proj = nn.Linear(input_dim, 3 * embed_dim)\n","        self.o_proj = nn.Linear(embed_dim, embed_dim)\n","        self._reset_parameters()\n","\n","    def _reset_parameters(self):\n","        nn.init.xavier_uniform_(self.qkv_proj.weight)\n","        self.qkv_proj.bias.data.fill_(0)\n","        nn.init.xavier_uniform_(self.o_proj.weight)\n","        self.o_proj.bias.data.fill_(0)\n","\n","    def forward(self, x, mask=None, return_attention=False):\n","        batch_size, seq_length, _ = x.size()\n","        if mask is not None:\n","            mask = expand_mask(mask)\n","        qkv = self.qkv_proj(x)\n","        qkv = qkv.reshape(batch_size, seq_length, self.num_heads, 3 * self.head_dim)\n","        qkv = qkv.permute(0, 2, 1, 3)\n","        q, k, v = qkv.chunk(3, dim=-1)\n","        values, attention = scaled_dot_product(q, k, v, mask=mask)\n","        values = values.permute(0, 2, 1, 3).reshape(batch_size, seq_length, self.embed_dim)\n","        o = self.o_proj(values)\n","        if return_attention:\n","            return o, attention\n","        else:\n","            return o\n","\n","# Define the EncoderBlock class\n","class EncoderBlock(nn.Module):\n","    def __init__(self, input_dim, num_heads, dim_feedforward, dropout=0.0):\n","        super().__init__()\n","        self.self_attn = MultiheadAttention(input_dim, input_dim, num_heads)\n","        self.linear_net = nn.Sequential(\n","            nn.Linear(input_dim, dim_feedforward),\n","            nn.Dropout(dropout),\n","            nn.ReLU(inplace=True),\n","            nn.Linear(dim_feedforward, input_dim)\n","        )\n","        self.norm1 = nn.LayerNorm(input_dim)\n","        self.norm2 = nn.LayerNorm(input_dim)\n","        self.dropout = nn.Dropout(dropout)\n","\n","    def forward(self, x, mask=None):\n","        attn_out = self.self_attn(x, mask=mask)\n","        x = x + self.dropout(attn_out)\n","        x = self.norm1(x)\n","        linear_out = self.linear_net(x)\n","        x = x + self.dropout(linear_out)\n","        x = self.norm2(x)\n","        return x\n","\n","# Define the TransformerEncoder class\n","class TransformerEncoder(nn.Module):\n","    def __init__(self, num_layers, **block_args):\n","        super().__init__()\n","        self.layers = nn.ModuleList([EncoderBlock(**block_args) for _ in range(num_layers)])\n","\n","    def forward(self, x, mask=None):\n","        for l in self.layers:\n","            x = l(x, mask=mask)\n","        return x\n","\n","    def get_attention_maps(self, x, mask=None):\n","        attention_maps = []\n","        for l in self.layers:\n","            _, attn_map = l.self_attn(x, mask=mask, return_attention=True)\n","            attention_maps.append(attn_map)\n","            x = l(x)\n","        return attention_maps\n","\n","# Define the PositionalEncoding class\n","class PositionalEncoding(nn.Module):\n","    def __init__(self, d_model, max_len=5000):\n","        super().__init__()\n","        pe = torch.zeros(max_len, d_model)\n","        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n","        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n","        pe[:, 0::2] = torch.sin(position * div_term)\n","        pe[:, 1::2] = torch.cos(position * div_term)\n","        pe = pe.unsqueeze(0)\n","        self.register_buffer('pe', pe, persistent=False)\n","\n","    def forward(self, x):\n","        x = x + self.pe[:, :x.size(1)]\n","        return x\n","\n","\n"],"metadata":{"id":"yXSNrmYyJJVq"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Define the Transformer-based text classifier\n","class TransformerClassifier(nn.Module):\n","    def __init__(self, vocab_size, d_model, num_heads, num_layers, dim_feedforward, num_classes, max_len=5000, dropout=0.1):\n","        super().__init__()\n","        self.embedding = nn.Embedding(vocab_size, d_model, padding_idx=0)\n","        self.positional_encoding = PositionalEncoding(d_model, max_len)\n","        self.encoder = TransformerEncoder(\n","            num_layers=num_layers,\n","            input_dim=d_model,\n","            num_heads=num_heads,\n","            dim_feedforward=dim_feedforward,\n","            dropout=dropout\n","        )\n","        self.pre_classifier = nn.Linear(d_model, d_model)\n","        self.classifier = nn.Linear(d_model, num_classes)\n","        self.dropout = nn.Dropout(dropout)\n","\n","    def forward(self, input_ids, mask=None):\n","        x = self.embedding(input_ids)\n","        x = self.positional_encoding(x)\n","        x = self.encoder(x, mask=mask)\n","        x = x[:, 0]  # Use the [CLS] token (first token) for classification\n","        x = F.relu(self.pre_classifier(x))\n","        x = self.dropout(x)\n","        logits = self.classifier(x)\n","        return logits\n"],"metadata":{"id":"pa2X_-W6O1BL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["\n","# Example usage\n","vocab_size = 30522  # Vocabulary size (BERT's vocab size)\n","d_model = 768  # Embedding size\n","num_heads = 12  # Number of attention heads\n","num_layers = 6  # Number of transformer layers\n","dim_feedforward = 3072  # Feedforward network hidden size\n","num_classes = 2  # Number of output classes (e.g., binary classification)\n","max_len = 512  # Maximum sequence length\n","dropout = 0.1  # Dropout rate\n","\n","model = TransformerClassifier(vocab_size, d_model, num_heads, num_layers, dim_feedforward, num_classes, max_len, dropout)\n","\n","# Assume we have a tokenizer that converts sentences to input_ids\n","# For demonstration, using random input_ids\n","input_ids = torch.randint(0, vocab_size, (1, max_len))  # Batch size 1, sequence length max_len\n","mask = (input_ids != 0).unsqueeze(1).unsqueeze(2)  # Mask for non-padding tokens\n","\n","logits = model(input_ids, mask)\n","print(logits)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"5EGiWisKO4DQ","executionInfo":{"status":"ok","timestamp":1716363457636,"user_tz":-120,"elapsed":3550,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"606d4b45-49d1-4821-c7ac-2754fab4e01f"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["tensor([[0.1719, 0.2699]], grad_fn=<AddmmBackward0>)\n"]}]},{"cell_type":"code","source":["mask.shape"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0sMeqFc6PjkF","executionInfo":{"status":"ok","timestamp":1716363638012,"user_tz":-120,"elapsed":447,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"7e1ce7aa-ae29-465e-d8c2-e9e9a9f3f162"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["torch.Size([1, 1, 1, 512])"]},"metadata":{},"execution_count":100}]},{"cell_type":"code","source":["#batch size, seq length\n","input_ids.shape\n","\n","\n","#In the attention mechanism,\n","#the mask needs to be broadcastable to the shape [batch_size, num_heads, seq_length, seq_length]\n","#[batch_size, 1, seq_length, seq_length]: the num_heads will be matched in the code by the number of heads"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"RcKvWciHPtmh","executionInfo":{"status":"ok","timestamp":1716363652735,"user_tz":-120,"elapsed":546,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"25327c84-1272-46eb-fdda-e40fd33085bb"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["torch.Size([1, 512])"]},"metadata":{},"execution_count":101}]},{"cell_type":"markdown","source":["# Transformers: Hugging Face\n","\n","[Hugging Face](https://huggingface.co/docs/transformers/main_classes/tokenizer)"],"metadata":{"id":"LmhLfblw52Xf"}},{"cell_type":"code","source":["#!pip install transformers\n","import torch\n","from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification\n","\n","# gpu or cpu?\n","device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n","print (device)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"WWNSmGAR4qlQ","executionInfo":{"status":"ok","timestamp":1716351041131,"user_tz":-120,"elapsed":6940,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"f057e4c0-e1bc-47bb-f653-20cafbe9909d"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["cpu\n"]}]},{"cell_type":"code","source":["model_name = 'distilbert-base-uncased' # huggingface model_ID or path to folder\n","model = DistilBertForSequenceClassification.from_pretrained(model_name)\n","print (model)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":815,"referenced_widgets":["282af430fa00450291123794063dbfcb","4167eb83be204793a5fb56480d3c6bab","af1cdab4ceea4a9cb58c0e93b13d0ed5","e55a0bdcfab44618b2cdc1977fd701ef","1819f15bf4a540a2890473796c74adc7","45ba675c075848fbad4aa442456b8403","328cc433a1aa47a889f95c2fefc81333","62e65433f16441a1b67a8c7df6b6ffcb","25cf36779d1d4d96b108a6d669257900","2e0a91b1d80243b5bf7b589cbadae919","8b8dbda730fd4b2ab43bc918f3fb7216","6babbc0a5826422dbac15760ea33642e","7fe58184c90b4b5492c5979e3b012229","2e1cc725897b4814977ceb183381436f","6a8c64df99cd4a809952037a6deffac1","01bc9a67cc7145a5829e255e9164d9a3","bf7f8bc1670b41d981bcfbf79481fceb","19442ee18742475fa63a75f7251c31cd","226b17f24a9b421c8f56823ac8696f2c","4d37e5c7d6b048cbbc89a64388eff3f5","2db2de8a8c8e4110a09d169e98909b10","409476d4e1a9441499bff6b26ab02839"]},"id":"fWibqqkV55fv","executionInfo":{"status":"ok","timestamp":1716351044705,"user_tz":-120,"elapsed":3600,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"b981cd37-9dbf-4de2-dff9-1d75e1c394a7"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n","The secret `HF_TOKEN` does not exist in your Colab secrets.\n","To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n","You will be able to reuse this secret in all of your notebooks.\n","Please note that authentication is recommended but still optional to access public models or datasets.\n","  warnings.warn(\n"]},{"output_type":"display_data","data":{"text/plain":["config.json:   0%|          | 0.00/483 [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"282af430fa00450291123794063dbfcb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["model.safetensors:   0%|          | 0.00/268M [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6babbc0a5826422dbac15760ea33642e"}},"metadata":{}},{"output_type":"stream","name":"stderr","text":["Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"output_type":"stream","name":"stdout","text":["DistilBertForSequenceClassification(\n","  (distilbert): DistilBertModel(\n","    (embeddings): Embeddings(\n","      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n","      (position_embeddings): Embedding(512, 768)\n","      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","      (dropout): Dropout(p=0.1, inplace=False)\n","    )\n","    (transformer): Transformer(\n","      (layer): ModuleList(\n","        (0-5): 6 x TransformerBlock(\n","          (attention): MultiHeadSelfAttention(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n","          )\n","          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","          (ffn): FFN(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n","            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n","            (activation): GELUActivation()\n","          )\n","          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","        )\n","      )\n","    )\n","  )\n","  (pre_classifier): Linear(in_features=768, out_features=768, bias=True)\n","  (classifier): Linear(in_features=768, out_features=2, bias=True)\n","  (dropout): Dropout(p=0.2, inplace=False)\n",")\n"]}]},{"cell_type":"markdown","source":["\n","\n","> Apply that to our dataset\n","\n"],"metadata":{"id":"KkjQopMgQn4q"}},{"cell_type":"code","source":[],"metadata":{"id":"QQOP3CxaQngY"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"2xgj5sWcQnLW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"gjpudA86QrQC"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"vBpbiokwQrMa"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)\n","inputs = tokenizer(df.iloc[0]['text'], return_tensors=\"pt\")\n","print(inputs)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":269,"referenced_widgets":["e5dca58d7b3b49ff9abd42ea95819f57","a3d7068708bd4fb2b7b610c20534d22a","d901e7f813234f5db3f2145d9b5a9c62","9255150b3c504e0d88d9665f03375dae","a66aa91967274708bdc99511281f7d29","2b801c5ac63e44d8b378a1ef2b23e573","055c03c50ff342338c271dc1c188984b","8d23762fcf1849b0b1f2add59d8f9ab6","642d930513b94e3480a95e42495eff6e","2c9d4c24d8bd42ca83a6272b50a86149","b944f5d370474d959ff036c233a6b0ea","543338b8d50742038eb1b828c34ea5cb","ffc4b1e95dde43fc9745d039d7dfebe4","30c1d394196942a6859f55cd7ee7ebb0","2835b6338f56442db829853f7f4a5bd1","a488b9f20f8f4b80af420741153b811b","97bb2f5a3a704dad9bd9715e030afc8a","48d89db83ad24b7a900b07a2c1726a27","1411034ecdda4d84b505afb18b6cd7ee","5f90607a776349d39fa6fc04f0633dd1","4487f892d2944d00ae7b42eeedbd00d9","67a0b0b630f64d73b249136465a47305","07238ecb15cf496ea5f95d56ce54724d","f8a974e34511401eb710f7e8650e3297","66689b65c5c24d3491a82acebfb0067c","7e940e755029483ea8fc712fc7975b85","216f50e3489343e48f5e8fd2bb31fb6a","5f6ffcc0a90d418f967bee4c64283228","b29688ccdcf84290a5de21990bc139d4","cff3a6ea28f048d0830625ec44ea50a9","8cc21ae9a0af405d88202b2dc027694a","46e8de5b795f4fffa4060c2160dbf8a9","0f877b8d8ef04bef83837903f1c80c59"]},"id":"WHYXvc9x59kY","executionInfo":{"status":"error","timestamp":1716351045933,"user_tz":-120,"elapsed":1243,"user":{"displayName":"Catarina Pereira","userId":"01889833079344592364"}},"outputId":"597cb4c6-51eb-4e25-8ff1-c56f3369ecc7"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["tokenizer_config.json:   0%|          | 0.00/48.0 [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e5dca58d7b3b49ff9abd42ea95819f57"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"543338b8d50742038eb1b828c34ea5cb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"07238ecb15cf496ea5f95d56ce54724d"}},"metadata":{}},{"output_type":"error","ename":"NameError","evalue":"name 'df' is not defined","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-34-90282be15ebc>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mtokenizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDistilBertTokenizerFast\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0minputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'opinion_text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_tensors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"pt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"]}]},{"cell_type":"code","source":["inputs = tokenizer(df['opinion_text'].tolist(), return_tensors=\"pt\", padding=True, truncation=True)\n","labels = torch.tensor(df['x_republican'].tolist()).long()\n","print(inputs, labels)"],"metadata":{"id":"c1kFUBRE5_eR"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["\n","\n","> Manifesto Berta\n","\n","[Documentation](https://manifesto-project.wzb.eu/information/documents/manifestoberta)\n","\n"],"metadata":{"id":"knnoJW4D9eVv"}},{"cell_type":"code","source":["import torch\n","from transformers import AutoModelForSequenceClassification, AutoTokenizer\n","\n","model = AutoModelForSequenceClassification.from_pretrained(\"manifesto-project/manifestoberta-xlm-roberta-56policy-topics-sentence-2023-1-1\")\n","tokenizer = AutoTokenizer.from_pretrained(\"xlm-roberta-large\")\n","\n","sentence = \"We will restore funding to the Global Environment Facility and the Intergovernmental Panel on Climate Change, to support critical climate science research around the world\"\n","\n","inputs = tokenizer(sentence,\n","                   return_tensors=\"pt\",\n","                   max_length=200,  #we limited the input to 200 tokens during finetuning\n","                   padding=\"max_length\",\n","                   truncation=True\n","                   )\n","\n","logits = model(**inputs).logits\n","\n","probabilities = torch.softmax(logits, dim=1).tolist()[0]\n","probabilities = {model.config.id2label[index]: round(probability * 100, 2) for index, probability in enumerate(probabilities)}\n","probabilities = dict(sorted(probabilities.items(), key=lambda item: item[1], reverse=True))\n","print(probabilities)\n","# {'501 - Environmental Protection: Positive': 67.28, '411 - Technology and Infrastructure': 15.19, '107 - Internationalism: Positive': 13.63, '416 - Anti-Growth Economy: Positive': 2.02...\n","\n","predicted_class = model.config.id2label[logits.argmax().item()]\n","print(predicted_class)\n","# 501 - Environmental Protection: Positive\n"],"metadata":{"id":"YgH0DoHb9g0S"},"execution_count":null,"outputs":[]}],"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyOMaZY4iv4redxY9x7bDTt4"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"282af430fa00450291123794063dbfcb":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_4167eb83be204793a5fb56480d3c6bab","IPY_MODEL_af1cdab4ceea4a9cb58c0e93b13d0ed5","IPY_MODEL_e55a0bdcfab44618b2cdc1977fd701ef"],"layout":"IPY_MODEL_1819f15bf4a540a2890473796c74adc7"}},"4167eb83be204793a5fb56480d3c6bab":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_45ba675c075848fbad4aa442456b8403","placeholder":"​","style":"IPY_MODEL_328cc433a1aa47a889f95c2fefc81333","value":"config.json: 100%"}},"af1cdab4ceea4a9cb58c0e93b13d0ed5":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_62e65433f16441a1b67a8c7df6b6ffcb","max":483,"min":0,"orientation":"horizontal","style":"IPY_MODEL_25cf36779d1d4d96b108a6d669257900","value":483}},"e55a0bdcfab44618b2cdc1977fd701ef":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2e0a91b1d80243b5bf7b589cbadae919","placeholder":"​","style":"IPY_MODEL_8b8dbda730fd4b2ab43bc918f3fb7216","value":" 483/483 [00:00&lt;00:00, 22.5kB/s]"}},"1819f15bf4a540a2890473796c74adc7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"45ba675c075848fbad4aa442456b8403":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"328cc433a1aa47a889f95c2fefc81333":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"62e65433f16441a1b67a8c7df6b6ffcb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"25cf36779d1d4d96b108a6d669257900":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"2e0a91b1d80243b5bf7b589cbadae919":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8b8dbda730fd4b2ab43bc918f3fb7216":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"6babbc0a5826422dbac15760ea33642e":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_7fe58184c90b4b5492c5979e3b012229","IPY_MODEL_2e1cc725897b4814977ceb183381436f","IPY_MODEL_6a8c64df99cd4a809952037a6deffac1"],"layout":"IPY_MODEL_01bc9a67cc7145a5829e255e9164d9a3"}},"7fe58184c90b4b5492c5979e3b012229":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_bf7f8bc1670b41d981bcfbf79481fceb","placeholder":"​","style":"IPY_MODEL_19442ee18742475fa63a75f7251c31cd","value":"model.safetensors: 100%"}},"2e1cc725897b4814977ceb183381436f":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_226b17f24a9b421c8f56823ac8696f2c","max":267954768,"min":0,"orientation":"horizontal","style":"IPY_MODEL_4d37e5c7d6b048cbbc89a64388eff3f5","value":267954768}},"6a8c64df99cd4a809952037a6deffac1":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2db2de8a8c8e4110a09d169e98909b10","placeholder":"​","style":"IPY_MODEL_409476d4e1a9441499bff6b26ab02839","value":" 268M/268M [00:01&lt;00:00, 222MB/s]"}},"01bc9a67cc7145a5829e255e9164d9a3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"bf7f8bc1670b41d981bcfbf79481fceb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"19442ee18742475fa63a75f7251c31cd":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"226b17f24a9b421c8f56823ac8696f2c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4d37e5c7d6b048cbbc89a64388eff3f5":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"2db2de8a8c8e4110a09d169e98909b10":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"409476d4e1a9441499bff6b26ab02839":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"e5dca58d7b3b49ff9abd42ea95819f57":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_a3d7068708bd4fb2b7b610c20534d22a","IPY_MODEL_d901e7f813234f5db3f2145d9b5a9c62","IPY_MODEL_9255150b3c504e0d88d9665f03375dae"],"layout":"IPY_MODEL_a66aa91967274708bdc99511281f7d29"}},"a3d7068708bd4fb2b7b610c20534d22a":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2b801c5ac63e44d8b378a1ef2b23e573","placeholder":"​","style":"IPY_MODEL_055c03c50ff342338c271dc1c188984b","value":"tokenizer_config.json: 100%"}},"d901e7f813234f5db3f2145d9b5a9c62":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_8d23762fcf1849b0b1f2add59d8f9ab6","max":48,"min":0,"orientation":"horizontal","style":"IPY_MODEL_642d930513b94e3480a95e42495eff6e","value":48}},"9255150b3c504e0d88d9665f03375dae":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2c9d4c24d8bd42ca83a6272b50a86149","placeholder":"​","style":"IPY_MODEL_b944f5d370474d959ff036c233a6b0ea","value":" 48.0/48.0 [00:00&lt;00:00, 2.41kB/s]"}},"a66aa91967274708bdc99511281f7d29":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2b801c5ac63e44d8b378a1ef2b23e573":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"055c03c50ff342338c271dc1c188984b":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8d23762fcf1849b0b1f2add59d8f9ab6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"642d930513b94e3480a95e42495eff6e":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"2c9d4c24d8bd42ca83a6272b50a86149":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b944f5d370474d959ff036c233a6b0ea":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"543338b8d50742038eb1b828c34ea5cb":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_ffc4b1e95dde43fc9745d039d7dfebe4","IPY_MODEL_30c1d394196942a6859f55cd7ee7ebb0","IPY_MODEL_2835b6338f56442db829853f7f4a5bd1"],"layout":"IPY_MODEL_a488b9f20f8f4b80af420741153b811b"}},"ffc4b1e95dde43fc9745d039d7dfebe4":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_97bb2f5a3a704dad9bd9715e030afc8a","placeholder":"​","style":"IPY_MODEL_48d89db83ad24b7a900b07a2c1726a27","value":"vocab.txt: 100%"}},"30c1d394196942a6859f55cd7ee7ebb0":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_1411034ecdda4d84b505afb18b6cd7ee","max":231508,"min":0,"orientation":"horizontal","style":"IPY_MODEL_5f90607a776349d39fa6fc04f0633dd1","value":231508}},"2835b6338f56442db829853f7f4a5bd1":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_4487f892d2944d00ae7b42eeedbd00d9","placeholder":"​","style":"IPY_MODEL_67a0b0b630f64d73b249136465a47305","value":" 232k/232k [00:00&lt;00:00, 2.68MB/s]"}},"a488b9f20f8f4b80af420741153b811b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"97bb2f5a3a704dad9bd9715e030afc8a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"48d89db83ad24b7a900b07a2c1726a27":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"1411034ecdda4d84b505afb18b6cd7ee":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5f90607a776349d39fa6fc04f0633dd1":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"4487f892d2944d00ae7b42eeedbd00d9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"67a0b0b630f64d73b249136465a47305":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"07238ecb15cf496ea5f95d56ce54724d":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_f8a974e34511401eb710f7e8650e3297","IPY_MODEL_66689b65c5c24d3491a82acebfb0067c","IPY_MODEL_7e940e755029483ea8fc712fc7975b85"],"layout":"IPY_MODEL_216f50e3489343e48f5e8fd2bb31fb6a"}},"f8a974e34511401eb710f7e8650e3297":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_5f6ffcc0a90d418f967bee4c64283228","placeholder":"​","style":"IPY_MODEL_b29688ccdcf84290a5de21990bc139d4","value":"tokenizer.json: 100%"}},"66689b65c5c24d3491a82acebfb0067c":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_cff3a6ea28f048d0830625ec44ea50a9","max":466062,"min":0,"orientation":"horizontal","style":"IPY_MODEL_8cc21ae9a0af405d88202b2dc027694a","value":466062}},"7e940e755029483ea8fc712fc7975b85":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_46e8de5b795f4fffa4060c2160dbf8a9","placeholder":"​","style":"IPY_MODEL_0f877b8d8ef04bef83837903f1c80c59","value":" 466k/466k [00:00&lt;00:00, 13.1MB/s]"}},"216f50e3489343e48f5e8fd2bb31fb6a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5f6ffcc0a90d418f967bee4c64283228":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b29688ccdcf84290a5de21990bc139d4":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"cff3a6ea28f048d0830625ec44ea50a9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8cc21ae9a0af405d88202b2dc027694a":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"46e8de5b795f4fffa4060c2160dbf8a9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0f877b8d8ef04bef83837903f1c80c59":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"nbformat":4,"nbformat_minor":0}