v3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:47:27.195438500Z",
     "start_time": "2023-12-19T18:47:27.166902700Z"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\mojee\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\mojee\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "import nltk\n",
    "import tiktoken\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn.utils.rnn import pad_sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "88717b8aea4b4003",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:20:55.057047800Z",
     "start_time": "2023-12-19T18:20:54.953973Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event</th>\n",
       "      <th>comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>Lionel Messi winning the World Cup</td>\n",
       "      <td>\"He was beautiful ,he was the point of differ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>Christiano Ronaldo's goal</td>\n",
       "      <td>\"Cristiano Ronaldo scores again! What a legen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>Christiano Ronaldo's goal</td>\n",
       "      <td>\"He plays off his left foot, he almost doesn'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>Christiano Ronaldo's goal</td>\n",
       "      <td>\"The man is a storybook all by himself. There...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>Christiano Ronaldo's goal for Manchester United</td>\n",
       "      <td>\"Ole Gunnar Solskjaer gives thanks again for ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              event  \\\n",
       "65               Lionel Messi winning the World Cup   \n",
       "66                        Christiano Ronaldo's goal   \n",
       "67                        Christiano Ronaldo's goal   \n",
       "68                        Christiano Ronaldo's goal   \n",
       "69  Christiano Ronaldo's goal for Manchester United   \n",
       "\n",
       "                                              comment  \n",
       "65   \"He was beautiful ,he was the point of differ...  \n",
       "66   \"Cristiano Ronaldo scores again! What a legen...  \n",
       "67   \"He plays off his left foot, he almost doesn'...  \n",
       "68   \"The man is a storybook all by himself. There...  \n",
       "69   \"Ole Gunnar Solskjaer gives thanks again for ...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/v2.csv\", sep=\";\", )\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8f20fcf88061933b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:20:59.297416500Z",
     "start_time": "2023-12-19T18:20:59.267588Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_test_ratio = 0.2\n",
    "\n",
    "train_set, test_set = train_test_split(df, test_size=train_test_ratio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "94ed5b395b3c95fa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:21:02.024868400Z",
     "start_time": "2023-12-19T18:21:01.975828400Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event</th>\n",
       "      <th>comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Tottenham win against Manchester City</td>\n",
       "      <td>\"Pep's Manchester City, beaten in the Premier...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Manchester city vs Tottenham</td>\n",
       "      <td>\"The hurricane that hit Manchester for the be...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Spain vs Russia</td>\n",
       "      <td>\"Spain on the plane, or Spain in the Russian ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>Tottenham win against Manchester City</td>\n",
       "      <td>\"Hotspur's game plan was spot without blemish...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Paul Walker comparison with Son-Heung Min</td>\n",
       "      <td>\"Walker is adhesive Son Heung-Min is elusive.\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Scott McTominay's score for Manchester United ...</td>\n",
       "      <td>\"A red-Letter day, Scott McTominay, Ederson s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Sergio Aguero's goal for Manchester City vs Qu...</td>\n",
       "      <td>\"Drama of the ultimate type! Tears of distres...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Peter Drury's commentary on Roma</td>\n",
       "      <td>\"Roma have risen from their ruins. Manolas, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Manchester City vs Leicester City. Vincent Kom...</td>\n",
       "      <td>\"Kompany... had a look, had a hit... ohhh! Ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Germany win against Brazil, 2014 FIFA World Cup</td>\n",
       "      <td>From Chancellor to the cheap seats, the German...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>Christiano Ronaldo's goal</td>\n",
       "      <td>\"The man is a storybook all by himself. There...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>South Africa taking the lead against Mexico in...</td>\n",
       "      <td>Goal Bafana Bafana! Goal for South Africa! Goa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Netherlands vs Senegal</td>\n",
       "      <td>\"De Jong… Goalkeeper's come and Holland lead…...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Bukayo Saka's Penalty kick for Arsenal against...</td>\n",
       "      <td>\"Now this is one brave boy. Bukayo Saka. Beca...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                event  \\\n",
       "54              Tottenham win against Manchester City   \n",
       "50                       Manchester city vs Tottenham   \n",
       "13                                    Spain vs Russia   \n",
       "55              Tottenham win against Manchester City   \n",
       "49          Paul Walker comparison with Son-Heung Min   \n",
       "10  Scott McTominay's score for Manchester United ...   \n",
       "23  Sergio Aguero's goal for Manchester City vs Qu...   \n",
       "9                    Peter Drury's commentary on Roma   \n",
       "45  Manchester City vs Leicester City. Vincent Kom...   \n",
       "12    Germany win against Brazil, 2014 FIFA World Cup   \n",
       "68                          Christiano Ronaldo's goal   \n",
       "0   South Africa taking the lead against Mexico in...   \n",
       "39                             Netherlands vs Senegal   \n",
       "25  Bukayo Saka's Penalty kick for Arsenal against...   \n",
       "\n",
       "                                              comment  \n",
       "54   \"Pep's Manchester City, beaten in the Premier...  \n",
       "50   \"The hurricane that hit Manchester for the be...  \n",
       "13   \"Spain on the plane, or Spain in the Russian ...  \n",
       "55   \"Hotspur's game plan was spot without blemish...  \n",
       "49     \"Walker is adhesive Son Heung-Min is elusive.\"  \n",
       "10   \"A red-Letter day, Scott McTominay, Ederson s...  \n",
       "23   \"Drama of the ultimate type! Tears of distres...  \n",
       "9    \"Roma have risen from their ruins. Manolas, a...  \n",
       "45   \"Kompany... had a look, had a hit... ohhh! Ca...  \n",
       "12  From Chancellor to the cheap seats, the German...  \n",
       "68   \"The man is a storybook all by himself. There...  \n",
       "0   Goal Bafana Bafana! Goal for South Africa! Goa...  \n",
       "39   \"De Jong… Goalkeeper's come and Holland lead…...  \n",
       "25   \"Now this is one brave boy. Bukayo Saka. Beca...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2bde5ac8d8f7fa77",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:21:07.600176600Z",
     "start_time": "2023-12-19T18:21:07.564309200Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train = train_set['event']\n",
    "y_train = train_set[' comment']\n",
    "X_test = test_set['event']\n",
    "y_test = test_set[' comment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "adc8329318d9236e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:31:06.091441Z",
     "start_time": "2023-12-19T18:31:06.030356Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22              Manchester City win against Aston Villa\n",
       "42    South Africa vs Mexico. Siphiwe Tshabalala's H...\n",
       "36                        Cameroon score against Brazil\n",
       "29                            Kylian Mbappe's hat trick\n",
       "34                   Senegal vs Ecuador, FIFA world cup\n",
       "28                                 Kylian Mbappe's goal\n",
       "62                   Lionel Messi winning the World Cup\n",
       "37                                   Morocco vs Belgium\n",
       "38                                  Argentina vs Mexico\n",
       "35                                   Cameroon vs Brazil\n",
       "8                         Peter Drury on Etihad stadium\n",
       "58                                  Lionel Messi's goal\n",
       "59                                  Lionel Messi's goal\n",
       "40        England vs Colombia. Eric Dier's Penalty goal\n",
       "15                                     Germany vs Korea\n",
       "18    Aaron Ramsey's Departure from Arsenal to Juventus\n",
       "7         Peter Drury's commentary on Cristiano Ronaldo\n",
       "3     England's defeat to Croatia in the 2018 World Cup\n",
       "43    Manchester United vs Manchester City. Wayne Ro...\n",
       "19    Mohamed Salah's Hat-Trick against Manchester U...\n",
       "2     When Peter Czech saved a penalty from Gabriel ...\n",
       "26    Chelsea's UEFA Champions League Victory. Chels...\n",
       "27                      Liverpool vs. Manchester United\n",
       "51                         Manchester city vs Tottenham\n",
       "41    AS Roma vs Barcelona. Kostas Manolas' Goal Aga...\n",
       "17                                      Spain vs Russia\n",
       "30            Messi's goal for Argentina against Mexico\n",
       "44         Real Madrid vs Arsenal. Thierry Henry's goal\n",
       "60                                  Lionel Messi's goal\n",
       "6      Peter Drury on Manchester United's style of play\n",
       "48    Harry Kane's goal for Tottenham against Manche...\n",
       "63                   Lionel Messi winning the World Cup\n",
       "4               Thomas Partey and Paul Pogba comparison\n",
       "1     When Pep Guardiola substituted Gabriel Jesus f...\n",
       "52                         Manchester city vs Tottenham\n",
       "61                                  Lionel Messi's goal\n",
       "56                         Manchester city vs Tottenham\n",
       "46      Cristiano Ronaldo's Return to Manchester United\n",
       "69      Christiano Ronaldo's goal for Manchester United\n",
       "47      Cristiano Ronaldo's Return to Manchester United\n",
       "24                               Arsenal vs Real Madrid\n",
       "5        Peter Drury on Manchester United's consistency\n",
       "33       Argentina beats France in the World Cup finals\n",
       "16                                    Roma vs Barcelona\n",
       "21              Liverpool win against Manchester United\n",
       "11        Michy Batshuayi score against Atletico Madrid\n",
       "20              Liverpool win against Manchester United\n",
       "32                           Lionel Messi World Cup win\n",
       "66                            Christiano Ronaldo's goal\n",
       "67                            Christiano Ronaldo's goal\n",
       "53                         Manchester city vs Tottenham\n",
       "57     Comparison between Peter Czech and Gabriel Jesus\n",
       "31      Argentina's win against France in the world cup\n",
       "64                   Lionel Messi winning the World Cup\n",
       "14                                 Sergio Aguero's goal\n",
       "65                   Lionel Messi winning the World Cup\n",
       "Name: event, dtype: object"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d5d8aca906f045a8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:21:16.673682Z",
     "start_time": "2023-12-19T18:21:16.573954100Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22     \"From the brink of despair, elation. How dare...\n",
       "42     \"It's Tshabalala... Goal Bafana Bafana! Goal ...\n",
       "36     \"Ooooooh! Cameroon! From Douala on the Gulf o...\n",
       "29     \"One feels for Mbappe, who scored a World Cup...\n",
       "34     \"Captain fantastic Kalidou Koulibally, the pe...\n",
       "28                  \"Mbappe…Into the galaxy of greats!\"\n",
       "62     \"The greatest player of his age finally has t...\n",
       "37     \"Drink it in Casablanca, relish it Rabat, thi...\n",
       "38     \"Perhaps Argentina's motto is - 'it is not ho...\n",
       "35     \"The Indomitable Lions have been tamed by the...\n",
       "8      \"Etihad is bubbling up like it only does on '...\n",
       "58     \"Lionel Messi dribbles past defenders effortl...\n",
       "59     \"Another incredible goal by Messi! He makes i...\n",
       "40     \"History has been made for this new team, new...\n",
       "15     \"It seems like the end of the world, and it c...\n",
       "18     \"One day, next autumn when Aaron Ramsey wakes...\n",
       "7      \"The man is a storybook all by himself. There...\n",
       "3      \"The Three Lions were toothless voiceless and...\n",
       "43     \"Rooney! Oh, wonderful! What a goal! And what...\n",
       "19     \"Henderson nicks it. Jota's one way, Salah's ...\n",
       "2                              Peter Denies Jesus Again\n",
       "26     \"Chelsea champions. The Blues from the King's...\n",
       "27     \"Liverpool utopia. An unimaginable zenith. In...\n",
       "51     \"City thought they've saved it, but you're ne...\n",
       "41     \"Roma have risen from their ruins! Manolas, t...\n",
       "17    There are decibels and then there's this. Ther...\n",
       "30     \"Messi's head ooooh! That's what they came fo...\n",
       "44     \"Here's Henry getting clear, it's Thierry Hen...\n",
       "60                  \"A breath. A heartbeat. And Messi!\"\n",
       "6      \"Rangnick's Man United are playing a new kind...\n",
       "48     \"Harry Kane has blown the league wide open. I...\n",
       "63     \"Lionel Messi has conquered his final peak, L...\n",
       "4     Right there you've got two midfielders battlin...\n",
       "1     Jesus for Silva, a move Judas Iscariot will be...\n",
       "52     \"The day Tottenham Hotspur ripped up logic an...\n",
       "61     \"Rise of Messi, rise of Messi, rise of Messi,...\n",
       "56     \"It is why we love our league and it is why t...\n",
       "46     \"18 years since that trembling teenager of to...\n",
       "69     \"Ole Gunnar Solskjaer gives thanks again for ...\n",
       "47     \"Madeira, Manchester, Madrid, Turin and Manch...\n",
       "24     \"It's Thierry Henry. He's scored. He's scored...\n",
       "5      \"Manchester united is the most consistent tea...\n",
       "33    Lionel Messi stares up at his final peak, Kyli...\n",
       "16           \"The unthinkable unfolds before our eyes!\"\n",
       "21     \"This for Manchester United was impossibly bad.\"\n",
       "11     \"Steal, steal, wonderful steal from Chelsea a...\n",
       "20     \"Ole Solskjaer has it to drink in. Has it som...\n",
       "32     \"Lionel Messi has conquered his final peak. L...\n",
       "66     \"Cristiano Ronaldo scores again! What a legen...\n",
       "67     \"He plays off his left foot, he almost doesn'...\n",
       "53     \"Breathtaking Premier League drama, pick the ...\n",
       "57                         \"Peter Denies Jesus Again!\",\n",
       "31     \"36 years since Maradona and Mexico. Here, fi...\n",
       "64     \"The little boy from Rosario, Argentina on be...\n",
       "14                      \"One turn, one touch, one-nil!\"\n",
       "65     \"He was beautiful ,he was the point of differ...\n",
       "Name:  comment, dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e333d6aa5e92ebb8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:21:21.219629500Z",
     "start_time": "2023-12-19T18:21:21.210014400Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54                Tottenham win against Manchester City\n",
       "50                         Manchester city vs Tottenham\n",
       "13                                      Spain vs Russia\n",
       "55                Tottenham win against Manchester City\n",
       "49            Paul Walker comparison with Son-Heung Min\n",
       "10    Scott McTominay's score for Manchester United ...\n",
       "23    Sergio Aguero's goal for Manchester City vs Qu...\n",
       "9                      Peter Drury's commentary on Roma\n",
       "45    Manchester City vs Leicester City. Vincent Kom...\n",
       "12      Germany win against Brazil, 2014 FIFA World Cup\n",
       "68                            Christiano Ronaldo's goal\n",
       "0     South Africa taking the lead against Mexico in...\n",
       "39                               Netherlands vs Senegal\n",
       "25    Bukayo Saka's Penalty kick for Arsenal against...\n",
       "Name: event, dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e333815a79521275",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:21:24.717027300Z",
     "start_time": "2023-12-19T18:21:24.676751500Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54     \"Pep's Manchester City, beaten in the Premier...\n",
       "50     \"The hurricane that hit Manchester for the be...\n",
       "13     \"Spain on the plane, or Spain in the Russian ...\n",
       "55     \"Hotspur's game plan was spot without blemish...\n",
       "49       \"Walker is adhesive Son Heung-Min is elusive.\"\n",
       "10     \"A red-Letter day, Scott McTominay, Ederson s...\n",
       "23     \"Drama of the ultimate type! Tears of distres...\n",
       "9      \"Roma have risen from their ruins. Manolas, a...\n",
       "45     \"Kompany... had a look, had a hit... ohhh! Ca...\n",
       "12    From Chancellor to the cheap seats, the German...\n",
       "68     \"The man is a storybook all by himself. There...\n",
       "0     Goal Bafana Bafana! Goal for South Africa! Goa...\n",
       "39     \"De Jong… Goalkeeper's come and Holland lead…...\n",
       "25     \"Now this is one brave boy. Bukayo Saka. Beca...\n",
       "Name:  comment, dtype: object"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9e3347a694923c98",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:21:33.614345800Z",
     "start_time": "2023-12-19T18:21:30.657155500Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"\\nThey tried their best not to show it, believe me. I'm surprised they\\ncouldn't find a sprint car race (mini cars through pigpens, indeed!)\\non short notice.\\n\\nGeorge\",\n",
       " '\\nStankiewicz?  I doubt it.\\n\\nKoufax was one of two Jewish HOFs: the other is Hank Greenberg.\\n\\nOther good players: Buddy Myer, Johnny Kling, Norm and Larry Sherry,\\nKen Holtzman, Saul Rogovin, Ed Reulbach.  \\n\\nThere have been over 150 Jewish major leaguers.  A few years ago there\\nwas an article about someone who keeps track of this in Spy magazine;\\nthe article was entitled \"Jews on First,\" of course.\\n\\nThere have also been at least two books on the subject.',\n",
       " '\\n[deletia- and so on]\\n\\nI seem to have been rather unclear.\\n\\nWhat I was asking is this:\\n\\nPlease show me that the most effective substance-absure recovery\\nprograms involve meetinsg peoples\\' spiritual needs, rather than\\nmerely attempting to fill peoples\\' spiritual needs as percieved\\nby the people, A.A, S.R.C. regulars, or snoopy. This will probably\\ninvolve defining \"spritual needs\" (is it not that clear) and\\nshowing that such things exist and how they can be filled.\\n\\nAnnother tack you might take is to say that \"fulfilling spiritual\\nneeds\" means \"acknowledging a \"higher power\" of some sort, then\\nshow that systems that do require this, work better than otherwise\\nidentical systems that do not. A correlation here would help you,\\nbut as you point out this might just be demonstrating swapping\\none crutch for annother. (however, I do feel that religion is\\nusually a better crutch than alchohol, as it is not usually\\npoisonous! :) )\\n\\nI hope with that clarification, my question will be answerable. I actually\\ndid know about the 12 step program, its the question of what it does,\\nrather than what it tries to do, that makes a difference to me.\\n---\\n\\t\\t\\t- Dan Johnson\\nAnd God said \"Jeeze, this is dull\"... and it *WAS* dull. Genesis 0:0',\n",
       " 'Excuse the sheer newbieness of this post, but I am looking for a decent\\nPaintProgram which will save to various file formats (.BMP  .PCX  etc etc)\\nvia ftp, freeware, or shareware.  I would like to check out the available\\nprograms for little $$ before I check out the commercial market.\\n\\nThanks in advance for any help or direction you can give me.',\n",
       " '==============================================================================\\n',\n",
       " '\\n\\nI watched him over his entire career.  I have NEVER seen a player, and that\\nincludes Russell Courtnall and Davie Keon, screw up as many breakaways as\\nBob Gainey.  And I will never forget the time Denis Potvin caught Gainey\\nwith his head down.  You have been sold a bill of goods on Bob Gainey.\\n\\nGainey was a plugger.  And when the press runs out of things to say about \\nthe stars on dynasties they start to hype the pluggers.  Grant Fuhr, Essa\\nTikkannen, Butch Goring, Bob Nystrom, Bob Gainey, Doug Jarvis, Derek\\nSanderson, Wayne Cashman, Bob Baun, Bob Pulford, Ralph Backstrom, Henri\\nRichard, Dick Duff...and so on...\\n\\ncordially, as always,\\n\\nrm\\n',\n",
       " 'A fine 26 year history came to a close tonight, as the Minnesota North Stars, \\nor Norm\\'s Stars (whichever you prefer) lost to the Red Wings by a score of\\n5-3.  The Stars goals were scored by Mike McPhee and Ulf Dahlen, who netted\\ntwo including the final one in franchise history, with less than a minute to\\nplay.\\n\\nTonight, on the air on the Stars TV telecast, announcer Al Shaver, the voice\\nof the North Stars, stated basically that he will not follow the team to \\nDallas.  Shaver, when asked by his son (who was doing the broadcast with him),\\n\"What will you do now?\" responded, \"First I\\'m going to get me a new pair of\\nslippers.  Then I\\'m going to sit in my easy chair and watch the world go by.\"\\n\\nThank you North Stars, and thank you Al Shaver, for 26 years of Minnesota\\nmemories.',\n",
       " 'I just installed a new TrueType font under MS-Windows 3.1\\nbut though all the applications display the font correctly on the\\nscreen, quite a few of them fail to print out the document correctly\\n(on a LaserJet 4 - non-PostScript printer).\\n\\nWhen I use the font in CorelDRAW, the document prints correctly, so I assume\\nCorelDRAW knows that the font has to be downloaded to the printer.\\n\\nBut when I use the Windows accessory Write, the printer prints square\\nboxes in place of the characters of the new font. Yet, Write does\\ndisplay the font correctly on the screen.\\n\\nI looked through all the Windows and LaserJet manuals, but got nowhere.\\nAll of them just make the statement that TrueType fonts will print\\nexactly as you see them on the screen---so I assume Windows knows that a font\\nhas to be downloaded automatically---but, how to make it do that????',\n",
       " \"This is the 2nd Reguest for Discussion ( RFD ) for the creation of \\ncomp.databases.ms-access \\n\\nThe discussion period started on April 6 and it will last until April 28,\\n( 22 days ).\\n\\nThis is an official RFD for the creation of a new newsgroup for the\\ngeneral discussion of the Microsoft Access RDMS.\\n\\nNAME: COMP.DATABASES.MS-ACCESS\\n\\nMODERATION: UNMODERATED. At this time, no need for a moderator has been\\nassertained. Future evaluation will determine if one is needed.\\n\\nPURPOSE: \\nAccess is a new RDBMS for the Windows Operating System. It includes WYSIWYG\\ndesign tools for easy creation of tables, reports, forms and queries and a\\ndatabase programming language called Access Basic.\\nTHe purpose of the group will be to provide help to people who use Access's \\nWYSIWYG design tools to create simple databases as well as to people who use \\nAccess Basic to create complex databases.\\n\\nRATIONALE:\\nEventhough Access is a new RDBMS, it is very popular because of its Graphical\\nDevelopment enviroment and its initial low price.\\nBeen a version 1.0 product means that all Access users are Novices.\\nFor that reason a newsgroup is needed where Access users can discuss \\ntheir experiences with the product and answer each other's questions.\",\n",
       " \"\\n\\nSince running any GUI over a network is going to slow it down by a\\nfair amount, I expect Windows NT will be multiuser only in the sense\\nof sharing filesystems. Someone will likely write a telnetd for it so\\none could run character-based apps, but graphics-based apps will have\\nto be shared by running the executables on the local CPU. This is how\\nthings are shaping up everywhere: client-server architectures are\\ntaking over from the old cpu-terminal setups. \\n\\nNote that the NeXT does this: you can always telnet into a NeXT and\\nrun character-based apps but you can't run the GUI. (Yeah, I know\\nabout X-Windows, just haven't been too impressed by it...)..\\n\\n\\n\\n\\n\\n\\n-- \"]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categories = [\n",
    "    'comp.graphics',\n",
    "    'comp.os.ms-windows.misc',\n",
    "    'rec.sport.baseball',\n",
    "    'rec.sport.hockey',\n",
    "    'alt.atheism',\n",
    "    'soc.religion.christian',\n",
    "]\n",
    "\n",
    "dataset = fetch_20newsgroups(subset='train', categories=categories, shuffle=True, remove=('headers', 'footers', 'quotes'))\n",
    "corpus = [item for item in dataset['data'][:10]]\n",
    "corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ee41ccdb67909f6e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:22:47.273168600Z",
     "start_time": "2023-12-19T18:22:35.651008Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "encoding = tiktoken.get_encoding(\"cl100k_base\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "482f1aea2be15f30",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:26:03.179631300Z",
     "start_time": "2023-12-19T18:26:03.131098Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Hello world. Lorem ipsum dolor sit'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"Hello world. Lorem ipsum dolor sit\"\n",
    "toks = encoding.encode(text)\n",
    "tk = encoding.decode(toks)\n",
    "tk"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ff2374b95bfa8f",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## Tokenization\n",
    "Using `tiktoken` library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1ee378233d69d9bc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:37:36.065656600Z",
     "start_time": "2023-12-19T18:37:36.019983Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train_tokens = []\n",
    "for i in X_train:\n",
    "    enc = encoding.encode(i)\n",
    "    X_train_tokens.append(enc)\n",
    "X_test_tokens = []\n",
    "for i in X_test:\n",
    "    enc = encoding.encode(i)\n",
    "    X_test_tokens.append(enc)\n",
    "y_train_tokens = []\n",
    "y_test_tokens = []\n",
    "for i in y_train:\n",
    "    enc = encoding.encode(i)\n",
    "    y_train_tokens.append(enc)\n",
    "for i in y_test:\n",
    "    enc = encoding.encode(i)\n",
    "    y_test_tokens.append(enc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd36e6f7a376ad27",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## Tokens to tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4410d4b37a28c21b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:45:13.239237700Z",
     "start_time": "2023-12-19T18:45:13.199550600Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train_tensors = [torch.tensor(data) for data in X_train_tokens]\n",
    "y_train_tensors = [torch.tensor(data) for data in y_train_tokens]\n",
    "X_test_tensors = [torch.tensor(data) for data in X_test_tokens]\n",
    "y_test_tensors = [torch.tensor(data) for data in y_test_tokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "e9a2ffd55a084aa8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T19:03:05.634145100Z",
     "start_time": "2023-12-19T19:03:05.588196600Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([56, 16])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_tensors_padded = pad_sequence(X_train_tensors, batch_first=True)\n",
    "X_train_tensors_padded.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "9949a24f23403199",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T19:04:50.443359700Z",
     "start_time": "2023-12-19T19:04:50.430851Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([56, 92])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_tensors_padded = pad_sequence(y_train_tensors, batch_first=True)\n",
    "y_train_tensors_padded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "49bed054d6ecce0a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:54:19.403712300Z",
     "start_time": "2023-12-19T18:54:19.370149300Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([14, 14])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_tensors_padded = pad_sequence(X_test_tensors, batch_first=True)\n",
    "X_test_tensors_padded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "3caf66c4fd90bbe5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T18:52:00.236692900Z",
     "start_time": "2023-12-19T18:52:00.189144700Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([14, 80])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_tensors_padded = pad_sequence(y_test_tensors, batch_first=True)\n",
    "y_test_tensors_padded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "39853f6e3c38eb7c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-12-19T19:02:47.608573100Z",
     "start_time": "2023-12-19T19:02:47.552215600Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([56, 16])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_length = max(X_train_tensors_padded.shape, X_test_tensors_padded.shape)\n",
    "max_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8d8a4a018d1df0f",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Network(nn.Module):\n",
    "    def __init__(self, x_):\n",
    "        super(Network, self).__init__()\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \n",
    "        return x"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}