word_embeddings_tutorial.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "word_embeddings_tutorial.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.6"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/winnechan/pytorch_practices/blob/master/word_embeddings_tutorial.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xf2aN7tC85ij",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "outputId": "11dd93a0-d234-440b-b1c4-5c263565023c"
      },
      "source": [
        "%matplotlib inline\n",
        "'''\n",
        "reference: https://pytorch.org/tutorials/beginner/nlp/word_embeddings_tutorial.html\n",
        "including:\n",
        "    N-Gram Language Modeling\n",
        "    Continuous Bag-of-Words\n",
        "'''\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "\n",
        "torch.manual_seed(1)\n",
        "\n",
        "CONTEXT_SIZE = 2\n",
        "EMBEDDING_DIM = 10\n",
        "# Dataset: we will use Shakespeare Sonnet 2\n",
        "test_sentence = \"\"\"When forty winters shall besiege thy brow,\n",
        "And dig deep trenches in thy beauty's field,\n",
        "Thy youth's proud livery so gazed on now,\n",
        "Will be a totter'd weed of small worth held:\n",
        "Then being asked, where all thy beauty lies,\n",
        "Where all the treasure of thy lusty days;\n",
        "To say, within thine own deep sunken eyes,\n",
        "Were an all-eating shame, and thriftless praise.\n",
        "How much more praise deserv'd thy beauty's use,\n",
        "If thou couldst answer 'This fair child of mine\n",
        "Shall sum my count, and make my old excuse,'\n",
        "Proving his beauty by succession thine!\n",
        "This were to be new made when thou art old,\n",
        "And see thy blood warm when thou feel'st it cold.\"\"\".lower().split()\n",
        "\n",
        "'''N-Gram Language Modeling'''\n",
        "class NGramLanguageModeler(nn.Module):\n",
        "\n",
        "    def __init__(self, vocab_size, embedding_dim, context_size):\n",
        "        super(NGramLanguageModeler, self).__init__()\n",
        "        self.embeddings = nn.Embedding(vocab_size, embedding_dim)\n",
        "        self.linear1 = nn.Linear(context_size * embedding_dim, 128)\n",
        "        self.linear2 = nn.Linear(128, vocab_size)\n",
        "\n",
        "    def forward(self, inputs):\n",
        "        embeds = self.embeddings(inputs).view((1, -1))\n",
        "        out = F.relu(self.linear1(embeds))\n",
        "        out = self.linear2(out)\n",
        "        log_probs = F.log_softmax(out, dim=1)\n",
        "        return log_probs\n",
        "\n",
        "'''Continuous Bag-of-Words Model'''\n",
        "class CBOW(nn.Module):\n",
        "\n",
        "    def __init__(self, vocab_size, embedding_dim, context_size):\n",
        "        super(CBOW, self).__init__()\n",
        "        self.embedding_dim = embedding_dim\n",
        "        self.embeddings = nn.Embedding(vocab_size, embedding_dim)\n",
        "        self.linear1 = nn.Linear(embedding_dim, 128)\n",
        "        self.linear2 = nn.Linear(128, vocab_size)\n",
        "\n",
        "    def forward(self, inputs):\n",
        "        embeds = self.embeddings(inputs).view((-1, self.embedding_dim))\n",
        "        embeds = torch.sum(embeds, dim=0, keepdim=True)\n",
        "        out = F.relu(self.linear1(embeds))\n",
        "        out = self.linear2(out)\n",
        "        log_probs = F.log_softmax(out, dim=1)\n",
        "        return log_probs\n",
        "      \n",
        "def test_ngram_model():\n",
        "    # we should tokenize the input, but we will ignore that for now\n",
        "    # build a list of tuples.  Each tuple is ([ word_i-2, word_i-1 ], target word)\n",
        "    trigrams = [([test_sentence[i], test_sentence[i + 1]], test_sentence[i + 2])\n",
        "            for i in range(len(test_sentence) - 2)]\n",
        "    # print the first 3, just so you can see what they look like\n",
        "    print(trigrams[:3])\n",
        "\n",
        "    vocab = set(test_sentence)\n",
        "    word_to_ix = {word: i for i, word in enumerate(vocab)}\n",
        "    \n",
        "    losses = []\n",
        "    loss_function = nn.NLLLoss()\n",
        "    model = NGramLanguageModeler(len(vocab), EMBEDDING_DIM, CONTEXT_SIZE)\n",
        "    optimizer = optim.SGD(model.parameters(), lr=0.001)\n",
        "\n",
        "    for epoch in range(10):\n",
        "        total_loss = 0\n",
        "        for context, target in trigrams:\n",
        "\n",
        "            # Step 1. Prepare the inputs to be passed to the model (i.e, turn the words\n",
        "            # into integer indices and wrap them in tensors)\n",
        "            context_idxs = torch.tensor([word_to_ix[w] for w in context], dtype=torch.long)\n",
        "\n",
        "            # Step 2. Recall that torch *accumulates* gradients. Before passing in a\n",
        "            # new instance, you need to zero out the gradients from the old\n",
        "            # instance\n",
        "            model.zero_grad()\n",
        "\n",
        "            # Step 3. Run the forward pass, getting log probabilities over next\n",
        "            # words\n",
        "            log_probs = model(context_idxs)\n",
        "\n",
        "            # Step 4. Compute your loss function. (Again, Torch wants the target\n",
        "            # word wrapped in a tensor)\n",
        "            loss = loss_function(log_probs, torch.tensor([word_to_ix[target]], dtype=torch.long))\n",
        "\n",
        "            # Step 5. Do the backward pass and update the gradient\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            # Get the Python number from a 1-element Tensor by calling tensor.item()\n",
        "            total_loss += loss.item()\n",
        "        losses.append(total_loss)\n",
        "    print(losses)  # The loss decreased every iteration over the training data!\n",
        "\n",
        "def test_cbow_model():\n",
        "    # By deriving a set from `raw_text`, we deduplicate the array\n",
        "    vocab = set(test_sentence)\n",
        "    vocab_size = len(vocab)\n",
        "\n",
        "    word_to_ix = {word: i for i, word in enumerate(vocab)}\n",
        "    data = []\n",
        "    for i in range(2, len(test_sentence) - 2):\n",
        "        context = [test_sentence[i - 2], test_sentence[i - 1],\n",
        "                   test_sentence[i + 1], test_sentence[i + 2]]\n",
        "        target = test_sentence[i]\n",
        "        data.append((context, target))\n",
        "    print(data[:5])\n",
        "    \n",
        "    losses = []\n",
        "    loss_function = nn.NLLLoss()\n",
        "    model = CBOW(len(vocab), EMBEDDING_DIM, CONTEXT_SIZE)\n",
        "    optimizer = optim.SGD(model.parameters(), lr=0.001)\n",
        "\n",
        "    for epoch in range(10):\n",
        "        total_loss = 0\n",
        "        for context, target in data:\n",
        "\n",
        "            # Step 1. Prepare the inputs to be passed to the model (i.e, turn the words\n",
        "            # into integer indices and wrap them in tensors)\n",
        "            context_idxs = torch.tensor([word_to_ix[w] for w in context], dtype=torch.long)\n",
        "\n",
        "            # Step 2. Recall that torch *accumulates* gradients. Before passing in a\n",
        "            # new instance, you need to zero out the gradients from the old\n",
        "            # instance\n",
        "            model.zero_grad()\n",
        "\n",
        "            # Step 3. Run the forward pass, getting log probabilities over next\n",
        "            # words\n",
        "            log_probs = model(context_idxs)\n",
        "\n",
        "            # Step 4. Compute your loss function. (Again, Torch wants the target\n",
        "            # word wrapped in a tensor)\n",
        "            loss = loss_function(log_probs, torch.tensor([word_to_ix[target]], dtype=torch.long))\n",
        "\n",
        "            # Step 5. Do the backward pass and update the gradient\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            # Get the Python number from a 1-element Tensor by calling tensor.item()\n",
        "            total_loss += loss.item()\n",
        "        losses.append(total_loss)\n",
        "    print(losses)  # The loss decreased every iteration over the training data!\n",
        "    \n",
        "if __name__ == \"__main__\":\n",
        "    test_cbow_model()\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[(['when', 'forty', 'shall', 'besiege'], 'winters'), (['forty', 'winters', 'besiege', 'thy'], 'shall'), (['winters', 'shall', 'thy', 'brow,'], 'besiege'), (['shall', 'besiege', 'brow,', 'and'], 'thy'), (['besiege', 'thy', 'and', 'dig'], 'brow,')]\n",
            "[513.7279975414276, 504.05146408081055, 494.793253660202, 485.9411971569061, 477.48647236824036, 469.42089462280273, 461.73359394073486, 454.3995406627655, 447.39133191108704, 440.67230010032654]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}