gpt_dev.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wJpXpmjEYC_T"
   },
   "source": [
    "## Building a GPT\n",
    "\n",
    "Companion notebook to the [Zero To Hero](https://karpathy.ai/zero-to-hero.html) video on GPT."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-15T02:15:06.668128Z",
     "start_time": "2024-02-15T02:15:06.544983Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "h5hjCcLDr2WC",
    "outputId": "ccc60f0c-fd78-4dbe-8598-0512d1036aad"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-02-16 08:02:49--  https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 0.0.0.0\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|0.0.0.0|:443... failed: Connection refused.\n"
     ]
    }
   ],
   "source": [
    "# We always start with a dataset to train on. Let's download the tiny shakespeare dataset\n",
    "! wget https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "O6medjfRsLD9"
   },
   "outputs": [],
   "source": [
    "# read it in to inspect it\n",
    "with open('input.txt', 'r', encoding='utf-8') as f:\n",
    "    text = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6xWI_VyAsN8F",
    "outputId": "ed819dd0-72e5-40a6-d2ed-928ff73bfda6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of dataset in characters:  1115394\n"
     ]
    }
   ],
   "source": [
    "print(\"length of dataset in characters: \", len(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2c5V0FvqseE0",
    "outputId": "25ca7adc-b8c0-42d1-b08c-e0863c5c314e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First Citizen:\n",
      "Before we proceed any further, hear me speak.\n",
      "\n",
      "All:\n",
      "Speak, speak.\n",
      "\n",
      "First Citizen:\n",
      "You are all resolved rather to die than to famish?\n",
      "\n",
      "All:\n",
      "Resolved. resolved.\n",
      "\n",
      "First Citizen:\n",
      "First, you know Caius Marcius is chief enemy to the people.\n",
      "\n",
      "All:\n",
      "We know't, we know't.\n",
      "\n",
      "First Citizen:\n",
      "Let us kill him, and we'll have corn at our own price.\n",
      "Is't a verdict?\n",
      "\n",
      "All:\n",
      "No more talking on't; let it be done: away, away!\n",
      "\n",
      "Second Citizen:\n",
      "One word, good citizens.\n",
      "\n",
      "First Citizen:\n",
      "We are accounted poor citizens, the patricians good.\n",
      "What authority surfeits on would relieve us: if they\n",
      "would yield us but the superfluity, while it were\n",
      "wholesome, we might guess they relieved us humanely;\n",
      "but they think we are too dear: the leanness that\n",
      "afflicts us, the object of our misery, is as an\n",
      "inventory to particularise their abundance; our\n",
      "sufferance is a gain to them Let us revenge this with\n",
      "our pikes, ere we become rakes: for the gods know I\n",
      "speak this in hunger for bread, not in thirst for revenge.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# let's look at the first 1000 characters\n",
    "print(text[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0e-Rbyr8sfM8",
    "outputId": "f34e94a9-5b44-4cf3-885b-986731929109"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " !$&',-.3:;?ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\n",
      "65\n"
     ]
    }
   ],
   "source": [
    "# here are all the unique characters that occur in this text\n",
    "chars = sorted(list(set(text)))\n",
    "vocab_size = len(chars)\n",
    "print(''.join(chars))\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Yw1LKNCgwjj1",
    "outputId": "86fcc21c-2cf7-40d9-cd7b-b5a253da4459"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[46, 47, 47, 1, 58, 46, 43, 56, 43]\n",
      "hii there\n"
     ]
    }
   ],
   "source": [
    "# create a mapping from characters to integers\n",
    "# 创建了一个名为stoi（\"string to integer\"的缩写）的字典。它使用了字典推导式，这是创建字典的一种简洁方式。\n",
    "# enumerate(chars)函数，它接收一个可迭代对象（在这个例子中是chars），并返回一个产生形如(index, element)的元组的迭代器。换句话说，它将可迭代对象的每个元素与其对应的索引配对。\n",
    "# for i,ch in enumerate(chars)部分的代码是一个for循环，它遍历这些(index, element)元组。i是索引，ch是来自chars可迭代对象的元素\n",
    "stoi = { ch:i for i,ch in enumerate(chars) }\n",
    "itos = { i:ch for i,ch in enumerate(chars) }\n",
    "encode = lambda s: [stoi[c] for c in s] # encoder: take a string, output a list of integers\n",
    "decode = lambda l: ''.join([itos[i] for i in l]) # decoder: take a list of integers, output a string\n",
    "print(encode(\"hii there\"))\n",
    "print(decode(encode(\"hii there\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "YJb0OXPwzvqg",
    "outputId": "db7297cc-36a9-4fae-e941-e7bb9e0e91d1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1115394]) torch.int64\n",
      "tensor([18, 47, 56, 57, 58,  1, 15, 47, 58, 47, 64, 43, 52, 10,  0, 14, 43, 44,\n",
      "        53, 56, 43,  1, 61, 43,  1, 54, 56, 53, 41, 43, 43, 42,  1, 39, 52, 63,\n",
      "         1, 44, 59, 56, 58, 46, 43, 56,  6,  1, 46, 43, 39, 56,  1, 51, 43,  1,\n",
      "        57, 54, 43, 39, 49,  8,  0,  0, 13, 50, 50, 10,  0, 31, 54, 43, 39, 49,\n",
      "         6,  1, 57, 54, 43, 39, 49,  8,  0,  0, 18, 47, 56, 57, 58,  1, 15, 47,\n",
      "        58, 47, 64, 43, 52, 10,  0, 37, 53, 59,  1, 39, 56, 43,  1, 39, 50, 50,\n",
      "         1, 56, 43, 57, 53, 50, 60, 43, 42,  1, 56, 39, 58, 46, 43, 56,  1, 58,\n",
      "        53,  1, 42, 47, 43,  1, 58, 46, 39, 52,  1, 58, 53,  1, 44, 39, 51, 47,\n",
      "        57, 46, 12,  0,  0, 13, 50, 50, 10,  0, 30, 43, 57, 53, 50, 60, 43, 42,\n",
      "         8,  1, 56, 43, 57, 53, 50, 60, 43, 42,  8,  0,  0, 18, 47, 56, 57, 58,\n",
      "         1, 15, 47, 58, 47, 64, 43, 52, 10,  0, 18, 47, 56, 57, 58,  6,  1, 63,\n",
      "        53, 59,  1, 49, 52, 53, 61,  1, 15, 39, 47, 59, 57,  1, 25, 39, 56, 41,\n",
      "        47, 59, 57,  1, 47, 57,  1, 41, 46, 47, 43, 44,  1, 43, 52, 43, 51, 63,\n",
      "         1, 58, 53,  1, 58, 46, 43,  1, 54, 43, 53, 54, 50, 43,  8,  0,  0, 13,\n",
      "        50, 50, 10,  0, 35, 43,  1, 49, 52, 53, 61,  5, 58,  6,  1, 61, 43,  1,\n",
      "        49, 52, 53, 61,  5, 58,  8,  0,  0, 18, 47, 56, 57, 58,  1, 15, 47, 58,\n",
      "        47, 64, 43, 52, 10,  0, 24, 43, 58,  1, 59, 57,  1, 49, 47, 50, 50,  1,\n",
      "        46, 47, 51,  6,  1, 39, 52, 42,  1, 61, 43,  5, 50, 50,  1, 46, 39, 60,\n",
      "        43,  1, 41, 53, 56, 52,  1, 39, 58,  1, 53, 59, 56,  1, 53, 61, 52,  1,\n",
      "        54, 56, 47, 41, 43,  8,  0, 21, 57,  5, 58,  1, 39,  1, 60, 43, 56, 42,\n",
      "        47, 41, 58, 12,  0,  0, 13, 50, 50, 10,  0, 26, 53,  1, 51, 53, 56, 43,\n",
      "         1, 58, 39, 50, 49, 47, 52, 45,  1, 53, 52,  5, 58, 11,  1, 50, 43, 58,\n",
      "         1, 47, 58,  1, 40, 43,  1, 42, 53, 52, 43, 10,  1, 39, 61, 39, 63,  6,\n",
      "         1, 39, 61, 39, 63,  2,  0,  0, 31, 43, 41, 53, 52, 42,  1, 15, 47, 58,\n",
      "        47, 64, 43, 52, 10,  0, 27, 52, 43,  1, 61, 53, 56, 42,  6,  1, 45, 53,\n",
      "        53, 42,  1, 41, 47, 58, 47, 64, 43, 52, 57,  8,  0,  0, 18, 47, 56, 57,\n",
      "        58,  1, 15, 47, 58, 47, 64, 43, 52, 10,  0, 35, 43,  1, 39, 56, 43,  1,\n",
      "        39, 41, 41, 53, 59, 52, 58, 43, 42,  1, 54, 53, 53, 56,  1, 41, 47, 58,\n",
      "        47, 64, 43, 52, 57,  6,  1, 58, 46, 43,  1, 54, 39, 58, 56, 47, 41, 47,\n",
      "        39, 52, 57,  1, 45, 53, 53, 42,  8,  0, 35, 46, 39, 58,  1, 39, 59, 58,\n",
      "        46, 53, 56, 47, 58, 63,  1, 57, 59, 56, 44, 43, 47, 58, 57,  1, 53, 52,\n",
      "         1, 61, 53, 59, 50, 42,  1, 56, 43, 50, 47, 43, 60, 43,  1, 59, 57, 10,\n",
      "         1, 47, 44,  1, 58, 46, 43, 63,  0, 61, 53, 59, 50, 42,  1, 63, 47, 43,\n",
      "        50, 42,  1, 59, 57,  1, 40, 59, 58,  1, 58, 46, 43,  1, 57, 59, 54, 43,\n",
      "        56, 44, 50, 59, 47, 58, 63,  6,  1, 61, 46, 47, 50, 43,  1, 47, 58,  1,\n",
      "        61, 43, 56, 43,  0, 61, 46, 53, 50, 43, 57, 53, 51, 43,  6,  1, 61, 43,\n",
      "         1, 51, 47, 45, 46, 58,  1, 45, 59, 43, 57, 57,  1, 58, 46, 43, 63,  1,\n",
      "        56, 43, 50, 47, 43, 60, 43, 42,  1, 59, 57,  1, 46, 59, 51, 39, 52, 43,\n",
      "        50, 63, 11,  0, 40, 59, 58,  1, 58, 46, 43, 63,  1, 58, 46, 47, 52, 49,\n",
      "         1, 61, 43,  1, 39, 56, 43,  1, 58, 53, 53,  1, 42, 43, 39, 56, 10,  1,\n",
      "        58, 46, 43,  1, 50, 43, 39, 52, 52, 43, 57, 57,  1, 58, 46, 39, 58,  0,\n",
      "        39, 44, 44, 50, 47, 41, 58, 57,  1, 59, 57,  6,  1, 58, 46, 43,  1, 53,\n",
      "        40, 48, 43, 41, 58,  1, 53, 44,  1, 53, 59, 56,  1, 51, 47, 57, 43, 56,\n",
      "        63,  6,  1, 47, 57,  1, 39, 57,  1, 39, 52,  0, 47, 52, 60, 43, 52, 58,\n",
      "        53, 56, 63,  1, 58, 53,  1, 54, 39, 56, 58, 47, 41, 59, 50, 39, 56, 47,\n",
      "        57, 43,  1, 58, 46, 43, 47, 56,  1, 39, 40, 59, 52, 42, 39, 52, 41, 43,\n",
      "        11,  1, 53, 59, 56,  0, 57, 59, 44, 44, 43, 56, 39, 52, 41, 43,  1, 47,\n",
      "        57,  1, 39,  1, 45, 39, 47, 52,  1, 58, 53,  1, 58, 46, 43, 51,  1, 24,\n",
      "        43, 58,  1, 59, 57,  1, 56, 43, 60, 43, 52, 45, 43,  1, 58, 46, 47, 57,\n",
      "         1, 61, 47, 58, 46,  0, 53, 59, 56,  1, 54, 47, 49, 43, 57,  6,  1, 43,\n",
      "        56, 43,  1, 61, 43,  1, 40, 43, 41, 53, 51, 43,  1, 56, 39, 49, 43, 57,\n",
      "        10,  1, 44, 53, 56,  1, 58, 46, 43,  1, 45, 53, 42, 57,  1, 49, 52, 53,\n",
      "        61,  1, 21,  0, 57, 54, 43, 39, 49,  1, 58, 46, 47, 57,  1, 47, 52,  1,\n",
      "        46, 59, 52, 45, 43, 56,  1, 44, 53, 56,  1, 40, 56, 43, 39, 42,  6,  1,\n",
      "        52, 53, 58,  1, 47, 52,  1, 58, 46, 47, 56, 57, 58,  1, 44, 53, 56,  1,\n",
      "        56, 43, 60, 43, 52, 45, 43,  8,  0,  0])\n"
     ]
    }
   ],
   "source": [
    "# let's now encode the entire text dataset and store it into a torch.Tensor\n",
    "import torch # we use PyTorch: https://pytorch.org\n",
    "data = torch.tensor(encode(text), dtype=torch.long)\n",
    "print(data.shape, data.dtype)\n",
    "print(data[:1000]) # the 1000 characters we looked at earier will to the GPT look like this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "f_WIXqxz0lU5"
   },
   "outputs": [],
   "source": [
    "# Let's now split up the data into train and validation sets\n",
    "n = int(0.9*len(data)) # first 90% will be train, rest val\n",
    "train_data = data[:n]\n",
    "val_data = data[n:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TD5Bj8Y6IAD4",
    "outputId": "bf23c586-1d33-4af1-b63d-ce6f90b0a528"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([18, 47, 56, 57, 58,  1, 15, 47, 58])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "block_size = 8\n",
    "train_data[:block_size+1]\n",
    "\n",
    "# :操作符用于指示切片或值范围。在这种情况下，train_data[:block_size+1]将返回一个新列表，该列表包含从train_data的开始到索引block_size（包含）的元素。\n",
    "# Python使用基于零的索引，所以第一个元素在索引0处。因此，如果block_size是5，那么这段代码将返回train_data的前6个元素（索引0到5）。\n",
    "# 需要注意的是，切片中的结束索引是独占的，这意味着它不包括该索引处的元素。然而，在这种情况下，我们对block_size加了1，所以索引block_size处的元素包含在切片中。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9HXDe8vGJCEn",
    "outputId": "588663aa-1de5-4ef7-aba0-4a96fe828353"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([18, 47, 56, 57, 58,  1, 15, 47])\n",
      "tensor([47, 56, 57, 58,  1, 15, 47, 58])\n",
      "when input is tensor([18]) the target: 47\n",
      "when input is tensor([18, 47]) the target: 56\n",
      "when input is tensor([18, 47, 56]) the target: 57\n",
      "when input is tensor([18, 47, 56, 57]) the target: 58\n",
      "when input is tensor([18, 47, 56, 57, 58]) the target: 1\n",
      "when input is tensor([18, 47, 56, 57, 58,  1]) the target: 15\n",
      "when input is tensor([18, 47, 56, 57, 58,  1, 15]) the target: 47\n",
      "when input is tensor([18, 47, 56, 57, 58,  1, 15, 47]) the target: 58\n"
     ]
    }
   ],
   "source": [
    "x = train_data[:block_size]\n",
    "y = train_data[1:block_size+1]\n",
    "print(x)\n",
    "print(y)\n",
    "for t in range(block_size):\n",
    "    context = x[:t+1]\n",
    "    target = y[t]\n",
    "    print(f\"when input is {context} the target: {target}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Q3k1Czf7LuA9",
    "outputId": "4ea8e8a0-443c-49bb-b3bf-ba36e1712999"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inputs:\n",
      "torch.Size([4, 8])\n",
      "tensor([[24, 43, 58,  5, 57,  1, 46, 43],\n",
      "        [44, 53, 56,  1, 58, 46, 39, 58],\n",
      "        [52, 58,  1, 58, 46, 39, 58,  1],\n",
      "        [25, 17, 27, 10,  0, 21,  1, 54]])\n",
      "targets:\n",
      "torch.Size([4, 8])\n",
      "tensor([[43, 58,  5, 57,  1, 46, 43, 39],\n",
      "        [53, 56,  1, 58, 46, 39, 58,  1],\n",
      "        [58,  1, 58, 46, 39, 58,  1, 46],\n",
      "        [17, 27, 10,  0, 21,  1, 54, 39]])\n",
      "----\n",
      "when input is [24] the target: 43\n",
      "when input is [24, 43] the target: 58\n",
      "when input is [24, 43, 58] the target: 5\n",
      "when input is [24, 43, 58, 5] the target: 57\n",
      "when input is [24, 43, 58, 5, 57] the target: 1\n",
      "when input is [24, 43, 58, 5, 57, 1] the target: 46\n",
      "when input is [24, 43, 58, 5, 57, 1, 46] the target: 43\n",
      "when input is [24, 43, 58, 5, 57, 1, 46, 43] the target: 39\n",
      "when input is [44] the target: 53\n",
      "when input is [44, 53] the target: 56\n",
      "when input is [44, 53, 56] the target: 1\n",
      "when input is [44, 53, 56, 1] the target: 58\n",
      "when input is [44, 53, 56, 1, 58] the target: 46\n",
      "when input is [44, 53, 56, 1, 58, 46] the target: 39\n",
      "when input is [44, 53, 56, 1, 58, 46, 39] the target: 58\n",
      "when input is [44, 53, 56, 1, 58, 46, 39, 58] the target: 1\n",
      "when input is [52] the target: 58\n",
      "when input is [52, 58] the target: 1\n",
      "when input is [52, 58, 1] the target: 58\n",
      "when input is [52, 58, 1, 58] the target: 46\n",
      "when input is [52, 58, 1, 58, 46] the target: 39\n",
      "when input is [52, 58, 1, 58, 46, 39] the target: 58\n",
      "when input is [52, 58, 1, 58, 46, 39, 58] the target: 1\n",
      "when input is [52, 58, 1, 58, 46, 39, 58, 1] the target: 46\n",
      "when input is [25] the target: 17\n",
      "when input is [25, 17] the target: 27\n",
      "when input is [25, 17, 27] the target: 10\n",
      "when input is [25, 17, 27, 10] the target: 0\n",
      "when input is [25, 17, 27, 10, 0] the target: 21\n",
      "when input is [25, 17, 27, 10, 0, 21] the target: 1\n",
      "when input is [25, 17, 27, 10, 0, 21, 1] the target: 54\n",
      "when input is [25, 17, 27, 10, 0, 21, 1, 54] the target: 39\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(1337)\n",
    "batch_size = 4 # how many independent sequences will we process in parallel?\n",
    "block_size = 8 # what is the maximum context length for predictions?\n",
    "\n",
    "def get_batch(split):\n",
    "    # generate a small batch of data of inputs x and targets y\n",
    "    data = train_data if split == 'train' else val_data\n",
    "    # 生成一个长度为batch_size的一维张量，张量中的每个元素都是一个随机整数，这个整数的范围是0到len(data) - block_size。这些随机整数将被用作从数据中提取序列的起始索引。\n",
    "    ix = torch.randint(len(data) - block_size, (batch_size,))\n",
    "    x = torch.stack([data[i:i+block_size] for i in ix])\n",
    "    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n",
    "    return x, y\n",
    "\n",
    "xb, yb = get_batch('train')\n",
    "print('inputs:')\n",
    "print(xb.shape)\n",
    "print(xb)\n",
    "print('targets:')\n",
    "print(yb.shape)\n",
    "print(yb)\n",
    "\n",
    "print('----')\n",
    "\n",
    "for b in range(batch_size): # batch dimension\n",
    "    for t in range(block_size): # time dimension\n",
    "        context = xb[b, :t+1]\n",
    "        target = yb[b,t]\n",
    "        print(f\"when input is {context.tolist()} the target: {target}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "qpyyAeIzQjlO",
    "outputId": "a650f8dc-da81-400b-bc59-0a595487fdb9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[24, 43, 58,  5, 57,  1, 46, 43],\n",
      "        [44, 53, 56,  1, 58, 46, 39, 58],\n",
      "        [52, 58,  1, 58, 46, 39, 58,  1],\n",
      "        [25, 17, 27, 10,  0, 21,  1, 54]])\n",
      "tensor([[43, 58,  5, 57,  1, 46, 43, 39],\n",
      "        [53, 56,  1, 58, 46, 39, 58,  1],\n",
      "        [58,  1, 58, 46, 39, 58,  1, 46],\n",
      "        [17, 27, 10,  0, 21,  1, 54, 39]])\n"
     ]
    }
   ],
   "source": [
    "print(xb) # our input to the transformer\n",
    "print(yb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "nql_1ER53oCf",
    "outputId": "5de90b1b-4603-428a-f571-fe4bd3c45436"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 65])\n",
      "tensor(4.8786, grad_fn=<NllLossBackward0>)\n",
      "\n",
      "SKIcLT;AcELMoTbvZv C?nq-QE33:CJqkOKH-q;:la!oiywkHjgChzbQ?u!3bLIgwevmyFJGUGp\n",
      "wnYWmnxKWWev-tDqXErVKLgJ\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import functional as F\n",
    "torch.manual_seed(1337)\n",
    "\n",
    "class BigramLanguageModel(nn.Module):\n",
    "\n",
    "    def __init__(self, vocab_size):\n",
    "        super().__init__()\n",
    "        # each token directly reads off the logits for the next token from a lookup table\n",
    "        # 嵌入表将每个词汇（token）映射到一个向量，向量的大小等于词汇表的大小。\n",
    "        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)\n",
    "\n",
    "    def forward(self, idx, targets=None):\n",
    "\n",
    "        # idx and targets are both (B,T) tensor of integers\n",
    "        logits = self.token_embedding_table(idx) # (B,T,C)\n",
    "\n",
    "        if targets is None:\n",
    "            loss = None\n",
    "        else:\n",
    "            B, T, C = logits.shape\n",
    "            # B代表批量大小（batch size），T代表时间步长（time steps），C代表类别数量（class number）。这里的C是词汇表的大小。\n",
    "            logits = logits.view(B*T, C)\n",
    "            targets = targets.view(B*T)\n",
    "            # PyTorch的F.cross_entropy函数期望的输入是这样的形状：logits应该是一个二维张量，其中每一行对应一个输入样本，每一列对应一个类别；targets应该是一个一维张量，其中每个元素对应一个输入样本的类别索引。\n",
    "            loss = F.cross_entropy(logits, targets)\n",
    "\n",
    "        return logits, loss\n",
    "\n",
    "    # generate方法是模型的生成函数。它接收当前的上下文idx和最大新生成token的数量max_new_tokens，并生成一个新的序列。这个方法首先获取模型的预测，然后只关注最后一个时间步的logits。\n",
    "    # 然后，它应用softmax函数来获取概率分布，从这个分布中采样下一个token的索引，然后将这个索引添加到当前的序列中。这个过程重复max_new_tokens次。\n",
    "    def generate(self, idx, max_new_tokens):\n",
    "        # idx is (B, T) array of indices in the current context\n",
    "        for _ in range(max_new_tokens):\n",
    "            # get the predictions\n",
    "            logits, loss = self(idx)\n",
    "            # focus only on the last time step\n",
    "            logits = logits[:, -1, :] # becomes (B, C)\n",
    "            # apply softmax to get probabilities\n",
    "            # F.softmax(logits, dim=-1)的作用是对logits的每一行（也就是每个样本的所有类别的输出）进行softmax操作。\n",
    "            # softmax操作会将每个元素的值转换为0到1之间的值，并且所有元素的值的和为1。这样，每一行的值就可以被解释为一个概率分布，表示模型预测每个类别的概率。\n",
    "            probs = F.softmax(logits, dim=-1) # (B, C)\n",
    "            # sample from the distribution\n",
    "            idx_next = torch.multinomial(probs, num_samples=1) # (B, 1)\n",
    "            # append sampled index to the running sequence\n",
    "            # torch.cat函数来将idx和idx_next两个张量在第二个维度（dim=1）上进行拼接。\n",
    "            idx = torch.cat((idx, idx_next), dim=1) # (B, T+1)\n",
    "        return idx\n",
    "\n",
    "m = BigramLanguageModel(vocab_size)\n",
    "logits, loss = m(xb, yb)\n",
    "print(logits.shape)\n",
    "print(loss)\n",
    "\n",
    "# 生成一个新的序列，将这个序列解码为文本，然后打印出这个文本。\n",
    "print(decode(m.generate(idx = torch.zeros((1, 1), dtype=torch.long), max_new_tokens=100)[0].tolist()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "eTyJ8qAaDdiF"
   },
   "outputs": [],
   "source": [
    "# create a PyTorch optimizer\n",
    "optimizer = torch.optim.AdamW(m.parameters(), lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Hs4kI8YdEkQj",
    "outputId": "42ded55c-2983-4d91-c528-675b2edfa849"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.531792163848877\n"
     ]
    }
   ],
   "source": [
    "batch_size = 32\n",
    "for steps in range(100): # increase number of steps for good results...\n",
    "\n",
    "    # sample a batch of data\n",
    "    xb, yb = get_batch('train')\n",
    "\n",
    "    # evaluate the loss\n",
    "    logits, loss = m(xb, yb)\n",
    "    optimizer.zero_grad(set_to_none=True)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "print(loss.item())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EcVIDWAZEtjN",
    "outputId": "0ad6f9d2-ad58-4498-a5f8-6f31407bb18b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "oiWIgkOIVFTBcJrb$T;AcqKlVpJZR\n",
      "RmntLvNAm:VHOdPyuFIGjsuJJv'VTbpQA'OEHA:CYSqo$-Ila;T;ABDC-zoiyxc;bO&X'FqVxBYW\n",
      "rXlcnL,iKD3fRb$GRj;CKHAINSgfMOsIG3S&alyln;ERjmylkteErSPTH;I&CJTASLT'gdco..JGixYGSP P?:CXSjRc,SPlvCnpBMlv.iv-urbWy;RcfRm vU?Bq-bZF\n",
      "pSnPl,ehiPhSA-cGNSP!z,OmEbfFA-gFOy:..dbfIotNmnErJpLfyFhi-iOl!IUCjRroxJBSGGSPZK;\n",
      "p.ltFoiUCJkzPOLXXEca-jalyKl-UiPi-oX&zkzVjk!kD3TBTnUVcGjoggCjxElaye.JG-Ko-\n",
      "SYguBigzPLLTRwNoa!PAfttS!PHQnC&XXcsullN:Cbf;aPytq--olgmQxfl, ajbtT!$\n",
      "hORSxL3sByzPyNwNfpDEHFZ;itkyH?dagqHNYwd&\n"
     ]
    }
   ],
   "source": [
    "print(decode(m.generate(idx = torch.zeros((1, 1), dtype=torch.long), max_new_tokens=500)[0].tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XinV8nmAnmKN"
   },
   "source": [
    "## The mathematical trick in self-attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tukiH-NbRBhA",
    "outputId": "d981f6d4-ac08-4ec2-8284-82f5fa1e0815"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a=\n",
      "tensor([[1.0000, 0.0000, 0.0000],\n",
      "        [0.5000, 0.5000, 0.0000],\n",
      "        [0.3333, 0.3333, 0.3333]])\n",
      "--\n",
      "b=\n",
      "tensor([[2., 7.],\n",
      "        [6., 4.],\n",
      "        [6., 5.]])\n",
      "--\n",
      "c=\n",
      "tensor([[2.0000, 7.0000],\n",
      "        [4.0000, 5.5000],\n",
      "        [4.6667, 5.3333]])\n"
     ]
    }
   ],
   "source": [
    "# toy example illustrating how matrix multiplication can be used for a \"weighted aggregation\"\n",
    "torch.manual_seed(42)\n",
    "a = torch.tril(torch.ones(3, 3))\n",
    "# torch.sum 计算张量a在第二个维度（dim=1）上的和。keepdim=True表示保持输出的维度\n",
    "a = a / torch.sum(a, 1, keepdim=True)\n",
    "b = torch.randint(0,10,(3,2)).float()\n",
    "c = a @ b\n",
    "print('a=')\n",
    "print(a)\n",
    "print('--')\n",
    "print('b=')\n",
    "print(b)\n",
    "print('--')\n",
    "print('c=')\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Hs_E24uRE8kr",
    "outputId": "8bf3ff5f-565e-48b8-de8e-7272706c8e12"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 8, 2])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# consider the following toy example:\n",
    "\n",
    "torch.manual_seed(1337)\n",
    "B,T,C = 4,8,2 # batch, time, channels\n",
    "x = torch.randn(B,T,C)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "86NuXX0fn7ps"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0\n",
      "tensor([[ 0.1808, -0.0700]])\n",
      "tensor([ 0.1808, -0.0700])\n",
      "0\n",
      "1\n",
      "tensor([[ 0.1808, -0.0700],\n",
      "        [-0.3596, -0.9152]])\n",
      "tensor([-0.0894, -0.4926])\n",
      "0\n",
      "2\n",
      "tensor([[ 0.1808, -0.0700],\n",
      "        [-0.3596, -0.9152],\n",
      "        [ 0.6258,  0.0255]])\n",
      "tensor([ 0.1490, -0.3199])\n",
      "0\n",
      "3\n",
      "tensor([[ 0.1808, -0.0700],\n",
      "        [-0.3596, -0.9152],\n",
      "        [ 0.6258,  0.0255],\n",
      "        [ 0.9545,  0.0643]])\n",
      "tensor([ 0.3504, -0.2238])\n",
      "0\n",
      "4\n",
      "tensor([[ 0.1808, -0.0700],\n",
      "        [-0.3596, -0.9152],\n",
      "        [ 0.6258,  0.0255],\n",
      "        [ 0.9545,  0.0643],\n",
      "        [ 0.3612,  1.1679]])\n",
      "tensor([0.3525, 0.0545])\n",
      "0\n",
      "5\n",
      "tensor([[ 0.1808, -0.0700],\n",
      "        [-0.3596, -0.9152],\n",
      "        [ 0.6258,  0.0255],\n",
      "        [ 0.9545,  0.0643],\n",
      "        [ 0.3612,  1.1679],\n",
      "        [-1.3499, -0.5102]])\n",
      "tensor([ 0.0688, -0.0396])\n",
      "0\n",
      "6\n",
      "tensor([[ 0.1808, -0.0700],\n",
      "        [-0.3596, -0.9152],\n",
      "        [ 0.6258,  0.0255],\n",
      "        [ 0.9545,  0.0643],\n",
      "        [ 0.3612,  1.1679],\n",
      "        [-1.3499, -0.5102],\n",
      "        [ 0.2360, -0.2398]])\n",
      "tensor([ 0.0927, -0.0682])\n",
      "0\n",
      "7\n",
      "tensor([[ 0.1808, -0.0700],\n",
      "        [-0.3596, -0.9152],\n",
      "        [ 0.6258,  0.0255],\n",
      "        [ 0.9545,  0.0643],\n",
      "        [ 0.3612,  1.1679],\n",
      "        [-1.3499, -0.5102],\n",
      "        [ 0.2360, -0.2398],\n",
      "        [-0.9211,  1.5433]])\n",
      "tensor([-0.0341,  0.1332])\n"
     ]
    }
   ],
   "source": [
    "# version 1: for loop and gather, use torch.mean\n",
    "# We want x[b,t] = mean_{i<=t} x[b,i]\n",
    "xbow = torch.zeros((B,T,C))\n",
    "# print(xbow)\n",
    "for b in range(B):\n",
    "    for t in range(T):\n",
    "        # 使用切片操作x[b,:t+1]来获取x在第b个批次中前t+1个时间步的所有元素，得到一个形状为(t,C)的张量xprev。\n",
    "        xprev = x[b,:t+1] # (t,C)\n",
    "        # 使用torch.mean(xprev, 0)来计算xprev在第一个维度（dim=0）上的平均值。这个操作会返回一个形状为(C,)的张量，它的每个元素是xprev在对应列上的元素的平均值。\n",
    "        xbow[b,t] = torch.mean(xprev, 0)\n",
    "        \n",
    "        print(b) if b==0 else None\n",
    "        print(t) if b==0 else None\n",
    "        print(xprev) if b==0 else None\n",
    "        print(xbow[b,t]) if b==0 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yhdOAd6-wXkZ",
    "outputId": "eaf6ab61-dff1-4bb7-e623-47f692bad5f9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1.]])\n",
      "wei===\n",
      "tensor([[1.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.5000, 0.5000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.3333, 0.3333, 0.3333, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.2500, 0.2500, 0.2500, 0.2500, 0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.2000, 0.2000, 0.2000, 0.2000, 0.2000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.1667, 0.1667, 0.1667, 0.1667, 0.1667, 0.1667, 0.0000, 0.0000],\n",
      "        [0.1429, 0.1429, 0.1429, 0.1429, 0.1429, 0.1429, 0.1429, 0.0000],\n",
      "        [0.1250, 0.1250, 0.1250, 0.1250, 0.1250, 0.1250, 0.1250, 0.1250]])\n",
      "torch.Size([4, 8, 2])\n",
      "torch.Size([8, 8])\n",
      "x[2]\n",
      "tensor([[-0.6631, -0.2513],\n",
      "        [ 1.0101,  0.1215],\n",
      "        [ 0.1584,  1.1340],\n",
      "        [-1.1539, -0.2984],\n",
      "        [-0.5075, -0.9239],\n",
      "        [ 0.5467, -1.4948],\n",
      "        [-1.2057,  0.5718],\n",
      "        [-0.5974, -0.6937]])\n",
      "xbow2===\n",
      "tensor([[-0.6631, -0.2513],\n",
      "        [ 0.1735, -0.0649],\n",
      "        [ 0.1685,  0.3348],\n",
      "        [-0.1621,  0.1765],\n",
      "        [-0.2312, -0.0436],\n",
      "        [-0.1015, -0.2855],\n",
      "        [-0.2593, -0.1630],\n",
      "        [-0.3015, -0.2293]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# version 2: using matrix multiply for a weighted aggregation\n",
    "wei = torch.tril(torch.ones(T, T))\n",
    "print(wei)\n",
    "wei = wei / wei.sum(1, keepdim=True)\n",
    "print(\"wei===\")\n",
    "print(wei)\n",
    "print(x.shape)\n",
    "print(wei.shape)\n",
    "xbow2 = wei @ x # (B, T, T) @ (B, T, C) ----> (B, T, C)\n",
    "print(\"x[2]\")\n",
    "print(x[2])\n",
    "print(\"xbow2===\")\n",
    "print(xbow2[2])\n",
    "torch.allclose(xbow, xbow2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "wOURrfG-ysoL",
    "outputId": "080b500d-8110-4602-fcef-7d6f2ebfc6bc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tril===\n",
      "tensor([[1., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1.]])\n",
      "wei===\n",
      "tensor([[0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0.]])\n",
      "wei===\n",
      "tensor([[0., -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n",
      "        [0., 0., -inf, -inf, -inf, -inf, -inf, -inf],\n",
      "        [0., 0., 0., -inf, -inf, -inf, -inf, -inf],\n",
      "        [0., 0., 0., 0., -inf, -inf, -inf, -inf],\n",
      "        [0., 0., 0., 0., 0., -inf, -inf, -inf],\n",
      "        [0., 0., 0., 0., 0., 0., -inf, -inf],\n",
      "        [0., 0., 0., 0., 0., 0., 0., -inf],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0.]])\n",
      "wei===\n",
      "tensor([[1.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.5000, 0.5000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.3333, 0.3333, 0.3333, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.2500, 0.2500, 0.2500, 0.2500, 0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.2000, 0.2000, 0.2000, 0.2000, 0.2000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.1667, 0.1667, 0.1667, 0.1667, 0.1667, 0.1667, 0.0000, 0.0000],\n",
      "        [0.1429, 0.1429, 0.1429, 0.1429, 0.1429, 0.1429, 0.1429, 0.0000],\n",
      "        [0.1250, 0.1250, 0.1250, 0.1250, 0.1250, 0.1250, 0.1250, 0.1250]])\n",
      "result===\n",
      "tensor([[ 0.1808, -0.0700],\n",
      "        [-0.3596, -0.9152],\n",
      "        [ 0.6258,  0.0255],\n",
      "        [ 0.9545,  0.0643],\n",
      "        [ 0.3612,  1.1679],\n",
      "        [-1.3499, -0.5102],\n",
      "        [ 0.2360, -0.2398],\n",
      "        [-0.9211,  1.5433]])\n",
      "tensor([[ 0.1808, -0.0700],\n",
      "        [-0.0894, -0.4926],\n",
      "        [ 0.1490, -0.3199],\n",
      "        [ 0.3504, -0.2238],\n",
      "        [ 0.3525,  0.0545],\n",
      "        [ 0.0688, -0.0396],\n",
      "        [ 0.0927, -0.0682],\n",
      "        [-0.0341,  0.1332]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# version 3: use Softmax\n",
    "tril = torch.tril(torch.ones(T, T))\n",
    "print(\"tril===\")\n",
    "print(tril)\n",
    "wei = torch.zeros((T,T))\n",
    "print(\"wei===\")\n",
    "print(wei)\n",
    "\n",
    "# 将wei中对应tril == 0为True的元素替换为负无穷。也就是说，如果tril的某个元素等于0，那么wei的对应元素就会被替换为负无穷。\n",
    "wei = wei.masked_fill(tril == 0, float('-inf'))\n",
    "print(\"wei===\")\n",
    "print(wei)\n",
    "wei = F.softmax(wei, dim=-1)\n",
    "print(\"wei===\")\n",
    "print(wei)\n",
    "xbow3 = wei @ x\n",
    "print(\"result===\")\n",
    "print(x[0])\n",
    "print(xbow3[0])\n",
    "torch.allclose(xbow, xbow3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EDarxEWIRMKq",
    "outputId": "07b587dd-a91c-4bb0-d7f1-e247cd5dacb5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 8, 16])\n",
      "torch.Size([4, 8, 16])\n",
      "========\n",
      "torch.Size([4, 8, 8])\n",
      "torch.Size([4, 8, 16])\n",
      "torch.Size([4, 8, 16])\n"
     ]
    }
   ],
   "source": [
    "# version 4: self-attention!\n",
    "torch.manual_seed(1337)\n",
    "B,T,C = 4,8,32 # batch, time, channels\n",
    "x = torch.randn(B,T,C)\n",
    "\n",
    "# let's see a single Head perform self-attention\n",
    "head_size = 16\n",
    "# 定义了一个线性变换层，它接收一个形状为(N, C)的输入，返回一个形状为(N, head_size)的输出。这个线性变换层的权重矩阵的形状是(C, head_size)，它没有偏置向量（bias vector），因为bias=False。\n",
    "key = nn.Linear(C, head_size, bias=False)\n",
    "query = nn.Linear(C, head_size, bias=False)\n",
    "value = nn.Linear(C, head_size, bias=False)\n",
    "k = key(x)   # (B, T, 16)\n",
    "q = query(x) # (B, T, 16)\n",
    "wei =  q @ k.transpose(-2, -1) # (B, T, 16) @ (B, 16, T) ---> (B, T, T)\n",
    "\n",
    "tril = torch.tril(torch.ones(T, T))\n",
    "#wei = torch.zeros((T,T))\n",
    "wei = wei.masked_fill(tril == 0, float('-inf'))\n",
    "wei = F.softmax(wei, dim=-1)\n",
    "\n",
    "v = value(x)\n",
    "out = wei @ v\n",
    "#out = wei @ x\n",
    "\n",
    "print(k.shape)\n",
    "print(q.shape)\n",
    "print(\"========\")\n",
    "print(wei.shape)\n",
    "print(v.shape)\n",
    "print(out.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vT1hdtzXCjgL",
    "outputId": "6d2c569b-7922-451f-9934-0fc564678d17"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.1574, 0.8426, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.2088, 0.1646, 0.6266, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.5792, 0.1187, 0.1889, 0.1131, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0294, 0.1052, 0.0469, 0.0276, 0.7909, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0176, 0.2689, 0.0215, 0.0089, 0.6812, 0.0019, 0.0000, 0.0000],\n",
       "        [0.1691, 0.4066, 0.0438, 0.0416, 0.1048, 0.2012, 0.0329, 0.0000],\n",
       "        [0.0210, 0.0843, 0.0555, 0.2297, 0.0573, 0.0709, 0.2423, 0.2391]],\n",
       "       grad_fn=<SelectBackward0>)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wei[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "M5CvobiQ0pLr"
   },
   "source": [
    "Notes:\n",
    "- Attention is a **communication mechanism**. Can be seen as nodes in a directed graph looking at each other and aggregating information with a weighted sum from all nodes that point to them, with data-dependent weights.\n",
    "- There is no notion of space. Attention simply acts over a set of vectors. This is why we need to positionally encode tokens.\n",
    "- Each example across batch dimension is of course processed completely independently and never \"talk\" to each other\n",
    "- In an \"encoder\" attention block just delete the single line that does masking with `tril`, allowing all tokens to communicate. This block here is called a \"decoder\" attention block because it has triangular masking, and is usually used in autoregressive settings, like language modeling.\n",
    "- \"self-attention\" just means that the keys and values are produced from the same source as queries. In \"cross-attention\", the queries still get produced from x, but the keys and values come from some other, external source (e.g. an encoder module)\n",
    "- \"Scaled\" attention additional divides `wei` by 1/sqrt(head_size). This makes it so when input Q,K are unit variance, wei will be unit variance too and Softmax will stay diffuse and not saturate too much. Illustration below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "4SNbLq5z3oBw"
   },
   "outputs": [],
   "source": [
    "k = torch.randn(B,T,head_size)\n",
    "q = torch.randn(B,T,head_size)\n",
    "wei = q @ k.transpose(-2, -1) * head_size**-0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Nl6I9n9IRTSo",
    "outputId": "0c5b9cd0-af8a-4564-fbad-41d844e54822"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.0966)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k.var()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "T1tQx7oeRvtc",
    "outputId": "3541ca1a-7447-4ef7-835e-81824aebc1b5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.9416)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q.var()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MLb_odHU3iKM",
    "outputId": "a687a222-5a2c-4cdb-c1bf-17cd05b45b69"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.0065)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wei.var()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JB82yzt44REI",
    "outputId": "f07da2f1-10bb-4a7a-bcaa-578587977d00"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.1925, 0.1426, 0.2351, 0.1426, 0.2872])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.softmax(torch.tensor([0.1, -0.2, 0.3, -0.2, 0.5]), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Mpt8569BB9_f",
    "outputId": "5d8b910a-6192-44ba-ebb2-497d88e0b629"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.0326, 0.0030, 0.1615, 0.0030, 0.8000])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.softmax(torch.tensor([0.1, -0.2, 0.3, -0.2, 0.5])*8, dim=-1) # gets too peaky, converges to one-hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAArUAAAIjCAYAAAAdn+MfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABf5UlEQVR4nO3deVxVdeL/8fcFBdwAFQQ1FJdy33IhaFFHiswWG/OrtqBk2qKWYjNKU6I2ZWWZjTlZjVuTTmbTtGk6SmJT4pJGuZcL4gZqpigqKJzfH/68deGCXLj7fT0fj/uYueeec/gcruCr4+ecazIMwxAAAADgwfxcPQAAAACgqohaAAAAeDyiFgAAAB6PqAUAAIDHI2oBAADg8YhaAAAAeDyiFgAAAB6PqAUAAIDHI2oBAADg8YhaoAqmT5+u5s2by9/fX507d3b1cHxWdHS0hg0b5uphOF16erpMJpM++ugju+1z2LBhql27doXWNZlMmjx5svn5ggULZDKZlJWVZV7Wq1cv9erVy27js4dNmzYpLi5OtWrVkslkUmZmpquHBMAOiFr4jK1bt+q+++5T06ZNFRQUpMaNG+vWW2/VrFmzKrW///73v/rzn/+sG2+8UfPnz9eLL76oI0eOaPLkyR77l2RWVpZMJpPVxw033ODSsa1bt06TJ0/WqVOnXDqOihg2bJjF9y44OFidOnXSa6+9poKCAlcPz6Vc/TNy8eJFDRw4UCdPntTrr7+uf/7zn2ratGml93f06FFNnDhRvXv3Vp06dWQymZSenm6/AQOosGquHgDgDOvWrVPv3r3VpEkTjRgxQpGRkTp48KDWr1+vN954Q2PGjLF5n1999ZX8/Pw0d+5cBQQESJK+++47TZkyRdHR0R595nbIkCG64447LJaFh4e7aDSXrVu3TlOmTNGwYcMUGhpq8dru3bvl5+de/40eGBiof/zjH5KkU6dO6d///reefvppbdq0SR988IGLR2cf58+fV7Vq5f818t///tfi+ZEjR1z6M7J3714dOHBA7777rh555JEq72/37t16+eWXde2116pDhw7KyMiwwygBVAZRC5/wwgsvKCQkRJs2bSoVRMeOHavUPo8dO6YaNWqYg9abXH/99XrwwQddPYwKCwwMdPUQSqlWrZrF9/CJJ55QTEyMlixZohkzZqhRo0altjEMQxcuXFCNGjWcOdRKCwoKuuo67vbzceXnveTvgcrq2rWrfvnlF9WrV08fffSRBg4caJf9ArCde53aABxk7969ateundW/yBo0aGDx/NKlS3r++efVokULBQYGKjo6Ws8884zFPxubTCbNnz9f+fn55n9iXrBggbp37y5JSkpKslguXZ5b2L59e/3444/q2bOnatasqZYtW5rnQ65du1YxMTGqUaOGWrVqpdWrV1uM68CBA3riiSfUqlUr1ahRQ/Xr19fAgQMt5i8ahqHevXsrPDzcItYLCwvVoUMHtWjRQvn5+VX5VpY5R3LYsGGKjo42P78yleHVV1/VO++8Y/5+du/eXZs2bSq1/a5du/R///d/Cg8PN38P/vKXv0iSJk+erD/96U+SpGbNmpm/t1eO3dqc2n379mngwIGqV6+eatasqRtuuEHLli2zWOfKnNQPP/xQL7zwgq655hoFBQWpT58+2rNnj8W6586d065du3TixAkbv2OX+fn5mb9vvx/3nXfeqZUrV6pbt26qUaOG3n777QqP/4qioiI988wzioyMVK1atXT33Xfr4MGDFuv873//08CBA9WkSRMFBgYqKipK48aN0/nz563uc9++fUpISFCtWrXUqFEjTZ06VYZhWKxTck6tNb//85Kenl7mz0hqaqqqV6+u48ePl9rHyJEjFRoaqgsXLpT7tb766ivdfPPNqlWrlkJDQ3XPPfdo586d5teHDRumnj17SpIGDhwok8lU7nzfsqbi/P7PXp06dVSvXr1yxwXAOThTC5/QtGlTZWRkaNu2bWrfvn256z7yyCNauHCh7rvvPo0fP14bNmzQtGnTtHPnTv3nP/+RJP3zn//UO++8o40bN5r/ifnaa6/V1KlTNWnSJI0cOVI333yzJCkuLs68719//VV33nmnBg8erIEDB+qtt97S4MGDtWjRIo0dO1aPPfaY7r//fk2fPl333XefDh48qDp16ki6fHHLunXrNHjwYF1zzTXKysrSW2+9pV69emnHjh2qWbOmTCaT5s2bp44dO+qxxx7Txx9/LElKTU3V9u3blZ6erlq1al31+3Xu3LlS8RYSEqLq1atX8Dv+m8WLF+vMmTN69NFHZTKZ9Morr+iPf/yj9u3bZ97fjz/+qJtvvlnVq1fXyJEjFR0drb179+rzzz/XCy+8oD/+8Y/66aef9K9//Uuvv/66wsLCJJU9JSI3N1dxcXE6d+6cnnzySdWvX18LFy7U3XffrY8++kj33nuvxfovvfSS/Pz89PTTT+v06dN65ZVX9MADD2jDhg3mdTZu3KjevXsrNTX1qiFXlr1790qS6tevb162e/duDRkyRI8++qhGjBihVq1a2Tz+F154QSaTSRMmTNCxY8c0c+ZMxcfHKzMz03zWd+nSpTp37pwef/xx1a9fXxs3btSsWbN06NAhLV261GJ/RUVFuv3223XDDTfolVde0YoVK5SamqpLly5p6tSplTp2SWrTpk2ZPyM33XSTpk6dqiVLlmj06NHmbQoLC/XRRx9pwIAB5Z4ZXr16tfr27avmzZtr8uTJOn/+vGbNmqUbb7xRW7ZsUXR0tB599FE1btxYL774op588kl1795dERERZe7zn//8Z6llzz77rI4dO1bhi+kAOJEB+ID//ve/hr+/v+Hv72/ExsYaf/7zn42VK1cahYWFFutlZmYakoxHHnnEYvnTTz9tSDK++uor87KhQ4catWrVslhv06ZNhiRj/vz5pcbQs2dPQ5KxePFi87Jdu3YZkgw/Pz9j/fr15uUrV64stZ9z586V2mdGRoYhyXjvvfcslr/99tuGJOP999831q9fb/j7+xtjx44t+xv0/+3fv9+QZPWxZs0a83H07Nmz1LZDhw41mjZtWmpf9evXN06ePGle/umnnxqSjM8//9y87JZbbjHq1KljHDhwwGKfxcXF5v8/ffp0Q5Kxf//+Ul+7adOmxtChQ83Px44da0gy/ve//5mXnTlzxmjWrJkRHR1tFBUVGYZhGGvWrDEkGW3atDEKCgrM677xxhuGJGPr1q3mZVfWTU1Ntfq9K/m9qFWrlnH8+HHj+PHjxp49e4wXX3zRMJlMRseOHS3GLclYsWKFxfa2jr9x48ZGXl6eed0PP/zQkGS88cYb5mXW/vxMmzbNMJlMFt/3oUOHGpKMMWPGmJcVFxcb/fr1MwICAozjx4+bl5f8fsyfP7/Ue1Tyz0t5PyOxsbFGTEyMxbKPP/7Y4s9fWTp37mw0aNDA+OWXX8zLfvjhB8PPz89ITEw0L7vyPVu6dGm5+7PmlVdesfrzdsXSpUsrNFYAjsH0A/iEW2+9VRkZGbr77rv1ww8/6JVXXlFCQoIaN26szz77zLze8uXLJUnJyckW248fP16Syvzn34qqXbu2Bg8ebH7eqlUrhYaGqk2bNoqJiTEvv/L/9+3bZ172+3mWFy9e1C+//KKWLVsqNDRUW7Zssfg6I0eOVEJCgsaMGaOHHnpILVq00IsvvljhcY4cOVKrVq2yeHTq1Mnm45WkQYMGqW7duubnV87OXTm248eP6+uvv9bDDz+sJk2aWGxrMpkq9TWXL1+uHj166KabbjIvq127tkaOHKmsrCzt2LHDYv2kpCSLuZ8lxyhd/md0wzAqfJY2Pz9f4eHhCg8PV8uWLfXMM88oNjbWfLb/imbNmikhIaFK409MTDSf0Zek++67Tw0bNjT/eZYs//zk5+frxIkTiouLk2EY+v7770uN//dnS00mk0aPHq3CwsJS02LsKTExURs2bDCf0ZakRYsWKSoqyjxtwJqjR48qMzNTw4YNs5gK0LFjR916660W34fKWrNmjVJSUsw/UwDcD1ELn9G9e3d9/PHH+vXXX7Vx40alpKTozJkzuu+++8yRcODAAfn5+ally5YW20ZGRio0NFQHDhyo0hiuueaaUqEWEhKiqKioUsuky9MVrjh//rwmTZqkqKgoBQYGKiwsTOHh4Tp16pROnz5d6mvNnTtX586d088//6wFCxbYdPHRtddeq/j4eIvH78PUFiVD9cp+rhzblXC82rQQWxw4cECtWrUqtbxNmzbm120ZY2UEBQWZ/4Pg66+/1sGDB/Xtt9+qefPmFus1a9asyuO/9tprLZ6bTCa1bNnSYr51dna2Ofpq166t8PBwcyiW/PPj5+dXapzXXXedJFns094GDRqkwMBALVq0yDyuL774Qg888EC5/4Fz5ftR1vfsxIkTVZpLfujQIQ0aNEg33nijZsyYUen9AHAs5tTC5wQEBKh79+7q3r27rrvuOiUlJWnp0qVKTU01r1PZM4RX4+/vb9Ny43cX5owZM0bz58/X2LFjFRsbq5CQEJlMJg0ePFjFxcWltk1PTzdf3LZ161bFxsba4Qguf2+MEhcMSZfnYVpTkWNzNUeM0d/fX/Hx8Vddzxl3OigqKtKtt96qkydPasKECWrdurVq1aqlw4cPa9iwYVb//LhC3bp1deedd2rRokWaNGmSPvroIxUUFLj0ThyFhYW67777FBgYqA8//PCqtzAD4Dr8dMKndevWTdLlf76ULl9QVlxcrJ9//tl8Vky6fOHRqVOnrnqTdkfFsCR99NFHGjp0qF577TXzsgsXLlj9MIKjR49qzJgxuu222xQQEKCnn35aCQkJVbrJ/BV169a1+Gf5Kyp7FvvKGcFt27aVu54t39umTZtq9+7dpZbv2rXL/Lo7s3X8P//8s8VzwzC0Z88edezYUdLl/6j56aeftHDhQiUmJprXW7VqldWvX1xcrH379pnPzkrSTz/9JEkWd7iojKu9j4mJibrnnnu0adMmLVq0SF26dFG7du3K3ebK96Os71lYWFiFLpC05sknn1RmZqa+/vrrci8qA+B6TD+AT1izZo3Vs25X5tpd+WfLKx84MHPmTIv1rvyTY79+/cr9Olf+4nTEp175+/uXOoZZs2ZZPUM6YsQIFRcXa+7cuXrnnXdUrVo1DR8+3C5nR1u0aKFdu3ZZ3Hrphx9+0Lffflup/YWHh+uWW27RvHnzlJ2dbfHa78dry/f2jjvu0MaNGy1uhJ+fn6933nlH0dHRatu2rc3jrOotvWxh6/jfe+89nTlzxvz8o48+0tGjR9W3b19Jv52J/v330zAMvfHGG2WO4c0337RY980331T16tXVp0+fKh3b1d7Hvn37KiwsTC+//LLWrl1bobO0DRs2VOfOnbVw4UKL/W7btk3//e9/S32QSEXNnz9fb7/9tmbPnq0ePXpUah8AnIcztfAJY8aM0blz53TvvfeqdevWKiws1Lp167RkyRJFR0crKSlJktSpUycNHTpU77zzjk6dOqWePXtq48aNWrhwofr376/evXuX+3VatGih0NBQzZkzR3Xq1FGtWrUUExNjdd6kre68807985//VEhIiNq2bauMjAytXr3a4vZQ0uW/iJctW6YFCxbommuukXQ5fh988EG99dZbeuKJJ6o0jocfflgzZsxQQkKChg8frmPHjmnOnDlq166d8vLyKrXPv/3tb7rpppt0/fXXa+TIkWrWrJmysrK0bNky88epdu3aVZL0l7/8RYMHD1b16tV11113WT0DN3HiRP3rX/9S37599eSTT6pevXpauHCh9u/fr3//+9+V+vQxe9zSq6JsHX+9evV00003KSkpSbm5uZo5c6ZatmypESNGSJJat26tFi1a6Omnn9bhw4cVHBysf//732XOGQ4KCtKKFSs0dOhQxcTE6Msvv9SyZcv0zDPPVPmT5a72M1K9enUNHjxYb775pvz9/TVkyJAK7Xf69Onq27evYmNjNXz4cPMtvUJCQir1fp04cUJPPPGE2rZtq8DAQL3//vsWr997773mP3t//etfJUnbt2+XdPlWYN98842ky7cAA+AkrrjlAuBsX375pfHwww8brVu3NmrXrm0EBAQYLVu2NMaMGWPk5uZarHvx4kVjypQpRrNmzYzq1asbUVFRRkpKinHhwgWL9azd0sswLt+yqm3btka1atUsbl3Us2dPo127dqXWb9q0qdGvX79SyyUZo0aNMj//9ddfjaSkJCMsLMyoXbu2kZCQYOzatcvidlYHDx40QkJCjLvuuqvU/u69916jVq1axr59+8r8Pl25Ddf06dPLXMcwDOP99983mjdvbgQEBBidO3c2Vq5cWeYtvaztS1ZujbVt2zbj3nvvNUJDQ42goCCjVatWxnPPPWexzvPPP280btzY8PPzs7h1VMlbehmGYezdu9e47777zPvr0aOH8cUXX1isU9btna6M/fe3narMLb2upqz33tbx/+tf/zJSUlKMBg0aGDVq1DD69etX6vZoO3bsMOLj443atWsbYWFhxogRI4wffvih1HFeGfvevXuN2267zahZs6YRERFhpKammm8ldkXJ70dFbullGGX/jFyxceNGQ5Jx2223lf8NLGH16tXGjTfeaNSoUcMIDg427rrrLmPHjh1Wv2dXu6VXebe3K3mM5a0HwHlMhuFGV2sAAHzeDz/8oM6dO+u9997j9lkAKow5tQAAt/Luu++qdu3a+uMf/+jqoQDwIMypBQC4hc8//1w7duzQO++8o9GjR1f6jgUAfBPTDwAAbiE6Olq5ublKSEjQP//5T4tPSQOAq/Go6Qdff/217rrrLjVq1Egmk0mffPKJxeuGYWjSpElq2LChatSoofj4+FL3b7Rm9uzZio6OVlBQkGJiYrRx40YHHQEAoCxZWVk6f/68PvnkE4IWcKKr9ZU16enpuv766xUYGKiWLVtqwYIFDh/n1XhU1Obn56tTp06aPXu21ddfeeUV/e1vf9OcOXO0YcMG1apVSwkJCbpw4UKZ+1yyZImSk5OVmpqqLVu2qFOnTkpISNCxY8ccdRgAAABu42p9VdL+/fvVr18/9e7dW5mZmRo7dqweeeQRrVy50sEjLZ/HTj8wmUz6z3/+o/79+0u6fJa2UaNGGj9+vJ5++mlJlz83PCIiQgsWLNDgwYOt7icmJkbdu3c332i8uLhYUVFRGjNmjCZOnOiUYwEAAHAHJfvKmgkTJmjZsmUWnwQ5ePBgnTp1SitWrHDCKK3zmgvF9u/fr5ycHIvPWg8JCVFMTIwyMjKsRm1hYaE2b96slJQU8zI/Pz/Fx8dbfJJPSQUFBSooKDA/Ly4u1smTJ1W/fn2HfkwqAACwH8MwdObMGTVq1KhSH8pSVRcuXFBhYaFD9m0YRqkmCQwMVGBgYJX3nZGRYdFbkpSQkKCxY8dWed9V4TVRm5OTI0mlPps7IiLC/FpJJ06cUFFRkdVtrnzGujXTpk3TlClTqjhiAADgDg4ePGj+BEZnuXDhgpo0raXjx4odsv/atWvr7NmzFsvs9YmIOTk5VtspLy9P58+fV40aNar8NSrDa6LWmVJSUpScnGx+fvr0aTVp0kQ9Q4eominAhSMDAMA+jOiGdt1f/jWVu0VbfkP/Cq97rkHF1iuMuChJKj5/QUf+PM0lFyYWFhbq+LFirdsYrtq17fuvvGfPGorrcVwHDx5UcHCwebk9ztK6M6+J2sjISElSbm6uGjb87QcxNzdXnTt3trpNWFiY/P39lZuba7E8NzfXvD9ryjp9X80UoGp+RC0AwLMZzRrbdX/5TWpVKjjONvJXRZP2XKQqtG5B5EX5lVjTlVMHa9c2qU4de099uHz2Nzg42CJq7SUyMtJqOwUHB7vsLK3kYXc/KE+zZs0UGRmptLQ087K8vDxt2LBBsbGxVrcJCAhQ165dLbYpLi5WWlpamdsAAODNHBG0lXG2kQ1naMs+D2WhIPJipcYCS7GxsRbtJEmrVq1yeTt5VNSePXtWmZmZyszMlHT54rDMzExlZ2fLZDJp7Nix+utf/6rPPvtMW7duVWJioho1amRxBV+fPn3MdzqQpOTkZL377rtauHChdu7cqccff1z5+flKSkpy8tEBAOBaBK1vKq+vpMvTLhMTE83rP/bYY9q3b5/+/Oc/a9euXfr73/+uDz/8UOPGjXPF8M08avrBd999p969e5ufX5nXOnToUC1YsEB//vOflZ+fr5EjR+rUqVO66aabtGLFCgUFBZm32bt3r06cOGF+PmjQIB0/flyTJk1STk6OOnfurBUrVpSaAA0AgDdzh6B1RMxKBO3VXK2vjh49ag5c6fK/ji9btkzjxo3TG2+8oWuuuUb/+Mc/lJCQ4PSx/57H3qfWneTl5SkkJER96g5lTi0AwON4a9BeLWaLz1/QoTGpOn36tEPmnpbnSjv8uKOB3efUnjlTrI5tj7nkuFzJo6YfAAAA+/LVoIX3IWoBAPBRBC28iUfNqQUAAPbh60FbN/KMis4V6FCFRwB3x5laAAB8DEF7psJfG56DqAUAwIcQtAStt2L6AQAAPsDeMSsRtHAvRC0AAF7OHc7OSgQtHIvpBwAAeDGClqD1FUQtAABeiqAlaH0JUQsAgBciaAlaX0PUAgDgZQhagtYXcaEYAABexNOCtqIxKxG0KB9RCwCAl3CHoHX1x97aErTt6x/VtgqvDXdH1AIA4AV8PWhtidlO4UckSYVnK7wJPABzagEA8HAEre1BC+9D1AIA4MEIWoIWlxG1AAB4KIKWoMVviFoAADwQQUvQwhIXigEA4EHsHbMSQQvvwJlaAAA8BEFL0KJsnKkFAMADuMN0A4mghfviTC0AAG6OoCVocXVELQAAboygJWhRMUQtAABuiqAlaFFxRC0AAG6IoHV80MaF7rF5G7gvLhQDAMDNuEPQOiJmJfcJ2ptDf9L5szZvBjdG1AIA4Ea8NWjdJWaly0EL78P0AwAA3ARBS9Ci8ohaAADcAEFL0KJqiFoAAFyMoCVoUXVELQAALkTQErSwDy4UAwDARQhax9/hAL6DqAUAwMnsHbMSQVsSQet7mH4AAIATEbQELRyDM7UAADiJO0w3kAhaeCfO1AIA4AQELUELxyJqAQBwMIKWoIXjEbUAADgQQUvQwjmIWgAAHISgJWjhPFwoBgCAA7hD0LoyZiWCtiLSzzdXDX/75tj585ckHbPrPj0BUQsAgJ35etDyKWFwBaYfAABgRwQtQQvXIGoBALATgpaghesQtQAA2AFB63lB26vGPrvuD65F1AIAUEUErecFbXzNPXbdH1yPC8UAAKgCgtbz7nBA0HonohYAgEqwd8xKBG1JBC1s4VXTD6Kjo2UymUo9Ro0aZXX9BQsWlFo3KCjIyaMGAHgaR5ydJWgtOWK6AUHr3bzqTO2mTZtUVFRkfr5t2zbdeuutGjhwYJnbBAcHa/fu3ebnJpPJoWMEAHg2d5huIBG0tiBmfYNXRW14eLjF85deekktWrRQz549y9zGZDIpMrKCP+3/X0FBgQoKCszP8/LybBsoAMAjEbQELdyXV00/+L3CwkK9//77evjhh8s9+3r27Fk1bdpUUVFRuueee7R9+/ar7nvatGkKCQkxP6Kiouw5dACAGyJoCVq4N6+N2k8++USnTp3SsGHDylynVatWmjdvnj799FO9//77Ki4uVlxcnA4dOlTuvlNSUnT69Gnz4+DBg3YePQDAnRC0BC3cn1dNP/i9uXPnqm/fvmrUqFGZ68TGxio2Ntb8PC4uTm3atNHbb7+t559/vsztAgMDFRgYaNfxAgDckzsErSNiViJo4V28MmoPHDig1atX6+OPP7Zpu+rVq6tLly7as4cfBgCA9wZtRWNWImjhObwyaufPn68GDRqoX79+Nm1XVFSkrVu36o477nDQyAAAnsLXg5ZPCYOn8bo5tcXFxZo/f76GDh2qatUsmz0xMVEpKSnm51OnTtV///tf7du3T1u2bNGDDz6oAwcO6JFHHnH2sAEAboSgJWjhebzuTO3q1auVnZ2thx9+uNRr2dnZ8vP7reN//fVXjRgxQjk5Oapbt666du2qdevWqW3bts4cMgDAjRC0BC08k9dF7W233SbDMKy+lp6ebvH89ddf1+uvv+6EUQEAPAFBS9DCc3nd9AMAACqDoCVo4dm87kwtAAC2sHfMSgRtSfaOWYmgRWlELQDAZxG0nhe0xCzKQtQCAHySO0w3kAhaWxC0KA9zagEAPoegJWjhfYhaAIBPIWgJWngnohYA4DMIWoIW3ouoBQD4BIKWoIV340IxAIDXc4egdUTMSgQtcAVRCwDwat4atO4SsxJBC/fA9AMAgNciaAla+A6iFgDglQhagha+hagFAHgdgpaghe8hagEAXoWgJWjhm7hQDADgNQhaz7rDgUTQwn6IWgCAx7N3zEoEbUkELdwdUQsA8GjucHZWImhtQczCEZhTCwDwWAQtQQtcQdQCADwSQUvQAr9H1AIAPA5BS9ACJRG1AACPQtAStIA1XCgGAPAY7hC0johZiaAFqoqoBQB4BG8N2orGrETQAuVh+gEAwO35etDWjTxT4aDtFH6EoIXNZs+erejoaAUFBSkmJkYbN24sd/2ZM2eqVatWqlGjhqKiojRu3DhduHDBSaO1jqgFALg1gpaPvYVjLVmyRMnJyUpNTdWWLVvUqVMnJSQk6NixY1bXX7x4sSZOnKjU1FTt3LlTc+fO1ZIlS/TMM884eeSWiFoAgNsiaAlaON6MGTM0YsQIJSUlqW3btpozZ45q1qypefPmWV1/3bp1uvHGG3X//fcrOjpat912m4YMGXLVs7uORtQCANwSQUvQomry8vIsHgUFBaXWKSws1ObNmxUfH29e5ufnp/j4eGVkZFjdb1xcnDZv3myO2H379mn58uW64447HHMgFcSFYgAAt0PQErS+Yt2plgq4FGDXfRaeLZS0XlFRURbLU1NTNXnyZItlJ06cUFFRkSIiIiyWR0REaNeuXVb3f//99+vEiRO66aabZBiGLl26pMcee8zl0w+IWgCA27B3zEoEbUn2jlmJoHVXBw8eVHBwsPl5YGCgXfabnp6uF198UX//+98VExOjPXv26KmnntLzzz+v5557zi5fozKIWgCAWyBoPS9oiVn3FhwcbBG11oSFhcnf31+5ubkWy3NzcxUZaf0P+HPPPaeHHnpIjzzyiCSpQ4cOys/P18iRI/WXv/xFfn6umd3KnFoAgMs5YroBQWuJoIU1AQEB6tq1q9LS0szLiouLlZaWptjYWKvbnDt3rlS4+vtf/tkxDMNxg70KztQCAFzKHebPSgStLQha75KcnKyhQ4eqW7du6tGjh2bOnKn8/HwlJSVJkhITE9W4cWNNmzZNknTXXXdpxowZ6tKli3n6wXPPPae77rrLHLeuQNQCAFyGoCVo4XqDBg3S8ePHNWnSJOXk5Khz585asWKF+eKx7OxsizOzzz77rEwmk5599lkdPnxY4eHhuuuuu/TCCy+46hAkSSbDleeJvUReXp5CQkLUp+5QVfOz7xWMAOCtCFqC1tXOnClWx7bHdPr06avOPbW3K+0wOO1BBdS2/90PPujzvkuOy5WYUwsAcDqClqAF7I3pBwAAp3KHoHVEzEoELeBKRC0AwGm8NWjdJWYlgha+i+kHAACnIGgJWsCRiFoAgMMRtAQt4GhELQDAoQhaghZwBqIWAOAwBC1BCzgLF4oBAByCoPWsOxxIBC08G1ELALAre8esRNCWRNACpRG1AAC7IWg9L2iJWXgLohYAYBfuMN1AImhtQdDCm3ChGACgyghaghZwNa+K2smTJ8tkMlk8WrduXe42S5cuVevWrRUUFKQOHTpo+fLlThotAHgHgpagBdyBV0WtJLVr105Hjx41P7755psy1123bp2GDBmi4cOH6/vvv1f//v3Vv39/bdu2zYkjBgDPRdAStIC78LqorVatmiIjI82PsLCwMtd94403dPvtt+tPf/qT2rRpo+eff17XX3+93nzzTSeOGAA8k6cF7blIgpaghTfzuqj9+eef1ahRIzVv3lwPPPCAsrOzy1w3IyND8fHxFssSEhKUkZFR7tcoKChQXl6exQMAfIk7BO3ZRv42BW1FFEReJGgBD+VVURsTE6MFCxZoxYoVeuutt7R//37dfPPNOnPG+i+enJwcRUREWCyLiIhQTk5OuV9n2rRpCgkJMT+ioqLsdgwA4O7cJWgryhFnZysatJ3CjxC0gJN4VdT27dtXAwcOVMeOHZWQkKDly5fr1KlT+vDDD+36dVJSUnT69Gnz4+DBg3bdPwC4K4KWj70F3JVX36c2NDRU1113nfbssf4DHRkZqdzcXItlubm5iows/7dgYGCgAgMD7TZOAPAEBC1BC7gzrzpTW9LZs2e1d+9eNWzY0OrrsbGxSktLs1i2atUqxcbGOmN4AOAxCFqCFnB3XhW1Tz/9tNauXausrCytW7dO9957r/z9/TVkyBBJUmJiolJSUszrP/XUU1qxYoVee+017dq1S5MnT9Z3332n0aNHu+oQAMDtELQELeAJvGr6waFDhzRkyBD98ssvCg8P10033aT169crPDxckpSdnS0/v986Pi4uTosXL9azzz6rZ555Rtdee60++eQTtW/f3lWHAABuhaD1rDscSAQtfJdXRe0HH3xQ7uvp6emllg0cOFADBw500IgAwDPZO2YlgrYkzs4C9uVVUQsAqDp3ODsrEbS2IGgBL5tTCwCoGoKWoAU8FVELAJBE0EoELeDJiFoAAEErghbwdEQtAPg4gpagBbwBF4oBgA9zh6B1RMxKBC3ga4haAPBR3hq0FY1ZiaAFvAlRCwA+yNeDlk8JA7wPc2oBwMcQtAQt4I2IWgDwIQQtQQt4K6IWAHwEQUvQAt6MObUA4AMIWs+6IEwiaAFbEbUA4MXsHbMSQVsSQQu4B6IWALwUQet5QUvMApVH1AKAF3KH6QYSQWsLghaoGi4UAwAvQ9AStIAv4kwtAHgRgpaghWfZ9ktD+Z8PtOs+i84V2HV/noIztQDgJQhaghbwZUQtAHgBgpagBXwd0w8AwMO5Q9C6MmYlghYAUQsAHs3Xg5ZPCQNwBdMPAMBDEbQELYDfELUA4IEIWoIWgCWiFgA8DEFL0AIojagFAA9C0BK0AKzjQjEA8BAErWfd4UAiaAFnImoBwM3ZO2YlgrYkghbwfEQtALgxdzg7KxG0tiBmAddgTi0AuCmClqAFUHFELQC4IYKWoAVgG6IWANwMQUvQArAdUQsAboSgJWgBVA4XigGAm3CHoHVEzEoELQDHI2oBwA14a9BWNGYlghZA1RC1AOBivh60fEoYAHtgTi0AuBBBS9ACsA+iFgBchKAlaAHYD1ELAC5A0BK0AOyLqAUAJyNoCVoA9seFYgDgJPaOWYmgLcneMSsRtICnIGoBwAkIWs8LWmIW8CxELQA4mDtMN5AIWlsQtIDnYU4tADgQQUvQAnAOohYAHISgJWgBOA9RCwAOQNAStACci6gFADsjaAlaAM7HhWIAYEfuELSOiFmJoAXg3rzqTO20adPUvXt31alTRw0aNFD//v21e/fucrdZsGCBTCaTxSMoKMhJIwbgTbw1aAsiL1YoaOtGnqlw0HYKP0LQArArr4ratWvXatSoUVq/fr1WrVqlixcv6rbbblN+fn652wUHB+vo0aPmx4EDB5w0YgDewpuDtiL4lDAAruZV0w9WrFhh8XzBggVq0KCBNm/erFtuuaXM7UwmkyIjbfg3OAD4HYKWoAXgel51prak06dPS5Lq1atX7npnz55V06ZNFRUVpXvuuUfbt28vd/2CggLl5eVZPAD4JoKWoAXgHrw2aouLizV27FjdeOONat++fZnrtWrVSvPmzdOnn36q999/X8XFxYqLi9OhQ4fK3GbatGkKCQkxP6KiohxxCADcHEFL0AJwH141/eD3Ro0apW3btumbb74pd73Y2FjFxsaan8fFxalNmzZ6++239fzzz1vdJiUlRcnJyebneXl5hC3gYwhaz7rDgUTQAt7OK6N29OjR+uKLL/T111/rmmuusWnb6tWrq0uXLtqzp+xffoGBgQoMDKzqMAF4IHvHrETQlkTQAqgMr5p+YBiGRo8erf/85z/66quv1KxZM5v3UVRUpK1bt6phw4YOGCEAT0bQel7QxtfcQ9ACPsKrztSOGjVKixcv1qeffqo6deooJydHkhQSEqIaNWpIkhITE9W4cWNNmzZNkjR16lTdcMMNatmypU6dOqXp06frwIEDeuSRR1x2HADcjztMN5AIWlsQs4Bv8aqofeuttyRJvXr1slg+f/58DRs2TJKUnZ0tP7/fTlD/+uuvGjFihHJyclS3bl117dpV69atU9u2bZ01bABujqAlaAG4P6+KWsMwrrpOenq6xfPXX39dr7/+uoNGBMDTEbQELQDP4FVzagHAnghaghaA5yBqAcAKTwvac5EELUELVN7s2bMVHR2toKAgxcTEaOPGjeWuf+rUKY0aNUoNGzZUYGCgrrvuOi1fvtxJo7XOq6YfAIA9uEPQuvLsrETQAr5kyZIlSk5O1pw5cxQTE6OZM2cqISFBu3fvVoMGDUqtX1hYqFtvvVUNGjTQRx99pMaNG+vAgQMKDQ11/uB/h6gFgN/x9aDlU8IA3zNjxgyNGDFCSUlJkqQ5c+Zo2bJlmjdvniZOnFhq/Xnz5unkyZNat26dqlevLkmKjo525pCtYvoBAPx/BC1BC3iTvLw8i0dBQUGpdQoLC7V582bFx8ebl/n5+Sk+Pl4ZGRlW9/vZZ58pNjZWo0aNUkREhNq3b68XX3xRRUVFDjuWiuBMLQCIoCVoAdc4lVtbfjWC7LrP4vOXz55GRUVZLE9NTdXkyZMtlp04cUJFRUWKiIiwWB4REaFdu3ZZ3f++ffv01Vdf6YEHHtDy5cu1Z88ePfHEE7p48aJSU1PtdyA2ImoB+DyClqAFvNHBgwcVHBxsfh4YGGiX/RYXF6tBgwZ655135O/vr65du+rw4cOaPn06UQsArkLQErSAtwoODraIWmvCwsLk7++v3Nxci+W5ubmKjLT+C6dhw4aqXr26/P1/+93Vpk0b5eTkqLCwUAEBAVUffCUwpxaATzKaNSZonXCHA4IWcG8BAQHq2rWr0tLSzMuKi4uVlpam2NhYq9vceOON2rNnj4qLi83LfvrpJzVs2NBlQSsRtQB8kL1jViJoS3JEzBK0gGMkJyfr3Xff1cKFC7Vz5049/vjjys/PN98NITExUSkpKeb1H3/8cZ08eVJPPfWUfvrpJy1btkwvvviiRo0a5apDkMT0AwA+xh3OzkoErS2IWcCxBg0apOPHj2vSpEnKyclR586dtWLFCvPFY9nZ2fLz++08aFRUlFauXKlx48apY8eOaty4sZ566ilNmDDBVYcgiagF4EMIWoIWgHWjR4/W6NGjrb6Wnp5eallsbKzWr1/v4FHZhukHAHwCQUvQAvBuRC0Ar0fQErQAvB9RC8CrEbQELQDfwJxaAF7LHYLWETErEbQAUBJRC8AreWvQVjRmpYoHLR+qAMAbELUAvI6vBy2fEgbAFzGnFoBXIWgJWgC+iagF4DUIWoIWgO8iagF4BYKWoAXg25hTC8DjEbSedYcDiaAFYH9ELQCPZe+YlQjakghaAJ6CqAXgkQhazwtaYhaAIxG1ADyOO0w3kAhaWxC0AByNC8UAeBSClqAFAGuIWgAeg6AlaAGgLEQtAI9A0BK0AFAeohaA2yNoCVoAuBouFAPg1twhaF0ZsxJBCwAVQdQCcFu+HrR8ShgAVBzTDwC4JYKWoAUAWxC1ANwOQUvQAoCtiFoAboWgJWgBoDKIWgBug6AlaAGgsrhQDIBbIGg96w4HEkELwL0QtQBcyt4xKxG0JRG0AHwBUQvAZdzh7KxE0NqCmAVgT9nZ2Tpw4IDOnTun8PBwtWvXToGBgZXaF1ELwCUIWoIWgG/KysrSW2+9pQ8++ECHDh2SYRjm1wICAnTzzTdr5MiRGjBggPz8Kn75FxeKAXA6gpagBeCbnnzySXXq1En79+/XX//6V+3YsUOnT59WYWGhcnJytHz5ct10002aNGmSOnbsqE2bNlV435ypBeBUBC1BC8B31apVS/v27VP9+vVLvdagQQP94Q9/0B/+8AelpqZqxYoVOnjwoLp3716hfRO1AJyGoCVoAfi2adOmVXjd22+/3aZ9E7UAnMIdgtYRMSsRtADgDphTC8DhvDVoCyIvErQAYGc7d+5U8+bNbd6OM7UAHMqbg7Yi+JQwALBNYWGhDhw4YPN2RC0AhyFoCVoAKCk5Obnc148fP16p/doctcuXL9fHH3+sevXq6eGHH1br1q3Nr/36668aMGCAvvrqq0oNBoD3IGgJWgCw5o033lDnzp0VHBxs9fWzZ89War82zaldvHix7r77buXk5CgjI0NdunTRokWLzK8XFhZq7dq1lRqIPc2ePVvR0dEKCgpSTEyMNm7cWO76S5cuVevWrRUUFKQOHTpo+fLlThop4J0IWoIWAMrSsmVLjRs3TmvWrLH6ePfddyu1X5uidvr06ZoxY4a++OIL/e9//9PChQv16KOPau7cuZX64o6wZMkSJScnKzU1VVu2bFGnTp2UkJCgY8eOWV1/3bp1GjJkiIYPH67vv/9e/fv3V//+/bVt2zYnjxzwDgQtQQsA5enWrZs2b95c5usmk8niU8YqymTYsFXt2rW1detWNWvWzLxszZo1uvvuuzV9+nTde++9atSokYqKimweiL3ExMSoe/fuevPNNyVJxcXFioqK0pgxYzRx4sRS6w8aNEj5+fn64osvzMtuuOEGde7cWXPmzKnQ18zLy1NISIj61B2qan4B9jkQwMPYO2YlgrYke8esRNDCt505U6yObY/p9OnTZf5TuKNcaYdrZk2RX40gu+67+PwFHRqT6pLjqoicnBwVFBSoadOmdt2vTXNqg4ODlZubaxG1vXv31hdffKE777xThw4dsuvgbFVYWKjNmzcrJSXFvMzPz0/x8fHKyMiwuk1GRkapCcsJCQn65JNPyvw6BQUFKigoMD/Py8ur2sABD0fQel7QErMAXCUy0oYbgdvApukHPXr00Jdffllqec+ePfX5559r5syZ9hpXpZw4cUJFRUWKiIiwWB4REaGcnByr2+Tk5Ni0vnT50zBCQkLMj6ioqKoPHvBQjphuQNBaImgB7xWQW12BOfZ9BORWd/VhVcjDDz+shQsXllqel5enhx9+2Ob92RS148aNU1CQ9VPkvXr10ueff66hQ4faPAhPk5KSotOnT5sfBw8edPWQAJdwh/mzEkFrC4IWgLtYsGCBnnjiCT355JMqLi42Lz9//rzV2L0am6K2Z8+eSklJ0R/+8AdNmTKl1OudO3dWVlaWzYOwl7CwMPn7+ys3N9dieW5ubpmnuiMjI21aX5ICAwMVHBxs8QB8DUFL0AJAVS1btkzLly9XQkKCfv311yrtq1Ifk5uenq4333xT/fv3V35+vnm5q2/pFRAQoK5duyotLc28rLi4WGlpaYqNjbW6TWxsrMX6krRq1aoy1wdA0EoELQDYQ9u2bbVhwwZdvHhRPXr00M6dOyu9r0pFrSStXr1aOTk5uuGGG1x6drak5ORkvfvuu1q4cKF27typxx9/XPn5+UpKSpIkJSYmWlxI9tRTT2nFihV67bXXtGvXLk2ePFnfffedRo8e7apDANwaQUvQAoA9mEwmSVL9+vW1evVq9ezZU7Gxsfrss88qtb9Kf0xuw4YNtXbtWiUlJal79+5aunSp2rRpU9nd2c2gQYN0/PhxTZo0STk5OercubNWrFhhvhgsOztbfn6/tXxcXJwWL16sZ599Vs8884yuvfZaffLJJ2rfvr2rDgFwW+4QtI6IWYmgBQBn+/1dZatVq6Z//OMfatu2rZ544olK7a9SUXulrAMDA7V48WL99a9/1e23364JEyZUahD2Nnr06DLPtKanp5daNnDgQA0cONDBowI8m7cGrbvErETQAvAta9asUb169SyWJScnq2PHjvr2229t3l+lorbk5zU8++yzatOmjU/c+QDwRQQtQQsA9tazZ0+ry+Pj4xUfH2/z/io1p3b//v0KDw+3WDZgwABt2LBB8+bNq8wuAbgpgpagBQB7eemll3T+/PkKrbthwwYtW7aswvuu1Jnasj7WrF27dmrXrl1ldgnADRG0BC0A2NOOHTvUpEkTDRw4UHfddZe6detmPlF66dIl7dixQ998843ef/99HTlyRO+9916F913pC8UAeDeClqAFAHt777339MMPP+jNN9/U/fffr7y8PPn7+yswMFDnzp2TJHXp0kWPPPKIhg0bVuaHfllD1AIohaD1rDscSAQtAM/RqVMnvfvuu3r77bf1448/6sCBAzp//rzCwsLUuXNnhYWFVWq/RC0AM3vHrETQlkTQAvB1xcXFmj59uj777DMVFhaqT58+Sk1NVY0aNaq030p/+AIA70LQel7QxtfcQ9AC8DgvvPCCnnnmGdWuXVuNGzfWG2+8oVGjRlV5v0QtAIdMNyBoLTF/FgAue++99/T3v/9dK1eu1CeffKLPP/9cixYtUnFxcZX2S9QCPs4d5s9KBK0tCFoAniw7O1t33HGH+Xl8fLxMJpOOHKncRbdXELWADyNoCVoAcLZLly6VuqtB9erVdfFixX5vl4ULxQAfRdAStADgCoZhaNiwYQoMDDQvu3Dhgh577DHVqvXb3yUff/yxTfslagEf5A5B64iYlQhaAHB3Q4cOLbXswQcfrPJ+iVrAx3hr0FY0ZiWCFgBcaf78+Q7ZL1EL+BBfD1o+JQwAvBcXigE+gqAlaAHAmxG1gA8gaAlaAPB2RC3g5QhaghYAfAFRC3gxgpagBQBfwYVigBeyd8xKBG1J9o5ZiaAFgKogagEvQ9B6XtASswBQdUQt4EXcYbqBRNDagqAFAPtgTi3gJQhaghYAfBlRC3gBgpagBQBfR9QCHo6gJWgBAEQt4NEIWoIWAHAZF4oBHsodgtYRMSsRtAAA2xG1gAfy1qB1l5iVCFoA8DRMPwA8DEFL0AIASiNqAQ9C0BK0AADriFrAQxC0BC0AoGxELeABCFqCFgBQPi4UA9wcQetZdziQCFoAcAWiFnBT9o5ZiaAtiaAFAO9B1AJuiKD1vKAlZgHAtYhawM24w3QDiaC1BUELAK7HhWKAGyFoCVoAcIXZs2crOjpaQUFBiomJ0caNGyu03QcffCCTyaT+/fs7doAVQNQCboKgJWgBwBWWLFmi5ORkpaamasuWLerUqZMSEhJ07NixcrfLysrS008/rZtvvtlJIy0fUQu4AYKWoAUAV5kxY4ZGjBihpKQktW3bVnPmzFHNmjU1b968MrcpKirSAw88oClTpqh58+ZOHG3ZiFrAxQhaghYAHCEvL8/iUVBQUGqdwsJCbd68WfHx8eZlfn5+io+PV0ZGRpn7njp1qho0aKDhw4c7ZOyVwYVigAu5Q9C6MmYlghaAb6t5TPIPsO8+iwov/29UVJTF8tTUVE2ePNli2YkTJ1RUVKSIiAiL5REREdq1a5fV/X/zzTeaO3euMjMz7TVkuyBqARfx9aDlU8IAwLEOHjyo4OBg8/PAwMAq7/PMmTN66KGH9O677yosLKzK+7MnohZwAYKWoAUARwsODraIWmvCwsLk7++v3Nxci+W5ubmKjCz9y3/v3r3KysrSXXfdZV5WXFwsSapWrZp2796tFi1a2GH0tmNOLeBkBC1BCwDuIiAgQF27dlVaWpp5WXFxsdLS0hQbG1tq/datW2vr1q3KzMw0P+6++2717t1bmZmZpaY8OBNnagEnImgJWgBwN8nJyRo6dKi6deumHj16aObMmcrPz1dSUpIkKTExUY0bN9a0adMUFBSk9u3bW2wfGhoqSaWWOxtRCzgJQUvQAoA7GjRokI4fP65JkyYpJydHnTt31ooVK8wXj2VnZ8vPz/3/cZ+oBZyAoPWsOxxIBC0A3zJ69GiNHj3a6mvp6enlbrtgwQL7D6gSiFrAgewdsxJBWxJBCwCQvOhCsaysLA0fPlzNmjVTjRo11KJFC6WmpqqwsLDc7Xr16iWTyWTxeOyxx5w0angzR5ydJWgtOWK6AUELAJ7Ja87U7tq1S8XFxXr77bfVsmVLbdu2TSNGjFB+fr5effXVcrcdMWKEpk6dan5es2ZNRw8XXs4dphtIBK0tiFkA8GxeE7W33367br/9dvPz5s2ba/fu3XrrrbeuGrU1a9a0ei+2shQUFFh81FxeXp7tA4bXImgJWgCA83nN9ANrTp8+rXr16l11vUWLFiksLEzt27dXSkqKzp07V+7606ZNU0hIiPnhynuywb0QtAQtAMA1vOZMbUl79uzRrFmzrnqW9v7771fTpk3VqFEj/fjjj5owYYJ2796tjz/+uMxtUlJSlJycbH6el5dH2IKgFUELAHAdt4/aiRMn6uWXXy53nZ07d6p169bm54cPH9btt9+ugQMHasSIEeVuO3LkSPP/79Chgxo2bKg+ffpo7969ZX7MW2BgoF0+Pxnewx2C1hExKxG0AADP4PZRO378eA0bNqzcdZo3b27+/0eOHFHv3r0VFxend955x+avFxMTI+nymV5XfXYxPIu3Bm1FY1YiaAEAruf2URseHq7w8PAKrXv48GH17t1bXbt21fz58yv16ReZmZmSpIYNG9q8LXyPrwctnxIGAHAXXnOh2OHDh9WrVy81adJEr776qo4fP66cnBzl5ORYrNO6dWtt3LhRkrR37149//zz2rx5s7KysvTZZ58pMTFRt9xyizp27OiqQ4GHIGgJWgCA+3D7M7UVtWrVKu3Zs0d79uzRNddcY/GaYRiSpIsXL2r37t3muxsEBARo9erVmjlzpvLz8xUVFaUBAwbo2Wefdfr44VkIWoIWAOBevCZqhw0bdtW5t9HR0ebAlaSoqCitXbvWwSODtyFoCVoAgPvxmukHgDMQtAQtAMA9ec2ZWsCR7B2zEkFbkr1jViJoAcCXELXAVRC0nhe0xCwA+B6iFiiHO0w3kAhaWxC0AOCbmFMLlIGgJWgBAJ6DqAWsIGgJWgCAZyFqgRIIWoIWAOB5iFrgdwhaghYA4Jm4UAz4/9whaF0ZsxJBCwDwXEQtIIKWD1UAAHg6ph/A5xG0BC0AwPMRtfBpBC1BCwDwDkQtfBZBS9ACALwHUQufRNAStAAA78KFYvA5BK1n3eFAImgBAFdH1MJn2DtmJYK2JIIWAOAqRC18gjucnZUIWlsQswAAWzCnFl6PoCVoAQDej6iFVyNoCVoAgG8gauG1CFqCFgDgO4haeCWClqAFAPgWLhSD13GHoHVEzEoELQAAZSFq4VW8NWgrGrMSQQsA8E1ELbyGrwctnxIGAPBlzKmFVyBoCVoAgG8jauHxCFqCFgAAohYejaAlaAEAkJhTCw9G0BK0AODpah0tUrXqRXbd56WL9t2fpyBq4XHsHbMSQVuSvWNWImgBAI5F1MKjELSeF7TELADAGYhaeAx3mG4gEbS2IGgBAM7ChWLwCAQtQQsAQHmIWrg9gpagBQDgaohauDWClqAFAKAiiFq4LYKWoAUAoKK4UAxuyR2C1hExKxG0AAA4AlELt+OtQesuMSsRtAAA78P0A7gVgpagBQCgMohauA2ClqAFAKCyiFq4BYKWoAUAoCqIWrgcQUvQAgBQVVwoBpciaD3rDgcSQQsAcE9ELVzC3jErEbQlEbQAAF9C1MLpCFrPC1piFgDg7ohaOJU7TDeQCFpbELQAAE/AhWJwGoKWoAUAwFG8Kmqjo6NlMpksHi+99FK521y4cEGjRo1S/fr1Vbt2bQ0YMEC5ublOGrHvIGgJWgAAHMmrolaSpk6dqqNHj5ofY8aMKXf9cePG6fPPP9fSpUu1du1aHTlyRH/84x+dNFrfQNAStAAAOJrXzamtU6eOIiMrViSnT5/W3LlztXjxYv3hD3+QJM2fP19t2rTR+vXrdcMNNzhyqD6BoCVoAQBwBq87U/vSSy+pfv366tKli6ZPn65Lly6Vue7mzZt18eJFxcfHm5e1bt1aTZo0UUZGRpnbFRQUKC8vz+KB0twhaM828q9w0J6LrFjQFkReJGgBAHAzXnWm9sknn9T111+vevXqad26dUpJSdHRo0c1Y8YMq+vn5OQoICBAoaGhFssjIiKUk5NT5teZNm2apkyZYs+hex13CdqK8rSzsxJBCwDA77n9mdqJEyeWuvir5GPXrl2SpOTkZPXq1UsdO3bUY489ptdee02zZs1SQUGBXceUkpKi06dPmx8HDx606/49HUFL0AIA4Gxuf6Z2/PjxGjZsWLnrNG/e3OrymJgYXbp0SVlZWWrVqlWp1yMjI1VYWKhTp05ZnK3Nzc0td15uYGCgAgMDKzR+X0PQErQAALiC20dteHi4wsPDK7VtZmam/Pz81KBBA6uvd+3aVdWrV1daWpoGDBggSdq9e7eys7MVGxtb6TH7KoKWoAUAwFXcPmorKiMjQxs2bFDv3r1Vp04dZWRkaNy4cXrwwQdVt25dSdLhw4fVp08fvffee+rRo4dCQkI0fPhwJScnq169egoODtaYMWMUGxvLnQ9sRNAStAAAuJLXRG1gYKA++OADTZ48WQUFBWrWrJnGjRun5ORk8zoXL17U7t27de7cOfOy119/XX5+fhowYIAKCgqUkJCgv//97644BI9F0HrWHQ4kghYA4H28Jmqvv/56rV+/vtx1oqOjZRiGxbKgoCDNnj1bs2fPduTwvJK9Y1YiaEsiaAEAqBiviVo4lzucnZUIWlsQswAAb+b2t/SC+yFoCVoAANwNUQubELQELQAA7oioRYURtAQtAADuiqhFhRC0BC0AAO6MC8VwVe4QtI6IWYmgBQDAWxC1KJe3Bm1FY1aqeNDyoQoAALgOUYsy+XrQ8ilhAAB4DubUwiqClqAFAMCTELUohaAlaAEA8DRELSwQtAQtAACeiDm1MCNoPesOBxJBCwDAFZyphYxmjQlaghYA4MNmz56t6OhoBQUFKSYmRhs3bixz3XfffVc333yz6tatq7p16yo+Pr7c9Z2FqPVx9o5ZiaAtyRHTDQhaAIC9LFmyRMnJyUpNTdWWLVvUqVMnJSQk6NixY1bXT09P15AhQ7RmzRplZGQoKipKt912mw4fPuzkkVsian2YI87OErSWmD8LAHB3M2bM0IgRI5SUlKS2bdtqzpw5qlmzpubNm2d1/UWLFumJJ55Q586d1bp1a/3jH/9QcXGx0tLSnDxyS0Stj3KH6QYSQWsLghYAYIu8vDyLR0FBQal1CgsLtXnzZsXHx5uX+fn5KT4+XhkZGRX6OufOndPFixdVr149u429MrhQzAcRtAQtAMA91DqUr2r+RXbd56WiC5KkqKgoi+WpqamaPHmyxbITJ06oqKhIERERFssjIiK0a9euCn29CRMmqFGjRhZh7ApErY8haAlaAIBvOHjwoIKDg83PAwMD7f41XnrpJX3wwQdKT09XUFCQ3fdvC6LWhxC0BC0AwHcEBwdbRK01YWFh8vf3V25ursXy3NxcRUaW/5fwq6++qpdeekmrV69Wx44dqzzeqmJOrY9wh6A928i/wkF7LrJiQVsQeZGgBQCgkgICAtS1a1eLi7yuXPQVGxtb5navvPKKnn/+ea1YsULdunVzxlCvijO1PsBdgraiPO3srETQAgA8V3JysoYOHapu3bqpR48emjlzpvLz85WUlCRJSkxMVOPGjTVt2jRJ0ssvv6xJkyZp8eLFio6OVk5OjiSpdu3aql27tsuOg6j1cgQtQQsAQHkGDRqk48ePa9KkScrJyVHnzp21YsUK88Vj2dnZ8vP77R/333rrLRUWFuq+++6z2I+1C9Gciaj1YgQtQQsAQEWMHj1ao0ePtvpaenq6xfOsrCzHD6gSmFPrpQhaghYAAF9C1HohgpagBQDA1zD9wMsQtJ51hwOJoAUAwB6IWi9h75iVCNqSCFoAANwXUesF3OHsrETQ2oKYBQDAvphT6+EIWoIWAAAQtR6NoCVoAQDAZUSthyJoCVoAAPAbotYDEbQELQAAsMSFYh7GHYLWETErEbQAAKDyiFoP4q1BW9GYlQhaAABgHVHrIXw9aPmUMAAAUB7m1HoAgpagBQAA5SNq3RxBS9ACAICrI2rdGEFL0AIAgIohat0UQUvQAgCAiuNCMTdj75iVCNqS7B2zEkELAICrEbVuhKD1vKAlZgEAcA9ErZtwh+kGEkFrC4IWAAD3wZxaN0DQErQAAKBqiFoXI2gJWgAAUHVErQsRtAQtAACwD6LWRQhaghYAANgPF4q5gDsErSNiViJoAQCAa3jNmdr09HSZTCarj02bNpW5Xa9evUqt/9hjjzlsnN4atAWRFysUtHUjz1Q4aDuFHyFoAQBAhXjNmdq4uDgdPXrUYtlzzz2ntLQ0devWrdxtR4wYoalTp5qf16xZ0yFj9OagrQg+JQwAADiK10RtQECAIiN/q7CLFy/q008/1ZgxY2QymcrdtmbNmhbbOgJBS9ACAADH8ZrpByV99tln+uWXX5SUlHTVdRctWqSwsDC1b99eKSkpOnfuXLnrFxQUKC8vz+JRHoKWoAUAAI7lNWdqS5o7d64SEhJ0zTXXlLve/fffr6ZNm6pRo0b68ccfNWHCBO3evVsff/xxmdtMmzZNU6ZMqdA4CFqCFgAAOJ7bR+3EiRP18ssvl7vOzp071bp1a/PzQ4cOaeXKlfrwww+vuv+RI0ea/3+HDh3UsGFD9enTR3v37lWLFi2sbpOSkqLk5GTz87y8PEVFRZVaj6D1rDscSAQtAACeyu2jdvz48Ro2bFi56zRv3tzi+fz581W/fn3dfffdNn+9mJgYSdKePXvKjNrAwEAFBgaWuQ97x6xE0JZE0AIAgN9z+6gNDw9XeHh4hdc3DEPz589XYmKiqlevbvPXy8zMlCQ1bNjQ5m0lglbyvKAlZgEA8Hxed6HYV199pf379+uRRx4p9drhw4fVunVrbdy4UZK0d+9ePf/889q8ebOysrL02WefKTExUbfccos6duxo89c2oisXwmXJb1KLoC2BoAUAANa4/ZlaW82dO1dxcXEWc2yvuHjxonbv3m2+u0FAQIBWr16tmTNnKj8/X1FRURowYICeffZZZw+7FD72tjSCFgAAlMXronbx4sVlvhYdHS3DMMzPo6KitHbtWmcMyyYEbWkELQAAKI/XTT/wdARtaQQtAAC4Gq87U+vJHB20FY1ZiaAFAACehah1E552QZhE0AIAAPdB1LoBTwtaPiUMAAC4G+bUuhhBWxpBCwAAbMWZWhciaEsjaAEAvsSUdVQmvwD77rO40K778xScqXURgrY0ghYAAFQWUesCBG1pBC0AAKgKph84GUFryd4xKxG0AAD4IqLWiQhaS5ydBQAA9kLUOgGfElYaQQsAAOyJObUORtCWRtACAAB7I2odiKAtjaAFAACOQNQ6CEFbGkELAAAchah1AIK2NIIWAAA4EheK2Zm73OFAImgBAIDvIGrtKP+aWjZ/Q115dlaqeNDyoQoAAMCdEbUu5M3TDSSCFgAAOA9zal2EoLUNQQsAAMpD1LoAQWsbghYAAFwNUetkBK1tCFoAAFARzKl1Im8OWnvHrETQAgCAiuNMrZMQtLYhaAEAgC04U+sEBG3FEbMAAKAyOFPrYARtxRG0AACgsohaByJoK46gBQAAVUHUOghBW3EELQAAqCqi1gEI2oojaAEAgD0QtXZG0FYcQQsAAOyFux/YUX5Df1Ukae0dsxJBCwAAfBtR62SednZWImgBAID7Y/qBExG0BC0AAHAMotZJCFqCFgAAOA5R6wQELUELAAAci6h1MIKWoAUAAI7HhWIO5GlBa++YlQhaAADgHJypdRCClqAFAADOw5laB/D1oCVmAQCAs3Gm1s4IWoIWAAA4H1FrR+caVGw9ghYAAMC+iFonI2gBAADsj6h1IoIWAADAMYhaJyFoAQAAHIe7HzhYRWNWImgBAAAqi6h1IHc5OysRtAAAwLsx/cBBCFoAAADnIWodgKAFAABwLo+J2hdeeEFxcXGqWbOmQkNDra6TnZ2tfv36qWbNmmrQoIH+9Kc/6dKlS+Xu9+TJk3rggQcUHBys0NBQDR8+XGfPnq30OAlaAADgaWbPnq3o6GgFBQUpJiZGGzduLHf9pUuXqnXr1goKClKHDh20fPlyJ420bB4TtYWFhRo4cKAef/xxq68XFRWpX79+Kiws1Lp167Rw4UItWLBAkyZNKne/DzzwgLZv365Vq1bpiy++0Ndff62RI0dWbowRBC0AAPAsS5YsUXJyslJTU7VlyxZ16tRJCQkJOnbsmNX1161bpyFDhmj48OH6/vvv1b9/f/Xv31/btm1z8sgtmQzDMFw6AhstWLBAY8eO1alTpyyWf/nll7rzzjt15MgRRURESJLmzJmjCRMm6Pjx4woICCi1r507d6pt27batGmTunXrJklasWKF7rjjDh06dEiNGjWq0Jjy8vIUEhKia2ZNkV+NoHLX9bQ7HEgELQDAO505U6yObY/p9OnTCg4OdurXvtIOfeoOVTW/0o1SFZeKC5X268IKH1dMTIy6d++uN998U5JUXFysqKgojRkzRhMnTiy1/qBBg5Sfn68vvvjCvOyGG25Q586dNWfOHPsdiI285u4HGRkZ6tChgzloJSkhIUGPP/64tm/fri5duljdJjQ01By0khQfHy8/Pz9t2LBB9957r9WvVVBQoIKCAvPz06dPS5KKz18od4yhEWdVdK5ix9O+/lEVVmIWxPlq5U+3sFWvGvt0puIdDgCAxzh7tliS5Mrze5eMQqnYAfvU5XD+vcDAQAUGBlosKyws1ObNm5WSkmJe5ufnp/j4eGVkZFjdf0ZGhpKTky2WJSQk6JNPPrHD6CvPa6I2JyfHImglmZ/n5OSUuU2DBg0sllWrVk316tUrcxtJmjZtmqZMmVJq+ZE/Tyt3jIfKfdVSZU/gf1DJ7QAA8FW//PKLQkJCnPo1AwICFBkZqbU5/3LI/mvXrq2oqCiLZampqZo8ebLFshMnTqioqMhqQ+3atcvqvstqrvLayRlcGrUTJ07Uyy+/XO46O3fuVOvWrZ00oopJSUmx+C+UU6dOqWnTpsrOznb6D4Ur5eXlKSoqSgcPHnT6P9u4EsfNcfsCjpvj9gWnT59WkyZNVK9ePad/7aCgIO3fv1+FhYUO2b9hGDKZTBbLSp6l9TYujdrx48dr2LBh5a7TvHnzCu0rMjKy1JV6ubm55tfK2qbkJOhLly7p5MmTZW4jWT99L0khISE+9cvgiuDgYI7bh3DcvoXj9i2+etx+fq65bj4oKEhBQeVfi+NoYWFh8vf3NzfTFbm5ueX2ky3rO4tL734QHh6u1q1bl/uwdoGXNbGxsdq6datFpK5atUrBwcFq27ZtmducOnVKmzdvNi/76quvVFxcrJiYmKodHAAAgJsLCAhQ165dlZaWZl5WXFystLQ0xcbGWt0mNjbWYn3pcnOVtb6zeMwtvbKzs5WZmans7GwVFRUpMzNTmZmZ5nvK3nbbbWrbtq0eeugh/fDDD1q5cqWeffZZjRo1ynxWdePGjWrdurUOHz4sSWrTpo1uv/12jRgxQhs3btS3336r0aNHa/DgwRW+8wEAAIAnS05O1rvvvquFCxdq586devzxx5Wfn6+kpCRJUmJiosWFZE899ZRWrFih1157Tbt27dLkyZP13XffafTo0a46hMsMDzF06FBDUqnHmjVrzOtkZWUZffv2NWrUqGGEhYUZ48ePNy5evGh+fc2aNYYkY//+/eZlv/zyizFkyBCjdu3aRnBwsJGUlGScOXPGprFduHDBSE1NNS5cuFDVw/QoHDfH7Qs4bo7bF3DcvnXc1syaNcto0qSJERAQYPTo0cNYv369+bWePXsaQ4cOtVj/ww8/NK677jojICDAaNeunbFs2TInj7g0j7tPLQAAAFCSx0w/AAAAAMpC1AIAAMDjEbUAAADweEQtAAAAPB5RWwEvvPCC4uLiVLNmTYWGhlpdJzs7W/369VPNmjXVoEED/elPf9KlS5fK3e/Jkyf1wAMPKDg4WKGhoRo+fLj5FmXuJj09XSaTyepj06ZNZW7Xq1evUus/9thjThx51UVHR5c6hpdeeqncbS5cuKBRo0apfv36ql27tgYMGFDqRtXuLCsrS8OHD1ezZs1Uo0YNtWjRQqmpqVf95BtPfb9nz56t6OhoBQUFKSYmptQHuZS0dOlStW7dWkFBQerQoYOWL1/upJHax7Rp09S9e3fVqVNHDRo0UP/+/bV79+5yt1mwYEGp99bVN4231eTJk0sdw9U+sdLT32vJ+u8wk8mkUaNGWV3fU9/rr7/+WnfddZcaNWokk8mkTz75xOJ1wzA0adIkNWzYUDVq1FB8fLx+/vnnq+7X1t8PcB2itgIKCws1cOBAPf7441ZfLyoqUr9+/VRYWKh169Zp4cKFWrBggSZNmlTufh944AFt375dq1at0hdffKGvv/5aI0eOdMQhVFlcXJyOHj1q8XjkkUfUrFkzdevWrdxtR4wYYbHdK6+84qRR28/UqVMtjmHMmDHlrj9u3Dh9/vnnWrp0qdauXasjR47oj3/8o5NGW3W7du1ScXGx3n77bW3fvl2vv/665syZo2eeeeaq23ra+71kyRIlJycrNTVVW7ZsUadOnZSQkFDq0wavWLdunYYMGaLhw4fr+++/V//+/dW/f39t27bNySOvvLVr12rUqFFav369Vq1apYsXL+q2225Tfn5+udsFBwdbvLcHDhxw0ojtp127dhbH8M0335S5rje815K0adMmi2NetWqVJGngwIFlbuOJ73V+fr46deqk2bNnW339lVde0d/+9jfNmTNHGzZsUK1atZSQkKALFy6UuU9bfz/AxVx8SzGPMn/+fCMkJKTU8uXLlxt+fn5GTk6Oedlbb71lBAcHGwUFBVb3tWPHDkOSsWnTJvOyL7/80jCZTMbhw4ftPnZ7KywsNMLDw42pU6eWu17Pnj2Np556yjmDcpCmTZsar7/+eoXXP3XqlFG9enVj6dKl5mU7d+40JBkZGRkOGKFzvPLKK0azZs3KXccT3+8ePXoYo0aNMj8vKioyGjVqZEybNs3q+v/3f/9n9OvXz2JZTEyM8eijjzp0nI507NgxQ5Kxdu3aMtcp6/efJ0lNTTU6depU4fW98b02DMN46qmnjBYtWhjFxcVWX/eG91qS8Z///Mf8vLi42IiMjDSmT59uXnbq1CkjMDDQ+Ne//lXmfmz9/QDX4kytHWRkZKhDhw6KiIgwL0tISFBeXp62b99e5jahoaEWZznj4+Pl5+enDRs2OHzMVfXZZ5/pl19+MX/aSHkWLVqksLAwtW/fXikpKTp37pwTRmhfL730kurXr68uXbpo+vTp5U4t2bx5sy5evKj4+HjzstatW6tJkybKyMhwxnAd4vTp06pXr95V1/Ok97uwsFCbN2+2eK/8/PwUHx9f5nuVkZFhsb50+efd099bSVd9f8+ePaumTZsqKipK99xzT5m/39zZzz//rEaNGql58+Z64IEHlJ2dXea63vheFxYW6v3339fDDz8sk8lU5nre8F7/3v79+5WTk2PxfoaEhCgmJqbM97Myvx/gWtVcPQBvkJOTYxG0kszPc3JyytymQYMGFsuqVaumevXqlbmNO5k7d64SEhJ0zTXXlLve/fffr6ZNm6pRo0b68ccfNWHCBO3evVsff/yxk0ZadU8++aSuv/561atXT+vWrVNKSoqOHj2qGTNmWF0/JydHAQEBpeZfR0REeMR7a82ePXs0a9Ysvfrqq+Wu52nv94kTJ1RUVGT153fXrl1Wtynr591T39vi4mKNHTtWN954o9q3b1/meq1atdK8efPUsWNHnT59Wq+++qri4uK0ffv2q/4ecBcxMTFasGCBWrVqpaNHj2rKlCm6+eabtW3bNtWpU6fU+t72XkvSJ598olOnTmnYsGFlruMN73VJV94zW97Pyvx+gGv5bNROnDhRL7/8crnr7Ny586oXEXi6ynwfDh06pJUrV+rDDz+86v5/P0e4Q4cOatiwofr06aO9e/eqRYsWlR94Fdly3MnJyeZlHTt2VEBAgB599FFNmzZNgYGBjh6qXVXm/T58+LBuv/12DRw4UCNGjCh3W3d9v1G2UaNGadu2beXOLZWk2NhYxcbGmp/HxcWpTZs2evvtt/X88887eph20bdvX/P/79ixo2JiYtS0aVN9+OGHGj58uAtH5jxz585V37591ahRozLX8Yb3Gr7JZ6N2/Pjx5f6XqiQ1b968QvuKjIwsdTXklSvdIyMjy9ym5ETzS5cu6eTJk2Vu4wiV+T7Mnz9f9evX1913323z14uJiZF0+cyfKyOnKu9/TEyMLl26pKysLLVq1arU65GRkSosLNSpU6csztbm5uY69b21xtbjPnLkiHr37q24uDi98847Nn89d3m/yxIWFiZ/f/9Sd6Yo772KjIy0aX13Nnr0aPNFqraegatevbq6dOmiPXv2OGh0jhcaGqrrrruuzGPwpvdakg4cOKDVq1fb/C8n3vBeX3nPcnNz1bBhQ/Py3Nxcde7c2eo2lfn9ANfy2agNDw9XeHi4XfYVGxurF154QceOHTNPKVi1apWCg4PVtm3bMrc5deqUNm/erK5du0qSvvrqKxUXF5tDwBls/T4YhqH58+crMTFR1atXt/nrZWZmSpLFLxVXqMr7n5mZKT8/v1LTR67o2rWrqlevrrS0NA0YMECStHv3bmVnZ1uc/XAFW4778OHD6t27t7p27ar58+fLz8/2Kfju8n6XJSAgQF27dlVaWpr69+8v6fI/x6elpWn06NFWt4mNjVVaWprGjh1rXrZq1SqXv7e2MAxDY8aM0X/+8x+lp6erWbNmNu+jqKhIW7du1R133OGAETrH2bNntXfvXj300ENWX/eG9/r35s+frwYNGqhfv342becN73WzZs0UGRmptLQ0c8Tm5eVpw4YNZd7ZqDK/H+Birr5SzRMcOHDA+P77740pU6YYtWvXNr7//nvj+++/N86cOWMYhmFcunTJaN++vXHbbbcZmZmZxooVK4zw8HAjJSXFvI8NGzYYrVq1Mg4dOmRedvvttxtdunQxNmzYYHzzzTfGtddeawwZMsTpx2eL1atXG5KMnTt3lnrt0KFDRqtWrYwNGzYYhmEYe/bsMaZOnWp89913xv79+41PP/3UaN68uXHLLbc4e9iVtm7dOuP11183MjMzjb179xrvv/++ER4ebiQmJprXKXnchmEYjz32mNGkSRPjq6++Mr777jsjNjbWiI2NdcUhVMqhQ4eMli1bGn369DEOHTpkHD161Pz4/Tre8H5/8MEHRmBgoLFgwQJjx44dxsiRI43Q0FDz3UweeughY+LEieb1v/32W6NatWrGq6++auzcudNITU01qlevbmzdutVVh2Czxx9/3AgJCTHS09Mt3ttz586Z1yl53FOmTDFWrlxp7N2719i8ebMxePBgIygoyNi+fbsrDqFSxo8fb6Snpxv79+83vv32WyM+Pt4ICwszjh07ZhiGd77XVxQVFRlNmjQxJkyYUOo1b3mvz5w5Y/77WZIxY8YM4/vvvzcOHDhgGIZhvPTSS0ZoaKjx6aefGj/++KNxzz33GM2aNTPOnz9v3scf/vAHY9asWebnV/v9APdC1FbA0KFDDUmlHmvWrDGvk5WVZfTt29eoUaOGERYWZowfP964ePGi+fU1a9YYkoz9+/ebl/3yyy/GkCFDjNq1axvBwcFGUlKSOZTd1ZAhQ4y4uDirr+3fv9/i+5KdnW3ccsstRr169YzAwECjZcuWxp/+9Cfj9OnTThxx1WzevNmIiYkxQkJCjKCgIKNNmzbGiy++aFy4cMG8TsnjNgzDOH/+vPHEE08YdevWNWrWrGnce++9FkHo7ubPn2/1z/zv/zvYm97vWbNmGU2aNDECAgKMHj16GOvXrze/1rNnT2Po0KEW63/44YfGddddZwQEBBjt2rUzli1b5uQRV01Z7+38+fPN65Q87rFjx5q/RxEREcYdd9xhbNmyxfmDr4JBgwYZDRs2NAICAozGjRsbgwYNMvbs2WN+3Rvf6ytWrlxpSDJ2795d6jVvea+v/D1b8nHl2IqLi43nnnvOiIiIMAIDA40+ffqU+n40bdrUSE1NtVhW3u8HuBeTYRiGU04JAwAAAA7CfWoBAADg8YhaAAAAeDyiFgAAAB6PqAUAAIDHI2oBAADg8YhaAAAAeDyiFgAAAB6PqAUAAIDHI2oBAADg8YhaACjHhQsXNGzYMHXo0EHVqlVT//79XT0kAIAVRC0AlKOoqEg1atTQk08+qfj4eFcPBwBQBqIWgM/LysqSyWQq9ejVq5dq1aqlt956SyNGjFBkZKSrhwoAKEM1Vw8AAFwtKipKR48eNT/PyclRfHy8brnlFheOCgBgC6IWgM/z9/c3n4W9cOGC+vfvr9jYWE2ePNm1AwMAVBhRCwC/8/DDD+vMmTNatWqV/PyYoQUAnoKoBYD/769//atWrlypjRs3qk6dOq4eDgDABkQtAEj697//ralTp+rLL79UixYtXD0cAICNiFoAPm/btm1KTEzUhAkT1K5dO+Xk5EiSAgICVK9ePe3YsUOFhYU6efKkzpw5o8zMTElS586dXTdoAIAFk2EYhqsHAQCutGDBAiUlJZVa3rNnT6Wnpys6OloHDhwo9Tq/PgHAfRC1AAAA8Hhc2gsAAACPR9QCAADA4xG1AAAA8HhELQAAADweUQsAAACPR9QCAADA4xG1AAAA8HhELQAAADweUQsAAACPR9QCAADA4xG1AAAA8Hj/D01ENquIGLy1AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 800x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 定义 Softmax 函数\n",
    "def softmax(z):\n",
    "    exp_z = np.exp(z)\n",
    "    return exp_z / np.sum(exp_z)\n",
    "\n",
    "# 创建一个二维网格，用于绘制图像\n",
    "z1 = np.linspace(-10, 10, 100)\n",
    "z2 = np.linspace(-10, 10, 100)\n",
    "Z1, Z2 = np.meshgrid(z1, z2)\n",
    "Z = np.array([softmax([z1, z2]) for z1, z2 in zip(np.ravel(Z1), np.ravel(Z2))])\n",
    "\n",
    "# 提取 Softmax 输出的第一个元素，即概率 P(z1)\n",
    "P_z1 = Z[:, 0].reshape(Z1.shape)\n",
    "\n",
    "# 绘制图像\n",
    "plt.figure(figsize=(8, 6))\n",
    "contour = plt.contourf(Z1, Z2, P_z1, cmap='viridis', levels=np.linspace(0, 1, 11))\n",
    "plt.colorbar(contour, label='P(z1)')\n",
    "plt.xlabel('z1')\n",
    "plt.ylabel('z2')\n",
    "plt.title('Softmax Function: Probability of z1')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2Num7sX9CKOH",
    "outputId": "929ceb78-a639-41d6-aac7-12997b5c93f0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 100])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class LayerNorm1d: # (used to be BatchNorm1d)\n",
    "\n",
    "# dim是输入的特征维度，eps是一个很小的数，用于防止除以0，momentum是用于计算移动平均的动量，但在这个类中并没有使用。\n",
    "  def __init__(self, dim, eps=1e-5, momentum=0.1):\n",
    "    self.eps = eps\n",
    "    self.gamma = torch.ones(dim)\n",
    "    self.beta = torch.zeros(dim)\n",
    "\n",
    "  def __call__(self, x):\n",
    "    # calculate the forward pass\n",
    "    xmean = x.mean(1, keepdim=True) # batch mean\n",
    "    xvar = x.var(1, keepdim=True) # batch variance\n",
    "    xhat = (x - xmean) / torch.sqrt(xvar + self.eps) # normalize to unit variance\n",
    "    self.out = self.gamma * xhat + self.beta\n",
    "    return self.out\n",
    "  # 计算了输入x在第二个维度（dim=1）上的均值和方差，然后用x减去均值，再除以标准差，得到了归一化后的xhat。最后，我们用gamma乘以xhat，再加上beta，得到了输出。\n",
    "\n",
    "  def parameters(self):\n",
    "    return [self.gamma, self.beta]\n",
    "\n",
    "torch.manual_seed(1337)\n",
    "module = LayerNorm1d(100)\n",
    "x = torch.randn(32, 100) # batch size 32 of 100-dimensional vectors\n",
    "x = module(x)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "633T2cmnW1uk",
    "outputId": "7720fa58-0478-4e8a-86a7-502d4cce9443"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.1469), tensor(0.8803))"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[:,0].mean(), x[:,0].std() # mean,std of one feature across all batch inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "LN9cK9BoXCYb",
    "outputId": "6368ece0-600e-417d-8a91-7c1e5d750ba8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(-3.5763e-09), tensor(1.0000))"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[0,:].mean(), x[0,:].std() # mean,std of a single input from the batch, of its features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dRJH6wM_XFfU"
   },
   "outputs": [],
   "source": [
    "# French to English translation example:\n",
    "\n",
    "# <--------- ENCODE ------------------><--------------- DECODE ----------------->\n",
    "# les réseaux de neurones sont géniaux! <START> neural networks are awesome!<END>\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZcvKeBXoZFOY"
   },
   "source": [
    "### Full finished code, for reference\n",
    "\n",
    "You may want to refer directly to the git repo instead though."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hoelkOrFY8bN",
    "outputId": "961304cd-e379-40d4-dd56-8de0b91d2861"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.209729 M parameters\n",
      "step 0: train loss 4.4116, val loss 4.4022\n",
      "step 100: train loss 2.6568, val loss 2.6670\n",
      "step 200: train loss 2.5091, val loss 2.5059\n",
      "step 300: train loss 2.4196, val loss 2.4336\n",
      "step 400: train loss 2.3498, val loss 2.3560\n",
      "step 500: train loss 2.2964, val loss 2.3130\n",
      "step 600: train loss 2.2406, val loss 2.2497\n",
      "step 700: train loss 2.2052, val loss 2.2185\n",
      "step 800: train loss 2.1633, val loss 2.1861\n",
      "step 900: train loss 2.1246, val loss 2.1511\n",
      "step 1000: train loss 2.1036, val loss 2.1309\n",
      "step 1100: train loss 2.0709, val loss 2.1196\n",
      "step 1200: train loss 2.0381, val loss 2.0794\n",
      "step 1300: train loss 2.0243, val loss 2.0633\n",
      "step 1400: train loss 1.9929, val loss 2.0362\n",
      "step 1500: train loss 1.9711, val loss 2.0313\n",
      "step 1600: train loss 1.9633, val loss 2.0484\n",
      "step 1700: train loss 1.9410, val loss 2.0126\n",
      "step 1800: train loss 1.9111, val loss 1.9973\n",
      "step 1900: train loss 1.9103, val loss 1.9877\n",
      "step 2000: train loss 1.8853, val loss 1.9945\n",
      "step 2100: train loss 1.8711, val loss 1.9765\n",
      "step 2200: train loss 1.8594, val loss 1.9612\n",
      "step 2300: train loss 1.8541, val loss 1.9503\n",
      "step 2400: train loss 1.8437, val loss 1.9446\n",
      "step 2500: train loss 1.8157, val loss 1.9425\n",
      "step 2600: train loss 1.8264, val loss 1.9394\n",
      "step 2700: train loss 1.8099, val loss 1.9330\n",
      "step 2800: train loss 1.8071, val loss 1.9250\n",
      "step 2900: train loss 1.8077, val loss 1.9323\n",
      "step 3000: train loss 1.7989, val loss 1.9207\n",
      "step 3100: train loss 1.7713, val loss 1.9230\n",
      "step 3200: train loss 1.7555, val loss 1.9141\n",
      "step 3300: train loss 1.7606, val loss 1.9092\n",
      "step 3400: train loss 1.7565, val loss 1.8923\n",
      "step 3500: train loss 1.7361, val loss 1.8905\n",
      "step 3600: train loss 1.7256, val loss 1.8840\n",
      "step 3700: train loss 1.7323, val loss 1.8863\n",
      "step 3800: train loss 1.7229, val loss 1.8916\n",
      "step 3900: train loss 1.7187, val loss 1.8654\n",
      "step 4000: train loss 1.7119, val loss 1.8552\n",
      "step 4100: train loss 1.7118, val loss 1.8778\n",
      "step 4200: train loss 1.7038, val loss 1.8559\n",
      "step 4300: train loss 1.6976, val loss 1.8423\n",
      "step 4400: train loss 1.7050, val loss 1.8618\n",
      "step 4500: train loss 1.6898, val loss 1.8519\n",
      "step 4600: train loss 1.6872, val loss 1.8321\n",
      "step 4700: train loss 1.6819, val loss 1.8385\n",
      "step 4800: train loss 1.6678, val loss 1.8449\n",
      "step 4900: train loss 1.6703, val loss 1.8382\n",
      "step 4999: train loss 1.6633, val loss 1.8223\n",
      "\n",
      "Flie?\n",
      "\n",
      "WARICENTIO:\n",
      "Shrungst bewiter are a toom here:\n",
      "In if the wouIes?\n",
      "Out; and sate, and for one that I are and those it;\n",
      "Git.\n",
      "\n",
      "WARIO:\n",
      "Advory's toble sear; the, will God\n",
      "of breath what Mear;\n",
      "Was can is your name Burry eyree facge\n",
      "For will dath thee herp too thy laments\n",
      "That would may clood, one these do spost I vour have weret, where sup.\n",
      "How sens Gortunt, which what fit,\n",
      "Out thre, is wwife that broth. Who', betch'd your.\n",
      "\n",
      "TRABELLANE:\n",
      "Shall\n",
      "My sworn must he anour,\n",
      "Buntius; in So metter those make me,\n",
      "And fliems, my chince.\n",
      "\n",
      "POMY:\n",
      "Yet my nourtwarly to be thrany's discont,\n",
      "If day a gends pmenaton him, say.\n",
      "\n",
      "DUKE OF YORK:\n",
      "The twerefory well streage was babolantand now thing:\n",
      "O, this to set myself, cour bid to shall her speesen he crown.\n",
      "Vnow as thou thront, plarter no adds in thyself croath.\n",
      "My lord, but terruly friend\n",
      "Ristomfurts to-mries\n",
      "Againt serviet, contandy that kisspy grave, we mine!\n",
      "Or he him my spast,\n",
      "I so unsinced, wen is bese zable gity;\n",
      "Not do to seing, then thee from whosame noby.\n",
      "Go and neer thou would may night.\n",
      "\n",
      "RUCHERSIO:\n",
      "That, by tongue will be in him our sitittion;\n",
      "She now to be, all goots stespers\n",
      "in An rentry. Towe pas-dayfull keep,\n",
      "That thy will your sould in him,\n",
      "And ladditlebaning that gentrand, which myself, betish end alwied's boy exides'd.\n",
      "\n",
      "ISABTH:\n",
      "My master the slands you great? I shalk;\n",
      "Stat the kngscanty on straight boys hitger;\n",
      "Becompely his doath; of us Voly.\n",
      "\n",
      "Sensure:\n",
      "And there your worts, all, save infict is a those astold\n",
      "of by my felling wit-be in his in\n",
      "Hard's deaths chanting me them is seedder'd was busweet.\n",
      "\n",
      "CORINIUS:\n",
      "No, for I have your merch'mord.\n",
      "Is you graXut affinzy houth this sele yourders?\n",
      "\n",
      "POLFORD NORWARD YOMIUS:\n",
      "You come you.\n",
      "\n",
      "POYCUS:\n",
      "Thy gleist the dongsorn:\n",
      "Nay fantle Becoleforfact tell servy inters! in I know tould lappread\n",
      "Goat you sucalf me wars non your\n",
      "art:\n",
      "As one thumself, who live requann.\n",
      "Anjury thee that with, and we not?\n",
      "Good the preasurs, comison toOH!\n",
      "\n",
      "Second Affordmn:-now.\n",
      "\n",
      "Must wear I pearince?\n",
      "And think\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import functional as F\n",
    "\n",
    "# 超参数设置\n",
    "# hyperparameters\n",
    "batch_size = 16 # how many independent sequences will we process in parallel?\n",
    "block_size = 32 # what is the maximum context length for predictions?\n",
    "max_iters = 5000\n",
    "eval_interval = 100\n",
    "learning_rate = 1e-3\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "eval_iters = 200\n",
    "n_embd = 64\n",
    "n_head = 4\n",
    "n_layer = 4\n",
    "dropout = 0.0\n",
    "# ------------\n",
    "\n",
    "torch.manual_seed(1337)\n",
    "\n",
    "# 1. 数据处理\n",
    "# wget https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt\n",
    "with open('./ng-video-lecture/input.txt', 'r', encoding='utf-8') as f:\n",
    "    text = f.read()\n",
    "\n",
    "# here are all the unique characters that occur in this text\n",
    "chars = sorted(list(set(text)))\n",
    "vocab_size = len(chars)\n",
    "# create a mapping from characters to integers\n",
    "stoi = { ch:i for i,ch in enumerate(chars) }\n",
    "itos = { i:ch for i,ch in enumerate(chars) }\n",
    "encode = lambda s: [stoi[c] for c in s] # encoder: take a string, output a list of integers\n",
    "decode = lambda l: ''.join([itos[i] for i in l]) # decoder: take a list of integers, output a string\n",
    "\n",
    "# 2. 数据集划分\n",
    "# Train and test splits\n",
    "data = torch.tensor(encode(text), dtype=torch.long)\n",
    "n = int(0.9*len(data)) # first 90% will be train, rest val\n",
    "train_data = data[:n]\n",
    "val_data = data[n:]\n",
    "\n",
    "# 3. 数据分批\n",
    "# data loading\n",
    "def get_batch(split):\n",
    "    # generate a small batch of data of inputs x and targets y\n",
    "    data = train_data if split == 'train' else val_data\n",
    "    ix = torch.randint(len(data) - block_size, (batch_size,))\n",
    "    x = torch.stack([data[i:i+block_size] for i in ix])\n",
    "    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n",
    "    x, y = x.to(device), y.to(device)\n",
    "    return x, y\n",
    "\n",
    "# 4. 模型评估\n",
    "# 评估模型在训练集和验证集上的平均损失。这个函数的实现非常简单：它首先将模型设置为评估模式，然后对每个数据集进行eval_iters次迭代。\n",
    "# 在每次迭代中，它获取一个小批量数据，然后计算模型的输出和损失。最后，它返回每个数据集的平均损失。\n",
    "@torch.no_grad()\n",
    "def estimate_loss():\n",
    "    out = {}\n",
    "    model.eval()\n",
    "    for split in ['train', 'val']:\n",
    "        losses = torch.zeros(eval_iters)\n",
    "        for k in range(eval_iters):\n",
    "            X, Y = get_batch(split)\n",
    "            logits, loss = model(X, Y)\n",
    "            losses[k] = loss.item()\n",
    "        out[split] = losses.mean()\n",
    "    model.train()\n",
    "    return out\n",
    "\n",
    "# 5. 模型head定义\n",
    "class Head(nn.Module):\n",
    "    \"\"\" one head of self-attention \"\"\"\n",
    "\n",
    "    def __init__(self, head_size):\n",
    "        super().__init__()\n",
    "        self.key = nn.Linear(n_embd, head_size, bias=False)\n",
    "        self.query = nn.Linear(n_embd, head_size, bias=False)\n",
    "        self.value = nn.Linear(n_embd, head_size, bias=False)\n",
    "        # 创建一个下三角矩阵，并将其注册为模型的一个缓冲区。\n",
    "        # 这个下三角矩阵将被用作self-attention的权重矩阵，它将确保模型只能在当前时间步之前的时间步上进行自注意力操作。\n",
    "        self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))\n",
    "\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, x):\n",
    "        B,T,C = x.shape\n",
    "        k = self.key(x)   # (B,T,C)\n",
    "        q = self.query(x) # (B,T,C)\n",
    "        # compute attention scores (\"affinities\")\n",
    "        wei = q @ k.transpose(-2,-1) * C**-0.5 # (B, T, C) @ (B, C, T) -> (B, T, T)\n",
    "        wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf')) # (B, T, T)\n",
    "        wei = F.softmax(wei, dim=-1) # (B, T, T)\n",
    "        wei = self.dropout(wei)\n",
    "        # perform the weighted aggregation of the values\n",
    "        v = self.value(x) # (B,T,C)\n",
    "        out = wei @ v # (B, T, T) @ (B, T, C) -> (B, T, C)\n",
    "        return out\n",
    "\n",
    "# 6. 模型MultiHeadAttention定义\n",
    "# 实现多头自注意力机制。在这个机制中，我们并行地进行多次自注意力计算，然后将结果拼接起来，通过一个线性层和一个dropout层进行处理，得到最终的输出。\n",
    "# 这种方法可以让模型在不同的表示子空间中学习输入的不同特征。\n",
    "class MultiHeadAttention(nn.Module):\n",
    "    \"\"\" multiple heads of self-attention in parallel \"\"\"\n",
    "\n",
    "    # __init__方法是类的构造函数，它接收两个参数：num_heads和head_size。num_heads是注意力头的数量，head_size是每个注意力头的大小。\n",
    "    # 创建了一个nn.ModuleList，它包含了num_heads个Head对象。我们还定义了一个线性层self.proj和一个dropout层self.dropout。\n",
    "    def __init__(self, num_heads, head_size):\n",
    "        super().__init__()\n",
    "        self.heads = nn.ModuleList([Head(head_size) for _ in range(num_heads)])\n",
    "        self.proj = nn.Linear(n_embd, n_embd)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    # forward方法定义了前向传播的计算过程。首先，我们对每个注意力头h进行计算，然后将结果在最后一个维度上拼接起来，得到out。\n",
    "    # 然后，我们将out输入到线性层和dropout层，得到最终的输出。\n",
    "    def forward(self, x):\n",
    "        out = torch.cat([h(x) for h in self.heads], dim=-1)\n",
    "        out = self.dropout(self.proj(out))\n",
    "        return out\n",
    "\n",
    "# 7. 模型FeedFoward定义\n",
    "class FeedFoward(nn.Module):\n",
    "    \"\"\" a simple linear layer followed by a non-linearity \"\"\"\n",
    "\n",
    "    # 在__init__方法中，我们首先调用了父类的构造函数，然后定义了一个神经网络self.net。这个神经网络是一个nn.Sequential对象，\n",
    "    # 它包含了两个线性层和一个ReLU激活函数，以及一个dropout层。\n",
    "    # 第一个线性层将输入的维度扩大到4 * n_embd，然后通过ReLU激活函数进行非线性变换，然后第二个线性层将维度缩小回n_embd，最后通过dropout层进行正则化。\n",
    "    def __init__(self, n_embd):\n",
    "        super().__init__()\n",
    "        self.net = nn.Sequential(\n",
    "            nn.Linear(n_embd, 4 * n_embd),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(4 * n_embd, n_embd),\n",
    "            nn.Dropout(dropout),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.net(x)\n",
    "\n",
    "# 8. 模型Block定义，layerNorm, multiheadattention, layerNorm, feedforward\n",
    "# Block类的作用是实现一个Transformer模型中的一个块。这个块包含了一个多头自注意力模块和一个前馈神经网络模块，以及两个层归一化操作。\n",
    "# 这种结构可以让模型在处理序列数据时，能够同时考虑到每个位置的信息和全局的信息。\n",
    "class Block(nn.Module):\n",
    "    \"\"\" Transformer block: communication followed by computation \"\"\"\n",
    "\n",
    "    def __init__(self, n_embd, n_head):\n",
    "        # n_embd: embedding dimension, n_head: the number of heads we'd like\n",
    "        super().__init__()\n",
    "        head_size = n_embd // n_head\n",
    "        self.sa = MultiHeadAttention(n_head, head_size)\n",
    "        self.ffwd = FeedFoward(n_embd)\n",
    "        self.ln1 = nn.LayerNorm(n_embd)\n",
    "        self.ln2 = nn.LayerNorm(n_embd)\n",
    "\n",
    "    # forward方法定义了前向传播的计算过程。首先，我们将输入x进行层归一化，然后输入到自注意力模块中，得到的输出与原始的x相加，得到新的x。\n",
    "    # 然后，我们将新的x进行层归一化，然后输入到前馈神经网络中，得到的输出与原始的x相加，得到最终的输出。\n",
    "    def forward(self, x):\n",
    "        x = x + self.sa(self.ln1(x))\n",
    "        x = x + self.ffwd(self.ln2(x))\n",
    "        return x\n",
    "\n",
    "# 9. 模型BigramLanguageModel定义\n",
    "# super simple bigram model，训练一个二元语言模型，并生成新的文本。\n",
    "class BigramLanguageModel(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        # each token directly reads off the logits for the next token from a lookup table\n",
    "        self.token_embedding_table = nn.Embedding(vocab_size, n_embd)\n",
    "        self.position_embedding_table = nn.Embedding(block_size, n_embd)\n",
    "        self.blocks = nn.Sequential(*[Block(n_embd, n_head=n_head) for _ in range(n_layer)])\n",
    "        self.ln_f = nn.LayerNorm(n_embd) # final layer norm\n",
    "        self.lm_head = nn.Linear(n_embd, vocab_size)\n",
    "\n",
    "    # forward方法定义了前向传播的计算过程。首先，我们从词嵌入表和位置嵌入表中获取嵌入，将它们相加得到x。\n",
    "    # 然后，我们将x输入到self.blocks中，进行层归一化，然后输入到self.lm_head中，得到logits。如果提供了目标，我们会计算交叉熵损失。\n",
    "    def forward(self, idx, targets=None):\n",
    "        B, T = idx.shape\n",
    "\n",
    "        # idx and targets are both (B,T) tensor of integers\n",
    "        tok_emb = self.token_embedding_table(idx) # (B,T,C)\n",
    "        pos_emb = self.position_embedding_table(torch.arange(T, device=device)) # (T,C)\n",
    "        x = tok_emb + pos_emb # (B,T,C)\n",
    "        x = self.blocks(x) # (B,T,C)\n",
    "        x = self.ln_f(x) # (B,T,C)\n",
    "        logits = self.lm_head(x) # (B,T,vocab_size)\n",
    "\n",
    "        if targets is None:\n",
    "            loss = None\n",
    "        else:\n",
    "            B, T, C = logits.shape\n",
    "            logits = logits.view(B*T, C)\n",
    "            targets = targets.view(B*T)\n",
    "            loss = F.cross_entropy(logits, targets)\n",
    "\n",
    "        return logits, loss\n",
    "\n",
    "    # 将当前的索引裁剪到最后的block_size个令牌，获取预测的logits，只关注最后一个时间步，应用softmax得到概率，从分布中采样，将采样的索引添加到运行的序列中。\n",
    "    def generate(self, idx, max_new_tokens):\n",
    "        # idx is (B, T) array of indices in the current context\n",
    "        for _ in range(max_new_tokens):\n",
    "            # crop idx to the last block_size tokens\n",
    "            idx_cond = idx[:, -block_size:]\n",
    "            # get the predictions\n",
    "            logits, loss = self(idx_cond)\n",
    "            # focus only on the last time step\n",
    "            logits = logits[:, -1, :] # becomes (B, C)\n",
    "            # apply softmax to get probabilities\n",
    "            probs = F.softmax(logits, dim=-1) # (B, C)\n",
    "            # sample from the distribution\n",
    "            idx_next = torch.multinomial(probs, num_samples=1) # (B, 1)\n",
    "            # append sampled index to the running sequence\n",
    "            idx = torch.cat((idx, idx_next), dim=1) # (B, T+1)\n",
    "        return idx\n",
    "\n",
    "\n",
    "# 10. 模型实例化\n",
    "model = BigramLanguageModel()\n",
    "m = model.to(device)\n",
    "# print the number of parameters in the model\n",
    "# 将其移动到设备device上。然后，我们打印出模型中的参数数量。\n",
    "print(sum(p.numel() for p in m.parameters())/1e6, 'M parameters')\n",
    "\n",
    "# create a PyTorch optimizer\n",
    "optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
    "\n",
    "for iter in range(max_iters):\n",
    "\n",
    "    # every once in a while evaluate the loss on train and val sets\n",
    "    if iter % eval_interval == 0 or iter == max_iters - 1:\n",
    "        losses = estimate_loss()\n",
    "        print(f\"step {iter}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}\")\n",
    "\n",
    "    # 从训练集中采样一个批次的数据xb和yb，将它们输入到模型中，得到logits和loss。然后将优化器的梯度清零，计算损失的反向传播，更新优化器的参数。\n",
    "    # 采样一个批次的数据，计算损失，清零梯度，计算反向传播，然后更新参数。这是训练神经网络模型的基本步骤。\n",
    "    # sample a batch of data\n",
    "    xb, yb = get_batch('train')\n",
    "\n",
    "    # evaluate the loss\n",
    "    logits, loss = model(xb, yb)\n",
    "    # 清零优化器的梯度。这是因为PyTorch的优化器在每次更新参数时都会累积梯度，所以在每次更新参数之前，我们需要清零梯度。\n",
    "    optimizer.zero_grad(set_to_none=True)\n",
    "    # 计算损失的反向传播。这会计算出每个参数的梯度。\n",
    "    loss.backward()\n",
    "    # 更新优化器的参数。这会根据每个参数的梯度和学习率来更新参数的值。\n",
    "    optimizer.step()\n",
    "\n",
    "# generate from the model\n",
    "# 用模型来生成新的文本，首先创建一个全零的上下文context，将其输入到模型的generate方法中，得到生成的文本，然后将其解码并打印出来。\n",
    "context = torch.zeros((1, 1), dtype=torch.long, device=device)\n",
    "print(decode(m.generate(context, max_new_tokens=2000)[0].tolist()))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fjjvMifYZf7x"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}