beginners-guide-to-text-generation-with-rnns.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Create the Environment","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import Dense, LSTM, Embedding\nimport matplotlib.pyplot as plt\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-27T13:53:16.236358Z","iopub.execute_input":"2021-05-27T13:53:16.236964Z","iopub.status.idle":"2021-05-27T13:53:22.987563Z","shell.execute_reply.started":"2021-05-27T13:53:16.236869Z","shell.execute_reply":"2021-05-27T13:53:22.986158Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"markdown","source":"# Download Text Data","metadata":{}},{"cell_type":"code","source":"file_URL = \"https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\"\nfile_name= \"shakespeare.txt\"\n# get the file path\npath = keras.utils.get_file(file_name, file_URL)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:22.991495Z","iopub.execute_input":"2021-05-27T13:53:22.991878Z","iopub.status.idle":"2021-05-27T13:53:23.264433Z","shell.execute_reply.started":"2021-05-27T13:53:22.991848Z","shell.execute_reply":"2021-05-27T13:53:23.263434Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Downloading data from https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\n1122304/1115394 [==============================] - 0s 0us/step\n","output_type":"stream"}]},{"cell_type":"code","source":"raw = open(path, 'rb').read()\nprint(raw[250:400])","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:23.266182Z","iopub.execute_input":"2021-05-27T13:53:23.266524Z","iopub.status.idle":"2021-05-27T13:53:23.276416Z","shell.execute_reply.started":"2021-05-27T13:53:23.266492Z","shell.execute_reply":"2021-05-27T13:53:23.275278Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"b\"\\nAll:\\nWe know't, we know't.\\n\\nFirst Citizen:\\nLet us kill him, and we'll have corn at our own price.\\nIs't a verdict?\\n\\nAll:\\nNo more talking on't; let it \"\n","output_type":"stream"}]},{"cell_type":"code","source":"text = raw.decode(encoding='utf-8')\nprint(text[250:400])","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:23.278652Z","iopub.execute_input":"2021-05-27T13:53:23.279197Z","iopub.status.idle":"2021-05-27T13:53:23.295190Z","shell.execute_reply.started":"2021-05-27T13:53:23.279151Z","shell.execute_reply":"2021-05-27T13:53:23.294329Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"\nAll:\nWe know't, we know't.\n\nFirst Citizen:\nLet us kill him, and we'll have corn at our own price.\nIs't a verdict?\n\nAll:\nNo more talking on't; let it \n","output_type":"stream"}]},{"cell_type":"code","source":"len(text)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:23.296456Z","iopub.execute_input":"2021-05-27T13:53:23.297041Z","iopub.status.idle":"2021-05-27T13:53:23.315207Z","shell.execute_reply.started":"2021-05-27T13:53:23.297007Z","shell.execute_reply":"2021-05-27T13:53:23.314275Z"},"trusted":true},"execution_count":5,"outputs":[{"execution_count":5,"output_type":"execute_result","data":{"text/plain":"1115394"},"metadata":{}}]},{"cell_type":"markdown","source":"# Vectorize Word Characters into Integers","metadata":{}},{"cell_type":"code","source":"# unique characters\nvocabulary = np.array(sorted(set(text)))\nlen(vocabulary)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:23.316523Z","iopub.execute_input":"2021-05-27T13:53:23.317090Z","iopub.status.idle":"2021-05-27T13:53:23.343718Z","shell.execute_reply.started":"2021-05-27T13:53:23.317055Z","shell.execute_reply":"2021-05-27T13:53:23.342446Z"},"trusted":true},"execution_count":6,"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"65"},"metadata":{}}]},{"cell_type":"code","source":"# assign an integer to each character\ntokenizer = {char:i for i,char in enumerate(vocabulary)}","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:23.345337Z","iopub.execute_input":"2021-05-27T13:53:23.345660Z","iopub.status.idle":"2021-05-27T13:53:23.358488Z","shell.execute_reply.started":"2021-05-27T13:53:23.345622Z","shell.execute_reply":"2021-05-27T13:53:23.357030Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"# check characters and its corresponding integer\n\nfor i in range(20):\n    char = vocabulary[i]\n    token = tokenizer[char]\n    print('%4s : %4d'%(repr(char),token))","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:23.361947Z","iopub.execute_input":"2021-05-27T13:53:23.362442Z","iopub.status.idle":"2021-05-27T13:53:23.377254Z","shell.execute_reply.started":"2021-05-27T13:53:23.362394Z","shell.execute_reply":"2021-05-27T13:53:23.375801Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"'\\n' :    0\n ' ' :    1\n '!' :    2\n '$' :    3\n '&' :    4\n \"'\" :    5\n ',' :    6\n '-' :    7\n '.' :    8\n '3' :    9\n ':' :   10\n ';' :   11\n '?' :   12\n 'A' :   13\n 'B' :   14\n 'C' :   15\n 'D' :   16\n 'E' :   17\n 'F' :   18\n 'G' :   19\n","output_type":"stream"}]},{"cell_type":"code","source":"vector = np.array([tokenizer[char] for char in text])\n\nprint('\\nSample Text \\n')\nprint('-'*70)\nprint(text[:100])\nprint('-'*70)\nprint('\\n\\nCorresponding Integer Vector \\n')\nprint('-'*70)\nprint(vector[:100])\nprint('-'*70)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:23.379523Z","iopub.execute_input":"2021-05-27T13:53:23.379929Z","iopub.status.idle":"2021-05-27T13:53:23.705783Z","shell.execute_reply.started":"2021-05-27T13:53:23.379897Z","shell.execute_reply":"2021-05-27T13:53:23.704742Z"},"trusted":true},"execution_count":9,"outputs":[{"name":"stdout","text":"\nSample Text \n\n----------------------------------------------------------------------\nFirst Citizen:\nBefore we proceed any further, hear me speak.\n\nAll:\nSpeak, speak.\n\nFirst Citizen:\nYou\n----------------------------------------------------------------------\n\n\nCorresponding Integer Vector \n\n----------------------------------------------------------------------\n[18 47 56 57 58  1 15 47 58 47 64 43 52 10  0 14 43 44 53 56 43  1 61 43\n  1 54 56 53 41 43 43 42  1 39 52 63  1 44 59 56 58 46 43 56  6  1 46 43\n 39 56  1 51 43  1 57 54 43 39 49  8  0  0 13 50 50 10  0 31 54 43 39 49\n  6  1 57 54 43 39 49  8  0  0 18 47 56 57 58  1 15 47 58 47 64 43 52 10\n  0 37 53 59]\n----------------------------------------------------------------------\n","output_type":"stream"}]},{"cell_type":"code","source":"# convert into tensors\nvector = tf.data.Dataset.from_tensor_slices(vector)\n\n# make sequences each of length 100 characters\nsequences = vector.batch(100, drop_remainder=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:23.707136Z","iopub.execute_input":"2021-05-27T13:53:23.707471Z","iopub.status.idle":"2021-05-27T13:53:23.748223Z","shell.execute_reply.started":"2021-05-27T13:53:23.707440Z","shell.execute_reply":"2021-05-27T13:53:23.747343Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"def prepare_dataset(seq):\n    input_vector = seq[:-1]\n    target_vector = seq[1:]\n    return input_vector, target_vector\n\ndataset = sequences.map(prepare_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:23.749381Z","iopub.execute_input":"2021-05-27T13:53:23.749804Z","iopub.status.idle":"2021-05-27T13:53:23.839938Z","shell.execute_reply.started":"2021-05-27T13:53:23.749773Z","shell.execute_reply":"2021-05-27T13:53:23.839015Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"# check how it looks\n\nfor inp, tar in dataset.take(1):\n    print(inp.numpy())\n    print(tar.numpy())\n    inp_text = ''.join(vocabulary[inp])\n    tar_text = ''.join(vocabulary[tar])\n    print(repr(inp_text))\n    print(repr(tar_text))","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:23.841236Z","iopub.execute_input":"2021-05-27T13:53:23.841739Z","iopub.status.idle":"2021-05-27T13:53:23.955160Z","shell.execute_reply.started":"2021-05-27T13:53:23.841696Z","shell.execute_reply":"2021-05-27T13:53:23.953678Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stdout","text":"[18 47 56 57 58  1 15 47 58 47 64 43 52 10  0 14 43 44 53 56 43  1 61 43\n  1 54 56 53 41 43 43 42  1 39 52 63  1 44 59 56 58 46 43 56  6  1 46 43\n 39 56  1 51 43  1 57 54 43 39 49  8  0  0 13 50 50 10  0 31 54 43 39 49\n  6  1 57 54 43 39 49  8  0  0 18 47 56 57 58  1 15 47 58 47 64 43 52 10\n  0 37 53]\n[47 56 57 58  1 15 47 58 47 64 43 52 10  0 14 43 44 53 56 43  1 61 43  1\n 54 56 53 41 43 43 42  1 39 52 63  1 44 59 56 58 46 43 56  6  1 46 43 39\n 56  1 51 43  1 57 54 43 39 49  8  0  0 13 50 50 10  0 31 54 43 39 49  6\n  1 57 54 43 39 49  8  0  0 18 47 56 57 58  1 15 47 58 47 64 43 52 10  0\n 37 53 59]\n'First Citizen:\\nBefore we proceed any further, hear me speak.\\n\\nAll:\\nSpeak, speak.\\n\\nFirst Citizen:\\nYo'\n'irst Citizen:\\nBefore we proceed any further, hear me speak.\\n\\nAll:\\nSpeak, speak.\\n\\nFirst Citizen:\\nYou'\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Batch and Prefetch Dataset","metadata":{}},{"cell_type":"code","source":"# number of batched sequences\nlen(sequences)//64","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:23.956581Z","iopub.execute_input":"2021-05-27T13:53:23.957086Z","iopub.status.idle":"2021-05-27T13:53:23.963852Z","shell.execute_reply.started":"2021-05-27T13:53:23.957039Z","shell.execute_reply":"2021-05-27T13:53:23.962734Z"},"trusted":true},"execution_count":13,"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"174"},"metadata":{}}]},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\n# buffer size 10000\n# batch size 64\ndata = dataset.batch(64, drop_remainder=True).repeat()\ndata = data.prefetch(AUTOTUNE)\n# steps per epoch is number of batches available\nSTEPS_PER_EPOCH = len(sequences)//64","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:23.965907Z","iopub.execute_input":"2021-05-27T13:53:23.966680Z","iopub.status.idle":"2021-05-27T13:53:23.987474Z","shell.execute_reply.started":"2021-05-27T13:53:23.966626Z","shell.execute_reply":"2021-05-27T13:53:23.986278Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"for inp, tar in data.take(1):\n    print(inp.numpy().shape)\n    print(tar.numpy().shape)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:23.988761Z","iopub.execute_input":"2021-05-27T13:53:23.989089Z","iopub.status.idle":"2021-05-27T13:53:24.042036Z","shell.execute_reply.started":"2021-05-27T13:53:23.989057Z","shell.execute_reply":"2021-05-27T13:53:24.041259Z"},"trusted":true},"execution_count":15,"outputs":[{"name":"stdout","text":"(64, 99)\n(64, 99)\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Build Model","metadata":{}},{"cell_type":"code","source":"model = keras.Sequential([\n    # Embed len(vocabulary) into 64 dimensions\n    Embedding(len(vocabulary), 64, batch_input_shape=[64,None]),\n    # LSTM RNN layers\n    LSTM(512, return_sequences=True, stateful=True),\n    LSTM(512, return_sequences=True, stateful=True),\n    # Classification head\n    Dense(len(vocabulary))\n])","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:24.043295Z","iopub.execute_input":"2021-05-27T13:53:24.043639Z","iopub.status.idle":"2021-05-27T13:53:25.087672Z","shell.execute_reply.started":"2021-05-27T13:53:24.043609Z","shell.execute_reply":"2021-05-27T13:53:25.086790Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:25.088773Z","iopub.execute_input":"2021-05-27T13:53:25.089173Z","iopub.status.idle":"2021-05-27T13:53:25.097934Z","shell.execute_reply.started":"2021-05-27T13:53:25.089144Z","shell.execute_reply":"2021-05-27T13:53:25.097072Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stdout","text":"Model: \"sequential\"\n_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\nembedding (Embedding)        (64, None, 64)            4160      \n_________________________________________________________________\nlstm (LSTM)                  (64, None, 512)           1181696   \n_________________________________________________________________\nlstm_1 (LSTM)                (64, None, 512)           2099200   \n_________________________________________________________________\ndense (Dense)                (64, None, 65)            33345     \n=================================================================\nTotal params: 3,318,401\nTrainable params: 3,318,401\nNon-trainable params: 0\n_________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"code","source":"keras.utils.plot_model(model, show_shapes=True, dpi=64)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:25.099454Z","iopub.execute_input":"2021-05-27T13:53:25.100114Z","iopub.status.idle":"2021-05-27T13:53:25.578971Z","shell.execute_reply.started":"2021-05-27T13:53:25.100068Z","shell.execute_reply":"2021-05-27T13:53:25.577812Z"},"trusted":true},"execution_count":18,"outputs":[{"execution_count":18,"output_type":"execute_result","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAWgAAAFYCAIAAADx/fEvAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3de1QT57YA8C8EBEIIIlWSEF6eLuUpCIpQ9F4vLRWRVlEavagoqEjFJfioeJCHSBFfR7wcXQSPRa9vsEWwiOb22FWPiGchIBpb1HKFGF4iVkBEQNK5f8zp3AgkzJDAJLh/fw1fZr7ZE2E7M9mzw8AwDAEAABV6dAcAANA9kDgAAJTpE0t/+9vfXr16RWMoAABtZmpqum7dOnz5/884UlJSaIoHjHFtbW2ZmZl0R0HZf//3f9fV1dEdhRZRTBEM4uaoQCCor6+nKSQwlslkMqFQePv2bboDoeaLL77YsmWLj48P3YFoC8UUAfc4AACUQeIAAFAGiQMADUhJSdm7dy/dUVDm4+Nz//79YWwIiQNoETs7u/Lycs3OKRAIND6nkZERg8FISEjAf3z58qVIJIqOjsZ/PHXq1NSpUzkcTmRk5Js3bxQ3lEqlbDY7ICBgyF24uLhwOJwXL17gP164cGHGjBkaPQiEEPrqq68SExOJPTIYjGXLlpHZEBIH0CJ1dXUj8ecxEmQy2ddff40vnzlz5pNPPjE1NUUIFRUVxcfHZ2VlNTU1+fr63rx5U3Gr6OhoLy8vkrswMDDYt2+fZsPuJygoqKSkBL/l+eDBg4yMDJIbQuIAWkTxjIPL5R48eNDZ2ZnD4Sxbtqynp4cY37t3r6Oj44QJE9avX9/b24uP6+vrNzc348uxsbH46UBISEhDQ8PMmTMZDMaRI0dGKOyrV6/OnTsXX05KSkpLS/Pz8zMxMVm1atWnn35KrJaXl8disRRHVNu8ebNIJCIOSlFNTY2/vz+Hw3F0dMzLyyPGlb1pz549EwqFFhYWtra2itlh3LhxXl5eYrGY4hFD4gBarLi4+Nq1aw8fPpRIJGfOnCHGc3Nzi4uLKysry8rK9u/fr2KGb7/91srK6s6dOxiGbdy4MSoqasWKFRqPUyKRTJkyBSHU09NTVVXV3NzM5XInTpy4YcOG7u5ufJ329vaEhIRDhw6Rn9bBwSE4OHjPnj39xuVy+eeffz59+nSZTJaRkREREXH37l3i1YFvGoZhwcHBlpaWNTU1YrE4KyvrypUrxPqOjo737t2jesiQOID2SkpKsra25vP5ixYtevDgATGemJhob29vZ2eXnJx8/vx5+gL8l7a2Nvw6paGhAcOwoqIiPKndunUrLS0NXycuLi4yMlIgEFCaOTk5+cSJEzKZTHGwsrKyqakpLS3NzMwsICBAKBSePXuWeHXgm3b//v3q6urDhw+bm5s7ODjExsZevHiRWJ/D4bS1tVE9ZEgcQHt98MEH+IKxsbHiXUZbW1t8wc7OrrGxkfyEIpFI8cxFU8zNzfHHNYyNjRFC27Zt4/P59vb2mzZtunr1KkKorKystLQ0NjaW6syTJ08ODQ3dvXu34mBjY6NAIDAwMCDWaWhoIF4d+KZJpdK2tjZDQ0Mmk6mnpxcdHa2YiTo6OsaPH081MP2hVwFAy0ilUk9PT4RQXV0dn8/HB42MjIjrgtbWVjabjS/r6Y34/47Tpk17/PjxnDlzeDwe8XeLEMIwDN97ZWWlRCIh/tT7RataQkKCo6OjnZ0dMcLn8+vr69++fYtP+OTJEysrKxUz4Ccg9fX1DAZj4KvV1dUhISFkIlEEZxxA96SlpdXV1Uml0tTU1KVLl+KDrq6uOTk5XV1dYrE4Pz+fWHnSpEkSiQR/tGKE7nEEBgbeuHEDXw4PDz9w4EBjY2NtbW1mZmZQUBC+X+wP6enp8+bNI7LGkCFZW1tHREQo3hzx8PDgcrkJCQnt7e1isTg3Nzc0NFTFDG5ubjY2Nlu3bm1paWltbc3JyTl69Cj+Um9vb1lZGfn7tQRIHED3hISEBAQEuLu7e3h4xMXF4YOZmZn5+fk8Hk8kEi1ZsoRYOT4+PjExkclkjtynKsuXL79+/XpnZydCKDU11cnJycHBwdvb28/PjwhPGZlM9tFHH6leJz4+XvH0hMlkFhYWlpeXCwSCmJiY48ePe3h4qNhcT0+voKCgpaXF1dXV2dm5tLSUyLZXrlzx9fW1trYmdZyKiERoZWWFjQBLS0s83w/D1q1b4+LiyI8D7fT06VNvb29NzabObxQlISEhpaWlg75kaGgok8kUR1JSUtLT06nuoq+vz8HBobu7e5ghqs3Hx6eqqor4MSMjY+nSpcpWVkwROn/GodlaQ7oqF11cXDR+067ffkdiF+8za2tronIUIZSUlLRjxw6qkzCZzOrqakNDQ42GRkFpaambmxu+7OLisnnzZpIb6vzNUc12TID+C4AMkvc1dYviB95DInXGoazsjMvlJicnOzk5mZiYrF27ViKRzJo1i8ViBQYGdnR0EKsVFRUNrPNTNucvv/wya9YsMzOzwMDAlpaWIcfJ1Bo+ePDAy8sL3zYsLEzF/wzaULk4mvudMWMGg8EwMjKaNWtWZWUlPnjw4MHg4GBinejoaOJzRBW/CQcOHHBxcVH8TGGENDc3u7i4jPRegGpDJw5MZdlZSUnJ9evX7969W1BQIBQKs7OzGxoaOjs7s7OziXUG1vkpm1MulwcHB8+bN08mk61Zs+bcuXP4DMrGBxpYNodvu2DBAplMtn79ekr1QnRVLmp8v8pWKy8vxzDs5cuXq1evXrp0qVwuRwiFhYX9z//8T2trK0Kop6fnwoULK1euREP9Jly6dOnKlSv4VmDMGzpxqC47S0xM5PF4U6ZMmT179rJly9zd3c3NzRcvXqz4rO7AOj9lc1ZUVPz222/JyckcDmfJkiXEQ4TKxgcaWDZXUVGBV/tyOJyFCxfOnz+f/LtDV+XiKO/X2Nj4yy+/lMvlNTU1CKFJkyb5+/vjxYiFhYVcLhcvmlD9m5CUlETUZYExb+h7HETZGXFD1c/Pj3iVODU1MjKysLAgllXX+SmbEy+JYzKZxPr4grLxgRTL5vBHkhsbG62srIhtKf1yD5xN2RGRJBKJaNmvMllZWRkZGVKpFL/waWpqmjp1KkIoIiIiOTk5Jibm5MmT+OkGGuo3QXUNUm1tLdVqa9r19vbeuHFj3LhxdAeiLV6/fk0sD504VJedkTGwzk/ZnHw+v6GhQS6X43/ndXV1+NWssnEyeDye4rZSqdTJyWl4B6LiiNCoVC5qdr+PHj2Kj48Xi8XTpk0zMjKyt7fv6+vDXwoMDIyMjLx69erf//73Y8eO4YOqfxNU/3rY29tDz1Fdp5j6h/6dVlF2RtLAOj9lc3p6epqZmaWmpnZ0dBQUFFy7dg2fQdk4GZ6enhwO5+uvv+7o6Lh8+TL+7ICa6KpcHPZ+B/X69Wsmkzl+/Pi+vr6DBw8qfqKkr6+/cuXKVatWzZkzh/h1Uf83AYwZQycOFWVnJA2s81M2J5PJvHTpUnFxsbW19bFjx4hCWmXjZOjr63/33XdFRUXW1tYikWjx4sXqn3zSVbmo5n5XrlzJ+MPq1as9PDxWr17t5eU1ZcqU//3f/3V2dlbcV0RExPPnz8PCwogR9X8TwNgxaFnYGObv73/s2DF1Zhi1ykV693v37l0Wi/Xq1Sv1p9Js5eioUVE5OtCuXbuGUTlKO29v73v37pFceUxVjpKRnZ1dUlLy6tWrkydPlpSUBAYG0h2RtpPL5RkZGaGhocQdEy2hZgNRTfUfhZ6j70Xi+Pjjj5OSkvh8fkZGRmFhoZWVVV1dHWMAKBvFdXZ26uvrl5eXJycn0x2L9oKeo2Pfhx9++OOPP7569erevXv+/v4IITs7u4FnYio+5VVEV+XiqO2XzWZjGPbzzz+PwgeoynpnkiyH1ZL+o9BzFIDRo7p35kCDlsOqU02rqfYc0HMUgNGjuncmSdrQfxR6jgIwelT3ziRJnWpaTbUghZ6jAIweFb0zyZfDakP/Ueg5CsDoUdE7k3w5rDrVtJq6xwE9RwEYPSp6Z5Ivh9WG/qPQcxQAzRu5ytERraaFnqP9KKYIuMcBwHBYW1vv3LmTqAFLSkoaxiR4z1GNxkVNaWkpsezi4vLzzz+TfP4IEgcAlEHPUUgcQIcNWlUJRgHcHAUAUAaJAwBA2TuXKlu2bKErDjCGvX79+vnz5zr32/X48eOjR48qVmcDAgP7oximqKioq6uL3miA9igqKkII4fVLACCEWCwW8fvw/2cc8CsCFOE9GoRCId2BAG0E9zgAAJRB4gAAUAaJAwBAGSQOAABlkDgAAJRB4gAAUAaJAwBAGSQOAABlkDgAAJRB4gAAUAaJAwBAGSQOAABlkDgAAJRB4gAAUAaJAwBAGSQOAABlkDgAAJRB4gAAUAaJAwBAGSQOAABlkDgAAJT9/9cjAIAQOnLkSGFhIUKotbUVIfTBBx8ghBYuXLhx40aaIwPaBL47FrzD0tLy1q1bb968IUaMjY3XrVtHY0hAC8EZB3hHd3c3l8ttb28nRsaPH9/Y2GhsbExjVEDbwD0O8A4jI6O5c+cqjvz7v/87ZA3QDyQO0F9UVJS5uTm+bG5u/uWXX9IbD9BCcKkC+uvr67O0tPztt98QQubm5i0tLfr6cC8MvAPOOEB/+vr6QUFBDAaDwWB89tlnkDXAQJA4wCDWrVtnbm5ubm4eGRlJdyxAG8GlChgEhmGWlpYIoWfPnjEYDLrDAVqH1FnoixcvoqKiRjoUoFXMzMx6enqEQiHdgVDT1dWlr68/btw4ugPRVSKRyMLCYsjVSCWOrq6uhw8fHjt2TO2ogM6oqamJiYnZsmUL3YFQk5mZ6ezs/PHHH9MdiE6KjIzs6urSWOJACLHZbB8fH/WiArrEx8fnz3/+s879o1+8eHHKlCk6F7aWYLPZJNeEm6MAAMogcQDwjpSUlL1799IdBWU+Pj73798ftd1B4gDqsrOzKy8v1+ycAoFA43OS8fLlS5FIFB0dTYycOnVq6tSpHA4nMjJS8dk/hJBUKmWz2QEBAUNO6+LiwuFwXrx4gf944cKFGTNmaDbyr776KjExUbNzqgCJA6irrq5O438GdDlz5swnn3xiamqK/1hUVBQfH5+VldXU1OTr63vz5k3FlaOjo728vEjObGBgsG/fPg2HqyAoKKikpKS+vn7kdqEIEgdQl+IZB5fLPXjwoLOzM4fDWbZsWU9PDzG+d+9eR0fHCRMmrF+/vre3Fx/X19dvbm7Gl2NjYxMSEhBCISEhDQ0NM2fOZDAYR44cGc1juXr1quIzfklJSWlpaX5+fiYmJqtWrfr000+Jl/Ly8lgsluKIaps3bxaJRMTBEmpqavz9/TkcjqOjY15eHjGu7J189uyZUCi0sLCwtbXNyMgg1h83bpyXl5dYLKZ4xMMEiQNoWHFx8bVr1x4+fCiRSM6cOUOM5+bmFhcXV1ZWlpWV7d+/X8UM3377rZWV1Z07dzAM27hxY1RU1IoVK0Y+cIQQkkgkU6ZMwZd7enqqqqqam5u5XO7EiRM3bNjQ3d2Nv9Te3p6QkHDo0CHyMzs4OAQHB+/Zs0dxUC6Xf/7559OnT5fJZBkZGREREXfv3iVeHfhOYhgWHBxsaWlZU1MjFouzsrKuXLlCrO/o6Hjv3r1hHzslkDiAhiUlJVlbW/P5/EWLFj148IAYT0xMtLe3t7OzS05OPn/+PH0BqtLW1kZcpzQ0NGAYVlRUhCe7W7dupaWl4S/FxcVFRkYKBAJKkycnJ584cUImkxEjlZWVTU1NaWlpZmZmAQEBQqHw7NmzxKsD38n79+9XV1cfPnzY3NzcwcEhNjb24sWLxPocDqetrW3Yx04JJA6gYXi3QYSQsbGx4t1EW1tbfMHOzq6xsZH8hCKRSPHMZUSZm5u/evUKX8a7kGzbto3P59vb22/atOnq1asIobKystLS0tjYWKqTT548OTQ0dPfu3cRIY2OjQCAwMDAgVmhoaCBeHfhOSqXStrY2Q0NDJpOpp6cXHR2tmIY6OjrGjx9PNarhgQcfwSiRSqWenp4Iobq6Oj6fjw8aGRkR5/+tra1EAZKeHj3/pU2bNu3x48dz5sxBCPF4POJPFyGEYRgeVWVlpUQiIf7a0btHoVpCQoKjo6OdnR3+I5/Pr6+vf/v2LT7bkydPrKysVGyOn4DU19cP+gBRdXV1SEgImTDUB2ccYJSkpaXV1dVJpdLU1NSlS5fig66urjk5OV1dXWKxOD8/n1h50qRJEokEfwJzNO9xBAYG3rhxg/gxPDz8wIEDjY2NtbW1mZmZQUFBeDzYH9LT0+fNm0dkjSFDtba2joiIIG6OeHh4cLnchISE9vZ2sVicm5sbGhqqYnM3NzcbG5utW7e2tLS0trbm5OQcPXoUf6m3t7esrIz8zVo1QeIAoyQkJCQgIMDd3d3DwyMuLg4fzMzMzM/P5/F4IpFoyZIlxMrx8fGJiYlMJnOUP1VZvnz59evXOzs78R9TU1OdnJwcHBy8vb39/PyIsJWRyWQfffSR6nXi4+OJRMNkMgsLC8vLywUCQUxMzPHjxz08PFRsq6enV1BQ0NLS4urq6uzsXFpaSqTgK1eu+Pr6WltbkzpO9WEkPH361Nvbm8yaYCyxsrLS1FSWlpb4GcRI27x5c25urjozpKSkpKenD2PDvr4+BweH7u5udfY+bD4+PlVVVWpO4u3t/fTpUzJr0nyPg8vl/v3vf3dxcRnGttu2bdPX1x9YHaxsXBu8b8eri5KSkoa3IZPJrK6u1mww5JWWlo7m7sb4pYrGq6FdXFwYCvCLXu0xEtXfAAw0xj9Vqaur0/icp0+fHrV7dVSNxPFqxMCKSaDT1D3jUFYAy+Vyk5OTnZycTExM1q5dK5FIZs2axWKxAgMDOzo6FGcoKioaWImsbNpffvll1qxZZmZmgYGBLS0tQ44T/wMrK+B98OCBl5cXvmFYWNiOHTuG9z68b8cL3nNqJQ5MZQFsSUnJ9evX7969W1BQIBQKs7OzGxoaOjs7s7OzFScZWImsbFq5XB4cHDxv3jyZTLZmzZpz587hMygb72dgAS++4YIFC2Qy2fr16xXLGYfxEaBOHy8AlKh1qYIXwN68eZPJZJqbm+MFsAsWLMBfTUxM5PF4PB5v9uzZ06dPd3d3RwgtXry4oqJCcRK8EhkhlJycvHPnzoSEBGXTVlRU/Pbbb8nJyUwmc8mSJcTjzMrG+8ELeBFCRAFvRUUF/tABk8lcuHDh/PnzyRz1ypUrV65ciS/HxcURdyXH3vHK5fLbt2+TeU+0R3Nz8+PHj3UubC1BsowNqZk4iAJY4kMaPz8/4lWi6s7IyIjoYmhkZNSvqcHASmRl0+L1uUwmk1gfX1A23o9iAS/eGaGxsdHKyorYkIgEISQSiZQdtbJ7HDp9vIPq6emh9ByXNqiurq6trR21Z73GmJcvX5JcU63EoboAlqSBlcjKpuXz+Q0NDXK5HP/Vr6urwz/XVDY+JB6Pp7ihVCp1cnIa9oGQpEPHy2KxFJ+h0glbtmzx9vbWuebsWoJ8r1a17nGoKIAlb2AlsrJpPT09zczMUlNTOzo6CgoKrl27hs+gbHxInp6eHA7n66+/7ujouHz5Mv4IE27kypy183gBoEStxKGiAJa8gZXIyqZlMpmXLl0qLi62trY+duwYUdWvbHxI+vr63333XVFRkbW1tUgkWrx4MZnv41i5ciVRx4HfyBjbxwvAIMiUl74nJef+/v7Hjh2jO4rRM+TxarDkfNSoLjnftWvX8MrJxwZvb+979+6pXoFkyfkYrxwdUnZ2dklJyatXr06ePFlSUhIYGEh3RCOLruNVp/mwphoX624jYhUoHYIGGxq/74nj448/TkpK4vP5GRkZhYWFqrshjAHv2/Eq0t1GxMpQPQRNNjQmc1rynlyqgH5UXKr8+uuv+B+hg4OD4qUBk8lsamrCl2NiYnbu3IlhmOLz8n/9618tLS3T09MdHBzMzc0jIyN7enrIbztkzCouVebPn3/8+HHix+nTp588eXLQNXNzc7/44gu818aQe3R2dk5NTTU1NcWDP3/+vKenJ/6SsnfJ0tLywIEDTk5OpqamS5cuJR6obW5u/uKLLyZMmGBjY3Po0KEhdz2MQwgICFB8E/qBSxUwglS32B2oX/NhpEbjYqTGB1463Yh4oOEdgqYaGkPiAJSpbrFLBi2Ni3W6EfFAwzsETTU0hsQBKFPdYpeMYTcuRmr0LtbpRsQDDe8QNNXQeIw/Vg9GgooWuySbD9PSuFinGxEPNLxD0FRDYzjjAJSpaLFLpvkwUqNxMVLjHofuNiJWtmuqh6DBhsaQOABlKlrskmw+TEvjYt1tRKxs11QPQZMNjcl89AIfx76fRqhydEQbF6uuHNXFRsQa3PWQDY11plkxAKNJFxsRa3DXGmxoDJcqAADK4IwDjDZoXDwGwBkHAIAySBwAAMoY2B8fj6sgk8mcnJyG9/1jQEe9ffv26dOnf/rTn+gOhJrnz58bGRkRpeWAkgcPHvzyyy9kPq8llTjkcjnVumCg6/72t78hhNatW0d3IGBU8fl8op21CqRujjKZzNH7FmygHfAnGuDfHQwK7nEAACiDxAEAoAwSBwCAMkgcAADKIHEAACiDxAEAoAwSBwCAMkgcAADKIHEAACiDxAEAoAwSBwCAMkgcAADKIHEAACiDxAEAoAwSBwCAMkgcAADKIHEAACiDxAEAoAwSBwCAMkgcAADKIHEAACgj9fUI4P2RnJyclZWFf0k6QkhfX5/BYHz55ZcpKSl0hwa0CJxxgHfMnTu3r6+vtbW1ra2tra2ttbW1r6/vP/7jP+iOC2gXOOMA7/j9998nTZr04sULYsTCwuLZs2dkvqQHvD/gjAO8Q09PLyQkRE/vX78YDAYjJCQEsgboBxIH6G/t2rX417ghhMzNzeFbIMFAcKkCBsHn85uamhBCXC4XXwBAEZxxgEGEhYUZGBjo6+uvWrWK7liANoIzDjCIhw8ffvTRRwihkpISJycnusMBWgcSBxjc5MmTEUJPnjyhOxCgjbQ0cSxcuLCiooLuKOjR19cnl8sNDQ3pDePVq1cIIVNTU5Lrv3792sTEZCQjeh95enoWFhbSHcUg9OkOYHAtLS23b9+2tramOxAa5OXl/fOf/zx06BC9YchkMoQQ+X8CgUBQX18/khG9d2QymVAopDuKwWlp4gC0ez+zNiAJPlUBAFAGiQOMtpSUlL1799IdBW18fHzu379PdxTqgsShw+zs7MrLyzU4oUAg0OyEA718+VIkEkVHR+M/njp1aurUqRwOJzIy8s2bN4prSqVSNpsdEBAw5JwuLi4cDod4vubChQszZszQeOQqUDqKr776KjExcTTDGwmQOHRYXV3dKP+FqO/MmTOffPIJ/mFNUVFRfHx8VlZWU1OTr6/vzZs3FdeMjo728vIiOa2BgcG+ffs0Hy4JVI8iKCiopKRE128kQ+LQYcQZB5fLPXjwoLOzM4fDWbZsWU9PD74Cl8vdu3evo6PjhAkT1q9f39vbi4/r6+s3Nzfjy7GxsQkJCQihkJCQhoaGmTNnMhiMI0eOjFDMV69enTt3Lr6clJSUlpbm5+dnYmKyatWqTz/9lFgtLy+PxWIpjqi2efNmkUhEHJSimpoaf39/Dofj6OiYl5dHjCt70549eyYUCi0sLGxtbTMyMobcNdWjGDdunJeXl1gsJnlo2gkSxxhRXFx87dq1hw8fSiSSM2fOEOO5ubnFxcWVlZVlZWX79+9XMcO3335rZWV1584dDMM2btyIEIqKilqxYoVm45RIJFOmTEEI9fT0VFVVNTc3c7nciRMnbtiwobu7G1+nvb09ISGB0gfSDg4OwcHBe/bs6Tcul8s///zz6dOny2SyjIyMiIiIu3fvEq8OfNMwDAsODra0tKypqRGLxVlZWVeuXFGx3+EdhaOj471798gfnRaCxDFGJCUlWVtb8/n8RYsWPXjwgBhPTEy0t7e3s7NLTk4+f/48fQH+S1tbG36d0tDQgGFYUVERntRu3bqVlpaGrxMXFxcZGSkQCCjNnJycfOLECbz8hFBZWdnU1JSWlmZmZhYQECAUCs+ePUu8OvBNu3//fnV19eHDh83NzR0cHGJjYy9evKhip8M7Cg6H09bWRunotA0kjjHigw8+wBeMjY0V78/Z2triC3Z2do2NjZTmFIlEiicvGmFubo7XpBobGyOEtm3bxufz7e3tN23adPXqVYRQWVlZaWlpbGws1ZknT54cGhq6e/duxcHGxkaBQGBgYECs09DQQLw68E2TSqVtbW2GhoZMJlNPTy86OrpfJupneEfR0dFBNC7QUVAANsZJpVJPT0+EUF1dHZ/PxweNjIyIM+rW1lY2m40vE/17Rs60adMeP348Z84cHo9H/N0ihDAMw/deWVkpkUiIP/V+0aqWkJDg6OhoZ2dHjPD5/Pr6+rdv3+ITPnnyxMrKSsUM+AlIfX09g8Egs8fhHUV1dXVISAiZ+bUWnHGMcWlpaXV1dVKpNDU1denSpfigq6trTk5OV1eXWCzOz88nVp40aZJEIiEeXxqJexyBgYE3btzAl8PDww8cONDY2FhbW5uZmRkUFITvFPtDenr6vHnziL+3IeOxtraOiIhQvK3g4eHB5XITEhLa29vFYnFubm5oaKiKGdzc3GxsbLZu3drS0tLa2pqTk3P06FHVe6d6FL29vWVlZeTv+2onSBxjXEhISEBAgLu7u4eHR1xcHD6YmZmZn5/P4/FEItGSJUuIlePj4xMTE5lM5sh9qrJ8+fLr1693dnYihFJTU52cnBwcHLy9vf38/IjwlJHJZPjD/irEx8crnp4wmczCwsLy8nKBQBATE3P8+HEPDw8Vm+vp6RUUFLS0tLi6ujo7O5eWlhLZVtneqR7FlStXfH19db6iH9NK3t7eT58+HfQlW1tb/M7/WJWbm7t582aNTGVpaYmfQYwCKysrkmumpKSkp6dTnb+vr8/BwaG7u5vqhhqhwb37+PhUVVWRWfPp06fe3t7q73EkjLUzDg3WPrq4uAy8NXjnzp25c+dyOB5p5lEAACAASURBVJzJkyfv3Lmzs7Ozu7ubMRh8WxcXFz09vdraWmIGX19fBoNRVVWlkSB1UVJS0o4dO6huxWQyq6ur6eo2oMG9l5aWurm5qT8PvcZa4hhRPT09CxYs+Pjjj2Uy2T/+8Q8zM7P8/HwjIyMiDVtYWPzwww/4MnE97OjoePLkSXz50aNHLS0ttPfaAEBNOpw4du3axeVyLSwsoqKi8M/S+tU+crnc5ORkJycnExOTtWvXSiSSWbNmsViswMDAjo6OYeyxtrb2+fPn27dvNzMzEwgE27dvDwsLG3KrVatWnTp1CsMwhFBOTs7q1auHsevhaW5udnFxGbXdgfeHriaOioqKY8eO3bx588mTJ56envgDAgNrH0tKSq5fv3737t2CggKhUJidnd3Q0NDZ2ZmdnU1MRf6zA1tbW0tLyz//+c+PHj3CSHdOmzx5sq2t7Y8//iiXy8+ePQvtf8EYoKt1HAYGBoaGhqampmZmZiq++CMxMZHH4/F4vNmzZ0+fPt3d3R0htHjx4uH1JTQ2Nv7pp5927949Z84cuVy+ZMmS9PR0CwuLITcMDw8/ceLEmzdvXFxcqBZEAqCFdDVxTJs2LSUlZenSpX19ffPnz9+yZQuLxRq4GlGcY2RkRPyFGxkZKdZWikQi8vt1cHA4d+4cQqimpiYyMnLNmjUFBQVDbhUSErJ9+/Znz56R/HIjiURCe+tAql6/fq1zMWu5trY2kpVvo09XL1UQQmFhYTdu3BCLxVVVVampqfjgKNQ+4j788MMNGzb885//JLOyiYlJUFBQZWXlwoULRzowAEaBrp5xXL58ubq6euXKlSwWC8Owrq4ufByvffT09CRZMowQioqK6uzsJPNQRnV19Z49ezZt2uTs7NzU1JSVlUW+HcY333zzzTffkFzZ1dV1y5YtJFfWEocOHdK5mLWcTCb74Ycf6I5icLp6xuHv79/R0TFz5kw7OzsMw4iWSpqtfVy5ciVRl7F69eqpU6cGBgZu2rRp4sSJs2fP5vF4x48fV38vAOgcLf1eFR8fn7y8PJ0vyx0WLfl6BKrg6xE0Dv96hNu3b9MdyCB09YwD6Jb3vEGxMrrbuBgSx1imZgG+pur3x16D4qCgIOIaVrGzxqJFi/DBqKgoxfUvXbo0a9YsNpvt5uZ2+fJlYlx3GxdD4gAjbuw1KEYInThxAn+2QLGXV0FBAYZhRIrEtbe3nz59+siRI83NzVu3bhUKhTU1NfhLutu4GBKHLlHWd5dk82G6ehePvQbFlODPNM2cOZPNZoeFhdnY2BD1h7rbuBgSh85Q3Xd3oIEF+Iim3sVjrEExLjY2lsViubu7U/pS6KamptraWsUHiHS0cTEkDp2huu8uSbT0Lh5jDYoRQkVFRW1tbS0tLTExMUKhsLKykky0vb29K1asiIiIcHZ2JgZ1tHExJA6dobrvLkm09C4eYw2KCWw2Ozw8PCAggMxJR19f37Jly1gsVr/rPh1tXKyrlaPvIRV9d8k3H6ald/EYa1A80JAbyuXy5cuXd3Z2fv/994qHiXS2cTGccegMFX13STYfRjT1Lh5jDYpfvHgRGRn566+/dnZ2njp1SiwWf/bZZyrm//3338PDw1taWgoLC/v1cNLdxsWQOHSGir675JsP09K7eIw1KLawsPD19V24cCGXy83MzMzNzcVP4hBCCQkJDAbj6NGj2dnZDAYD75D49OnT06dP//TTTywWC6/yOHz4ML6+DjcuHtGOpsOmolnxmKfBZsX9jGjvYtXNit/zBsXKqG5crM3NiuEeBxgNSUlJw9gKbxGs8WC0Z++lpaUjOv/IgUsVAABlcMbxHhm0yBKAYYAzDgAAZZA4AACUaemlyqRJk3x8fOiOgh59fX1yuVzx0Sxa4I/AjRs3juT6r1+/hgbuGkd80KtttLQDGKAdXlIFbUTBoOBSBQBAGSQOAABlkDgAAJRB4gAAUAaJAwBAGSQOAABlkDgAAJRB4gAAUAaJAwBAGSQOAABlkDgAAJRB4gAAUAaJAwBAGSQOAABlkDgAAJRB4gAAUAaJAwBAGSQOAABlkDgAAJRB4gAAUAaJAwBAmZZ+PQKgS2dn59u3bxFCb968QQi9fPkSIWRgYMBms2mODGgT+HoE8I4dO3b813/9l7Gx8e+//44Q0tPTe/PmTUxMzN69e+kODWgRSBzgHb/88sucOXN+++03YmTChAk3btxwcXGhMSqgbeAeB3iHk5OTiYmJ4gibzYasAfqBxAH6i4iIIL750cDAICIigt54gBaCSxXQn1QqnT59On5b1NzcvKKiwt7enu6ggHaBMw7Qn62traWlJb5saWkJWQMMBIkDDGL9+vXGxsZGRkbr16+nOxagjeBSBQzi2bNnU6dORQg9evSIOPsAgEB/AVhTU1NfXx/dUYD+bGxsGAxGb2+vTCajOxbwDn19fR6PR28M9J9xCAQCa2tremPQft3d3S9fvhzNX5fnz58jhCZOnKjOJDKZjMfj6evT///TWCKTyerr6+mNQSv+RW/fvk13CNru9u3bhw4dunjx4qjtsb29HSFkZmamziQ+Pj55eXnwH4NmCQQCukPQjsQBtJCaKQOMbfCpCgCAMkgcgDYpKSnw7NxAPj4+9+/fpzuKIUDiGGvs7OzKy8s1O6dAIND4nC9fvhSJRNHR0fiPp06dmjp1KofDiYyMxJ/oJ0ilUjabHRAQMOScLi4uHA7nxYsX+I8XLlyYMWOGZsNWISgoiPGH8ePHE+OLFi3CB6OiohTXv3Tp0qxZs9hstpub2+XLl4nxr776KjExcdTCHh5IHGNNXV3daP61DNuZM2c++eQTU1NThFBRUVF8fHxWVlZTU5Ovr+/NmzcV14yOjvby8iI5rYGBwb59+zQfLjknTpzAMAzDsLa2NmKwoKAAwzAiReLa29tPnz595MiR5ubmrVu3CoXCmpoa/KWgoKCSkhLaPzdRDRLHWKN4xsHlcg8ePOjs7MzhcJYtW9bT00OM792719HRccKECevXr+/t7cXH9fX1m5ub8eXY2NiEhASEUEhISENDw8yZMxkMxpEjRzQV59WrV+fOnYsvJyUlpaWl+fn5mZiYrFq16tNPPyVWy8vLY7FYiiOqbd68WSQSEUehqKamxt/fn8PhODo65uXlEePK3qVnz54JhUILCwtbW9uMjIzhHaYyZmZm+fn5M2fOZLPZYWFhNjY2FRUV+Evjxo3z8vISi8Wa3aNmQeIY44qLi69du/bw4UOJRHLmzBliPDc3t7i4uLKysqysbP/+/Spm+Pbbb62srO7cuYNh2MaNG6OiolasWKF+YBKJZMqUKQihnp6eqqqq5uZmLpc7ceLEDRs2dHd34+u0t7cnJCQcOnSI/LQODg7BwcF79uzpNy6Xyz///PPp06fLZLKMjIyIiIi7d+8Srw58lzAMCw4OtrS0rKmpEYvFWVlZV65cGXLvsbGxLBbL3d29sLCQfMxNTU21tbWKvQscHR3v3btHfobRB4ljjEtKSrK2tubz+YsWLXrw4AExnpiYaG9vb2dnl5ycfP78+dEPrK2tDb9OaWhowDCsqKgIz2K3bt1KS0vD14mLi4uMjKRatpCcnHzixIl+Ba+VlZVNTU1paWlmZmYBAQFCofDs2bPEqwPfpfv371dXVx8+fNjc3NzBwSE2NnbIIpqioqK2traWlpaYmBihUFhZWUkm2t7e3hUrVkRERDg7OxODHA5H8WJHC0HiGOM++OADfMHY2FjxpqOtrS2+YGdn19jYSH5CkUikeOYybObm5q9evcIDQwht27aNz+fb29tv2rTp6tWrCKGysrLS0tLY2FiqM0+ePDk0NHT37t2Kg42NjQKBwMDAgFinoaGBeHXguySVStva2gwNDZlMpp6eXnR0NMnSezabHR4eHhAQQOako6+vb9myZSwWq981YEdHh+LtVS0EBWDvKalU6unpiRCqq6vj8/n4oJGREXGZ0NraSjQo1tPT/H8w06ZNe/z48Zw5c3g8HvF3ixDCMAzfXWVlpUQiIf7U+4WnWkJCgqOjo52dHTHC5/Pr6+vfvn2LT/jkyRMrKysVM+AnIPX19QwGg+KR/cuQG8rl8uXLl3d2dn7//feKh4kQqq6uDgkJGd5+Rweccbyn0tLS6urqpFJpamrq0qVL8UFXV9ecnJyuri6xWJyfn0+sPGnSJIlEgj/WpKl7HIGBgTdu3MCXw8PDDxw40NjYWFtbm5mZGRQUhO8I+0N6evq8efOIrDFkDNbW1hEREYo3Rzw8PLhcbkJCQnt7u1gszs3NDQ0NVTGDm5ubjY3N1q1bW1paWltbc3Jyjh49qmLvL168iIyM/PXXXzs7O0+dOiUWiz/77DMV8//+++/h4eEtLS2FhYWGhoaKL/X29paVlZG/H0wLSBzvqZCQkICAAHd3dw8Pj7i4OHwwMzMzPz+fx+OJRKIlS5YQK8fHxycmJjKZTA1+qrJ8+fLr1693dnYihFJTU52cnBwcHLy9vf38/Ih4lJHJZB999JHqdeLj4xVPT5hMZmFhYXl5uUAgiImJOX78uIeHh4rN9fT0CgoKWlpaXF1dnZ2dS0tLifQ66N4tLCx8fX0XLlzI5XIzMzNzc3PxEzqEUEJCAoPBOHr0aHZ2NoPB2LFjB0Lo6dOnp0+f/umnn1gsFl7lcfjwYXz9K1eu+Pr6avsDPhjdrKyslL1ka2uL38wHpaWlISEhmprN0tISP4MYad7e3k+fPlX2akpKSnp6OtU5+/r6HBwcuru71QttmEZh7z4+PlVVVSpWUPEnM2p0+IxDU+WMygr7lHFxcRl4d/DOnTtz587lcDiTJ0/euXNnZ2dnd3c3YzD4ti4uLnp6erW1tcQMvr6+DAajqqpK/SPSFUlJSfh/v5Qwmczq6up+p/ejZhT2Xlpa6ubmNnLza4QOJw5NGbSwj5Kenp4FCxZ8/PHHMpnsH//4B17bY2RkRKRnCwuLH374AV8mLo8dHR1PnjyJLz969KilpYWuPwYAqNKNxLFr1y4ul2thYREVFYV/WtavnJHL5SYnJ+PfCbJ27VqJRDJr1iwWixUYGNjR0THS4dXW1j5//nz79u1mZmYCgWD79u1hYWFDbrVq1apTp05hGIYQysnJWb169UjHSWhuboavSgHq0IHEUVFRcezYsZs3bz558sTT0xN/kKFfOSNCqKSk5Pr163fv3i0oKBAKhdnZ2Q0NDZ2dndnZ2cPbL/mPD/C24H/+858fPXqEke6oNnnyZFtb2x9//FEul589e3bVqlXDixOA0acDdRwGBgaGhoampqZmZmbr1q1TtlpiYiKPx+PxeLNnz54+fbq7uztCaPHixcQjACPH2Nj4p59+2r1795w5c+Ry+ZIlS9LT0y0sLIbcMDw8/MSJE2/evHFxcdGGtk4AkKQDiWPatGkpKSlLly7t6+ubP3/+li1bWCzWwNWIIiIjIyPij9bIyKjfM9rkiUQi8is7ODicO3cOIVRTUxMZGblmzZqCgoIhtwoJCdm+ffuzZ89UJETCzz///MUXX5APSRvU1dVFRUUN+u8Fho1kFdyI0oHEgRAKCwsLCwvr7OxcvXp1ampqeno6GplyRvV9+OGHGzZswK+ehmRiYhIUFFRQULBw4cIhV7a2tt6yZYvaAY6qR48erVu3Dr5gQbNKSkroDkEXEsfly5erq6tXrlzJYrEwDOvq6sLH8XJGT0/PYRcFqxYVFdXZ2UnmuYzq6uo9e/Zs2rTJ2dm5qakpKyuLfEeMb7755ptvviGzJofD8fHxITmtljAxMfH09NT2WiZdw2Qy6Q5BF26O+vv7d3R0zJw5087ODsMwojmSpsoZBy3sU23lypVEXcbq1aunTp0aGBi4adOmiRMnzp49m8fjHT9+XJ2QANB2o15y1p82lMFpP81Wjo4a1ZWju3btGkbl6Jjn7e197949FStow5+MDpxxAM1Ss+JWUwW70HMUeo7qgLq6uoHV33V1dXTH9f6CnqPQc1QH4PdH+lHs16C7lLXSJNlAlK7+o9BzFHqOAtqobqU50MCKW6RG/1GkRnsO6DlKgJ6jYLSpbqVJEi39R6HnKA56jgIaqG6lSdKw+48iNVqQQs9RBD1HAV1UtNIk30CUlv6j0HMUeo4C2qhopUmmgShu2P1HkRr3OKDnqE73HIUCMN2gogDs0aNHfn5+bDZ76tSp586dI8bLysrwTwoWLVq0YsWKnTt34uPfffedlZUVg8H461//imGYpaXlnj17pk6dOn78+LVr1xJN8Uhuvn79+uXLlysLW0UBWFtbG5/Pf/XqFYZh3d3d69atMzU1nTRpUkxMzMDGfHjiIH4MDAw8evTowDmdnZ0vXryILzc1NbFYLE9PzyHfKMVGiqmpqevXr8eXm5ubly9fPmnSpEmTJq1Zs+b58+eq937y5ElHR0e8yh7/CBa3c+dOxb+4uLg4DMMUm7/hMjIy8PXz8/M/++yzQd80nDb8yUDi0A0jVzk6ov1HoefoMOhEz1G4xwFok5SUNIyt8K6fGg9Ge/ZeWlo6ovNrBNzjAABQBmcc77tBiywBUA3OOAAAlEHiAABQphWXKrdv36Y7BG334MGDFy9e6Nwb9erVq4qKCi1/0BMMAwMj3c5/hERFRRENFIAyfX19b968wR/uGB1NTU0IIR6Pp84kHR0dbDZbO7vD6i4LCwtKnbRHAv2JA2gnvOxS59ojg9EB/xUAACiDxAEAoAwSBwCAMkgcAADKIHEAACiDxAEAoAwSBwCAMkgcAADKIHEAACiDxAEAoAwSBwCAMkgcAADKIHEAACiDxAEAoAwSBwCAMkgcAADKIHEAACiDxAEAoAwSBwCAMkgcAADKIHEAACjTiu9VAdqjtLS0oaEBIXTv3j2E0MWLFxFCVlZWH330Ec2RAW0CiQO849atW0lJSePGjcN/LCgo6O3t3b17NyQOoAi+VwW8o7Gx0dnZua2tjRgxNzeXSCRWVlY0RgW0DdzjAO/g8/k2NjaKIzY2NpA1QD+QOEB/X375pYmJCb7MYrG+/PJLeuMBWgguVUB/L1++tLe3b29vRwiNHz++pqbGwsKC7qCAdoEzDtCfubm5i4sLvuzi4gJZAwwEiQMMYsOGDaampmw2e8OGDXTHArQRXKqAQbx+/Rq/IdrQ0EDc7wCAQGcdx+3bt2UyGY0BABUmT56MELpy5QrdgYDBWVtb+/j40LV3Os84vvjiC319fR6PR1cAOkcikSCEXF1dR2FfT548QX+kDzWdO3cuNDRU/XkAoampqa+vD6/rpQXNlaObNm2iMWvqnEOHDiGEtmzZMgr76u3tRQgRJaTqyMvLwyMHmnL79m1631IoOQeD00jKAGMVfKoCAKAMEgegWUpKyt69e+mOQuv4+Pjcv3+f7iiUgsQxNtnZ2ZWXl2twQoFAoNkJcS9fvhSJRNHR0fiPp06dmjp1KofDiYyMfPPmjeKaUqmUzWYHBAQMOaeLiwuHw3nx4gX+44ULF2bMmKHxyJUJCgpi/GH8+PHE+KJFi/DBqKgoxfUvXbo0a9YsNpvt5uZ2+fJlYvyrr75KTEwctbCpgsQxNtXV1Y3mX8uwnTlz5pNPPjE1NUUIFRUVxcfHZ2VlNTU1+fr63rx5U3HN6OhoLy8vktMaGBjs27dP8+GSc+LECQzDMAxTfMi4oKAAwzAiReLa29tPnz595MiR5ubmrVu3CoXCmpoa/KWgoKCSkpL6+vpRDZ00SBxjE3HGweVyDx486OzszOFwli1b1tPTg6/A5XL37t3r6Og4YcKE9evX45+hIIT09fWbm5vx5djY2ISEBIRQSEhIQ0PDzJkzGQzGkSNHNBjn1atX586diy8nJSWlpaX5+fmZmJisWrXq008/JVbLy8tjsViKI6pt3rxZJBIRB6KopqbG39+fw+E4Ojrm5eUR48reqGfPngmFQgsLC1tb24yMjOEdpjJmZmb5+fkzZ85ks9lhYWE2NjYVFRX4S+PGjfPy8hKLxZrdo6ZA4hj7iouLr1279vDhQ4lEcubMGWI8Nze3uLi4srKyrKxs//79Kmb49ttvrays7ty5g2HYxo0bEUJRUVErVqxQPzaJRDJlyhSEUE9PT1VVVXNzM5fLnThx4oYNG7q7u/F12tvbExISKH366ODgEBwcvGfPnn7jcrn8888/nz59ukwmy8jIiIiIuHv3LvHqwDcKw7Dg4GBLS8uamhqxWJyVlUWmIi42NpbFYrm7uxcWFpKPuampqba2lnhKCCHk6OiI92HTQpA4xr6kpCRra2s+n79o0aIHDx4Q44mJifb29nZ2dsnJyefPn6cltra2Nvw6paGhAcOwoqIiPJHdunUrLS0NXycuLi4yMlIgEFCaOTk5+cSJE/1KkysrK5uamtLS0szMzAICAoRC4dmzZ4lXB75R9+/fr66uPnz4sLm5uYODQ2xs7JA1V0VFRW1tbS0tLTExMUKhsLKykky0vb29K1asiIiIcHZ2JgY5HI7ixY5WgcQx9n3wwQf4grGxseIdR1tbW3zBzs6usbGR0pwikUjx5GXYzM3NX716hceGENq2bRufz7e3t9+0adPVq1cRQmVlZaWlpbGxsVRnnjx5cmho6O7duxUHGxsbBQKBgYEBsQ7eYBU38I2SSqVtbW2GhoZMJlNPTy86OprkQxJsNjs8PDwgIIDMSUdfX9+yZctYLFa/y8COjg7F26taBQrA3l9SqdTT0xMhVFdXx+fz8UEjIyPiGqG1tZXNZuPLenoj8n/MtGnTHj9+PGfOHB6PR/zdIoQwDMP3WFlZKZFIiD/1fhGqlpCQ4OjoaGdnR4zw+fz6+vq3b9/iEz558kR1czP8BKS+vp7BYFA8sn8ZckO5XL58+fLOzs7vv/9e8TARQtXV1SEhIcPb70iDM473V1paWl1dnVQqTU1NXbp0KT7o6uqak5PT1dUlFovz8/OJlSdNmiSRSIgnmzR1jyMwMPDGjRv4cnh4+IEDBxobG2trazMzM4OCgvAdYX9IT0+fN28ekTWGjMHa2joiIkLx5oiHhweXy01ISGhvbxeLxbm5uaofonFzc7Oxsdm6dWtLS0tra2tOTs7Ro0dV7P3FixeRkZG//vprZ2fnqVOnxGLxZ599pmL+33//PTw8vKWlpbCw0NDQUPGl3t7esrIy8veDRxkkjvdXSEhIQECAu7u7h4dHXFwcPpiZmZmfn8/j8UQi0ZIlS4iV4+PjExMTmUymZj9VWb58+fXr1zs7OxFCqampTk5ODg4O3t7efn5+REjKyGSyIXuvx8fHK56eMJnMwsLC8vJygUAQExNz/PhxDw8PFZvr6ekVFBS0tLS4uro6OzuXlpYSGXbQvVtYWPj6+i5cuJDL5WZmZubm5uLndAihhIQEBoNx9OjR7OxsBoOxY8cOhNDTp09Pnz79008/sVgsvMrj8OHD+PpXrlzx9fW1trZWfYC0wegTEhJSWlo65Gpbt26Ni4sbhXi031/+8pe//OUvGpnK0tISP4MYBVZWVipeTUlJSU9PpzpnX1+fg4NDd3e3GnEN3yjs3cfHp6qqStmrpaWlISEhI7f3IcE9jqG5uLj8/PPPDAbD1NTU0dExNDQ0OjqayWTSHdcYkZSUNIytmExmdXW1xoPRnr2XlpaO6PxqgksVUk6fPv37778/fPhw+/bthw8f/s///E+6IwKATlqaOH755ZdZs2aZmZkFBga2tLQQ48rK+JSV/e3atYvL5VpYWERFRRGfRA67FpDH4y1evDg/P/+77767c+eOpkIa0dpEZZqbmxVrjQCgRBsTh1wuDw4OnjdvnkwmW7Nmzblz5/BxTGUZ38Cyv4qKimPHjt28efPJkyeenp74sw8qJiH5SYG7u/uf/vSnIWcjGZLqGQDQTtp4j6OiouK3335LTk5mMplLliwhHojEy/hu3rzJZDLNzc3xMr4FCxbgr+JlfwghouzPwMDA0NDQ1NTUzMxs3bp1ZCYhic/nv3z5UiMh3bt3T/14ABhl2pg48PI+4u4jUcBDlPERt3b9/PyIrRTL/vBHqqdNm5aSkrJ06dK+vr758+dv2bKFxWKpmEQkEpGMsKGhYcKECRoJSfUMA+3evVvn2vC9ePGCasE4UK23t9fNzY3GALQxcfD5/IaGBrlcjueOuro6/Gp8GGV8YWFhYWFhnZ2dq1evTk1NTU9PV78WsKqq6smTJ//2b/+mkZCEQiGlGZKSkkan56gGCQQCrX08XEfR3nNUG+9xeHp6mpmZpaamdnR0FBQUXLt2DR9XUcY3qMuXL+/bt6+xsbGvrw/DsK6uLtWTDHmPo7m5OT8/f/HixUKhEC/sUT8kqjMAoA20MXEwmcxLly4VFxdbW1sfO3aMKApWUcY3KH9//46OjpkzZ9rZ2WEYhvdTojoJbuXKlQwGY8qUKfv379+8eTPxfJf6IQ0vHgBoNrL1ZSqRrBwFBA1Wjo4m1ZWjinbt2jWMKtIxw9vb+969e2TWpL1yVBvPOMAoULOH6Ei0IB17/UeRkqNQ1pdUy/uMKoLEAbTF2Os/quIoBu1LquV9RhVB4hgjlLXSJNlDVBtakI69/qMqjmJQWt5nVBEkjrFAdSvNgQb2EEVqtCCF/qODUnEUSHlfUm3uM6oIEsdYoLqVJkm0tyAdY/1HVRyFir6k2txnVBEkjrFAdStNkobdghT6jw5K2VEQBu1Lqs19RhVpY+UooEpFK03yPURpb0E6xvqPKjuKgRQn1OY+o4rgjGMsUNFKk2QPUaRGC1LoP6ps74MehYq+pFreZ1QRJI6xQEUrTfI9RGlvQTrG+o8qOwoVfUm1vc+oolEtN3sXVI5SNXKVoyPagpR85eh73n9UdZ9RRbRXjsI9DqBF3vP+o1reZ1QRXKoAACiDMw6AEEKDFlYCoAyccQAAKIPEAQCgjOZLlQsXLty+fZveGHQIUeagWzo7O3WuT6qWk0ql9AbAB1gukAAAADBJREFUwBRKgEZZUVHR48eP6do7ADptypQpeEUZLehMHAAAHQX3OAAAlEHiAABQ9n+YT7WNIPVZzAAAAABJRU5ErkJggg==\n","text/plain":"<IPython.core.display.Image object>"},"metadata":{}}]},{"cell_type":"markdown","source":"# Train the Model","metadata":{}},{"cell_type":"code","source":"# test whether model performs good\n\nfor example_inp, example_tar in data.take(1):\n    example_pred = model(example_inp)\n    print(example_tar.numpy().shape)\n    print(example_pred.shape)\n    ","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:25.580652Z","iopub.execute_input":"2021-05-27T13:53:25.580968Z","iopub.status.idle":"2021-05-27T13:53:26.876155Z","shell.execute_reply.started":"2021-05-27T13:53:25.580935Z","shell.execute_reply":"2021-05-27T13:53:26.874609Z"},"trusted":true},"execution_count":19,"outputs":[{"name":"stdout","text":"(64, 99)\n(64, 99, 65)\n","output_type":"stream"}]},{"cell_type":"code","source":"ids = tf.random.categorical(example_pred[0], num_samples=1)\nids.shape","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:26.877931Z","iopub.execute_input":"2021-05-27T13:53:26.878251Z","iopub.status.idle":"2021-05-27T13:53:26.889712Z","shell.execute_reply.started":"2021-05-27T13:53:26.878220Z","shell.execute_reply":"2021-05-27T13:53:26.888433Z"},"trusted":true},"execution_count":20,"outputs":[{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"TensorShape([99, 1])"},"metadata":{}}]},{"cell_type":"code","source":"ids[0][-1].numpy()","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:26.891215Z","iopub.execute_input":"2021-05-27T13:53:26.891863Z","iopub.status.idle":"2021-05-27T13:53:26.906621Z","shell.execute_reply.started":"2021-05-27T13:53:26.891819Z","shell.execute_reply":"2021-05-27T13:53:26.905458Z"},"trusted":true},"execution_count":21,"outputs":[{"execution_count":21,"output_type":"execute_result","data":{"text/plain":"27"},"metadata":{}}]},{"cell_type":"code","source":"# callback to save checkpoints\ncheckpoint_path = os.path.join(\"./checkpoints\", \"ckpt_{epoch}\")\ncheckpoint_callback = keras.callbacks.ModelCheckpoint(filepath=checkpoint_path, save_weights_only=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:26.908178Z","iopub.execute_input":"2021-05-27T13:53:26.908612Z","iopub.status.idle":"2021-05-27T13:53:26.915951Z","shell.execute_reply.started":"2021-05-27T13:53:26.908574Z","shell.execute_reply":"2021-05-27T13:53:26.914872Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam', \n              loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True))\n\nhistory = model.fit(data, \n                    epochs=10, \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    callbacks=[checkpoint_callback])","metadata":{"execution":{"iopub.status.busy":"2021-05-27T13:53:26.919791Z","iopub.execute_input":"2021-05-27T13:53:26.920918Z","iopub.status.idle":"2021-05-27T14:52:49.026604Z","shell.execute_reply.started":"2021-05-27T13:53:26.920866Z","shell.execute_reply":"2021-05-27T14:52:49.025185Z"},"trusted":true},"execution_count":23,"outputs":[{"name":"stdout","text":"Epoch 1/10\n174/174 [==============================] - 364s 2s/step - loss: 3.4031\nEpoch 2/10\n174/174 [==============================] - 355s 2s/step - loss: 2.4450\nEpoch 3/10\n174/174 [==============================] - 357s 2s/step - loss: 2.1154\nEpoch 4/10\n174/174 [==============================] - 357s 2s/step - loss: 1.9102\nEpoch 5/10\n174/174 [==============================] - 354s 2s/step - loss: 1.7560\nEpoch 6/10\n174/174 [==============================] - 355s 2s/step - loss: 1.6403\nEpoch 7/10\n174/174 [==============================] - 356s 2s/step - loss: 1.5536\nEpoch 8/10\n174/174 [==============================] - 353s 2s/step - loss: 1.4871\nEpoch 9/10\n174/174 [==============================] - 357s 2s/step - loss: 1.4351\nEpoch 10/10\n174/174 [==============================] - 353s 2s/step - loss: 1.3939\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Performance Evaluation","metadata":{}},{"cell_type":"code","source":"plt.plot(history.history['loss'], '+-r')\nplt.title('Performance Analysis', size=16, color='green')\nplt.xlabel('Epochs', size=14, color='blue')\nplt.ylabel('Loss', size=14, color='blue')\nplt.xticks(range(10))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-27T14:52:49.028732Z","iopub.execute_input":"2021-05-27T14:52:49.029070Z","iopub.status.idle":"2021-05-27T14:52:49.236134Z","shell.execute_reply.started":"2021-05-27T14:52:49.029039Z","shell.execute_reply":"2021-05-27T14:52:49.235228Z"},"trusted":true},"execution_count":24,"outputs":[{"output_type":"display_data","data":{"text/plain":"<Figure size 432x288 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAYwAAAEcCAYAAADUX4MJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAtMElEQVR4nO3deZhT9dn/8ffNsAkii4yogEAtLrghnSIWNxQX3FBxQStMtUqtYl2o1vZpDVGfX+vTVu2i1gWrVsQFpKBSVKiKuwwoCoIWxQURmYqAuIDA/fvje8YJcTJkZjI5mZnP67pyJTnn5Jw7Ycid727ujoiIyJY0izsAERFpGJQwREQkK0oYIiKSFSUMERHJihKGiIhkRQlDRESy0jzuAKSwWdJ+BPw9ZdNa4B3gNuBvnvANObpOW+BvwJFAMfAnT/jFuTh3U2RJuw04B7jBE35JHq73I8LfSS9P+Ls5PG9PYAlwlif8zlydV2pHJQzJ1inA/sAw4GXgL8CVOTz/BcDpwM+j61yfw3M3KZa0rYBTo6dnWNIa8g/Djwh/D4/GHYiohCHZe9UTvjh6/Lgl7bvARdQxaVjSWnnC1wG7A8s84XfXMc708zZFJwDbANOAo4GjgEfiDKi2on/DF+OOQwIlDKmt2cAhlrTtPOErLGn7AFcDBwKtgbnAFZ7wZypeYEm7ExhMKK38EdgXuNWS9rOUYyqmHhjkCX/KkrYr8DtgENAKmAeM9YRPT3nNWCAB7BWddyAwExgane9/gU+BnxGqu54Ezope/ldCNdga4K+e8GtTzlsMXBNduxvwCfAMcJkn/MMqrr8LcANwcHTsOOAaT/imtHMmgeOA7YAVwFPAORUJLpvPcgtKo/f7I+C96PlmCSPbmC1prYHfAocDPQlVkrOjz2BRpgAsaQ8D3Tzh+6Zt7wW8DZzvCf+bJW174Nro/NsCK4E5wNnR31VP0qqkLGnfB/4f0A9oQyiFTPeEn5/l5yO1pCopqa1ewEZgrSWtH/A80Ak4l1Bt9Qkww5L2vbTXtQfuAyYAQ4B7CVUOjwHLo8f7A3MtaTsCzwL7AKMJ1SyrgEctaUOqiGkK8DRwPJtXaY0ADgXOj85zIHA3MBl4LYp3GvA7S9rRKa/rBHwF/JLwK/0yoDfwXPRFmm4y8G/CL/x/EhJDacVOS1rH6HM6DbiO8Ov/cqAF0DI6piaf5bdEn9lg4H5PeHkUx3HRtatSbcyEJN2OkDiPAX5KSGIvRF/2mdwM9LWk9U/bPgr4HBgfPf8H4d/7MkLS+BmwlJAIqnp/WxP+VjYSEuIQ4Cr04zcv9CFLtoqiuvB2hC/uk4CHPeFfWNJ+D7wPHOoJXw9gSXsMmA/8hvBlVGFr4ExP+JTUk1vS/gus84S/mLLtSqAjsH9FdZglbRrwBqHU8K+0GP/sCf9TFbGvA4ZWNNBb0vYELgF+4wm/Jtr2FHAiofQzDcAT/iah2q0iniLguei9DiF82ab6oye8ooPADEvaoYR2mYptlwDfAUo84a+kvG5CyuOafJZVORMoIiREgLuiGE4jdCpIV23MnvDVhMZzoliKCF/YH0fHZWprmk7oHPETQpsXlrQWhJLdeE/4Z9Fx+wO/8oSPT3ntg9W8v90IfxOXe8JfS9l+ZzWvkRxRCUOytQj4mlBlcBPhF+LZUQPrwYT/5Jssac2jxGLADOCgtPN8Tfb16QcBL6a0neAJ30j4gu1rSdsm7fj0L/AKT6T15qqoSnks5bwbgMVA99QXWtJ+akmbZ0lbC2wgfJkD7FrFddIbZucDO6U8PwKYnZYsUq9V08+yKqXAfzzhL0TPZwDL2LzUUJOYsaSdakl7yZK2ivAZfE5I/FV9BgBEVVq3AMMtae2jzScAXaLtFWYDl1nSLrKk7WVJs+rfHv8hlDJvsaSdaUnrvoXjJYeUMCRbJwLfJ/zCa+sJH+kJX0moOiki/Pr9Ou02GuhoSUv9OyuPvvSz0YlQP51uOeFLNL2apapjIdTnp1pfzfZvqposaRcSkuMMQomqPzAg2l1VldTKtOfr0o7bllDdkklNP8vNWNJKgD7AQ5a0Dpa0DoQS4UPAAEvaLjWN2ZJ2HHA/sBA4A9iP8HdQTtWfQapx0fsZET0/D3g5LWGeBkwlVM29BnxoSbsy0/uMSjyDCEnwJuB9S9p8S9qwLcQiOaAqKcnW/NRf+ilWAZuAG6msBtlMaqMvUJP59FcCVdWTbx+dJ/0LP9dz9Q8HZnrCx1RsiBpta+u/QNdq9q+iZp9luopSxC+iW7qRwK+3GOXmhgOLPeE/qtgQVS112tILPeGfWNIeAH4SVasNIqV6KzpmBaFL9QVRB4dSQjtKOaEdpKrzvgoMi0pfJYQ2pgcsaft4wufX8P1JDShhSJ14wj+3pD1DaJieu4UvtJp6GrjYktazYjBYVId+GvCKJ3xNDq9VlTaE3lOpzqrqwCw9Dvw6+mKbl76zLp+lJa0loU3hJeCKKg65HhhhSfuNJ2q0CE4bQjVUqhGEkkM2bgJeAG4HVhM6PFQpajP6lSXtPGDPLZ04qkZ80ZL2G0JHh90JVWpST5QwJBcuBWYBj1nSxhGqhjoTuj0WecKr+gLLxvWEnjBPWNIShC/v8wldQY+pa9BZmA78wpL2K0LD7aHAyXU43/WEap0ZlrRrgNcJn9NQ4LyoIbi2n+UxhCqvMZ7wp9J3WtJuIfxiP4TQrThb04ETLGnXE9qeSoALCaWhLfKEv2hJe4XQ/vIXT/gXKTG1J1T3jaeyjWwooarx8arOZ0k7ltDT6p+E7rZtCT2rPiMkJqlHasOQOvOEzyXUa38C/Jnwn/1PhHERs+pw3mXAAcACwpfdREJVyDGp4zDq0VWEBtpLCA3qexPGbNSKJ3wVYYzIZEIpYDph3MgGonaVOnyWpYQvzUw9jCYAX5K58TuT2wg90k4DHiZ0BT6OUFrIVkVMt6Rt/4owxuRcwr/tZEKvqR+m96JL8R/C+/gNoZfc3wmf3+Ge8OrahyQHTEu0ikh9sqQ9B2zyhB8YdyxSN6qSEpGcs6S1IlSjDQZ+QKhqkgZOCUNE6sMOhBHrq4D/5wmfGm84kguqkhIRkayo0VtERLLSqKukOnfu7D179ow7DBGRBmPOnDn/dffiqvY16oTRs2dPysrK4g5DRKTBMLP3Mu1TlZSIiGRFCUNERLKihCEiIllRwhARkawoYYiISFaUMDIZOzbuCERECooSRibJZNwRiIgUlEY9DqNWvvwSbrop7ihERAqOShipxo6FNm3g5z8Pz83CTdVTIiJKGJsZOxbcYUy0hPOKFeG5EoaIiBJGlUqjRcnuy7j8sIhIk6OEUZW99oIuXeCuu+KORESkYChhZHLFFTBnDixYEHckIiIFQQkjk9NPh6IiuPvuuCMRESkIShiZdOkCQ4bAPffAxo1xRyMiEjsljOqUlsKyZTBzZtyRiIjETgmjOsceCx06qPFbRAQljOq1bg3Dh8PkybBmTdzRiIjESgljS0aODNOFTJwYdyQiIrFSwtiSAQOgd29VS4lIk6eEsSVmofF71ixYsiTuaEREYqOEkY0zzwz3//hHvHGIiMRICSMbPXrAoEFhEJ973NGIiMQibwnDzFqb2ctmNs/MFpjZt1YoMrNWZna/mS02s5fMrGfKvl9G2980syPzFfc3Ro6Et9+G55/P+6VFRApBPksY64BD3X0foC9wlJkNSDvmx8Cn7v5d4HrgWgAz6wMMB/YAjgJuMrOifAUOwLBhYa0MNX6LSBOVt4ThwdroaYvoll6/MxSo+EaeCBxmZhZtv8/d17n7EmAx0D8PYVdq1y4kjQceCN1sRUSamLy2YZhZkZm9CqwAnnD3l9IO6Qp8AODuG4DVwLap2yNLo21VXWOUmZWZWVl5eXlu38DIkbB6NUydmtvziog0AHlNGO6+0d37At2A/ma2Zz1c41Z3L3H3kuLi4tyefNAg6NZN1VIi0iTF0kvK3VcBTxLaI1J9CHQHMLPmQHvgk9TtkW7RtvwqKoIRI+Cxx2D58rxfXkQkTvnsJVVsZh2ix1sBhwOL0g6bCkTro3Iy8G9392j78KgXVS+gN/ByXgJPN3IkbNoE48fHcnkRkbjks4SxA/Ckmb0GzCa0YTxiZleZ2fHRMeOAbc1sMXApcAWAuy8AHgDeAKYDF7h7PItU7LYb9O+vhZVEpMkxb8QD0UpKSrysrCz3J77xRhg9Gl55Bfr2zf35RURiYmZz3L2kqn0a6V0bw4dDixZq/BaRJkUJoza23RaOOw7uvRe+/jruaERE8kIJo7ZGjoQVK0KPKRGRJkAJo7aGDIHOnVUtJSJNhhJGbbVsCaefHkZ9f/pp3NGIiNQ7JYy6KC2F9evh/vvjjkREpN4pYdRFv36wxx4akyEiTYISRl2YhcbvF16At96KOxoRkXqlhFFXZ54JzZqplCEijZ4SRl3tuCMMHhzW+960Ke5oRETqjRJGLpSWwvvvw9NPxx2JiEi9UcLIhRNOCCvyqVpKRBoxJYxcaNMGTjkFJk6Ezz+POxoRkXqhhJErpaWwdi089FDckYiI1AsljFw54ADo2VPVUiLSaClh5EqzZmFMxsyZ8MEHcUcjIpJzShi5NHIkuGv5VhFplJQwcmnnnWHgwDCDbSNeyVBEmiYljFwrLYVFi2D27LgjERHJqbwlDDPrbmZPmtkbZrbAzC6q4pjLzOzV6DbfzDaaWado37tm9nq0rx4W6s6RU0+FVq3U+C0ijU4+SxgbgDHu3gcYAFxgZn1SD3D337t7X3fvC/wSeNrdV6YcMijaX+UC5QWhffswkG/CBFi3Lu5oRERyJm8Jw90/cve50ePPgIVA12pecjowIR+x5VxpKaxcCdOmxR2JiEjOxNKGYWY9gX2BlzLsbwMcBUxK2ezA42Y2x8xGVXPuUWZWZmZl5eXlOYy6Bg4/HLbfXsu3ikijkveEYWZbExLBxe6+JsNhxwHPpVVHHeDu/YAhhOqsg6p6obvf6u4l7l5SXFyc09iz1rw5/PCH8OijEFfSEhHJsbwmDDNrQUgW4929ujk0hpNWHeXuH0b3K4DJQP/6ijMnSkthwwa47764IxERyYl89pIyYByw0N2vq+a49sDBwJSUbW3NrF3FY+AIYH79RlxHe+0FffuqWkpEGo18ljAGAiOAQ1O6zh5tZueZ2Xkpx50IPO7uqdO+dgGeNbN5wMvAo+4+PX+h11JpKcyZAwsWxB2JiEidmTfiEcklJSVeVhbjkI2PP4auXWHMGLj22vjiEBHJkpnNyTR0QSO961OXLjBkCNxzD2zcGHc0IiJ1ooRR30pLYdmyMIutiEgDpoRR3449Fjp0UOO3iDR4Shj1rXVrGD4cJk+GNZmGnYiIFD4ljHwYORK+/DKs+S0i0kApYeTDgAHQu7eqpUSkQVPCyAez0Pg9axYsWRJ3NCIitaKEkS9nnhnu//GPeOMQEaklJYx86dEDBg0KCys14sGSItJ4KWHkU2kpvP02PP983JGIiNSYEkY+nXQStGmjxm8RaZCUMPKpXTsYNgweeCB0sxURaUCUMPJt5EhYvRqmTo07EhGRGlHCyLdBg6BbN1VLiUiDo4SRb0VFMGIEPPYYLF8edzQiIllTwojDyJGwaROMHx93JCIiWVPCiMNuu0H//mFMhohIA6GEEZeRI+G11+DVV+OOREQkK0oYcRk+HFq0UOO3iDQYeUsYZtbdzJ40szfMbIGZXVTFMYeY2WozezW6XZmy7ygze9PMFpvZFfmKu95suy0cdxzcey98/XXc0YiIbFE+SxgbgDHu3gcYAFxgZn2qOO4Zd+8b3a4CMLMi4EZgCNAHOD3DaxuWkSNhxYrQY0pEpMDlLWG4+0fuPjd6/BmwEOia5cv7A4vd/R13Xw/cBwytn0jzaMgQ6NxZjd8i0iDE0oZhZj2BfYGXqti9v5nNM7N/mdke0bauwAcpxywlQ7Ixs1FmVmZmZeXl5bkMO/datoTTT4cpU+DTT+OORkSkWnlPGGa2NTAJuNjd0xe5ngv0cPd9gL8A/6zp+d39VncvcfeS4uLiOsdb70pLYf16uP/+uCMREalWXhOGmbUgJIvx7v5Q+n53X+Pua6PH04AWZtYZ+BDonnJot2hbw9evH+yxh6qlRKTg5bOXlAHjgIXufl2GY7aPjsPM+kfxfQLMBnqbWS8zawkMBxrH7H1mofH7hRfgrbfijkZEJKN8ljAGAiOAQ1O6zR5tZueZ2XnRMScD881sHvBnYLgHG4DRwGOExvIH3H1BHmOvX2eeCc2aqZQhIgXNvBEvF1pSUuJlZWVxh5GdI4+ERYtgyZKQPEREYmBmc9y9pKp9+mYqFKWl8P778PTTcUciIlIlJYxCccIJYUU+VUuJSIFSwigUbdrAKafAxInw+edxRyMi8i1KGIWktBTWroWHvtXjWEQkdkoYheSAA6BXL1VLiUhBUsIoJM2aheVbZ86EDz7Y8vEiInmkhFFoRo4Edy3fKiIFp84Jw4wWuQhEIjvvDAMHhoWVGvEYGRFpeGqUMMz4mRnDUp6PA740400zds15dE1VaWkYxDd7dtyRiIh8o6YljJ8B5QBmHAScCpwBvAr8MaeRNWWnngqtWoXG77Fj445GRASA5jU8viuwJHp8HPCgOw+Y8TrwTE4ja8ratw8D+SZMgJUrlTREpCDUtISxBtguenw4MDN6/DXQOldBCaFaauXKuKMQEflGTRPG48BtZtwOfBf4V7R9DypLHlJXY8fC0UdXPjcLN5U0RCRGNU0YFwDPAcXAye5U/ATuB0zIZWBN2tixoYfUhg2V20aPhiuvjC0kEZEatWG4swa4sIrtiZxFJJWKisL9pZfCddfBqlVwxx3QQj2ZRST/apQwzOgDbHTnzej54UApsAD4P3c25j7EJi6RCLeOHeE3v4E1a8L6363VZCQi+VXTKqk7gH0BzOgOTAE6EaqqrsltaAKE6ikz+PWv4S9/galT4Zhj4LPP4o5MRJqYmiaM3YC50eOTgZfcOZqw9OrpuQxMqjB6dBib8fTTMHgwfPJJ3BGJSBNS04RRBKyPHh8GTIsevw10yVVQUo0RI2DSJJg3Dw4+GJYtizsiEWkiapow5gM/NeNAQsKYHm3vCvy3uheaWXcze9LM3jCzBWZ2URXH/NDMXjOz183seTPbJ2Xfu9H2V82sgSzUXU+GDoVp0+Ddd8OU6O+8E3dEItIE1DRh/AI4F3gKmODO69H244GXt/DaDcAYd+8DDAAuMLM+accsAQ52972Aq4Fb0/YPcve+mRYob1IOPRT+/W9YvTokjQUL4o5IRBq5GiUMd2YRxmB0dufslF23AD+t/rX+kbvPjR5/BiwklExSj3ne3T+Nnr4IdKtJfE1O//6hPQPgoIPg5S3lbBGR2qvx9OZR19kvzdjTjD3MaO3Ou+6syPYcZtaT0NvqpWoO+zGVI8kBHHjczOaY2ahqzj3KzMrMrKy8vDzbkBquPfeEZ58N808ddlgodYiI1IOaTm/e3IzfA58C84DXgU/N+L9s18Uws62BScDF7r4mwzGDCAnjFymbD3D3fsAQQnXWQVW91t1vdfcSdy8pLi7O+r01aN/5TkgaPXqEKUWmTIk7IhFphGpawvg/4EzgPGAXoDehKmoE8NstvdjMWhCSxXh3fyjDMXsDtwND3f2bfqPu/mF0vwKYDPSvYeyN2447huqpvfeGYcPgnnvijkhEGpmaJowzgB+7c5c7b0e3O4FzgB9W90IzM2AcsNDdr8twzE7AQ8AId38rZXtbM2tX8Rg4gtBjS1Jtu21YD/ygg0L327/+Ne6IRKQRqel6GO0JYy7SvQ102MJrBxJKIq+b2avRtl8BOwG4+9+AK4FtgZtCfmFD1COqCzA52tYcuNfdpyPf1q5d6HI7fDhceGGYf+p//ieMFhcRqYOaJox5hFX3LkjbflG0LyN3fxao9lvL3c8hlFbSt78D7PPtV0iVWreGBx+Es88O80+tWgW//72ShojUSU0TxuXANDMGE7q9QhhTsSOhMVoKRYsWcNddoffUH/8YksYtt1TOgCsiUkM1nd58lhm7EEoYu0WbHyRMEXIx8GxOo5O6adYsTFjYsSNcc00Y5HfPPWG9cBGRGqppCQN3lgH/k7rNjH2AYbkKSnLIDK6+OiSNMWPC9OgPPQRt28YdmYg0MDUeuCcN1KWXwrhxMGMGHHFEqKISEakBJYym5Oyzw+JLs2fDoEHw8cdxRyQiDYgSRlNz8snw8MPw5ptw4IHw/vtxRyQiDURWbRhmTN3CIdvkIBbJlyOPhCeeCCv3HXBAeLzrrnFHJSIFLtsSxidbuC0B7q6PAKWeDBwITz0F69aFksYrr8QdkYgUuKxKGO6cVd+BSAz69oVnngnLvR5yCDzySEgeIiJVUBtGU7fLLvDcc7DDDqGq6l//2vJrRKRJUsIQ6N4dZs2C3XaD44+HBx6IOyIRKUBKGBJstx08+SQMGBAmLrzttrgjEpECo4Qhldq3h8ceg6OOglGjwoSFAGPHxhqWiBQGJQzZXJs28M9/wmmnweWXw69+Bclk3FGJSAGo8VxS0gS0bAnjx8M228Bvo4UUv/46zIArIk2WShhStauv3rwdo2XLMJGhqqdEmiwlDKna2LHgHm4A3bqFhLFihSYuFGmilDAkO2+8AT/7WViEaffdw4p+FclERJoEJQzZskQirBV+ww3w0kthkN+pp8Kxx8K778YdnYjkSd4Shpl1N7MnzewNM1tgZhdVcYyZ2Z/NbLGZvWZm/VL2lZrZf6Jbab7iFjZvtygpgZdfhuuug6efhj32gD/8ITSKi0ijls8SxgZgjLv3IawDfoGZ9Uk7ZgjQO7qNAm4GMLNOQALYD+gPJMysY74ClzTNm8Mll4RqqsMOg8sug+9/P5Q+RKTRylvCcPeP3H1u9PgzYCHQNe2wocDdHrwIdDCzHYAjgSfcfaW7fwo8ARyVr9glg512gilTYNIkKC+H/feH0aPDMrAi0ujE0oZhZj2BfYH0n6RdgQ9Sni+NtmXaXtW5R5lZmZmVlZeX5yxmycAMTjoJFi4MyeKmm0Kj+KRJahQXaWTynjDMbGtgEnCxu+f8p6i73+ruJe5eUlxcnOvTSybbbAN//jO8+CIUF4eV/YYO1Yp+Io1IXhOGmbUgJIvx7v5QFYd8CHRPed4t2pZpuxSa/v2hrCw0hM+cCX36hAbyDRvijkxE6iifvaQMGAcsdPfrMhw2FRgZ9ZYaAKx294+Ax4AjzKxj1Nh9RLRNClHz5jBmDCxYAAcfHB5XJBIRabDyWcIYCIwADjWzV6Pb0WZ2npmdFx0zDXgHWAzcBpwP4O4rgauB2dHtqmibFLKePcMqfg8+CMuXw377wUUXwWefxR2ZiNSCeSNumCwpKfEy/aotDKtXh5lvb74ZdtwR/vpXOOGEuKMSkTRmNsfdS6rap5Hekh/t28ONN8Lzz0OnTnDiiSFhfPDBFl8qIoVBCUPya8AAmDMHrr0WHn88NIr/6U+wcWPckYnIFihhSP61aBEWZ1qwAA44AC6+OLRvzJ0bd2QiUg0lDIlPr14wbRrcdx8sXRqmF7n0Uli7Nu7IRKQKShgSL7OwHOyiRWEd8euvD9VUU6fGHZmIpFHCkMLQoUPoQfXcc2HU+NChMGwYfKjxmSKFQglDCssPfhDaMn7721BdtfvuoQvuxo1aHlYkZkoYUnhatoQrroD588MMuBdeGO6TybgjE2nSlDCkcO28M0yfDvfeC++9F7adeKLW3RCJiRKGFLZkEs44A1asCM//+c8wlqNXL3jiCU2hLpJHShhS2MaODUmhIjGsWRNmwl2/Ho44InTFnThRA/9E8kAJQxqWdu3C7LfvvAO33RbmqDrllNAVd9y4kEhEpF4oYUjDkUhUPm7VCs45J4zfuP9+aNs2PP/Od8L6Gxr8J5JzShjScFTVrbaoCE49NcxPNX069O4dSiA9eoTjP/kk31GKNFpKGNI4mMGRR8KTT8ILL4Q5qpJJ2GknuOQSzYorkgNKGNL4DBgAU6aEcRzDhsFf/hK66J59Nrz5ZtzRiTRYShjSeO2xB9x9NyxeDD/5CUyYEEaOn3yylosVqQUlDGn8evYMpYz33gur/s2YEbrjDh4MM2dqLIdIlvKWMMzsDjNbYWbzM+y/LGWt7/lmttHMOkX73jWz16N9+mkotbPddnDNNfD++2EBpwULQtLYbz+YPBk2bYo7QpGCls8Sxp3AUZl2uvvv3b2vu/cFfgk87e4rUw4ZFO2vcq1Zkaxts01YwGnJEvjb30JPqpNOClVYd96psRwiGeQtYbj7LGDlFg8MTgcm1GM4ItC6dWjbePPN0L7RqhWcdRZ897th2djPP487QpGCUnBtGGbWhlASmZSy2YHHzWyOmY2KJzJptJo3h+HD4ZVXwpTqPXuGZWN79ICrroKVab9zNM26NFEFlzCA44Dn0qqjDnD3fsAQ4AIzOyjTi81slJmVmVlZeXl5fccqjYkZDBkCs2bBs8+GKdUTiZA4fv7zysWcNM26NFGFmDCGk1Yd5e4fRvcrgMlA/0wvdvdb3b3E3UuKi4vrNVBpxAYOhIcfhnnzwup/N9wQZsg955y4IxOJTUElDDNrDxwMTEnZ1tbM2lU8Bo4AquxpJZJze+8N99wDF1wAX38dJjiEUBoxg0svjTc+kTzKZ7faCcALwK5mttTMfmxm55nZeSmHnQg87u6prY1dgGfNbB7wMvCou0/PV9wiQGgEd4fly8Pzkqiz3g03wKGHhkSyalVc0YnkhXkjHrRUUlLiZRrRK7lmFpLHW2+F1QDvvRf+85+wtOwxx4QFn445BrbaKu5IRWrMzOZkGr5QUFVSIg1CxTTru+wSeky9+SbMng3nnx8mPjzlFOjSBX70o7Aq4IYNcUYrkjMqYYjk0saN8NRTMH48TJoUVgjs0iV02z3jjDAliVncUYpkpBKGSL4UFcFhh8Edd8DHH4ekMXAg3HxzmIJkl11CCUWz5koDpIQhUl9atw5TjkyaFJLHuHFhTMfVV8Nuu4WG8+uuqxzfIVLglDBE8qFDh7Aex4wZsHQpXH99qJoaMwa6d1dPK2kQlDBE8m3HHcPUI7Nnh6qpK68MSeScc0J7x0knwcSJ8OWXcUcqshklDJE4VdfTavvtw2SImXpaaU4ryTP1khIpNNn2tKoYDyKSQ+olJdKQZNvTCrTok+SVEoZIIUvvaXX88WGN8quuCvuLikJJ47jj4J134o1VGj0lDJGGokMHmDIlVEN99FHYNnIkdO0KjzwCO+8cZtQ991y47z5YsSLWcKXxURuGSENV0YbhHhrNZ84Mt3//G1avDsfsvXeo3ho8GA46CLbeOt6YpeBV14bRPN/BiEiOVLRjmIWBgLvtFqZh37gR5s4NYz5mzoSbbgrjPpo3hwEDQgI57LDQHtKyZbzvQRoUlTBEGrsvv4Tnn69MIHPmhMbytm1DqWPw4JBA9toLmqmWuqmrroShhCHS1Hz6aei2O3NmSCIV81oVF4cR5xUJpFevWMOUeChhiEhmS5dWtn/MnAnLloXt3/lOZfXVoYeGhFKVsWM1iLARUcIQkey4w6JFlaWPp56qbEDfZ5/K0seBB1Y2oGsAYaOihCEitbNhQ2jzqEggzz0H69dDixaVDehjx8Inn0CnTnFHKzmghCEiufHFFyFpJJPhPt3uu0NpKXzve+HWsWP+Y5Q6KYiEYWZ3AMcCK9x9zyr2HwJMAZZEmx5y96uifUcBfwKKgNvd/XfZXFMJQ6SerVoVksK110JZWSiNpI4433nnkDhKSsJ9v35hAKIUrEIZh3En8Ffg7mqOecbdj03dYGZFwI3A4cBSYLaZTXX3N+orUBHJUsWX/+WXV25buTKMAykrC7eXX4YHHqjc/93vViaQkpKQRLbZJq9hS+3kLWG4+ywz61mLl/YHFrv7OwBmdh8wFFDCECkEFQMIK3TqFBrHBw+u3Pbf/1YmkTlzwriQ++6r3L/LLt8uibRrl5/4JWuFNtJ7fzObBywDfu7uC4CuwAcpxywF9osjOBGpQjZdajt3hiOOCLcK5eUhecyZExLJM8/AhAlhnxnsuuvmSWTffauf2kTde+tdXhu9oxLGIxnaMLYBNrn7WjM7GviTu/c2s5OBo9z9nOi4EcB+7j46wzVGAaMAdtppp++999579fRuRCTnPv548yQyZ07lmucVU6CkVmf17RtGrFfsb8SdePKlUNowquXua1IeTzOzm8ysM/Ah0D3l0G7RtkznuRW4FUKjdz2FKyL1oUsXOProcKuwfPnmCWTGDPjHP8K+Zs1Cz6zvfS88nzo1PO/VK8ydJTlVMJ+omW0PfOzubmb9CVOvfwKsAnqbWS9CohgOnBFboCKSX9tvD8ccE24Vli0LyeMPf4BZs2DBgrB96NBwX5FIdtst3Fc83nXXyhKJ1FjeEoaZTQAOATqb2VIgAbQAcPe/AScDPzWzDcCXwHAP9WUbzGw08BihW+0dUduGiDRVO+4YbscdV7nNLKyHvmgRLFwY7l97DSZP3nxlwp122jyJVNwXF4dzSEYauCcijUOmNox168IqhRVJpOJ+0aIwELFCp07fLpHsvjv06BFWNmwiGkQbhohInaR3763QqhXssUe4pdq0KUy8uHDh5snk4Ydh3LjK41q3Dt1+05PJLrvAVlttfs5G3lNLJQwRkXQrV1YmkNRksmRJZSnGDHr23Lw0cu65IQntsEODXVtEJQwRkZro1Al+8INwS/XVV/DWW5tXbS1cGJbF/eqrcEy3bqFU07NnmCK+V6/K+4rH7dvn/S3lghKGiEi2WrcO66TvvXfltrFjw2SMqdatCyWM5ctDQ/yqVZvv79jx20mk4r5Hj4JdOldVUiIiuZKp4f3TT0N11pIlYXLG1MfvvhumjE89R9eumUsn22+/5equOrSlFMRstXFQwhCRvKrNaPNNm8K4kvSEUnH/Ydo45VatKhNIpuquOox6VxuGiEg+ZOqpVZ1mzUK7R7duYSXDdF99Be+9V3Xp5PnnK1dErFCPa5CohCEi0pBVVHf97nfw4IPf3p9I1Kh6SiUMEZHGqmPHcEtdc6SeJmJsmB2FRUQk75QwREQam9q0pWRBCUNEpLGpp+lJlDBERCQrShgiIpIVJQwREcmKEoaIiGRFCUNERLLSqEd6m1k58F4tX94Z+G8Ow6mtQoijEGIAxZFOcWyuEOIohBigbnH0cPfiqnY06oRRF2ZWlml4fFOLoxBiUByKoyHEUQgx1GccqpISEZGsKGGIiEhWlDAyuzXuACKFEEchxACKI53i2FwhxFEIMUA9xaE2DBERyYpKGCIikhUlDBERyYoSRhozO8rM3jSzxWZ2RYxx3GFmK8xsfowxdDezJ83sDTNbYGYXxRRHazN72czmRXEk44gjiqXIzF4xs0fiiiGK410ze93MXjWzWJaVNLMOZjbRzBaZ2UIz2z+GGHaNPoOK2xozuzjfcUSxXBL9fc43swlm1jqmOC6KYliQ689CbRgpzKwIeAs4HFgKzAZOd/c3YojlIGAtcLe775nv60cx7ADs4O5zzawdMAc4Id+fh5kZ0Nbd15pZC+BZ4CJ3fzGfcUSxXAqUANu4+7H5vn5KHO8CJe4e2yAxM7sLeMbdbzezlkAbd18VYzxFwIfAfu5e2wG7tb12V8LfZR93/9LMHgCmufudeY5jT+A+oD+wHpgOnOfui3NxfpUwNtcfWOzu77j7esIHPzSOQNx9FrAyjmunxPCRu8+NHn8GLAS6xhCHu/va6GmL6Jb3Xzpm1g04Brg939cuNGbWHjgIGAfg7uvjTBaRw4C3850sUjQHtjKz5kAbYFkMMewOvOTuX7j7BuBp4KRcnVwJY3NdgQ9Sni8lhi/IQmRmPYF9gZdiun6Rmb0KrACecPc44rgBuBzYFMO10znwuJnNMbNRMVy/F1AO/D2qorvdzNrGEEeq4cCEOC7s7h8CfwDeBz4CVrv74zGEMh840My2NbM2wNFA91ydXAlDtsjMtgYmARe7+5o4YnD3je7eF+gG9I+K3nljZscCK9x9Tj6vW40D3L0fMAS4IKrCzKfmQD/gZnffF/gciLPNryVwPPBgTNfvSKiN6AXsCLQ1szPzHYe7LwSuBR4nVEe9CmzM1fmVMDb3IZtn427RtiYrajOYBIx394fijieq9ngSOCrPlx4IHB+1HdwHHGpm9+Q5hm9Ev2hx9xXAZEJ1aj4tBZamlPQmEhJIXIYAc93945iuPxhY4u7l7v418BDwgzgCcfdx7v49dz8I+JTQLpsTShibmw30NrNe0S+W4cDUmGOKTdTYPA5Y6O7XxRhHsZl1iB5vReiUsCifMbj7L929m7v3JPxd/Nvd8/4LEsDM2kadEIiqgY4gVEXkjbsvBz4ws12jTYcBee8ckuJ0YqqOirwPDDCzNtH/m8MIbX55Z2bbRfc7Edov7s3VuZvn6kSNgbtvMLPRwGNAEXCHuy+IIxYzmwAcAnQ2s6VAwt3H5TmMgcAI4PWo/QDgV+4+Lc9x7ADcFfWCaQY84O6xdmuNWRdgcvheojlwr7tPjyGOC4Hx0Y+rd4CzYoihImkeDvwkjusDuPtLZjYRmAtsAF4hvmlCJpnZtsDXwAW57IygbrUiIpIVVUmJiEhWlDBERCQrShgiIpIVJQwREcmKEoaIiGRFCUOkQJnhZpwcdxwiFZQwRKpgxp3RF3b6Le8z5IoUCg3cE8lsBmHgYqr1cQQiUghUwhDJbJ07y9NuK+Gb6qLRZjxqxhdmvGfGZlOFmLGXGTPM+NKMlVGppX3aMaVmvG7GOjM+NuOutBg6mfGgGZ+b8U4V17gyuvY6M5abcXe9fBIiKGGI1EWSMNdYX8I0EHebUQJgRlvCFDNrCRMDnkiYjO6Oiheb8RPgFuDvwN6EqajT54S6EpgC7APcD9xhxk7R64cBPwfOB3oDxwIv5/5tigSaGkSkCmbcCZwJfJW260Z3fmGGA7e7c27Ka2YAy90504xzCesjdHPns2j/IYSZdnu7s9iMpcA97lVPCx5d43fu/DJ63hxYA4xy5x4zLiXMn7SnO1/n7M2LZKA2DJHMZgHpixOtSnn8Qtq+Fwgr8kFY+ey1imQReZ6w+FIfM9YQFueauYUYXqt44M4GM8qB7aJNDwIXAUvMeIyw/sFUd9Zt4ZwitaIqKZHMvnBncdotF2to16RYn15ycKL/t+58AOxKKGWsAf4IzImqw0RyTglDpPYGVPG8Yg2EhcBeZrRL2f8Dwv+5he6sICzOdVhdAnDnK3cedecS4PvAHoRp6UVyTlVSIpm1MmP7tG0b3SmPHp9kxmzgKeBkwpf/ftG+8YRG8bvNuBLoSGjgfsidxdEx/wtcb8bHwKNAG+Awd/6YTXBm/Ijwf/glQuP6aYQSyX9q+D5FsqKEIZLZYOCjtG0fEpbuBRgLDAP+DJQDZ7kzG8CdL8w4EriB0HPpK0Jvp4sqTuTOzWasB8YQ1mFeCdRkcapVwC8IjestCCveneTOkhqcQyRr6iUlUgtRD6ZT3JkYdywi+aI2DBERyYoShoiIZEVVUiIikhWVMEREJCtKGCIikhUlDBERyYoShoiIZEUJQ0REsvL/AbhNJq+koKgxAAAAAElFTkSuQmCC\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":"# Inference - Next Character Prediction","metadata":{}},{"cell_type":"code","source":"# reset previous states of model\nmodel.reset_states()","metadata":{"execution":{"iopub.status.busy":"2021-05-27T14:52:49.252417Z","iopub.execute_input":"2021-05-27T14:52:49.252724Z","iopub.status.idle":"2021-05-27T14:52:49.258655Z","shell.execute_reply.started":"2021-05-27T14:52:49.252687Z","shell.execute_reply":"2021-05-27T14:52:49.257777Z"},"trusted":true},"execution_count":26,"outputs":[]},{"cell_type":"code","source":"sample = 'ANTHONIO:'\n# vectorize the string\nsample_vector = [tokenizer[s] for s in sample]\npredicted = sample_vector\n# convert into tensor of required dimensions\nsample_tensor = tf.expand_dims(sample_vector, 0) \n# broadcast to first dimension to 64 \nsample_tensor = tf.repeat(sample_tensor, 64, axis=0)\n\n# predict next 1000 characters\n# temperature is a sensitive variable to adjust prediction\ntemperature = 0.6\nfor i in range(1000):\n    pred = model(sample_tensor)\n    # reduce unnecessary dimensions\n    pred = pred[0].numpy()/temperature\n    pred = tf.random.categorical(pred, num_samples=1)[-1,0].numpy()\n    predicted.append(pred)\n    sample_tensor = predicted[-99:]\n    sample_tensor = tf.expand_dims([pred],0)\n    # broadcast to first dimension to 64 \n    sample_tensor = tf.repeat(sample_tensor, 64, axis=0)\n    \n","metadata":{"execution":{"iopub.status.busy":"2021-05-27T15:05:50.180597Z","iopub.execute_input":"2021-05-27T15:05:50.180989Z","iopub.status.idle":"2021-05-27T15:06:21.247443Z","shell.execute_reply.started":"2021-05-27T15:05:50.180960Z","shell.execute_reply":"2021-05-27T15:06:21.246388Z"},"trusted":true},"execution_count":42,"outputs":[]},{"cell_type":"code","source":"pred_char = [vocabulary[i] for i in predicted]\ngenerated = ''.join(pred_char)\nprint(generated)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T15:06:21.249144Z","iopub.execute_input":"2021-05-27T15:06:21.249611Z","iopub.status.idle":"2021-05-27T15:06:21.256780Z","shell.execute_reply.started":"2021-05-27T15:06:21.249566Z","shell.execute_reply":"2021-05-27T15:06:21.255619Z"},"trusted":true},"execution_count":43,"outputs":[{"name":"stdout","text":"ANTHONIO:\nWhat are you which never was the trie would\nAnd in the strong will night not did the moon:\nPridot me you then be so good the more than the moor\nA mistress in the way a most about so then\nCome with the steed but what with him that swore.\n\nGRUMIO:\nAnd then you come to way the mine of\na right to be made thee well.\n\nPETRUCHIO:\nWhat is the mistress of my son:\nAnd Mirth you him to your life.\n\nANTONIO:\nHark, sir, what, not here in my sense out.\n\nPETRUCHIO:\nAy, sir, this young.\n\nProvost:\nI am head, and all the sen where the house of a little with the\nhouse, my love, having the pettit is all\nthe curner of the lives be ready to my sister then\nFrom the new company.\n\nPROSPERO:\nI have been from the midst such a desperit of his time.\n\nBAPTISTA:\nI may be then to be a master 'stall in her death.\n\nGRUMIO:\nBut what you must be beauthing sorry.\n\nPETRUCHIO:\nPray, be long too shall be presently,\nstays to thy consciench.\n\nPETRUCHIO:\nI can lie thee to the bid and\nThe word of my father, what is is the which \n","output_type":"stream"}]},{"cell_type":"markdown","source":"### vary temperature to see yet different prediction","metadata":{}},{"cell_type":"code","source":"sample = 'ANTHONIO:'\n# vectorize the string\nsample_vector = [tokenizer[s] for s in sample]\npredicted = sample_vector\n# convert into tensor of required dimensions\nsample_tensor = tf.expand_dims(sample_vector, 0) \n# broadcast to first dimension to 64 \nsample_tensor = tf.repeat(sample_tensor, 64, axis=0)\n\n# predict next 1000 characters\n# vary temperature to change randomness\ntemperature = 0.8\nfor i in range(1000):\n    pred = model(sample_tensor)\n    # reduce unnecessary dimensions\n    pred = pred[0].numpy()/temperature\n    pred = tf.random.categorical(pred, num_samples=1)[-1,0].numpy()\n    predicted.append(pred)\n    sample_tensor = predicted[-99:]\n    sample_tensor = tf.expand_dims([pred],0)\n    # broadcast to first dimension to 64 \n    sample_tensor = tf.repeat(sample_tensor, 64, axis=0)\n    \n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_char = [vocabulary[i] for i in predicted]\ngenerated = ''.join(pred_char)\nprint(generated)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T15:02:55.051760Z","iopub.execute_input":"2021-05-27T15:02:55.052046Z","iopub.status.idle":"2021-05-27T15:02:55.059127Z","shell.execute_reply.started":"2021-05-27T15:02:55.052018Z","shell.execute_reply":"2021-05-27T15:02:55.058078Z"},"trusted":true},"execution_count":39,"outputs":[{"name":"stdout","text":"ANTHONIO:\nNow, I have not so early debeits and secret what then\nHath been and leaves and speak to please you, your widow driblest.\n\nMIRANDA:\nWhat is naw't shall now call her cheer him\nThine own reputed anchor all. Pest, not hell!\n\nDAKE ANTIUS:\nSir, then he that have been the dank in the\ngreat amorn'd myst you the Pribs Henry,\nThere are children of the still stand him.\n\nGRUMIO:\nWhat shall no pass are love not so receiveds and all,\nWhat mistress lost and talk! shall he come proclaim in dich,\nWhen 'tis begire the servant rather father\nfrom thy chimate, sir; suppiar with the met\nAs widow the lost me well and I may never;\nTherefore Prance your good remimble bid her birds.\n\nPETRUCHIO:\nGod good with his fanst from my wife.\nO at thus thou artwain on the world!\nSo more off me mean a will of shame,\nHave they seem the house the names not some too;\nI pray you, or olders arispr o' the\nmore tobergey.\n\nPedant:\nSir, sir, I proceed?\n\nGREMIO:\nThy did and for a gentlewat for his temper old\nAnd made hears for appr\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Thank You For Your Time!","metadata":{}}]}