43-seq2seq-序列到序列-自然语言处理-pytorch.ipynb

{
 "cells": [
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfkAAACGCAYAAADetfNEAAAZ70lEQVR4Ae2dXYgeRb7GR70QPCfOSFwl7m4SRQ4u8WQCyu7qxSScXVHQmByF4OLmw5hFDhoihFy4iUz82OSAkURkr4yOIHJuIhEvRBxnE40XmoAhQhAVyZU3AYl66U0ffr37jzU9/X50v9WV7qqnoan+qO5+36rn//yqqvvtd+ynn37KNKsMpAFpQBqQBqSB+DQwpkqNr1JVp6pTaUAakAakATQgyGskQyM50oA0IA1IA5FqQJCPtGLVilcrXhqQBqQBaUCQF+TVgpcGpAFpQBqIVAOCfKQVqxa8WvDSgDQgDUgDgrwgrxa8NCANSAPSQKQaEOQjrVi14NWClwakAWlAGhDkBXm14KUBaUAakAYi1YAgH2nFqgWvFrw0IA1IA9KAIC/IqwUvDUgD0oA0EKkGBPlIK1YteLXgpQFpQBqQBgR5QV4teGlAGpAGpIFINSDIR1axX3zxRTY7O6u5Yhn88MMPMrmIYuGrr75SDFSMAXzj+++/VxxEFAeM5AjysVXo2Fg2prlyGezYsUPmFlEsXH31eGUNKG7GskcffVRxEFEcCPKRVeaPP/6YG9vdG/6S/e2NY5qHLIPxa67Ntm3bJnOLKB4A9pr7/6wYGDIG8ItfLPl19tBDDykOIooDQT6yymTIGXPbtHN/9s6XmeYhy2Dx9TdkW7dulblFFA9XXHFFtuF/disGhowB/OJXN/5HtmHDBsVBRHEgyEdWmYJ8vYaNIB/fE8iCfPVYEOTjiwNBXpBXT+fLLBPk4zM3QV6QB3Ca9eBdVCJQT766sTFMKcjHZ4aCfPVYUE8+vjhQTz6ylp4gX93YBPk4jU2Qrx4LgnycsaCf0EUEekG+urEJ8nEamyBfPRYE+ThjIQjk33777Wxqairp+cEHH2z81oAgX93YQkL+/fffz1avXp10HKxfvz7jp55N3ysV5KvHQijIf/jhh9maNWuSjoN77703O3/+fONxEGy4nt8g89MuDC7F+ZZbbsm//9dff91opQry1Y0tJOSffPLJpONgxYoV+fc/ffp0o3GAsQny1WMhFOT37NmTdBysXLky//4nTpxoPA6CQZ7fIC9ZsiRLdXr99dfzSuVVm032YAT56sYWEvK8VW/RokWphkF25MiRPA4+++yzRuNAkK8XB6Egv3v37uyyyy5LNg7ee++9PA4++uijxuNAkA8kM0G+nukA4BBzqKfrBXlBPoSe615DkA8DBEE+TDkHvYogHwbWdc1NkA8TDurJtzsOBPkwcSDIhynnoFcR5NttboJ8mHAQ5NsdB4J8mDgQ5MOUc9CrCPLtNjdBPkw4CPLtjgNBPkwcCPJhyjnoVQT5dpubIB8mHAT5dseBIB8mDgT5MOUc9CqCfLvNTZAPEw6CfLvjQJAPEweCfJhyDnoVQb7d5ibIhwkHQb7dcSDIh4kDQT5MOQe9SqyQP/yPc0F+4lb3qflhjxPks+zcuXP53GRgdB3y/3fqQhR67xUXXYf8hQsXRpZviDgQ5CtU0/j4+ILcvE1r2bJl2ebNmxfsczdMT09nW7ZscTc1ttx1yK/dtCPb9teDCwzuuhuWZWWgZ9u+N47Pm//2xrHsT9v3LpgPvX163nn/9MT0vDwYUvHYHftn5h3Ty7SG3R4D5NEzszsRHxgfsTDIAPkN/8GDB93DvS93HfL/9d+bM1d7QL+oc9YtJsrigG1lcYDGXb1eijjoOuTx/aLOWZ+cnMzn4r4ygZfFUVm+UbYJ8kOUHq0twLlq1ars+PHj+cxhVCIVDegxtkEQJw/m1vTUdci7MAf4//nbNfl83S+XZzf9ZtXFdTPAHMpPTOdvdcrN6onp3PjY7s433jKZr7vmZtvIxzL7OAcGa8fe+tvV8wzRPb7Oclchj85nZmZy+ZaZE6+SZgLevDPeJo7h3eHuPDExkceTu41lO78dO0raZcgD56sWjWdoj3hAjzmwn5jOfveHdfl2dMp+UnSIXlmmHkiZaRiYji3lGJZd7V6KOOgq5OEBHFi+fPlFHsACZhhx9OjRXMdlHb9i3BTXjx07lr+KfRTdF48V5IslUrKO8QBzZnsXPoaHUVEpNhnoe7XgrFGASJqcfEK+3/vvm3itLeaDoZlplRkS+8zEzKjooZPX1kkZDXDNjP0YpZsHc7PeEQ0I9nHuPzyw5eL24nnd4+ss+4J8v7rh9ZK+33hn+iUtmhN6NsizzLUtDtA7cWIzxxJL69aty/PZdlKfseET8qdOner7SlDf765Hc9aIRaOuboG86bosDqyxatrkPH/9+9GLuufcxRGtSxEHviA/KA58v9a2jAdsA/BuIxUe0HC1OCAuinFTXCcGYIzPSZAfsjQxJCoAuGNEmBQVatAnxdiotH69dc5T1sIb8mMMlc0X5DE2jJvhp1dffXXBvxg1AXkDMcYE7Fn/y+5DF4FrQAbCGJwZGUB319nOsWaGrPNdLL+l5HFntnNt24b5MZpg+X2kviC/du3avKG5adOmrAxCviGP+NAuujdzYnnv3r35TPnaMmkZsDE86+1YHBFTTUy+IA9ELA5efPHFBXHg+931AJ3eO9d0Z7SMxtEmeYiF3/9x/TyNEwdFvZLfjQNGCIo6Nr1bGiIOfEF+48aNfePAN+TRKrFFj92024sHxIfr9xY3pvfiuiA/5H+x+/6DGowJ0LkVQOXa0L1VNBVHpfaaOA9iYC4zwF7HVd1ukH/33XezM2fO1J4//vjj7Pnnn8/sX4wwHIDy1ltv5b0a35DHiP7t6ol8SJ5rYWSYDqA183FT6+lgSOSxxoD1Ulwz41zkKZqbu871Ob9tKzNM2zdKes2112cPPPBA7XqxOn3ttdcyDI574ZQXugJC1rNpAvIYG6Zl5uQCn8/g7itqnPhhiJMRMOvdEDt8bhoFvieDPKmVWZ2UOHjuuefmxcH9999/MQ58Qx5tAXODupuiSYbg0TN5WHe1iH6LcUCDwR7iI3Xjwj3WlotxwCiAja5ZHh/pkl/flN1zzz0j1Q31SQekXxw0AXl4gIat143WjQekNhEfbhwQN/TurTFst6tsnVu+dk47x6ipevJDlCAVxT3GQ4cO5b0QKpHKojIsJQ+G1a+CyMP9SjPIIS5dK4tBHtNtasaY9+/fn59/087984ymrgGYeblwxbRYtx68m2J0XAtT4ntiRNb751w0GCw/xsdwPOvuMcOWjwv/ut/Pjhv2mqPkA0KPPPJIY/9CV9Qw2ufzlj1MR17gTmwAeuIAk7TGMWZJgwRN8fe4vqadO3c2pn+rGxosBw4c8P5XswCc0SqeR7EUnTMTD0UQu5p244DtfNayODDw23cZJvUZB5dffkXj9UMcPP74417/hQ4NA2c4YM9pWQcQrVOOaBudm+ZN0+Sz2CFlP7O7jeN8ToL8EKVJS4xKoJeCEbkVwuFWSVbRZafEyDA6zsUy5yFtYjLI0zp8+eWXa8/PPPNMdvfdd+c9LzMAeo20mhke9t2Tx5Csx20GhKnYA0iW0jOhV44Rcgx5mM0EWbZ79LbP0rJeDKbJOW0EgHPSe7HzsO5z/vfxa7I777yzdr1YnTKqsnTp0nlGyajL4cOH8yHlJnryplduOxEHNmFMbGMuToDf7c2wn/wGectPPPg0uFdeeSUvm6eeemrksr7rrrtK42B2djYf1fJ9Tx5tW4PVUvTNULw9gEojlkaADc+jV+Ji2Dggn6vrsjiwOCrmdY+ru7z4uiXZbbfdNnLd0DB0RxvxKjcOfPfkjQfo33hg8EbLXN/WXcij76LmjSUWA6Scvxgv7v6qy4J8hRJzIW4V7LbE3P3F05If07WpuG7bfaQG+VH/Tx4DM7hz/xd4MDRps2/IA3Z629xnJMV0DOwYCSZGigFiOgZ5hu1tG9stX5n52L58uHP73twwMVFrVLjHkIf8GOmT//v6PEN081Vd9nVPfmpqKq8fQL99+/aLw/RWP01AHqOiF0NPHA3bBNwZysf0ygzKzI3UzM7t9XMM8eNzsuH6Uf9P3p5NIRYoc2tEWTmTNgF507il6JzGKOvEhgHdGqc0TOvEAaNcNByIAxsdczXNNh72IyZ9xoGve/KD4sA35F2N4v9MaBfts94L8mV8IIbcOOJcZdvca1ZdFuSHKDFMiaEZWowMz9EjdyuCimW9rBI5PdvtN8R2Oc6JKNjne/IFecyLoUi7x+uaGsu+IY+xYFKYFdBlxsgALb/1xYhIbQjTIG/HYX7M5M/BX/idPGbF+cjPshkl+ZlpUJih2nVJmd1ruQZYZ9kX5GmEAbJivdi6b8gDaHuC2NU/gEbLTHY7qqhpjA/NEyto3vRvDQLSYowUz1F13RfkKc9+ccB+35AHuEDVeu2kxAWQZejdvf1kt6CqxAF6Jz8xxkiA6b1XHLCdPMRMHc2XHeML8oPiwDfkjQf4LDwgJtx4EOSdnqCZke/U94N3GBfmxHC9PXBBpVrLrR/kMUaEYD0Z16jYhrGV7XPzVV32Cfl+ddME5M0MgDQzBoTBAF9MhhQDxGxc8LLMPmby07uxdUsxNALQrkFKXlu369i52G7nszw+Ul+Q71c37PMNeYbS6a0zuaZGL9565QZvV9McY8P4BnnOwTHsA/qc2x7aq6r3Xvl9Qn5QWfuGPFqkIWpaJEXzbGcmFmyUi3XTpeU33faKA0YE7BhS9xwsc7ydy/K5eWzbKKkvyA+qG9+QR6vm+fDARqGMB4J8ByFvJoIZUblMrsnZNra7y4gBwJP2mtgH6H326LsKeUyNoXoMyOBu9wRdIzLzITWTYRnjY+5lRm5P3o4jrz2URM/JzM2eUCbtdT47R9W0q5B3NWz6R78YnTsBbmsMs52YsBhguah1GiOczx0RcM9Xd7nLkLdnR1yto03TmkHc1i0dJg5o7JZBvl8csM93HHQV8q4eze/RNBpmAvI2uXonD6PA9iQ9afHpettm57LzjJJquL5C6VFJVqlmcvQ+mOm58PMHgM1Ejwajc3s0vS7FeW24s1eeKtu7DHmGJDEsgGymUkxpAHCPkJ6+mRvLNAg4zt1u52GIH2Nz93Es57a513C9Xd+uNWoaE+R7aZz4IC6YDPAssx0TdGe0b8P2VXQ+KG9XIQ/AGa1CZ0AbvdMARd+mvV6Q7xUHgB0dEwc8sFccdrcYIC2LA9tv1/eRxgh59G0NXBqvdPJsMp83dvRKYQz7fE2CfIWSBNwGbUyJmYqzieUmzMrOP2zaVcgXjQNjYptrbpanbJvtK6YYIrAvbmfdPQ/LXJO8dm1SN0/ZOapuiwHypv9+miReLuXUVcijJ1evLLvr7K+jS+Kgl5bd7WVxwDXdPFU1X5Y/BsgbD9A6MWHr6L7IA5cf/eJimNjqd3xxnyBfLJEI1mOBfJkxxLAtBsh3IUy6DPkYdD7oO8QA+S7EgSDfhVqq+BkFeb+/ax9kVlX3C/IVBV0zuyDf7jgQ5GsKu+JhgnzFAutCdkG+3eYmyIeJIkG+3XEgyIeJA0E+TDkHvYog325zE+TDhIMg3+44EOTDxIEgH6acg15FkG+3uQnyYcJBkG93HAjyYeJAkA9TzkGvIsi329wE+TDhIMi3Ow4E+TBxIMiHKeegVxHk221ugnyYcBDk2x0HgnyYOIgS8tu2bcsWL16c/06R3yqmNvOvW7xspNc75we9BnLY/U2+1rbqE+tdyh8K8vzXwlVXXZWc/i3en3322TwO+O3ysJqum8/3a227pOe6nzUU5Pfs2ZPrwHSRWvrCCy/k3//EiRONxwHxM1Y3iKocx5uG3Ddqpbr87bffNlqpgny9nlIoyPPO7lS1737vb775ptE4wJsE+eqxEAry+/btUxz86z/uq3C0bt4gkAc+/DNRyvOZM2caNzZBvrqx0esJBXmC9IMPPkg6Dkb9m9lhjU6Qrx4LoSBPHc7NzSUdB59++mnjPLBYCQJ5u5jSn//7vYmyEOSrG1toyDdR7zrnwrgS5KvHQkjIS7MLNdtUmQjyAf6Fr6nKK55XkK9ubIJ8OLMp6rXJdUG+eiwI8nHGgiAvyJf+UUzdh3e6eFzI4fomwaZz/2zSgrwgr3j4ZzwI8oK8IH/9DdnWrVuD3SOT+fwM46bKQpAX5JvSVtfOK8gL8oK8IB9dA0eQF+S7BuOmPq8gL8gL8oK8IP9ldSh28dZUv8+se/LNjzA1BfJ+5xXkI4I8Fe3+HlnLY0OXx2OPPRYd6PoFfuz7pP3hte+W1cMPP6w4iI0JsQd7at9vZmYme/rppzVXLIOzZ8/K3CIytzfffLO1MbBr167WfrYQ7/NIzZMv9fdVTz4iY7vUYtL14xzuU736q9fz589na9euVYNSvhtMA4K8xBZMbIKFP1ioLLtZlocPH85vIQF71WE367Br9SbIC/IyG2lAGgikgZUrV+aQP3DggMo8UJl3Dcq+P68gL6HJbKQBaSCABvgXSnvIbXJyUmUeoMx9A7OL5xPkJTSZjTQgDQTQwPbt2y9CHtifOnVK5R6g3LsIZp+fWZCXyGQ00oA0EEADS5cunQd5oO/TzHUu3eMv04AgHyC4ywpe2xSQ0kA6Gjhy5EgOeLsnv3HjxmxiYkKQl/82rgFBXiJrXGSCWTowU12X1/Xs7Gw2NTWVAXeG6l966aX8p3Tcp1eZlZeZysVPuQjygrxMRhqQBgJpALgD+c8//1xlHqjMU28sCPISmsxGGpAGAmlAkPfTO00d3FW+vyAfKLirVIryygikgTg1IMjHWa9tjldBXpBXL04akAYCaUCQF+RDNwgE+UDBHbpidT2ZiTTQPg0I8u2rk9jjRJAX5NWLkwakgUAaEOQF+dCNCkE+UHCHrlhdT2YiDbRPA4J8++ok9jgR5AV59eKkAWkgkAYEeUE+dKNCkA8U3KErVteTmUgD7dOAIN++Ook9TgR5QV69OGlAGgikAUFekA/dqBDkAwV36IrV9WQm0kD7NCDIt69OYo8TQV6QVy9OGpAGAmlAkBfkQzcqBPlAwR26YnU9mYk00D4NCPLtq5PY40SQF+TVi5MGpIFAGhDkBfnQjQpBPlBwh65YXU9mIg20TwOCfPvqJPY4EeQFefXipIFkNDA3N5etWbMm/293/t899HzzzTfnfzV7++23B7+2fdf77rsv++6775Kp89ghPuj7CfIyeAW7NJCMBnbt2pVDdvXq1dmlmFesWJEtWrQou+OOOy7J9W+99db8+588eTKZOh8Ewdj3C/IyeAW7NJCMBoD8lVdemaU6vfPOOznkP/nkk2TqPHaID/p+grwMXsEuDSSjAUFekB8Exdj2C/Iy+GQMPrbg1fep/hCXIC/IpxY3grwgL8hLA8loQJAX5AV5GV4yhpea2PV9q/d8YyszQV6Qj03Tg76PevJq1KhRIw0kowFBXpAfBMXY9gvyMvhkDD624NX3qT4yIcgL8qnFjSAvyAvy0kAyGhDkBXlBXoaXjOGlJnZ93+o939jKTJAX5GPT9KDvo568GjVq1EgDyWigKcgvW7Zswft1Lly4kK1atSrbsmXLgn3uhpmZmYF53PyjLOtlOOk1dAV5GXwyBj+oxav98RtgE5AH5pOTk/PYa4AH4Js3bx4IcfLs2LFj3jmaWBHk49d40ccEeUFekJcGktGAb8gDcXrxzPYu/GPHjuU9ePbZNAj0NAo4x7lz5+yQRlJBXpBPJtiLrR2tpyd+1Xl6de4b8pCYHvjRo0ez06dPZ8B6fHw8A/AGfVLyMAP7XhP7+u3vdVyV7YJ8eppXT169ODXspIFkNNAE5OmBA3dgztSrN26NgF5QthGBXsf3Oq7KdkFekE8m2NWLS0/sqnPVuW/I29D88ePH8yF6Ura5w/fA3Xr2vYDMSAC9+IMHDzZ6b16QTy8G1JNXL04NO2kgGQ34hjwAt2F5hulZBvLT09M5z9116+mXgd7y2b150iYmQV6QTybY1atLT+yqc9W5b8i7IDaIF3vyBn3b7x7DcrGXTwOhqSftBfn0YkA9efXi1LCTBpLRQBOQ5x46w/TLly/P1q9fX6knz0jAxMTEvPv41punceB7EuQF+WSCXb269MSuOled+4Y8vW5+I79u3bo8BdrD9uSBOS/LcX9qZ1DnHAz/cz6fkyCfXgyoJ69enBp20kAyGvANeRfANhxvw/PsYxswp7dv+9kOvOn586Bdrwn48wCfT9AL8oJ8MsGuXl16Yledq85DQp5evkGeZaDOBPTp/Zf14IvA56l7+4lecV+ddUE+vRhQT169ODXspIFkNNAk5A3a9NrpzdMDB+hMtl4HzHaOOscWjxHkBflkgl29uvTErjpXnTcJ+SJQ27guyKcXA+rJqxenhp00kIwGBHn91WxqjX1BXgafjMGnFtz6vgt7bYK8IJ9aXAjygrwgLw0kowFBXpAX5GV4yRheamLX913Ys02tTAR5QT41zasnr0aNGjXSQDIaEOQFeUFehpeM4aUmdn1f9eQFeUE+NR9QT16NGjVqpIFkNADkx8bG8t+t89v11OZ9+/bl3//kyZPJ1HlqUC9+X0FeBq9glwaS0cDevXtzyAH6lOezZ88mU+dF6KW2LsjL4BXs0kBSGpibm8tmZ2eTndWLT+u2lSAvg0/K4FNrxev7pmXoqm/Vd1EDgrwgL8hLA9KANCANRKoBQT7Sii225rSuFr40IA1IA+lpQJAX5NWClwakAWlAGohUA4J8pBWrFnt6LXbVuepcGpAGihoQ5AV5teClAWlAGpAGItWAIB9pxRZbc1pXC18akAakgfQ0IMgL8mrBSwPSgDQgDUSqAUE+0opViz29FrvqXHUuDUgDRQ0I8oK8WvDSgDQgDUgDkWpAkI+0YoutOa2rhS8NSAPSQHoaEOQFebXgpQFpQBqQBiLVgCAfacWqxZ5ei111rjqXBqSBogb+H91OUUxybqXtAAAAAElFTkSuQmCC"
    }
   },
   "cell_type": "markdown",
   "id": "0be010de",
   "metadata": {},
   "source": [
    "### 编码器和解码器架构\n",
    "![image.png](attachment:image.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b6b3341d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 编码器\n",
    "\n",
    "from torch import nn\n",
    "\n",
    "class Encoder(nn.Module):\n",
    "    \"\"\"编码器-解码器架构的基本编码器接口\"\"\"\n",
    "    def __init__(self, **kwargs):\n",
    "        super(Encoder, self).__init__(**kwargs)\n",
    "\n",
    "    def forward(self, X, *args):\n",
    "        raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1cb3dc98",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 解码器\n",
    "class Decoder(nn.Module):\n",
    "    \"\"\"编码器-解码器架构的基本解码器接口\"\"\"\n",
    "    def __init__(self, **kwargs):\n",
    "        super(Decoder, self).__init__(**kwargs)\n",
    "\n",
    "    def init_state(self, enc_outputs, *args):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def forward(self, X, state):\n",
    "        raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9baa3ae1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 合并编码器和解码器\n",
    "class EncoderDecoder(nn.Module):\n",
    "    \"\"\"编码器-解码器架构的基类\"\"\"\n",
    "    def __init__(self, encoder, decoder, **kwargs):\n",
    "        super(EncoderDecoder, self).__init__(**kwargs)\n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "\n",
    "    def forward(self, enc_X, dec_X, *args):\n",
    "        enc_outputs = self.encoder(enc_X, *args)\n",
    "        dec_state = self.decoder.init_state(enc_outputs, *args)\n",
    "        return self.decoder(dec_X, dec_state)"
   ]
  },
  {
   "attachments": {
    "image-2.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcgAAADnCAYAAACaLMa9AAAgAElEQVR4Ae2dO6wcy2GmGdtYHwICfCVfgMd6XMiCLNJKvA6Iw8XaCg/9AG5wA9IrwJvtoQ0sNrFBrBM/AhKrZBVRWAPWAwZIyLBCEl7lDDY2GBg3uAmTzZW08Y38j+r0dPc5p6dnpqb7CxrVXa/uqfrr/6pqXrd+9rOfNR62gRpQA2pADaiByxq4ZYNcbhDbw/ZQA2pADagBNCAgXUG7g6AG1IAaUAMdGhCQHY3i7NHZoxpQA2pADQhIAenMUQ2oATWgBjo0ICA7GsWZozNHNaAG1IAaEJAC0pmjGlADakANdGhAQHY0ijNHZ45qQA2oATUgIAWkM0c1oAbUgBro0ICA7GgUZ47OHNWAGlADakBACkhnjmpADagBNdChAQHZ0SjOHJ05qgE1oAbUgIAUkM4c1YAaUANqoEMDArKjUZw5OnNUA2pADagBASkgnTmqATWgBtRAhwYEZEejOHN05qgG1IAaUAMCUkA6c1QDakANqIEODQjIjkZx5ujMUQ2oATWgBgSkgHTmqAbUgBpQAx0aEJAdjeLM0ZmjGlADakANCEgB6cxRDagBNaAGOjQgIDsaxZmjM0c1oAbUgBoQkALSmaMaUANqQA10aEBAdjSKM0dnjmpADagBNSAgBaQzRzWgBtSAGujQgIDsaBRnjs4c1YAaUANqQEAKSGeOakANqAE10KEBAdnRKM4cnTmqATWgBtSAgBSQGzPHFy9edMa9fft2I14T0UT2rQF0+PLlS7Wod+1cAwJSkV0SGXB89OjRKu79+/fNmzdvVseTJ0+ahw8frq8DS8IHDx5cOoi7devWxnF2dnbpXlzfvXu3Ibxz507z9OnTVfrJyckqjnjO923A3q9u6KMLtIheMpkjrq3DpBG2016/fr2hTzQbDUYDxFF3tEg5DvSaeMLkN6xbOzftHwEpIC8N7vPz85UBIKQuI2ibAhAlH6B79uzZ6py4thAxqbb5UBdlyUta0kvDKc/bdXo9LzO6Tn+ycoxu0Go0g44uLi5WkzvOmeQl7d27dyudATvSorn2/SjTTqNM8uW+5EndpKnR+epQQArItQFgJBnsmAVmVBpBjKLLFEojIR0TwcBShnoAaK5Tx+PHj1f3YIafe2U1yfXp6emlMmV5z+drTF19iz5v377dPH/+vHn16tVKG4TEMylDK5xTtq039Bhtp+52HtKzM5I86Jp80SL1cESvxLfrTVnD49engBSQawBlRo4BMejbRhCjaJsC+UoYYgzJG5NIfbkmxIxiOISUweA4Z8XZNaMvy3t+/AZ00z5EZ+gCPSVEN+xgoBm0A7yYZAWW3IM8XRM08ucZ2pM84kkvD1ap2TWJxttQTX2Gx69PASkg1waB6WAAvO8XUwikcp2wNAXyMHvP+zwYA3HlBykwsNKwsuUak+E9znK2TjyrhaRrNsdvNlP0IXpAg2XIOXrkQGNoq30vgHnv3r2VRtE46cTl7QBCrsty0V5C3vdE57lGs7nuumdZl+fHqV8BKSAvmQJQA24MaAY95zGEMowhYEiADbhSJrNwrmNIgJN4wpQjTH3kBcq55p7lNfEazHEazNT9hhbQFWBKSBy6BWDsfhDPh8syiUNrmXwB11KrmdRRhgkZ1ykXPRICTyZ5iaOO8jq6nvr1Wt9hdS8gBeQaPplFYxCZpWdLC2ixRYUplCaDYTAjL02HQU2e9hHYZtBzH0ytvbokHcOhTt5jSn7Dw5pFDe2P3qLNhMShlUCLyRXnWSlGwylLWrZJ2xqNvvNaAS/QRLtZbSaN+qk7QE284Xx0KiAF5BpADPasBgFUgIaxlOaCAWAyhDGaXGcF2WUSqSflyZstqtTPijT5+AAPeTA8TWg+ptOljevGoQ1WiWgjIToNDAFetJg60Wg01ZWefISUJQ86ZNWI/gBkyhNPeq5Ji47Lejyfh14FpIBcA5LBXw5sDIIZNGG2mDI7L02oNB3MgmvC8oihkMY9AsJAmPrIQzr3Sn7O2zP38hk9n4cRXbcf0QsHeknIxA5QcZRbpVyn3ugpWuW61Gd0W+oajUaX5EeLlOe8jOdcjc5ThwJSQK5NBDMBiMzOYz4xBEwh5kK+0kiIz0oQo4kptUPqJG/iUyfXnOe6rLs8TznDeZrRdfo1MIpWCFk9oiuO8m2AttYAJtoe0hQQLJ+jzMs5dUaryVfmSZzhPDQqIAXk2hB4/yUQI2SbtTQEzmMGCTEC4vPeTRuQpPGBCd5L5EMQ5SoVo8GwCDEvQvIHtlxzrtnMw2y27UdWaQFYtAIcy69vlBot70d+NE3eUrvUmV+LYss29acseSkbLVI/59Er52V9KWc4D80KSAHZCSAGPoOc9/4CSuDGeduUEk9+DKQ0B/JiIAFvmUZdqZv7cc79kpe6OC/LeD4P4xnTj+gsMEQraIbt1XLShX7QXLt+tMQBENvvZ6NPDsqVdVEHcSmLFjMeuHfiu+7Xvr/Xx6lbASkgN8zEwXycg9l+s9/UwLQaEJACUkCqATWgBtRAhwYEZEejOAubdhZme9qeakANHKMGBKSAdOaoBtSAGlADHRoQkB2NcowzHZ/ZGboaUANqYFoNCEgB6cxRDagBNaAGOjQgIDsaxVnYtLMw29P2VANq4Bg1ICAFpDNHNaAG1IAa6NCAgOxolGOc6fjMztDVgBpQA9NqQEAKSGeOakANqAE10KEBAdnRKM7Cpp2F2Z62pxpQA8eoAQEpIJ05qgE1oAbUQIcGBGRHoxzjTMdndoauBtSAGphWAwJSQDpzVANqQA2ogQ4NCMiORnEWNu0szPY8vvb87LPP1n/nlL91Mvz5X2Ydazt8+umnQvCGfi8gb9hgmv3xmb19dvM+44+vPebXBo6Fm40FASkgnVWqgQ0NAMfyT4GPddXkc/981Utf0qcCUkAqAg1fDWypgQBSQ72ZodbaXgJyXD+6gtzSSGodED7XuAFhu/283QTkvPQjIMf1p4AUkK621MCGBgTkOEOtdYIlIMf1p4DUHDfMsdZB7nONG+Rj2k1A7q+tx/TPTcsIyHH9KSAFpIBUAxsaEJDjDPWm4NpXfgE5rj8FpOa4YY77GrTeZ9yg3Ue7Cch6+2ZM/wvIcf0pIAWkgFQDGxqoEZDv379vHj9+3Lx48WLjeUtovH37diOdr3s8e/ZsI74sR71ddZ+fnzfcu8yb83fv3jVv3ry58iBfyhwiFJAC8qACPITovec40dtuV7dbbYAEUPfu3VtB7u7du83z5887xzsgIj2gA4wA7OHDh82TJ082QFZqAYhwEEe5XJ+enq7AnGvSUo64O3fuNGdnZ70H6ak35fYdcn/6dN/3Pfb7uYJ09eCgUQMbGqgJkKwIgVSgByyBIKvJrpVd0gO5i4uL5uTkZA28gK4NrcRj6pR99OjR6kg8oKOuNiDb9bShkPLt+H1e8wwC8uqJYbtPBKTmuGGObZF4ffOBdextVgMgAR2rPuBWQiltC6yA1suXLzc0XIIT0AEIVpeBbOogBMCsAKmLI3kIKUse6iOtrJd46gXeDx486D1IJ195z32fC8hxY1hACsiDDtx9G4X3u55RHBqQQOv27dsrQAFCDL7vAG68T9jVt4AVsJHGOXm78hGX+jkHjoEKIVu0bPFyHoCmDBCl7qHD9yCvp7u+vjlUvIAUkL2GcShRet/Dm8mhAclKLR+2CbgIgV1WhFwDJfSSvMAUCGbVyQdzeC1dRxuWuQ/1BZCJK8M2IEmjTJmnfS4gD6/pMb4iIAWkgFQDGxo4NCD7zAyoBYp9eYjvyke5NhTLOoBrezs0oCvzledl+tAKsr01W9axj3Oekz7dx73mdA8BqTk6aNTAhgbmAsgAjJAP9QSAiWdlB9iIZ9WZ9xy7VpxlXCBAPaxSqeOqr3scEpI8p4C8+SpWQGqOG+aYwW948wE1lzarBZDZMmXlxwHE+ARrrgmzvVq2PfFAKyAkBH5s0ZZxADJH4lMP1+V3J3mfs32v9n14Xp4xdZXhIbdZeQ4BefPxLCAFpIBUAxsaqAWQgVXCACnXfWFXPoBJfF+ZwCzpwJRVJ1CkLGBOWkJgWK4MgWj5qVrud0gw5jkF5M3hSNsJSM1xY9BnUBmOG1RzaLdjBiQfogFcwK3si5sCkrKUoS7ao1xNkgY4S+CSni3a3BdY8unXEqJJ22coIMeNZQEpIC+ZyD4HrfcaN2j30W41AJLtyvb3C/nqB8Bpx+eTpVnpASTiWAXyXcqsDtshkCNvQAbkuH716tWqHO9NAkHqIuTe3IM+AIa5LyHPluciX7kVzNdE9tFvffcQkOPGmoAUkAcduH0D2vhxA3qqdqsBkNnaBEhXHdnGTJm0A/GUBXxtOHKdcknLSg/4USbpqY+6soXKvRKf+7TzJz0gzfW+QwE5bjwJSAG5HuT7HrTeb9yg3Ue71QDIfbzOpdxDQI4bawJSQApINbChAQE5zlBrBa6AHNefAlJz3DDHWge5zzVukI9pNwG5v7Ye0z83LSMgx/WngBSQAlINbGhAQI4z1JuCa1/5BeS4/hSQmuOGOe5r0HqfcYN2H+0mIOvtmzH9LyDH9aeAFJACUg1saCCAvOrTo6Zf/QnbGtpIQArIjUE+ZqZlmXFCst3m1W4A0mN+beA4vdk4dQXp6sGJhRrY0MBnn3125XcPa1gZbfsMP/7xj5vvf//7i3itn3766UY/C8xhYApIzdFBowYWq4H8gICgGAbFUttHQGqOizXHpQ56X/cvYMDPwfFzcrbJL9rEtvhFWwhIAak5qIFFaoCfisv7rGzVCoZfgMG2+HlbCEjNUWNQA4vUAD+GHkC2/4VDQAhLNCAgNcdFmqMGqAHmb6yAJP/EkR8qVxtqIxoQkAJSQKqBxWmAf+TI6jGgPPQ/bsSUDesBtIDUHBdnjhpQPQZ0qL44Pz9f/VckH9L5xje+sTrnvxwP9Tzet05NCkgBqSmogcVpgBUkx0cffdR87nOfW53nfx6FVZ2wOkS/CEjNcXHmeIiB5j3rNN379+/7NQ89sNcDBaTi6BWHpl6nqdsv0/WLgJyuLeeoSwEpIAWkGlisBgSkgBwCu4DUHBdrjkMDw7RlGKeAXEY/jx3PAlJACkg1sFgNCEgBOQRPAak5LtYchwaGacswTgG5jH4eO54FpIAUkGpgsRoQkAJyCJ4CUnNcrDkODQzTlmGcAnIZ/Tx2PAtIASkg1cBiNSAgBeQQPAWk5rhYcxwaGKYtwzgF5DL6eex4FpB7BCT/OecxvzYYO/hqLrcUnfI7rB988MFixmXNmqvx2QTkHgGZfw8wvLX+J4U5tEWNA3vbZ5pDv/gaNsfZtrpYWnkBuWdAfnLxl81f/f3/9ZhBG9CXmPAcTYPXtQSt/o/v/EPz35//YPbjcc5a3eX4E5AHAOQ//UvTeBx/G8zZdAJIdXr8OqUP56xVAblHiO20sf99Vq7paDq71NkUdQvIeWg0XiMgx30YyRXkHuGr6Wg6U8BrH3WoVbW6D53Vfg8BKSDd7h255T3nWbmAFJC1w2sfzycgBaSAFJAbHzQSkAJyHwCq/R4CUkAKSAEpIEdqIO/x1R7Oebdjl5AVkALyEiC/84//r/mTP/9f67gX//yvzV///U9Xx3/9i+80D//4z5rf/aP/sk5vG8OP3v7/df6US0jdyT+UL/kJuX/K1BbO2XSOYQWJhkqtlpqKVn/n9/5gUD985arUW87HaLUso1bHfShml7AbU7eAFJCXDOQ3f/tB8+f/+8fNJ//tfza/+munDdcc54//dHVgKIEWhvCN3/5Pl44//Zv/sy5DuV/6DyfNF3/j3rqOGAf1UD/36TsoT1rK1BYKyMNuQ/7nP/zjFSDREjqLVtEMekXHJbRIL/U6pVZ5FuqvTaN5njlrdQz4rltGQArI9aBmkP/H3/391XWglQGGCfUZQDtvyjCjx7hyXYZD9SVfX71JP3Q4Z9OpfQUJ3Jh4oYG2loBin1bbeUsNMWHL5K+MHyqTfNfJk7yHCMdq9d27dxvb79eFyxzyLRaQdHw6n9+dzHnZqcR9+ctf3ki7d+9e89Of/vTGwqnZdIARgGTwYjBclytIzsvrcmbeB7KyzrYpYCi0x1UHdbTL1nI91nRKjR3i/P37983t27ebJ0+eNG/fvu3Ucc1aZWUIHJmAATS0VK4gSSuvgWk00wcy6hyC6lU6Jb2vfO59yHCsVr/97W83HKVOf/CDHzRnZ2eX4sp0/JRr/LNPX2X+ms8XCUg68M6dO82LFy9WB519fn7e2eFtSAJHBDKmU2s2nQAv5tOGXvu6HOxdaZjXL//K7dVW15/97d+tDSrl+owq6ccQjjWdMdqZsgyafvTo0Xpycnp62jx//vzSRLBmrQJF9IVmAWEbbmgrOyFtHfXpLpO/EqYp21cm6ccQbqPVEpJ4Hx7YpUcmXhcXF83du3dXYHz27NnR/xTjIgFJ556cnKzgSKc+ffp0dXR1OnGB5DZwpJ6aTYdBHqhlBQn4mBXnPUTOc8QUeE3lkfi8J5M62saD6WTFSp3le0PlecCdemsKYzrl6z+W8w8//HBldO3nffjwYfO9733vKLT6pa/91gqOARhaC+iiU15fNENc+XopRxrlonE0yXXKEKZ+znlvs9Rned7WeFnHoc+j1T6PuyoeSLKQ6INjygNIDlaOeCs+m7RjDBcJSFaQbDEBPjqNjh/aCiAfHd213XqTTmdwItRDD5au+wMiVnyYCOmYBAfXMZKUS55cJ2+uMYqUIw3wYlwl7MhDGmWov++gbOqtLYzpfPLJJ82xHV/5yldWmi6Bwa4Ks370XrNW0QRw5BmjH/TW1iFpyRPtlLBLWbSZVSh1A8tS82UZNNyn1a73L3PfQ4fR6k38qszLyvHzn//8xnZrmYdzVo8sODh/+fJl785cu1yt14sEJB3IFhOdcp1ZDsJAIFlJBqw37dSaTYeZc1Z9DOaYDcaT2TXnOcoBn7zEAT7yYzQYSSCYrduUS5lVePGXq4kDg7jrqNV4tjWdm+pnqvxMBtlWRY9M/BgL7QlizVpFM3y9I/CLzogfWkGivTbsMnFrx7N1m8kZ2mVsrCZ1PRqNbqknGq8p3Ear5bZqud3a1iNeSp8QkoaueBurne+YrhcJSGY5eSM5s5xctzuvva26DSRrNp22eeRj8gCR75Tl+2GExJWDP7Bjdk25AC3GlbwYTEwHw8F4yMNBGqaU6zJMmdRTS7iN6bR1tu9r3nMfMq+atRo9BJDRHyHf0S21mjzRDLpCv2gUDWZXI/HJV2o19Wf1iG6z6qRcqeNoP/XUEo7VagnHaLQPkuip/CwHOxJMvPq8NfXVHC4SkOUnsOhAOrU9g6bTgGHXp1WJH7OKrNl0GMilSbAKxBAwE4wkJkEYQGJUf/Hdf1y9J8Nr44MRpUFQH/m7TAKDidGRTp3kT16uMalc1xiONZ2aDSHPVrtW0UPghz6jNXRTajV5olUAStzQNmpba+g6ICWN+5U/UMCkcK5aBZDRRBl2+SLb8yw4ks8V5B6/Q5hGP+awdtMJIDETAIYZYDh9K8jMwplBl7CLwcS0cp2QchhUrgPgXGNGgPdXP/z1S8aU9FpCAXnYHwoI/LIdChiHVpDoDJCVk7hoKdrPdUJ0Tf25pjz15Dpa5f17xkHiawvnrNVdMmGRK8hdNuhQ3ccCSGbHWfkByAAsM3PirmMAfYAsZ9wYDu3Chy6AJnUzY8+92ivN69x3X3nmbDq1a5U+5hmBUoCFZtBPtENInuvooQ+Q5VgAhtEquuRe5f2IK1ea17nvvvLMWatDnrttmoDc4yq4dtNhZg2wmDVnRdg14ybuOgOb+jCedt7U3Y7vuq7VcHjWOZtO7Vpdtf+/b99nRYjW2noDkl26asdF++14tJr622nta7U6j99fLaEqIAXktQykbQZeC0g1cNgt5pu0/5wncyXQpj4XkAJSQI78q6M5m84xrCBvAoil552zVqeGYlmfgBSQAlJArj95GHMQkMezOrwO/AXkuO1fASkgBaSAFJAjNXAdONWQR0AKyI1BntlwLaGzcmfltWjxqudQq2r1Ko0sId0V5AFWkMzmPObRBoBkjkYRQKrTeejUFaQryOqNCtPxmF8bzBWQalWtzlHbN3lNriD3uIK8Scccc15+bio/WHzMr8NnHzfrPqZ24ycju35m8pheg8+6O50KSAE5+cqb32DkdxkduLsbuLbtNG3LP/vw/4W25zTtObd2FJACclJzyF/eXPXHqnMbSL6e4zRY/nGC/4a1/46z/3bdbwJSQE5qDvzlTd67cutK09m1gW1TP3/DFK2W/0KxTZ2WnZfmBaSAnBSQ/NdmTMetq3mZxdzMn7cCotXyfwzn9jp9PePHoYAUkJMBkg88xHAI3boaPzA1td22HW8FoM9Sr36wbLdtfoyaFpACcjJAsmLEcMpVpFtXmk6Nxpi3Ak5OThreh0S3frBMrba1KiAF5GSAxGjYqvrSl77UfO1rX2swH7euNJ226dRwfXZ2tgLjV7/61eajjz5aTer8YJlabWtTQArIyQCZ75QxG//Wt77V8CEIt600nbbp1HAdrX7zm99svvCFL6y+C0lcDc/mM9QzZgSkgJzcFADkxx9/PHm9Gkc9xjGXvrh//35zenqqVvXBTg0ISIXRKYxtDFBACrJt9LPPsgJSrQ7pTUAKSAGpBibXwJDp1JQmIAXkkB4FpOY4uTm6gtR0hkynpjQBqVaH9CggBaSAVAOTa2DIdGpKE5ACckiPAlJznNwcXUFqOkOmU1OagFSrQ3oUkAJSQKqByTUwZDo1pQlIATmkRwGpOU5ujq4gNZ0h06kpTUCq1SE9CkgBKSDVwOQaGDKdmtIEpIAc0qOA1BwnN0dXkJrOkOnUlCYg1eqQHgXkHgH53e9+t+EfzOd+AMivf/3rs3+d9CN9OjTAjjXtRz/60SL6L3+YPPcxOWet7nKMCcg9AhJweMyvDXY5QA9Vtzqdn07p00Pp6VjvKyD3DEhmcscqFp/78nYUfTlX0+F1qdXL/X3M+p+zVnfZLwJSQArskRqYs+kIyPnAEYDMWasCcqSB7bLhxtSt6Wg6Y3RziDJqVa0eQne13dMV5B7hq+loOrUZQN/zqFW12qeNJcULSAHpFutIDcx520pACsglgbDvtQrIkebY16BD8ZqOpjOkj5rS1KparUmPh3oWASkgXUGO1IAryHlB5FAmvI/7zlmru2w/ATnSHMd0Su2z8tevXzcc5WvjmcvrFy9eNO/evbsUl/RHjx41T5486UxLnoTUwaC96mg/T8rXEM7ZdOag1S49RzfPnj1rHj9+PKlWGRupv7ZwzlrdZVsLSAG5HtQMIkzjzZs36wOjLK/v3bu3AdEI9P37983du3eb6xgF5kXemFhXCHB5ptRfWzhn06kdkNfRKloe0g/6uri4uFJf19EqwD07O7uyrkNpeM5a3WWbCkgBuR7UgI1BftXx9u3bdZm2OFkZ8vNdwLKdVl5jOlcZCoN6yODK+g5xPmfTqR2Q19Xq0GQNjaLVvh2RaOo6Wr1OntR3iHDOWt1lewpIAbkGGYOcA8G1z4eMpi1QTOcqsFH/ycnJlTC+yX3bz7Hr6zmbTu2ABGrRRnmOrq6zKow2zs/PG1aSue4Kr6vVm9y36z67jJuzVnfZbgJSQK7MAZNhSypbrDknnoH/4MGDFfQYaEPwY3XJyhD4DQkX07lqBTlUvoa0OZtOzYBk5ff8+fOG7X62/3OO9oDmdbWaFeRVq0i1utwPYwlIAbkCGcaCUXAArpwnPmAElqT1AYp0yjArJ+zLV870MeO+g3x9dRw6XkAexjgBIe9fZwci52iPFSFh9Iqu+nSCPsu8fflKQDI2+rTKPfvqOHT8nLW6y7YVkAJyPahjKggu55gDpkNI3MuXL3u3sDIjJ2TlOQRSzIn6dinuXdfN8w8Z8K7vv8v6eV01908JrZwDTjSXazSIdvvaibzkKXXblTf1daUdS9yctbrLPhCQAnJtIAyibE8l5NN5GA8h6QASw+gSZftTp5hTn8kSz8GMnHsNHdy/636HjuP5BeRhVpFo8PT0dKUh3g5AR9EmaZnUEdelE/qufO+Ra1aTXXkzKSR9SKekDe2adNW9r7g5a3WXbSggBeTaFBhEGE0ZRnyYw8OHD9d5E58QY2ivGDMz7wJcDAwz4wDA5dc+uE4a9eQ+NYVzNp1jWEGiN/oA0KHbaAPtAM9ct0P0yPZsqatoFc2183MP6qQc6Wid8tEn1zlnRdouX8P1nLW6y/YVkAJyPaAZRBwIrjzHPDCczNLbgozhdIEQ8wB8pRlRvjQo0vjARVke08OU2veq6XrOpnMMgAwUgVPO0Qd6YwLWpR80dvv27RXQ2lrq0zH1ldrkXtwz5dEBq9hc1xjOWau7bG8BKSDXA5tBxDZRGQIvBj8rSM6zyosoMSFgVxpI0hKSpwQg5oKBkU6drEwBKXXwqcRXr16tngGTJj711BbO2XSOAZDtLVb0Ua4m0WqpH7ZKgWMZ19YUaaWe2++lZ+JGfKlV6kUP7fpquZ6zVnfZxgJSQK4HNYOI2TEhR2bmmEJEiDEEkjkfgmPKYTyBIrDNDBzT4p45cm/iOdhGA6Kpp6aQZwUkNT3TVM9yDIDMFiv9gL4Cx+iFMHqmXch/Ha2iTfJShsldVqIZD22tcm/S2ivNqfpiinrmrNUp2qevDgEpINcGD/A4SrHEbNpxXfFlnqnO93WfMc87Z9OpHZDoog27tnbTp9tq6Lrlr5svz7XPcM5a3WU7CkgBeQmIuxTb3Oqes+nUDsi5aWnXr2fOWt1l2wlIASkgR2pgzqYjIA/z9ZVdmf2ctbqrNqNeATnSHMd0iqaj6YzRzSHKqFW1egjd1XZPASkgXUGO1MCcZ+UCUkDWBqtDPI+AHGmOYzpL09F0xujmEGXUqlo9hO5qu6eAPAAgWXl4zKMNAEltg3qK5wkg1ek8dAg6xSwAAAZwSURBVDrn3Y4p9N5Xh4DcMyAxHo95tUHf4DrmeDU6L42mP49Zk4d4dgG5R0AeooMPcU++ON33nbRDPI/3nNd24ZT9yXcp86MVU9ZrXfPQnIAUkJNvEfKLOWzpaBLzMIk59yO/vlP+UtScX6uv7ebjUUAKyElBxsqR7Zyhf1NwoN58oNpmu2kzfneV31Gt+Vdw7Pvd9P112lVACshJAcnvVub9DreuDjewrzP4l56HtwKiVc6X3h6+/s3xKiAF5KTGwI88x3TcutoccJpQPW3CWwHRKv9iY9/U0ze19IWAFJCTGQMfeIjhELp1peHUYnTt58hbAaVe/WCZem3rREAKyMkAyYqR93T42x9CzMetK02nbTo1XOetAHY8suvhB8vUalubAlJATgZIjAZIfvDBB80Xv/jFlfG4zarptE2nhmv+05GJHJr98MMPV/9VynUNz+Yz1DNmBKSAnNQUsnX18ccf+8lAtTWptqYGB1q9f//+6hPXbq/WA6Wp+3mb+gSkJja5ibG1CiC3EaZlNax9aCCA3Me9vMfxaVpACsjJQSYgj88IlmreAlKtDmlfQApIAakGJtfAkOnUlCYgBeSQHgWk5ji5ObqC1HSGTKemNAGpVof0KCAFpIBUA5NrYMh0akoTkAJySI8CUnOc3BxdQWo6Q6ZTU5qAVKtDehSQAlJAqoHJNTBkOjWlCUgBOaRHAak5Tm6OriA1nSHTqSlNQKrVIT0KSAEpINXA5BoYMp2a0gSkgBzSo4DUHCc3xx/+8If+S7u6mlxXQ0Y2Nu0nP/lJ8+rVq6N41rGv0XLjJwECUiPTHNSAGlADaqBDAwKyo1GccY2fcdl2tp0aUANz0YCAFJDOHNWAGlADaqBDAwKyo1HmMvvxdTiTVwNqQA2M14CAFJDOHNWAGlADaqBDAwKyo1GccY2fcdl2tt2xaoD/hPR/IdVvqV8BKSC3mjk+evSoefr06aqOt2/fNg8ePOg9Li4utrpXKVzPNbI+DVylQzTapUV0HC1Td67boRBdjvYEpIDcClrv379vzs/Pm8ePH6+++3h2drauD6Mirc/IjF+O0eyzr1+/ft3cvXt3pUfO28ezZ8+aUqd5toAw1+1yuUbzyWM4bw0LSAE5yWB/8eLFGpCZwZ+enjYcWVVqJvM2k1r6F5B1ATDP15VO/lKvaDj5DZerWwEpIEcbATNpVo6ZUcd4CMutKgxmyLA0oOUa0C76Hv0Bu6wI2yGabeuRMux28JYB5wDyzZs3vccunts66xsHAlJAjgYkA5qVI9tZvC+DsWA8MaisHAlv37691X00j/rMo9Y+QX9jtljv3LnT3Lt3bzXhQ9domYO6Tk5O1tdtuNbaDj7X9mNGQArIrcGFmWSLlVk4BuUKcvvBqcGNa8NM1PraryudOCCIfgEiZZn0sTtS5vcDOuP6pK8vao8XkAJya0BG5C9fvlyBEUMp33t0BbksU4keDhWiP1aCfVukz58/X60Gy+djVZhPZDO5Y8LHNZoOINl2dfW4LC0LSAG5NSAxEGbeea+nbTzltefLMphD9Dd6ZLs0emyHgK8EXWCYfHlmVpTtFST1uopcjoYFpIDcGpCZaRNiThycYzQYESaDscR4DJdjMIfoa/RXArD9DO10gMdRAhJoMumjbJmf70+S1q7T63lqWkAKyK0GO8bCTBuDAILAEDjGRGJUZZxmMk8zqaVfS6B1PVNfeglItEw+ypf52XINOLvqNm5e2haQAnIrQAI+jCUmwvs0+ZADsAwgAamryHmZR60wQIt8arr8FHV5zvuT0WX5GgJIdFv+0k65miRNQC5HxwJSQG4FSMwC02BmnVUjMOTXSm7durWCZ0yIfDk3XI7J7LuvM1nru29fegBZlgOkHOi7jPd8GfoVkALSga8G1IAaUAMdGhCQHY3i7HAZs0P72X5WA2pgSAMCUkA6c1QDakANqIEODQjIjkYZmlGY5oxTDagBNbAMDQhIAenMUQ2oATWgBjo0ICA7GsXZ4TJmh/az/awG1MCQBgSkgHTmqAbUgBpQAx0aEJAdjTI0ozDNGacaUANqYBkaEJAC0pmjGlADakANdGhAQHY0irPDZcwO7Wf7WQ2ogSENCEgB6cxRDagBNaAGOjQgIDsaZWhGYZozTjWgBtTAMjQgIAWkM0c1oAbUgBro0ICA7GgUZ4fLmB3az/azGlADQxr4N6seSqb8rRAlAAAAAElFTkSuQmCC"
    },
    "image-3.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAKIAAAAvCAYAAABg3BEdAAAGnklEQVR4Ae2avWvbWhjG+6ecf0JjyNIpXaLtmg6COwgyGALBFIK7iEARGYooBBMwplBCoeByIYVeNAS0BGUI6lB8h6BAQUNAQ0BDwctTjizZ+nIkWR+Or9/BKHbso3Oe8zvP+5736MV0OgW9SIN1M/Bi3R2g+9Mi4AwQiBQRnkVEJBAJRAKRwjKF5ZABckRyRHLEcDXQlZyRHJEckRyRnJCcMGSAHJEckRwxXA10JWckRyRHJEckJyQnDBmoxxE9B5ZhwOCv/9xnscLCAW7G1YNzG+hnTOBuoUvXA+KvMWTGwPjrvfk/AdGDfWPCdttwLQfjg0A/psFcC4gu7GsT9mMb403fY6tBtL8rkCQJ0r4ERXciC8iD+aGDznuzJXd6DiBO4d1oEPc1mGuAcXtBvLuAxBh6b3oQuJMfXc6h864UCPsDWL/TK7eZUP88QORjs87FmBbNjDet69aCaH/qgDEZF1c6RmcjXP4MclvPhLYnQNHbzHWfD4jTRwOK8BLqtReJEGlw6ga0GRAdA4MjETuMYWdfhvJ1Am8tec8yAV3oxzwnU2EkXM/91gMTFBiR8GSeS+i87viv/rdoCLcwOph93vlbi/2m3EQlQPztwDjrQdxlYLsi5JMxJpH+lGt7mQbLPvdgvBPADi/htDhn9YN40EVXCBPv8Cqg/71Nh1kmcvi5BY33Ub6AHRM7APStHl84Dzr64ZgEFYY3a8f5Ks82aKzq+KIgyugeCkG7oX4MwrE+Tx2aBXEKnpow1sPlQ6hX89f6QWQCukO+27RhDruz/IvnYC2vsOzJsjDynW3m1kx4OXO6k2CSPQMKY+h8nKTCkvPPYizSJxtTP4TPQBFOjDi4MbiLTGIURAbhcATz3oV9M4os6i4unSJt1fCduwt0GINq1NBWQS3qBzGS9E+n5sx5/NJO+bLE5HN3HhLD0PjUtfs5DVAmkDea7zjyFzsOXFCG6n3Lcm97UWIRVIyG0sy1EmE88365kxEFMe5E5vuFO2o3LYHh6ugzhpQ+ueNYvX/1gxirI0YFLg/iapOaL4b9hYdUAamJDQBVr5e0Eey0/Xqpv7gEKFd1JPXLdVqEf5bub2NgmND4+E7bqwmvCUQX9g+7YjhbAkuByTFPeUiVMf6VaCMPxEcT6t4ib2Ps1dLdpXtnlSgOlwVxdf28ex1adwTrSZ22BET3ex8skvQ35XzZ7doYywxsb5CejCdB9GCevgrCsQxZCoDc12AFm5f5/fzQJkA1irplORBX08+GfjaAdtwBE7T02GNgbgmI3JGEt/m7wLI5ovypQI4YbEiiBew5QE/kiN61ild+OGbgmxWXF72D9+IHK+7u1yqY0IcePR58MHFxNsDoKpGX+gCUA7GofvNxRSDzQ30eiEtyRM8turASkSZy/6w+8c/aDc2OgdFZ3z/REI9UDIZGq7UqX4RgR5i1M+abK5UxiMlds1/kDRxwT4PpO2AUHhGDH1x8B8bHAfrcLfd7UM9GMILwz3fds9yyDz1VE4y2Fc+lYzniv9X1KwRioFFfX4Bnfw42Z5UrBNmQtgsiXxn+IEWMfmZ3aNmKqevzWY1sWWkiq47owTgJd67xeiE/mw1dkklhTdLGxWsGcZhw59/8CaVLqK9ljO+TYy8IIt81V9SvCIhZdURX72OHCeicW/FKQwG3KzJ39YBYojN+frObl6MkJ6q+99Y5z/PiJZKoUP7JClPmRevo/wr97Ye1HWi3WX02oe2qMBOnOYXaDTRO6+diEj6Cl3W9dWJpQz6IwcnK0SaerBQGkbtLkB/ejdH/YMZEKjMh5b5rwxjy/MzE+JCBHYwTJyoRaIJCdTQslbmX7yY8P3ywMT6OP8nCIeL5ZJn24t+trl8uiH4asrwaEO9PRLfCDGT/pmVH9GCdS9g5UKC+GcCMJvMVB/KUQL6LMAbhQIbM8h9o4N8XsnbVBfro3Q4g7cpQ3vUwuI4Uxh91KH9pFcdcRb/Zrlk95BFBRO90gPHtIgcM9eNP3wjhSVOB8Ya/q3ptGcTs1VB1ELm//6VDkUR0JAndoVXgzHb2PKJ42pZjr0mXBGg85+1UXiyrjWU7QEwInguu/33+hLaBSYsH/8X6tdpE57cdPKHdUpRK9odAXAnSpmDY3nYJRAKxwuapvoVDIBKIBGIyT6D39TnMpmlJjkiOSI64aauW+tucY5MjkiOSI5LDNOcwm6YtOSI5Ijnipq1a6m9zDk6OSI5IjkgO05zDbJq25IjkiOSIm7Zqqb/NOfgfMEFkJSY7Fj4AAAAASUVORK5CYII="
    },
    "image-4.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAALMAAAAkCAYAAADCdwsIAAAGVUlEQVR4Ae2bPWvjShSG96fMn1AZUlyTYtOsupgUghSCFILA4ib4whIMwaQIIhDEwmIuBBNYcBoHLqi4oCY4RVCK4C0WBQIqAioCKgJu3svocyTLshzJiuOdwij6PnPmmXfOOaN8mkwm4D/ug3Vg4NM6NIK3gQ9GygCHmc9MazMzc5g5zBxmPrXzqX3VGODKzJWZK/OqjUpuD58puDJzZebKzJWQK+GqMcCVmSszV+ZVG5XcHj5TcGXOU+bnMQzD8H7mk7s2CpYc+C7sO7+NhjGGk+ePFT/HYU53EAX43vbBvVVBCPF+8lVwLH39h9+3Mdj320iIilFV7XkyYTw4tQoAh5ntvMchWlsKBo/BlM1hLgGjhcHBBpQrq8QzFgudOMwRzBb6koD2NaPAHOZyID7raAvb0O4XgzIZBhW/l8McwGxfySBSD+NXxnkc5nIwTyawLiQQeQArEg3GvxUfqw/mVxujKxWtXREbXhy6AXG3Be223rgqe9Sb0D4TSBepKTEBswXnYYCjvQYEIqCx04J2swq2l4UjFTO/2jDOWxA3CcimCLkzwPilxDt+9yGRBtS7Es8oCH09ML+MoO4IUTIVJlV0uxKJ1b2GbdJE71fK4QzMzQMFYpAMxvaLtU2h2YMwZW/BTk8+i4VZhnIw3U/CoV6iymGhv0vQODdLq3zS7um21wCzA/0wdpD4tQf9lw3HsWBea1DZGPVNnTHdqKxGe6q630SrcwRZOsLgugd5sw39hU6FTRDi/524l4GZEBHd6zFse4zhsRgNTOFs+Z2UsKlyH7EwEwgHPYweHVi3PShCWOVQMLSL+XnaVhdGh4Ds9ZceaiwfZm+aCZyyXzx2Gl8qaO42C/+Uy/HMke/eqhCJCDUIC+j+NlXZwJ7RKQH5rMFMg8LAvM0qy9MAcqjS+wPY6fs+1D4LcwvD5xja0WksQuptfHwa2Pxz5vdtENKtruw3w79Lh9n97yhSsXcJKbyMmoB8HcbQeWEFQfMfOgCCzsxKUhiYk7aPoK4lzMk6s5cUB+0sA7P/HDkuec6AcdFBkr5+6TCzDkkCkT+a04a+dd+6lPxFj59xcufbJKB7Q22oDmb32UT/8Ag6o25vtbu++1hlzoeZ+k3YktA+0dA9oGq7AbmjQTtpQSQC2np2Qrw2ME/uVAjB6BaODbhLGpXZnU/jdRrisMldeCyMkauA2cHoUkOvI4OQMvFlPQM86auiMFvoyzL6v30bzTMBROjCcENBkKJzyedPsD4wuwa6USIhoqvbMdDPJsZP2R24aMwsX2TFzDaGBxTmLoyofjyCSu1h4ncvZv4rP2ZOziozwgwvLJkFsw+89sOYmQi5Tv73H28/n/fugjAPB2h1wqqGX6EQ/g73x+grKkYe2NP9mRkzuw7cqE+m70kPiCL7Sw8zqBGO3g5qy2F2HG/LxGJFGuiHGaEKuxidNb2Zgk3orJ9UUcNrGMcuGjPnwWwPoQQzVFtPQ+vCOPaTrWa61h3MZGG4RDrZs1vu+dx3F4SZTQCDBFi6jEO32X0RVDN2exiHs3JUFDgKlJ3xeXjNG7a1wOwBfaNB2YqzY69WKzSgLbuY/mph2JHQ2GuhrSiQpYYXQ3cNxoFeQijm1pnLK/ME9r2B4UkTMhO/hxCYP+gg20D7X2Y5nelQXxAENL9nlwLnnZ/97sVh9pN6Eb0HxoeMrWGb/K2v4qx4TGhSvkkg7GgwK1Tn2mAOG+g6DpzgV2/8TB0fqARJlqAmE38FcJYqhrbP3eYpc9DZo9ONIPEsAkK111T1bq9kJySTxZm+8VR4e41WAGeO2mo7a6ZDo/f70BJ2ygvOeUlKxvH5z2TaMA9mqkhf1ErVqLB9lb3b92EcLzPtj/wcH/O+zWDyk8L2Zjxr3r21K/M8g5Z6/jFY7Dg2MhZYTGhfCFrX2eWlfLuCBOuQlgEFSN80aHo6nnShf2tGCzf5z4thqOa6Kt5tQT/XoB0r/oKT1IZ2PoCZ992GV+OXpsO3N4BaxA9/CMy2B1JiRXG/N73iR79n3lx+cb9Ix3z8a+j3zA20+PfMVSvTAs+j/2lyx5QPl6QiHx/WOT6l/2kS/sdOTT78Q5R5juNrcvbaA/zOfuQwv3MHcMCrExoOM4c5IxmuDrA6ByuHmcPMYa5zxPF3fUylrLvf/gdSGYVk55GWkAAAAABJRU5ErkJggg=="
    },
    "image-5.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAANcAAAAsCAYAAAD2KC1vAAAIkUlEQVR4Ae2bTWvb2BrH56OcL6Fl6KZ00WwiZhOThWAWBi8EhSAuBA8MJlBMuAQRCKIwmMJgcpk7LgUPFLQICIZBuVCURXEWQYGCFwEtAloEtPlfjnSOLcuW/CYJT3gWxnWqI5/n5fe8HfmHKIpAL9IB+UD5PvADKbV8pZJOSafcBwguytxUuVTkAwRXRYql7EXZi+AiuChzVeQDBFdFiqXMRZmL4CK4KHNV5AMEV0WKpcxFmYvgIrgoc1XkAwRXRYqlzEWZi+AiuChzVeQDBFdFiqXMRZmL4CK4KHNV5AME1xaKDR9cuPcBOecWOlw/wwfw/3bhP+1+ZiS4NnSM8MZE49CEG+y+kdd34N2WietePTDh7jhgBNcmcD056CgqrNvddsKXBlVaHu+DCnY8RLCJ/WpaQ3CtregQ7vk+lFN7pw2bdsQX+e84wL1G9+9wZ8vy2uEKvg1hHjegvmJgTMHrwwaMD+4/x1EfhzCYgs717hq1GKYAvt1Du9XAa0XaoIn2Z39lJx076fV7UI/WW1+8v1WrgRDOewXs3RDjtQPkqt+x3XW1wjX+rENh3KCZV2uwswrKOkLwpQ3G2rB3vN7P7jv57GN4vDevf26Pc3cFuEJ4F+oW67dz1qxM4XUHjBkYPpZ73+z3bPq5Rrh89I8EVG+7cOQg4MmHe+3lZ66nAOFzvcqLI7Ouo33ShNqy0L+y0HPGiKIQzikDO+phlImW/rUJo6VDO9TQsfm1yZ7D2x6MljmVN7Mueg4R1ARq3KeIwKb81MXgxkcQjDFy+uh89CZ7lnufew8ddGRgbPXhC7uEjyPYNwWZryob3vfRYAxdp17/mNNL1qbic41weTDjMoSBKTr631YYYYcOunzNqYMwR4DwuovGUWP111n+vbjSRlcaFEXH4CExmH+lxZF6/5I73xiDFptvpO/70GJnEwGkOYAf71fAqJhwF+5flDasAyes2EGkLjkcb7ubTdrGQ+gSroMu7PEKe5bfW2DDVZ117rrARpsxNH8vAHuh3lfYdwnraoQrQjpy8tJQeaPD/HOUn7W+mnEZafy5AoglKCO6taAyhv2LaRRPykCGts17LBcmd66z2RJqdKXDdEJEHDK+PgaRG9BNAkruVEsEnNz/L9EJRJSPS/KVSsBF3x3APlFmysI9rS2y+qLrI0SV2nCxPeYgLMM3NrhHrXBFUYDRpw60eJgx7buUdzLSR4jGHkaihvZ/a4Ap3eqjeqw4mUX2Z0bs7jl3piYG3wUsC+CSxvQu98FYA7074WjCoZtXqciako/D2GAKuhzMDYy31pobcwrFxnBFiJ7HcC51MQyZ2lA9d6fVRUrGam1IcC1wnBDBnQ3zJxkFFZhfuUN6sN4yaLEz8iiZzgI5kbE0p0y+m7EunEmPJ/426bGKjCmy1OTaCONPTTCWhjUtX4Q4K7614G0gQ/jooX/Sgb1qM58u6bS+KFu30OlzCP+mj/aBBEzHMC4T0zJuZ8PwwYap9wr0U2SPLWTbwB6LAl2NmWsM72ba6PPNJNOexDjmTYToYYAmU9H7loBmKhr698VKWrvnep93PiVKtEm/FCH6zvfDoEzKwJyeixtDXDstGUW/lYZ1Rr4I3oUiAkmxjLOGC+DyAcspB1c6tFwfwv/Sg3U5gDc3JPHR1yQICvTfPAQyiIRjjORjXI8u+pcWetepbCudLfDhyevE3+LMFPdhIrvPyMh1utyGs/JxWXzYlxbME165mPlw5fRcYVBDJSB1UvBeI1wiyrxSJ8OH5JyFN9j/xh//tdB9tw+maOhcWhj8bwT79zrPv0RZ+KOcBPoYvEsya9JvJQ7snjGwyTXSqSNEwtDsvZNk6ocBdD6MifupEKPPGfluA/hfBnBXzTxZI8ZlXhau6dBI/Tiarxhue2jIoZIcTIj35qck8PHjkuSoZMFxgww2b+QAScWevE/rP/grK+OWNowzfxFcouxO20cOoIqGYPMwp+yY1fMWn2uEy4P1RpaB0wj6WreEgyWRXvk5L7NUo4AZRT956B2rUFsG2k0D2iHfZ3MyOeTXFp1zxeP4Qw3Gv/ja5Dxp2m+VLN9CuCIE9y6cj8bc0EXKGd4N0NGyZ117ML6IquJ5DM8Zons0K3e8fmzDyPTLjO1BOx1gFGfKcmVcBteic67AbmOPKWh8mA6lpOx1v9cIl4DjKUAQiNfM+JnX6gr0z7OlY90KmX6f6LeyPVH8hAabf0LjcYRRajSdlEvpkqhk+XLg4vvnTrl0PJ2yw/w5ogvzVReuLBsz0TuU9guyZ5DlylgMl6g0jukJjfkSJWOwZHLG+60QzoWBwZJeawpBORktvBui96sNXwIvS6BJvyW/J3m2kP1sT6djst9i8jwrAXNmChqXMMXyBXcOHCfv5WEs98Z1lwuXB+vAEMMFuef13nl2VlPHESvreqmMAUa58jlwvo6nOhVBIrfnip8t3KdnC1cyzvchjDca2qcGOp8WNNNZGEv9LLLUZIwewDnj/d/0MHlGhkcbbSU1BYzPchQ0LpJxtH/VnF9btnw5cI0+atC3OVR9stHZ9Kc0JctYlLn4memuPzxdf1lYKhTrReQZQGb2EcL71UDjsInOeRdtXYNxPoCbKvOya+Pfcx3IJx182Kca1FYH5qkO/aQHp2Bt9l7rfU6mhdYJf3JEgfaLBcuuOxiVpfe8+yTTwnjAxVQYZxYGX6cTwH/Kb+kIrhnI8oy9+O/8l8jOXU1Pj2yxz/XgXSzr7txD/BJZPpu6w3ohuHbYOLvj0LsO3G7uj+AiuJYPm0hHG+mI4CLH2chxKKsuz5YEF8FFcFXkAwRXRYqlyL48sr90HRFcBBdlrop8gOCqSLEvPSqTfMszM8FFcFHmqsgHCK6KFEuRfXlkf+k6IrgILspcFfkAwVWRYl96VCb5lmdmgovgosxVkQ/8Hw+Eg7KxtFKfAAAAAElFTkSuQmCC"
    },
    "image-6.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAKsAAAAfCAYAAACYsz16AAAINklEQVR4Ae1bv2vbXBd+/5T7T2gMWUyHZrH4lpoMgg6CDIaCER8EdzGFIjIEUTCiUMwLH6bwgUvBhYKGgpaiDEUZijMEBQIaAhoCGgpano/7Q9LVL0vxW/drqAYhx1e+59xznnvOc85V/kqSBP3V2+AxYOCvx6Bkr2O/mSgGerD2meXRZNYerD1Ye7Dy1B1h4/oIf+w/jcW3Pvzb+NEYvqc2NZi49eF+jxp92Cmybt6PMTpWcUAICCE4GI4wOubXQFEweD7DqiIkwHoywPi/QaPwn+mw8IMO/RfJ+pl693PJoA2wenGA8Yd6zHQCKzeoB5OBdQrnXhLwI8DyhIL4CPZl/n3wHw3K6RrhL0qzPVhz2z/qDXDnYKoUsZSupztYr5cYUbDqSwQlAFKg0Ig7mPs8it6uoBMNi6tfZ8AerL/O1il49nWngY7oqwrOOoM1+jxlgFTOvUpa998esTFyxsf8+RGIVgX1vhZH5+3B+kjAGnpYnRsYDQ84Zg5VjCY2vEjS/3oJjQxgfZO+e0jryjvjfNX4VCbAIVaMBhDwMR/2U4LR35sKqHuwFo2/T3v8jnPHFxZGCscRzcT5pWN1K9smwPJYytQik3eMrByAhIyqqf2GpnwCMrTh06pf0IWpU67MY/jvp9AnOrQnYywu8/HQmUGfVMN+avA4ihCXqEc6lt7bI+vu8lMZv+393sfyVIdxomEwXsCPU8eHcF7pMBoLzxhRlPthr+tjXDQFqArjnYNNGCG68bGeW1gXwBrDfUVAnhezczewhmuMKSAVE57Uhoqu1jCHBMozE04oDHRhVYotaoT4ywzqmYvo3sWMziUoQ5LwXUROVvXFGEsJBNr7+goxNXAbWHeW37JJUvn/vzt1rArTjZiNabQyvwpfiMChfwhrs1zwXgMhGpbXKbj3d+eyOFi7dG04tTThSfbvBFbqaBayKb8QLSt6109NLJ0NIgnAFDSE6FjdyAuP4Z4ZWF0niF0TCkkpQ4KEFWMER2lxJilHARB+HNeCvwyO7WDdXX5Zzm/3970Lk2WlGO5rBYQYWN9x23Nf1FfWSRJi/YKAPLXhl2z+89coIiVL/eWUL+Mk/1yHo05gTQuoLjy0Tki++AjrCd1defsr3QimmyuaPy8W2aFY2w7WdO5u8uMbBxZNp3t3YqrXT7jfrWFQMLx0BGVKAWLClYJJZtuYZ7i2jJU9/49skdc1LJAVUn792utw1AGsIk0TBdZF/cTyguqEZOMiiuYGTeCdF6MBfTa69hGwXi7nytWirqpHJ7C2yg/gzG1YpyMQxXpUYOV2J8hrBQ8WLWYma0QZ0CIElwEH86WNIykKZz7Knq3aOHsmjlj9MftcLrabf+O/oX6mgUqB6bbz5DoctYM1cjBlQvJomCldtzDGWQeFA4LseTZGpJMmsRHk3i2TJxZEubJiws0KhmZjdAJrF/miDVYL1jsPy7mNxZcG/vwjRrRV1xiRfKBSsV/zeHztYDG3sZIK08yuCd30FAgS/Ur5qsT1WftR2JPSK+W121q4yjLo5+hiCfvdjBXV449FLrxNx5T+McAOTTjS0Xh0uanUKxXO+n3R4a2rrybfEYUd2gyaRHQH8h0uPct2c14s0VaGSkp8lcpTpnBo342C43O3FlgnsHaRvwWsnD8XaUzmzNiFydoyo4aCJcBSo78lmH2piyzbx7PI9K8FNhWQJ+DO1bBktUIM71ytcH3aflReOizSUtA5Ox/aeLAIQRms23WM4LwUvVUW/LgteLS1CoVUkggKc5yvlfq3MbJGjsmKKfVQTKoMMDrWsJCOVDNHFYzHU7da22cV7aOhDuPlGKMnPDVwvhrC/dvGlDp0aMCcL+B24DapDp3AmmyTn28qOldtZP0RwnfXMI+lCJat3cfimQJyOM07I9kYnVs4SxnVZ5228SiA5y5gkGKFnK4/Ee0r9cTAdDzCgG0cwVdDF4v5FBohUCcm7HduJZJl8xR0zm1SHK8Ha9KmYxLBm4+FbjlYlSflIo9n3ELRfWk3g7WoXJPS9d+zE6zjYo+MzhffbhBkJxVi9ygzuFlq5Eqq77pFU1nHLmBtl8/X0whW5kgP1mGxhSfrsdfPlHOf1PWjY4RXQc5NRfGkvJLSPKMFKhbf6332ML0bwErt06ijLJf2dyNx1WQZ1q48Kp1gxfsBK29HlQ4QLizWssr6qaJ6Vc+9nDcxvnpQUlJeZPPnVrB2kS+iyjawUt6nvhHvQHSKQs06PwwgNNWrMEo8kc7Bi1SCtJ8afTJAiArrIgcC46uHW4rGmGYNF27TdSUXU81gbdLxIWtl7wbUbMpGGvCQyeuepUrLlSjnewcwPtLiJIJ7dgQytApnwqyNRfnqXYDVqQUvi7jtDm8Daxf59ICCdgPMF/RdBxXGmY3Vt9zhyb2D2bOiznVr38t3VwtoL1YIKm0o0S89NLCmfDVyYT4loEEg7wLwLMb46vUK0zdSgNhpwzWAtVHHdv9lNmMnXfUvQe0NrNTx68lBXvnf+1hMVGgTEyY9GjxfY5NRAr6Y+JsN7VDH7LUB+6u8k9sX2wZWyuna5GcG28mB7Trua/74cgFjqME4MzHVDVifNhJQqV4x/LcaDk5mMP9demnkgWtl3QDBf5XnM9hvncrbUbuvk77POoDxz99n3cUZ4j8FtrZzdpm3+hv6nwLejRQFH+iE3Q1c1aWfa0eb0P8UuCy2w2Rb7jGy7qhwD7Lac3zZaX/q5x6s/eZ4NJujB2sP1h6sf2qq6te9P/rXR9Y+svaRtY8w+4swf6pt+8jaR9Y+sv6pu79f9/4yyv8AjInPA0erpTEAAAAASUVORK5CYII="
    },
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoEAAAC2CAYAAACms1eEAAAgAElEQVR4Ae2dd/AfRfnH4QcqNoiKigIJih2TUEKAUELvCXZFTRCxgRBEBRXERpMSaaFpoqLYiaiDwzBEZRx1HGIdCyoqHQREEFAZ/rnfvI55f9kc9+n3uc/e7fuPm7273dvd53ne++x7n2trPfLII5k368AYMAaMAWPAGDAGjIG0MLCWDZ6WwW1v29sYMAaMAWPAGDAGwIBJoCOhjgQbA8aAMWAMGAPGQIIYMAlM0OheAXoFaAwYA8aAMWAMGAMmgSaBXv0ZA8aAMWAMGAPGQIIYMAlM0Ohe/Xn1ZwwYA8aAMWAMGAMmgSaBXv0ZA8aAMWAMGAPGQIIYMAlM0Ohe/Xn1ZwwYA8aAMWAMGAMmgSaBXv0ZA8aAMWAMGAPGQIIYMAlM0Ohe/Xn1ZwwYA8aAMWAMGAMmgSaBXv0ZA8aAMWAMGAPGQIIYMAlM0Ohe/Xn1ZwwYA8aAMWAMGAMmgSaBXv0ZA8aAMWAMGAPGQIIYMAlM0Ohe/Xn1ZwwYA8aAMWAMGAMmgSaBXv0ZA8aAMWAMGAPGQIIYMAlM0Ohe/Xn1ZwwYA8aAMWAMGAMmgSaBrV39LV++/HGycW716tWPO29naGdoDBgD11xzTcZmLBgLqWDAJNAksJUO7/LLL8922WWXXLa77747W7VqVb4tWbIkW7hw4dSxCCHp/Pnz19g4t9Zaaz1uU71yEhzPmjUrb2/69On5JMJEssEGG+TnyGdf5Z16gjEG4sQA43fx4sXZiSeemI/jMr+An5DfOOqoo9bwGeRxbZnfCMll0T9QHkxwrXwJKcfGSpxYaYtdTAJNAlvpZBYtWpRBBBmoRYcLKQs3ykAUKYfjPeuss/J9zhUHOpHEomOmLjl48thn47yuD/d1zqmduzEQDwYY9wsWLJgau4x1+QXOa2zjWzTeIYOUw29o3JfZlPF/ww03TPmDon8ISaDqJi36mrK6fS4eDDXRFiaBJoFTjqmJAC7rM86WFT15OFGcdCcSVjxfjNjhiJkA1A71MVnomJQ6FD0gEsA1bDNmzMjb5xr2w2u8b8dtDMSDAXzGtGnTspUrV2YrVqzIxyt3DyCBbPgTLQpFFGU//AsRQR2T4hPwATonkqfjon9QPr5CvkSprnEaD17aZAuTQJPAKUfVFmCzUoeo4dB1ezYkZDhabSEJZFUfEj70oXLSTdG5c57rRPxIaVvniEYyQYQTgupyaqduDMSBARE9fAdjXCnjFx9CCtljkRcSQuxHWd11kD3DMhBMIoXKI6W90GdQNxtlOY/P0HF4nffjwEub7GASaBK4hnNqA7ghcjhmonpytFq961gpZE0ycw1kEUfPxvmig8eZ46h1DdfreUNSiCcRhfCcIgyc03VO7cyNgbgwoAUeCz/8g1LGOGOerUj2IHOMb/kMiBt2hQTKvtQVLja5JvQP7POcMs8r6zxt69nl0EepTqdxYafJ9jAJNAmcclZNBnKx76HjZR8yp6hemMpp4+C5JaNne3R7Buet2zJcx3lS6qRNrld9lIV46hjiWYxAFvvpYztzYyAODDB+Z8+enY9ZpYxzIvv4AG0QNS0E5RMowybfEI57yBwkkTxszbXyEaT4DHyFzhWP5aOMkzhw0jY7mASaBLaOBLLShsyxgiYyh3PGwUME2TgW2eM8gxoHjCPWsUggk0BxE6GUM8Cps4pXmzpPym0d6vRq3g48xIX348MD45Sxji9QSuQPQkce0T029kXMOJYv0TXFW73yH/Ipsj11QzaLzxPKf9EP9lXeaXyYaYNNTAJNAlvnZBSBwznjYNm0ypaDx5EzgJXKceu46LDDwa66dD1ldetGUQLq062dpUuX5reMmEwUQQjr876duzEweQww9hmrRPyUKrqHfcJxL3sxzrlORI8yyium8imUxRdwG5l25Ccoz2JRx/gUrimSxGK9Pp48dppsA5NAk8COTqupwC4SLcggK3ccanET6UNWnLOOKcdxsbwmAvK4Rrd2FB3kesqQT5RA5dkv9qup+nW/Pem0EQOMXTZF90jDxz303B9jWpFA9MA1jHc28jgu+g3yOEd5iCO+oOgfVJd8iVL7DY+3cY43k0CTwNaRQAYMK2pu0bKK1u0aETIcMo5aTlcDjPN6lkcOW3lhinOmrM5Rr47L2qCc2tM1Tu3YjYG4MICfwCYaz6TcsmVsszHuNfbDxzsY23pekGvK7KqIYZgX+gTtk8qXkHaqL6zH+3HhqGn2MAk0CSx1Wk0DcthfHCmbVvHc0pFDlYMPna6upYxW50USKKfPpMBtnHB1Tp1MAqR6AJy6RCg5z77acWqnbQzEhQHGs3yCfAQLSMaxbMV5Nh0r5ToRxDAf4qeoHwtSHlPRNaTyNVwj/0Bd8iVKw2u8Hxdu2mAPk0CTwDUcUxtAHcqAg8URiwRCDCF04XOCKs9EoNs8ofMnn/I4aDaIoK4hpQ0RTSYDjrleEUjth9d4387cGIgHA4xRjX1SxjGkDd8hOzGu2XSslPL4DhE+nRexFEkM66JMmX+grHxJ+Dyi6nQaD2baYguTQJPAxzm1toDbcthhGgPGgDFgDBgDnTFgEmgSaBJoDBgDxoAxYAwYAwliwCQwQaN7VdR5VWTdWDfGgDFgDBgDqWDAJNAk0Ks/Y8AYMAaMAWPAGEgQAyaBCRo9lRWO5fRq3hgwBowBY8AY6IwBk0CTQK/+jAFjwBgwBowBYyBBDJgEJmh0r4o6r4qsG+vGGDAGjAFjIBUMmASaBHr1ZwwYA8aAMWAMGAMJYsAkMEGjp7LCsZxezRsDxoAxYAwYA50xYBJoEujVnzFgDBgDxoAxYAwkiAGTwASN7lVR51WRdWPdGAPGgDFgDKSCAZNAk0Cv/owBY8AYMAaMAWMgQQyYBCZo9FRWOJbTq3ljwBgwBowBY6AzBkwCTQK9+jMGjAFjwBgwBoyBBDFgEpig0b0q6rwqsm6sG2PAGDAGjIFUMGASaBLo1Z8xYAwYA8aAMWAMJIgBk8AEjZ7KCsdyejVvDBgDxoAxYAx0xoBJoEmgV3/GgDHQAgwcccQR2QknnJDbctmyZRnHnvw6T37o5oYbbshOPPHESvR09913Z2zWeXedWz9x6ccksAXO34MqrkFle9gek8DA/Pnz1yCBO+64owlJD/9+zTXXZGuttVYlepo+fXpGfZOwvdu0zxkWAyaBPZzEsIr1dR6UxoAxUCcGTAIHx1uVJBAyaRI4uA0GGSNEWlesWJEdddRRYyXbu+yyS97OIH1ralmTQJPAsQ6mpg4M93u8ztz6rV6/nUggEyd5kBQ29id525I+MImT6lYs6bRp06b6t3r16im/dPnll2czZsyYyluwYMHUdcixePHiPI/6qGP58uVT13Ju0aJFU21RfuHChfkxZZUnPNKudEW++kc++/Q7zFdbkAbaCmVSnU5Hxzq37ZcsWZLbF12DCen1rLPOmsLO7NmzsxA77HNO2KAOYZ884Yp8cKQ87Dpr1qw8H7vTvtprW2oSaBLYWnC3bbBantEnkzbrEHISPhOo28EQHYiTZGd/3JEUtVWWMuHSJ6JmTK5M4txK1UTLBMwxEzLnQmLHxB0SraJs1EW+2i22hdxM7tTNhi5UnmNIAXVwPceUFdGDDGywwQZTJINj+ha25UhgtWMUe0Pa0Tu25lj6JhVWQuxQVvbFntgptKeOsX1oazCnY7WBPWkXjEASi+2rXJNTk0CTwDUGVZPB7L5X64Ctz2bpsxsJJBqycuXKfHKctF2ZUEOyFBIt9U3P1zEpk6/zpEzKmsiZ/EUAyKO8SB3H7IcTNwQhjCLRD5XnPO2GbUEyiDxxjjZDMk29ulZthXKF9Xh/uLGEztHx0qVLS7GLbYrEDbxgS2xRtCfnRdyxpcZFN/tAKNUP4a5b+ablmQSaBK7h9JoGYPd3OOdqvbVPb51IIJMYxAkCxIRKZCUkRnVjISROtM1x2caEyyYSpn7qPMfIgTxM7MgHAQjrD/fVVpGoqQz1lvVDpCFsV33RtZ3qVjmnw483iBsYwA7czg1JvzBdtJtsVcROSPp7jQuwRfSPdiGMRdy0xaYmgSaBJoHGgDHQAgx0IoFMVkx4pExsTGjcJpvUJBYSJ/pAtIaJPuyP+kuUh2hNmMctXSZ5XQvBFTEIJ3nyi20VI4HoQ2WI+hXbUj+oS8Qi7IuuVVttJQqhzJPax8ZazJDSD0iebterX7JZmT1DfFBOZcEfCwhFeqkXrIA1YUv1ty01CWyB828bKC3P8Ktm6y5d3XUigUxsmjTBB5Nj8TZZnbgJiRPtMtHSd03IImaknGMy1kTPOSIzIoHUFRJI6grrD/dpCz0QOZS8YXkm+2J9oe76IYFhX9TGpNOrrroqY5t0P6pqH0xIz9gE4l7EDmQPexZJP/YU0WMMhLeSwYLywJvqrKrfsdZjEmgS2BrnEOsgc7/SJWZ12r4TCYQ4MeExWVKmODHW2UfaKhIzJlsiL0Qn6R/5In2UZ8Knz5xXtEYkkEmcPK7jepE6ZO7UFnVQFn1ACsP+0G5YH2VFBnqRQAgEdalvdeu1U3vo5rDDDmuln8c26F3YYYEABqSLbvYUrsABG2NEuNH1KaQmgSaBUwMmBcBbRhOytmLg5ptvztiQL9yXvERHYr5dSeSG/ol00W/2ixMzk35ItMquk8ydUurkuk759KPYbqey4XmuCfsf5k1qvwwLk+rLuNoVBjrZtJM9sRV5bOPqW+z1mgSaBCYL/tgHp/tnwpo6BiBVRNc0SRO9IdozDEFLXZeW3/6kDAMmgSaBJoHGgDFgDESLAYgft4shg6Qcl01mPmeSYwwMjgGTQDt/O1RjwBgwBowBY8AYSBADJoEJGt2rpcFXS9aZdWYMGAPGgDHQNgyYBJoEevVnDBgDxoAxYAwYAwliwCQwQaO3bSVjebw6NwaMAWPAGDAGBseASaBJoFd/xoAx0FIMpP4WbYyfbKmbqPDZlNg+W1O3DlIfB930bRLYUuffzejOG3y1ZJ1ZZ03EAB877vTttCbKM2if+XBw+OHpQa9vQ/nUdQD++Zh0G2w5DhlMAk0CPTiMAWOghRhg8uOzKuEfFMYxicRcp/7+EXMfx903/S1m3O3EWj9/lWEcOBpYvpA3CWyh8491MLpf5YPQerFexoEByB+TX6pREL4niPxsqUZDrYNH8t/BgYHw/9njGG9NrdMk0CTQUSBjwBhoIQaIAIkE6Y8bTZ2ohum3/guMDsLfzA1TV1OvCXWQYkSY6J/GAH+aaaodx9lvk8AWOv9xAsZ1O2plDMSPgTAClGIUhBchNPmTphgNRQcQH+khRR2EJBg9+G8zj/ddJoEmgV4dGQPGQMswwOTH83AiAJCBlN4Q5WUQZA91kFo0tEwHqZGg0P5ExhcsWGBfV/B1JoEFhTjK8fiVgnVinRgDzcGAomB6JnDffffNCVFKb8nyVjTbzJkzsy233DIngxDjlHDMf5YhPqnqQCR4xx13zPGv8ZDSYqgfvJsEmgQm5Rj7GRQu0xzCY1s93lZMcrwRuWrVqnzyO/LII/M3hFN6OQICQNRr0003zbbeeuv8MzEpRgLRwWabbZbNnj0710FKkUDwzrOgxxxzTD4OrrrqqnxcmASu6TNMAk0CTQKNAWOghRi4884788nv3HPPTda+RAMPOuigZOVnkbTttttm+++/f7I6uPDCC/NxcPPNNyerg26LZZPAFjr/bgZ33pqrIOvD+mgrBu64447kSSARsIULFyY9+UMC99tvv2R1cMEFF+Tj4NZbb01WB918nEmgSaAHhjFgDLQQA7fffns++Z133nnJ2pfnAVMngXPnzk2aBC5btiwfB7fddluy48AksIUOvptRnefoljFgDIgEnn/++clOfpDA1N8I3W677TJeDkrVJ7AI4k1xxkOqOugmtyOBJogeGMaAMdBCDBD5YPJLmQRutdVWJoHbbZfts88+yY5xnollHPB4RDcylGqeSWALnX+qYLbcjn4ZA49hQCSQ22Gp6gUSeOCBByYrP3bffvvtkyaB55xzTk4C//GPfySNg04+wCTQJNADwxgwBlqIAR6EJwKSMgnk8zAHHHBA0vjeYYcdsr333jtZHZx99tn5OLjrrruS1UEnAsh5k8AWOv9uBnfeY5ES68K6aDMGRAJ5O7LNcnaTzSTwkQwSuNdeeyWLgaVLl+Yk0N8HLPf3JoEmgck6h26Th/PKHYb10hy93HLLLfnklzIJ3GabbZKPBM6bNy9pEsiH04mI33PPPZ7rSviOSWCJUjzRNWeis61sK2OgHAN8HJfJj4/lpqojSGDKH0rG7vw2bc8990wWA2eeeWY+Du69995kddBt/JsEmgR6YBgDxkALMWAS+Eg2Z84ck8Add8z22GOPZMf4GWeckZPAf/3rX8nqwCSwhQ6+m1GdVx4ZsV6sl5QwIBJ40UUXJTv5pf63DPC+0047JU0CTz/99JwE3nfffcmOg25+z5FAE0QPDGPAGGghBkwCH/1vbsq/TGPy33nnnbPdd9892TF+2mmn5STw/vvvT1YHJoEtdPDdjOo8R7yMAWPgpptuyie/iy++ONnJj1+mpfy3DPwAJHC33XZLFgOnnnpqPg4eeOCBZHXQbT5wJNAE0QPDGDAGWogBk8BHMkhgyn/LYPLfZZddsl133TXZMX7KKafkJPDBBx9MVgcmgS108N2M6jxHgYwBY+DGG2/MJ79LLrkk2cmP/+amTgLnz5+fNAk8+eST83Hw0EMPJTsOus0HjgSWEMQbbrghO/HEE6Pa+OBlN0MOkhebbPSnig95XnPNNVHZDLkuv/zyke0Wo1zLly8fWa4Qs9QXGy5Xr15dqYyhvL32q9DF0UcfnU9+/Dativrwi736XVU+46aKPm+88cbZ5ptvXkldjMOq5OunnqrG/YwZMzK2KvRZhT/rR3bKMP6q6DO3wvlU0gknnDByfVX7vX51Mc5yJoElJBDgAZrYtiqAgGOJTS76U4WD5bZHbLLRp1HtFiMed9p5dLlCvcRmN/qD3sM+1rkfoz6qGKP96rCtY7lf+SmXug5i9HuMy0Fs2ISyJoFdSOD3/pxlMWwHH/nxnNxUASiRwFO+/KMoZKMfDKwqJhic5sy586OQC9y8cu783JGPajc5wxiwKLnGQQLBeSwyxkACqyChDz/88MiTlnxGFWO037HAWGbrt/y4y02iP5Nos5se6+6P/F63PtWZF1t/qpLdJNAkcKITr0lg72fX5HxiIUiQW5PA3nYbxUlPmoSGfTcJfDQqVzcprZt0hTYv26+7P/J7ZX2ZxLnY+lOVDkwCTQJNAscU8XUksH+iBOlxJPAxfZkEOhJYN+nqRSrq7k9spCu2/vSyV7/5JoEmgSaBJoEDYcCRwMfIWr+OdtByJoEmgXWTrl4Yrbs/sZGu2PrTy1795psEmgQORACqviXp28G9CYWcT9W6H7Y+k8DeNuvXAXcqZxJoElg36eqERZ2vuz/ye2p/0mls/alKHyaBJoEmgY4EDoQBk0CTwKomoE711E04OvVD5yfRn0m0KXnL0rr7Exvpiq0/ZTYa5pxJoEngQARg2OhRp+scCexNKOR8Oumw7vMmgb1tNowzDq9xJNCRwLpJV4i/sv26+yO/V9aXSZyLrT9V6cAk0CTQJNCRwIEwYBJoEljVBNSpnroJR6d+6Pwk+jOJNiVvWVp3f2IjXbH1p8xGw5wzCTQJHIgAVB11ciSwN6GQ86la98PWZxLY22bDOOPwGkcCHQmsm3SF+Cvbr7s/8ntlfZnEudj6U5UOTAJbSAK7/YJN3/yCfA1LAqq8bhAS2E0uBgROqokfi+71Oy45nyr1Pkpdw5DAXraD9PgTMY+Ry7aTwF54qJtw9JpQx9Gfa6+9tuvHsMfRZi85u+VX3Z9+/V63PtWZJz9cZ5t1tGUS2DISyL8dmUAWLlyYrVy58nFOpskkcNasWVP/wCxzIDipJpLABQsWZNOmTcuWLFmS/y+zOPDlfEYhblVeOygJZMIHk7Nnz85WrFhR+p9ok8DHCCD2bzsJnD59es+xzHgujoVJHVdNgJBjvfXWy9Zdd91s/vz5WRkhHEebo+iv6v4sWrSoL783Sp+rvFZ+uMo6Y6jLJLBBJPD666/Pem2rVq3KjjrqqAwny0RSJBexksBLL720p2xnnHFGBmFCLjacZ0gqcFIxkcCXbz0v23bbbXvKtWzZsgyHKLkgS0uXLp0iS3I+VRK5Uep6xZydszlzesslrF533XXZSSedlEHiJePixYvXWKRw3pHAx4gg+sDuMUwS8hn9jFHZvFfaz1hmPMcgP32YM2dOjt+rr746q2o74IADso022mhqTOCrDzvssEwL3KpJ16i6nDevP3/Wy/bKB0/9+L1R+13V9fLDVdUXSz0mgQ0ggVvttPeUo9AkOkwKuTjrrLPyumK5Hfy2484YWTacJ8QXJxUTCXzK09YfWTYiuscee2xezyjErcprn/Hs540sl/A7Y8aMKUzGRgLVx14p+GNBwiRB9F2T+ChOnjZjIYHnn39+ZfbuR5cayzGRwHXWWac2Hay99tp51Hzu3LlR/T8ZnPeyX1X5od8bZRxVea1JYAlZqlLBMdUlY1c5cY5S135vek8++CBwvbbjjz8+O/DAA9cYrBtssEFOklavXp1pVR8LCTz+wivyvh5++OE9ZSN6pAinnA0TxfLly/OoGfsxkcDpL94i23zzzXvKdfDBB68RJUM2ombYmlupseFxs5fNyl74wt5yhVgFk8VJhCgAjy8w9pE5NhIIntB9rw3SQllhEjmJ5I7i06iLdkepo6prr7zyyr7HaGjzbvuHHHJIz7GMTquSYdR6Ntlkk2zDDTfMDjrooMo2fAO3g4UbUnz1oYcemi8kkD8mHWyxRX/+rJvdw7wjjjgimzlz5hryl/m9UW1X1fXyw1XVF0s9jgSWkFsZexTiVuW1TI44iH5AAyGSU+HWqSZZXRsbCRzkxRARQFJsVIy44DBjIoE8O9ePE9ct7pCsy16kseFx0GcCsZMwiZMXaQ9lJD82Eojewz72s89CS/YkOljEaD91UAZ9DNN+v/UPUk4+g3SQ67qV7Wcs9zN2urVRZR59qbo/IoCk1F18LnAcbY6ik6r7o3HSy++N0ucqr5UfrrLOGOoyCWwZCWTSYbXV6e07OfRYIoGDkEDIQ7eJCCfVRBIIUS+S9dA5yPlUubAYpa5BSSCyIEM3QtQWEii7YU8mN253dxqLKluWtp0E9jOWGc9lupnEuaoJEDIcd9xxua/uJM842uzUVj/nq+4Pvrwfv9dP3+ooIz9cR1t1tmES2DIS2As8TSaBvWTDSTWRBPaSS85nFOJW5bXDkMBeMraNBCIvUUHk4nZxL/mL+W0ngUV5i8dVE45i/YMeT6I/k2izm17q7o/8Xrc+1ZkXW3+qkt0k0CRwot8LHCQS2Av0OCmTwGzs9jQJfOwt3l6Y1MTRLYJdVodJYPW3X8v03O+5ugkQ/ZpEm930UXd/NHa69anOvNj6U5XsJoEmgWMnDd2iUCaBvQmFnE83PdaZZxLY22ahg+b5NybQ8FyvfZNAk8C6SVcvTNbdH/m9Xv2qKz+2/lQlt0mgSaBJoP8dPBAGTAIHI4FMHrwxPIjTNgk0CaybdPXCZ939iY10xdafXvbqN98k0CRwIAJQdcTJkcDehELOp2rdD1ufSWBvm4UOWG/sd3sxJizPvkmgSWDdpKuIweJx3f2R3yv2Y1LHsfWnKj2YBJoEmgQ6EjgQBkwCByOBekGk25uQRYduEmgSWDfpKmKweFx3f2IjXbH1p2ifYY9NAk0CByIAw0aPOl3nSGBvQiHn00mHdZ83Cexts6JDHpTUDVq+2F6Vx/qiwKAvt4zSh7oJR6++TqI/k2izmx7q7o/8Xrc+1ZkXW3+qkt0k0CTQJNCRwIEwYBJoEljVBNSpnroJR6d+6Pwk+jOJNiVvWVp3f2IjXbH1p8xGw5wzCTQJHIgAVB11ciSwN6GQ86la98PWZxLY22ZFZzxoZG/Q8sX2qjx2JHAyn2upm3T1wkzd/ZHf69WvuvJj609VcpsEdiGBBx/1iSyGbebcXfMHxaswuhz6Hq95WxSy0Q8mvCpuNeGknrvxZlHIBW6es/FmA38apMzGcj4xYFFy7bTzYJ88KZMrPAcGwHksMlZNwgatj/L8dg7bT3rjn930p4oxGtq82z5jmb+tTFp2tU9f6FO3Pledl7oO0L3GjewwyZTxSH+qtvOk6zMJLCGBIkoYPJZtk+kzKgEfbyiuv/4G0ciFfunPIG9Odho0/JkhFnupH8P8LaIon94uVZ0xpAccuLASPErWLWbOis52g7zIITk6pdiMCaxTfvE8v5yLwc7qA/3hBZdiP8d13NaxPIi+UtdBjH6v7oXAIHgZtqxJYAkJHFaZZdcN4vjLro/5HP8ojrl/w/YNQlolARi2H1Vfxz9scaxV1xtTfcg3zL96xy3DoCRw0P5UJXMVi7FB+15F+WF9UdPHA8R8VHJexUK1ChsOW8ewtqc9rm06BobVm64zCRwjCWRwDvqRWBkm9pRJB9maOml00y/EfcGCBa0jSzi72bNnt06u0Jas1EeZFMK6qtwfNwmsarHZ1EgHf2UZdOGG78KHVUWgq8RLv3UtWrRoqH9Tq3501mQdaB4algjPmjWrlb5e9u0nNQkcIwlUOL+NKw1kYmJr+iqybJAwoSBbkyeHMrkgtsg1rMMsqzOmc0zqyBcj0aVfVRG1os41EY76zB64oJ9N81d6fGfQhRv2aKK8of25TT9KoEE+oWk2lw5GmYeEdzDQxmCGdNQrNQkcIwnUcz2DOqdeRoshn4gBg2cUBxSDHMU+hI4hxohSsb/9HosgYbM2Enf0oEkdGWMjuvRpXCRQEyFRoX7xUFZOi9am+SvkRr9sgyzctNhrmryyneyO3INGQakDXUlvvPSgepuUjjIPCe/ooE2+flD7mabrd+EAABjGSURBVASOiQQyKDXASNu00ggJBbIN44AGBWpd5cMJJcaI0rB6wMkJj20j7tKJJnXkjI3o0qdxkUBFc0a9rRfqr0n+SottdNzvZK7oocZEk+QV3mV3ZBiGyIY+gTqapoNR56EQN7z9Lb2mlpoEjokEhgOUATauCWASgC06j2Ec0CT63U+boWPAbrFFlPqRoaxMOMEjV5uIO/KGEVzki43ojssHFCfCYW/rFRet/ZKpMqzVeS6MhqHjfhdu4WKP65oir3RbtDsyDBIFpR6eh+M6bU2bo+iv+k46yDxUxDvXt8XXCyP9piaBYyCBCrNrkEEs2rTSgFCILEnGQR1QvwCts5wmFGSTXLFFlIbRhwiSiCDpIA5zmDbrvoZJvYjJmIguk8w4JtnigqxfElS0T5EUNcVfgeMQ1/1M5vgqFgm6ron+WXZHBskxCJEt+gRuqzbF5sIuchfHfL/zkHCzww47ZFtssUVeD2NAdaeUmgSOgQSKTBx99NH5SuX1r399nrZhpSHn8aY3vSmX6bjjjsvTYSMQMQ02OYYXvvCF2fOe97wMpxBbRGkYfWnFvO++++a20rMw/TrMYdqs+xomAxGZvffeO58YY3Lq4yKBLFa0YNGEOOhtPXBA/1SP9Bi7v1K/hW/5pF5kW/751FNPzeVuirzhmMJWELeXvvSl+abjsEy3fengLW95y5T/Bj+DYqdbG+PM0zwk251wwgm5HP0SYTBC2Ze97GXZ3Llz8wVaG+awYXRuEjgGEoghAOk3vvGNHJi/+c1vGjO4+gERjuKXv/xlLtuKFStaFUbHbvwhgVUxcraFKCHXxz72sdxmd911V2vkEl6xE/bCbtgPeZUXQzoOEoi81KsJXSSo34lQeqEeFkCnn356Xh/XQzBGfdtY9Y8z1RiVfjnupz2ixJdddlku71VXXRUdXnrJILxjJ7Z+5S7Wu2zZslwHv//97/vSW/H6SR4j8xVXXJH3H6wOM+bnzJmT7bfffo2TvUq9mwSOiQRipG9+85s5QH/961+3DmR//etfc9nauHo65JBDsk033bR1Nvv4xz+e2+x///tf62STU9xss83yiKCOY0lFUqruD2SATfWzP2wbelmiCeSvKKPkL57vdvytb30r1xsL2m7lYs4TCRy2jyKBt9xySyN1MCpmTQIfyUwCx0gC5WR+9atfNXKAdXMsf/vb33IH+rnPfa51sr3tbW/LNtlkk9bJ9clPfjK32X//+9/WySasciv/rW99a3TyDUNSJFM/aRX1jzqh9tPPcZUZRn75Z5PAtbJUSeA222yTHXDAAdH5i3GNk7J6TQLHSAL1BtIvfvGL1oHs73//e04oPvvZz7ZOtkMPPTTbeOONWyfXpz71qdxm//nPf1onm5zb5ptvnvGck45jSYchKYP0vYr6UyOBX/jCF/LxwIJ2EF3HVHbUSCDPxj3hCU/ImuoTuAUM9oeNXvMcZBte/hsFkyaBYySBK1euzAE6zLMKoxi1jmtvvPHGXLZLLrmksQ60k57e/va3Z89//vNbJ9fJJ5+c2+yhhx5qnWyy5Yte9KLszW9+c3TyVUHSJGNZSv2DPgtYrEeL1ib6K55pHPRtcF7cgwQU9dCkY2QeVO5QPqJgW265ZaN1AAEc5jEInilk3Hz+859vtPyhPYfZNwkcIwn89re/nYPsuuuuax3Ibrrpply2iy++uHWyveMd78jfDh5mQMV8zSmnnJLb7IEHHmidzaT3F7/4xdnBBx8cnXyjRmwkX6e0CuKmt2w7tdG289ikqX/KqMoW3PHgzkdV9TWpHj0O0MbHtQaxg0ngGEmgVtY//vGPWzfI/vCHP+SE4sILL2ydbO985ztzEvjwww+3SjZ9EuPmm29ulVyhw3vJS16S8ZZseC6GfW45xf65IaJpfGokBn2Nuw/XX399ts4662Tve9/7kpC3TJ/f+973ch9+7rnnJqcDfDufhuHFkDLdpHTOJHCMJBAgDfusQhNAuGrVqlYOIB6S/ulPf9o62Xgr+Ic//GHr5ArHys9//vOMN9fDczHsa0E46i3bccmiZ6ti+rbiuGSl3j333DN74hOfmEEGx9lOrHU/+OCD+ctvz372szM+GRVrP8fVr0984hPZ2muvnX/qbFxtNKVek8Axk8CmAMH9fCQ5R2ib12tzIm1EA4f9pts47cWfRvgDwzDPVo2zX+Oo+7vf/W4eAeNFqXHU34Q69XHl73//+8npQI9pHXbYYcnJXoZNk0CTQA8EY8AYqAEDECxeRIBwxUS2lixZkpOiNt+10OT3ox/9KH/zf6uttkoS80QAjznmmPxW+Lve9a6kdEDEUx9Uf85znpPdeeedScmvMVBMTQJrcP5Fpfu43giM9W19x4IBbgtDBIkITpp0EZHkxQjekGz7ZzL+/e9/Z0cccUQuK2/D/ulPf0qKANx6660ZEVA+oYS9jz/++CTk59GQL37xi9l73vOeDOKH7AsXLsxuv/32JOTvx++ZBPZBAlm18/xbpw1nSpkq3tDrx2guUx+pwaadbt8JF7ZHffZog67BlP7Tyy/uli5dmvuWKmWD9Nx3333Zvffem/3zn//M/dM//vGPjC8V8EkM3oBff/31s6c//ekZb/gzWf7lL3/JydEf//jHjBe/fve732W//e1vM/54xAeV6TfX89zlz372s+wnP/lJxktv1157bUaE7Qc/+EFObK+++uqMX7Fxq/HKK6/MeAHhO9/5Tv6LLz6bxVuZ/FLz61//evbVr341/33bl770pXyy5teGH/zgB7P3v//9+UsbkNP3vve9+ST+7ne/O+OlLW7j8UF3dMeHwfkkEBGeN7zhDdnrXve67DWveU32qle9KjvwwAPzB/95A5bJn23//ffPLrjggow/ZZx33nkZL0WcffbZ2Wc+85n8Eztnnnlm/vu80047LeNFKj6rdNJJJ2V8aJ3nyOgfb1F/9KMfzbil+pGPfCT78Ic/nPEP9WOPPTb7wAc+MNV3PkFD/4888sicgB5++OGZZED/fIqKN3P5QxGy8LjAzJkzsy222CJ7xStekb385S/P/23Ly05sfP4IEscH0V/wghdk/B2H3yRyG58/HPGBe2Tl81b8+3yjjTbKNtxww/zZR8mPvffaa69cT6997WuzV7/61bmuIEboi0/GoCN+pbbPPvtk/Ieb5yf32GOPbLfddst23XXX/Dd1O++8c7bjjjtm8+bNy7bffvtsu+22y7bddttc31tvvXVGpJWINzh/5StfOSUP/zoOZZEckqHYf4gbzy0+61nPyp75zGfmiycWUcjx1Kc+NXvKU56SrbfeelP2lZxlKdfxm9Mqx1kb6jIJ7IMEsmIvA5XO4RQowycH2gCKFGTAOfcTicGm2LdMJ8JFWZ7PmRj2wgC4YvKTH3H6KFFrsh4gJE9+8pNzgvK0pz0tJ9qQj2c84xk5iYGUQWqe+9zn5iQNsgbxgcARHe4kOy8x/N///d/UxpvN2tZdd92MjY8+s/HCC9uTnvSknCBB9mmTNiCRkEmIGIQMognhhKhBQCFuREohcvxNA2LHW7QQvR122CEnfhBA/CKEEGIIQYQoQhghjhBIiCSEEmIJwTzooINywglBh6hD2N/4xjfmn3OCxEPmeSkJQgzBhyBD9iH93LYmkgeJZkEAqeYRBkg2t7ZZMLBwgIh/6EMfyok5kU6IOqSeRUgT/43cy39UlW8S2AcJLCqbgVokECaBzZr0y2xYtDPH3UggkcAiDsrq8LnhsdEtEluFXrFfp0hvFfX3U4dwBCkcZmPyBKfc2mVjctYEzSTNtvvuu+cb0R8WQES6Pv3pT+dRrzPOOCOfLHlzmagkG5ExImTnnHNOHi0janb++efnETQiaXwa6qKLLsqjiEQS+Wg8fw/iN5L8T5yICxFH/srB7bhLL700I9r35S9/Od8uu+yy7Ctf+UoeCfza176WRwWJDtIvom5E34jCEY0L+0kf6Zv6RUSP/tAX+kD7aptI45///Oc8wkmkk78c8ZF7PpHExi3S2267Lbvjjjvy58N4ZgxbEDklgkok9f7778/a/F3NfvDpMsP7ryboziSwYhLIpMUqhY39EARMOMoLyQOOvzgRlZ0L66pjH4fIhMBKjn6HX6bH0XPMefqq/uCkubUxjv6jo7AtHdNPtU++jukj/aP/yKHzlIEE0k/pXXVxLrxlIBIoucJ6dA1ts097of1DG1OGfOmGcqEs6r/TRx0uuoLQEEUJx1GIufC8bIBO0TF2CnUpuxSxQDu0QVvcqgyv8X67Jz/b1/Y1Bh7JTAIrJIGE8yENEAZ9DkLEg3Pc+iFlY+IRqaJs+H0uJizyJwlQ+o089A0yI+IkYoOc5NNv9Z2ynEcuog3IIJJVhSz0CfKmOtEjx9IjekPHtEX7PI9CHhM9+/SPPMlCv6mL65CFayhPWfYpKzk5ph7ql7zogvYpxz7P51CecpQJ+0qbXEse++hG13K9t0eJNLpBj9gAXWn8oB/ZNNSh8IgN0adsiB24tcV12Bj7UjflwQGkTzqnDerEPrQdEn2VcWqMGgPGQBsxYBI4xATMZKPJR6DgmPPhpBWWY9IPr4FsMDFxfbjPMROZiIbqrztl4mTSDNtlYtU59kWqKIMMyBjKPw45aBfyR5u0D1mQrsL2KFPUt0gB14a24XrVQR6ySzbao17OsyGn6pHNOa/9UH5IBec5F7ZHefTIOdXr9FGbYE8IXVEfInnYRnkQN9mJNLQTOgePlMEGkDvZhjo4r3rClLL0wbbxhB/iwvvGQ1sxYBJYIQmEMIRA4ZhJhY1JhehDuHFOExOEAYLB9UxeZRNhWHcd+0yWREW4pUq/6W9IArVPX9iH1IbyKTJWZV8hd5r40RM6ox3aYPKWDjlmoqdf3AKkTDixs49dKIedOpEC8kI5ZUuu67RPnurl2rCc8srOKS/VFF1hUzAU3pJHH9gVm4X4gozLpiLcoe5kO8aYiB3Xl0X6KEOb1Eldoc3DOr1vMmAMGANtwoBJYI0kkIm/uIkEKorFZMckNGmQQUIhdUTIIF4ca1Klb+E+x0yaTLRF+aomsxBTkWTao22OaZdUeiNPkzl5IhHKr5MEogORFbVPn4rnlJd6CiEHX7qFyxjBfhD5Ir44Rl/CQKi7IkaxA+MMXFA39YInFjkcU77TYiCs1/smAcaAMdAWDJgE1kACAUtIOjhm8iIqISAxGVFGpEvnJ5VC6pgUw/ZFqjhXnGCZPDWx6hrIY3iLTudHTSF4tK+6iQxyrAihSBc6VVvIE5Ku0B5cV7wdTD4koShnSN467avN8FpISkgw6HvYH13j9LHJBTtiF/RcZlOIoeyGDYUHdIjtwCM65/owj3zZH4xSR4gV2+AxG1gXcesC3KaOXcZ26joYZZyaBNZEAiEhTErcitJtp+LEBLlhcooB0EywEBf6yluT3FKFBKrPIcERAOk/t9O4RqSLepRfVUof0JPqZiLnOCRZ9J23RPnAN33hmDLqA/vkQy7QN/lEhJAVGTrJ2Yn4hefVRqgj+qY2aBddhv3RNeNM+Ugv3wjjsxfjbGdcdUPWiAaCL8aRSB7toX/pl3wWWOCRvNC+4AE7UxaiOK6+1l0vGNd4qLvtGNrDnviBGPpSZx/wMWx1thlTW/K7pDH1q0l9MQkcggTicItEjeOQhAACjsNyOGnIBVuxLOWpV9GsGEBEH5l46S+DjE39Ji0beIqsIEtZfhVyoVPqV106Did1nUOf9Ik8rlEZbBH2UeWRVzJSf1FOlSOv0776VbwWwklfaBvd1O28IYF8QLWpJFD20Bgq4kv2CG0uWyhPxFw4UH7TUxYU4ZhoujyD9j9V+U0CH32spugLBsVPyuVNAocggeMCDJGLkICMqx3XW/8tHmwbOiqIDITTtqjfFm3UeaokSLZMVX6TQJNAjYFhU5PACEggxI/bXLp9NawxfV28hIIIILesuE2pN1CJTtlm8dqsSbZJlQTJRqnKbxJoEqgxMGxqEhgBCYQMECVq2y2qYUHZ1utk5zAi2FZZLVe95JYFZMqRZZPAevEWy/jGl2J7+9Th7W8SGAEJjGVAuR/DDyTrzrqbJAaICKVOAom2T9IGk2gbu8f0HHndOmBhbRI4mu81CTQJTM5x1u2o3N5oTsr6660/JsOU7yTw0lWK8mN3tpTHCLZPWf5RZTcJNAn0ADIGjAFjwBgwBoyBBDFgEpig0UddOfj63pEZ68g6MgaMAWPAGIgdAyaBJoFe/RkDxoAxYAwYA8ZAghgwCUzQ6LGvTNw/r56NAWOgEwbC5+DC/U7lfd5YMgY6Y8Ak0CTQqz9jwBiIGAN++3HNCWzXXXfNjj/++ByzF154YTZv3jzjtwt+q8IPfzpK8Q3sthNIk8Aug6ftxrd8a04u1of1EQsGiHDp22dVTeKhbNTd1LdKTQIHG6dV4YfP0aT8a8Jw/LRp3yTQJNCraGPAGIgEA0RbFi5cmP9dRr+QZBLXv67D/11rIoLMcZ5t6dKla9iymKfPqFA39dKWyKbqiz3tRgKRS7qYlFy0z2dLSLEb+kTv6lfRRuSr3+RhM9mevNCG1MGxbFTWVlie/SIJDLEUfl4FfWlTX9UW/eGvVvzxaFJ6lcxOB1sE9NKXSWAkzr+XoZxfLfCtT+szFgxAEJj8mWSnT5+e33ITWaOPTOKcZ2Lmn9P8flDkgkl62rRp+Xlu1fHnECZqrqMO5TGJh3nK5xrqpm36ELYbi36K/ehEAiUjMrGhp5BMFesZ1zH20kec6Qc6Rb98zJtj8iDfap/z2IY8PvxMWcqQD+FCDmyPLORjU11LW1xLea6nvGzOMbYNSSD4Eca0L1JHG/zSkrrYJ6UvtEXbXMc5lVcfnDbbl5oEmgROORQP5mYPZtuvmfZjku82uTKJh2RGkzX2hhSEfwqBcIj8iBCI2EEYIQZlOIFUMsmHBKOsXAznOpFAdAGxUR+RX2RZ5+pIi/bCPqGN6AO6pn9E4iiviBt5YIGNffof2gxbUl4RvGJbRTzQBmVIaaPYloglbYEr+sU+W7Et+kQZ5Tttpr8p2s0k0CTQg9oYMAYmiAFF94jgrVix4nG2KE7cmthx5kzaRaKjyZpJnCgRxI7IU6dIH0SAthVxKk4SsR13IoHoARmIXi1evDhbuXLl43RZhywiXWoLe9AndKwNm0CoIHgifCrP+fCcbi0vWbIkj9SF9Yf7XF+GB5URbtQHUiJ/5HNtsV3O6Vr2hSv2vbVHByaBBrQHtDFgDEwYAxA2yBikDcLAhKzoEBMx+Zp4NZlzrIiS8kiLkzXkSLf+IB/URd20wTFthpHGsK4Y9zuRQPoqPRIRQ29hZLAuWULiRJvol35gt3DDBugdIhb2LSRjkESILddTVpFD6uGaYluULS4KVIbzYCvsg/apK2xX/dG1HBdxpTJOm00ITQIn7Pw9gJo9gGw/269qDDAxc/uQSZm6mYjDiZ3zEAvyirdAOQcxFGEokiBN6tRBGyITVcswzvo6kUAIU6gn9pF3nH0pq1s6Vl7xFi3n0T2EruwWLWSLjXLYWTjguBcJLLaFfdUfXavFBfWhI2GEdtQueWy6ln2TwHb6OpNAk8DanaQcjNN2OhXbtVq7MhFD7LhVzC1doncib5rkmcC5/cltX4iDon26xbtq1aqcTHAcRhWbaKtOJBAdoBv0hC6IsIng1ClnSJxoF9KF3rmdix2wUWgH+qjor/JExiB1yMF1yIVtw/rDfdqC6Mnm6IB6wjKQT+qjLjb0pWcO+yGB3EIOiXadei1r65577snlueKKKzyPDcllTAKHVFwZIH2u2snP+rQ+jYFHb9MxuUMWyqJ3ymPCZ0IPSR4EhOvIIw2jQE3V7THHHLPGx6I5lixEQCFO0oXO15lCpop65lgkqywfYsV5pfSfPmNLXSf7KcpLflld4AGcKNpYLMP1ymdfumFBER4X6xfOimV0/SRSSCD2Nwkcfq4wCTQJnHICkxjEbnP4wWvdWXfGQLMxAMmD8IXEnWOIm23bbNs2xX4mgSaBdjbGgDFgDBgDE8IAkUtuB/NGM7dqdTu/KSTC/Ww2WTUJnNDA98Bp9sCx/Ww/Y8AYqAoD3GrlVrCe9ayqXtdjjPbCgEmgSaAjAMaAMWAMGAPGgDGQIAZMAhM0eq+VgfO9ejQGjAFjwBgwBtqPAZNAk0Cv/owBY8AYMAaMAWMgQQyYBCZodK/u2r+6s41tY2PAGDAGjIFeGDAJNAn06s8YMAaMAWPAGDAGEsSASWCCRu+1MnC+V4/GgDFgDBgDxkD7MWASaBLo1Z8xYAwYA8aAMWAMJIgBk8AEje7VXftXd7axbWwMGAPGgDHQCwMmgSaBXv0ZA8aAMWAMGAPGQIIYMAlM0Oi9VgbO9+rRGDAGjAFjwBhoPwZMAk0CvfozBowBY8AYMAaMgQQxYBKYoNG9umv/6s42to2NAWPAGDAGemHAJNAk0Ks/Y8AYMAaMAWPAGEgQA/8PXvBowkFHc+YAAAAASUVORK5CYII="
    }
   },
   "cell_type": "markdown",
   "id": "14a5321d",
   "metadata": {},
   "source": [
    "### 序列到序列学习（seq2seq）\n",
    "使用循环神经网络编码器和循环神经网络解码器的序列到序列学习\n",
    "![image.png](attachment:image.png)\n",
    "编码器：\n",
    "![image-3.png](attachment:image-3.png)\n",
    "![image-4.png](attachment:image-4.png)\n",
    "解码器：\n",
    "![image-5.png](attachment:image-5.png)\n",
    "![image-6.png](attachment:image-6.png)\n",
    "编码器-解码器\n",
    "![image-2.png](attachment:image-2.png)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4dd5972f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import math\n",
    "import torch\n",
    "from torch import nn\n",
    "import dltools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "19bee046",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 多层GRU循环神经网络\n",
    "class Seq2SeqEncoder(dltools.Encoder):\n",
    "    \"\"\"用于序列到序列学习的循环神经网络编码器\"\"\"\n",
    "    def __init__(self, vocab_size, embed_size, num_hiddens, num_layers,\n",
    "                 dropout=0, **kwargs):\n",
    "        super(Seq2SeqEncoder, self).__init__(**kwargs)\n",
    "        # 嵌入层\n",
    "        self.embedding = nn.Embedding(vocab_size, embed_size)\n",
    "        self.rnn = nn.GRU(embed_size, num_hiddens, num_layers,dropout=dropout)\n",
    "\n",
    "    def forward(self, X, *args):\n",
    "        # 输出'X'的形状：(batch_size,num_steps,embed_size)\n",
    "        X = self.embedding(X)\n",
    "        # 在循环神经网络模型中，第一个轴对应于时间步\n",
    "        X = X.permute(1, 0, 2)\n",
    "        # 如果未提及状态，则默认为0\n",
    "        output, state = self.rnn(X)\n",
    "        # output的形状:(num_steps,batch_size,num_hiddens)\n",
    "        # state[0]的形状:(num_layers,batch_size,num_hiddens)\n",
    "        return output, state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13f06620",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Seq2SeqEncoder(vocab_size=10, embed_size=8, num_hiddens=16,num_layers=2)\n",
    "encoder.eval()\n",
    "X = torch.zeros((4, 7), dtype=torch.long)\n",
    "output, state = encoder(X)\n",
    "output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d1d73a35",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Seq2SeqDecoder(dltools.Decoder):\n",
    "    \"\"\"用于序列到序列学习的循环神经网络解码器\"\"\"\n",
    "    def __init__(self, vocab_size, embed_size, num_hiddens, num_layers,\n",
    "                 dropout=0, **kwargs):\n",
    "        super(Seq2SeqDecoder, self).__init__(**kwargs)\n",
    "        self.embedding = nn.Embedding(vocab_size, embed_size)\n",
    "        self.rnn = nn.GRU(embed_size + num_hiddens, num_hiddens, num_layers,\n",
    "                          dropout=dropout)\n",
    "        self.dense = nn.Linear(num_hiddens, vocab_size)\n",
    "\n",
    "    def init_state(self, enc_outputs, *args):\n",
    "        return enc_outputs[1]\n",
    "\n",
    "    def forward(self, X, state):\n",
    "        # 输出'X'的形状：(num_steps,batch_size,embed_size)\n",
    "        X = self.embedding(X).permute(1, 0, 2)\n",
    "        # 广播context，使其具有与X相同的num_steps\n",
    "        context = state[-1].repeat(X.shape[0], 1, 1)# 沿着X轴重复N次\n",
    "        X_and_context = torch.cat((X, context), 2)\n",
    "        output, state = self.rnn(X_and_context, state)\n",
    "        output = self.dense(output).permute(1, 0, 2)\n",
    "        # output的形状:(batch_size,num_steps,vocab_size)\n",
    "        # state[0]的形状:(num_layers,batch_size,num_hiddens)\n",
    "        return output, state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba86f702",
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder = Seq2SeqDecoder(vocab_size=10, embed_size=8, num_hiddens=16,num_layers=2)\n",
    "decoder.eval()\n",
    "state = decoder.init_state(encoder(X))\n",
    "output, state = decoder(X, state)\n",
    "output.shape, state.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6e3b99a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 0, 0],\n",
       "        [4, 0, 0]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def sequence_mask(X, valid_len, value=0):\n",
    "    \"\"\"在序列中屏蔽不相关的项\"\"\"\n",
    "    maxlen = X.size(1)\n",
    "    mask = torch.arange((maxlen), dtype=torch.float32,device=X.device)[None, :] < valid_len[:, None]\n",
    "    X[~mask] = value\n",
    "    return X\n",
    "\n",
    "X = torch.tensor([[1, 2, 3], [4, 5, 6]])\n",
    "sequence_mask(X, torch.tensor([0, 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d53e3cff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 1.,  1.,  1.,  1.],\n",
       "         [ 1.,  1.,  1.,  1.],\n",
       "         [-2., -2., -2., -2.]],\n",
       "\n",
       "        [[ 1.,  1.,  1.,  1.],\n",
       "         [ 1.,  1.,  1.,  1.],\n",
       "         [-2., -2., -2., -2.]]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用指定的非零值来替换这些项\n",
    "X = torch.ones(2, 3, 4)\n",
    "sequence_mask(X, torch.tensor([2,2]), value=-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "bdbfd2f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 交叉熵损失\n",
    "class MaskedSoftmaxCELoss(nn.CrossEntropyLoss):\n",
    "    \"\"\"带遮蔽的softmax交叉熵损失函数\"\"\"\n",
    "    # pred的形状：(batch_size,num_steps,vocab_size)\n",
    "    # label的形状：(batch_size,num_steps)\n",
    "    # valid_len的形状：(batch_size,)\n",
    "    def forward(self, pred, label, valid_len):\n",
    "        weights = torch.ones_like(label)\n",
    "        weights = sequence_mask(weights, valid_len)\n",
    "        self.reduction='none'\n",
    "        unweighted_loss = super(MaskedSoftmaxCELoss, self).forward(pred.permute(0, 2, 1), label)\n",
    "        weighted_loss = (unweighted_loss * weights).mean(dim=1)\n",
    "        return weighted_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0bb34b6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2.3026, 1.1513, 0.0000])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss = MaskedSoftmaxCELoss()\n",
    "loss(torch.ones(3, 4, 10), torch.ones((3, 4), dtype=torch.long),torch.tensor([4, 2, 0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "79762544",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练\n",
    "def train_seq2seq(net, data_iter, lr, num_epochs, tgt_vocab, device):\n",
    "    \"\"\"训练序列到序列模型\"\"\"\n",
    "    def xavier_init_weights(m):\n",
    "        if type(m) == nn.Linear:\n",
    "            nn.init.xavier_uniform_(m.weight)\n",
    "        if type(m) == nn.GRU:\n",
    "            for param in m._flat_weights_names:\n",
    "                if \"weight\" in param:\n",
    "                    nn.init.xavier_uniform_(m._parameters[param])\n",
    "\n",
    "    net.apply(xavier_init_weights)\n",
    "    net.to(device)\n",
    "    optimizer = torch.optim.Adam(net.parameters(), lr=lr)\n",
    "    loss = MaskedSoftmaxCELoss()\n",
    "    net.train()\n",
    "    animator = dltools.Animator(xlabel='epoch', ylabel='loss',xlim=[10, num_epochs])\n",
    "    for epoch in range(num_epochs):\n",
    "        timer = dltools.Timer()\n",
    "        metric = dltools.Accumulator(2)  # 训练损失总和，词元数量\n",
    "        for batch in data_iter:\n",
    "            optimizer.zero_grad()\n",
    "            X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]\n",
    "            bos = torch.tensor([tgt_vocab['<bos>']] * Y.shape[0],device=device).reshape(-1, 1)\n",
    "            dec_input = torch.cat([bos, Y[:, :-1]], 1)  # 开头加上bos，Y去掉最后一列，确保形状不变\n",
    "            Y_hat, _ = net(X, dec_input, X_valid_len)\n",
    "            l = loss(Y_hat, Y, Y_valid_len)\n",
    "            l.sum().backward()      # 损失函数的标量进行“反向传播”\n",
    "            dltools.grad_clipping(net, 1) # 梯度裁剪\n",
    "            num_tokens = Y_valid_len.sum()\n",
    "            optimizer.step()\n",
    "            with torch.no_grad():\n",
    "                metric.add(l.sum(), num_tokens)\n",
    "        if (epoch + 1) % 10 == 0:\n",
    "            animator.add(epoch + 1, (metric[0] / metric[1],))\n",
    "    print(f'loss {metric[0] / metric[1]:.3f}, {metric[1] / timer.stop():.1f} 'f'tokens/sec on {str(device)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0c43ee39",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss 0.106, 4138.4 tokens/sec on cpu\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Created with matplotlib (https://matplotlib.org/) -->\r\n",
       "<svg height=\"180.65625pt\" version=\"1.1\" viewBox=\"0 0 259.00625 180.65625\" width=\"259.00625pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       " <metadata>\r\n",
       "  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\r\n",
       "   <cc:Work>\r\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\r\n",
       "    <dc:date>2022-01-07T17:10:43.027741</dc:date>\r\n",
       "    <dc:format>image/svg+xml</dc:format>\r\n",
       "    <dc:creator>\r\n",
       "     <cc:Agent>\r\n",
       "      <dc:title>Matplotlib v3.3.4, https://matplotlib.org/</dc:title>\r\n",
       "     </cc:Agent>\r\n",
       "    </dc:creator>\r\n",
       "   </cc:Work>\r\n",
       "  </rdf:RDF>\r\n",
       " </metadata>\r\n",
       " <defs>\r\n",
       "  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\r\n",
       " </defs>\r\n",
       " <g id=\"figure_1\">\r\n",
       "  <g id=\"patch_1\">\r\n",
       "   <path d=\"M 0 180.65625 \r\n",
       "L 259.00625 180.65625 \r\n",
       "L 259.00625 0 \r\n",
       "L 0 0 \r\n",
       "z\r\n",
       "\" style=\"fill:none;\"/>\r\n",
       "  </g>\r\n",
       "  <g id=\"axes_1\">\r\n",
       "   <g id=\"patch_2\">\r\n",
       "    <path d=\"M 50.14375 143.1 \r\n",
       "L 245.44375 143.1 \r\n",
       "L 245.44375 7.2 \r\n",
       "L 50.14375 7.2 \r\n",
       "z\r\n",
       "\" style=\"fill:#ffffff;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"matplotlib.axis_1\">\r\n",
       "    <g id=\"xtick_1\">\r\n",
       "     <g id=\"line2d_1\">\r\n",
       "      <path clip-path=\"url(#pd19a7cf81c)\" d=\"M 50.14375 143.1 \r\n",
       "L 50.14375 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_2\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 0 0 \r\n",
       "L 0 3.5 \r\n",
       "\" id=\"m78bd039291\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n",
       "      </defs>\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#m78bd039291\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_1\">\r\n",
       "      <!-- 10 -->\r\n",
       "      <g transform=\"translate(43.78125 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 12.40625 8.296875 \r\n",
       "L 28.515625 8.296875 \r\n",
       "L 28.515625 63.921875 \r\n",
       "L 10.984375 60.40625 \r\n",
       "L 10.984375 69.390625 \r\n",
       "L 28.421875 72.90625 \r\n",
       "L 38.28125 72.90625 \r\n",
       "L 38.28125 8.296875 \r\n",
       "L 54.390625 8.296875 \r\n",
       "L 54.390625 0 \r\n",
       "L 12.40625 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-49\"/>\r\n",
       "        <path d=\"M 31.78125 66.40625 \r\n",
       "Q 24.171875 66.40625 20.328125 58.90625 \r\n",
       "Q 16.5 51.421875 16.5 36.375 \r\n",
       "Q 16.5 21.390625 20.328125 13.890625 \r\n",
       "Q 24.171875 6.390625 31.78125 6.390625 \r\n",
       "Q 39.453125 6.390625 43.28125 13.890625 \r\n",
       "Q 47.125 21.390625 47.125 36.375 \r\n",
       "Q 47.125 51.421875 43.28125 58.90625 \r\n",
       "Q 39.453125 66.40625 31.78125 66.40625 \r\n",
       "z\r\n",
       "M 31.78125 74.21875 \r\n",
       "Q 44.046875 74.21875 50.515625 64.515625 \r\n",
       "Q 56.984375 54.828125 56.984375 36.375 \r\n",
       "Q 56.984375 17.96875 50.515625 8.265625 \r\n",
       "Q 44.046875 -1.421875 31.78125 -1.421875 \r\n",
       "Q 19.53125 -1.421875 13.0625 8.265625 \r\n",
       "Q 6.59375 17.96875 6.59375 36.375 \r\n",
       "Q 6.59375 54.828125 13.0625 64.515625 \r\n",
       "Q 19.53125 74.21875 31.78125 74.21875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-48\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_2\">\r\n",
       "     <g id=\"line2d_3\">\r\n",
       "      <path clip-path=\"url(#pd19a7cf81c)\" d=\"M 98.96875 143.1 \r\n",
       "L 98.96875 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_4\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"98.96875\" xlink:href=\"#m78bd039291\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_2\">\r\n",
       "      <!-- 15 -->\r\n",
       "      <g transform=\"translate(92.60625 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 10.796875 72.90625 \r\n",
       "L 49.515625 72.90625 \r\n",
       "L 49.515625 64.59375 \r\n",
       "L 19.828125 64.59375 \r\n",
       "L 19.828125 46.734375 \r\n",
       "Q 21.96875 47.46875 24.109375 47.828125 \r\n",
       "Q 26.265625 48.1875 28.421875 48.1875 \r\n",
       "Q 40.625 48.1875 47.75 41.5 \r\n",
       "Q 54.890625 34.8125 54.890625 23.390625 \r\n",
       "Q 54.890625 11.625 47.5625 5.09375 \r\n",
       "Q 40.234375 -1.421875 26.90625 -1.421875 \r\n",
       "Q 22.3125 -1.421875 17.546875 -0.640625 \r\n",
       "Q 12.796875 0.140625 7.71875 1.703125 \r\n",
       "L 7.71875 11.625 \r\n",
       "Q 12.109375 9.234375 16.796875 8.0625 \r\n",
       "Q 21.484375 6.890625 26.703125 6.890625 \r\n",
       "Q 35.15625 6.890625 40.078125 11.328125 \r\n",
       "Q 45.015625 15.765625 45.015625 23.390625 \r\n",
       "Q 45.015625 31 40.078125 35.4375 \r\n",
       "Q 35.15625 39.890625 26.703125 39.890625 \r\n",
       "Q 22.75 39.890625 18.8125 39.015625 \r\n",
       "Q 14.890625 38.140625 10.796875 36.28125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-53\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_3\">\r\n",
       "     <g id=\"line2d_5\">\r\n",
       "      <path clip-path=\"url(#pd19a7cf81c)\" d=\"M 147.79375 143.1 \r\n",
       "L 147.79375 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_6\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"147.79375\" xlink:href=\"#m78bd039291\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_3\">\r\n",
       "      <!-- 20 -->\r\n",
       "      <g transform=\"translate(141.43125 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 19.1875 8.296875 \r\n",
       "L 53.609375 8.296875 \r\n",
       "L 53.609375 0 \r\n",
       "L 7.328125 0 \r\n",
       "L 7.328125 8.296875 \r\n",
       "Q 12.9375 14.109375 22.625 23.890625 \r\n",
       "Q 32.328125 33.6875 34.8125 36.53125 \r\n",
       "Q 39.546875 41.84375 41.421875 45.53125 \r\n",
       "Q 43.3125 49.21875 43.3125 52.78125 \r\n",
       "Q 43.3125 58.59375 39.234375 62.25 \r\n",
       "Q 35.15625 65.921875 28.609375 65.921875 \r\n",
       "Q 23.96875 65.921875 18.8125 64.3125 \r\n",
       "Q 13.671875 62.703125 7.8125 59.421875 \r\n",
       "L 7.8125 69.390625 \r\n",
       "Q 13.765625 71.78125 18.9375 73 \r\n",
       "Q 24.125 74.21875 28.421875 74.21875 \r\n",
       "Q 39.75 74.21875 46.484375 68.546875 \r\n",
       "Q 53.21875 62.890625 53.21875 53.421875 \r\n",
       "Q 53.21875 48.921875 51.53125 44.890625 \r\n",
       "Q 49.859375 40.875 45.40625 35.40625 \r\n",
       "Q 44.1875 33.984375 37.640625 27.21875 \r\n",
       "Q 31.109375 20.453125 19.1875 8.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-50\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_4\">\r\n",
       "     <g id=\"line2d_7\">\r\n",
       "      <path clip-path=\"url(#pd19a7cf81c)\" d=\"M 196.61875 143.1 \r\n",
       "L 196.61875 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_8\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"196.61875\" xlink:href=\"#m78bd039291\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_4\">\r\n",
       "      <!-- 25 -->\r\n",
       "      <g transform=\"translate(190.25625 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_5\">\r\n",
       "     <g id=\"line2d_9\">\r\n",
       "      <path clip-path=\"url(#pd19a7cf81c)\" d=\"M 245.44375 143.1 \r\n",
       "L 245.44375 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_10\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"245.44375\" xlink:href=\"#m78bd039291\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_5\">\r\n",
       "      <!-- 30 -->\r\n",
       "      <g transform=\"translate(239.08125 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 40.578125 39.3125 \r\n",
       "Q 47.65625 37.796875 51.625 33 \r\n",
       "Q 55.609375 28.21875 55.609375 21.1875 \r\n",
       "Q 55.609375 10.40625 48.1875 4.484375 \r\n",
       "Q 40.765625 -1.421875 27.09375 -1.421875 \r\n",
       "Q 22.515625 -1.421875 17.65625 -0.515625 \r\n",
       "Q 12.796875 0.390625 7.625 2.203125 \r\n",
       "L 7.625 11.71875 \r\n",
       "Q 11.71875 9.328125 16.59375 8.109375 \r\n",
       "Q 21.484375 6.890625 26.8125 6.890625 \r\n",
       "Q 36.078125 6.890625 40.9375 10.546875 \r\n",
       "Q 45.796875 14.203125 45.796875 21.1875 \r\n",
       "Q 45.796875 27.640625 41.28125 31.265625 \r\n",
       "Q 36.765625 34.90625 28.71875 34.90625 \r\n",
       "L 20.21875 34.90625 \r\n",
       "L 20.21875 43.015625 \r\n",
       "L 29.109375 43.015625 \r\n",
       "Q 36.375 43.015625 40.234375 45.921875 \r\n",
       "Q 44.09375 48.828125 44.09375 54.296875 \r\n",
       "Q 44.09375 59.90625 40.109375 62.90625 \r\n",
       "Q 36.140625 65.921875 28.71875 65.921875 \r\n",
       "Q 24.65625 65.921875 20.015625 65.03125 \r\n",
       "Q 15.375 64.15625 9.8125 62.3125 \r\n",
       "L 9.8125 71.09375 \r\n",
       "Q 15.4375 72.65625 20.34375 73.4375 \r\n",
       "Q 25.25 74.21875 29.59375 74.21875 \r\n",
       "Q 40.828125 74.21875 47.359375 69.109375 \r\n",
       "Q 53.90625 64.015625 53.90625 55.328125 \r\n",
       "Q 53.90625 49.265625 50.4375 45.09375 \r\n",
       "Q 46.96875 40.921875 40.578125 39.3125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-51\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-51\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"text_6\">\r\n",
       "     <!-- epoch -->\r\n",
       "     <g transform=\"translate(132.565625 171.376563)scale(0.1 -0.1)\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 56.203125 29.59375 \r\n",
       "L 56.203125 25.203125 \r\n",
       "L 14.890625 25.203125 \r\n",
       "Q 15.484375 15.921875 20.484375 11.0625 \r\n",
       "Q 25.484375 6.203125 34.421875 6.203125 \r\n",
       "Q 39.59375 6.203125 44.453125 7.46875 \r\n",
       "Q 49.3125 8.734375 54.109375 11.28125 \r\n",
       "L 54.109375 2.78125 \r\n",
       "Q 49.265625 0.734375 44.1875 -0.34375 \r\n",
       "Q 39.109375 -1.421875 33.890625 -1.421875 \r\n",
       "Q 20.796875 -1.421875 13.15625 6.1875 \r\n",
       "Q 5.515625 13.8125 5.515625 26.8125 \r\n",
       "Q 5.515625 40.234375 12.765625 48.109375 \r\n",
       "Q 20.015625 56 32.328125 56 \r\n",
       "Q 43.359375 56 49.78125 48.890625 \r\n",
       "Q 56.203125 41.796875 56.203125 29.59375 \r\n",
       "z\r\n",
       "M 47.21875 32.234375 \r\n",
       "Q 47.125 39.59375 43.09375 43.984375 \r\n",
       "Q 39.0625 48.390625 32.421875 48.390625 \r\n",
       "Q 24.90625 48.390625 20.390625 44.140625 \r\n",
       "Q 15.875 39.890625 15.1875 32.171875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-101\"/>\r\n",
       "       <path d=\"M 18.109375 8.203125 \r\n",
       "L 18.109375 -20.796875 \r\n",
       "L 9.078125 -20.796875 \r\n",
       "L 9.078125 54.6875 \r\n",
       "L 18.109375 54.6875 \r\n",
       "L 18.109375 46.390625 \r\n",
       "Q 20.953125 51.265625 25.265625 53.625 \r\n",
       "Q 29.59375 56 35.59375 56 \r\n",
       "Q 45.5625 56 51.78125 48.09375 \r\n",
       "Q 58.015625 40.1875 58.015625 27.296875 \r\n",
       "Q 58.015625 14.40625 51.78125 6.484375 \r\n",
       "Q 45.5625 -1.421875 35.59375 -1.421875 \r\n",
       "Q 29.59375 -1.421875 25.265625 0.953125 \r\n",
       "Q 20.953125 3.328125 18.109375 8.203125 \r\n",
       "z\r\n",
       "M 48.6875 27.296875 \r\n",
       "Q 48.6875 37.203125 44.609375 42.84375 \r\n",
       "Q 40.53125 48.484375 33.40625 48.484375 \r\n",
       "Q 26.265625 48.484375 22.1875 42.84375 \r\n",
       "Q 18.109375 37.203125 18.109375 27.296875 \r\n",
       "Q 18.109375 17.390625 22.1875 11.75 \r\n",
       "Q 26.265625 6.109375 33.40625 6.109375 \r\n",
       "Q 40.53125 6.109375 44.609375 11.75 \r\n",
       "Q 48.6875 17.390625 48.6875 27.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-112\"/>\r\n",
       "       <path d=\"M 30.609375 48.390625 \r\n",
       "Q 23.390625 48.390625 19.1875 42.75 \r\n",
       "Q 14.984375 37.109375 14.984375 27.296875 \r\n",
       "Q 14.984375 17.484375 19.15625 11.84375 \r\n",
       "Q 23.34375 6.203125 30.609375 6.203125 \r\n",
       "Q 37.796875 6.203125 41.984375 11.859375 \r\n",
       "Q 46.1875 17.53125 46.1875 27.296875 \r\n",
       "Q 46.1875 37.015625 41.984375 42.703125 \r\n",
       "Q 37.796875 48.390625 30.609375 48.390625 \r\n",
       "z\r\n",
       "M 30.609375 56 \r\n",
       "Q 42.328125 56 49.015625 48.375 \r\n",
       "Q 55.71875 40.765625 55.71875 27.296875 \r\n",
       "Q 55.71875 13.875 49.015625 6.21875 \r\n",
       "Q 42.328125 -1.421875 30.609375 -1.421875 \r\n",
       "Q 18.84375 -1.421875 12.171875 6.21875 \r\n",
       "Q 5.515625 13.875 5.515625 27.296875 \r\n",
       "Q 5.515625 40.765625 12.171875 48.375 \r\n",
       "Q 18.84375 56 30.609375 56 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-111\"/>\r\n",
       "       <path d=\"M 48.78125 52.59375 \r\n",
       "L 48.78125 44.1875 \r\n",
       "Q 44.96875 46.296875 41.140625 47.34375 \r\n",
       "Q 37.3125 48.390625 33.40625 48.390625 \r\n",
       "Q 24.65625 48.390625 19.8125 42.84375 \r\n",
       "Q 14.984375 37.3125 14.984375 27.296875 \r\n",
       "Q 14.984375 17.28125 19.8125 11.734375 \r\n",
       "Q 24.65625 6.203125 33.40625 6.203125 \r\n",
       "Q 37.3125 6.203125 41.140625 7.25 \r\n",
       "Q 44.96875 8.296875 48.78125 10.40625 \r\n",
       "L 48.78125 2.09375 \r\n",
       "Q 45.015625 0.34375 40.984375 -0.53125 \r\n",
       "Q 36.96875 -1.421875 32.421875 -1.421875 \r\n",
       "Q 20.0625 -1.421875 12.78125 6.34375 \r\n",
       "Q 5.515625 14.109375 5.515625 27.296875 \r\n",
       "Q 5.515625 40.671875 12.859375 48.328125 \r\n",
       "Q 20.21875 56 33.015625 56 \r\n",
       "Q 37.15625 56 41.109375 55.140625 \r\n",
       "Q 45.0625 54.296875 48.78125 52.59375 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-99\"/>\r\n",
       "       <path d=\"M 54.890625 33.015625 \r\n",
       "L 54.890625 0 \r\n",
       "L 45.90625 0 \r\n",
       "L 45.90625 32.71875 \r\n",
       "Q 45.90625 40.484375 42.875 44.328125 \r\n",
       "Q 39.84375 48.1875 33.796875 48.1875 \r\n",
       "Q 26.515625 48.1875 22.3125 43.546875 \r\n",
       "Q 18.109375 38.921875 18.109375 30.90625 \r\n",
       "L 18.109375 0 \r\n",
       "L 9.078125 0 \r\n",
       "L 9.078125 75.984375 \r\n",
       "L 18.109375 75.984375 \r\n",
       "L 18.109375 46.1875 \r\n",
       "Q 21.34375 51.125 25.703125 53.5625 \r\n",
       "Q 30.078125 56 35.796875 56 \r\n",
       "Q 45.21875 56 50.046875 50.171875 \r\n",
       "Q 54.890625 44.34375 54.890625 33.015625 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-104\"/>\r\n",
       "      </defs>\r\n",
       "      <use xlink:href=\"#DejaVuSans-101\"/>\r\n",
       "      <use x=\"61.523438\" xlink:href=\"#DejaVuSans-112\"/>\r\n",
       "      <use x=\"125\" xlink:href=\"#DejaVuSans-111\"/>\r\n",
       "      <use x=\"186.181641\" xlink:href=\"#DejaVuSans-99\"/>\r\n",
       "      <use x=\"241.162109\" xlink:href=\"#DejaVuSans-104\"/>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "   </g>\r\n",
       "   <g id=\"matplotlib.axis_2\">\r\n",
       "    <g id=\"ytick_1\">\r\n",
       "     <g id=\"line2d_11\">\r\n",
       "      <path clip-path=\"url(#pd19a7cf81c)\" d=\"M 50.14375 120.882401 \r\n",
       "L 245.44375 120.882401 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_12\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 0 0 \r\n",
       "L -3.5 0 \r\n",
       "\" id=\"mf6d362fa74\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n",
       "      </defs>\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#mf6d362fa74\" y=\"120.882401\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_7\">\r\n",
       "      <!-- 0.12 -->\r\n",
       "      <g transform=\"translate(20.878125 124.681619)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 10.6875 12.40625 \r\n",
       "L 21 12.40625 \r\n",
       "L 21 0 \r\n",
       "L 10.6875 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-46\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_2\">\r\n",
       "     <g id=\"line2d_13\">\r\n",
       "      <path clip-path=\"url(#pd19a7cf81c)\" d=\"M 50.14375 97.207811 \r\n",
       "L 245.44375 97.207811 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_14\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#mf6d362fa74\" y=\"97.207811\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_8\">\r\n",
       "      <!-- 0.14 -->\r\n",
       "      <g transform=\"translate(20.878125 101.00703)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 37.796875 64.3125 \r\n",
       "L 12.890625 25.390625 \r\n",
       "L 37.796875 25.390625 \r\n",
       "z\r\n",
       "M 35.203125 72.90625 \r\n",
       "L 47.609375 72.90625 \r\n",
       "L 47.609375 25.390625 \r\n",
       "L 58.015625 25.390625 \r\n",
       "L 58.015625 17.1875 \r\n",
       "L 47.609375 17.1875 \r\n",
       "L 47.609375 0 \r\n",
       "L 37.796875 0 \r\n",
       "L 37.796875 17.1875 \r\n",
       "L 4.890625 17.1875 \r\n",
       "L 4.890625 26.703125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-52\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-52\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_3\">\r\n",
       "     <g id=\"line2d_15\">\r\n",
       "      <path clip-path=\"url(#pd19a7cf81c)\" d=\"M 50.14375 73.533221 \r\n",
       "L 245.44375 73.533221 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_16\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#mf6d362fa74\" y=\"73.533221\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_9\">\r\n",
       "      <!-- 0.16 -->\r\n",
       "      <g transform=\"translate(20.878125 77.33244)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 33.015625 40.375 \r\n",
       "Q 26.375 40.375 22.484375 35.828125 \r\n",
       "Q 18.609375 31.296875 18.609375 23.390625 \r\n",
       "Q 18.609375 15.53125 22.484375 10.953125 \r\n",
       "Q 26.375 6.390625 33.015625 6.390625 \r\n",
       "Q 39.65625 6.390625 43.53125 10.953125 \r\n",
       "Q 47.40625 15.53125 47.40625 23.390625 \r\n",
       "Q 47.40625 31.296875 43.53125 35.828125 \r\n",
       "Q 39.65625 40.375 33.015625 40.375 \r\n",
       "z\r\n",
       "M 52.59375 71.296875 \r\n",
       "L 52.59375 62.3125 \r\n",
       "Q 48.875 64.0625 45.09375 64.984375 \r\n",
       "Q 41.3125 65.921875 37.59375 65.921875 \r\n",
       "Q 27.828125 65.921875 22.671875 59.328125 \r\n",
       "Q 17.53125 52.734375 16.796875 39.40625 \r\n",
       "Q 19.671875 43.65625 24.015625 45.921875 \r\n",
       "Q 28.375 48.1875 33.59375 48.1875 \r\n",
       "Q 44.578125 48.1875 50.953125 41.515625 \r\n",
       "Q 57.328125 34.859375 57.328125 23.390625 \r\n",
       "Q 57.328125 12.15625 50.6875 5.359375 \r\n",
       "Q 44.046875 -1.421875 33.015625 -1.421875 \r\n",
       "Q 20.359375 -1.421875 13.671875 8.265625 \r\n",
       "Q 6.984375 17.96875 6.984375 36.375 \r\n",
       "Q 6.984375 53.65625 15.1875 63.9375 \r\n",
       "Q 23.390625 74.21875 37.203125 74.21875 \r\n",
       "Q 40.921875 74.21875 44.703125 73.484375 \r\n",
       "Q 48.484375 72.75 52.59375 71.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-54\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-54\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_4\">\r\n",
       "     <g id=\"line2d_17\">\r\n",
       "      <path clip-path=\"url(#pd19a7cf81c)\" d=\"M 50.14375 49.858631 \r\n",
       "L 245.44375 49.858631 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_18\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#mf6d362fa74\" y=\"49.858631\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_10\">\r\n",
       "      <!-- 0.18 -->\r\n",
       "      <g transform=\"translate(20.878125 53.65785)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 31.78125 34.625 \r\n",
       "Q 24.75 34.625 20.71875 30.859375 \r\n",
       "Q 16.703125 27.09375 16.703125 20.515625 \r\n",
       "Q 16.703125 13.921875 20.71875 10.15625 \r\n",
       "Q 24.75 6.390625 31.78125 6.390625 \r\n",
       "Q 38.8125 6.390625 42.859375 10.171875 \r\n",
       "Q 46.921875 13.96875 46.921875 20.515625 \r\n",
       "Q 46.921875 27.09375 42.890625 30.859375 \r\n",
       "Q 38.875 34.625 31.78125 34.625 \r\n",
       "z\r\n",
       "M 21.921875 38.8125 \r\n",
       "Q 15.578125 40.375 12.03125 44.71875 \r\n",
       "Q 8.5 49.078125 8.5 55.328125 \r\n",
       "Q 8.5 64.0625 14.71875 69.140625 \r\n",
       "Q 20.953125 74.21875 31.78125 74.21875 \r\n",
       "Q 42.671875 74.21875 48.875 69.140625 \r\n",
       "Q 55.078125 64.0625 55.078125 55.328125 \r\n",
       "Q 55.078125 49.078125 51.53125 44.71875 \r\n",
       "Q 48 40.375 41.703125 38.8125 \r\n",
       "Q 48.828125 37.15625 52.796875 32.3125 \r\n",
       "Q 56.78125 27.484375 56.78125 20.515625 \r\n",
       "Q 56.78125 9.90625 50.3125 4.234375 \r\n",
       "Q 43.84375 -1.421875 31.78125 -1.421875 \r\n",
       "Q 19.734375 -1.421875 13.25 4.234375 \r\n",
       "Q 6.78125 9.90625 6.78125 20.515625 \r\n",
       "Q 6.78125 27.484375 10.78125 32.3125 \r\n",
       "Q 14.796875 37.15625 21.921875 38.8125 \r\n",
       "z\r\n",
       "M 18.3125 54.390625 \r\n",
       "Q 18.3125 48.734375 21.84375 45.5625 \r\n",
       "Q 25.390625 42.390625 31.78125 42.390625 \r\n",
       "Q 38.140625 42.390625 41.71875 45.5625 \r\n",
       "Q 45.3125 48.734375 45.3125 54.390625 \r\n",
       "Q 45.3125 60.0625 41.71875 63.234375 \r\n",
       "Q 38.140625 66.40625 31.78125 66.40625 \r\n",
       "Q 25.390625 66.40625 21.84375 63.234375 \r\n",
       "Q 18.3125 60.0625 18.3125 54.390625 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-56\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-56\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_5\">\r\n",
       "     <g id=\"line2d_19\">\r\n",
       "      <path clip-path=\"url(#pd19a7cf81c)\" d=\"M 50.14375 26.184042 \r\n",
       "L 245.44375 26.184042 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_20\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#mf6d362fa74\" y=\"26.184042\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_11\">\r\n",
       "      <!-- 0.20 -->\r\n",
       "      <g transform=\"translate(20.878125 29.98326)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"text_12\">\r\n",
       "     <!-- loss -->\r\n",
       "     <g transform=\"translate(14.798437 84.807812)rotate(-90)scale(0.1 -0.1)\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 9.421875 75.984375 \r\n",
       "L 18.40625 75.984375 \r\n",
       "L 18.40625 0 \r\n",
       "L 9.421875 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-108\"/>\r\n",
       "       <path d=\"M 44.28125 53.078125 \r\n",
       "L 44.28125 44.578125 \r\n",
       "Q 40.484375 46.53125 36.375 47.5 \r\n",
       "Q 32.28125 48.484375 27.875 48.484375 \r\n",
       "Q 21.1875 48.484375 17.84375 46.4375 \r\n",
       "Q 14.5 44.390625 14.5 40.28125 \r\n",
       "Q 14.5 37.15625 16.890625 35.375 \r\n",
       "Q 19.28125 33.59375 26.515625 31.984375 \r\n",
       "L 29.59375 31.296875 \r\n",
       "Q 39.15625 29.25 43.1875 25.515625 \r\n",
       "Q 47.21875 21.78125 47.21875 15.09375 \r\n",
       "Q 47.21875 7.46875 41.1875 3.015625 \r\n",
       "Q 35.15625 -1.421875 24.609375 -1.421875 \r\n",
       "Q 20.21875 -1.421875 15.453125 -0.5625 \r\n",
       "Q 10.6875 0.296875 5.421875 2 \r\n",
       "L 5.421875 11.28125 \r\n",
       "Q 10.40625 8.6875 15.234375 7.390625 \r\n",
       "Q 20.0625 6.109375 24.8125 6.109375 \r\n",
       "Q 31.15625 6.109375 34.5625 8.28125 \r\n",
       "Q 37.984375 10.453125 37.984375 14.40625 \r\n",
       "Q 37.984375 18.0625 35.515625 20.015625 \r\n",
       "Q 33.0625 21.96875 24.703125 23.78125 \r\n",
       "L 21.578125 24.515625 \r\n",
       "Q 13.234375 26.265625 9.515625 29.90625 \r\n",
       "Q 5.8125 33.546875 5.8125 39.890625 \r\n",
       "Q 5.8125 47.609375 11.28125 51.796875 \r\n",
       "Q 16.75 56 26.8125 56 \r\n",
       "Q 31.78125 56 36.171875 55.265625 \r\n",
       "Q 40.578125 54.546875 44.28125 53.078125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-115\"/>\r\n",
       "      </defs>\r\n",
       "      <use xlink:href=\"#DejaVuSans-108\"/>\r\n",
       "      <use x=\"27.783203\" xlink:href=\"#DejaVuSans-111\"/>\r\n",
       "      <use x=\"88.964844\" xlink:href=\"#DejaVuSans-115\"/>\r\n",
       "      <use x=\"141.064453\" xlink:href=\"#DejaVuSans-115\"/>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "   </g>\r\n",
       "   <g id=\"line2d_21\">\r\n",
       "    <path clip-path=\"url(#pd19a7cf81c)\" d=\"M 50.14375 13.377273 \r\n",
       "L 147.79375 90.491353 \r\n",
       "L 245.44375 136.922727 \r\n",
       "\" style=\"fill:none;stroke:#1f77b4;stroke-linecap:square;stroke-width:1.5;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_3\">\r\n",
       "    <path d=\"M 50.14375 143.1 \r\n",
       "L 50.14375 7.2 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_4\">\r\n",
       "    <path d=\"M 245.44375 143.1 \r\n",
       "L 245.44375 7.2 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_5\">\r\n",
       "    <path d=\"M 50.14375 143.1 \r\n",
       "L 245.44375 143.1 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_6\">\r\n",
       "    <path d=\"M 50.14375 7.2 \r\n",
       "L 245.44375 7.2 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "  </g>\r\n",
       " </g>\r\n",
       " <defs>\r\n",
       "  <clipPath id=\"pd19a7cf81c\">\r\n",
       "   <rect height=\"135.9\" width=\"195.3\" x=\"50.14375\" y=\"7.2\"/>\r\n",
       "  </clipPath>\r\n",
       " </defs>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "embed_size, num_hiddens, num_layers, dropout = 32, 32, 2, 0.1\n",
    "batch_size, num_steps = 64, 10\n",
    "lr, num_epochs, device = 0.005, 30, dltools.try_gpu()\n",
    "\n",
    "train_iter, src_vocab, tgt_vocab = dltools.load_data_nmt(batch_size, num_steps)\n",
    "encoder = Seq2SeqEncoder(len(src_vocab), embed_size, num_hiddens, num_layers,\n",
    "                        dropout)\n",
    "decoder = Seq2SeqDecoder(len(tgt_vocab), embed_size, num_hiddens, num_layers,\n",
    "                        dropout)\n",
    "net = dltools.EncoderDecoder(encoder, decoder)\n",
    "train_seq2seq(net, train_iter, lr, num_epochs, tgt_vocab, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1d09adcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 预测评估\n",
    "def predict_seq2seq(net, src_sentence, src_vocab, tgt_vocab, num_steps, device, save_attention_weights=False):\n",
    "    \"\"\"序列到序列模型的预测\"\"\"\n",
    "    # 在预测时将net设置为评估模式\n",
    "    net.eval()\n",
    "    src_tokens = src_vocab[src_sentence.lower().split(' ')] + [src_vocab['<eos>']]\n",
    "    enc_valid_len = torch.tensor([len(src_tokens)], device=device)\n",
    "    src_tokens = dltools.truncate_pad(src_tokens, num_steps, src_vocab['<pad>'])\n",
    "    # 添加批量轴\n",
    "    enc_X = torch.unsqueeze(torch.tensor(src_tokens, dtype=torch.long, device=device), dim=0)\n",
    "    enc_outputs = net.encoder(enc_X, enc_valid_len)\n",
    "    dec_state = net.decoder.init_state(enc_outputs, enc_valid_len)\n",
    "    # 添加批量轴\n",
    "    dec_X = torch.unsqueeze(torch.tensor([tgt_vocab['<bos>']], dtype=torch.long, device=device), dim=0)\n",
    "    output_seq, attention_weight_seq = [], []\n",
    "    for _ in range(num_steps):\n",
    "        Y, dec_state = net.decoder(dec_X, dec_state)\n",
    "        # 我们使用具有预测最高可能性的词元，作为解码器在下一时间步的输入\n",
    "        dec_X = Y.argmax(dim=2)\n",
    "        pred = dec_X.squeeze(dim=0).type(torch.int32).item()\n",
    "        # 保存注意力权重（稍后讨论）\n",
    "        if save_attention_weights:\n",
    "            attention_weight_seq.append(net.decoder.attention_weights)\n",
    "        # 一旦序列结束词元被预测，输出序列的生成就完成了\n",
    "        if pred == tgt_vocab['<eos>']:\n",
    "            break\n",
    "        output_seq.append(pred)\n",
    "    return ' '.join(tgt_vocab.to_tokens(output_seq)), attention_weight_seq\n",
    "\n",
    "# 评估机器翻译的结果bilingual evaluation understudy\n",
    "def bleu(pred_seq, label_seq, k):  #@save\n",
    "    \"\"\"计算BLEU\"\"\"\n",
    "    pred_tokens, label_tokens = pred_seq.split(' '), label_seq.split(' ')\n",
    "    len_pred, len_label = len(pred_tokens), len(label_tokens)\n",
    "    score = math.exp(min(0, 1 - len_label / len_pred))\n",
    "    for n in range(1, k + 1):\n",
    "        num_matches, label_subs = 0, collections.defaultdict(int)\n",
    "        for i in range(len_label - n + 1):\n",
    "            label_subs[' '.join(label_tokens[i: i + n])] += 1\n",
    "        for i in range(len_pred - n + 1):\n",
    "            if label_subs[' '.join(pred_tokens[i: i + n])] > 0:\n",
    "                num_matches += 1\n",
    "                label_subs[' '.join(pred_tokens[i: i + n])] -= 1\n",
    "        score *= math.pow(num_matches / (len_pred - n + 1), math.pow(0.5, n))\n",
    "    return score"
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUQAAABACAYAAABiDA5kAAAQqklEQVR4Ae2dTWvbzBbHn48yX0JLk43JotnE3E1EFoIuDFkYAsEUggJFBIrJIvg+EEQhiEIxgYJLwYWCuLegTXEWxVn0SRfFhYIvFLQoaFHQ5n+ZF9myopHtxK/pWRgnY1kezfznN2fOzJn5K45j0IvKgDRAGiANxPiLCoEaAmmANEAakBogIJKFTCME0gBpQGmAgEiNgRoDaYA0QECkYRINk0gDpIFxDZCFSNYBWQekAdIAWYjjPQP1lFQepAHSAFmIZB2QdUAaIA2QhUi9IVlEm6eBQeDBthjYlosegXzuICcLkUQ1d1ERaBcJ2gHaBwzGiwARaXfu2iUgkqjmLioC4gKBGAVwmIH6+wHV2wLaLgFxAYVKQFggEP70+vrchMEstL6F8E+rsPbKaF5Tec+rzREQ//QGRs+/UZZW/7UJtuWg9dqB88JGxTBg++FGPcO84LWI+xAQCQgrbkw9eHtlGIyBsSra38na0Tf0EP4xLyeG2hs+ZI4QhtGK6+9x1RcBkYC4Fg2qe0ZA1IMwgU4XDcZQfeXDO2CoXPTWou4m5zvJ//q/ExAJiGvRqLrnBMSJYPnWgskq8L7EEOV12IH/0kTjE1mJE8tuynZOQJyyoOZV4HSffCuBgJhfLmm9RL4NY7uJbhQjum7C3K3AqrVwSxqeW6dOQCQxzU1M6cY7698ExMlAnLVM6frZy5SASEBcXyD+7MI7rsI8slG3KrBO27j9FSMe+HB2Syhtl2FUPfjvm7BPGnAOSihZDtpfaQhJMJwdhrzMCIgExPUE4o8OagbDzlmAUNRRHy0esnbQxkD9367KGdfqVV9FbfTgPmFg+y304xDdKweWwWD8XTT50Id/VkeFz9y+m26xc3jdgvPUADOaFD4X9+DuNhBEegAN3tdhPqnAfGbDPvHgnlZgXhbVif5eiwY9AZGAuIZAjBC8MMCYCe/rqHGINXh8UkGkyRA2xmz43GoU9ZikNdBN/V8MRP7dLpozAJH/1uBt9Y8AYvSzh9axA/9nUsaZ9xsX5llXdki/evCe1dE4raK8bcHxeQdzi1atCfd8R3Rm/ThG/00V7LCjOrbM/VbcHgmIK66ARfd4m3L/cR+isvSYAfOgjvoz9TqwYO6bcD/zRpSFX16aivsttBD59wiId3XCLWwX3mkVjNXQGeSDq3teHs5yd88NWK/7onPqv7ZS60plPVTf8M9CdI4YrCt53d3fzf+dZV1HQCQgrqGFmACxaKH27ECMrl1YVgMtPpR+6sD/kTQ+CURjz4b7ro3mQQW1Vz01DI/Qu6yjetqEbSVWz59jIcbXTT0QowCN7cRtEMI/MWDseXLWW3xPuSFU/HWTd2TfWrB46OEbF7Wr27XQXhq2BMQCIIphEY+eGDacpAGt9l1aU001LFxtXtJiesjf4xZijN7FjojIaHwaf77wS4BbMXybHYi3lxWwrSa6vyMEpwzsxFfQk0CsvlU+xB9tVJkB/ttR0IBh2PDDGNFHB8xooPu7AIgKBMN7JfrSpSefq3epOekb5REpha+hP3W8jB5SD3e+WwBEXjbmy3xfYP/KAtuqo8PbzhcPFUP5Gb+1UTUqMGtNdIeujgXmP1O+d54v8zkBMVMgwwJTAl63wPl1hfSw3HTlOSE9C8T4VxfNXT6J4smZZf79723U9pKGNDsQRR6jEP2bAN4zlvJjZYfM8v/SxbUEZwIeAcqSGLIX+RB1daRLT5edvIZN2LBBPXuSrwllm77/zH9rgcjDCOUi8Tv35PW0bcG93rzZfgJirpiU4M67czHpe5cmDGbA/vDQIPyMJZPkPeSO7xqsZw6cmgnruIVeOHuvG37toLFnTGiMs9/3ToNJ8i3ee/D2TZQNaQ0Z2yacD8pSC2/RPrVQNkqo7Jswj1wE3JfFl93sV1BSFlRp14T3ISfts6xHOakS4faqjvKeg/ZNiIBHxgwd+zogfhR7DzKriU4QIFAvbqEWAXHo37yjn8m62hgg/uygbvHZ/IweeEf2tC5j0n+FygLPXJP9zhr9T0DMqwxhCUzqpaetZOlb4UOfimZ4UQyM1O+I3jozhI96wpIaLU+J0D3bAXsiIxom31sCqbJbQWVLQik7TJ18j1Qe88pzZWkpIPIGPJxJVkPmwxpqYulMpqMRfq4K3JsYYpj9ZLQ79e0HD8GPSUCMIX1vmbri5ZBXh6ny2RQghu9tyEmSVN1zGFo1tD6HCMM+2kc5z5961nXUFQExp4KkKOfpo+O7kjy8txTDyswwKXxfB2MZq07smTf9urpEmHLYyoTvLEnb3Hc+S9pAja9LtGy4/ld0LyyU9xpov23COaqiZJRhnvoI+SyzYaL2zEbzyoNzUIXz9lZZNyG6F3VhnXb49y57+N91C41D7uO0YL/071pJQlMZyA51pkuXYFkfIMpZZveYzxYbsJ67cP1kZniA9mEV7W8pGMajjn/k99TPTq+rrgiIQ6EmlTt5WLOaysxrSMl2UJnZWGXhsqOOWtScPFvx++MCYvGzLr4OdTrSpcv8rg8QC8rvWwvVY96RFFyzoZ9tDhB1YVyfPZgpf5Kx7aLHoxySGTpuBbzsgfvxEj9V5ciBXbNgie8ZMJ+rkDBRiTng0YaKlVF+ykPFBrh920B9GD5mo3UjHcqDDw7Mbb7ImIGdSZ+kyMtWGeUthsbbAN6pjcaLOsytMmoX3Xyh5Q7j+2jt82GuBoj/UksgphQnAXG+DTzPoucg1qXzz5YFxP67OizLRGnPQec6gHdSQ/24DnO7Ck9pV9dp8DWGD/eHz7esdXmdNX0zgDgxjCtG+MEWm4zyMC7u1O4cGqi97iH8nSr4Tw0Bpp1THwMVahTduLA4rA7acuiTCx5+jz60oWJ8j7rc8DH129/5Mo4REEUlqbywJw0EKgpADn8ZnI85s3O5vicJby0QWRKxkSqDAjgSEKcrp2kbmc71okvn910KEEMf9pOmMBy4X5UZteHGvDIayCkIxeMhlMWhetOWzzpetwFAnCaMiwuZO8q5JVZF88JG6bCNfhqGHAQCKizjDFYOdqZApAWiGurkhooVhY/FiJMhrLIQhRBUXsxXqcWpuflTjZSAOJcZ/2U2Qh34dOk8b8sAYvSxAfOii1j5mqURIXU23e/Pt+NYZp1M+q0NAGIStVAUxqUqKFm7xmx08mIvNcBJRMD4MomJQExbXQkkJ6QVAHFspk6TP1GJuUCcMGTeX8yQWTQovgxmmteZ/rjMkfN9wgJkbsWs+SuvoenAp0vn90i0WLz+VekuM8GWl4eiNGkN7ojZdHldYhyk0x4v/PLKZoOAmPGT5Q79+nLdGGPYOVcB5+nrNMARweaMwRDrxnJ8iOIeU8IvL8Z2HkDMBfVmT6p8/foVj+WV17h0vkJdOr/H8oCotKMib2T+lfFhOAjWMIokr4znnbYBQJwmjIv3YhF6f1dQOffhn1fA2M4w6HxYaLlAjOCfcAvEUL47BT7dotoxv9yUkJwHENUGBNmQsMTvOGZRqOesJ9tZ/eyideHC+5gsm8jv9cmHmF8uQ/2kO9eJf0/Q0R19yd9eHhDl+SxjKxFuXOzw9bL8rJbvHdhPTVS2a7D5pN+FA2tbheJNfPZZy1Et8bkMNEuYZr3f/a/fCCBODuOKMfAbqLAGAuE35Hu0MfAJi7F4SQWK4QRKLLdi53vhGdznqCpagiG7DnFK+C3KQkzumx0mqYXZ5RfJ0FQtzN5NQtxiDN7V1JAz7eu8KxoC4t0yuR8M+X10Iw1duvztpQFRwW8UrdNH+9CQ7eA3nwkvizWZJiupY05lvqfdM3KmchuMVoXYfs6E4twBrK/nzQAiLxBNGBdf1pJEWHA/k7CUPrvDsC7Gl93se3IjTwVE87gBuzbat82+DDBIT8Co64ZW17ShYjnhY/92HZi7JQWkEirPffz30hwtxeH5e+7Dz0mTG6GOKk82lpzV/yJ0z4JZc+A8s1A79mSIWyKk3wP0gg4a+3luhwH859wfOAqFk2WWCqFL7rOQ90d6DKnQUE5d6dJV2S4LiPJ3dmDx5WdHDupPTdQvgtHqizDE4L0tNrMQm7+KUY6BRrAYYA1uAnTOzMyE50j7MwH2ATrdHCA+4CGHhalANzaRkXtf3XBn+RU0zLvIZ7F1MX5tNq9dNLfkTi3F12W/t5z/H9cxpDr96NJHZbwcICaTJxNGDPxoWDUi4cvaGN/150eIKG085Laf0fPMorXueWnlUVIERF2FZq1E3XVLTtdaiRPywQVdmbhR6v2EPIvoddfK4XqeBbu6POnyOjFdZwXq0lN1txwg9tDkm2lUczZnGOZFrmAoibOfJUCNkw46L8oQ+xoOr5tT/fz0Ye820VsQbCfWmXqePwuIajH0tLv1DhvpWu2HmPgysz7OAmH+8uHwbbPusQPOtEJ66HXDsv5e8BzzboSLuF8yCkn2Vkx+Q5eefK7elwJEcb5zJlAgk4/4lw+bcYtNDpHFjk27Fuo8qCF77YP/j4TbpvnpobtBPVw7fwwQeYWOrZs7Xs8zHR4Klk39/qMB4gPhsGgg9t/Wx9vB0WgycVO1M898/zFAnGeh0b0e3hNnyzAXiLr4dW1sOR1Dmi1X+n82rRIQH9ijk+BmE5yuvO4AcWL8ekFsuTiGdD750uWX0h9n+RIQCYhrESM8DsRp4tcTX2p6pjRJS4dSzrPhzn6GM4FznuW/+HsREAmIawjEaeLX8+CXlzbvRrTABcqkxZVrkYBIIly5CLkVNW4hJkAsWoaTB7+8NAIiWanTa4CASEBcQyBOE7+eB790Go+PdWDx9XbbNTRet9B8WoYllo2kPuMb9RoMpVMe+ph/BnMc9uAd8UiONtovLBEFtZAQNtLiyrVIQCQRrlyEdy3EGJPj19PwSyyAbJr8X+5iFCNWB03JSCX12XEH/W9dBF9CzRnM8rrScFG73BSBgJiU+eN6JyASEFcMxGUdQ8obrgKmOGtGAXEIOhXOlmyeIWJ3S3D/IzceGAFwM3yI0U9+NK0DP29fUNK8VvMERBKHVhyb73vKQk8u1TFO+AFJ2c8ULLNnMP+zaUCUW2l5p1Uwtnmn3q1acwREAuKjB6LYCZ3XsxgyG5DWXhaIujOY+RkiDPKw+xhxFMAZnu883+FidO2iblVQsmzYxzaaf9swLRc9df7PTLAQoYIExJnKLI5BQCQgPnogGnxS5U0bzZqJ+mUPYZyaVNmto/Gmpz2DOeL6+O7DsSzYVx14J6aYVDH4aYuf57kVFj/ruIbWawfMMOHdJGfxGGozhRC3QYCg4NVLx9wTEO+lawIiAfFewpm1513N9XetwNXkYzpLMgoj9F7ugFlqF5rrJgxmoTV2IPx095IHqpGFOGt9ExAJiI8YiMpnOJw4mRImK9OEyq86XkDAcddD71eoLNgZ8k8W4r10TUBcmfhnEDfl8R7ilpML7oUL8fpQfJ7MrJbEQq4X/kkD9fcDcQ54a5+hctlBq1pD58c0Q+bUEJ6AeA/NkA/xXoW2kMZA0KO6+OKhYlTRFkPkEP5JCTt7FuqvEh/nNJ2o6giOLXFwmvXchetvQGewJvonC3FNKoIgO01jp2tIJ4vVAAGRgEiWGWmANKA0QECkxkCNgTRAGiAgLtb0pqENlS9pYPM0QBYiWQdkHZAGSANkIW5e70UWB9UZaWCxGiALkawDsg5IA6QBshAX29NQT07lSxrYPA38HyAShIZgiRAgAAAAAElFTkSuQmCC"
    }
   },
   "cell_type": "markdown",
   "id": "9100b3d4",
   "metadata": {},
   "source": [
    "### bleu定义\n",
    "![image.png](attachment:image.png)\n",
    "label:  A、B、C、D、E、F\n",
    "predict:A、B、B、C、D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "dd9da5c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "go . => <unk> !, bleu 0.000\n",
      "i lost . => je suis <unk> ., bleu 0.000\n",
      "he's calm . => il est <unk> ., bleu 0.658\n",
      "i'm home . => je suis <unk> ., bleu 0.512\n"
     ]
    }
   ],
   "source": [
    "# 开始预测\n",
    "engs = ['go .', \"i lost .\", 'he\\'s calm .', 'i\\'m home .']\n",
    "fras = ['va !', 'j\\'ai perdu .', 'il est calme .', 'je suis chez moi .']\n",
    "for eng, fra in zip(engs, fras):\n",
    "    translation, attention_weight_seq = predict_seq2seq(\n",
    "        net, eng, src_vocab, tgt_vocab, num_steps, device)\n",
    "    print(f'{eng} => {translation}, bleu {bleu(translation, fra, k=2):.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18a245a3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}