48-注意力机制-多头注意力-自然语言处理-pytorch.ipynb

{
 "cells": [
  {
   "attachments": {
    "image-2.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAASoAAAApCAYAAACC2WQ1AAAOlElEQVR4Ae1cTWvbWBeen3L/hJYmi4Yu4ncR824ishDMwpCFoVBMIXhgMIUgsgiiEER5QygUUQi4dHChIHgL3hRnEZzF4C6CCwUvCl4UtBjw5hnO/ZCuvuzYtWeUmbsIdixZOufc5zzn4x75p/l8DvNnbGAwYDBQZQz8VGXhjGzGeQwGDAYIA4aoTEZpMmqDgcpjwBCVAWnlQWqyKpNVGaIyRGWIymCg8hgwRGVAWnmQmozKZFSGqAxRGaIyGKg8BgxRGZBWHqQmozIZlSEqQ1SGqAwGKo8BQ1QGpJUHqcmoTEZliMoQlSEqg4HKY8AQlQFp5UFqMiqTURmiMkRliMpgoPIYMET1EEEazTCLSqLs9xmiP0qOVU3XRXrMI8xmUeUdSGV70WyGqMS+0QPSQ+lTtVdDVCXgqtpCxfJ86aF14GH4vYSMlh2vir66nNMB/Cd1MNZBOFN6RRie2Wi9nVSerCZvW7DPhqVEtex4vLZVWZsKymGIqoKLUg7cGcLjGtzB4kxj8sZB4+Wowg6e12P0sg52GGCir0c0gLvTQfhNkVcFX7+F6Oy4GJRluFyfCQKnAf+2gvLr9q7we0NUFV6cHGHdBXAsD8NlMn/tocm6S5znb3SanB5T9J8wWC/y5Do8s2C/rm5WRUHBOhsuDQrTt02w54PSrCu31svW+F923BDVA1rw6bsW2HGImSbz5EMXTsuDf+Kg8bgG74YIaAR/z5Lv/0ZC0uTUHTGnRzRAlzF0PowxOGnCOajHWePsQwfsSR/Tkmvp1/3r3wuC7XyYSaIaI2g5aOzV4H6KMDqvJet166NueRhVUo/qYSS7loaotgGc72MET9sIPi8u0bKLsez/4RkD06P3rY8Ga6L3dY75JxcszqKm6B0xtN5Nl0b6ZffcxvEiPeqsgfZxG967AC1mxUQ1v/bAKuvgQ3iMwbsWjh59dNG5CuBaRLozzD9fwlaBhWe5LfSnq5FCdO3BPu6ngtM21qTq16wsUU3ftrD7uAbGmOaAqy3yto3Ps5mjJpp7TQR3SjbRf1FNYO6UXAfSQ/3ZCO4EyJPP5DHZp+GlQny+sMH/TtNENaT/mz3e1+E9nqOezDwEUTXfboqoxPVyshIx3lGZqfTSXmWZk9P/MMBvGcKdXDXBrBoaR10En1R2Iu1JRMW0cvfaQ21H4YLsqOx+z1dOGJqcSvbnA0yoPFP/a6/djxRw8utlv/4tRVSEt+hjN8HrrY+mKlv5fWVQWTE4jl42YJ1sqWy8C+GfttHYa8E999A+qKH53Id77KBm7cJ+6mMQk+sM4S826ns1OK023LMOHMcv39hZUc9F/lpZouJC8yxBOOniZuU9QbpBw82pz8IsNJsOGLPi7IWAajmXGKeaq1SKKedwUz2maODCkk5RvxinM6BvfbRZnZcRZI/JaxvsVPVDiBAVcQkiSXo5E/SaDMLBNmgbLo/UQxKkAhf1apSTJ6WQvDeVPZT5fRH/p/WIMHjOeJ9nFnZ49jS47aF3I7NRwkC2yf5FkeMaRCUxMHvfjuVtXuk9MGp8q7XSdyGF7KPzOtiRCA7z+QTBIYP7KbExDzA8YEQYnNi4/F0e4zKv2TekTQXLijM3ZfPNvQ7hcRsTjhIZSVf3/QjBExmIv4XonvjwDhlEEBTknVvvTfqZvFa1iYpH02oSFXc2WXYlczIC5Gnga84pCSm1sLrzP/JTPQzuTHsehpL0eLTWCIKXBQcdeM8d1JgO5CFcjRg2B2hBKoKQ6okTEphufeyqLORZulRRzq36TGk9xrjcF6VedOPDdjroPEvGL3i2FZOzdPo4K1qfqOayL8Z1+c8lxppzjV7uxiTWfq9neCLgJJmqsEdqvb/00X7cgnfRRedilDTPiXC1tVt1TbgNVRmpyVp8nQjT2xC9Cx/+efLXV6SZ+34xUVHZzYPfXQD7ZCCCKLd9XexgyoCx8YCYk6/qv5leWaISTVR26GOkZ060oMzG5eckwsZA4hmYjNQa4HizWDk406OzID3nYpyAnUfWNvrZ7fprD1bcnxKkUXcyW/0Fix/LtsKxsgyQl56xHkn2RIRA2UCKnMv0yMkxQ/+p1q9SxzdBVHPKeCxJSDrp6tlvUlqTrbjuVnpcgn/2NE3MRXYlonHe6JlbAUaUfkWvvLooWPvMudHnAG2njcv3I0x1bGbOS8u4hKiIkGTpKUpbURXwQLrv4+P7Luy9GrphCH9n9T5cWpZiuzyQjKoN/8pD+9BGY4eBWbtov/uBRV+4aMWGUsYcntuwDxuokVPuNGAf2mi+FJFTkE6SOqvviFdyOlVSKEcWn9UdB3Xl5GoLm5OeOi+RiQ8PajNSs+sA3Z8tsEdNdD+QTagktOGp0knTNfrch/eUZLewe9BAY7+D4DqEu8dWGAHQHVmVseIzx0nKP1WGcmDvubk+RlaPtK2kvrc+7KIdvyxRxf9L+yobarqXXT+2u8raeJnqwInLP5W1ieypfpod7Jyg98RePCNFs1YHXjqoxUFYYULeJ/W5hiWpo2rcF+rztY/WgYtBNpAtswP13wpKv8lrwtQY4UkT7kBkljQuwiwb7oWH1s8eBt8iTG/6cA9FRj8802Reet8E14X6aN9/IETF0DgJMaVHQ7gD0+Lq0WXZ4xYRRq/baD+7z5+LkHbRNCPl3suU136V7imJocV0GaF/V++LqJTaYQ4ufx/n+iKqXEoNQHKZIgzPOwhK0vhZ6KJbROJ8h5DsVkc3lOUMTYdbwlEUsejylr3nsklipbQ/zjS+UvYkHY+XU5KIc85N9l2sx/yPMYLjkkZtTEzCuWc3PhzLgvNiIDCyaO1Sx3TSJQcTWZZ1HGKa7R1m+oUp23wfwj8OMC58dGmG8KSLvuzPpb5HPS5JiMkslrBZIzfpTuU8Q7oUTeOUmu6pzDWla/rctBw6UdXROvHRbe3CYtRYDxDeqfJX2CspfeU1p320eEY/QfBsO5n8AyEqrZzKgJQMzglgp6s9frFoUX78mEh/8yUJ3+HKNn51sPCSJ3HkPjWgZYmm7/I13/TgZcsl/TprvOeRkMglU67ynUO2SkYlgoWjMsDjHvonFqx4l0+VUwzuFTW9iYg3O6YxjzGwi/YvbdTYLjpvJ0mJvIJ9UpsAb/u8TI13+RTpMhc92pnMbZL8OJaS4CUb95mNh4RQROO6PKCIDRXnOOlJ6f0pen/fHlX/fReNl0MMXzQ0YpxheNFGgzE4xz2M9Ee4ohH8gxa88y786w2vtVzLB0NU8TZ0DFKVkv84WBIw3O9aoh+T32peSlSpvoggrLgRqzfViQQyvZBVZUyfr40XZIhUjQ+UO0CRTZJMQDXWacCR35M7miRj0mMLzp0QlX6fNSO53jskebUyVc8cSc9Uv3AFMkyvRcaeWvBqXtHAqwXK6PShXvH9ZURFJf/u4hK0VGY9o6LMkq7VgH8zgr+vVy4Z2Uuvt/nzHghRdRAqBk8RVYThixbs/WQ8oBgUmyz9Shrpanzgv+WlX96RdRDozV2GfC9ELv4fy8rcYpDEGVVmZ3E9opJZrMqqUllamsRiIs6Cek09uA1jDHQQfh3C2xeEpbK6YgwU24V6eknvkMGW/UZ+jRSJ5fuF8X2+L/g1i6zeBf/rWR1jDXiFWYko/XJll3Y9ys7qWv8ylk87p/izLFHNMae+GgUZahnE83llNtz+5w+EqLQGXQxSG8H/A7Re9BAcsTUXaB0DC8CwX8JcqbG4ma7uJWZvKEJnI2eyo1YvAeucP5ZhHSwhwyJgxj0qhk4os59cjyrC8JSGKfNlbQ7g2vZ+NtNIHC+fdarr0OMla+lBusUYkLjQ9HDUgOWcZsmoDHWWDoWKUp7ILlumaqRb6qwj+DsWsv1Kpee9XvVsOp7RUniRr1znclzw+0QjePvL9c3LVEBU8zmoHUE7lbTxEQ+uFmGr7DMaBqbyeQM70JUlKr1nI8oLG8FrmlLW0n3G0H3zCg77C8tA2cwvjGz8WKN4PEFbTOHIySBnAhzZ3N1GuUTb65976P5MTVILu/s27KdttOQgqij9IoxetWHv18DU3IwmdyInOY/KAAsyDel4WSJOfz/jiKX3yZyX2hUjLNgIPqoBUIUNIrApwudN2I+t5WMBqvwqIAnRQ8qMV9xX1nufRz9pQz9zs+A+NJ5wn3bAtwHcAxvu+3H5b5bpcv3eF5PpzEHn3EVrz4Lzq+pnEVE30D7tovmIofFruNozl19DdI9s7FrrkGd63StLVPcFNXd65xLBiw76y3br9AVa9b0sVab0kGxplBYRuLTcWfWeqfMn6D+jB14bhaMH97VX+rykd6X3qCjDSJVAKTnSAEpf7z7HtqFH+X1pMHfRTtnq8mv3ouFOp4H6fmb0YB17cXyV//ge9cvuT/zFA5/Bjdq903RYR9aVvkMVhN7iWO/eD5+orpqw9m20XmTnW9YzSDFw6RknKiNsOA41O8sH/Kj8szKTzsXXXE0+euC1fRXAUw+8rgSWsnsVERU1Upub36mT8m5HjzL9aDcqPaC5ibUQ14gwOG2j94aGbfOP2mzuPnOIoVkH/u12dtQ2KmsWlzQLV7g5ULZmxZ8/eKLaqpE1o49eOag92kXjyMdw4UAdNdst+SxUsdHXlVn0sLR+nSbf6tfMP2RLv5c0C7v8cYn8rtPmdNmsHuVyjV85aL3WJvt/yF5F95Hl730HTNe8P81H5eeqiuSp2mdjXDqtjfyKiCGqNcGzkBjoZ17Ug5wbu74cRjy5RPDMx7BwuLBqQC2S55+ih8p0LLgXAdov1cPiRTqv/xk9z9k86WOidr03hqf1ZVqI/S3JZ4hqS4bd/GKKHbn6vvO3PT60GZ3+KXoQUQ3h7tTRcNolk+cPiww2s77b0dkQ1YMhqu0AoMrgNLKZNVcYMERliGrxc43GPsY+FcCAIaoKLIKKGubVZBAGA8UYMERliMpkDAYDlceAISoD0sqD1GQZxVnGv8kuhqgMURmiMhioPAb+BEDKNq3zIJ+NAAAAAElFTkSuQmCC"
    },
    "image-3.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAALcAAABXCAYAAABGKtQdAAAJxUlEQVR4Ae2dMYvjSBbHN5hvMNFk9SUUmgnObNC+YMxdMGICcRsYOhAs15iGRgOLaWjEBI046DMNjZmlEQN7uNmlmx0Q3ICSQR00nmDwBIMHDhw0OFhwcKDkf5Qk22XZbsuypHLrXmAsyXI9qer3/vVUUj195/s+6EN1UEYGvivjSdE5kbNyBghu6rlK23MT3AQ3wU1dPXX1j40BUm5SblLux+a1dLzU05Byk3KTcpMSJlHCEfquCzf+uRtinIcTjYfoxW3x9S+j0gK7CYek3JlC58FiDCz+eWVjkKmdyNH+00UjbouvH7v5OFMe55BjmQR3ppUbwS0NLtn2k/Ruxe1DcBcOtwfreRVVJVR49XKQYQhBcIthC8FdONxcuQboNghuEcQ8lgluKXAP0d0nuPMAWiyT4Ca4MwyLiounRYhXLRPckuGun3TQOWpAfVmFwhgqWhveH2khoZhbBJ3glgw3U3TYn/m49BjucRiqVM97gpqOMRqNhfWHwCe4Ce5MgRZhSwrXLOZmr53pmLR3GsIdjFOPPNhnJpovFLD9LoaJjjmpffGYy7tMyp0ImqQAJIVLgFsYE5+DOzquYBvBnbDnmm8nglsy3Orb/rThCO55OMUQI80ywS0bbuEmDsFNcE+VLo035/ufJGFJtI/4TMixDTsa954+lxI9j0JhSXrgSbmlKHfyBiO4k9dVXLgI7p2FewDnrA1DY2B7OswzG95oXUMn6TnWlVGe3wnunYU7DWQEt6jeBDfBvcPXMGkcfPYfgpvgJrhFuaflmTrM14XssEC2/VX1Imc7KTcpNyn3vELJ8cTdPwbZyinb/m5xQcpNyk3KvfuquQuqIVs5ZdvfhTaYHQMpNyk3KTcp90wRVteFbOWUbT9JHRW3Dyl3psr9G/SnT/H0b/+aTkBY7Qh5NLJs+3mcU/oyCe5M4f4Zf33yBE++/ydGmZabtIFl2188zpFjQN2roaLpaJ5YMDQN7duk0+YWy9tELMoF9wPpxQZXjcU0Z4yh9YFXdNSdC4+hpkuWIxuuAuwH099a0BQNxlkbrf0a1EMT7WMdVaWC+n4L3WBOKAdzBOfYRPtUnU6VC55yPHIKcf5ywR2p5eimOQW58YuY0WkAmz9lF0BswIk9Zdc7qwWNkD6v37/x+tkzPNN/K6TxFlWsKPt8mlwL7tiHd6rC/hoqLK/3xqUH50SNRINvD6fU1YJJz1EyImFq3eI5bKfWYnmlhNsfu2hNVPjPHfSFEKF3Xp2C37wRs6H20N5jaFwNtxg92O6CrnehQmEVmB/Tdtvp7Y++eLh+10b7bPbpuKvqYjncPu85Obi8/rUo+WfUFuZHH/79NZqsjvan7AAWYY4vlxNunibhRIkgrqHzeVKZIcDT2S6N7jT76tg1oSgGnPvJvmm+08PFGyaAW1FhpY5JU9i/d2FqDZhXLvqJz30N3L4HcwL3rQWFKVDfdGDpGqwPoaCMP3XQ/MnG9XkTzZtVTpSmDWb/KSncPvxPbdQm6v3GC9U42KZBm4Ymky41zBlSe+NtOcqRAi6hV4krz+brG9of92C91GF/2bSnWA73+EMLjbceepdNNN8NgrrsnYeh3nxqCi4ytVDBP5rTeHzz852BvOy/5YXbF1Wax4ehmitHDoZcpSPwaxf9qLusbREOTCpZ9lDcZvZH7w1oF/0UDi3CzaAdtWEeaahwhT5so3sbKvHotoPmCwamGejeCQ4UXPib8Hwf/bcqamdiEqJJXW7/XWK4fQzeadP42ri6hqkos9GRKIUwYya6vzTAtA76420rtIDRigeVfjP7fOSi9qM5F2enibk7Nza0w2v0ftVRn8uWtao+h7g+bMJ2ujBPuuinTh+3qvxwe6nh9r/a0CahCf/eM6d5+IKREeG3dAoWr9zN4FrWlW63bTP7PGSYv6iOn8+qdVG5w9Cud15H86YP56i4C8Z1dVVuuP0Rrg8mQ38M6nlv1gXPgd9A99tiQ46/dNH6QYd1ZcPQdHQ+CV3rUgXdMOaNlVH4aAm/Bjm8TjFsuQi3z8PAFwacbw6M51YQcqyDL+/fSw63D36RE46OaOh8FuEUxryXpSu7d2BMwxg+nsvA1t58yADuQkdLhrj+sSKMSS86+HIAl8HtY3xroX7iYnRrQU05lt17q6GiVNG+S3osq/crPdx8zNvk8fX+bNhv0mDhzR4FxntxvDusrMGlCqZMFIg3Zv5wT44r/XcK5/qjD/ugCv3CwyBJ7Dt00Tnjdyhr0E/C1BP1AxPhmDgfdapA+8lC8yWD8kMH/f+uhm/xPMfo/aMOxuow3cU2Wdz/4bLLD3es609aQXNDWN/4W8PqsMQr/qXlRncI//77LPxZut/DjZL0GBf3S29/6U0cR7y7m9cx51cuwb0KvnsXltYI4u3WgQH7LomSbHZBtwjntg0t2/62x5/t/wnuVXCn2i4bLtn2s4VzW+cnuFNBvKoRZcMl2/6qepGzneDOFO4UF3SC/cKHAgXb26rkLv6f4M60gTOAu9ChQDmKWpQjENw7BPf2jb6dc21vf7echeAmuLd4fn23YI47J8FNcBPcca+g9WWqJTsskG1/WZ3I20bKTcpNyk1KnUSBZCunbPtJ6qi4fUi5SblJuUm5kyiObOWUbT9JHRW3Dyk3KTcpNyl3EsVZr5zeaQXV76O0E9GLVLOr2/X2s7OVpD7k7kPKLUG5B3xCMp+/SXDn2msQ3BLgHk7yFhLcBPfj6UqThQUEdzHhCim3TOX+i4nOhYHGKxVVPs/z+bbpfZM51+MRi+2cgOCWCTdToF/2g9QK01n6f2qjl/qYCG7RcQnu1CAtU5VkcE3DEmbAmcw4v7WiFBRhamD/qwOD5zTc2wT2ZPZFAMq8THBLhTsCmR9DHG6eqfaYgW2U/4PgFp2V4JYJ9yshd/gC3D1YCoP+6ybpfQlugjtToMXwJBlc07BEHAqMw/3VhsoYqroF+7SByuskr9pIZl8EoMzLpNyZgr4eriAtm5CAk7EW7Mv4+3pUXPxsgCktuDwm5zms2ST7lehM8eX19ssMc/zcCO6C4Y43wKp1780s3g4y0q7NU8hBJ7jF+iS4dxLuAexXk3jbg6UoMN87sHR77v0+YkOGywS3WCcEdx5wz4UdKZ4hCV6MpIVvCRt7MJ+r0A/0xXflPPBqwnGm5xUPfx7HOsGdKQQj9F0X7sInvFEjqkomy+Mhegu2XLh3Q0mJOHcLeoI7U7h3q3EzcaBHXD8E9yNuvP93eNedP8FNcOf62Ok6APP8neAmuAnuPD2MyqZYPQ8GSLlJuUm58/AsKpMUO08GSLlJuUm58/QwKpsUPA8GSLlJuUur3P8DlI7XTR2r4hQAAAAASUVORK5CYII="
    },
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfYAAADWCAYAAAAjMKA+AAAgAElEQVR4Ae2df4wmRZnHSTSocDJr/HEHhzOcnoBk3QFJFk9xhxy5Y/+4WeEW//DiLhg5uOSYZY0hR0BcCHdwJrsRIRg3NxsETQhxVz2IuQQ2gBpjZE2UaATcoIdRo5tFSDT+QXLpy6fXZ6ampvv90W9Xd1W93z861V2/up6q53k+VfV293vSq6++WuhQH0gHpAPSAemAdCAPHThJA5nHQGocNY7SAemAdEA6gA4I7Nqx0I6NdEA6IB2QDmSkAwJ7RoOp2bpm69IB6YB0QDogsAvsmqlLB6QD0gHpQEY6ILBnNJiaqWumLh2QDkgHpAMCu8Cumbp0QDogHZAOZKQDAntGg6mZumbq0gHpgHRAOiCwC+yaqUsHpAPSAelARjogsGc0mJqpa6YuHZAOSAekAwK7wK6ZunRAOiAdkA5kpAMCe0aDqZl62jP1559/vrj11lt1BO6D++67TxCT38taBwR2KXjWCp7SZGfPnj3FKaecUmzZskVHoD7YuHFjcdJJJxUvvfSS9F6+L1sdENil3Nkqd0pQp62s1s855xyNR0CbfPTRR0uwHz9+XP0csJ9Ts73c2iuwS7nl4CLRAYE9/E8pAnv4Ps4NkinKI7BH4tRTVB61uV0nKbC3259V+imwh+/jqn5XXLf9LrAL7FqxR6IDAnt45yewh+9jQbz/PhbYI3HqMob+jaHvMRDYw+uAwB6+j/u2I91f/8eu1aomNtHoQJdgP3jwYHHs2LFa2avSKDPIaS4vL69Lr4obVEfoNIFdYA+tYzHUrxW7wLbOGcegmNPYhq7AvmPHjmJxcbGYnZ0tjhw5sm78gbqlcX748OHi0KFDxdzcXHnOtR3uOPGaHtdHjx5dSXfLuPd6/PHHy3vUvdrH/ekPt/42zgV2gb0NPYq9DoFdYG/decau9LG2LzTYgfTOnTtXgAloAWvVShw4k0Ye2rVp06aCCQHn7uH2pYEdaJPX8i8tLZXl3dU7eajHLe+e2z3cuDbOBXaBvQ09ir0OgV1gr3WusStvbu0DZqHeYweqrIJduNJ/wB4gLywslCttt09JszyAnXODt5uPOomfmZkpgDhpBnfOSfchTjqree5bdZDml3Hv2fRcYBfYm+pOSuUEdoFdYI9EB0KBHdiyemZlzj2qDtIAPyt1dysdZ2Yrb7bf+WqbbcOTz3V2Bn0D+YYNG8p7Aeldu3atAfWwFbtbb5vnArvA3qY+xVqXwB6JU49VQdSu7hxhKLDbyhsQA1QOVuB79+5duXbH2aAL6Ik3UNM+wG4TA/K55QA7cWzfE1I/eTm3w/JbGnmrVuzEkWb52woF9u70ua0xUz3jj5nALrC37jxliOMbIn0WCuxV42EQrkqzOFuB2zUhYHevOQfA+/btK7fiAbZNBObn51d+07fJgK3yuSavX1foa4G9mW6GHhfV3+64COwCe+fOVUZcbcQhwe6vjNkmB7y2Wrbfxt2xAeyUI7QDsNs5IflpNzC3a8556p7DgG5PuRvYuR9p1DfocNvTxrnAXq17bfSt6oinbwV2gV1gj0QHQoKd7XjbDif0t+Krtr0N1K7DrlqxW7qbH5AfOHCg1C3bdrd8hOS1nwgIaY9dMzGwc7dMG+cCezzwaWM8VUf1eArskTh1KWi1gk5Tv4QEu9+PgBXA+/F2zcN0rOqBrMURjgp2wIw8AJt6bKVOHQZyq9ce7LNrJhmUs+s2Q4FddtamPsVal8AusAdxoLEqfMztCgV2fxue7Xd/K54424637XcADHS3bdu2sqVOG90DYJOf39htxW5P3/M0POWZHABqyhnw7fd17un+JMD2PfVw8M592+MlsAvsbetUjPUJ7AJ7684zRkVPoU2hwA5M3W34unMATT+R38655ryujKXTdisDyIE79bj9bqt/8lkaEwN3Ne/mpw73uo1zgV1gb0OPYq9DYBfYW3eesSt9rO0LBfZY5e2jXQK7wN6H3nV9T4FdYBfYI9EBgT08dAT28H3cNcR0v/VjKrBH4tSlnOuVc9r6RGAPrwMCe/g+nja7jVFegV1g14o9Eh0Q2MNDR2AP38cxgm7a2iSwR+LUp03xJO96B7tnz57ye+11D6opfvXTtE374s477yxf2XvppZc0oZXvy1YHBHYpd7bKndrk4cEHHxz4FbZBX2hT2uAv2Ln9c/rpp0vn5fey1gGBXQqetYKnBvfU2hvilbTU+kDtXb/7pD7pt08EdoFdYJcONNYBPjxj76fLmffrzNX/6n/TAYFdTr2xUzclUjidDoUPy7DFzdfipAPTqQMa9zjHXWAX2OWUpQONdIDPwdpv13Vfj5Pjj9Pxa1zyHheBXU69kVOXY8jbMYwyvjMzMytgt2+/j1JOeaQ70oGwOiCwC+wCu3RgbB3goTlbrRPyhy9y1mGdtfpX/TuqDgjscupyyNKBsXXA/oUNqF988cUl5O1PYEZ1PsonUEkHwuiAwC6nPrZTlzGGMcaU+pXf1AE5YL/pppsG/rd7SnKprdLtHHRAYBfYBXbpQGMdAOyf+cxnGpfPwYlKBk0GYtMBgV1OXU5ZOtBYBwR2QS02qKk9rxYCu5x6Y6cuA5JTF9ilA/ID8emAwC6wC+zSgcY6ILDH59QFWo2JwC6n3tipy4HIgQjs0gH5gfh0QGAX2AX2iHTg8OHD5RPmTf+WtOtygP26665Lqs0/+tGPpPMR6bwmBu1PDAR2KbicXCQ6cNddd6356AvQ1BGmDwST9mGiPo2nTwX2SJy6jCIeo+hrLG699dZi9p3nFo88XyRzLD/5v8V/P/t/ybT3tuX/KSdLx48f14RWvi9bHRDYpdzZKndfgG563xTBntIkhLYK7JpAN7XPlMoJ7AK7wB6JDgjs4XcqBHaBPSVAN22rwB6JU286gCqXj6MS2AV22XM+9tznWArsArtW7JHogMAusPcJA907n0mFwB6JU5dR5WNUTccydrA/dOTlZB6Sq/vtX1vxsrOm9plSOYFdYNeKPRIdiAXsAPyiSz9U+CB/2xlzA8G+cfNCsfzEz9fk+cj1e4r/+NKTa+LqoNtFvMAusKcE6KZtFdgjcepNB1Dl8nFUsYAdwN5w1/3FX507X0L5PZsvKd7x7vOLU0/bUHDuHgZjgA7YuQbkH1m6rTzIe+k/Xr1y7YPfyncVCuz52It8X/1YCuwCu1bsHerA4uJisbCwUBw4cKA4duzYmr6PCeyA9ub7vrayagfyrODrVt9/e8VVZRrpTAoIOYi/5ubPrlz7uwBdAd3uMwjsfLVv586dBf81L2jUQ0N9E3/fCOwdOnUZxGgGcejQoQLIAcBBx7Zt28p85A11AGA+89rWsbS0VMzNzZUfSdmwYUMJkiNHjpQgiQ3sBkO20zm45kt4Fm/h3V//Qblav/NLT5WreXdVHvtWPBDft2/fyphs2rSpsPGQvY5mr+qn+PpJYBfYo1md4GQBOfDAwW7ZsiX4MTs7W96vq0+3vu51r6u8H/IC/Ri+PMeKm231U944U0J8cecN5TVx9BMhB/mAOyG/yRNnEwDL44bUY5OBvkJbsd97770FE0N33Jlozc/PFxdeeOG6CaWbL9VzbIvJo0AcH4jbHhOBXWCPwtDZlsax5rhiYgVof87irtgBxI4dO4qDBw9GuWIHyj6AabMfxzVb7LZdX5UeS5yB/YUXXiiWl5fLiaML6q1bt5YTrFA7QNTLeJs+dBVyT3RvZmamnLxoVyJvuAvsAnsUYGf1BNT9353bnsn2XR+/sbMTAVR8WWPbigfsbLO7D8sBQffagM2Knd/S//mWu8sVPGUBva3YObfVvJXpIzSwu9+KZ6eo7Ps/7d7YRKtvXQlxf2TFztiZCFG/6oxjwiCwC+y9GzjOBmCweplmxxAj2H34Vq3YeUgOgLPVzgN3lOGaeCvPeaxgd3WOlaw/4XLTczjH3li5o285yCMZ1k8mBHaBvXfjZoUEMKbdQGMCO1vrb/vLs1aeijdAV4Hd0gh5cA64pwr2adFBtuXZOZoWeadNToFdYO/duAEa24PTZny+vLGAHagDZnuIbvd/frHgiXcO3k+3c0LyshrnXXW229mSN7C776+X5396st6dCHR9XrUV74/DNFzzU5Am0+tXurmMvcAusPcOVICm1cOrJ37njeD/2AE1v5cDXcDNuftkPNC3g3R+h+dwIU1+JgbUxcE5h5unj3OB/QTM+NlLYBfYe3f+ucykJMd6YxLYT/RJLCv2PoDb1T0FdoF9GnywVuxasfc+aRPYBXaBff2ENySAtGLvtr9DjmVV3QK7wC6wR6IDWrHrb1urnHSIOIFdYO/d8YdQbNUZj2JrxX5iLAR2gb0rvySwx+P/Qoy5VuyRrNZCDG4qdQrsAru24rsFjcDebX937YsFdoG99x0Zgf2Ek7n99tvLJ5V5WllH2D545ZVXetf7rp29ez+BXWCfagNwjUHnYYxBYD/Rr7/4xS86/4Y4Dn7ajqeeemrqfZrAHsaXxcIIrdi1Yu/dyQnseTuZWJyd2rGqZwL7al/kqBcCu8AusEsHeteBHJ1rzDIJ7AK7jF6OP6gOaMWet5OJGXDT2jaBPW+b04pd0A4K7VEcp8Cet5MZRQeUp1sdENi77e+u9VtgF9gFdulA7zrQteOb9vsJ7AK7jF6OP6gOaMWet5OZdojGKL/AnrfNacUuaAeF9ihOTWDP28mMogPK060OCOzd9nfX+i2wC+wCu3Sgdx3o2vFN+/0EdoFdRi/HH1QHtGLP28lMO0RjlF9gz9vmtGIXtINCexSnJrDn7WRG0QHl6VYHBPZu+7tr/RbYBXaBXTrQuw507fim/X4Cu8Auo5fjD6oDWrHn7WSmHaIxyi+w521zWrEL2kGhPYpTE9jzdjKj6IDydKsDAnu3/d21fgvsAcD++c9/Xn+7qb8eDaoDf7d4Ze8Tsq6d1aD7PfTQQ0H7W3+jG/ZvdFPo33eec14yNiewBwA7K9AzL7ikuPDq23SM0Afn/9NNxbu3/Yv6aoS+Mp3CEQ4C3bSlYXP0ifWPwsG+B5s7e+vV6q9MbU5gDwj2a58sCh3qg7Z14B8++2QJsWmD9yB5Dext97Xqk/2iA6nZnMAusGvykdgELDUnMwjIbaUJ7AJwyElYajYnsAvsArvAnvy2vsAusAvsqw8ECuwCu8AusAvsielASIip7vWTJK3YA4Cyre26ruph9cDDczKQ9QaiPpm8T1JzMl3YnVbsk+uVbLO+D1OzOa3YA0xEpg3sf3/H14qrH315zUTmIw/9vLjwqj1r4uocB+U56tL9+I3bbxg5r182h+vUnIzAXg+Mpvq4/b9+UHD45c/6wIfWxfl5uEaHxrGjS/7t/sr7VdWdY1xqNiewC+wjOYJBxgqU/+zP59YZ/unzCwWA98viJEijDAfndpx86kx5PsjpkNevc5quU3MyAnsYsL/5nfMFtuTq/tmXXVU5SWYSgN1Qxrc5XhM0+3Prcs+pF71z46bpPDWbE9gF9laMFcfBCh0gn3H+JeXxxr84q3jzX5+/cm1OCCNZvPup4v3X312mcc7Bu8dnXXx5ec61TQoILQ8hdbrXnFetXnJ1PKk5GYG9fbCj2+ySYXPYldkctlFlc9gHdvL3//71Ars0+7nkpi+usUGzOeq2PITUib26cbK51YfVutDxce4hsAvsE4MdmPtb8TgenE7VLJ/4UQ4rS+jmLz9E4tTBamLULcgcYC+wr3eo0/YbO/ZgEHZ12uzEjeMc+FvaoNBsjrrdfKzysXM3bpp2zlKzOYFdYJ8Y7DgNtvh8uOMEzFH4jsa2BW0L0A3Zjvfzu9fcy73mHtzLjcv5PDUnM85Ko2neaQM7OgBs/VWzgbdK/0nDdlxbs3Nsrs5WqavKJilbdZ8c41KzOYFdYG/FOKseoBsG9rpPWrJVWOccMDBW6G46cQL7+lVsU0imWG7awI7+A/VxwT7I5urAzoTdn0wTJ7DHa3MCu8C+BpIuMMc5x9BxDDgaWwWwonBXCG595Pmbf/1sWYZy7uE7Ebcc24H2W73FU1Zgj9fJdDFRmEawo/9VNuc+HOeCHxsZZHPUZTblhtgbdufGkVdgj9fmBHaBfY3BusY7zjkOg8MtM2jFjqOwCUBV6NZj50weWM0TWhwh9xHY43UyAnuYh+eAq797NcgWAHSVrVmcOwlw7Yt0P417+7B3y+R2jrwp/fGSwC6wr4FkU4NkleA/zFMHdoyESQDOAmdDOVbpBmzS/VU57cKJVQG87j5NZYm9XGpORmAPA3YeGEUXXH2tAzu2BYixOUKuXZsl/j1X7l5TF/Vih1UPphJfZYtuW3I6T83mBHaBfZ0xj2uQGLm/cqCOOuCaQ8JYbNZPHTgXKwPouba2UH/VPUh3HZTlzzlMzckI7O2Dnckweu/reR3YibdteFbglEOPOIg3gLsfiiLenXC796IOyrpxOZ+nZnMCu8A+sXG6ELZtvarQVgoGc3NO9g4uoTkSoG6Og7qsjDkP0snP1nwd8C1vbmFqTkZgbx/s2AMwRrc5r7I3126wEdsRY0Lg2pxNjEl362QCbmW4D+fUWb4r772ZkpuN+fKkZnMCu8A+Mdh9I+jiGieDsfnb/13cu+97pOZkBPb2wR5aB12gu/dC99ydNDct5/PUbE5gF9iTBHvOTmSYbKk5GYE9PbAP08FpS0/N5gR2gV1gT+wvO1NzMgK7wJ76RCA1mxPYBXaBXWDX/7EnpgOpgzK19gvsAUDZxYy/zXvwsQz9H7tWKaGcV2pOpk3bqqtrWj9QE0rHVO9a/5WazWnFHmAiYmB3nzzV+Yl/fFM/tNMPKX0sow7GbcYb2KVf7eiX+nF9P6ZkcwJ7ALA//vjjxZaFhWDH+z/wgWB1h2z3oLpzlGmQvJOmAbI2wZh6Xc8880xQm8hRPyXTeD56+/btydicwB4A7KGd5I4dO5JRsFH64ujRo8XS0lJWMh08eLBYXl7OSqZRxjLXPIuLi9mNZY5+RBPeE5+WFtgTAzuwYEvo2LFj2TgajHFubi4beYAbIMgRBrmCe5BcTDyxOXbiBuVLKS1HP8LiIDc/0lSnBPbEwA4scDJ79+7NxsnMzs6WMrHKbarIMZVj0sUYcQCFmNqmtoz/ZzkAg7HMaYWbsx/JaQLW1F4F9oTAbisHnEwuM1OM0CCYi+Nk0mUy5TQBa+pkUi9nE88NGzZksVPm+pH5+fksJp4sCszmcvEjk9iNwJ4Q2F1goMRHjhxJ3igxQjNIwhx+Yti0adOKTLlMwCZxMimXdYGBfubw3ETufiSXCdgkdiOwJwR2Vg62eiBMfWYKxDFCZDEYpu44mWwBgJmZmfLgPIcJ2CROJuWy2BhjaWO6sLCQ/GTa9yOpP7hqP33l5EcmtRmBPRGws32G4n7yk58sTj755OLTn/50sWXLlqSdDNvwOM0Pf/jDxZlnnllOVFKfrDAxYZze+973Fmxzcp76ZAUng/M8fPjwuh0VxtB/joCJDHkndU4xlGf8PvrRjxavfe1ri49//OPleKa8q2R+5BOf+ETpR/bs2ZONH7nyyivL8cGHpD5ZmVT3BfZEwG4DbVuDzz33XBaOE7nuuOOOckVkMuYQXn755cXWrVuzGSN7FoLQHR9Wsv4rRkw4iXfzpXz+9NNPl/I89thj2cj08MMPlzL99Kc/zUam2267LSu9m8RmBHaBvXfDFtjHf1J7EqNvUpaVHgD3V+fE+bBnh8KHfZN7xlJGYI9fP9EVgX11nAR2gV1gD6ADua3YY4FsH+0Q2FeB0Uf/j3pPgX11nAT2AE59VEVskk9b8avK26T/uiojsKcxTqPog8CexlgK7KvjJLAL7FqxB9ABgX3VyYwCz5jzCOxpjKXAvjpOAnsApx7SSWnFvqq8Ift50roF9jTGaZRxFtjTGEuBfXWcBHaBXSv2ADogsK86mVHgGXMegT2NsRTYV8dJYA/g1EM6Ka3YV5U3ZD9PWrfAnsY4jTLOAnsaYymwr46TwC6wa8UeQAcE9lUnMwo8Y84jsKcxlgL76jgJ7AGcOk6KL2+FOG6//fbyIwwPPvhgkPrrPn9qXx4LIRNf9OKDJiHqps46mXgnO9Q9L7744mLz5s3B6vffJ48ZjF21LdRYfuELXyj1k2+sh7pHXR+Fuh9fnMPmvvzlLweRqc7mQvqRj33sY0H9SEo2J7AHADsf58BoUj38D47gdOxrYqnKVOVo+E59qvLQ9joYTGO8/8cmqY1r1Qd95Efi86Gp2JbAHgjs/KkJgAxx3HvvvUHqpa04xDqw8w3mEPJ89atfLVgVhajbHH6VTMhKeoj7sqPywAMPBKnbZErFyXTRToNgiLGkznvuuSfIWFI3vqIO7KH8CJ/Hzc2P8PzR/v37g4xTajYnsAcCe6p/0DII7FXOpwunPck9cJx1MtXFT3K/LsqaTF3cK5V7GNhTaa/bTnxFlW0RJz+y+rux22ddn6dmcwK7wL5mS7cOdnXOp2sDG/d+ZpCEftk6Wf18sV2bTLG1q8/2COxxANB0oM62UvcjJl/socAusK8BXq4GKbDH5fjbdowCe1zjm6sfaVtvQ9UnsAvsAnsAHQhlsNSrFft6iAns6/skpA4Oq1tg73c8BPYATn3c38Z4BWSQoVSlE+e+fuGeD6prWFpbBjmsPVUyUcaNH1bHMFlINwhOumKnXW7bqu5d1V4/blgdVfX6cSaTHz/N1+OC3R8Xv+/qxtst18ZYct+67elx/YjbNl8erqva68s5rI6qeqvicvUjVbLGGCew9wz2paWl8sEZ/sPaVRDieUULJ75z587i0KFD5fum9toW8Ri+hQsLC2vqoT6rm3x2+Pdx78n5pAaJo+C+PEFvbaVe4okjpM3btm1beX/WnInJYzLNz8+vyEgd9j/fJguh9ZMvh11TV51MdfFW1kJru7Xf4mk39yed940ZA3vv2JWJ/LR9165dZR6Tk3hXFjvnPnaPqtBkqkqb1jj6jvEcRX7Ggj7mSWc3P9foLAf17du3b2U8LR/xjC2h6SfnpFOuic21AXbuy+HbN3G0y2yuyo+QxoEco/iRqvtY/1hYZ1t1slo5C12bo/1u/OLiYm9+xNoReyiw9wR2nAPOxZTWoIVCozSEvL7BgSKTjvGRn7LAHiMkrxmz66gwUtcxGXDMCdUp5iQGSftc+HEvt03IZA6ENJMJeTjMqdA2K+e2l3OTw0JkpM46eUirk6ku3q2L/jeZzNkQZ3kYDw5zdq5M3BuZiLPxJDTZqINyJouFjK3VXxWaTFVp0xqHbjCeg+Sn722cyEc/MrbonpVjbBlP1+bIRzr3YDzJb2NqodXXxObqYMf9SLO2VYW0ZVI/YvpmemmhyezLxDVtq2qPxdXZVp2sVo7QxoXx4pp7uW0yWyKeo8rmrH1WzkKrz2zNwlH9iNvOmM8F9h7AjjGasppymoIaRFA48uGImGVjfBaHQqH8lCFk9ctqkBUESm7KizKT7h7UOUghmxok9+B+rjx2TjzGSPsJcZqcm0yuAVMPMiCLrXBdmVxZ7NzKV8lFnjqZ6uKtHu5bJRNxpNGXJhMOq04mGw+gwFghN+0invE1OdzQ2lAVmkxVadMah64xnnXyoyP0tT+eXBNPOv0K1InD0Zt+Emf1ms2RxuFORilPWXccOR9mc3Ww416DwE695KH9hHagm74f4brOj1h7kSUlPzLI5pCJfjCZCLlmHOknk9kNR/EjpgexhwJ7D2D3lcI3XnMyKCDGi/JhmITmJFBSPl6B08Fg3Vm1KS/1mLFbaOX9Nth1HezqnI+V80Pu58bRFnN6tNmVyQwKBwX0SUOeAwcOlLA0x0qaOTOTh3CQTNRVJ1NdvNtuO6ceDrsm5L4mk6XbOJlMXDNWpNNW4M9hMhFnaa5M7n38c/IPgpiffxqu6btx+oT8fr/Qr+gg48q5jaWNFfFMOEljDIGgjSfjTTwH4+2OJXH+vdzrOtuiDt83uOX8cz+vtYl6fJszm6Gt2BXp2FtMfoR+88cJmczmfJnM5hhDDsq37Uf8Po/1WmDvCewoJbNIDj4PaueuophhY7Czs7OlkXOOUaK4pBvgLa8ZAqEZBobLNQf53Xv453Ww475Wt1/Grqnb5JibmyvPaael0x6MkrpcmcwIibM229e4iLM2W/sJcbqEFmf38EPqq5OpLt6tg7YhEw6dg3NrD/nqZCKP/YxCPuqxcUA2q8PkIJ3DZKJetx3uucnkxk37Of02CtjRJ8bQ9JNzbNH6j3oYB1c/OTegkE7/E8c5Y8oYYpPoNgfxNq6cc1j9VaHV5adRjjQ/3r1uw48gD/poNmch96ENpBOa/nJu+uu2xT2vs606Wd2y1G02Z+PEmFge2lPnR7A57mFtNlmIszZb+6nTHSdLt/u4IfWNol9umT7PBfaewO4OepXxGrwtH8po54SmuJyTxoGSWh47R1nJSzyKbLNay+eHkxikW5ffXtIwOtd4/Dxc01Zk59ycpNVrxk0aeZDF4iyPH5pBWh+46XWyunnsnPJVddBG5LJ8VTKRRlnSWOVZfoMB6TY2OGq3j6xeN6SulJyM2/ZQ5/TtOH3ijxPtQp/4qcTaWJXH4gg5zM4oY7rIGKIXxBEOs7k62FF/lW+w9vlhVd5R/IjpNfdDBpOD+k0+5LB8brrfBruus606Wa2cG5rNuHGcYz9mQ1zTbjcP15R1/YiNB/ms/eQb14+494n5XGDvCewoOLNSDnfFbkqHQZlio4D8nkfIgaMgHQCg4JQhHuU1R0IaDsadZVv5QQrZ1CDdlQMy2Uybc9piDoZza6/JZOnkJR8ykMfyEtIX3AN5kd1kIZ9rtL5slKuTqS7e6nBlclfstNPawr1piy8T9+Wg/6mPPLSb/O7YMpaWRuimWzv80GTy46f5mr5jPAf1AWPE2HG4+kk85QjpW9Mv00/qtnHinLEntDF3y1OPrRLJw0F9g9pVBzvKVsHarWsUP4LNWFt8mf9FsUsAAA1ESURBVEwXY/Qj2Jw7TuYn6G/zDchlMlm660eQz/IiIwfjZ3HWL1an27fueWo2J7D3BHZXaSY591cD/vW4ddfBrs75jFP/qG3z8/nX49zTDJLQL1cnq5+v7nqcdo2Tt+5+Fm8y2bXCExOnYWBvq5+YfLZVF/XU2dYoYG+rHb5++tf+fYal19lWnax+/YOuh93byvr5/GvLN0qYms0J7ImDfRSlHCdPSIMcpx1t5TWDDAH2tto4bj0m07jlcs4PBLsCe9v9WAe7LsHetky5+pG2+ylUfQK7wL5m9ZGrQQrs/X7iMpQDs3oF9rjGN1c/YvoWeyiwC+wCewAdCGn4WrGvh5jAvr5PQurgsLoF9n7HQ2AP4NS1hdavUrtOxyCoFXs8Y+KOT1vnAntc4yuw9zseArvArhV7AB1oC1hV9dhkpSptWuME9n5B4uudwN7veAjsAZw6TsZ9hc1esUkhHGSQ7qsnKchCG3llpk4m4u2jM6nI48rkO9NpvjawpzSO1lZ8Be33x09+5MSridZPfYbmR/wxivVaYA8AdlZUGGWow96fDVV/1es8vPcZ6n7Ue+ONNwatv8oA7R30UHKFlIm2V8k0rXHobKhx7EI/ebfaH7vQfiSkftJn0+JH/HGL4VpgDwD20AM77IMXoe/fdv04APvIR9t191UfEyGOvu6v+7a7FYrNVYEq1X7mne7c/EiOMjXVL4E9MbADC7aQc3IyQJ3tyKZKHGM5vjjH1mGMbVObxof+zMxMVpNPdnzwI5N8tCU2PcpRpqZ9LLAnBnY+ZoFB5rQVyx/cIBOfe2yqyDGVY9KFPBw5TcBi6uMu24JeMpY5TT7N5nLaVeJTvoxTTr6xqZ4L7AmB3QUGD7I1HfSYypnTxCDtu+oxta9JW2zlgEz81tikDpUZf1Udqs/YsmYsOXIAIb/nmzy57Cq5vpEH3ULpQir1CuwJgR1ImEESVj1wk4riWTtdp4lMOWwN2moIeXKZgNl4TVuIPrJSN7vLYfLp21wOu0r8nGdjRJiDb5zE1gT2hMAOMGy7iTD1h19wmhihycXvmKlvo9lqCFk4kK/q4ziTGK3Kdreat2dasDezvZRBaBMV003CHHaVzIeYP0ndN05q4wJ7ImDHIHEsu3fvLmHxqU99KnmwA0FkuuKKK4o3velNpTypOxlAwHMQGzduLM4777zyPIft20kdTarl0UdW6Zs3by6uueaacjxTnqgxKQGC/NXpW9/61vKBwNTfSME3ItMtt9xSysTiQGBPBGypOoa2222/ST/33HPZ/I50xx13lJOVtvuqz/ouv/zyYuvWrdmMUZ99GcO9+S2aSXUMbWmjDUxYXvOa12QjD33ChIsV+x//+Mes5Goy3lqxJzaxEdi724ZtYlBWRmBPY5xsvIaFgP2qq67KBhiA/Q1veEM28jB+Bvbf//73Wck1TDer0gV2gb13I9CKPS8IVjma1OMA+7Zt23q3lbb6EbDz+3pb9cVQj4H95ZdfzkquJn0rsAvsvRuBwC6wN3FeXZYB7Bxd3jPkvQA7v7GHvEfXdRvYjx8/npVcTfpRYBfYezcCgV1gb+K8uizzrne9q3x4rst7hrwXYD/jjDN6t/02ZTSw/+Y3v8lKriZ9JLAL7L0bgcAusDdxXl2VefHFF4vXv/71xf79+3u3lbZkBuxnnXVWNvLQLwb2X/7yl1nJ1WTMBXaBvXcjENgF9ibOq6sy1157bfnq4u9+97vebaUtmQH72WefnY089IuBnYlYW/2Uaj0Cu8DeuxEI7AJ7jA70Jz/5SXHppZeWr1B985vf7N1O2uwjwM63Ftqss++6DOw/+9nPspKrSb8K7ImC/frrrw/6/9MYftvHr371q0qDGwb2ttvRZn11Rjfsdbc226C62tdVnoDnU7Knn3568cADD1Tqbd3YpxCPzlxwwQVZyWVgT/nLgG3pjsCeKNj5EpY9qZtK+Mwzz1Q6kmFg3759e7Sy/va3v62UaRDYf/3rX0crTyq6FLqdu3btKu6///4i1wexAPtFF11UqbttwaXregzszz77bFZyNelHgT1RsOvLc3FvXw8CexNDVZm4xzu18QHsH/zgB7MCoIH9xz/+cVZyNdEtgV1g790Ihq3Ymyh232UEdoG4bx0cdP+rr766/I+GQXlSSzOw//CHP+zdp/XddwK7wN67EQjsgmDfjnDa7s9DgTfccEPvtt9mvxvYv//972clV5M+EtgF9t6NQGAX2Js4L5Vppjdf+cpXilNPPbWoe+Yl1X41sH/ve9/r3af13YcCu8DeuxEI7M0cdN/OQ/dPb9yuu+668vW93KCOLhrYv/Od7/Tu0/q2DYFdYO/dCHgCmb9bDP2kc4j69VT8iW+oh+hb1dle35533nnFaaedVrzlLW8pAdg3eELc38D+rW99q3efFkK+ceoU2AX23o3g5ptvLsGe4vvYdcY27OG5FGVVm9t/X76rPr3nnnuKhx9+uMj592cD+xNPPNG7T6vzC13FC+wCe+9GoK349LZ0u3JQuo90Y1QdeOSRR8oFwne/+93efdqobQ6VT2AX2Hs3AoFdzjuUg1O906Nbn/vc54qTTz65+MMf/tC7T+tb7wR2gb13IxDYp8f59u3wdP98de2aa64p3ve+9/Xuz2LQMYFdYO/dEAT2fJ1tDE5Obchfv77xjW+U2/B6cO7EWAvsArvAHkAHhj08J9jkDxuNcdgxfuWVV4qnn3664Ct6vFXDg4jqc4E9SSU4ePBgqcT6VnxYpzGpgzj33HPLccLh6FAfSAfC6cDb3/728g97JrXZnMprxR5gtRZSQQT2uIFuY7+4uFjwD3y8gqNDfSAdaF8Hvv3tbxcvvPCCHparYJjAXtEp5pxjDAX2NMCurfg0xilGG1ebpDuT6oDALrD3/pOEHp6TI5vUkam8dEg6sKoDArvALrAH0AGt2FedjByu+kI60K0OCOwBnHpIJX7ssceKd7zjHcWLL77YO5DbknP//v3FZZddlo089MuNN95Y7N69OyuZ2hpv1dOtk1d/T19/C+yJgV1GOn1GqjHXmEsHpAPj6IDALrBrVSkdkA5IB6QDGemAwJ7RYI4zo1NerQCkA9IB6UCeOiCwC+yaqUsHpAPSAelARjogsGc0mJp95zn71rhqXKUD0oFxdEBgjxjsS0tLtbPoY8eOFXysxh1svnbmXsd6fuTIkWLv3r2VbeULXUePHq1Mi1Ueaxdj4svFNfJaHoVy0NIB6UBoHRDYIwE7QONPDNxjbm5uzTVpLvR27NhR7Ny5s4QG8NiwYcO6/NQbWonGrR85aPfhw4fLyQnXy8vLBRMZZJ6fny8WFhaia/cwOU0GQhtH5EFWu3bHb1h9ShcApAPSgSY6ILBHAnYcvn1PGsAB7U2bNpUHqz5LY1XoDjR5Kbtly5YSjAYQQuII3fwxnLOzYO1ELkCIvLSNeOJoewxtHacNjBdjwSTLxos4Oyf0x2+c+pVXTl46IB0YRQcE9kjAzmCxtQ70bDsXuAECA33dlq5BnBWwHZQzeI6iCF3lAXzAzr0fUEc25J2dnS3TZ2ZmSuC7+WI+t2/400ZkYceBg3/1svO68YtZLrVNIJEOpKcDAnskYAd2AO7QoUMrcLbtamC9b9++Mt1WtkCQMsAbYBCSRggcWR3GCnbaDrgJASLb1cjAgczIynZ8Kg6FSZRNSGgzfY9sbvsZC+LdOJ2n5zA1ZhqzFHRAYI8E7CgLjh9AGJDdkHhfoQzexLvgACqWRrxfLoZrAx8reNrINWBncmI/PcTQzlHawE8JTFBMJvreJi7Ecdhka5T6lEfwkA5IBybRAYE9MrCzUrXtdDesWsEavFEA4GgPnfEQnaXFCHZARxvZoqZ9rHiJ8x+eA/STKHfXZZGBe9L3/jiyE0F8123S/QQI6cD06YDAHhnYp23Fbk+SA3UcEKBPFYAu2P1xJC1VuQSG6QODxjztMRfYIwN71StuwI5439gABWlsXRPaa1XkddP8cjFcGwRZrbNtzVY2MhBvzwqkBkKTiXb748juRGryxKAnakPagNH49TN+AntEYAdyOP+6wzcSYMiWNqtefqt2y1EXgIwVJkDQfpvmIUDeBkAG4lLbgrdxccFu55bGpCXWsbA2KuzHCavf1e9t64DAHgnY2Yq216IGhYDPlIDzKlgQz+qd7WAAb/ljCgE57QR4HPbwGZA3+YmPqc3D2gLMmZTw+7o9Q2CysIIn3h2/YfUpXQ5fOiAdaKIDAnskYG8yeHVlbPVelx5TfKwTjyZ9lOpOQxNZVUbAkQ7EqwMCe4Zgl8HFa3AaG42NdEA6EFoHBHaBPant7tAGofrldKUD0oHUdUBgF9gFdumAdEA6IB3ISAcE9owGM/VZptqvlZJ0QDogHZhcBwR2gV0zdemAdEA6IB3ISAcE9owGUzPdyWe66kP1oXRAOpC6DgjsArtm6tIB6YB0QDqQkQ4I7BkNZuqzTLVfKyXpgHRAOjC5DgjsArtm6tIB6YB0QDqQkQ4I7BkNpma6k8901YfqQ+mAdCB1Hfh/2TcxF3ZWilUAAAAASUVORK5CYII="
    }
   },
   "cell_type": "markdown",
   "id": "ba1137d5",
   "metadata": {},
   "source": [
    "多头注意力：多个头连结然后线性变换\n",
    "![image.png](attachment:image.png)\n",
    "\n",
    "模型计算方式：\n",
    "![image-2.png](attachment:image-2.png)\n",
    "\n",
    "![image-3.png](attachment:image-3.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "405a8a85",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import torch\n",
    "from torch import nn\n",
    "import dltools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e4a40cda",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 具体实现\n",
    "class MultiHeadAttention(nn.Module):\n",
    "    \"\"\"多头注意力\"\"\"\n",
    "    def __init__(self, key_size, query_size, value_size, num_hiddens,num_heads, dropout, bias=False, **kwargs):\n",
    "        super(MultiHeadAttention, self).__init__(**kwargs)\n",
    "        self.num_heads = num_heads\n",
    "        self.attention = dltools.DotProductAttention(dropout)\n",
    "        self.W_q = nn.Linear(query_size, num_hiddens, bias=bias)\n",
    "        self.W_k = nn.Linear(key_size, num_hiddens, bias=bias)\n",
    "        self.W_v = nn.Linear(value_size, num_hiddens, bias=bias)\n",
    "        self.W_o = nn.Linear(num_hiddens, num_hiddens, bias=bias)\n",
    "\n",
    "    def forward(self, queries, keys, values, valid_lens):\n",
    "        # queries，keys，values的形状:(batch_size，查询或者“键－值”对的个数，num_hiddens)\n",
    "        # valid_lens　的形状:(batch_size，)或(batch_size，查询的个数)\n",
    "        # 经过变换后，输出的queries，keys，values　的形状:(batch_size*num_heads，查询或者“键－值”对的个数，num_hiddens/num_heads)\n",
    "        print(queries.shape) # 2 4 100\n",
    "        print(keys.shape) # 2 6 100\n",
    "        print(values.shape) # 2 6 100\n",
    "        queries = transpose_qkv(self.W_q(queries), self.num_heads) # 10 4 20\n",
    "        keys = transpose_qkv(self.W_k(keys), self.num_heads) # 10 6 20\n",
    "        values = transpose_qkv(self.W_v(values), self.num_heads) # 10 6 20\n",
    "\n",
    "        if valid_lens is not None:\n",
    "            # 在轴0，将第一项（标量或者矢量）复制num_heads次，然后如此复制第二项，然后诸如此类。\n",
    "            print(\"valid_lens前的形状{}\".format(valid_lens.shape))\n",
    "            valid_lens = torch.repeat_interleave(valid_lens, repeats=self.num_heads, dim=0)\n",
    "            print(\"valid_lens后的形状{}\".format(valid_lens.shape))\n",
    "\n",
    "        # output的形状:(batch_size*num_heads，查询的个数，num_hiddens/num_heads)\n",
    "        output = self.attention(queries, keys, values, valid_lens)\n",
    "        print(\"output后的形状{}\".format(output.shape))\n",
    "        \n",
    "        # output_concat的形状:(batch_size，查询的个数，num_hiddens)\n",
    "        output_concat = transpose_output(output, self.num_heads)\n",
    "        print(\"output_concat后的形状{}\".format(output_concat.shape))\n",
    "        print(\"output_W_o后的形状{}\".format(self.W_o(output_concat).shape))\n",
    "        return self.W_o(output_concat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bc0bc8c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transpose_output函数反转了transpose_qkv函数的操作\n",
    "def transpose_qkv(X, num_heads):\n",
    "    \"\"\"为了多头注意力的并行计算而变换形状\"\"\"\n",
    "    # 输入X的形状:(batch_size，查询或者“键－值”对的个数，num_hiddens)\n",
    "    # 输出X的形状:(batch_size，查询或者“键－值”对的个数，num_heads，num_hiddens/num_heads)\n",
    "    X = X.reshape(X.shape[0], X.shape[1], num_heads, -1)\n",
    "    print(\"transpose_qkv中输出X的形状{}\".format(X.shape))\n",
    "\n",
    "    # 输出X的形状:(batch_size，num_heads，查询或者“键－值”对的个数,num_hiddens/num_heads)\n",
    "    X = X.permute(0, 2, 1, 3)\n",
    "    \n",
    "    # 最终输出的形状:(batch_size * num_heads,查询或者“键－值”对的个数,num_hiddens/num_heads)\n",
    "    print(X.reshape(-1, X.shape[2], X.shape[3]).shape)\n",
    "    return X.reshape(-1, X.shape[2], X.shape[3])\n",
    "\n",
    "def transpose_output(X, num_heads):\n",
    "    \"\"\"逆转transpose_qkv函数的操作\"\"\"\n",
    "    X = X.reshape(-1, num_heads, X.shape[1], X.shape[2])\n",
    "    X = X.permute(0, 2, 1, 3)\n",
    "    return X.reshape(X.shape[0], X.shape[1], -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "91801176",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultiHeadAttention(\n",
       "  (attention): DotProductAttention(\n",
       "    (dropout): Dropout(p=0.5, inplace=False)\n",
       "  )\n",
       "  (W_q): Linear(in_features=100, out_features=100, bias=False)\n",
       "  (W_k): Linear(in_features=100, out_features=100, bias=False)\n",
       "  (W_v): Linear(in_features=100, out_features=100, bias=False)\n",
       "  (W_o): Linear(in_features=100, out_features=100, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_hiddens, num_heads = 100, 5\n",
    "attention = MultiHeadAttention(num_hiddens, num_hiddens, num_hiddens,num_hiddens, num_heads, 0.5)\n",
    "attention.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0919fe76",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 4, 100])\n",
      "torch.Size([2, 6, 100])\n",
      "torch.Size([2, 6, 100])\n",
      "transpose_qkv中输出X的形状torch.Size([2, 4, 5, 20])\n",
      "torch.Size([10, 4, 20])\n",
      "transpose_qkv中输出X的形状torch.Size([2, 6, 5, 20])\n",
      "torch.Size([10, 6, 20])\n",
      "transpose_qkv中输出X的形状torch.Size([2, 6, 5, 20])\n",
      "torch.Size([10, 6, 20])\n",
      "valid_lens前的形状torch.Size([2])\n",
      "valid_lens后的形状torch.Size([10])\n",
      "output后的形状torch.Size([10, 4, 20])\n",
      "output_concat后的形状torch.Size([2, 4, 100])\n",
      "output_W_o后的形状torch.Size([2, 4, 100])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 4, 100])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size, num_queries = 2, 4\n",
    "num_kvpairs, valid_lens =  6, torch.tensor([3, 2])\n",
    "X = torch.ones((batch_size, num_queries, num_hiddens)) # 2 4 100\n",
    "Y = torch.ones((batch_size, num_kvpairs, num_hiddens))# 2 6 100\n",
    "attention(X, Y, Y, valid_lens).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "711b3355",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bd4e457",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}