13_loading_and_prprocessing_data_3_TFDS+Excerises.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF Transform\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.config.experimental.set_memory_growth(tf.config.experimental.list_physical_devices('GPU')[0], True)\n",
    "\n",
    "import numpy as np\n",
    "import sklearn\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    import tensorflow_transform as tft\n",
    "\n",
    "    def preprocess(inputs):  # inputs is a batch of input features\n",
    "        median_age = inputs[\"housing_median_age\"]\n",
    "        ocean_proximity = inputs[\"ocean_proximity\"]\n",
    "        standardized_age = tft.scale_to_z_score(median_age - tft.mean(median_age))\n",
    "        ocean_proximity_id = tft.compute_and_apply_vocabulary(ocean_proximity)\n",
    "        return {\n",
    "            \"standardized_median_age\": standardized_age,\n",
    "            \"ocean_proximity_id\": ocean_proximity_id\n",
    "        }\n",
    "except ImportError:\n",
    "    print(\"TF Transform is not installed. Try running: pip3 install -U tensorflow-transform\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "\n",
    "datasets = tfds.load(name=\"mnist\")\n",
    "mnist_train, mnist_test = datasets[\"train\"], datasets[\"test\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['abstract_reasoning', 'accentdb', 'aeslc', 'aflw2k3d', 'ag_news_subset', 'ai2_arc', 'ai2_arc_with_ir', 'amazon_us_reviews', 'anli', 'arc', 'bair_robot_pushing_small', 'bccd', 'beans', 'big_patent', 'bigearthnet', 'billsum', 'binarized_mnist', 'binary_alpha_digits', 'blimp', 'bool_q', 'c4', 'caltech101', 'caltech_birds2010', 'caltech_birds2011', 'cars196', 'cassava', 'cats_vs_dogs', 'celeb_a', 'celeb_a_hq', 'cfq', 'chexpert', 'cifar10', 'cifar100', 'cifar10_1', 'cifar10_corrupted', 'citrus_leaves', 'cityscapes', 'civil_comments', 'clevr', 'clic', 'clinc_oos', 'cmaterdb', 'cnn_dailymail', 'coco', 'coco_captions', 'coil100', 'colorectal_histology', 'colorectal_histology_large', 'common_voice', 'coqa', 'cos_e', 'cosmos_qa', 'covid19sum', 'crema_d', 'curated_breast_imaging_ddsm', 'cycle_gan', 'deep_weeds', 'definite_pronoun_resolution', 'dementiabank', 'diabetic_retinopathy_detection', 'div2k', 'dmlab', 'downsampled_imagenet', 'dsprites', 'dtd', 'duke_ultrasound', 'e2e_cleaned', 'emnist', 'eraser_multi_rc', 'esnli', 'eurosat', 'fashion_mnist', 'flic', 'flores', 'food101', 'forest_fires', 'fuss', 'gap', 'geirhos_conflict_stimuli', 'genomics_ood', 'german_credit_numeric', 'gigaword', 'glue', 'goemotions', 'gpt3', 'groove', 'gtzan', 'gtzan_music_speech', 'hellaswag', 'higgs', 'horses_or_humans', 'i_naturalist2017', 'imagenet2012', 'imagenet2012_corrupted', 'imagenet2012_real', 'imagenet2012_subset', 'imagenet_a', 'imagenet_r', 'imagenet_resized', 'imagenet_v2', 'imagenette', 'imagewang', 'imdb_reviews', 'irc_disentanglement', 'iris', 'kitti', 'kmnist', 'lambada', 'lfw', 'librispeech', 'librispeech_lm', 'libritts', 'ljspeech', 'lm1b', 'lost_and_found', 'lsun', 'malaria', 'math_dataset', 'mctaco', 'mlqa', 'mnist', 'mnist_corrupted', 'movie_lens', 'movie_rationales', 'movielens', 'moving_mnist', 'multi_news', 'multi_nli', 'multi_nli_mismatch', 'natural_questions', 'natural_questions_open', 'newsroom', 'nsynth', 'nyu_depth_v2', 'omniglot', 'open_images_challenge2019_detection', 'open_images_v4', 'openbookqa', 'opinion_abstracts', 'opinosis', 'opus', 'oxford_flowers102', 'oxford_iiit_pet', 'para_crawl', 'patch_camelyon', 'paws_wiki', 'paws_x_wiki', 'pet_finder', 'pg19', 'piqa', 'places365_small', 'plant_leaves', 'plant_village', 'plantae_k', 'qa4mre', 'qasc', 'quac', 'quickdraw_bitmap', 'radon', 'reddit', 'reddit_disentanglement', 'reddit_tifu', 'resisc45', 'robonet', 'rock_paper_scissors', 'rock_you', 'salient_span_wikipedia', 'samsum', 'savee', 'scan', 'scene_parse150', 'scicite', 'scientific_papers', 'sentiment140', 'shapes3d', 'smallnorb', 'snli', 'so2sat', 'speech_commands', 'spoken_digit', 'squad', 'stanford_dogs', 'stanford_online_products', 'starcraft_video', 'stl10', 'story_cloze', 'sun397', 'super_glue', 'svhn_cropped', 'ted_hrlr_translate', 'ted_multi_translate', 'tedlium', 'tf_flowers', 'the300w_lp', 'tiny_shakespeare', 'titanic', 'trec', 'trivia_qa', 'tydi_qa', 'uc_merced', 'ucf101', 'vctk', 'vgg_face2', 'visual_domain_decathlon', 'voc', 'voxceleb', 'voxforge', 'waymo_open_dataset', 'web_nlg', 'web_questions', 'wider_face', 'wiki40b', 'wiki_bio', 'wikihow', 'wikipedia', 'wikipedia_toxicity_subtypes', 'wine_quality', 'winogrande', 'wmt14_translate', 'wmt15_translate', 'wmt16_translate', 'wmt17_translate', 'wmt18_translate', 'wmt19_translate', 'wmt_t2t_translate', 'wmt_translate', 'wordnet', 'xnli', 'xquad', 'xsum', 'xtreme_xnli', 'yelp_polarity_reviews', 'yes_no']\n"
     ]
    }
   ],
   "source": [
    "print(tfds.list_builders())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x216 with 5 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAABYCAYAAABWMiSwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAo0UlEQVR4nO2dWXBc132nv4vuRm/oHY1GY98JgKu4SJRp0ZSoiu2RK47HdkWesZNK5SkpP2Rq5mEeMg/JzMs8pCZTE8+4UpnVLrvKVtmKY7soKopMyqYkCgJBmcSOxo5e0QuARu995wE5RwBJUaQEoBvg/apYkgAIPPf0vb97/ruiqioaGhoaGvtDTaUXoKGhofEkoYmuhoaGxj6iia6GhobGPqKJroaGhsY+oomuhoaGxj6iia6GhobGPqKJroaGhsY+UnWiqyhKr6IoWUVRvl/ptVQaRVG+rSjKkKIoOUVR/k+l11MtKIriVhTlp4qipBVFmVcU5V9Vek2VRFGUjXv+lBRF+W+VXlelURSlQ1GUXyqKklAUJaQoyt8oiqKv9LqqTnSB7wDvVXoRVcIK8J+A/1XphVQZ3wHygA/418D/UBTlaGWXVDlUVa0Tf9jakwzw4wovqxr470AE8AOngM8Bf1rJBUGVia6iKC8DSeCNCi+lKlBV9Seqqr4KrFZ6LdWCoihW4KvAf1BVdUNV1V8DPwO+VdmVVQ1fY0to3qr0QqqATuBHqqpmVVUNAVeAir+cq0Z0FUWxA38J/NtKr0WjqukDSqqqTm772m2q4GGqEv4Q+H+qVt8P8F+BlxVFsSiK0gx8kS3hrShVI7rAfwT+p6qqi5VeiEZVUwek7vlaCrBVYC1VhaIobWyZ0P+30mupEq6x9TJeA5aAIeDVSi4IqkR0FUU5BbwI/JcKL0Wj+tkA7Pd8zQ6sV2At1cYfAL9WVXW20gupNIqi1ACvAT8BrEA94AL+cyXXBVUiusAloANYUBQlBPw74KuKogxXclEaVckkoFcUpXfb104Cdyu0nmriD9BOuQI30Ar8jaqqOVVVV4H/DfyLyi6rekT3b4FutiKMp4DvAr8APl+5JVUeRVH0iqKYAB2gUxTFVA0pL5VEVdU0W6eXv1QUxaooygXgy8D3KruyyqIoymeAZrSsBQBUVY0Bs8Cf/PNz5GTL3327ogujSkRXVdVNVVVD4g9bJmRWVdVopddWYf6crfSffw9885///c8ruqLq4E8BM1tR+h8Cf6Kq6pN+0v1D4Ceqqmpulg/5l8AXgCgwDRSBf1PRFQGKFuTU0NDQ2D+q4qSroaGh8aSgia6GhobGPqKJroaGhsY+oomuhoaGxj6iia6GhobGPvJxOZ9PSmqD8hg/q+3Jg9H25X60PbmfJ35PtJOuhoaGxj6iia6GhobGPqKJroaGhsY+oomuhoaGxj7yRDdP0TgcqKpKuVwmn89TLpcplUqoqoper6empgaDwYBer93qGtWBdidqHGhKpRLxeJxUKsWVK1dYWlpibm6OdDrN0aNH8fv9XLp0icHBQXQ6HTU1mnGnUVk00dU40JTLZdLpNPF4nA8++IDx8XHu3r3L2toa6+vrdHZ2MjAwQFdXF0ajkdra2kovWaPKEJZRqVSiXC4DoCgKer1+TywkTXQ1DjTFYpHFxUUWFxcZHx9nYmKCzc1NSqUS4+PjLC8vYzabCQQCnD9/nuPHj1NTU4OiPG4assZhI5/PUygU+O1vf0swGOS9994jEAhgNpsxGo184Qtf4Hd+53cwGAwYDIZd+3sPjOiKFpTlchlVVVEU5Yl/eMReCJ60/VBVlWKxyOrqKuFwmHA4TDT6YQvmSCRCNBrlzp075HI52tvb6e/v37MTjEb1oKoq97at3f7fqqqSz+fJZrPMz88zOTnJa6+9xvDwMHa7HYvFQnt7O88//7yMC+wWB+LOK5VKBAIB4vE4b7zxBrOzszz77LP09vbS1dVFc3NzpZe4rxSLRXK5HG+99RYrKysUi0UALl68yJEjR54I4c3lciwsLBAMBvnZz37G7OzsDsEVqKrK1NQU0WiUhoYGFEWht7eX7u7uCqxaY6/I5/Osr69TLBYpFAqk02lmZmbI5XKUSiVKpRKpVIp8Ps/a2hqlUgmHw0FNTQ1vvfUWgUCAxcWtmbjZbJZCocDm5iaFQmFXBRcOiOiWy2VCoRDz8/P8wz/8A0NDQxQKBcrlMi6X64kT3VKpRDab5YMPPmB0dJR8Pg9AT08PfX19AIdeeIvFIsFgkLm5OYaHhwkEAmSz2Qf+bDgcJhKJMDY2Rn19PU6n88CJ7oOGDRz2z/hRUVWVQqHA2toahUKBTCZDPB5nZGSEzc1N6UYIh8Nsbm4SDofJ5/O0t7djtVq5efMm8/Pz8veJ5ymfz+/w8+4WB0J0S6US09PTjI6Okkwmga0N2dzclKe8J4VMJsO1a9dYWFjg+vXrBAIByuUyNTU1zMzM0NfXh9PppK6urtJL3RNEwCOZTMrrj8Vi8gF52P83MTFBNpvF5/Nx/vz5fVz1p+POnTu89dZbpFIpIpEIbrebY8eO4fP5OHPmzBMVHBTpgdlslnQ6zcLCAkNDQySTSZaXl8nn82QyGTKZDKFQSB7OisUimUyGYrHIxsYGqqqyvr6OTqcjkUjs6zUcCNEtl8ssLS0xOTnJxsYGAIVCgWw2+8SJbj6fZ2hoiDt37jA8PEwwGATAYDCwvLxMLBbDaDQeatEtFAqsr68zPDzMzMwMyWRSnk4exvz8PKFQiMuXL+/DSneP2dlZXnnlFVZWVpiYmKCzs5Pf/d3fZWBggBMnTjxRortdQBOJBGNjY/zkJz8hGo0yNzcn3QLCMniQNSC+Fw6HK2ItVLXoqqpKOp0mlUoxNzfHzMwMer2epqYment7OX78OPX19ZVe5r5QLpdJJpNEo1HGx8e5c+eOfAEZDAaMRiNmsxmLxXKog0SJRII33nhD3g/hcJhCoSC/rygKTqeT2tpaUqnUDpeDCK4clLmAmUyGdDpNJBIhGAxKKy+ZTPLee+8Rj8fx+/24XC7a2tqk71Gn02Gz2eR9UC6XiUQibG5ukk6nyefzWCwWjEYj9fX1uFwuFEWpandFNpslk8kwPj7OrVu3iMfjhMNhlpeXmZmZYXNzk2w2S21tLS0tLej1eux2OyaTiaamJoxGIwaDgVwux7Vr1wiHww/8exRFkamFJpNJFtjsJlX9dArRTSQSLCwsMDMzQ1tbG263myNHjnD8+HGcTmell7kvCJM6HA4zNjbG3bsfDr/V6/XU1tZK0d1tx381EY/H+fnPf87s7CwzMzOk02ngwxNNTU0Nbreburo6stnsfX7eezM+qhVVVclkMqyurhKNRgmFQvJakskkQ0NDhMNhPB4PTU1NAJjNZhRFkRV4RqMR+DCtLhaLSfF1u93Y7XZqamqw2+3odLqqFl1xsr158ybf//73iUajLCws3PdzQmStVit+vx+n08nZs2ex2WxYrVbW1tYYGxv7SNEVv8NqtWI2m5880S0UCkxOTjI/P08ymURVVcxmMw6HA7vdjs1mO9QCs51CocD8/Dxzc3NkMhn5dUVRMJvNcj8O654I/524H0Kh0A7XkqIoWK1WrFYrly9fpq2tjTfeeINAIMDq6qr045XLZTKZDKlUCqPRiMlkquBVPRhxIh0aGuL69ev89re/JZvNSp+1uI5kMsnw8DDT09PMz8/vOOk6HI4dJ91QKMTGxoY86VqtVmpraxkbG6OtrY2BgQEGBgbQ6XRVZSmJasP333+f4eFhRkdHCYfDZDIZedBwu9243W66u7txuVwcOXIEs9mMzWbDbDbT0tJCTU2NfOGI+8ZgMOwQ1EKhgKqqOJ1O/H4/brcbo9G46/tRPbv7AAqFArdv32ZiYoJYLEa5XMZiseB2u3E6nTgcjkovcd/I5/NMTk4yPT3N5ubmju/V1dXhdDoP9Z6sr6/z/vvvMzU1xeTkJIlEYocfV6fTYbfb8Xq9fOUrX+Hs2bPk83lqampk8ET0ZUin08RiMZxOZ1WK7vr6OpFIhH/6p3/ir//6rx8YIBT5ydevXwceLZPh3hO+oih0d3fT3NzMyy+/THt7uzSpq4VwOMzs7CyvvvoqP/jBD+Q11NbWSvdIf38/vb29fOlLX8Lr9cpc7O17srGxQTAYlPcBcF++drlcplAo4PV66evrw+fz7cn9sS+7u7m5STKZJBaLMTMzg9PppL+/X55aP+qGKZfLrK+vs7q6Kt9Odrud+vr6qnxY9pJCocD09DRjY2M7TGqDwUBHRwddXV14PJ4Kr3L3SafThEIh5ubmuHHjBsvLy2QyGVm6qSgKdXV1WK1WnnvuOdrb2/H7/fKhNJlM6HQ64MMTYiqVIhgMUlNTU1V7ls/nyefzjI2NMTQ0xMTExA7BFZ+3+HchEsJPLZ6jB4mrQJjL23NXFUVhYmKC9957j46ODpl2WEmEj/b27du88847zMzMoKqqtGba29s5duwYHo+H9vZ2fD4fzc3N2Gy2B7pKdDodTqcTn89Hb28vBoOBuro69Ho9S0tLJJNJWVzU1dXFZz/7Wdra2vbk2vZFdJPJJJOTkwwPD/OjH/2I/v5+/uiP/oiGhgZ54Q+iXC5Lf1Yul5P+upaWFqxW634svWrIZrO899573L59e4fo6nQ6nnrqKZ5++ulDma8s/Jd3797llVdeYW1tjXw+L4VFr9fjcrlobGzkG9/4BsePH8fn86HX67FYLFitVim6Qmii0SiTk5MYjUY6OzsreXk7SKfTrK+vc/36db73ve/JwJlAr9djMpmkoJRKpR1BxEehtrYWg8EgX1zRaJRoNMqNGzfI5/Ncvny5KkQ3mUwSj8e5evUqP/zhD+XLx26309bWxosvvsgf//EfY7PZdgQCP+oAJwLwJpOJZ555hvb2djweD3q9ntdee43NzU1yuRw6nY4zZ87wjW98Y8+yQvZFdEXJbqFQYHV1lUgkQjgcxmAwPDDxWJTopdNpGUgQlSF+v5/Ozk5sNtt+LL0qKBaL5PN5crncDt+e8Gk1NjYeuj0plUrk83lisRijo6NMT0/L6iLYCpjV1tZSV1fH8ePHaW1tpbGxEbvdjsFgeGiwbGNjg1AoRHt7+35dziMRi8Vkld3a2hq5XA4Ap9NJU1MTLpeL1tZW9Ho9Op2ObDZLOBx+aH7ydkRmh9FoZGhoiEAgIL9XKBTI5XJVkYKpqqq0jkVubblcRlEU2trauHTpEseOHZPZCeKl+jBqamqoq6tDURSOHTtGc3PzjkBrLpdDVVXp066trX2k3/tJ2BfRFQ9IPp9neXkZo9HI5OQk5XKZp5566r6fL5fLrK2tEYvFCAQCTE9Py5PL0aNH+cxnPoPL5dqPpVccEfjZ3Nwkk8nIm0QE0BwOB4ODgzzzzDNVHX1+XPL5PMlkkkAgwJUrV4hEIvJ0Bh+mRTU2NvJ7v/d79Pb20tvbi9PpRFGUh+btRqNRRkdH6ejo2Ker+XhUVWV6epq3336bsbExYrGYfHG0t7fzwgsvMDAwwOXLl2X6UyqV4u7du4+Uowxbz2FzczMWi4W/+Iu/2CG6osqxGkQXtgJoi4uLrK+v73ipnD59mj/7sz/DYrFgs9ke+Z7X6XR4vV68Xi+tra3k83neeecdFhYWKJVKrK2tSStgtxvc3Mu+iK5Op5NRQOFXe1jqjjB7RNleqVTCZrNJ391hz0XdTj6fZ3Z2ltnZ2R3pTyJab7fbMRqNh6ZPrDCZQ6EQH3zwAXfv3iUej+9IeIetU357ezstLS00NzfT0NCA0Wh8pIcwm83el8NbSdLpNNlslqWlJaamplhdXd1xrQ6Hg97eXlpbW3E4HDIlTFEUWlpaHlkohf+7pqZG+j3F35NOp4lGo1LkKtk86d6T7nZEYYTBYHjs9Ymf1+v1lEolGaQT7jqv14vH49nzNNR9Ua7a2locDgcWi+WRfj6XyzE6OkogEGBtbQ1VVfF4PPh8Plk7f5hOdQ9jbW2NK1euyMorgU6no76+nubmZsxmc+UWuMvkcjmSySQ3b97kO9/5jszHLBaLO4Sorq6O559/nu7ubk6dOkVDQ8Mjm4OpVIqlpSVSqdSOAFQlUFWVUChEMBjkxo0bXLly5b6XQWtrK5///Oex2Wzy3lcUBZPJ9FgCoaoq0WiUVCp13/fC4TAbGxs8/fTTFAqFindiW11dZX5+nvX19R1fX1tbY3Z2lubmZtxu9yf+7AqFAu+++64Mzup0Ok6ePMmJEyf23ALal10VLfgetXGESAwXp1xFUbDZbDJv7kkRXNi6OZaXl1lcXJQ+PtgyFX0+H62trY/8MjsIJBIJJiYmCAQCRCIRUqnUDsHV6XSYTCYcDoc85ZrN5sfyv9XV1dHQ0FA1wVgREC0Wi9JnL+55h8OB1+vFYrFgMpl2WDSi0fajIrKBhKtmO0ajEbvdLgssKv2M1dXV4fF4MJvNO1624XCYW7dusby8TDAYxGQyyZJ3kZmh1+sxGAwyZ91sNst9E/nNiUSCWCxGIpGgXC5jMBhoamriyJEjuN3uPb22fRHdQqHAxsbGfR/0RyHSepLJJMViEZ1OR3t7O729vdjt9j1ebXWRTqd5++23mZyclGYQbN1g586d4+zZs/j9/gqucHcZHR3l7/7u71hYWGB+fv6+E67BYKClpYXu7m4uXLhAe3v7Y/eZ6Ozs5NKlS/T09FRcXBRFwWKx4HQ60ev15HI5KRq9vb2cO3eO06dP70rRS7lcZnx8XBYYbKexsZHBwUFaWlqora2t6L4oikJfXx8Oh4Ph4eEd33vnnXcYGRmR4trY2MjRo0cxmUxYLBZqa2tlzvrp06dxOp309vbK6rxCocDIyAjz8/OMjo4yPz+P1WrF7Xbzmc98hq9//et73stiX0Q3l8sRj8dlVdDDIsvibS+yHERKjNvtxu/3HypT+lFQVZVcLndf5N5gMOB2u2loaDgUOcuiofTq6irLy8usrq7KQZOAPH1ZrVZ6enro7u7G6XRisVgeyZ+9/Z7T6/WyIqsaEFkoLS0tHDt2TEbQBwcHOXLkCI2Njbs23y2Xy8k+sdvR6XTU1tbeV1RQCba/iFpaWujt7WV9fV26gzKZjMxRBmTGislkwmg0SgvBaDTicDjY3NyUz0gul2N8fJylpSU2NjYoFouyd8l+Xfu+iG44HObmzZvMzMw81MUg+gsEg0GuXbvG2NgY8Xgco9HIqVOneO6552hoaNiPJVctImvBZrPR29vLiRMnDoXoRiIRAoEAt27dYmRkRLbkEyiKIpuZfPvb36atrY3m5uaPdTc96CUvTM5qKZd2OBzYbDa+9a1vcfnyZWpra6mtrcXlcuHxeGQDlk8rCNtTMe/d22rD5XJhs9n46le/yvHjx/nNb37Db37zG5natr6+TiKRIB6P8/777+8I/NXU1FBTU4PZbJYnX+F+Eu6FbDZLMpmUbhyv18vGxgZzc3P4fL49dTHsueiKSGQwGCSRSMh+qKlUikQiQSgUwmw2yzdXNBplaWmJRCLBxsYGNTU1mEwmGUSoltPJXiNaGOZyuR2ZHqLcVTR1udfPd9AQophMJpmdnZV19feKgtlspqGhgZaWFvx+P16v96ERbHGf3euegK09NBgMe5aH+biIU6zH45HpSrW1tVgsll1p0Sl6x25sbEg/pogPbBeoaiqLFv7Z+vp6mc0SjUZlvrpIKRVTVERWFCD7S5fLZXK5nHSliKwpkXooUu1KpRK5XI5gMCjbx6ZSKVn9Jj6PXbu2XftND0DUus/NzfH666+TTCYplUqsrq5y48YNpqamWFpaQqfTybSZhYUF2cqxUCjg9/tpaGiQaUFPSqpYLpdjaWmJpaUlGVxRVRWDwcD58+fp6enB5/NVjXB8UsSL5d133+W73/0u0Wj0PleAwWCgq6uLb37zm3R0dNDW1ibLPR+ESDvb2NiQ0wTgQ4GxWq14PJ6qCkCKwgWReyoKinaDQqHAm2++yfT0ND//+c8ZGxtjbW0N+LBC7ciRI7z44otVM1FDfFZtbW34/X76+/v5/d//fSm6oj1AKBRibGxM9tcW4itOvhsbG7z77rusr6/LdDjxLAlCoRCRSIS//du/xWQyyZf6Zz/7WS5cuEBTU9OuZjTsqYKJLu5ra2vE43EZSCsUCjJqaLFYUBRFiu7i4uKOIgCn0ymjt58kN++gIk79sVhsh6ktggfNzc1Vcyr5NIgX89raGsvLy7KZz/acyrq6OlwuF52dnbS0tDy0CklYVuJkJ/x2QsREoY7NZpPBlWpBp9Pt6ktUtEbd2NiQrVFDoZA8/MCHbQzdbjc+n6/qAtXC1SIyTQqFAvl8no2NDemCEF/LZDJSTIWVk0gk0Ol0UmyLxaKs6BMuJjF1ROxLNpslkUjQ1NQkXViiU9luvAj3VHSXlpa4e/cuo6OjcmgcbPXGXFhYQK/Xy6oY8fYRpkI+n8dms3Hp0iUGBgZobGx8YgQXtvIUf/zjHzM7O0symZRvb4vFwrPPPsvZs2fxer2VXuanRpxKRd/ke0taHQ4Hp06d4vjx45w/fx6Xy/VQsSwWi9y8eZPZ2Vlu3rzJ6OiofIGLAFVbWxunT5+uOtHdbbLZLK+//jqzs7P89Kc/ZXp6mlQqJZvkAPT19TE4OMizzz7LsWPHqt59JwRT5P43NTXR39+/w70AW1k/ExMTLCwscPXqVelm0uv1NDc343Q6uXjxIn6/n8nJSaLRqBxumslkSCaT/OIXv+Cdd97hK1/5Cj6fT3bz+9TX8Kl/w0PY3NwkEonI6ZsiKrnddMrn89KcEn5M8eApiiL9eE9S1oLwOy0tLbG8vCz9b8LUdrlceL3eqn9AHoYY+Le+vk48Hr+vkY3AaDTi9XrlhIOP83GKJknLy8vE4/H7kutFp65HzXo4iIjnKJ1Os7i4yPT0NEtLSzvSxIR4eb1eOjo6ZPOpame760VkLGw/nYv5aQJxwi2Xy7LgQ9xPPT09sqLParWSzWYpl8uEw2Gy2SyRSIRkMkkkEiGbze7aS3pPRTeZTDI9PU0sFkOv1+P3+zl58qTcKL1eL4/4FouFWCzGD37wA2KxGLBlWoixPNVm9uwVhUKBZDJJKBRiamqK5eVlaRJ5PB4aGxvxeDw7mlQfRMLhMNFolCtXrvDaa6+xtLT0wFRCh8PByZMn6e7ufqTrFUNMb926dd/AQTFfS7QYdbvdh8JauJdsNsvIyAjLy8v88pe/lCc5gaIoNDc34/V6+fznP88Xv/jFPS8I2C9EH+LFxUWuXLlCLBZjZWWFUqlEW1sbHo+Hb37zm3R3d9PT04Pdbuepp54il8sxNTVFMBjkRz/6Ef/4j/8o3Q6izeRuFdPs6VMrKtFqa2ulz6i7u1vm4G3vaWk2m1lZWZGnNxExdDgcuFyuqknv2WuKxSJra2uyOEQ4/w0GA3a7XXZWOuj7sba2RigUYmJigvfff/++pi2iSktkLbhcro89mQpXRTweJxKJ7Kjggw8zJYRPsFoq0nYDYV7n83nW19dZXl5mfn6epaUlgsGg3F+xr6JzWWtrK52dnQf+1C8yEOLxONPT08zNzTE6OirbCBiNRjweDy0tLfT19dHX1ydz3L1e747ezNubaW13WezWmKc9Fd1jx47h9XpJp9Mkk0k5WkOk7IjGG+l0mqmpKXmBRqOR/v5+Wltbqa+v31HGd9hZXV3l6tWrTE1N7TC5LRYLFy9epKenp6oab38SVFXl5s2b/PKXv2RsbGxHAERgNBpxOp10dnbKrnIPO+kWi0Xm5uaIRCKMj48zPT39QNeCyPe1Wq2HxqdbLBZlaeuvfvUrVlZWuHbtGpFIhMXFRXkPKYoiUw1feuklLly4QH9/f0Wb23xaxMtmcnKSn/70pywuLvLrX/+atbU12TlscHAQv9/P1772NVpbW+nr68Nut9/nnotEIszOzsreFA6HA6fTSWtr664GrvdUdD0ezyMJRCKRYGpqSvpehK/J5/M9dl39QSeTyTA7O8v8/PyOiiyDwUBbWxtdXV2HImshEokwMTFBJBJ5YJes2tpa7HY7LpcLv9//0FOpyPFeXV0lGAwSj8cfGJQTARgxVeIw3FfCmlxfXycajXLnzh3m5+cZGRmR7hVxuhU9Zd1uNz09PZw8efKhk1uqGZFzK073kUiEoaEhVlZWmJqaIp/Py0ozkfJ16tQp2avk3s9eZL3E43HpExYNhUTxym4d/KrCKbixscHw8DCBQIBsNovBYKC5uVnObHqS2NzclPnL28VIVOWJB+Wg09bWxtNPPy2n2t6L2+3m7Nmz9PX1PVQcRUe6SCTClStXmJ+flz1S7+XEiRMMDAxw5swZGhoaDryLJpfLsbq6Sjgc5urVq6ysrPCrX/2KRCKxY46eXq+nu7sbj8fD5cuX6e3t5cyZMwfSbSca+YfDYWnZTE1NEQqFZOc4RVGor6/nmWeeoampiZdeekmmWT7IahZDAu7evcubb74ppwz39/fz3HPPMTg4uGvpYlAlopvP52UhgKiFFkf7gxws+iSIaQmrq6s7sjhEo5fW1tYKr3B3EFMQpqamHvh9q9Uq3Uuw07cmUFVV9qFdXFxkZGSEubk5mfgvENHupqYmOWXiMPhzhf86GAzKwFkgELhvnJNOp8Pn89HS0sLZs2c5efIkbrf7QGUECTeCKIwIh8NMTEwwOzvL0NCQrAcQhzabzUZ/fz/t7e2cPXsWj8fzkaXUojotFArJtDrY6q979OhRGhsbd9W9WRWKlslkmJmZYXFxkUKhIAfPdXd3H6gb49OwvdlLOByW5rHJZKKzs/NQ7YXoBetwOD7SkhFmcywW4+2336ZUKhGJRGSwo1gssry8LMeQiwCK8IMLdDodbW1tNDQ0cPHiRS5fvkxjY+N+XeqeUCwWSafTzM3N8corr7C8vMytW7dIpVI7rt1kMnHx4kWampp4/vnnaW1tpaurC7fbfWAsSOFyvH37Nh988AELCwtMTk7KirR0Ok0kEpEBV7vdTnd3N42NjVy+fFlODv+oZjaqqrK6uipfXsLqEr7co0eP7nqGS1WIbrFYlAPySqUSer2e+vp6fD7fgc5FfRzy+TzxeJx4PE4qlZKjog0GA62trbLl3mFBNJ152FBSUak2NTVFOp0mEAjIHgXZbJbR0VESiQR37twhnU7vSPqHD3saiKyZ/v5+jh8/fiB9mNsRohsKhbhx44Y0te/NADEajRw9epTBwUEuXbp0IK0kUUUWCAS4fv06d+/eZWhoCPgwMApbp1KbzUZzc7Ms3T1z5szHHlREX4p4PE4ymSSVSuFwOKirq5OjfXY74FpR0c3n89InJQZPdnZ20tzcTEdHhyz5fBIIhUL8/d//PVNTUztmgVmtVi5cuCBzCp8UwuEw165dw2g0YrVaZYQekA2SotGobFW4vZ5elIyLSsaLFy/S399Pf3//gRdc2JofduPGDcbHx5mbm5Plq9sLP86dO4ff7+dzn/scHR0dez6CZrdZXV1lY2ODW7duMTk5ycjICCMjI8TjcQD8fj9HjhzB7/czODgoq9NsNhttbW2yUc1HIUqkNzc3uXLlCrdu3WJiYgKAI0eOyLaaexHIr7joRqNR4vE4hUIBnU5HU1MT7e3tNDU1PVFtHGOxGNevXycYDMrGHbAlMCdPnqS3t7eqGrTsNYlE4r7ihu3cm2ImxFSkhNXV1XHs2DH6+/t58cUXOXbs2J6udz9JpVLcvn2b6elpgsGg7GkiqrTsdjvnzp2ju7ubp556Cr/ff6AyNVRVJZVKEYlEuHHjBm+++aacFCHwer089dRTHD9+nC996UuYzebHrqgT5b43btzg6tWrch/b29s5f/487e3te2JdVlR0RV7hzMwMGxsbbG5uMj8/T6FQ4NatW6yvr9PT03MgyhM/KcKMFjeAaPSuKIo85YnUu8MUVPR6vfT29tLS0oLX6yWTybCxsfGJfpfJZMJkMtHV1YXT6WRwcJD6+npOnDghu9QdBsLhMGNjY4yPj3Pjxg1isZgcIima+Jw5c4ampibOnz8vB1kexDzclZUVJicnmZ2dZXl5+b57w2w24/V6sVqtsiHU9skqH0WpVCIWi8mMKfH35HI52UK2t7eXs2fP0tTUtCfXVnHRvXbtGktLSzLymMvlyGQyDA8Pk06nZaOJw4rIMxSiu7a2JnOVRQcoj8eD2+0+dKJrNptpa2vD6/VKc/JxEb1g7XY7p06dkuPK29raqK+vP1TWQSQS4dq1a0xMTHDz5k1yuZwsETcajbhcLi5cuEBXVxfPPvvsgS2iUVWVYDDI2NgYc3NzhEIhgB3Ti00mk/x8RbluOp3+2KqxQqHA+Pg40WiUV199lUAgQCKRIJvNyorPnp4eTp8+vWfXVxVPsXgLGwwG2cqxp6eHrq6uQxOx/yjy+TyJREKeckWXNb1ej8/nk6PFxcjsw4JwAZw+fZpiscjS0hKBQECOxRZ9du/FZDLR3t6O2WymsbERq9WK3++nrq6OEydO4PF4aG5uliNcDgPbiz8mJiZYXFykWCxKP7bRaKSrq4u2tjZOnDhBa2vrgX5uRG+Io0ePMjMzQyAQ2NHuFWBxcZHXX38dl8vFyMiIDC5+3PBb0dAmnU6zsrLC5uYmHo9HWge9vb0MDAzs6fVVXHS357/pdDr50AwMDNDX13co8ikfRi6XIxaLyY5YwkQSlTRNTU0YjcZDVwYtXALPPfccZ8+eZWJigpGREYaGhmQq0INEV4irz+fj3LlzuFwu+vv7sdvth3ayiJgbKCbhJpPJHdWKFouFY8eO0d3dzTPPPIPP56vwij8diqLQ0dGB3W6XrWFXV1d3iG4gEGB2dhaj0YjFYpEn3e1FMQ+bKgIfdlrr7u6mqamJr3/963zuc5/b8+B9RUW3WCzK9CidTofNZuO5556js7MTr9d74EfRPArpdJqFhQUikciOG8ZsNtPb20tXV9ehFBKBXq/HZDLh8/kYHByU7hQxqPJec9FqtTIwMIDdbpeTgEW+70EKFj0Oa2trLCwssLS0xPr6ugz4iNaGHo+H48eP09HRcWiyfcRh6+mnn8ZgMMjpDul0mrW1NTmmR2RsiAPL9vtFWIx1dXU7RrKLAZyNjY3U1dUxMDCAz+ejvb19X8rDK569EIlEWF1dRafT0dDQwMsvvyxH0RyWhiQPI5lMcvfuXebm5naYRnV1dZw7d46Ojo4DbSp+HGImWFdXF52dnTsGST7IP7c9N/Pefx5WotEoQ0NDjI6OEovFZD6uXq/H7XbT0dHBCy+8QFNT06GxDB0OBw6Hgy9/+cu89NJLzM/PMz8/z8rKCnNzczK3tlAoUCwWCQaDrKys3Pd7RP9cMUXaarXKKdLnzp2jsbFRNpHafm/tJRUVXVF1BFvmtJhyW1dXd2hPLfdis9no7u4mlUphNpvlh261WqVP9zAF0B7Gft30Bw29Xo/FYpGHEPEyslqtHDlyhJ6eHhwOx6FszC4GVDqdTorFIhaLBYfDIbNdRI/kZDJJZ2fnfc2TRFtZk8lEY2OjzAgyGo20tbXhdDr3vYthxUVXpHtYrVYcDoeM1D8p+Hw+Ll26BGyVHqbTadmC78iRIzQ3Nx9q94LGxyNcCDabbcfX3W43ly9fprOzU5rKhw3RaKaxsRGfz/eRltC943q2I17m91pGIpVuv1/0FRVdl8vFpUuXSKfTmM3mQ2UePSqigXtLSwsvvPCC9FO1t7dLH5R2+nuyEY22PR4PTqeTTCZDLpeT/Qbq6+sPvWV4mKygiopuX18ff/VXfyWLAUTO5ZOEcPQ//fTTnDhxQr69RZ7uYTMXNR4fl8uF1WolGAzS09NDPB4nFArhdrvp7+/H7/cfmvS4J4GKiq5OpzuUJtHjoiiKFF8NjXsR5b0NDQ2cOXOG9fV1YrEYPT09uFwuOexV42CgfEwFx+4MBap+HueO1fbkwWj7cj+7uif5fJ7NzU3pv9Tr9Vit1l1tsP0J0Z6f+/nIPdFEdwvtprkfTXQfjHav3I+2J/fzkXuiOQw1NDQ09hFNdDU0NDT2kY9zL2hoaGho7CLaSVdDQ0NjH9FEV0NDQ2Mf0URXQ0NDYx/RRFdDQ0NjH9FEV0NDQ2Mf0URXQ0NDYx/5/8sHUngxeZLiAAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(6, 3))\n",
    "mnist_train = mnist_train.repeat(5).batch(32).prefetch(1)\n",
    "for item in mnist_train:\n",
    "    images = item[\"image\"]\n",
    "    labels = item[\"label\"]\n",
    "    for index in range(5):\n",
    "        plt.subplot(1, 5, index + 1)\n",
    "        image = images[index, ..., 0]\n",
    "        label = labels[index].numpy()\n",
    "        plt.imshow(image, cmap=\"binary\")\n",
    "        plt.title(label)\n",
    "        plt.axis(\"off\")\n",
    "    break # just showing part of the first batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(32, 28, 28, 1)\n",
      "[4 1 0 7 8 1 2 7 1 6 6 4 7 7 3 3 7 9 9 1 0 6 6 9 9 4 8 9 4 7 3 3]\n"
     ]
    }
   ],
   "source": [
    "datasets = tfds.load(name=\"mnist\")\n",
    "mnist_train, mnist_test = datasets[\"train\"], datasets[\"test\"]\n",
    "mnist_train = mnist_train.repeat(5).batch(32)\n",
    "mnist_train = mnist_train.map(lambda items: (items[\"image\"], items[\"label\"]))\n",
    "mnist_train = mnist_train.prefetch(1)\n",
    "\n",
    "for images, labels in mnist_train.take(1):\n",
    "    print(images.shape)\n",
    "    print(labels.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "datasets = tfds.load(name=\"mnist\", batch_size=32, as_supervised=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "mnist_train = datasets[\"train\"]#.repeat()#.prefetch(1)\n",
    "\n",
    "# TODO: Fix this\n",
    "\n",
    "# tf.data.Dataset.from_generator(\n",
    "#     mnist_train,\n",
    "#     (tf.uint8, tf.int64)\n",
    "# )\n",
    "\n",
    "#\n",
    "# mnist_train_categorical = tf.keras.utils.to_categorical(\n",
    "#     y=[(image, label) for item in mnist_train],\n",
    "#     num_classes=10\n",
    "# )\n",
    "#\n",
    "# model = tf.keras.models.Sequential([\n",
    "#     tf.keras.layers.Flatten(input_shape=[28, 28, 1]),\n",
    "#     tf.keras.layers.Lambda(lambda images: tf.cast(images, tf.float32)),\n",
    "#     tf.keras.layers.Dense(10, activation='softmax')\n",
    "# ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# tf.keras.losses.sparse_categorical_crossentropy,\n",
    "\n",
    "#\n",
    "# model.compile(\n",
    "#     loss='sparse_categorical_crossentropy',\n",
    "#     optimizer=tf.keras.optimizers.SGD(lr=1e-3),\n",
    "#     metrics=[tf.keras.metrics.Accuracy()]\n",
    "# )\n",
    "#\n",
    "# model.fit(\n",
    "#     mnist_train,\n",
    "#     steps_per_epoch=60000 // 32,\n",
    "#     epochs=5\n",
    "# )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow Hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "keras_layer (KerasLayer)     (None, 50)                48190600  \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 16)                816       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 17        \n",
      "=================================================================\n",
      "Total params: 48,191,433\n",
      "Trainable params: 833\n",
      "Non-trainable params: 48,190,600\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import tensorflow_hub as hub\n",
    "\n",
    "hub_layer = hub.KerasLayer(\"https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1\",\n",
    "                           output_shape=[50], input_shape=[], dtype=tf.string)\n",
    "\n",
    "model = tf.keras.Sequential()\n",
    "model.add(hub_layer)\n",
    "model.add(tf.keras.layers.Dense(16, activation='relu'))\n",
    "model.add(tf.keras.layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "sentences = tf.constant([\"It was a great movie\", \"The actors were amazing\"])\n",
    "\n",
    "embeddings = hub_layer(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "<tf.Tensor: shape=(2, 50), dtype=float32, numpy=\narray([[ 7.45939985e-02,  2.76720114e-02,  9.38646123e-02,\n         1.25124469e-01,  5.40293928e-04, -1.09435350e-01,\n         1.34755149e-01, -9.57818255e-02, -1.85177118e-01,\n        -1.69703495e-02,  1.75612606e-02, -9.06603858e-02,\n         1.12110220e-01,  1.04646273e-01,  3.87700424e-02,\n        -7.71859884e-02, -3.12189370e-01,  6.99466765e-02,\n        -4.88970093e-02, -2.99049795e-01,  1.31183028e-01,\n        -2.12630898e-01,  6.96169436e-02,  1.63592950e-01,\n         1.05169769e-02,  7.79720694e-02, -2.55230188e-01,\n        -1.80790052e-01,  2.93739915e-01,  1.62875261e-02,\n        -2.80566931e-01,  1.60284728e-01,  9.87277832e-03,\n         8.44555616e-04,  8.39456245e-02,  3.24002892e-01,\n         1.53253034e-01, -3.01048346e-02,  8.94618109e-02,\n        -2.39153411e-02, -1.50188789e-01, -1.81733668e-02,\n        -1.20483577e-01,  1.32937476e-01, -3.35325629e-01,\n        -1.46504581e-01, -1.25251599e-02, -1.64428815e-01,\n        -7.00765476e-02,  3.60923223e-02],\n       [-1.56998575e-01,  4.24599349e-02, -5.57703003e-02,\n        -8.08446854e-03,  1.23733155e-01,  3.89427543e-02,\n        -4.37901802e-02, -1.86987907e-01, -2.29341656e-01,\n        -1.27766818e-01,  3.83025259e-02, -1.07057482e-01,\n        -6.11584112e-02,  2.49654502e-01, -1.39712945e-01,\n        -3.91289443e-02, -1.35873526e-01, -3.58613044e-01,\n         2.53462754e-02, -1.58370987e-01, -1.38350084e-01,\n        -3.90771806e-01, -6.63642734e-02, -3.24838236e-02,\n        -2.20453963e-02, -1.68282315e-01, -7.40613639e-02,\n        -2.49074101e-02,  2.46460736e-01,  9.87201929e-05,\n        -1.85390845e-01, -4.92824614e-02,  1.09015472e-01,\n        -9.54203904e-02, -1.60352528e-01, -2.59811729e-02,\n         1.13778859e-01, -2.09578887e-01,  2.18261331e-01,\n        -3.11211571e-02, -6.12562597e-02, -8.66057724e-02,\n        -1.10762455e-01, -5.73977083e-03, -1.08923554e-01,\n        -1.72919363e-01,  1.00515485e-01, -5.64153939e-02,\n        -4.97694984e-02, -1.07776590e-01]], dtype=float32)>"
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercises\n",
    "\n",
    "## 1. to 8.\n",
    "See Appendix A\n",
    "\n",
    "## 9.\n",
    "### a.\n",
    "_Exercise: Load the Fashion MNIST dataset (introduced in Chapter 10); split it into a training set,\n",
    "a validation set, and a test set; shuffle the training set; and save each dataset to multiple TFRecord files.\n",
    "Each record should be a serialized `Example` protobuf with two features: the serialized\n",
    "image (use `tf.io.serialize_tensor()` to serialize each image), and the label. Note: for large images,\n",
    "you could use `tf.io.encode_jpeg()` instead. This would save a lot of space, but it would lose a bit of image quality._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "(X_train_full, y_train_full), (X_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n",
    "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n",
    "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "train_set = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(len(X_train))\n",
    "valid_set = tf.data.Dataset.from_tensor_slices((X_valid, y_valid))\n",
    "test_set = tf.data.Dataset.from_tensor_slices((X_test, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.train import Feature, Features, Example, BytesList, Int64List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def create_example(image, label):\n",
    "    image_data = tf.io.serialize_tensor(image)\n",
    "    #image_data = tf.io.encode_jpeg(image[..., np.newaxis])\n",
    "    return Example(\n",
    "        features=Features(\n",
    "            feature={\n",
    "                \"image\": Feature(bytes_list=BytesList(value=[image_data.numpy()])),\n",
    "                \"label\": Feature(int64_list=Int64List(value=[label.numpy()]))\n",
    "            }\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features {\n",
      "  feature {\n",
      "    key: \"image\"\n",
      "    value {\n",
      "      bytes_list {\n",
      "        value: \"\\010\\004\\022\\010\\022\\002\\010\\034\\022\\002\\010\\034\\\"\\220\\006\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\001\\000\\000\\rI\\000\\000\\001\\004\\000\\000\\000\\000\\001\\001\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\003\\000$\\210\\177>6\\000\\000\\000\\001\\003\\004\\000\\000\\003\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\006\\000f\\314\\260\\206\\220{\\027\\000\\000\\000\\000\\014\\n\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\233\\354\\317\\262k\\234\\241m@\\027M\\202H\\017\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\001\\000E\\317\\337\\332\\330\\330\\243\\177yz\\222\\215X\\254B\\000\\000\\000\\000\\000\\000\\000\\000\\000\\001\\001\\001\\000\\310\\350\\350\\351\\345\\337\\337\\327\\325\\244\\177{\\304\\345\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\267\\341\\330\\337\\344\\353\\343\\340\\336\\340\\335\\337\\365\\255\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\301\\344\\332\\325\\306\\264\\324\\322\\323\\325\\337\\334\\363\\312\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\001\\003\\000\\014\\333\\334\\324\\332\\300\\251\\343\\320\\332\\340\\324\\342\\305\\3214\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\006\\000c\\364\\336\\334\\332\\313\\306\\335\\327\\325\\336\\334\\365w\\2478\\000\\000\\000\\000\\000\\000\\000\\000\\000\\004\\000\\0007\\354\\344\\346\\344\\360\\350\\325\\332\\337\\352\\331\\331\\321\\\\\\000\\000\\000\\001\\004\\006\\007\\002\\000\\000\\000\\000\\000\\355\\342\\331\\337\\336\\333\\336\\335\\330\\337\\345\\327\\332\\377M\\000\\000\\003\\000\\000\\000\\000\\000\\000\\000>\\221\\314\\344\\317\\325\\335\\332\\320\\323\\332\\340\\337\\333\\327\\340\\364\\237\\000\\000\\000\\000\\000\\022,Rk\\275\\344\\334\\336\\331\\342\\310\\315\\323\\346\\340\\352\\260\\274\\372\\370\\351\\356\\327\\000\\0009\\273\\320\\340\\335\\340\\320\\314\\326\\320\\321\\310\\237\\365\\301\\316\\337\\377\\377\\335\\352\\335\\323\\334\\350\\366\\000\\003\\312\\344\\340\\335\\323\\323\\326\\315\\315\\315\\334\\360P\\226\\377\\345\\335\\274\\232\\277\\322\\314\\321\\336\\344\\341\\000b\\351\\306\\322\\336\\345\\345\\352\\371\\334\\302\\327\\331\\361AIju\\250\\333\\335\\327\\331\\337\\337\\340\\345\\035K\\314\\324\\314\\301\\315\\323\\341\\330\\271\\305\\316\\306\\325\\360\\303\\343\\365\\357\\337\\332\\324\\321\\336\\334\\335\\346C0\\313\\267\\302\\325\\305\\271\\276\\302\\300\\312\\326\\333\\335\\334\\354\\341\\330\\307\\316\\272\\265\\261\\254\\265\\315\\316s\\000z\\333\\301\\263\\253\\267\\304\\314\\322\\325\\317\\323\\322\\310\\304\\302\\277\\303\\277\\306\\300\\260\\234\\247\\261\\322\\\\\\000\\000J\\275\\324\\277\\257\\254\\257\\265\\271\\274\\275\\274\\301\\306\\314\\321\\322\\322\\323\\274\\274\\302\\300\\330\\252\\000\\002\\000\\000\\000B\\310\\336\\355\\357\\362\\366\\363\\364\\335\\334\\301\\277\\263\\266\\266\\265\\260\\246\\250c:\\000\\000\\000\\000\\000\\000\\000\\000\\000(=,H)#\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\"\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "  feature {\n",
      "    key: \"label\"\n",
      "    value {\n",
      "      int64_list {\n",
      "        value: 9\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for image, label in valid_set.take(1):\n",
    "    print(create_example(image, label))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following function saves a given dataset to a set of TFRecord files. The examples are written to the files\n",
    "in a round-robin fashion. To do this, we enumerate all the examples using the `dataset.enumerate()` method,\n",
    "and we compute `index % n_shards` to decide which file to write to. We use the standard `contextlib.ExitStack`\n",
    "class to make sure that all writers are properly closed whether or not an I/O error occurs while writing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from contextlib import ExitStack\n",
    "\n",
    "def write_tfrecords(name, dataset, n_shards=10):\n",
    "    paths = [\"{}.tfrecord-{:05d}-of-{:05d}\".format(name, index, n_shards)\n",
    "             for index in range(n_shards)]\n",
    "\n",
    "    with ExitStack() as stack:\n",
    "        writers = [stack.enter_context(tf.io.TFRecordWriter(path))\n",
    "                   for path in paths]\n",
    "        for index, (image, label) in dataset.enumerate():\n",
    "            shard = index % n_shards\n",
    "            example = create_example(image, label)\n",
    "            writers[shard].write(example.SerializeToString())\n",
    "    return paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "train_filepaths = write_tfrecords(\"my_fashion_mnist.train\", train_set)\n",
    "valid_filepaths = write_tfrecords(\"my_fashion_mnist.valid\", valid_set)\n",
    "test_filepaths = write_tfrecords(\"my_fashion_mnist.test\", test_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b.\n",
    "_Exercise: Then use tf.data to create an efficient dataset for each set. Finally, use a Keras model to train\n",
    "these datasets, including a preprocessing layer to standardize each input feature. Try to make the input\n",
    "pipeline as efficient as possible, using TensorBoard to visualize profiling data._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def preprocess(tfrecord):\n",
    "    feature_descriptions = {\n",
    "        \"image\": tf.io.FixedLenFeature([], tf.string, default_value=\"\"),\n",
    "        \"label\": tf.io.FixedLenFeature([], tf.int64, default_value=-1)\n",
    "    }\n",
    "    example = tf.io.parse_single_example(tfrecord, feature_descriptions)\n",
    "    image = tf.io.parse_tensor(example[\"image\"], out_type=tf.uint8)\n",
    "    #image = tf.io.decode_jpeg(example[\"image\"])\n",
    "    image = tf.reshape(image, shape=[28, 28])\n",
    "    return image, example[\"label\"]\n",
    "\n",
    "def mnist_dataset(filepaths, n_read_threads=5, shuffle_buffer_size=None,\n",
    "                  n_parse_threads=5, batch_size=32, cache=True):\n",
    "\n",
    "    dataset = tf.data.TFRecordDataset(filepaths,\n",
    "                                      num_parallel_reads = n_read_threads)\n",
    "    if cache:\n",
    "        dataset = dataset.cache()\n",
    "    if shuffle_buffer_size:\n",
    "        dataset = dataset.shuffle(shuffle_buffer_size)\n",
    "\n",
    "    dataset = dataset.map(preprocess,\n",
    "                          num_parallel_calls = n_parse_threads)\n",
    "    dataset = dataset.batch(batch_size)\n",
    "    return dataset.prefetch(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "train_set = mnist_dataset(train_filepaths, shuffle_buffer_size=60000)\n",
    "valid_set = mnist_dataset(train_filepaths)\n",
    "test_set = mnist_dataset(train_filepaths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 5 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAABYCAYAAABWMiSwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA3TUlEQVR4nO19WWyk13Xmd2vfNxaruDWbvanZ6pbklhJblmC7rdiwPVasxJ6HeJbkZV4SzMPM2zxkXgYBBgMMggwce14Sz3g8gIExksijQBkbjtS2FiBSqxerJdHN7ubOIllVrCrWvv7zQH+Xp/4u9squqpb+DyDYXSwW7///9557zne+c64yDAMWLFiwYKE/sA16ABYsWLDwSYJldC1YsGChj7CMrgULFiz0EZbRtWDBgoU+wjK6FixYsNBHWEbXggULFvoIy+hasGDBQh8xNEZXKRVTSv2dUqqslFpSSv2LQY9p0FBK/W+lVEoptaOUuqaU+jeDHtMwQCk1o5R6VSmVU0ptKKX+UinlGPS4BgWl1L9VSl1QStWVUv9z0OMZBiilSqavtlLqO4MeFzBERhfAdwE0ACQB/EsA/10pdXqwQxo4/jOAGcMwQgC+AeDPlFLPDHhMw4DvAdgCMA7gUwC+AOBPBjmgAWMdwJ8B+P6gBzIsMAwjwC/s2pQqgB8PeFgAhsToKqX8AL4F4D8ahlEyDONNAP8XwL8e7MgGC8MwPjAMo87//ubr2ACHNCw4AuD/GIZRMwxjA8D/A/CJ3aANw/hbwzBeBpAd9FiGFP8cu5v0G4MeCDAkRhfAYwDahmFcE69dwSd4IRFKqe8ppSoA5gCkALw64CENA/4bgD9QSvmUUpMAvoZdw2vBQi/8EYD/ZQxJz4NhMboBAAXTawUAwQGMZahgGMafYPc+fA7A3wKo3/43PhH4BXY35B0AqwAuAHh5kAOyMJxQSk1jl376waDHQgyL0S0BCJleCwEoDmAsQwfDMNq/oVymAPzxoMczSCilbAB+it0NyA8gDiAK4L8MclwWhhZ/COBNwzAWBj0QYliM7jUADqXUCfHaUwA+GNB4hhUOWJxuDMAhAH9pGEbdMIwsgP8B4J8NdlgWhhR/iCHycoEhMbqGYZSx67n8J6WUXyn1PICXAPxwsCMbHJRSCaXUHyilAkopu1LqKwC+DeC1QY9tkDAMIwNgAcAfK6UcSqkIdjm7KwMd2ADxm/vgAWAHYFdKeT7JEjpCKfUcgEkMiWqBGAqj+xv8CQAvdrOMPwLwx4ZhfJI9XQO7VMIqgByA/wrg3xmG8ZOBjmo48E0AXwWQBnAdQAvAvx/oiAaLP8WuJOo/APhXv/n3nw50RMOBPwLwt4ZhDBVNqYYkoWfBggULnwgMk6drwYIFCx97WEbXggULFvoIy+hasGDBQh9hGV0LFixY6CMso2vBggULfcSdtHx3JW2gAkIpdcf3tdttvPvuu/jzP/9zNJtNBAIBuFwujI6Owm636/c6nU60222sr6+j1WrhzJkzSCaT+PznP48jR470/GzDMG4Zw53GxLfdzZv4p+7hvXdENpvF9evXsbGxgYsXL2J8fBy/93u/h0AgAL/fv+/4a7UafvCDH+DatWt46qmnMDExgVOnTmFycvKghnYv9wQ44PsyxBjYXKnVasjn81heXsb58+dhs9kwMjICAGg0GrDZbAgEAnA4HPB4PF1zx+FwQCmFXC6Her2O5557DidOnNjvT90rBnZPhhj73pO+CKjb7TZarRaq1Sp2dnaQzWbR6XSglILdbofdbker1QKwZ8A7nQ46nY42puVyGYVCAcViEaVSCR6PBw7H3vDlBOO/h1kOV6vVUC6Xkc1mkc/nUSgUUKlUUCwWsbm5iVKpBK/XC5vNBqfTCaUUWq0WDMNAp9PRv99sNlEqlVAoFJDNZuFyuRAIBOD1egd9iRYOGHzWxWIR+XweNpsNNpsNSinU63XYbDY0m004nU49dzqdDgDA7XbDZrOhWCyiXq+jXq+j3W7r37fQPxyI0b3TQ9ve3sbS0hKuXr2Kn/zkJ2g2m+h0OvD7/XqnzufzaLfb+ovv4cT55S9/iVarhVKphNnZWTz99NOYmZm57TiGeTK9//77eOWVV7Czs4NMJoNKpYJsNgu73Y5f/vKXejF4vV4cOXIEDocDq6urqNVqqNVqXZtSKpWCw+FAMBiE1+vF7//+7+PcuXODvkQLB4yNjQ289tprmJubw8svvwzDMBCNRgHsbuJKKYTDYbjdbkQikS5nJhqNwuVyIZPJoNFoYGxsDKOjowgGg/D5fIO8rE8cDtTTlUaTO6xhGCgUCtje3sb6+jquX78Oh8OBZDLZ9XulUgnNZhOtVgudTgfNZhOGYegJkc/nUa1WkUqlEAqFcOTIESQSCW1YHQ4H7HY7lFJDbWybzSYajQbS6TSuX7+OSqWCnZ0dNJtNlMtldDodbGxsaA/f6/Wi0+nAbrdjeXkZlUoFjUYDnU4Ho6Oj8Hg8qFQqMAwD2WwWDodDLyxGERY+HqhUKlhZWcHa2ho2NzcB7NIKhmFoo1uv1+F2u9FqteBwOFCv7zalMwwDbrdb0wvlclnPIwv9xX0Z3f043NXVVWxsbCCdTiOTyWhvjN5ZJpPB+Pj47h92OFAul/HRRx+hXC7j5s2begLZ7XYEg0G43W6Mj4/D7XYjGo0iFApha2sL9XodjUYDc3Nz8Hg8cDqdmJqawuTkJHw+H/x+/wPeloeHmzdv4sqVK/jVr36FxcVFuFwuBINBGIaBQCCAdruNRqOBVquFcrkMAFhY2G2QxAUUCoXgdrsRj8fh8Xg0zVAoFFCtVjE/P49Lly7pe2Lh44H19XX8wz/8AyqVCsbHx+F0OuH3+9HpdFAqlWC325FMJuHxeDS3S083GAzCbrfr+eV0OvX6tNBfHIin22q10G63USgUkE6nkUqlkEql0Ol00G63AUBzkn6/X3vC7XYbOzs7KJfL2N7eRrPZ3B2Uw6GJ/1arBafTCZfLBWCP19ra2kKn04HH44HL5dITzTAMOJ3OofXyisUiVlZWsLW1hXK5DMMwEIlEoJTSyUNyc/yqVqs6SWi32+F2u+HxeOB2u+F0OvUCarVaqNfr+jkw9HxU0CsRCkDz/3eTqOX7JR513rLVaqHRaCCfzyOVSsFmsyEWi8HhcMDv96PVaqHZbMJut8Pn88Hj8cDn82kjCwAulws2m02vIwCawrPQX9yX0ZUT2DAMXL58GQsLC9je3kahUIBSCj6fT1MNNIoOhwOFQgGlUgmZTEaHzy6XC6dPn9YJN5vNpjkpANpYu91uHD16FCMjI9pokwve2trChQsXEI/HMTY2hsOHD+PMmTNDt9iKxSJWV1d1IgTYyzzTsHi9Xng8Hk0tkLsFdu+9y+WC3W5HpVJBpVIBsGtYSLGk02nMzc1pvvxRQLPZRL1eh1IKNpsNdrsdLpcLtVoN29vbcDgciEQi+jrNMAwDOzs7aDQaKBQKXaHz5OTkI7cBSVy9ehU/+9nPMDc3B5fLpT1cOhbccOnF2u12PV/43ZyoTqVSmJubw+nTpxGJRAZyXZ9UPJCnSwnY1tYWbt68iWq1ilqthkAggEAgoI2Iy+WC3+9HtVqFx+NBvV5Hs9nUmXn+3DAMvfDC4TCUUprndLlc8Hq9iMViSCQSyOfzqFQqerFWKhWtjqjVatqjJs87LGg0GtjZ2dFUAQCtSKDhpZfvdDoBoMvoAtCbUb1eR6vVgtvt7vIEq9UqstksqtVqH6/s3sH50+l0UK1WUalUtMGl916pVJDP5+FyueB2u/XPmLkHoI1OsVjU185ko1IKoVBIe3406uY5sZ+XPQxIp9O4ePEitra29PyQOQxgb/ycSxLcuPlzADq6rNVqfb+eTzru2+gahoG1tTXk83lsbm6iXC7D4XBo3a0Mh7mo7HY7JiYm4Pf70Wg0dMadEwiANj6hUEhLXNrtNhKJBHw+H9xut+Y6uQj9fr/mQ5VSyOfzWFtbw9zcHKLRKCYmJoZmQdXrdeRyOVQqFX1/6Ln1Co+lsaWxcblcUErB4/HoaIJyMgDI5XKw2WwoFMwnID187Mf3m1/f2dlBqVTC1atX8cEHHyCXyyGTycDtdnfJ3RwOhw6N+cVN+oknnoBSCpcuXUIul8Pa2hoqlUqXnFApBb/fD5/Ph5MnT+Lw4cOYmprCxMRE19juVms+CGxvb+PKlStdkSFpA8Lj8XRFO4wE2+22vj7+3GazoVKpIJPJDP3G/HHEAxldGtxSqaTJeeoBAWjvyzAM7dmGQrun8oTDYR0y0ntRSulMPY2u2+3uMrrM3ssdH9iddDTu5XIZxWIR6XQaNpsN4+PjQ7OYWq0WarWa5q+lh2oOCQmzppkLiwtKLhzDMFCpVGC32/vqxfQa+36GF9jztK5du4Y333wT2WwWGxsb8Hq9mptvtVrw+XxIJpM6Qw9AS6JCoRCUUnjnnXewsbGh1R2cKxwDHQDOG36m9HiH2ehWKhVsbGzA4/EgmUxq+oBOBrCXB+E1yShCgs5No9HQkaKF/uK+jW6n08HKygrm5+dRLpd1BYxZcE01Ar2UTqeDWCymFxZhDpPcbjcAIBaL6QlUKpW0oellrBiSU0Y1NzeHdruNkydP3u9lHjjMxQ61Wg2lUkmPG4CWywHoCoe5gBqNBgBob4dhdb1e116NmZLoF6TRYtQiX9/c3EShUMAvfvELXLx4Eel0Gtvb22i1Wjrq4XPk/8lXkpKhjvunP/0pAGBrawutVksbZyaWyH+WSiVUq1VcvnwZq6urmJubw6VLl/D444/jmWee0feYYx422SHnDO+NdFIA6GiP60xuNHI+8LpotGV0ZKF/eCBPN5vNYm1tTcu2bDZbVygjDQcnDHfkRCKBZrOJSqXSZSD4nZOH2VZW0kiDLg2RTMCQNy4UCojH40M1sThGpZT2/mlEvV7vLYuBBkjeHxplcnXksym54+uDyEzz2ez3twuFAtbX13Hx4kX89Kc/hd/vh9frhcPh0LQC5wm9VUmhANBe78rKii6ycTgciEajcDqd2Nra0vkDKkEqlQqWl5eRSqX0vPX5fHj66ae7IrJh9HhpKLkxAOii47hO5NwCcMsGLK9tP0/YwsPHfRndZrOJWq2mkx+ULQHoMhqcBHyNYAEEX+cik+/jJJFhuOTpJOTkAqA5rVqtpkseAfTMevcbjUZD0zEulwuhUAhTU1Oo1+vIZrMAoGkEuYA6nY7msunh1mo1nSyk0oGv9Vv4fqdqQJY5v/POO3j//ffRaDRw9uxZZDIZpNNpHSmRonI4HLpIhhtzp9OBz+fD8ePHoZRCJpNBvV7XERDnXTAY1DpVcrrcoElVnT59GuVyGa+88gpmZmY0PywN27CAnq4s/mHEJ9cVx06Hh9dsGIa+nzTeTEwOw5r4pOG+jW6j0dBFD4FAoGuycoFIUh/oTlhwsph3cUn8833SmBK9eE+ZLAB2DVyj0dAaxmGYYOxBQWlcIBDA+Pg4tre3sbCwoLPtALoSbFR2yPuzs7ODdruNWCwGr9eLarWqF9qgNZhmWWGpVEIul8PVq1fx9ttv49ixY5idncWvfvUrLC8v68QgN1ebzaY3DyYEnU4nAoEADh06pBNs1WoVKysrWhFjGAb8fr+ODgzD0JFEtVpFu91GNBrF0aNHsbKygvfffx/NZlPLC4fJw5WQXD6pKV6b5PeBbh6bnnCz2ezaVNgUZxi17B933LMV6nQ6yOVyeiFwgkvjye/0LOQObA59OFn48Llw9hO0m40tF4qcYPxMLlypdhjUopIGoFwua68uEolgdnYWCwsLeO+997q8GkIWmMh7y2KI2dlZzMzM4PLly1hcXNQ9KiqVCur1el83nF7heavVwuuvv44rV67g2rVraDabyGazujiG7zU/N2qQmXT1+XxwuVy6Qq9araLT6WgNLvMG9HBZOl4ul1Gr1TQ3vLy8jHfeeQeRSARPPfUU8vk8vvvd7+Kxxx7Dl770paEyRFwjtVpNJ5ZlZEnnxeVy6fJyGlhyvXy/jJwYqUpP+VHE7eig2/2s0+lgeXkZpVIJo6Ojug5ARuz7faaMPpvNJjKZDC5cuAC/34/nn3/+js2m7nklGoah5T0A4PP5NJ9LvpbaW9IPAPQDpwfCnZoeHw0zX+fCkeh1E2SSid/NNEO5XIbdbkc4HL7Xyz0wcPE0Gg1Uq1W43W64XC5EIhGcPHlSe2qS9zYL2qWX0ul00Gg0UK/X8dhjj+HZZ5/F9vY2Njc3kcvltNGt1Wp9DSN7TfR2u43XX38df//3f49IJAKfz4dsNotisYhisXhLlMN/8z5Fo1G43W6Ew2E0Gg0sLCxofbLb7cb09LRWr/B3ea+LxSJqtRoajYZ+fXV1FTs7O/jc5z6HJ554Aj//+c/xV3/1V/ja176Gc+fODZXRBaCNLqsQWWwE3Do30um0djJYFuzz+RAIBLRzxM2/Wq0+0uoFGREDt9qHXhEy0W63sbS0pCtbR0dHtcNj1s33+nu0NbVaDWtra/jJT36CZDKJs2fPPhyjy7JEGkiK9JvNpuYYGb4wGcIBm2+U9I7lLsL39Lph8vdbrZbmhXkzaLjcbjeazSY2NzfRbrdv6dnbT9RqNVQqFZRKJW0IQ6EQIpEIYrEYgsEgWq3WLRSKecc1byxKKUSjUYyNjSEYDOrFWKvVdAezSCSiX3/YMHu4ly5dwvr6OjKZjPa4WO5Mzp1zxufz6c2GXimvkxOcGw1pGhpl8pXAbo9itgKlzIzFFoZhYHt7G9lsFrOzs/qeh0IhbGxs4Ic//CFmZmbwhS98QXs9gwLXlNkwyuhOysfcbjdOnz6Ner2OpaUlfb8ol2OkRyfHnJx7FCEToGaj2+vaDGO3AVe5XMb6+jo2NjZw6NChnhG0+ff4OvMHGxsbeO+993ROwul06g29V89v4r7cH058LgpmzfP5PMrlst4hpqamEI1GtcQHQFc4I1UNtzO6DKc5ueRNYVKPk4hcHyVY9GwMwzjIps33jGq1qsukK5UKwuEwIpEI4vE4xsfHEQ6H9aZFmHdWeW+k0Y3H45iensbIyIhWQFAHu76+3tXs+mFDTtZGo4Hz589rw0uD63A4tFaZzYtsNptu3kIevtlsdiVYSZfQWNdqNTidTq1Q4bWvra0hl8vpsTBsJLfL5N3Zs2f134hEIlheXsZf/MVf4POf/zw++9nPDoXRpSqFoCFhopUbWLvdhs/nw+nTp3V/EiYauf6Y2+A6e9SN7u0MLn9uRrvdRjabxfb2NhYXF7GxsYEnnniip8dsXm/csBg9Lyws4G/+5m90LYDL5cLNmzfhdrsP1ujabDYkEgm43W4YhoF4PI5MJqPbE8pEDxeLlITJnZkXI7OysvGyGVJrKPk/j8ejb2QoFNK9GbgQDx06hHg8PtAkCbv+U5DudDoRjUYRDAbhcrm66BQp+5KQ9whAVytMYJdfj8fjWkpVqVS0bK6faLfbWFlZQTabxerqKtLpNNrttk6UST7a4XDoHrCcL/Ry2WOCkQw9P27WlEmxrJoUCqV4ZjB09Pv9mld+7bXXsLq6qsPtUqmke1fE43FMTU0NLDoiPUL1jZRFElxnjJ5OnTqFTqeDd999Fzs7O5pCkOtNFtsMa+LwbiG9folKpYKbN292zZfp6WkopTA/P4/NzU24XC6Mj48jEolo2eGd/g6w22Z2bm4Oq6ur8Hg8iEQieOaZZ+Dz+XQ3t9vhvozuzMwMOp0OxsbGUCqVcOXKFSwvL+vFIj1buUtz8MCeYWFVDQdqLgSQSRZgjxvmzk0v5te//jUuXryImZkZ7fE6nU6EQiGcPn164JnacrmMra0tFAoFtFoteDweTExMYHR0VJd2Aujy5uWmRJ4b2Dt6hZ4Q72E0GsXk5CR+/etf69aQmUymq3dxP9BoNHRS78MPP8TS0hL8fj9CoZC+PvbLiEajiEQimqcm7cCGN3a7XUdSsjCGvLdhGEin013ev6xUBPaiJeYeotEoAoEAlpeX8dd//de6Cq5SqSCXy2FlZQVvvfUWjhw5oivABoFarYZsNtvVk0J6p7xmPutIJILnn38eDocDr776qv5dKln4xeQ278ejjl7XsLOzg9dff13nDAKBAL72ta/B7XbjnXfeweLioj4IYWxsrGfTH+n9yr+xubmJ119/HbVaDeFwGEePHsXXv/512O12rKys3JEnv+/sis1mg9frhVIK09PT8Hq9mJiYQLlcRiqVwsbGBux2u+65ILWD5guShsb8fyba5KIC9rpSxeNxjI6OotPpYGRkBKFQSEuogsEgJicnh0KPSM+O6gx6unfq2i+PZCEYmpNXp1fv9/sRi8W6SqLz+XzfyoEZdhUKBVy/fh03btxAqVTq0pPSO2eoy+ujJltWscnkKrBnZHppuM1cpZkH5/hk4kkW7EiDXa1WcePGDR26Dwqyt7K5KIj/l538GDExnyIjC/6emZp61D1dMyqVClZXV7GysoKrV6+iWq0iHo+j0WhgfX0dHo9Hz71cLqc3bfbnYCWstFP8987ODtLpNJaWlpDL5QDsqUbm5+d1oc+d+nk/kCUKhUIIBoOaU+PXG2+8gTfeeEPLteh1sjqIXBQhFyV3YhonTjB6OVwErMJKJBI4e/YsPvOZz8DlcumzwkZGRjAxMTE0vBVPhmCo5/f7MTk5iVgsBuDWngVyQbDBDX+XndRY+st7MzIygsOHDyMYDALYnSQrKys4fvx4X66x3W5jc3MTqVQKb7zxBj766CN9XeRuZRc4boakEJj0kY1ZgG6jK707Wcknu4+ZS6ClYef8ofyKzWOYUPN6vSgWi3jjjTdQq9Xw7W9/uy/3rhdYGCI1trwvVOaw9JtNn3g//X6/ngfSyMpN5ONmcIHdJOrPfvYz3Lx5E6+++ira7TY+/elPY2RkBFeuXIHP59P0HqWHiUQCADA9Pa2NLiHv0fr6Ot566y2sra1hfX0dfr8fhw4dQrlcxs9//nOMj4/jpZde0vd9Pzyw+9drt5QLyVwxY9buSgkL/y1BPo8LiaGerECi5yj7z1IiMywgp8vmNC6XS0uh2DsB2CsTlkUl3IBYbWbmyavVKorFIux2OyKRiM7UM+HZL1lQo9HA9evXsby8rFtLxuNxHb47HA59D0gdGIahm7TLpKHcxGWEYy5p5ft7RVD8GX+PxlrymrLKkYaNYyoUClhdXdURVb9pBtIwXAPmBA8AXdk4OjqK0dFR7b3T05Vj3o+q+ziA871YLGJ9fR3pdFpLVBkNsI2Az+fTlGa73cba2hrq9To2NjYQDAZ1spdrkVWtCwsL+OCDD9BsNnUuplwuw+v16tah+9UXSDxwP10J/jHuyMwy87wmuVsD6OqbyjOeZG9YytHkIpPNYhh+NRoN7TlR+2rW18nxDQK5XA4LCwvIZrMwDAOhUAjHjh3TDz2TyehJwgQRr5PnWRUKha4Ni9fEhJXP50MoFILf79ecKT3LfqBcLuPVV1/FRx99hBs3bqBWq+Hs2bOYmprCkSNHEIlEsLm5qZOurVYL8/PzmJ+f1y0dZcWV1O3yWs3crqQUpLwMQFdlG+cSCwLYs4FRhORLSZOkUim8+eabmJ6exrlz5/p+gKPUGct7wDVEBYfH48GZM2c0ldZqtRCLxfRGIRNn3MAlZ/5xQKPRwPb2NtbW1nD58mXkcjn9fPnsi8UiDMNAIpGAzWbD9vY2KpUK/umf/kkbaWC3GIe/6/V6kclkkEql9Enmx48fxwsvvKCpjHA4jJmZGd0Z8U54KESn7IpE42Em/7l4zG3q+N1s0G8XfksM885t5tRsNpvWC9IYyy9WUUku2Px5Nttu39ytrS1MTk7C7/d3dcyi8XrYuH79OjKZDLa2tpDP5/Xzn56extGjR+FyufSBiDs7O4jFYgiFQshkMjqJyPagkn/nPeu1iZrR69nL97KCiHSXbNIE7FEOjNDq9TpWV1fhcrkGVrm137XKnzscDoyMjGBkZAQOhwPtdrurCs3cx4OqH0nhPKogBZPJZPDBBx9gcXFRqz3ojdIhK5fLei1wntFRkUogOog01rRTjJ6Z+DUMQyfqKGe80/MCHtDo7rcQnE6n5k7otdHNJyXA3ZeZafJzMuwzS8jkjs2Fw51ahpFMPEgp2qDBh+9wOHRypN1uI5VK4fz583qyyJ4Dm5ubupIK6C4kocdrs9m0p/jCCy9gbGysq6S6X6We3//+91Eul/Hhhx8in88jFoshFovh61//Op588kntAc/NzSGVSuFb3/oWzp07p0Xm29vb2NraQiQSQSQS0c8R6M4c3+5auMFzE5cSQ3LilUoFsVgMIyMj2hiRH2aFG7nnnZ0d3ev3d3/3dx/6PeyF/eau3Jy9Xi/OnDmDsbExPX6PxwO/369PbDE3zKGOfViNbi/dbC8UCgXcuHED7733Hr73ve/p6rJAIKCLhcbHx2Gz2bCxsYFWq4VwOAyPx4PDhw8jEonoAxDK5TLq9bqmQEkrxONxLe8k/2sYu/1ElpaWEA6HNbVzN2vtwDxdeWPMoYtMZMj3mrOxkkagd0MjK8t8ZTKEBnjYvV5pCADof/MkCYY+1NdyIzKH2IS8L5SGsRfBIMDzzCTHTg43lUphfX0dqVRKqyko3m82m/qkh/14S8IcCfA183d5j+UGTk83GAxiampKU1PFYhHlchl+vx/JZLKrUT7PoWPBRr/zBPtReFKnTo/N6/V2KT8k9cLP4hfnz7Aa3duBHiY36vn5eayvr3cdzul2u3WvDq41RlB0amq1WlfbAtIuzJ3ICJwVroyEpP6Zne64bu+Eh0IvsGUh+y4Aex2zzCWIFKzTo5VebafT0SdR8AbQM5bJM/IvwJ4xGiaDC+xVzslMfL1eRz6fx+LiIjKZjH6Y6+vr+ncA9NxUeO8A6GTPpz/96a7TACTn+bDx/PPPI5vN4sKFCyiXy5qb//GPfwyHw4ErV65gY2MD4XAYPp8P7777LpaXlwEA0WhU3wtOfrm5yqQhFxBfY7TEsJoUlfRyOd9YAHHy5El885vfxOLiIq5du4a5uTnMz8/jU5/6FL74xS8inU7j7bff1r1DisUicrkcnE6n9nT6AUnFceGTgybPGwwGdXVjMBjU3r589jJ6kv0IzI2V+g25eZrn6H7rl9Hy5cuX8fLLLyOTyWBhYQEOhwOzs7Pwer263J+yQp6gTHUCN/ulpSU4HA7MzMwgGo3qzZ/PW55ByOpWrq9yuayT37VaDUtLS/B6vf31dLs+9De7BxfAfhrZ23k20tOlIQXQ5S0Ce8bHnFwaNqNLL4uGhEUA1WpVl3pyzOaKqv2uh54ue/RKCRnQrUV92IjH47DZbJicnIRSSntdpVKpq4iD11OtVpHL5bS+1DAMbVCA3lxmL+UC3ys9XRorc+UeOTt6JqVSCcViUTcGUkohFouh1WrpZGyxWNQlyGwk0y+YFR1yHvD62ARnP6+119yhRzdoTrdX7sYMGll+1Wo1lMtlTUmx9UAgEIDf74fH49HziRGKtBdy8yb3m81muw4G4Ou0Ydy0SE/JBCc3ORZr9c3T5WLgAFwuFwKBgC6OAKDLPPeTAMmMtPmzAWihdz6f1ycGc+eXyaO7eZCDALOn7DPAzCeTaLVaTRsqeupmvpqg8aIYvlQqoV6vY2trC1tbWzrCILfOKOBh4tSpU/oaMpkMzp8/rxdGu93G5OQkxsfH9SJot9t63OSyOWfMx8jIeSOTglKlIPszc2NmhJTJZPS5efV6HW+99RZu3rypNd1cZIFAANPT0wiHwyiXy1hbW8Pq6ioKhQJu3ryJXC5325r6g4Zh7JXRS4UFjY/P58PY2FjPqjneMxZL0BHi53k8HoTD4b7MjdvhTkafhnVzcxNbW1tYW1vDwsKC3sT9fj9mZmbg8XgQi8X05lMulzE3NwcAmJqagsvl0gUNNOBbW1sol8u4dOmS1o57PB4cP34cU1NTmk4oFArY3t7WSW9WLpJPt9lsWp55N3gonq4M8Xt5Jr1wJ+KcO47cuWnoHwXZi1QSsEiEXpa56uh26gwJWTxAfpKJAKJfHj9Ds6mpKV34AewafupwO52OPiON94OVdfTWbrdpmiuruEnvx++ay6mZWOJ5Yh6PB6FQSHPhTCx5vV6MjY1ph4GRRL+rGrnJcJ4D6PLeuejlYbB3i2HwdIFb9dNykwF2pZa5XA7ZbFY3KuKBszI5yEIb0pTNZhM7OzswDAOxWExrdQm2DpWnyzAq5FqS0jwm2fhFFQybLMm5diccyCwyLxSpLJCcjZkW4MXLz+DNNhsgesQktB+1Q/U4ERja5fN5fPDBB1hZWdHKDclL3w1kqAnsnZRBbhNAX0+QsNvtGBsbQzweRyKR6FJp7OzsoFKp4Pvf/z7Onz+vIxWK+OU17TdehnnmTUnOG/k6OXSv14tQKIRnnnkGs7OzeOyxx3D8+HG9SV24cAGvvvoqpqamsLGxgdHRUbz44ou4fPkyXnnlFdTrdaRSqb7TC+wWxkb0cs1Uq1VdDRqLxXpyolKiKPuh0GMetNHtdHYPROB5dyxs2NnZ0WMj9SMLfdg7IxqNolAoYG1trcvgUtO+urqKarWKpaWlrqiR1091w2/91m8hkUjojXxjY0MfeQ/sns9IGaTH4+lqLxAKhVCr1ZBKpbQOmgUr++GhqBeAO1e7SM5tv8+QMGddzTvXsMPMz1GWJBfy/XqlcmFJmsac6X/YILXkcrluKSRgY3Wfz9eVGOPzlBRTr+vbL4vf672y8pHqCDYYOnXqFB577DGcOHFC83DZbBbRaBRer1fz6YlEQvfplY1i+gnpQclIT1J0VC30SrTKDcysCmKCrd+5DybCybNzDWQyGeTzeayvr6NQKGgjyveZmx1RcUCKjXOI90zmTkg9ceNiqS83fbZYpUKl1WppLt8wdtuklkolHc0ZhqGTa1RHyHt5p/X20OiF/QbQS9rTCzLLyockEwCyV8OjAGlUDGO38CGTyeiKIrMhuhtIkp+eNCf1vXxOPxAMBuF2u/V3eX6ebDsoN1c5h8hv9gJ/j2EpF2Kj0dC9i3mg5Wc/+1l0Oh2k02n92cxSA9BVjcCup5nP5xEOh3HkyJG+t8i02Wz6WqQSAdjNETidThw7dgxjY2NdUjabzYZgMKg7Z7HvBZOsNFw8yqefuHDhAur1Oq5du6YPtQWgJY98FrIZEm2ALIfe3t5GOp2G0+nUyU9SazabDeFwGM8995yeA8BesUQymdQbsc1mQ71eRyaTQSaTQbFY1DkWGlKWU9PLdbvdOglO1RHvLfNMt8OBGd1eSbDbvfdeDAMXVKezd5rpsCbM9oNZysYsqTwt4l65WLNh4g7fK4oYNGTptzkKkh65VB7IZ2w2yMAeZdVrHkiPkD+nTKpQKKBQKOi5xASnYezpi+XY2OSczYn6hV7cvrxPNpsNoVAIgUCg6z1KqVtO6DZvTP1Utkik02ndxa1UKunz7wqFgk6Qm2kjswNHtUC1WoXf70c4HNaSQM4Jj8eDZDKpcyedTkcXRYyPj+uSc2BXclkul5HP53XP61ar1aVe4MEIMo/ChCZxt9HDgRhd86TvRYjLgcnJtJ/xlaV4MovLCjdpiKUmc1gRDAYxNjaGzc1N7bmTo+oVVt/NJmaekDz0ktzdnaKJh4nbbaoyDJT/J8wJUz57yX2bw22pnmFXLql4yeVy+M53voMf/ehHOoRk+89UKoX5+Xm4XC5dnkw1xZNPPonR0VGcPHkSo6OjD/u2dYH0CMNcUlI8kt5utyMajWqjIyFVC3JTl5HQIHS6CwsLWtdKvpYeJw2b3W7XfbJlxCufO8N7Rk18do1GA/l8HgB0BLm9vY1araa1t+x+l0gk4PF4kEqlUCwWdV8QHp/Fv0WZGhNzPp8PiURCR5c+nw+HDx/Wp1TfCQ+FXjB7GAQX0n78R68kmvR+ZPZa/h3pGQ0rKKMjn8SN5EF5Quk5c5GaDe2wUjBmtQFwa7Uify69NLMH1KtrFucEJT+VSgXpdLpL1hMKhRAOh/Xx8JVKBT6fTyf2GIoyWcVTh/sFM+VCbal8xjQ8Zk/XvNbkPRxklzHq0ql7Jsfv9Xq7ju6SZ+mx9wGw503yOkk9kJ+V3qbL5UKn09FSOaoQtra2tFooEAigWCxqnXu1WoXNZuvqicuGXaTvuJlzTD6fD/F4HJFI5K4izIdidOn69ypjNXuk5Jh4oziBpH5XGlvJASq1d3oCubhhpRzC4TAOHz6MxcVFnfhgVtYcNt8L5EYlj7DhZN0vOTUosDwT2DtVxGxEzZuEnAP0+BhCc+7wZ3IDZkTEkzl4eKBcyC6XS9fUHzt2DIcOHYLH49Gc6alTpxCJRLoKN/oJMxVHQ3m7zbqXkyIpBSqAOFf6iZdeegmFQkFXim1sbOgDDwBo6SCb1Ehu1xy+08lgiE97wRLfra0tPXecTieOHz+OyclJfPnLX0YwGNQnJXONUG1D73m/vBQrYfkzp9OpO5PdTSe6A5OMScjqq15eltmLkeCkkhyUWZ8pP5M33szDDZvXyywpOxVR5C6N4oOOWTbylp7NsBhdGg2Xy9XTIzdn281Rzt1cR68NnoU5XBjUB/P9TCwxrKRUCQCSySRCodDQaMF7rZ3bUTjm90hvchC9Fx5//HHk83m89957qNfrWF5e1mE7sMf5s7SfBlB2TJPzm+1LpfFVardnxrVr19But7Ue+8knn0QymcTTTz+NRCKBeDyuE2oyWUnjy8+Szsx+TbTuZQ0fqNHld4rJAXQdOiiTIvRwmemTfBzfI39HTg7+PkOGSqWihcrDCp4GzPJBRgH3k/QyP2Du8KzO45FFNPCDPqpIgp4uT/6QnovcaCXkQiN6RUzlchmGYdzSwJ7ziOGhVJEwtA0EAigUCnjttddQrVaxvLyMQCCA2dlZ7en2GwyZ2Z9EPndKplgc0csIyGtkkom/6/f7EY1GB3JdPp8PL7zwAn77t38bX/ziF1GpVLC8vKw1t4VCAfl8HoVCQRcw8JQaUgBmW0KD6fF4EI/HcejQIfzO7/wOgsEgDh06pAt2fD4fkskkvF6vViWYaRaPx9N1r2UkLosvZCcynj7NBulTU1P7Xv9D0ely1yHpLHcmsxLB7MabvWO50GS3JC5QKdfoNZZhAZtkULAvPT1q/x6E3yWvxQZA5I8Hdfjgfs+Aoa25MXcv7t6cUJONyntx+LVarYsf5PukMZeyM75mt+8e01OtVvHhhx9ie3sbH330EY4ePYpz584hFosNZONi1ZV8fkwY0ljczabK65aHCXCDHgRcLhcef/xxAHvVlBcvXsTW1hbsdjvW19e1c5LNZnXpf6fTwc7ODjKZjP4szifOfY/Hg0AggPHxcXz5y19GIpHA448/flcOGefJnfJDNLpc0+wFoZTSSc2+GF0JVtLQwEguRu4WvABgb+ExU8v3y8UjFydRLpeRzWbh9Xr7nui4F7Bt4NTUFI4fP67DJbMhIHoll3pB3g/yWUwMUfA9qAbcvSA93V4hmfnIk06no1vtsUcDPRFu5uSGjx07BpfLhXw+r3lfs6diVjswU+7z+dBqtbC2toZgMIivfvWrGB8fx8jISBd/188N3ev1Ih6Pw+/394wU+bzZt1pCrjnz5jVMTglVC0ePHsXY2BjGx8e1UoDUAakepRRKpRIKhYL+fVaJyT4jPFI9mUzC5XJhaWkJwF7pdK8x8LOYD+l1GCrRK49ADjqRSNxxE3woRpct9NjchBMduNWr65U0kYmVXuS59AhrtRoKhYIWrsu/NUzgppBMJnHo0CEtGZPSuv2IezP2WzT0oNmOUB5NMyxgtVqvEnHzddnt9q5a92q1qvk9nuBLr8Nms2FiYgLBYBAffvihfi8NqyyNBrpPT2Co3m7vHqwZiUTw3HPPIRKJaM/lQZKd9wuPx4NoNHqL0ZUqBHO1FiENg4RcV8MAru+JiQkAuOUQVRZM7OzsoFAoaPkWAC0h9fl8mjKR6gZq4dfX1/XRRr1yA5TVMTJkMYmUpLIwA9hLpNHLdjqdustZPB4fjNHlxZKvZCgk+TrZ3EROFt6wXtlKuUjJ2bGya3x8/JbPGCaMjIzg5MmTCIfDOHbsGObn5/Huu+9qrpEVL9Lbl9gvgSKTi16vF+FwGF/5yldw7NgxjI6OIh6P4+TJk3291v1AzisajSKTyXRJgYDuk0HYTtHr9WJ2drZrDlBEz0Ym8/PzaLfbSCaTiMViWFxcRKFQuMXA8N4SPGLl2LFj+MY3vqFVAbFYTFctSXlSvz1Ej8eDkZERnWE3J5Clp2vmdXuVLcsE2jB5u7cDvU+ldnXXdCxk4o1GkwoTGc2022243W69QZkjZWAvipYH6kren58jbROVM/Jv3y2V91CMLi9QGl3J6wL79/mUFybDQ/MOxYut1+v65NlhBs+wOnHiBADgH//xHzE/P49KpdLVSep+1AZcgG63G+FwGF/60pfw4osv6vsdDocfxiXdF3w+X8/ElDl8a7VaKJVKCIVCmJ2d1Vnmer2OlZUV2O12HTGkUinUajUkEgkkEgl9LJKcQ6Ry5HHm7Ok7MjKCF198cWgUCoTX69VGt5cahUaX9IqEWS7ItXa3VVPDAilxu1+MjIwc4IgeHA9sdKWnygddq9VQqVT0A5Y7Bt9P4yrDvttJMaSrL9/Lnb5XL8uDkmI9DEiukjDrUYleEYGEPM68Xq8jHA4jFArpz5BdvAYNt9utjSKfZ68EIr0INqQhDdBoNHTfBIaAtVoNzWYTly9fht/vRzqd1jJCuXET0nMhFzeMYMhsVv84HA4EAgHdoMfMVdIrlMf38PVBeOwWunGgR7BTSVCtVnVXHi54s9HlopNHy/B18wLpFXKbW7+ZjxkfBAd3L2DdtjwxYr+yWF6H5LqBvc1E3mMeQshmJ8MGtlmUMihz9CO5fB5cSePRaDSQyWTQ6ez25gWgN6+3334bnU5Hc3z8TIaCBCMxh8Oh2/sNIxiySqNLlUooFILP59P8o9noer1ezZ3zNfOXhcHgQD1dHjjIA/wkD2eWgdGI0tCQfDYbcrPx4XeZiJNJBRk+DWNCjeA1mA0p0K2/vZfP63UOW6/P7yfMBlUaXXLZsjELqSS+V54+InXN0oOjEoTePTlL6enKecr5wq5ng5JO3Q14HeSXqbeVyaJe3K3su2CmWoB7m1sWDhYH4ulyQlO+xSa+Zl2t/D1ZsSYNZK8kkdnwSm6LlAYpBmai5WcMIyRVIo0OcPd0iPm+8OwvWW3FMHpQfKX5mkKhkJbyUOpkfv6dTkdXAXU6HRSLRf2MmYyjtAfYq7LiuWbcePg6m8Ywt0AuNBaLYXJyEuFweGjnCbBHMzCzTqqFzkuvUnveP2l493NsLPQXB+rpUq5E4b+5DaPZm5Nid/PnyWwhIfW9+/F0w1R9dSdw/JSj9CoiAfZ6Cpvvo/RaeP37NRoaFKRBVWpXPM6DHynxkdQQDaJ50yUVxeSp+dkrpbSek4aIHi/nIA0vjXU8Hsfs7CzGxsb6eEfuHZQzMQropdM1b8Cy1NmsBHqUEmkfRxyohapWq7qjExeLlOsQfPC9tJnS2MikG40paYleBqdXr4dh5XXlZsRkIr1RLhbCrOOVCg8AXdfey+sZNOQYxsfHkUwmMT09jZGREa2/5PWVSiXk8/mu+cNIQD5bczJW0ijmzTocDuuzrPiZSinMzMzgK1/5CpLJ5L7zdBjgdDoRDAb13Cd9xMgmGAzeYnTZZFteLwsRBlWlaGEXB2p0zTSC9MjoxdET6+WtSdDQMBy8XY8CufubaYxhWThmUJfLMJHoFfqZvXrp7Uuws/2gT3i9HfhMjxw5gmeffRbpdFonxqhEKJVKPZUuZqpJcrTSmzZXL5qTZUzwnjhxAiMjIwgEAvpng+bAbweuH7mWZPWUhJn3JWTzFguDwYEbXYZucjclp8iQR4bJ9O74e1KMbv5uDo3oLXPiyeqrYeetKHwnHWPOMkuYCwh6VZjZ7XaMjIxgenoawWBQv242VP1GL+OllMILL7yAz3zmM1hcXMTS0pKuGJIbs7njmLkJkvm7pGPkZs25yMKAZDKJRCKBiYkJHD58+BYvd9g2a1Ir5v4J7XZbV2yZNyNZZUf6QSmlN+Z+H9NjYQ8PZHTNhrFWq3UdfcLyOclH7mdUpdHhd/N7CLnIZNu/+yksGDSkisHcYU02dzFDblxSrWD+7GG9H+zwNDo62nUUuzR85sjJbISJXkaX/CfnHL942gLbOMoNrVfuYRhAeoH9THhNvG+9egqYVTx8X6/Em4X+4kCMLrA78bPZLG7cuKHPGOKikYkAszzIzE9KIywNr5mz5KThZGLybpgnkwyB6b2Ql5OLn5677A9K2Gw23VWJYGGIbBk57GD/hJmZGRw6dOi+Nojbvd+cwJOQRtiMYQy7Q6EQjhw5grW1NaRSKSil4Pf7US6XUSwW9fE9BNeZVCuUy2U0Gg09d4apCdInDQ9ML0hvlO3VmPyQ2lmGfLJ5iIT5//JIEeBWHpOLgzrPYSvh3A8cv9frRSKRQKVSQTKZ1AsJ2Mu+V6tVreKQhoNyPL5O7Sa5y0dFwUF66VF5doOC1+tFMplEo9HQR8LwQEaeimFWBvl8PoTDYe3V8xgiHiszrFV4nwQc6OocHx/HmTNndJUMM6j06KSnSs+VoZGZt+vFa1LFwGQAX2P1jXniDZvXK8d3+PBhvPjii1haWsLExASUUjrDTp0pm3JLyJCbBpYbz6c+9SlMT0/rpJE5irDwaGJiYgJf+MIXsLa2pk8kdrvdGB0dxYkTJxCJRLo2WpfLhWPHjmFkZATPPvsspqamkEwm4ff7MTo6ilAohOnp6UFdziceB2p0ZWMKKYcC9rSoNBr7ScVu13tAZpZJTZiPHTEb3mEF28EFg0F98qjX69VaY0YIZspE3iPyopQBUQ1hGdmPFxwOB3w+H3w+H/x+v6YP6OX28nRZPszfY+vBYDCIQCDwyERDH0eoYfMGLViwYOHjDMslsmDBgoU+wjK6FixYsNBHWEbXggULFvoIy+hasGDBQh9hGV0LFixY6CMso2vBggULfcT/B4LubNXYkujhAAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for X, y in train_set.take(1):\n",
    "    for i in range(5):\n",
    "        plt.subplot(1, 5, i + 1)\n",
    "        plt.imshow(X[i].numpy(), cmap=\"binary\")\n",
    "        plt.axis(\"off\")\n",
    "        plt.title(str(y[i].numpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "class Standardization(tf.keras.layers.Layer):\n",
    "    def adapt(self, data_sample):\n",
    "        self.means_ = np.mean(data_sample, axis=0, keepdims=True)\n",
    "        self.stds_ = np.std(data_sample, axis=0, keepdims=True)\n",
    "    def call(self, inputs):\n",
    "        return (inputs - self.means_) / (self.stds_ + tf.keras.backend.epsilon())\n",
    "\n",
    "standardization = Standardization(input_shape=[28, 28])\n",
    "# or perhaps soon:\n",
    "# standardization = tf.keras.layers.Normalization()\n",
    "\n",
    "sample_image_batches = train_set.take(100).map(lambda image, label: image)\n",
    "\n",
    "sample_images = np.concatenate(\n",
    "    list(sample_image_batches.as_numpy_iterator()),\n",
    "    axis=0\n",
    ").astype(np.float32)\n",
    "\n",
    "standardization.adapt(sample_images)\n",
    "\n",
    "model = tf.keras.models.Sequential([\n",
    "    standardization,\n",
    "    tf.keras.layers.Flatten(),\n",
    "    tf.keras.layers.Dense(100, activation=\"relu\"),\n",
    "    tf.keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "\n",
    "model.compile(\n",
    "    loss=\"sparse_categorical_crossentropy\",\n",
    "    optimizer=\"nadam\",\n",
    "    metrics=[\"accuracy\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n"
     ]
    },
    {
     "ename": "InternalError",
     "evalue": " Blas GEMM launch failed : a.shape=(32, 784), b.shape=(784, 100), m=32, n=100, k=784\n\t [[node sequential/dense/MatMul (defined at <ipython-input-32-728c76230b32>:14) ]] [Op:__inference_train_function_426304]\n\nFunction call stack:\ntrain_function\n",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mInternalError\u001B[0m                             Traceback (most recent call last)",
      "\u001B[1;32m<ipython-input-32-728c76230b32>\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m     12\u001B[0m )\n\u001B[0;32m     13\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 14\u001B[1;33m model.fit(\n\u001B[0m\u001B[0;32m     15\u001B[0m     \u001B[0mtrain_set\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     16\u001B[0m     \u001B[0mepochs\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;36m5\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\anaconda3\\envs\\dads\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001B[0m in \u001B[0;36m_method_wrapper\u001B[1;34m(self, *args, **kwargs)\u001B[0m\n\u001B[0;32m    106\u001B[0m   \u001B[1;32mdef\u001B[0m \u001B[0m_method_wrapper\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;33m*\u001B[0m\u001B[0margs\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;33m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    107\u001B[0m     \u001B[1;32mif\u001B[0m \u001B[1;32mnot\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_in_multi_worker_mode\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m  \u001B[1;31m# pylint: disable=protected-access\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 108\u001B[1;33m       \u001B[1;32mreturn\u001B[0m \u001B[0mmethod\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;33m*\u001B[0m\u001B[0margs\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;33m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    109\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    110\u001B[0m     \u001B[1;31m# Running inside `run_distribute_coordinator` already.\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\anaconda3\\envs\\dads\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001B[0m in \u001B[0;36mfit\u001B[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001B[0m\n\u001B[0;32m   1096\u001B[0m                 batch_size=batch_size):\n\u001B[0;32m   1097\u001B[0m               \u001B[0mcallbacks\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mon_train_batch_begin\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mstep\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 1098\u001B[1;33m               \u001B[0mtmp_logs\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mtrain_function\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0miterator\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   1099\u001B[0m               \u001B[1;32mif\u001B[0m \u001B[0mdata_handler\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mshould_sync\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1100\u001B[0m                 \u001B[0mcontext\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0masync_wait\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\anaconda3\\envs\\dads\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001B[0m in \u001B[0;36m__call__\u001B[1;34m(self, *args, **kwds)\u001B[0m\n\u001B[0;32m    778\u001B[0m       \u001B[1;32melse\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    779\u001B[0m         \u001B[0mcompiler\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;34m\"nonXla\"\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 780\u001B[1;33m         \u001B[0mresult\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_call\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m*\u001B[0m\u001B[0margs\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;33m**\u001B[0m\u001B[0mkwds\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    781\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    782\u001B[0m       \u001B[0mnew_tracing_count\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_get_tracing_count\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\anaconda3\\envs\\dads\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001B[0m in \u001B[0;36m_call\u001B[1;34m(self, *args, **kwds)\u001B[0m\n\u001B[0;32m    838\u001B[0m         \u001B[1;31m# Lifting succeeded, so variables are initialized and we can run the\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    839\u001B[0m         \u001B[1;31m# stateless function.\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 840\u001B[1;33m         \u001B[1;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_stateless_fn\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m*\u001B[0m\u001B[0margs\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;33m**\u001B[0m\u001B[0mkwds\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    841\u001B[0m     \u001B[1;32melse\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    842\u001B[0m       \u001B[0mcanon_args\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mcanon_kwds\u001B[0m \u001B[1;33m=\u001B[0m\u001B[0;31m \u001B[0m\u001B[0;31m\\\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\anaconda3\\envs\\dads\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001B[0m in \u001B[0;36m__call__\u001B[1;34m(self, *args, **kwargs)\u001B[0m\n\u001B[0;32m   2827\u001B[0m     \u001B[1;32mwith\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_lock\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   2828\u001B[0m       \u001B[0mgraph_function\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0margs\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mkwargs\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_maybe_define_function\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0margs\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mkwargs\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 2829\u001B[1;33m     \u001B[1;32mreturn\u001B[0m \u001B[0mgraph_function\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_filtered_call\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0margs\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mkwargs\u001B[0m\u001B[1;33m)\u001B[0m  \u001B[1;31m# pylint: disable=protected-access\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   2830\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   2831\u001B[0m   \u001B[1;33m@\u001B[0m\u001B[0mproperty\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\anaconda3\\envs\\dads\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001B[0m in \u001B[0;36m_filtered_call\u001B[1;34m(self, args, kwargs, cancellation_manager)\u001B[0m\n\u001B[0;32m   1841\u001B[0m       \u001B[0;31m`\u001B[0m\u001B[0margs\u001B[0m\u001B[0;31m`\u001B[0m \u001B[1;32mand\u001B[0m\u001B[0;31m \u001B[0m\u001B[0;31m`\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;31m`\u001B[0m\u001B[1;33m.\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1842\u001B[0m     \"\"\"\n\u001B[1;32m-> 1843\u001B[1;33m     return self._call_flat(\n\u001B[0m\u001B[0;32m   1844\u001B[0m         [t for t in nest.flatten((args, kwargs), expand_composites=True)\n\u001B[0;32m   1845\u001B[0m          if isinstance(t, (ops.Tensor,\n",
      "\u001B[1;32m~\\anaconda3\\envs\\dads\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001B[0m in \u001B[0;36m_call_flat\u001B[1;34m(self, args, captured_inputs, cancellation_manager)\u001B[0m\n\u001B[0;32m   1921\u001B[0m         and executing_eagerly):\n\u001B[0;32m   1922\u001B[0m       \u001B[1;31m# No tape is watching; skip to running the function.\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 1923\u001B[1;33m       return self._build_call_outputs(self._inference_function.call(\n\u001B[0m\u001B[0;32m   1924\u001B[0m           ctx, args, cancellation_manager=cancellation_manager))\n\u001B[0;32m   1925\u001B[0m     forward_backward = self._select_forward_and_backward_functions(\n",
      "\u001B[1;32m~\\anaconda3\\envs\\dads\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001B[0m in \u001B[0;36mcall\u001B[1;34m(self, ctx, args, cancellation_manager)\u001B[0m\n\u001B[0;32m    543\u001B[0m       \u001B[1;32mwith\u001B[0m \u001B[0m_InterpolateFunctionError\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    544\u001B[0m         \u001B[1;32mif\u001B[0m \u001B[0mcancellation_manager\u001B[0m \u001B[1;32mis\u001B[0m \u001B[1;32mNone\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 545\u001B[1;33m           outputs = execute.execute(\n\u001B[0m\u001B[0;32m    546\u001B[0m               \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0msignature\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mname\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    547\u001B[0m               \u001B[0mnum_outputs\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_num_outputs\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\anaconda3\\envs\\dads\\lib\\site-packages\\tensorflow\\python\\eager\\execute.py\u001B[0m in \u001B[0;36mquick_execute\u001B[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001B[0m\n\u001B[0;32m     57\u001B[0m   \u001B[1;32mtry\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     58\u001B[0m     \u001B[0mctx\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mensure_initialized\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 59\u001B[1;33m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001B[0m\u001B[0;32m     60\u001B[0m                                         inputs, attrs, num_outputs)\n\u001B[0;32m     61\u001B[0m   \u001B[1;32mexcept\u001B[0m \u001B[0mcore\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_NotOkStatusException\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0me\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mInternalError\u001B[0m:  Blas GEMM launch failed : a.shape=(32, 784), b.shape=(784, 100), m=32, n=100, k=784\n\t [[node sequential/dense/MatMul (defined at <ipython-input-32-728c76230b32>:14) ]] [Op:__inference_train_function_426304]\n\nFunction call stack:\ntrain_function\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "logs = os.path.join(\n",
    "    os.curdir, \"my_logs\",\n",
    "    \"run_\" + datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
    ")\n",
    "\n",
    "tensorboard_cb = tf.keras.callbacks.TensorBoard(\n",
    "    log_dir=logs,\n",
    "    histogram_freq=1,\n",
    "    profile_batch=10\n",
    ")\n",
    "\n",
    "model.fit(\n",
    "    train_set,\n",
    "    epochs=5,\n",
    "    validation_data=valid_set,\n",
    "    callbacks=[tensorboard_cb]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning:** The profiling tab in TensorBoard works if you use TensorFlow 2.2+. You also need to make\n",
    "sure `tensorboard_plugin_profile` is installed (and restart Jupyter if necessary)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Launch tensorboard from terminal\n",
    "\n",
    "# %load_ext tensorboard\n",
    "# %tensorboard --logdir=./my_logs --port=6006"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10.\n",
    "_Exercise: In this exercise you will download a dataset, split it, create a `tf.data.Dataset` to load it\n",
    "and preprocess it efficiently, then build and train a binary classification model containing an `Embedding` layer._\n",
    "\n",
    "### a.\n",
    "_Exercise: Download the [Large Movie Review Dataset](https://homl.info/imdb), which contains 50,000 movies reviews\n",
    "from the [Internet Movie Database](https://imdb.com/). The data is organized in two directories,\n",
    "`train` and `test`, each containing a `pos` subdirectory with 12,500 positive reviews and a `neg` subdirectory\n",
    "with 12,500 negative reviews. Each review is stored in a separate text file. There are other files and\n",
    "folders (including preprocessed bag-of-words), but we will ignore them in this exercise._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "DOWNLOAD_ROOT = \"http://ai.stanford.edu/~amaas/data/sentiment/\"\n",
    "FILENAME = \"aclImdb_v1.tar.gz\"\n",
    "filepath = tf.keras.utils.get_file(FILENAME, DOWNLOAD_ROOT + FILENAME, extract=True)\n",
    "path = Path(filepath).parent / \"aclImdb\"\n",
    "path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for name, subdirs, files in os.walk(path):\n",
    "    indent = len(Path(name).parts) - len(path.parts)\n",
    "    print(\"    \" * indent + Path(name).parts[-1] + os.sep)\n",
    "    for index, filename in enumerate(sorted(files)):\n",
    "        if index == 3:\n",
    "            print(\"    \" * (indent + 1) + \"...\")\n",
    "            break\n",
    "        print(\"    \" * (indent + 1) + filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def review_paths(dirpath):\n",
    "    return [str(path) for path in dirpath.glob(\"*.txt\")]\n",
    "\n",
    "train_pos = review_paths(path / \"train\" / \"pos\")\n",
    "train_neg = review_paths(path / \"train\" / \"neg\")\n",
    "test_valid_pos = review_paths(path / \"test\" / \"pos\")\n",
    "test_valid_neg = review_paths(path / \"test\" / \"neg\")\n",
    "\n",
    "len(train_pos), len(train_neg), len(test_valid_pos), len(test_valid_neg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b.\n",
    "_Exercise: Split the test set into a validation set (15,000) and a test set (10,000)._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "np.random.shuffle(test_valid_pos)\n",
    "\n",
    "test_pos = test_valid_pos[:5000]\n",
    "test_neg = test_valid_neg[:5000]\n",
    "\n",
    "valid_pos = test_valid_pos[5000:]\n",
    "valid_neg = test_valid_neg[5000:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c.\n",
    "_Exercise: Use tf.data to create an efficient dataset for each set._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since the dataset fits in memory, we can just load all the data using pure Python code and use\n",
    "`tf.data.Dataset.from_tensor_slices()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def imdb_dataset(filepaths_positive, filepaths_negative):\n",
    "    reviews = []\n",
    "    labels = []\n",
    "    for filepaths, label in ((filepaths_negative, 0), (filepaths_positive, 1)):\n",
    "        for filepath in filepaths:\n",
    "            with open(filepath, encoding='utf8') as review_file:\n",
    "                reviews.append(review_file.read())\n",
    "            labels.append(label)\n",
    "    return tf.data.Dataset.from_tensor_slices((tf.constant(reviews), tf.constant(labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for X, y in imdb_dataset(train_pos, train_neg).take(3):\n",
    "    print(X)\n",
    "    print(y)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "%timeit -r1 for X, y in imdb_dataset(train_pos, train_neg).repeat(10): pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It takes about 20 seconds to load the dataset and go through it 10 times."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "But let's pretend the dataset does not fit in memory, just to make things more interesting. Luckily, each\n",
    "review fits on just one line (they use `<br />` to indicate line breaks), so we can read the reviews\n",
    "using a `TextLineDataset`. If they didn't we would have to preprocess the input files (e.g., converting\n",
    "them to TFRecords). For very large datasets, it would make sense a tool like Apache Beam for that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def imdb_dataset(filepaths_positive, filepaths_negative, n_read_threads=5):\n",
    "\n",
    "    dataset_neg = tf.data.TextLineDataset(\n",
    "        filepaths_negative,\n",
    "        num_parallel_reads=n_read_threads\n",
    "    )\n",
    "\n",
    "    dataset_neg = dataset_neg.map(lambda review: (review, 0))\n",
    "\n",
    "    dataset_pos = tf.data.TextLineDataset(\n",
    "        filepaths_positive,\n",
    "        num_parallel_reads=n_read_threads\n",
    "    )\n",
    "\n",
    "    dataset_pos = dataset_pos.map(lambda review: (review, 1))\n",
    "    return tf.data.Dataset.concatenate(dataset_pos, dataset_neg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "%timeit -r1 for X, y in imdb_dataset(train_pos, train_neg).repeat(10): pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now it takes about 34 seconds to go through the dataset 10 times. That's much slower, essentially because the\n",
    "dataset is not cached in RAM, so it must be reloaded at each epoch. If you add `.cache()` just before\n",
    "`.repeat(10)`, you will see that this implementation will be about as fast as the previous one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "%timeit -r1 for X, y in imdb_dataset(train_pos, train_neg).cache().repeat(10): pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "\n",
    "train_set = imdb_dataset(train_pos, train_neg).shuffle(25000).batch(batch_size).prefetch(1)\n",
    "valid_set = imdb_dataset(valid_pos, valid_neg).batch(batch_size).prefetch(1)\n",
    "test_set = imdb_dataset(test_pos, test_neg).batch(batch_size).prefetch(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d.\n",
    "_Exercise: Create a binary classification model, using a `TextVectorization` layer to preprocess each review.\n",
    "If the `TextVectorization` layer is not yet available (or if you like a challenge), try to create your own\n",
    "custom preprocessing layer: you can use the functions in the `tf.strings` package, for example `lower()` to\n",
    "make everything lowercase, `regex_replace()` to replace punctuation with spaces, and `split()` to split words\n",
    "on spaces. You should use a lookup table to output word indices, which must be prepared in the `adapt()` method._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's first write a function to preprocess the reviews, cropping them to 300 characters, converting them to\n",
    "lower case, then replacing `<br />` and all non-letter characters to spaces, splitting the reviews into words,\n",
    "and finally padding or cropping each review so it ends up with exactly `n_words` tokens:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def preprocess(X_batch, n_words=50):\n",
    "    shape = tf.shape(X_batch) * tf.constant([1, 0]) + tf.constant([0, n_words])\n",
    "    Z = tf.strings.substr(X_batch, 0, 300)\n",
    "    Z = tf.strings.lower(Z)\n",
    "    Z = tf.strings.regex_replace(Z, b\"<br\\\\s*/?>\", b\" \")\n",
    "    Z = tf.strings.regex_replace(Z, b\"[^a-z]\", b\" \")\n",
    "    Z = tf.strings.split(Z)\n",
    "    return Z.to_tensor(shape=shape, default_value=b\"<pad>\")\n",
    "\n",
    "X_example = tf.constant([\"It's a great, great movie! I loved it.\", \"It was terrible, run away!!!\"])\n",
    "preprocess(X_example)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's write a second utility function that will take a data sample with the same format as the\n",
    "output of the `preprocess()` function, and will output the list of the top `max_size` most frequent words,\n",
    "ensuring that the padding token is first:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "def get_vocabulary(data_sample, max_size=1000):\n",
    "    preprocessed_reviews = preprocess(data_sample).numpy()\n",
    "    counter = Counter()\n",
    "    for words in preprocessed_reviews:\n",
    "        for word in words:\n",
    "            if word != b\"<pad>\":\n",
    "                counter[word] += 1\n",
    "    return [b\"<pad>\"] + [word for word, count in counter.most_common(max_size)]\n",
    "\n",
    "get_vocabulary(X_example)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we are ready to create the `TextVectorization` layer. Its constructor just saves the\n",
    "hyperparameters (`max_vocabulary_size` and `n_oov_buckets`). The `adapt()` method computes the vocabulary using\n",
    "the `get_vocabulary()` function, then it builds a `StaticVocabularyTable` (see Chapter 16 for more details).\n",
    "The `call()` method preprocesses the reviews to get a padded list of words for each review, then it\n",
    "uses the `StaticVocabularyTable` to lookup the index of each word in the vocabulary:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class TextVectorization(tf.keras.layers.Layer):\n",
    "    def __init__(self, max_vocabulary_size=1000, n_oov_buckets=100, dtype=tf.string, **kwargs):\n",
    "        super().__init__(dtype=dtype, **kwargs)\n",
    "        self.max_vocabulary_size = max_vocabulary_size\n",
    "        self.n_oov_buckets = n_oov_buckets\n",
    "\n",
    "    def adapt(self, data_sample):\n",
    "        self.vocab = get_vocabulary(data_sample, self.max_vocabulary_size)\n",
    "        words = tf.constant(self.vocab)\n",
    "        word_ids = tf.range(len(self.vocab), dtype=tf.int64)\n",
    "        vocab_init = tf.lookup.KeyValueTensorInitializer(words, word_ids)\n",
    "        self.table = tf.lookup.StaticVocabularyTable(vocab_init, self.n_oov_buckets)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        preprocessed_inputs = preprocess(inputs)\n",
    "        return self.table.lookup(preprocessed_inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's try it on our small `X_example` we defined earlier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "text_vectorization = TextVectorization()\n",
    "\n",
    "text_vectorization.adapt(X_example)\n",
    "text_vectorization(X_example)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks good! As you can see, each review was cleaned up and tokenized, then each word was encoded as its index\n",
    "in the vocabulary (all the 0s correspond to the `<pad>` tokens).\n",
    "\n",
    "Now let's create another `TextVectorization` layer and let's adapt it to the full IMDB training set (if the\n",
    "training set did not fit in RAM, we could just use a smaller sample of the training set by\n",
    "calling `train_set.take(500)`):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "max_vocabulary_size = 1000\n",
    "n_oov_buckets = 100\n",
    "\n",
    "sample_review_batches = train_set.map(lambda review, label: review)\n",
    "\n",
    "sample_reviews = np.concatenate(\n",
    "    list(sample_review_batches.as_numpy_iterator()),\n",
    "    axis=0\n",
    ")\n",
    "\n",
    "text_vectorization = TextVectorization(\n",
    "    max_vocabulary_size,\n",
    "    n_oov_buckets,\n",
    "    input_shape=[]\n",
    ")\n",
    "\n",
    "text_vectorization.adapt(sample_reviews)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's run it on the same `X_example`, just to make sure the word IDs are larger now, since the vocabulary bigger:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "text_vectorization(X_example)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Good! Now let's take a look at the first 10 words in the vocabulary:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "text_vectorization.vocab[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These are the most common words in the reviews."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now to build our model we will need to encode all these word IDs somehow. One approach is to create bags of\n",
    "words: for each review, and for each word in the vocabulary, we count the number of occurences of that\n",
    "word in the review. For example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "simple_example = tf.constant([[1, 3, 1, 0, 0], [2, 2, 0, 0, 0]])\n",
    "tf.reduce_sum(tf.one_hot(simple_example, 4), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The first review has 2 times the word 0, 2 times the word 1, 0 times the word 2, and 1 time the word 3, so\n",
    "its bag-of-words representation is `[2, 2, 0, 1]`. Similarly, the second review has 3 times the word 0, 0 times\n",
    "the word 1, and so on. Let's wrap this logic in a small custom layer, and let's test it. We'll drop the counts\n",
    "for the word 0, since this corresponds to the `<pad>` token, which we don't care about."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class BagOfWords(tf.keras.layers.Layer):\n",
    "    def __init__(self, n_tokens, dtype=tf.int32, **kwargs):\n",
    "        super().__init__(dtype=tf.int32, **kwargs)\n",
    "        self.n_tokens = n_tokens\n",
    "\n",
    "    def call(self, inputs):\n",
    "        one_hot = tf.one_hot(inputs, self.n_tokens)\n",
    "        return tf.reduce_sum(one_hot, axis=1)[:, 1:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's test it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "bag_of_words = BagOfWords(n_tokens=4)\n",
    "bag_of_words(simple_example)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It works fine! Now let's create another `BagOfWord` with the right vocabulary size for our training set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "n_tokens = max_vocabulary_size + n_oov_buckets + 1 # add 1 for <pad>\n",
    "bag_of_words = BagOfWords(n_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We're ready to train the model!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "model = tf.keras.models.Sequential([\n",
    "    text_vectorization,\n",
    "    bag_of_words,\n",
    "    tf.keras.layers.Dense(100, activation=\"relu\"),\n",
    "    tf.keras.layers.Dense(1, activation=\"sigmoid\"),\n",
    "])\n",
    "\n",
    "model.compile(\n",
    "    loss=\"binary_crossentropy\",\n",
    "    optimizer=\"nadam\",\n",
    "    metrics=[\"accuracy\"]\n",
    ")\n",
    "\n",
    "model.fit(\n",
    "    train_set,\n",
    "    epochs=5,\n",
    "    validation_data=valid_set\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We get about 75% accuracy on the validation set after just the first epoch, but after that the model makes no\n",
    "progress. We will do better in Chapter 16. For now the point is just to perform efficient preprocessing\n",
    "using `tf.data` and Keras preprocessing layers."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### e.\n",
    "_Exercise: Add an `Embedding` layer and compute the mean embedding for each review, multiplied by the square\n",
    "root of the number of words (see Chapter 16). This rescaled mean embedding can then be passed to the\n",
    "rest of your model._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To compute the mean embedding for each review, and multiply it by the square root of the number of words in\n",
    "that review, we will need a little function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def compute_mean_embedding(inputs):\n",
    "    not_pad = tf.math.count_nonzero(inputs, axis=-1)\n",
    "    n_words = tf.math.count_nonzero(not_pad, axis=-1, keepdims=True)\n",
    "    sqrt_n_words = tf.math.sqrt(tf.cast(n_words, tf.float32))\n",
    "    return tf.reduce_mean(inputs, axis=1) * sqrt_n_words\n",
    "\n",
    "another_example = tf.constant([\n",
    "    [[1., 2., 3.], [4., 5., 0.], [0., 0., 0.]],\n",
    "    [[6., 0., 0.], [0., 0., 0.], [0., 0., 0.]]\n",
    "])\n",
    "\n",
    "compute_mean_embedding(another_example)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check that this is correct. The first review contains 2 words (the last token is a zero vector, which\n",
    "represents the `<pad>` token). The second review contains 1 word. So we need to compute the mean embedding\n",
    "for each review, and multiply the first one by the square root of 2, and the second one by the square root of 1:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "tf.reduce_mean(another_example, axis=1) * tf.sqrt([[2.], [1.]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Perfect. Now we're ready to train our final model. It's the same as before, except we replaced the\n",
    "`BagOfWords` layer with an `Embedding` layer followed by a `Lambda` layer that calls\n",
    "the `compute_mean_embedding` layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "embedding_size = 20\n",
    "\n",
    "model = tf.keras.models.Sequential([\n",
    "    text_vectorization,\n",
    "    tf.keras.layers.Embedding(\n",
    "        input_dim=n_tokens,\n",
    "        output_dim=embedding_size,\n",
    "        mask_zero=True), # <pad> tokens => zero vectors\n",
    "    tf.keras.layers.Lambda(compute_mean_embedding),\n",
    "    tf.keras.layers.Dense(100, activation=\"relu\"),\n",
    "    tf.keras.layers.Dense(1, activation=\"sigmoid\"),\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### f.\n",
    "_Exercise: Train the model and see what accuracy you get. Try to optimize your pipelines to make training as\n",
    "fast as possible._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "model.compile(loss=\"binary_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\n",
    "model.fit(train_set, epochs=5, validation_data=valid_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model is not better using embeddings (but we will do better in Chapter 16). The pipeline looks fast enough\n",
    "(we optimized it earlier)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### g.\n",
    "_Exercise: Use TFDS to load the same dataset more easily: `tfds.load(\"imdb_reviews\")`._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "\n",
    "datasets = tfds.load(name=\"imdb_reviews\")\n",
    "train_set, test_set = datasets[\"train\"], datasets[\"test\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for example in train_set.take(1):\n",
    "    print(example[\"text\"])\n",
    "    print(example[\"label\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}