colab_notebooks/models_evaluator.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"models_evaluator.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"DUpN56XCZL5D","executionInfo":{"status":"ok","timestamp":1622045190905,"user_tz":-120,"elapsed":422,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}},"outputId":"cddf1aef-76c1-4f80-e129-7dbe2bf192e3"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"nNMG-cNaZ0z6","executionInfo":{"status":"ok","timestamp":1622045199694,"user_tz":-120,"elapsed":8413,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}},"outputId":"30d6dcfe-00d0-46a7-b4dc-5d82b467ad5c"},"source":["!mkdir \"/content/images\"\n","!mkdir \"/content/images/test\"\n","!unrar x \"/content/drive/Shareddrives/Intercambio/MATERIAS/Deep Learning/Food Recognition/images/test/test.rar\" \"/content/images/test\"\n","!cp \"/content/drive/Shareddrives/Intercambio/MATERIAS/Deep Learning/Food Recognition/images/test/annotations.json\" \"/content/images/test\"\n","!mkdir \"/content/images/train\"\n","!cp \"/content/drive/Shareddrives/Intercambio/MATERIAS/Deep Learning/Food Recognition/images/train/annotations.json\" \"/content/images/train\""],"execution_count":2,"outputs":[{"output_type":"stream","text":["mkdir: cannot create directory ‘/content/images’: File exists\n","mkdir: cannot create directory ‘/content/images/test’: File exists\n","\n","UNRAR 5.50 freeware      Copyright (c) 1993-2017 Alexander Roshal\n","\n","\n","Extracting from /content/drive/Shareddrives/Intercambio/MATERIAS/Deep Learning/Food Recognition/images/test/test.rar\n","\n","\n","Would you like to replace the existing file /content/images/test/images/006452.jpg\n"," 33085 bytes, modified on 2020-09-01 09:47\n","with a new one\n"," 33085 bytes, modified on 2020-09-01 09:47\n","\n","[Y]es, [N]o, [A]ll, n[E]ver, [R]ename, [Q]uit q\n","\n","Program aborted\n","mkdir: cannot create directory ‘/content/images/train’: File exists\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"aIL2RVeNfis1","executionInfo":{"status":"ok","timestamp":1622045201420,"user_tz":-120,"elapsed":1729,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["import cv2\n","import numpy as np\n","import sys\n","from os import listdir\n","from os.path import isfile, join\n","from google.colab.patches import cv2_imshow\n","from tqdm import tqdm\n","from pycocotools.coco import COCO\n","import matplotlib.pyplot as plt\n","import matplotlib.cm as cm\n","\n","import math\n","\n","sys.path.append(\"/content/drive/Shareddrives/Intercambio/MATERIAS/Deep Learning/Food Recognition/scripts/\")\n","from data_generation import DataGeneration\n","from filter_cats import filtered_cats"],"execution_count":3,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9nm4h82NhiWe","executionInfo":{"status":"ok","timestamp":1622045209154,"user_tz":-120,"elapsed":7736,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}},"outputId":"e9dcb6b8-ed54-4940-e927-06b0fbce8344"},"source":["import tensorflow as tf\n","from tensorflow.keras.models import Model\n","from tensorflow.keras.utils import normalize\n","from tensorflow.keras.utils import to_categorical\n","from tensorflow.keras.layers import Conv2D, BatchNormalization, Activation, MaxPool2D, Conv2DTranspose, Concatenate, Input, Dropout\n","from tensorflow.keras.applications import VGG19, EfficientNetB5, ResNet50, DenseNet121, VGG16\n","from tensorflow.keras.metrics import MeanIoU\n","from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard\n","from tensorflow.keras import backend as K\n","\n","%env SM_FRAMEWORK=tf.keras\n","!pip install segmentation_models\n","import segmentation_models as sm"],"execution_count":4,"outputs":[{"output_type":"stream","text":["env: SM_FRAMEWORK=tf.keras\n","Requirement already satisfied: segmentation_models in /usr/local/lib/python3.7/dist-packages (1.0.1)\n","Requirement already satisfied: keras-applications<=1.0.8,>=1.0.7 in /usr/local/lib/python3.7/dist-packages (from segmentation_models) (1.0.8)\n","Requirement already satisfied: image-classifiers==1.0.0 in /usr/local/lib/python3.7/dist-packages (from segmentation_models) (1.0.0)\n","Requirement already satisfied: efficientnet==1.0.0 in /usr/local/lib/python3.7/dist-packages (from segmentation_models) (1.0.0)\n","Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.7/dist-packages (from keras-applications<=1.0.8,>=1.0.7->segmentation_models) (1.19.5)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras-applications<=1.0.8,>=1.0.7->segmentation_models) (3.1.0)\n","Requirement already satisfied: scikit-image in /usr/local/lib/python3.7/dist-packages (from efficientnet==1.0.0->segmentation_models) (0.16.2)\n","Requirement already satisfied: cached-property; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from h5py->keras-applications<=1.0.8,>=1.0.7->segmentation_models) (1.5.2)\n","Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image->efficientnet==1.0.0->segmentation_models) (2.5.1)\n","Requirement already satisfied: scipy>=0.19.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image->efficientnet==1.0.0->segmentation_models) (1.4.1)\n","Requirement already satisfied: matplotlib!=3.0.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image->efficientnet==1.0.0->segmentation_models) (3.2.2)\n","Requirement already satisfied: imageio>=2.3.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image->efficientnet==1.0.0->segmentation_models) (2.4.1)\n","Requirement already satisfied: pillow>=4.3.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image->efficientnet==1.0.0->segmentation_models) (7.1.2)\n","Requirement already satisfied: PyWavelets>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image->efficientnet==1.0.0->segmentation_models) (1.1.1)\n","Requirement already satisfied: decorator<5,>=4.3 in /usr/local/lib/python3.7/dist-packages (from networkx>=2.0->scikit-image->efficientnet==1.0.0->segmentation_models) (4.4.2)\n","Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet==1.0.0->segmentation_models) (1.3.1)\n","Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet==1.0.0->segmentation_models) (2.8.1)\n","Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet==1.0.0->segmentation_models) (2.4.7)\n","Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet==1.0.0->segmentation_models) (0.10.0)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet==1.0.0->segmentation_models) (1.15.0)\n","Segmentation Models: using `tf.keras` framework.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"szzjlpIkxvgv","executionInfo":{"status":"ok","timestamp":1622045209154,"user_tz":-120,"elapsed":10,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["def batch_generator(batchsize, images_path, annotation_path):\n","  print(\"Generator created for \" + annotation_path)\n","  i_img = 0\n","  coco = COCO(annotation_path)\n","\n","  categories_ids, categories_names, img_ids = filtered_cats(coco, n=N_CATS)\n","  np.random.shuffle(img_ids)\n","\n","  images = coco.loadImgs(img_ids)\n","  img_paths = [img[\"file_name\"] for img in images]\n","\n","  data_gen = DataGeneration(coco, SIZE_X, SIZE_Y, categories_ids, add_background=ADD_BACKGROUND)\n","\n","  while True:\n","    inputs = np.zeros((batchsize, SIZE_X, SIZE_Y, 3))\n","    outputs = np.zeros((batchsize, SIZE_X, SIZE_Y, N_CHANNELS))\n","\n","    for i in range(batchsize):\n","      try:\n","        inputs[i] = data_gen.x_sample(join(images_path, img_paths[i_img]))\n","        outputs[i] = data_gen.y_sample(img_ids[i_img])\n","      except:\n","        i_img = (i_img + 1) % len(img_paths)\n","        inputs[i] = data_gen.x_sample(join(images_path, img_paths[i_img]))\n","        outputs[i] = data_gen.y_sample(img_ids[i_img])\n","\n","      i_img = (i_img + 1) % len(img_paths)\n","\n","    yield inputs / 255, outputs"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"id":"RCvX0HNgmmW6","executionInfo":{"status":"ok","timestamp":1622045209155,"user_tz":-120,"elapsed":11,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["def conv_block(input, num_filters):\n","  x = Conv2D(num_filters, 3, padding=\"same\")(input)\n","  x = BatchNormalization()(x)\n","  x = Activation(\"relu\")(x)\n","  x = Dropout(0.1)(x)\n","\n","  x = Conv2D(num_filters, 3, padding=\"same\")(x)\n","  x = BatchNormalization()(x)\n","  x = Activation(\"relu\")(x)\n","\n","  return x\n","\n","\n","def decoder_block(input, skip_features, num_filters):\n","  x = Conv2DTranspose(num_filters, (2, 2), strides=2, padding=\"same\")(input)\n","  x = Concatenate()([x, skip_features])\n","  x = conv_block(x, num_filters)\n","  return x\n","\n","\n","def build_vgg19_unet(input_shape, n_classes=1):\n","  \"\"\" Input \"\"\"\n","  inputs = Input(input_shape)\n","\n","  \"\"\" Pre-trained VGG19 Model \"\"\"\n","  vgg19 = VGG19(include_top=False, weights=\"imagenet\", input_tensor=inputs)\n","  for layer in vgg19.layers:\n","    layer.trainable = False\n","\n","  \"\"\" Encoder \"\"\"\n","  s1 = vgg19.get_layer(\"block1_conv2\").output         ## (512 x 512)\n","  s2 = vgg19.get_layer(\"block2_conv2\").output         ## (256 x 256)\n","  s3 = vgg19.get_layer(\"block3_conv4\").output         ## (128 x 128)\n","  s4 = vgg19.get_layer(\"block4_conv4\").output         ## (64 x 64)\n","\n","  \"\"\" Bridge \"\"\"\n","  b1 = vgg19.get_layer(\"block5_conv4\").output         ## (32 x 32)\n","\n","  \"\"\" Decoder \"\"\"\n","  d1 = decoder_block(b1, s4, 512)                     ## (64 x 64)\n","  d2 = decoder_block(d1, s3, 256)                     ## (128 x 128)\n","  d3 = decoder_block(d2, s2, 128)                     ## (256 x 256)\n","  d4 = decoder_block(d3, s1, 64)                      ## (512 x 512)\n","\n","  \"\"\" Output \"\"\"\n","  outputs = Conv2D(n_classes, (1,1), padding=\"same\", activation=ACTIVATION)(d4)\n","\n","  model = Model(inputs, outputs, name=\"VGG19_U-Net\")\n","  return model"],"execution_count":6,"outputs":[]},{"cell_type":"code","metadata":{"id":"fFk54AZKFrg2","executionInfo":{"status":"ok","timestamp":1622045209497,"user_tz":-120,"elapsed":352,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["def conv_block(input, num_filters):\n","  x = Conv2D(num_filters, 3, padding=\"same\")(input)\n","  x = BatchNormalization()(x)\n","  x = Activation(\"relu\")(x)\n","  x = Dropout(0.1)(x)\n","\n","  x = Conv2D(num_filters, 3, padding=\"same\")(x)\n","  x = BatchNormalization()(x)\n","  x = Activation(\"relu\")(x)\n","\n","  return x\n","\n","\n","def decoder_block(input, skip_features, num_filters):\n","  x = Conv2DTranspose(num_filters, (2, 2), strides=2, padding=\"same\")(input)\n","  x = Concatenate()([x, skip_features])\n","  x = conv_block(x, num_filters)\n","  return x\n","\n","\n","def build_efficientnetb5_unet(input_shape, n_classes=1):\n","  \"\"\" Input \"\"\"\n","  inputs = Input(input_shape, name=\"input_1\")\n","\n","  \"\"\" Pre-trained EfficientNetB5 Model \"\"\"\n","  efficientnetb5 = EfficientNetB5(include_top=False, weights=\"imagenet\", input_tensor=inputs)\n","  for layer in efficientnetb5.layers:\n","    layer.trainable = False\n","\n","  \"\"\" Encoder \"\"\"\n","  s1 = efficientnetb5.get_layer(\"block2a_expand_activation\").input\n","  s2 = efficientnetb5.get_layer(\"block3a_expand_activation\").output\n","  s3 = efficientnetb5.get_layer(\"block4a_expand_activation\").output\n","  s4 = efficientnetb5.get_layer(\"block6a_expand_activation\").output\n","\n","  \"\"\" Bridge \"\"\"\n","  b1 = efficientnetb5.get_layer(\"top_activation\").output\n","\n","  \"\"\" Decoder \"\"\"\n","  d1 = decoder_block(b1, s4, 512)\n","  d2 = decoder_block(d1, s3, 256)\n","  d3 = decoder_block(d2, s2, 128)\n","  d4 = decoder_block(d3, s1, 64)\n","\n","  \"\"\" Last Upsampler \"\"\"\n","  x = Conv2DTranspose(32, (2, 2), strides=2, padding=\"same\")(d4)\n","  x = conv_block(x, 32)\n","\n","  \"\"\" Output \"\"\"\n","  outputs = Conv2D(n_classes, (1,1), padding=\"same\", activation=ACTIVATION)(x)\n","\n","  model = Model(inputs, outputs, name=\"EfficientNetB5_U-Net\")\n","  return model"],"execution_count":7,"outputs":[]},{"cell_type":"code","metadata":{"id":"xm7SvCU97DQl","executionInfo":{"status":"ok","timestamp":1622045209497,"user_tz":-120,"elapsed":8,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["def conv_block(input, num_filters):\n","    x = Conv2D(num_filters, 3, padding=\"same\")(input)\n","    x = BatchNormalization()(x)\n","    x = Activation(\"relu\")(x)\n","\n","    x = Conv2D(num_filters, 3, padding=\"same\")(x)\n","    x = BatchNormalization()(x)\n","    x = Activation(\"relu\")(x)\n","\n","    return x\n","\n","def decoder_block(input, skip_features, num_filters):\n","    x = Conv2DTranspose(num_filters, (2, 2), strides=2, padding=\"same\")(input)\n","    x = Concatenate()([x, skip_features])\n","    x = conv_block(x, num_filters)\n","    return x\n","\n","def build_resnet50_unet(input_shape, n_channels):\n","    \"\"\" Input \"\"\"\n","    inputs = Input(input_shape, name='input_1')\n","\n","    \"\"\" Pre-trained ResNet50 Model \"\"\"\n","    resnet50 = ResNet50(include_top=False, weights=\"imagenet\", input_tensor=inputs)\n","\n","    \"\"\" Encoder \"\"\"\n","    s1 = resnet50.get_layer(\"input_1\").output           ## (512 x 512)\n","    s2 = resnet50.get_layer(\"conv1_relu\").output        ## (256 x 256)\n","    s3 = resnet50.get_layer(\"conv2_block3_out\").output  ## (128 x 128)\n","    s4 = resnet50.get_layer(\"conv3_block4_out\").output  ## (64 x 64)\n","\n","    \"\"\" Bridge \"\"\"\n","    b1 = resnet50.get_layer(\"conv4_block6_out\").output  ## (32 x 32)\n","\n","    \"\"\" Decoder \"\"\"\n","    d1 = decoder_block(b1, s4, 512)                     ## (64 x 64)\n","    d2 = decoder_block(d1, s3, 256)                     ## (128 x 128)\n","    d3 = decoder_block(d2, s2, 128)                     ## (256 x 256)\n","    d4 = decoder_block(d3, s1, 64)                      ## (512 x 512)\n","\n","    \"\"\" Output \"\"\"\n","    outputs = Conv2D(n_channels, 1, padding=\"same\", activation=ACTIVATION)(d4)\n","\n","    model = Model(inputs, outputs, name=\"VGG16_U-Net\")\n","    return model"],"execution_count":8,"outputs":[]},{"cell_type":"code","metadata":{"id":"HGMzMM8puBZP","executionInfo":{"status":"ok","timestamp":1622045209498,"user_tz":-120,"elapsed":9,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["def conv_block(inputs, num_filters):\n","    x = Conv2D(num_filters, 3, padding=\"same\")(inputs)\n","    x = BatchNormalization()(x)\n","    x = Activation(\"relu\")(x)\n","\n","    x = Conv2D(num_filters, 3, padding=\"same\")(x)\n","    x = BatchNormalization()(x)\n","    x = Activation(\"relu\")(x)\n","\n","    return x\n","\n","def decoder_block(inputs, skip_features, num_filters):\n","    x = Conv2DTranspose(num_filters, (2, 2), strides=2, padding=\"same\")(inputs)\n","    x = Concatenate()([x, skip_features])\n","    x = conv_block(x, num_filters)\n","    return x\n","\n","def build_densenet121_unet(input_shape, n_channels):\n","    \"\"\" Input \"\"\"\n","    inputs = Input(input_shape, name=\"input_1\")\n","\n","    \"\"\" Pre-trained DenseNet121 Model \"\"\"\n","    densenet = DenseNet121(include_top=False, weights=\"imagenet\", input_tensor=inputs)\n","\n","    \"\"\" Encoder \"\"\"\n","    s1 = densenet.get_layer(\"input_1\").output       ## 512\n","    s2 = densenet.get_layer(\"conv1/relu\").output    ## 256\n","    s3 = densenet.get_layer(\"pool2_relu\").output ## 128\n","    s4 = densenet.get_layer(\"pool3_relu\").output  ## 64\n","\n","    \"\"\" Bridge \"\"\"\n","    b1 = densenet.get_layer(\"pool4_relu\").output  ## 32\n","\n","    \"\"\" Decoder \"\"\"\n","    d1 = decoder_block(b1, s4, 512)             ## 64\n","    d2 = decoder_block(d1, s3, 256)             ## 128\n","    d3 = decoder_block(d2, s2, 128)             ## 256\n","    d4 = decoder_block(d3, s1, 64)              ## 512\n","\n","    \"\"\" Outputs \"\"\"\n","    outputs = Conv2D(n_channels, 1, padding=\"same\", activation=ACTIVATION)(d4)\n","\n","    model = Model(inputs, outputs)\n","    return model"],"execution_count":9,"outputs":[]},{"cell_type":"code","metadata":{"id":"MMgBXVaQwMcc","executionInfo":{"status":"ok","timestamp":1622045209498,"user_tz":-120,"elapsed":8,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["def conv_block(input, num_filters):\n","    x = Conv2D(num_filters, 3, padding=\"same\")(input)\n","    x = BatchNormalization()(x)\n","    x = Activation(\"relu\")(x)\n","\n","    x = Conv2D(num_filters, 3, padding=\"same\")(x)\n","    x = BatchNormalization()(x)\n","    x = Activation(\"relu\")(x)\n","\n","    return x\n","\n","def decoder_block(input, skip_features, num_filters):\n","    x = Conv2DTranspose(num_filters, (2, 2), strides=2, padding=\"same\")(input)\n","    x = Concatenate()([x, skip_features])\n","    x = conv_block(x, num_filters)\n","    return x\n","\n","def build_vgg16_unet(input_shape, n_channels=1):\n","    \"\"\" Input \"\"\"\n","    inputs = Input(input_shape)\n","\n","    \"\"\" Pre-trained VGG16 Model \"\"\"\n","    vgg16 = VGG16(include_top=False, weights=\"imagenet\", input_tensor=inputs)\n","    vgg16.trainable = False\n","    \"\"\" Encoder \"\"\"\n","    s1 = vgg16.get_layer(\"block1_conv2\").output         ## (512 x 512)\n","    s2 = vgg16.get_layer(\"block2_conv2\").output         ## (256 x 256)\n","    s3 = vgg16.get_layer(\"block3_conv3\").output         ## (128 x 128)\n","    s4 = vgg16.get_layer(\"block4_conv3\").output         ## (64 x 64)\n","\n","    \"\"\" Bridge \"\"\"\n","    b1 = vgg16.get_layer(\"block5_conv3\").output         ## (32 x 32)\n","\n","    \"\"\" Decoder \"\"\"\n","    d1 = decoder_block(b1, s4, 512)                     ## (64 x 64)\n","    d2 = decoder_block(d1, s3, 256)                     ## (128 x 128)\n","    d3 = decoder_block(d2, s2, 128)                     ## (256 x 256)\n","    d4 = decoder_block(d3, s1, 64)                      ## (512 x 512)\n","\n","    \"\"\" Output \"\"\"\n","    outputs = Conv2D(n_channels, (1,1), padding=\"same\", activation=ACTIVATION)(d4)\n","\n","    model = Model(inputs, outputs, name=\"VGG16_U-Net\")\n","    return model"],"execution_count":10,"outputs":[]},{"cell_type":"code","metadata":{"id":"khWX92iestW1","executionInfo":{"status":"ok","timestamp":1622045209498,"user_tz":-120,"elapsed":8,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["def iou_score(ground_truth, predicted_mask, with_background=True):\n","  iou = 0\n","  batch_size = ground_truth.shape[0]\n","  categories = ground_truth.shape[3]\n","\n","  for i_mask in range(batch_size):\n","    for i_cat in range(categories - int(with_background)):\n","      intersection = np.sum(np.multiply( ground_truth[i_mask, :, :, i_cat], predicted_mask[i_mask, :, :, i_cat]))\n","      union = np.sum(np.add(ground_truth[i_mask, :, :, i_cat], predicted_mask[i_mask, :, :, i_cat]))\n","\n","      if union == 0 and intersection == 0:\n","        iou += 1\n","      elif union == 0:\n","        print(\"MATH ERROR: UNION IS 0 BUT INTERSECTION NO!!\")\n","      else:\n","        iou += intersection / union\n","\n","  return iou / (batch_size * categories)\n","\n","\n","def only_true_iou_score(ground_truth, predicted_mask, with_background=True):\n","  iou = 0\n","  total_masks_computed = 0\n","  batch_size = ground_truth.shape[0]\n","  categories = ground_truth.shape[3]\n","\n","  for i_mask in range(batch_size):\n","    for i_cat in range(categories - int(with_background)):\n","      intersection = np.sum(np.multiply( ground_truth[i_mask, :, :, i_cat], predicted_mask[i_mask, :, :, i_cat]))\n","      union = np.sum(np.add(ground_truth[i_mask, :, :, i_cat], predicted_mask[i_mask, :, :, i_cat]))\n","\n","      if union != 0:\n","        iou += intersection / union\n","        total_masks_computed += 1\n","      elif intersection != 0:\n","        print(\"MATH ERROR: UNION IS 0 BUT INTERSECTION NO!!\")\n","\n","  return iou / (total_masks_computed)\n","\n","\n","def iou_metric(y_true, y_pred):\n","  threshold = 0.7\n","  tf.py_func(iou_score, [y_true, np.where(y_pred > threshold, 1, 0)], tf.float64)\n","\n","\n","def only_true_iou_metric(y_true, y_pred):\n","  threshold = 0.7\n","  tf.py_func(only_true_iou_score, [y_true, np.where(y_pred > threshold, 1, 0)], tf.float64)\n","\n","\n","def jaccard_loss(y_true, y_pred):\n","  threshold = 0.7\n","  return (1 - iou_score(iou_score(y_true, np.where(y_pred > threshold, 1, 0))))\n","\n","\n","def only_true_jaccard_loss(y_true, y_pred):\n","  threshold = 0.7\n","  return (1 - only_true_iou_score(iou_score(y_true, np.where(y_pred > threshold, 1, 0))))"],"execution_count":11,"outputs":[]},{"cell_type":"code","metadata":{"id":"FFWtiOW-rsd7","executionInfo":{"status":"ok","timestamp":1622046268595,"user_tz":-120,"elapsed":217,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["SIZE_X = 128\n","SIZE_Y = 128\n","N_CATS = 16\n","ADD_BACKGROUND = False\n","N_CHANNELS = N_CATS + int(ADD_BACKGROUND)   # One more channel for background\n","ACTIVATION = \"softmax\"\n","CHANNEL_CORRECTION = False\n","weights_file = \"/content/drive/Shareddrives/Intercambio/MATERIAS/Deep Learning/Food Recognition/weights/vgg19_pretrained_16_cats_softmax_no_bkg_weights.h5\"\n","build_network = build_vgg19_unet\n","\n","INPUT_SHAPE = (SIZE_X, SIZE_Y, 3)\n","BATCH_SIZE = 32\n","\n","TRAIN_IMAGES_PATH = \"/content/images/train/images\"\n","TRAIN_ANNOTATIONS_PATH = \"/content/images/train/annotations.json\"\n","TEST_IMAGES_PATH = \"/content/images/test/images\"\n","TEST_ANNOTATIONS_PATH = \"/content/images/test/annotations.json\""],"execution_count":36,"outputs":[]},{"cell_type":"code","metadata":{"id":"5GGt7Nm0GrJW","executionInfo":{"status":"ok","timestamp":1622046268933,"user_tz":-120,"elapsed":2,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["BACKBONE = 'efficientnetb5'\n","preprocess_input = sm.get_preprocessing(BACKBONE)"],"execution_count":37,"outputs":[]},{"cell_type":"code","metadata":{"id":"Vi3iJAVQATyL","executionInfo":{"status":"ok","timestamp":1622046270445,"user_tz":-120,"elapsed":1514,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["#Load Model\n","loaded_model = build_network(INPUT_SHAPE, N_CHANNELS)\n","loaded_model.load_weights(weights_file)"],"execution_count":38,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vZ3xz2pNJvS0","executionInfo":{"status":"ok","timestamp":1622046282326,"user_tz":-120,"elapsed":11892,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}},"outputId":"263568f2-a778-40ff-ec82-f1181657fe8e"},"source":["gen_viewer = batch_generator(697, TEST_IMAGES_PATH, TEST_ANNOTATIONS_PATH)\n","x_test, y_test = next(gen_viewer)"],"execution_count":39,"outputs":[{"output_type":"stream","text":["Generator created for /content/images/test/annotations.json\n","loading annotations into memory...\n","Done (t=0.10s)\n","creating index...\n","index created!\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"8mB5ZUKLHqw4","executionInfo":{"status":"ok","timestamp":1622046283208,"user_tz":-120,"elapsed":893,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}},"outputId":"d85e67a5-bf54-411f-d21a-9068b691582f"},"source":["# Create DataGeneration instance to obtain images and masks\n","coco = COCO(TEST_ANNOTATIONS_PATH)\n","\n","categories_ids, categories_names, img_ids = filtered_cats(coco, n=N_CATS)\n","\n","images = coco.loadImgs(img_ids)\n","img_paths = [img[\"file_name\"] for img in images]\n","\n","data_gen = DataGeneration(coco, SIZE_X, SIZE_Y, categories_ids, add_background=ADD_BACKGROUND)"],"execution_count":40,"outputs":[{"output_type":"stream","text":["loading annotations into memory...\n","Done (t=0.09s)\n","creating index...\n","index created!\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"LtVsdITB4oLw","executionInfo":{"status":"ok","timestamp":1622046283209,"user_tz":-120,"elapsed":8,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}},"outputId":"e7e99499-ac8a-473b-a655-ee24dacc502b"},"source":["print(categories_names)\n","names = np.asarray(categories_names)\n","\n","print(np.where(names == 'apple')[0])\n","print(np.where(names == 'rice')[0])\n","print(np.where(names == 'banana')[0])\n","print(np.where(names == 'cucumber')[0])\n","print(np.where(names == 'carrot')[0])\n","print(np.where(names == 'coffee-with-caffeine')[0])\n","print(np.where(names == 'egg')[0])\n","print(np.where(names == 'mixed-vegetables')[0])"],"execution_count":41,"outputs":[{"output_type":"stream","text":["['water', 'bread-white', 'salad-leaf-salad-green', 'tomato', 'butter', 'bread-wholemeal', 'coffee-with-caffeine', 'carrot', 'apple', 'mixed-vegetables', 'egg', 'tea', 'rice', 'banana', 'mixed-salad-chopped-without-sauce', 'cucumber']\n","[8]\n","[12]\n","[13]\n","[15]\n","[7]\n","[6]\n","[10]\n","[9]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"eSNNMpUprrwS","executionInfo":{"status":"ok","timestamp":1622046283209,"user_tz":-120,"elapsed":5,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["def plot_labeled_masks(masks, category_names, alpha):\n","\n","  cmap = cm.get_cmap('viridis')\n","  plt_mask = np.zeros(masks.shape[:2])\n","  n = masks.shape[2]\n","  for k in range(n):\n","    idx_arr = (masks[:, :, k] == 1)\n","    plt_mask[idx_arr] = k + 1\n","\n","  uniques = np.unique(plt_mask)\n","  has_zero = int(0.0 in uniques)\n","  n_colors = len(uniques) - has_zero # Don't count zeros\n","  print(uniques)\n","  print(n_colors)\n","  for i, val in enumerate(uniques):\n","    if val > 0:\n","      # Plot point to show labels\n","      color_arg = (i-has_zero) / (n_colors-1) if n_colors > 1 else 0\n","      plt.plot(0, 0, '-', label=category_names[int(val-1)], color=cmap( color_arg )) \n","\n","  plt_mask[ plt_mask==0 ] = None\n","  plt.legend()\n","  plt.imshow(plt_mask, alpha=alpha)"],"execution_count":42,"outputs":[]},{"cell_type":"code","metadata":{"id":"f_50DrOcsCmp","executionInfo":{"status":"ok","timestamp":1622046283210,"user_tz":-120,"elapsed":5,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["def plot_labeled_results(image, ground_truth, predicted_masks, category_names):\n","  plt.figure(figsize=(16, 8));  plt.axis('off')\n","  alpha = 0.6\n","\n","  # Show image superposed to ground truth\n","  plt.subplot(121)\n","  plt.title('Ground truth')\n","  plt.imshow(cv2.cvtColor(image.astype(np.float32), cv2.COLOR_BGR2RGB)); plt.axis(\"off\")\n","  plot_labeled_masks(ground_truth, category_names, alpha)\n","\n","  # Show image superposed to prediction\n","  plt.subplot(122)\n","  plt.title('Predicted mask')\n","  plt.imshow(cv2.cvtColor(x_sample.astype(np.float32), cv2.COLOR_BGR2RGB)); plt.axis(\"off\")\n","  plot_labeled_masks(predicted_masks, category_names, alpha)\n","  plt.show()"],"execution_count":43,"outputs":[]},{"cell_type":"code","metadata":{"id":"j6XMsBrU8Iqb","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1622046329488,"user_tz":-120,"elapsed":46283,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}},"outputId":"6ae50c50-b437-4050-dda2-5e0c0a4146ae"},"source":["total_prediction = loaded_model.predict(x_test)\n","\n","# Channel Correction\n","if CHANNEL_CORRECTION:\n","  total_prediction[:,:,:,[8, 12]] = total_prediction[:,:,:,[12, 8]]\n","  total_prediction[:,:,:,[13, 15]] = total_prediction[:,:,:,[15, 13]]\n","  total_prediction[:,:,:,[6, 7]] = total_prediction[:,:,:,[7, 6]]\n","  total_prediction[:,:,:,[9, 10]] = total_prediction[:,:,:,[10, 9]]\n","\n","thresholds = [ 0.5, 0.7, 0.8, 0.85, 0.9 ]\n","ious = []\n","true_ious = []\n","for threshold in thresholds:\n","  print(f\"Threshold = {threshold}...\")\n","  iou = iou_score(y_test, np.where(total_prediction > threshold, 1, 0), ADD_BACKGROUND)\n","  true_iou = only_true_iou_score(y_test, np.where(total_prediction > threshold, 1, 0), ADD_BACKGROUND)\n","  print(f\"IoU score for the whole evaluation batch is {iou}\")\n","  print(f\"IoU score, only accounting for ground truth with objects in it, for the whole evaluation batch is {true_iou}\")\n","  ious.append(iou)\n","  true_ious.append(true_iou)"],"execution_count":44,"outputs":[{"output_type":"stream","text":["Threshold = 0.5...\n","IoU score for the whole evaluation batch is 0.6598931991652498\n","IoU score, only accounting for ground truth with objects in it, for the whole evaluation batch is 0.04268777311733193\n","Threshold = 0.7...\n","IoU score for the whole evaluation batch is 0.7524269060925008\n","IoU score, only accounting for ground truth with objects in it, for the whole evaluation batch is 0.052202147869403734\n","Threshold = 0.8...\n","IoU score for the whole evaluation batch is 0.7963049553412237\n","IoU score, only accounting for ground truth with objects in it, for the whole evaluation batch is 0.058205995839687434\n","Threshold = 0.85...\n","IoU score for the whole evaluation batch is 0.8147171857335708\n","IoU score, only accounting for ground truth with objects in it, for the whole evaluation batch is 0.05992996146532065\n","Threshold = 0.9...\n","IoU score for the whole evaluation batch is 0.8380989646668486\n","IoU score, only accounting for ground truth with objects in it, for the whole evaluation batch is 0.0615798617280176\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"yVq9h6nnHnT_","executionInfo":{"status":"ok","timestamp":1622046329488,"user_tz":-120,"elapsed":20,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}}},"source":["def plot_ious(title, x, y):\n","  plt.title(title)\n","  plt.scatter(x, y)\n","  plt.xlabel(\"Thresholds\")\n","  plt.ylabel(\"Metric\")\n","  plt.grid()"],"execution_count":45,"outputs":[]},{"cell_type":"code","metadata":{"id":"VRd4iqLHi9vO","colab":{"base_uri":"https://localhost:8080/","height":339},"executionInfo":{"status":"ok","timestamp":1622046329489,"user_tz":-120,"elapsed":19,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}},"outputId":"97d0d4be-cdfc-4d4c-b1e2-1bc66cdbb266"},"source":["plt.figure(figsize=[6.4 * 2, 4.8])\n","plt.subplot(121)\n","plot_ious(f\"IOUs\", thresholds, ious)\n","plt.subplot(122)\n","plot_ious(f\"Only True IOU\", thresholds, true_ious)\n","plt.show()"],"execution_count":46,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAwoAAAFCCAYAAABRrCoJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3df5xdVX3v/9fbBDD+IgiaiyEaWhGL2kLJ1bZUm8pV0LaSVtSgVbilUr8tWK3mSq79WkrrLUoVr1e/WlQqYhUsYkwrNlrD2NaLFDBIhDYY0JYM+AsIEg0Qwuf7x96DhzkzycnMnJmTmdfz8TiP2Wfttdd89mHYK5+z9lo7VYUkSZIkdXrETAcgSZIkafCYKEiSJEnqYqIgSZIkqYuJgiRJkqQuJgqSJEmSupgoSJIkSepioiBJkjSOJN9O8t9mOg5pJpgoSBPQ2XEkOSTJ3yS5I8mPkvxrkl/vqLs0SSWZP6qNjyb58+mOXZLmkiSnJNmY5MdJvpPkA0kW9vl3fjDJtvZ1f5IdHe8/3+ffXUme2vH+iCRrk9yd5J4kVyT5pY79y5NsGaOdoSS/289YNfhMFKRJSPJ44F+A+4FnAAcB5wGfSHLiTMYmSXNdkjcB7wBWAfsDvwA8Bfhikn379Xur6nVV9Ziqegzwv4BLRt5X1Ys64ps/fiuTl+Snga8AG4FDgScBnwG+kOQX+/m7NTuYKEiT80ZgG3BqVX2nqrZX1SeBtwPvSpJeGkny1CRfbr/x+UGSS/oZtCTNdkkeB/wpcEZV/UNV7aiqbwMvB5YCv93WOyvJp5J8rP3G/YYky8Zo77+0oxIHdpT9fJLvJ9lnD+L6dpK3JLke+FGS+WOMAjxsxDnJrye5LsnWJP83yc/2+OvOAq6sqrdW1Z1VdU9VvRe4iCaBknbJREGanBcAn66qB0eVfwp4MvC0Htv5M+ALwAHAIcD/mbIIJWlu+iXgkcBlnYVVtQ24nOb6PeIlwMXAQmAt8L7RjVXVd4AhmkRjxKuBi6tqxx7GdhLwa8DCqnpgVxWTHAVcAPwecCDwV8DaJPv18HteAPztGOWfAo5JsmCPotacY6IgTc5BwO1jlN/esb8XO2iGw59UVfdW1b9MRXCSNIcdBPxgnH+I387Dr8//UlWXV9VOmm/bf26cNi/kJyMR82j+wX/RBGJ7b1XdWlXbe6h7GvBXVXVVVe2sqguB+2huo9qdXfVRjwAe33PEmpNMFKTJ+QFw8BjlB3fsH+mkRg9N70OTIAD8DyDAv7bD3r8z1YFK0hzzA+CgceYBHNzuH/Gdju0fA48c57jPAkckOZTm2/q7q+pfJxDbrXtQ9ynAm9rbjrYm2QosoZlvsDu76qMeBO6i6aPGunWqs4/SHGWiIE3OPwK/lWT0/0svp+kIbqL55mYHzT2xnQ4F/gOaIe2qem1VPYlmePn/67xfVZK0x66k+eb9tzoLkzwGeBHwpT1tsKrupblt57dpbjuayGgCQI16/2PgUR3v/0vH9q3A26tqYcfrUe18uN35R+BlY5S/nGbuwo+B/6RJqB4zsrOdX/cU2j5Kc5eJgjQ559GspPGRdqLbI5OcBLwVWFWNncCngbcnOTDJPm2dI4DPAyR5WZJD2jbvoulERs97kCT1qKruppnM/H+SHN9ee5fS/EN/CxP/R/7HgFNo5jVMtI3RrgNemWRekuOBX+nY9yHgdUmek8ajk/xaksf20O6fAr+U5O1JHp/ksUnOAF4DvAWgqv4TuAp4R5LHtHMfVtF8wfXVKTo/7aVMFKRJqKo7gF+mmTB3I3AH8EfAq6uqc+Wi3wfuBK4HvgecDvxaVX233f9fgauSbKOZSPeHVXXL9JyFJM1OVfVO4H8Cfwn8kOYfxLcCx1bVfRNs8ys0X+R8raqm6hv3PwR+A9gKvApY0/H7rgFeSzPB+i5gM02i0kus36Tpo34O+DbNCPdLgePa8xjxCuCJbdvDwLE0fdS9kzgnzQKpGj36JUmSpPEkWQ98oqo+PNOxSP1koiBJktSjJP8V+CKwpKrumel4pH7y1iNJkqQeJLmQZoLwG0wSNBeYKEiSplQ7cXRTks1Jzhxj/35JLmn3X9VOMB3Z97NJrmyXCd7YLhDwqCSfS/Lvbfk5HfVPaZ+Me137+t3pOUvNRVV1clXtX1UfnelYpOlgoiBJmjLtQ6jeT7P85BHASUmOGFXtVOCuqnoqzcph72iPnQ98HHhdVT0DWM5P1nH/y6p6OnAUzRNlX9TR3iVVdWT78p5xSZoiJgqSpKn0bGBzVd1SVfcDFwMnjKpzAs0TbgEuBY5t121/IXB9VX0dmlXF2ifR/riqrmjL7ge+BhyCJKmvxnrq4Kxz0EEH1dKlSyd07I9+9CMe/ehHT21AkzBo8YAx9cqYemNMvZloTNdee+0PquoJfQhpxGIe/tTZLcBzxqtTVQ8kuRs4EHgaUEnWAU8ALm6Xt3xIkoU0y0j+747ilyZ5Hs0DDt9YVV1PvU1yGnAawIIFC45esmTJhE7uwQcf5BGPGKzv2IypN8bUG2PavUGLByYX00033TR+v1BVs/519NFH10RdccUVEz62HwYtnipj6pUx9caYejPRmIBrqo/XW+BE4MMd718NvG9UnW8Ah3S8vxk4CHgz8K12+1E0T9Y9tqPefJqHFL6ho+xAYL92+/eA9buLcTb1CVXG1Ctj6o0x7d6gxVM1uZh21S8MVjokSdrbDQOdX9cf0paNWaedl7A/zcMKtwD/VFU/qKofA5cDP99x3PnAN6vqPSMF1dyeNPLgrA8DR0/huUjSnGaiIEmaSlcDhyU5NMm+wEqap413Wguc3G6fSDMKUMA64FntKkfzgV+heeI5Sf6cJqF4Q2dDSQ7uePsS4N+m+Hwkac6aE3MUJEnTo5o5B6fT/KN/HnBBVd2Q5Gya4e21wEeAi5JsBu6kSSaoqruSvJsm2Sjg8qr6XJJDgLcC/w58rZn3zPuqWeHo9UleAjzQtnXKNJ6uJM1qJgqSpClVVZfT3DbUWfa2ju17gZeNc+zHaZZI7SzbAmSc+quB1ZMMWZI0Bm89kiRJktTFREGSJElSFxMFSZIkSV1MFCRpL7JmwzDHnLOejcN3c8w561mzYfTKo5KkuaLffYKTmSVpL7FmwzCrL9vI9h07YQkMb93O6ss2ArDiqMUzHJ0kaTpNR5/giIIk7SXOXbep6RA6bN+xk3PXbZqhiCRJM2U6+gQTBUnaS9y2dfselUuSZq/p6BNMFCRpL/GkhQv2qFySNHtNR59goiBJe4lVxx3Ogn3mPaxswT7zWHXc4TMUkSRppkxHn+BkZknaS4xMTmvuP72HxQsXsOq4w53ILElz0HT0CSYKkrQXWXHUYlYctZihoSHOeNXymQ5HkjSD+t0neOuRJEmSpC4mCpIkSZK6mChIkiRJ6mKiIEmSJKmLiYIkSZKkLiYKkiRJkrqYKEiSJEnqYqIgSZIkqUtfE4UkxyfZlGRzkjPH2P/kJFck2ZDk+iQvbstfkOTaJBvbn8/vOGaobfO69vXEfp6DJEmSNBf17cnMSeYB7wdeAGwBrk6ytqpu7Kj2x8CnquoDSY4ALgeWAj8AfqOqbkvyTGAd0Pk86ldV1TX9il2SJEma6/o5ovBsYHNV3VJV9wMXAyeMqlPA49rt/YHbAKpqQ1Xd1pbfACxIsl8fY5UkSZLUoZ+JwmLg1o73W3j4qADAWcBvJ9lCM5pwxhjtvBT4WlXd11H21+1tR/9vkkxhzJKkSerhttP9klzS7r8qydKOfT+b5MokN7S3nz6yLT+6fb85yXtHrv1JHp/ki0m+2f48YLrOU5Jmu77detSjk4CPVtW7kvwicFGSZ1bVgwBJngG8A3hhxzGvqqrhJI8FPg28GvjY6IaTnAacBrBo0SKGhoYmFOC2bdsmfGw/DFo8YEy9MqbeGFNvBjEm6Pm201OBu6rqqUlW0lznX5FkPvBx4NVV9fUkBwI72mM+ALwWuIrmi6Xjgc8DZwJfqqpz2qTkTOAtfT9RSXPOmg3DnLtuEyuX3MNbz1nPquMOZ8VRo78Dn136mSgMA0s63h/SlnU6leZiT1Vd2X5zdBDwvSSHAJ8BXlNVN48cUFXD7c97knyC5hanrkShqs4HzgdYtmxZLV++fEInMTQ0xESP7YdBiweMqVfG1Btj6s0gxtR66LZTgCQjt512Jgon0IwoA1wKvK8dIXghcH1VfR2gqu5o2zgYeFxVfbV9/zFgBU2icAKwvG3rQmAIEwVJU2zNhmFWX7aR7Tt2whIY3rqd1ZdtBJjVyUI/bz26GjgsyaFJ9gVWAmtH1flP4FiAJD8DPBL4fpKFwOeAM6vqKyOVk8xPclC7vQ/w68A3+ngOkqQ908ttpw/VqaoHgLuBA4GnAZVkXZKvJfkfHfW3jNPmoqq6vd3+DrBoqk5Ekkacu25TkyR02L5jJ+eu2zRDEU2Pvo0oVNUDSU6nWbFoHnBBVd2Q5GzgmqpaC7wJ+FCSN9JMbD6lqqo97qnA25K8rW3yhcCPgHVtkjAP+EfgQ/06B0nStJoP/DLwX4EfA19Kci1NIrFbbf9RY+2brbejgjH1yph6Y0xjW7nknofuk1m0AN70rAfaPffMeGzQv8+or3MUqupymntJO8ve1rF9I3DMGMf9OfDn4zR79FTGKEmaUr3cdjpSZ0s7L2F/4A6akYJ/qqofACS5HPh5mnkLh4zT5neTHFxVt7e3KH1vrKBm6+2oYEy9MqbeGNPY3nrOeoa3bgeaJOFdG5t/Qi9euIAzXrV8BiNr9Osz8snMkqSp1Mttp2uBk9vtE4H1VVU0I9DPSvKoNoH4FeDG9taiHyb5hXYuw2uAz47R1skd5ZI0ZVYddzgL9pn3sLIF+8xj1XGHz1BE02OmVz2SJM0iPd52+hGaVe42A3fSJBNU1V1J3k2TbBRweVV9rm3694GPAgtoJjF/vi0/B/hUklOB/wBePg2nKWmOGZmw3MxJuIfFCxe46pEkSXuqh9tO7wVeNs6xH6e51Wh0+TXAM8cov4N2UQxJ6qcVRy1mxVGLGRoaGojbjaaDtx5JkiRJ6mKiIEmSJKmLiYIkSZKkLiYKkjSONRuGOeac9WwcvptjzlnPmg2jV/mUJGn2cjKzJI1hzYZhVl+2sXkS5xIY3rqd1ZdtBJj1q1xIkgSOKEjSmM5dt6lJEjps37GzXRpPkqTZz0RBksZwW/sEzl7LJUmabUwUJGkMT1q4YI/KJUmabUwUJGkMq447nAX7zHtY2YJ95rHquMNnKCJJkqaXk5klaQwjE5abOQn3sHjhAlYdd7gTmSVJc4aJgiSNY8VRi1lx1GKGhoY441XLZzocSZKmlbceSZIkSepioiBJkiSpi4mCJEmSpC4mCpIkSRo4azYMc8w569k4fDfHnLOeNRuGZzqkOcfJzJIkSRooazYMs/qyjWzfsROWwPDW7ay+bCOAq89NI0cUJEmSNFDOXbepSRI6bN+xs12yWtPFREGSJEkD5bat2/eoXP1hoiBJkqSB8qSFC/aoXP1hoiBJkqSBsuq4w1mwz7yHlS3YZx6rjjt8hiKam5zMLEmSpIEyMmG5mZNwD4sXLmDVcYc7kXmamShIkiRp4Kw4ajErjlrM0NAQZ7xq+UyHMyf19dajJMcn2ZRkc5Izx9j/5CRXJNmQ5PokL+7Yt7o9blOS43ptU5I0s3q49u+X5JJ2/1VJlrblS5NsT3Jd+/pgW/7YjrLrkvwgyXvafack+X7Hvt+dznOVpNmsbyMKSeYB7wdeAGwBrk6ytqpu7Kj2x8CnquoDSY4ALgeWttsrgWcATwL+McnT2mN216YkaYb0eO0/Fbirqp6aZCXwDuAV7b6bq+rIzjar6h7gobIk1wKXdVS5pKpOn/qzkaS5rZ8jCs8GNlfVLVV1P3AxcMKoOgU8rt3eH7it3T4BuLiq7quqbwGb2/Z6aVOSNHN6uU6fAFzYbl8KHJskvTTefmn0ROCfpyheSdI4+jlHYTFwa8f7LcBzRtU5C/hCkjOARwP/rePYr446dmT2yu7aBCDJacBpAIsWLWJoaGiPTwBg27ZtEz62HwYtHjCmXhlTb4ypN4MYU6uXa/9DdarqgSR3Awe2+w5NsgH4IfDHVTU6IVhJM4JQHWUvTfI84CbgjVV1K5KkSZvpycwnAR+tqncl+UXgoiTPnIqGq+p84HyAZcuW1fLlyyfUztDQEBM9th8GLR4wpl4ZU2+MqTeDGNMUuB14clXdkeRoYE2SZ1TVDzvqrARe3fH+74BPVtV9SX6PZqTi+aMbnq1fHoEx9cqYemNMuzdo8UD/YupnojAMLOl4f0hb1ulU4HiAqroyySOBg3Zz7O7alLQXWrNhmHPXbWLlknt46znrXQZv79XLtX+kzpYk82luPb2jHSW4D6Cqrk1yM/A04BqAJD8HzK+qa0caqqo7Otr9MPDOsYKarV8egTH1yph6Y0y7N2jxQP9i6ucchauBw5IcmmRfmm+B1o6q85/AsQBJfgZ4JPD9tt7KdmWMQ4HDgH/tsU1Je5k1G4ZZfdlGhrduB2B463ZWX7aRNRv8HmAv1Mt1ei1wcrt9IrC+qirJE9rJ0CT5KZpr/y0dx50EfLKzoSQHd7x9CfBvU3YmkjTH9W1Eob3v9HRgHTAPuKCqbkhyNnBNVa0F3gR8KMkbaSY2n9J+o3RDkk8BNwIPAH9QVTsBxmqzX+cgaXqcu24T23fsfFjZ9h07OXfdJkcV9jI9Xvs/QnOr6WbgTppkAuB5wNlJdgAPAq+rqjs7mn858GIe7vVJXkLTV9wJnNKnU5OkOaevcxSq6nKaJU87y97WsX0jcMw4x74deHsvbUrau93WjiT0Wq7B1sO1/17gZWMc92ng07to96fGKFsNrJ5MvJKksfX1gWuS1IsnLVywR+WSJKn/TBQkzbhVxx3Ogn3mPaxswT7zWHXc4TMUkSRJmunlUSXpoXkI567bBNzD4oULXPVIkqQZZqIgaSCsOGoxK45azNDQEGe8avlMhyNJ0pznrUeSJEmSupgoSJIkSepioiBJkiSpi4mCJEmSpC4mCpIkSZK6mChIkiRJ6mKiIEmSJKmLiYIkSZKkLiYKkiRJkrqYKEiSJEnqYqIgSZIkqYuJgiRJkqQuJgqSJEmSupgoSJIkSepioiBJkiSpi4mCJEmSpC4mCpIkSZK6mChIkiRJ6mKiIEmSJKmLiYIkaUolOT7JpiSbk5w5xv79klzS7r8qydK2fGmS7Umua18f7DhmqG1zZN8Td9WWJGny+poo9NBZnNdx0b8pyda2/Fc7yq9Lcm+SFe2+jyb5Vse+I/t5DpKk3iWZB7wfeBFwBHBSkiNGVTsVuKuqngqcB7yjY9/NVXVk+3rdqONe1bHvez20JUmahPn9arijs3gBsAW4OsnaqrpxpE5VvbGj/hnAUW35FcCRbfnjgc3AFzqaX1VVl/YrdknShD0b2FxVtwAkuRg4Abixo84JwFnt9qXA+5Jkgr9vzLaqqibYniSp1bdEgd46i04nAX8yRvmJwOer6sd9iVKSNJUWA7d2vN8CPGe8OlX1QJK7gQPbfYcm2QD8EPjjqvrnjuP+OslO4NPAn7fJwHht/aDzFyY5DTgNYNGiRQwNDU3o5LZt2zbhY/vFmHpjTL0xpt0btHigfzH1M1HopbMAIMlTgEOB9WPsXgm8e1TZ25O8DfgScGZV3Tf5cCVJM+x24MlVdUeSo4E1SZ5RVT+kue1oOMljaRKFVwMf67XhqjofOB9g2bJltXz58gkFODQ0xESP7Rdj6o0x9caYdm/Q4oH+xdTPRGFPrAQuraqdnYVJDgaeBazrKF4NfAfYl+ai/xbg7NENztZvjwYtHjCmXhlTb4ypN4MYU2sYWNLx/pC2bKw6W5LMB/YH7mhHCO4DqKprk9wMPA24pqqG2/J7knyCZtT6Y+O11a+Tk6S5pJ+JQi+dxYiVwB+MUf5y4DNVtWOkoKpubzfvS/LXwJvHanC2fns0aPGAMfXKmHpjTL0ZxJhaVwOHJTmU5pq/EnjlqDprgZOBK2luL11fVZXkCcCdVbUzyU8BhwG3tAnAwqr6QZJ9gF8H/nFXbfX3FCVpbuhnotBLZ0GSpwMH0FzkRzuJZgShs/7BVXV7O/FtBfCNqQ5ckjQx7TyB02lGgucBF1TVDUnOphkZWAt8BLgoyWbgTpr+AeB5wNlJdgAPAq+rqjuTPBpY1yYJ82iShA+1x4zXliRpkvqWKPTYWUBzUb949DdA7VrYS4Avj2r6b9pvnQJcB4xePk+SNIOq6nLg8lFlb+vYvhd42RjHfZpm/sHo8h8BR4/zu8ZsS5I0eX2do7C7zqJ9f9Y4x36bZkL06PLnT12EkiRJksbik5klSZIkdTFRkCRJktTFREGSJElSFxMFSZIkSV1MFCRJkiR1MVGQJEmS1MVEQZIkSVIXEwVJkiRJXUwUJEmSJHUxUZAkSZLUxURBkiRJUhcTBUmSJEldTBQkSZIkdTFRkCRJktTFREGSJElSFxMFSZIkSV16ShSS/GaS/TveL0yyon9hSZIGgdd/SZq7eh1R+JOqunvkTVVtBf6kPyFJkgaI139JmqN6TRTGqjd/KgORJA0kr/+SNEf1mihck+TdSX66fb0buLafgUmSBoLXf0mao3pNFM4A7gcuaV/3AX/Qr6AkSQPD678kzVE9DR9X1Y+AM/sciyRpwEzk+p/keOB/A/OAD1fVOaP27wd8DDgauAN4RVV9O8lS4N+ATW3Vr1bV65I8Cvhb4KeBncDfVdWZbVunAOcCw+0x76uqD+/haUqSxrDLRCHJe6rqDUn+DqjR+6vqJX2LTJol1mwY5tx1m1i55B7ees56Vh13OCuOWjzTYUm7NNHrf5J5wPuBFwBbgKuTrK2qGzuqnQrcVVVPTbISeAfwinbfzVV15BhN/2VVXZFkX+BLSV5UVZ9v911SVadP6EQlSePa3YjCRe3Pv+x3INJstGbDMKsv28j2HTthCQxv3c7qyzYCmCxo0E30+v9sYHNV3QKQ5GLgBKAzUTgBOKvdvhR4X5KM12BV/Ri4ot2+P8nXgEP2MC5J0h7a5RyFqrq2/XbotKr68ujX7hpPcnySTUk2J+kauk5yXpLr2tdNSbZ27NvZsW9tR/mhSa5q27yk/XZJGkjnrtvUJAkdtu/YybnrNo1zhDQYJnH9Xwzc2vF+S1s2Zp2qegC4Gziw3Xdokg1JvpzkuaMbT7IQ+A3gSx3FL01yfZJLkyzZoxOVJI1rt3MUqmpnkqck2beq7u+14V6Gn6vqjR31zwCO6mhi+zjDz+8Azquqi5N8kGYI+wO9xiVNp9u2bt+jcmmQTPT6Pwm3A0+uqjuSHA2sSfKMqvohQJL5wCeB946MWAB/B3yyqu5L8nvAhcDzRzec5DTgNIBFixYxNDQ0oQC3bds24WP7xZh6M0gxbd2+g+/efS8H7Psgf3Xx37Fo/0eycME+Mx0WMFif04hBi2nQ4oH+xdTrWti3AF9pv9n/0UhhVb17F8f0Mvzc6SR28xCfdmj6+cAr26ILaYavTRQ0kJ60cAHDYyQFT1q4YAaikSZkT6//w0Dnt/qH8JOJxqPrbGn/8b8/cEdVFc2qSiMjGjcDTwOuaY87H/hmVb2nI447Otr9MPDOsYKqqvPb41m2bFktX758vPPdpaGhISZ6bL8YU28GJaY1G4ZZ/aWNbN/xCN70rAd518ZHsGCfnfzFbx0xELekDsrn1GnQYhq0eKB/MfW6POrNwN+39R/bvh6zm2N6GX4GIMlTgEOB9R3Fj0xyTZKvJlnRlh0IbG2HqnfZpjQIVh13OAv2mfewsgX7zGPVcYfPUETSHtvT6//VwGHtbaL7AiuBtaPqrAVObrdPBNZXVSV5QjsaTZKfAg6jSVRI8uc0CcUbOhtKcnDH25fQrJokDSxvSdXepNcRhRur6m87C5K8bArjWAlcWlWd/+c8paqG285ifZKNNPex9mS2DjMPWjxgTLuyEPiLX5rHd+/ewQH7wuojH2TR/vuy8O5vMjT0zZkOb2A+p07G1JtpjGmPrv9V9UCS04F1NMujXlBVNyQ5G7imqtYCHwEuSrIZuJOmDwB4HnB2kh3Ag8DrqurOJIcAbwX+HfhaO+95ZBnU1yd5CfBA29YpU3XiUj94S6r2Jr0mCqtp1rDeXVmnXoafR6xk1AN8qmq4/XlLkiGa+QufBhYmmd+OKozb5mwdZh60eMCYejU0NMTLBzCmQfycjGn3pjGmPb7+V9XlwOWjyt7WsX0v0JVsVNWnaa7zo8u3AGOuilRVq9t4pL2Ct6Rqb7K75yi8CHgxsDjJezt2PY7m25tdeWj4meYf8yv5ydyCzt/xdOAA4MqOsgOAH7eT0w4CjgHe2Q5NX0EzVH0xzdD1Z3cThyRpD03y+i9pHKuOO/wny2a3vCVVg2p3Iwq30UwiewlwbUf5PcAbxzyi1ePwMzQJxMXtJLYRPwP8VZIHae6LPadjtaS3ABe396tuoBnCliRNrQlf/yWNb2TCcjMn4R4WL1zggzg1sHaZKFTV14GvJ/lEW/fJVdXzbJvdDT+3788a47j/CzxrnDZvoVlRSZLUJ5O9/ksa34qjFrPiqMUMDQ1xxquWz3Q40rh6XfXoeOA64B8AkhzZ+RA0SdKs5fVfkuaoXhOFs2i+xd8KUFXX0SxnKkma3c7C678kzUm9Jgo7qmr00qQ1Zk1J0mzi9V+S5qhel0e9IckrgXlJDgNeD/zf/oUlSRoQXv8laY7qdUThDOAZwH3AJ4EfMurpmJKkWcnrvyTNUT2NKFTVj2meivnW/oYjSRokXv8lae7a3QPXdrmyRVW9ZGrDkSQNAq//kqTdjSj8InArzXDzVUD6HpEkaRB4/ZekOW53icJ/AV4AnAS8Evgc8MmquqHfgUmSZpTXf0ma43Y5mbmqdlbVP1TVycAvAJuBoSSnT0t0kqQZ4fVfkrTbycxJ9gN+jeZbpaXAe4HP9DcsSdJM8/ovSXPb7iYzfwx4JnA58KdV9Y1piUqSNKO8/kuSdjei8NvAj4A/BF6fPDSXLUBV1eP6GJskaeZ4/ZekOW6XiUJV9fpANknSLOL1X5JkRyBJkiSpi1hY8qEAABirSURBVImCJEmSpC4mCpIkSZK6mChIkiRJ6mKiIEmSJKmLiYIkaUolOT7JpiSbk5w5xv79klzS7r8qydK2fGmS7Umua18f7Djm6CQb22Pem3a91iSPT/LFJN9sfx4wXecpSbOdiYIkacokmQe8H3gRcARwUpIjRlU7Fbirqp4KnAe8o2PfzVV1ZPt6XUf5B4DXAoe1r+Pb8jOBL1XVYcCX2veSpClgoiBJmkrPBjZX1S1VdT9wMXDCqDonABe225cCx6bjiW6jJTkYeFxVfbWqCvgYsGKMti7sKJckTZKJgiRpKi0Gbu14v6UtG7NOVT0A3A0c2O47NMmGJF9O8tyO+lvGaXNRVd3ebn8HWDQlZyFJ2vWTmSVJmka3A0+uqjuSHA2sSfKMXg+uqkpSY+1LchpwGsCiRYsYGhqaUIDbtm2b8LH9Yky9MabeGNPuDVo80L+YTBQkSVNpGFjS8f6QtmysOluSzAf2B+5obyu6D6Cqrk1yM/C0tv4h47T53SQHV9Xt7S1K3xsrqKo6HzgfYNmyZbV8+fIJndzQ0BATPbZfjKk3xtQbY9q9QYsH+hdTX2896mHli/M6Vre4KcnWtvzIJFcmuSHJ9Ule0XHMR5N8q+O4I/t5DpKkPXI1cFiSQ5PsC6wE1o6qsxY4ud0+EVjfjgY8oZ0MTZKfopm0fEt7a9EPk/xCO5fhNcBnx2jr5I5ySdIk9W1EoWPlixfQ3E96dZK1VXXjSJ2qemNH/TOAo9q3PwZeU1XfTPIk4Nok66pqa7t/VVVd2q/YJUkTU1UPJDkdWAfMAy6oqhuSnA1cU1VrgY8AFyXZDNxJk0wAPA84O8kO4EHgdVV1Z7vv94GPAguAz7cvgHOATyU5FfgP4OX9PkdJmiv6eevRQytfACQZWfnixnHqnwT8CUBV3TRSWFW3Jfke8ARg6zjHSpIGRFVdDlw+quxtHdv3Ai8b47hPA58ep81rgGeOUX4HcOwkQ5YkjaGficJYK188Z6yKSZ4CHAqsH2Pfs4F9gZs7it+e5G20a2ZX1X1jHDcrJ64NWjxgTL0ypt4YU28GMSZJ0uwyKJOZVwKXVtXOzsJ2YtpFwMlV9WBbvJpmCbx9aSamvQU4e3SDs3Xi2qDFA8bUK2PqjTH1ZhBjkiTNLv2czNzLyhcjVgKf7CxI8jjgc8Bbq+qrI+VVdXs17gP+muYWJ0mSJElTqJ+JQi8rX5Dk6cABwJUdZfsCnwE+NnrScjvKQLvyxQrgG307A0mSJGmO6tutRz2ufAFNAnFxu372iJfTrH5xYJJT2rJTquo64G+SPAEIcB3wun6dgyRJkjRX9XWOwu5WvmjfnzXGcR8HPj5Om8+fwhAlSZIkjaGvD1yTJEmStHcyUZAkSZLUxURBkiRJUhcTBUmSJEldTBQkSZIkdTFRkCRJktTFREGSJElSFxMFSZIkSV1MFCRJkiR1MVGQJEmS1MVEQZIkSVIXEwVJkiRJXUwUJEmSJHUxUZAkSZLUxURBkiRJUhcTBUmSJEldTBQkSZIkdTFRkCRNqSTHJ9mUZHOSM8fYv1+SS9r9VyVZOmr/k5NsS/Lm9v3hSa7reP0wyRvafWclGe7Y9+LpOEdJmgvmz3QAkqTZI8k84P3AC4AtwNVJ1lbVjR3VTgXuqqqnJlkJvAN4Rcf+dwOfH3lTVZuAIzvaHwY+01H/vKr6y36cjyTNZY4oSJKm0rOBzVV1S1XdD1wMnDCqzgnAhe32pcCxSQKQZAXwLeCGcdo/Fri5qv5jyiOXJD2MiYIkaSotBm7teL+lLRuzTlU9ANwNHJjkMcBbgD/dRfsrgU+OKjs9yfVJLkhywGSClyT9hLceSZIGxVk0txFtawcYHibJvsBLgNUdxR8A/gyo9ue7gN8Z49jTgNMAFi1axNDQ0IQC3LZt24SP7Rdj6o0x9caYdm/Q4oH+xWSiIEmaSsPAko73h7RlY9XZkmQ+sD9wB/Ac4MQk7wQWAg8mubeq3tce9yLga1X13ZGGOreTfAj4+7GCqqrzgfMBli1bVsuXL5/QyQ0NDTHRY/vFmHpjTL0xpt0btHigfzGZKEiSptLVwGFJDqVJCFYCrxxVZy1wMnAlcCKwvqoKeO5IhSRnAds6kgSAkxh121GSg6vq9vbtbwLfmLpTkaS5ra9zFHpYIu+8jiXtbkqytWPfyUm+2b5O7ig/OsnGts33ZqzxaUnSjGjnHJwOrAP+DfhUVd2Q5OwkL2mrfYRmTsJm4I+Arv5htCSPpllJ6bJRu97Z9gnXA78KvHGKTkWS5ry+jSj0skReVb2xo/4ZwFHt9uOBPwGW0dx3em177F0096O+FrgKuBw4no5l9CRJM6uqLqe5PneWva1j+17gZbtp46xR738EHDhGvVdPJlZJ0vj6OaLQyxJ5nTqHlI8DvlhVd7bJwReB45McDDyuqr7aDlN/DFjRv1OQJEmS5qZ+Jgq9LJEHQJKnAIcC63dz7OJ2e7dtSpIkSZq4QZnMvBK4tKp2TlWDs3UpvEGLB4ypV8bUG2PqzSDGJEmaXfqZKPSyRN6IlcAfjDp2+ahjh9ryQ3ppc7YuhTdo8YAx9cqYemNMvRnEmCRJs0s/bz16aIm89iE5K2mWxHuYJE8HDqBZJm/EOuCFSQ5on7L5QmBduwTeD5P8Qrva0WuAz/bxHCRJkqQ5qW8jClX1QJKRJfLmAReMLJEHXFNVI0nDSuDidnLyyLF3JvkzmmQD4OyqurPd/n3go8ACmtWOXPFIkiRJmmJ9naOwuyXy2vdnjXPsBcAFY5RfAzxz6qKUJEmSNFpfH7gmSZIkae9koiBJkiSpi4mCJEmSpC4mCpIkSZK6mChIkiRJ6mKiIEmSJKmLiYIkSZKkLiYKkiRJkrqYKEiSJEnqYqIgSZIkqYuJgiRJkqQuJgqSJEmSupgoSJIkSepioiBJkiSpi4mCJEmSpC4mCpKkKZXk+CSbkmxOcuYY+/dLckm7/6okS0ftf3KSbUne3FH27SQbk1yX5JqO8scn+WKSb7Y/D+jnuUnSXGKiIEmaMknmAe8HXgQcAZyU5IhR1U4F7qqqpwLnAe8Ytf/dwOfHaP5Xq+rIqlrWUXYm8KWqOgz4UvtekjQFTBQkSVPp2cDmqrqlqu4HLgZOGFXnBODCdvtS4NgkAUiyAvgWcEOPv6+zrQuBFZOIXZLUwURBkjSVFgO3drzf0paNWaeqHgDuBg5M8hjgLcCfjtFuAV9Icm2S0zrKF1XV7e32d4BFkz8FSRLA/JkOQJKk1lnAeVW1rR1g6PTLVTWc5InAF5P8e1X9U2eFqqokNVbDbXJxGsCiRYsYGhqaUIDbtm2b8LH9Yky9MabeGNPuDVo80L+YTBQkSVNpGFjS8f6QtmysOluSzAf2B+4AngOcmOSdwELgwST3VtX7qmoYoKq+l+QzNLc4/RPw3SQHV9XtSQ4GvjdWUFV1PnA+wLJly2r58uUTOrmhoSEmemy/GFNvjKk3xrR7gxYP9C8mbz2SJE2lq4HDkhyaZF9gJbB2VJ21wMnt9onA+mo8t6qWVtVS4D3A/6qq9yV5dJLHAiR5NPBC4BtjtHUy8Nl+nZgkzTWOKEiSpkxVPZDkdGAdMA+4oKpuSHI2cE1VrQU+AlyUZDNwJ00ysSuLgM+0tyPNBz5RVf/Q7jsH+FSSU4H/AF4+5SclSXOUiYIkaUpV1eXA5aPK3taxfS/wst20cVbH9i3Az41T7w7g2EmEK0kaR19vPdrdQ3faOi9PcmOSG5J8oi371fahOiOve9sl80jy0STf6th3ZD/PQZIkSZqL+jai0PHQnRfQLI93dZK1VXVjR53DgNXAMVV1V7uaBVV1BXBkW+fxwGbgCx3Nr6qqS/sVuyRJkjTX9XNEoZeH7rwWeH9V3QXNahZjtHMi8Pmq+nEfY5UkSZLUoZ+JQi8P3Xka8LQkX0ny1STHj9HOSuCTo8renuT6JOcl2W/qQpYkSZIEMz+ZeT5wGLCcZq3tf0ryrKraCtCuif0smtUzRqymefrmvjRrYr8FOHt0w7P14TqDFg8YU6+MqTfG1JtBjEmSNLv0M1Ho5aE7W4CrqmoH8K0kN9EkDle3+18OfKbdD0BV3d5u3pfkr4E3j/XLZ+vDdQYtHjCmXhlTb4ypN4MYkyRpdunnrUe9PHRnDc1oAkkOorkV6ZaO/Scx6rajdpSBNAtqr+AnD92RJEmSNEX6NqLQ40N31gEvTHIjsJNmNaM7AJIspRmR+PKopv8myROAANcBr+vXOUiSJElzVV/nKPTw0J0C/qh9jT7223RPfqaqnj/lgUqSJEl6mL4+cE2SJEnS3slEQZIkSVIXEwVJkiRJXUwUJEmSJHUxUZAkSZLUxURBkiRJUhcTBUmSJEldTBQkSZIkdTFRkCRJktTFREGSJElSFxMFSZIkSV1MFCRJkiR1MVEYx5oNwxxzzno2Dt/NMeesZ82G4ZkOSZI0Q+wTJM1F82c6gEG0ZsMwqy/byPYdO2EJDG/dzurLNgKw4qjFMxydJGk62SdImqscURjDues2NR1Ch+07dnLuuk0zFJEk7T2SHJ9kU5LNSc4cY/9+SS5p91+VZOmo/U9Osi3Jm9v3S5JckeTGJDck+cOOumclGU5yXft68VSfj32CpLnKRGEMt23dvkflkqRGknnA+4EXAUcAJyU5YlS1U4G7quqpwHnAO0btfzfw+Y73DwBvqqojgF8A/mBUm+dV1ZHt6/IpPB3APkHS3GWiMIYnLVywR+WSpIc8G9hcVbdU1f3AxcAJo+qcAFzYbl8KHJskAElWAN8CbhipXFW3V9XX2u17gH8Dpu2eH/sESXOVcxTGsOq4w39yP2prwT7zWHXc4TMYlSTtFRYDt3a83wI8Z7w6VfVAkruBA5PcC7wFeAHw5rEab29TOgq4qqP49CSvAa6hGXm4a4zjTgNOA1i0aBFDQ0M9n9Cqn9vJ8F07ebCKRQvgTc96gEckLD5g5x610y/btm0biDg6GVNvjKk3gxbToMUD/YvJRGEMI5PTmvtP72HxwgWsOu5wJ61JUn+dRXMb0bZ2gOFhkjwG+DTwhqr6YVv8AeDPgGp/vgv4ndHHVtX5wPkAy5Ytq+XLl+9RYGs2DHPuuk2sXHIPF9/62IHqE4aGhtjT8+k3Y+qNMfVm0GIatHigfzGZKIxjxVGLWXHUYoaGhjjjVctnOhxJ2lsMA0s63h/Slo1VZ0uS+cD+wB00Iw8nJnknsBB4MMm9VfW+JPvQJAl/U1WXjTRUVd8d2U7yIeDv+3BO9gmS5iQTBUnSVLoaOCzJoTQJwUrglaPqrAVOBq4ETgTWV1UBzx2pkOQsYFubJAT4CPBvVfXuzoaSHFxVt7dvfxP4xtSfkiTNTSYKkqQp0845OB1YB8wDLqiqG5KcDVxTVWtp/tF/UZLNwJ00ycSuHAO8GtiY5Lq27H+2Kxy9M8mRNLcefRv4vSk/KUmao0wUJElTqv0H/OWjyt7WsX0v8LLdtHFWx/a/AN2TFpp9r55MrJKk8bk8qiRJkqQuJgqSJEmSupgoSJIkSepioiBJkiSpS5oV6Wa3JN8H/mOChx8E/GAKw5msQYsHjKlXxtQbY+rNRGN6SlU9YaqD2ZvMsj4BjKlXxtQbY9q9QYsHJhfTuP3CnEgUJiPJNVW1bKbjGDFo8YAx9cqYemNMvRnEmOaCQfzcjak3xtQbY9q9QYsH+heTtx5JkiRJ6mKiIEmSJKmLicLunT/TAYwyaPGAMfXKmHpjTL0ZxJjmgkH83I2pN8bUG2PavUGLB/oUk3MUJEmSJHVxREGSJElSlzmbKCQ5PsmmJJuTnDnG/lOSfD/Jde3rdzv2nZzkm+3r5AGJaWdH+drpiqmt8/IkNya5IcknOspn5HPaTUwz8jklOa/j996UZGvHvpn6e9pVTDP1OT05yRVJNiS5PsmLO/atbo/blOS4mYwnydIk2zs+ow9ORTw9xvSUJF9q4xlKckjHvr78Lc0V9gtTE1NbZ9r6BfuEaYlpTvQJk4lp1vYLVTXnXsA84Gbgp4B9ga8DR4yqcwrwvjGOfTxwS/vzgHb7gJmMqd23bYY+p8OADSOfAfDEAficxoxpJj+nUfXPAC6Y6c9pvJhm+O/pfOD/abePAL7dsf11YD/g0LadeTMYz1LgGzP0Gf0tcHK7/Xzgon7+Lc2VV4+f/SnYLwxUvzCZeGbyMxpV3z5hAPqEKYhpKbOwX5irIwrPBjZX1S1VdT9wMXBCj8ceB3yxqu6sqruALwLHz3BM/dJLTK8F3t9+FlTV99rymfycxoupX/b0v91JwCfb7UH5e+qMqV96iamAx7Xb+wO3tdsnABdX1X1V9S1gc9veTMXTL73EdASwvt2+omN/v/6W5gr7hamLaTr7BfuE/sfUL4PWJ0w2pn6Z0X5hriYKi4FbO95vactGe2k7jHNpkiV7eOx0xgTwyCTXJPlqkhVTEE+vMT0NeFqSr7S/+/g9OHa6Y4KZ+5yAZniQ5tuPkf+hZ/rvaayYYOY+p7OA306yBbic5lutXo+dzngADm2Hnr+c5LmTjGVPYvo68Fvt9m8Cj01yYI/Hanz2C1MX03T2C/YJ/Y8J5kafMNmYYBb2C3M1UejF3wFLq+pnaTKwC2c4Hth1TE+p5ol8rwTek+Snpymm+TTDustpvoH4UJKF0/S7x7OrmGbqcxqxEri0qnZO8+/dlbFimqnP6STgo1V1CPBi4KIkM3mdGi+e24EnV9VRwB8Bn0jyuF20M5XeDPxKkg3ArwDDwCD9Pc1m9gu9GbR+wT5hz9gnTCymWdkvzPSHPVOGgc5vXQ5pyx5SVXdU1X3t2w8DR/d67AzERFUNtz9vAYaAo6YjJprsdG1V7WiH/26iuSDP2Oe0i5hm8nMasZKHD+fO5Oc0Xkwz+TmdCnyq/d1XAo8EDurx2GmLpx3uvqMtv5bm/tGnTTKenmKqqtuq6rfazuitbdnWHs9H47NfmKKYmN5+wT6h/zHNlT5hUjHN2n6hpnjSxd7wovl24RaaobWRiSHPGFXn4I7t3wS+Wj+ZGPItmkkhB7Tbj5/hmA4A9mu3DwK+yS4mKU1xTMcDF3b87luBA2f4cxovphn7nNp6Twe+Tfv8kpn+e9pFTDP59/R54JR2+2do7v0M8AwePnHtFiY/mXky8Txh5PfTTDAbnsa/74OAR7TbbwfO7uff0lx59fjZ2y8MWL8wyXjsE3qLaU70CVMQ06zsFyYV/N78ohkuuokm43trW3Y28JJ2+y+AG9r/IFcAT+849ndoJs5sBv77TMcE/BKwsS3fCJw6jTEFeDdwY/u7Vw7A5zRmTDP5ObXvzwLOGePYGfmcxotphv+ejgC+0v7u64AXdhz71va4TcCLZjIe4KXt/4vXAV8DfmMaP6MTaTrqm2i+Qd6v339Lc+XVw2dvv9BbTNPaL0w0npn8jNr3Z2GfMFB9wmRiYpb2Cz6ZWZIkSVKXuTpHQZIkSdIumChIkiRJ6mKiIEmSJKmLiYIkSZKkLiYKkiRJkrqYKGhOS3Jgkuva13eSDLfbW5Pc2Iffd1aSN+/hMdvGKf9okhOnJjJJkn2C9HAmCprTqnmq6ZFVdSTwQeC8dvtI4MHdHZ9kfr9jlCRND/sE6eFMFKTxzUvyoSQ3JPlCkgUASYaSvCfJNcAfJjk6yZeTXJtkXZKD23qvT3JjkuuTXNzR7hFtG7ckef1IYZI/SvKN9vWG0cGk8b4km5L8I/DEjn3ndPyuv+zbJyJJc5d9guYcM19pfIcBJ1XVa5N8iuapix9v9+1bVcuS7AN8GTihqr6f5BU0j0//HeBM4NCqui/Jwo52nw78KvBYYFOSDwA/C/x34Dk0TxG9KsmXq2pDx3G/CRxO81TIRTRPGb0gyYHtvqdXVY36XZKkqWGfoDnHREEa37eq6rp2+1pgace+S9qfhwPPBL6YBGAecHu773rgb5KsAdZ0HPu5qroPuC/J92gu8L8MfKaqfgSQ5DLguUBnp/A84JNVtRO4Lcn6tvxu4F7gI0n+Hvj7SZ21JGks9gmac7z1SBrffR3bO3l4Yv2j9meAG0buaa2qZ1XVC9t9vwa8H/h54OqOe1d31e4eq6oHgGcDlwK/DvzDZNqTJI3JPkFzjomCNDmbgCck+UWAJPskeUaSRwBLquoK4C3A/sBjdtHOPwMrkjwqyaNpho3/eVSdfwJekWRee8/rr7a/8zHA/lV1OfBG4Oem8PwkSb2zT9Cs4q1H0iRU1f3tcnTvTbI/zf9T7wFuAj7elgV4b1VtbYeix2rna0k+CvxrW/ThUfeiAnwGeD7Nfaj/CVzZlj8W+GySR7a/64+m6vwkSb2zT9Bsk6qa6RgkSZIkDRhvPZIkSZLUxURBkiRJUhcTBUmSJEldTBQkSZIkdTFRkCRJktTFREGSJElSFxMFSZIkSV1MFCRJkiR1+f8BivRZ77SVU/EAAAAASUVORK5CYII=\n","text/plain":["<Figure size 921.6x345.6 with 2 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000,"output_embedded_package_id":"1-0nfLjIAQoXvZZKJj93y0FI4bRqCRFjf"},"id":"kg2N-2bauWWC","executionInfo":{"status":"ok","timestamp":1622046380823,"user_tz":-120,"elapsed":51350,"user":{"displayName":"Rafael Nicolas Trozzo","photoUrl":"","userId":"14921564160525915517"}},"outputId":"f40659c3-2d4d-46bd-b1a4-2d5d171d3f80"},"source":["# Load input and ground truth\n","for i_img in range(100):\n","  x_sample = x_test[i_img]\n","  y_ground_truth = y_test[i_img]\n","\n","  x_sample_input = np.expand_dims(x_sample, 0)\n","  prediction = loaded_model.predict(x_sample_input)\n","  threshold = 0.9\n","  predicted_mask = np.where(prediction > threshold, 1, 0)[0]\n","\n","  # Channel Correction\n","  if CHANNEL_CORRECTION:\n","    predicted_mask[:,:,[8, 12]] = predicted_mask[:,:,[12, 8]]\n","    predicted_mask[:,:,[13, 15]] = predicted_mask[:,:,[15, 13]]\n","    predicted_mask[:,:,[9, 10]] = predicted_mask[:,:,[10, 9]]\n","    predicted_mask[:,:,[6, 7]] = predicted_mask[:,:,[7, 6]]\n","\n","  plot_labels = np.append(categories_names, 'background') if ADD_BACKGROUND else categories_names\n","  # plot_labeled_masks(x_sample, y_ground_truth, predicted_mask, plot_labels)\n","  plot_labeled_results(x_sample, y_ground_truth, predicted_mask, plot_labels)\n","\n","  y_ground_truth_iou = np.expand_dims(y_ground_truth, 0)\n","  predicted_mask_iou = np.expand_dims(predicted_mask, 0)\n","\n","  print(f\"Total IoU score is {iou_score(y_ground_truth_iou, predicted_mask_iou, ADD_BACKGROUND)}\")\n","  print(f\"IoU score only for ground truth with objects in it is {only_true_iou_score(y_ground_truth_iou, predicted_mask_iou, ADD_BACKGROUND)}\")"],"execution_count":47,"outputs":[{"output_type":"display_data","data":{"text/plain":"Output hidden; open in https://colab.research.google.com to view."},"metadata":{}}]}]}