Fully_Supervised_from_Scratch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Linear_Evaluation_Flower_Full_Dataset_Supervised.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.1"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ayulockin/SwAV-TF/blob/master/Fully_Supervised_from_Scratch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9auttsxh1u0k",
        "colab_type": "text"
      },
      "source": [
        "## Initial setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "cZlN8EEM6NVC",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "!pip install wandb"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "R1c99jQL0KMr",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow_datasets as tfds\n",
        "\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras import models\n",
        "\n",
        "import matplotlib.pyplot as plt \n",
        "import numpy as np\n",
        "import random\n",
        "import time\n",
        "import os\n",
        "\n",
        "tf.random.set_seed(666)\n",
        "np.random.seed(666)\n",
        "\n",
        "tfds.disable_progress_bar()"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ATfRWo0x6Vkp",
        "colab": {}
      },
      "source": [
        "import wandb\n",
        "from wandb.keras import WandbCallback\n",
        "\n",
        "wandb.login()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "vnhWEo8_12FK"
      },
      "source": [
        "## Dataset gathering and preparation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "BAOhOFMs110h",
        "colab": {}
      },
      "source": [
        "# Gather Flowers dataset\n",
        "train_ds, validation_ds = tfds.load(\n",
        "    \"tf_flowers\",\n",
        "    split=[\"train[:85%]\", \"train[85%:]\"],\n",
        "    as_supervised=True\n",
        ")\n",
        "\n",
        "AUTO = tf.data.experimental.AUTOTUNE\n",
        "BATCH_SIZE = 128\n",
        "\n",
        "@tf.function\n",
        "def scale_resize_image(image, label):\n",
        "    image = tf.image.convert_image_dtype(image, tf.float32)\n",
        "    image = tf.image.resize(image, (224, 224)) # Resizing to highest resolution used while training swav\n",
        "    return (image, label)\n",
        "\n",
        "training_ds = (\n",
        "    train_ds\n",
        "    .map(scale_resize_image, num_parallel_calls=AUTO)\n",
        "    .batch(BATCH_SIZE)\n",
        "    .prefetch(AUTO)\n",
        ")\n",
        "\n",
        "testing_ds = (\n",
        "    validation_ds\n",
        "    .map(scale_resize_image, num_parallel_calls=AUTO)\n",
        "    .batch(BATCH_SIZE)\n",
        "    .prefetch(AUTO)\n",
        ")"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "eqOBLHJa5Ztp"
      },
      "source": [
        "## ResNet50 from scratch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Znqofzfr46-4",
        "colab": {}
      },
      "source": [
        "def get_training_model():\n",
        "    inputs = layers.Input(shape=(224, 224, 3))\n",
        "    EXTRACTOR = tf.keras.applications.ResNet50(weights=None, include_top=False,\n",
        "        input_shape=(224, 224, 3))\n",
        "    x = EXTRACTOR(inputs, training=True)\n",
        "    x = layers.GlobalAveragePooling2D()(x)\n",
        "    x = layers.Dense(5, activation=\"softmax\")(x)\n",
        "    classifier = models.Model(inputs=inputs, outputs=x)\n",
        "    \n",
        "    return classifier"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qtt4jmzzxe8Y",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "outputId": "a3d1dfb4-a04a-4873-c804-aa7fb77d30b7"
      },
      "source": [
        "model = get_training_model()\n",
        "model.summary()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_3 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
            "_________________________________________________________________\n",
            "resnet50 (Functional)        (None, 7, 7, 2048)        23587712  \n",
            "_________________________________________________________________\n",
            "global_average_pooling2d_1 ( (None, 2048)              0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 5)                 10245     \n",
            "=================================================================\n",
            "Total params: 23,597,957\n",
            "Trainable params: 23,544,837\n",
            "Non-trainable params: 53,120\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "I0o5LHV0pCPu"
      },
      "source": [
        "## Callback"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "vECu62eyCdut",
        "colab": {}
      },
      "source": [
        "# Early Stopping to prevent overfitting\n",
        "early_stopper = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=2, verbose=2, restore_best_weights=True)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Rp1fJIEYpFIC"
      },
      "source": [
        "## Without Augmentation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "HdC-897fDXxC"
      },
      "source": [
        "#### Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "RArUvFToDJKn",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 612
        },
        "outputId": "24f406b7-b1b6-4f31-8844-aaab9167af7f"
      },
      "source": [
        "# get model and compile\n",
        "tf.keras.backend.clear_session()\n",
        "model = get_training_model()\n",
        "model.summary()\n",
        "\n",
        "model.compile(loss=\"sparse_categorical_crossentropy\", metrics=[\"acc\"],\n",
        "                     optimizer=\"adam\")\n",
        "\n",
        "# initialize wandb run\n",
        "wandb.init(entity='authors', project='swav-tf', id='resnet50-scratch')\n",
        "\n",
        "# train \n",
        "history = model.fit(training_ds,\n",
        "                 validation_data=(testing_ds),\n",
        "                 epochs=100,\n",
        "                 callbacks=[WandbCallback(),\n",
        "                            early_stopper])"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
            "_________________________________________________________________\n",
            "resnet50 (Functional)        (None, 7, 7, 2048)        23587712  \n",
            "_________________________________________________________________\n",
            "global_average_pooling2d (Gl (None, 2048)              0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 5)                 10245     \n",
            "=================================================================\n",
            "Total params: 23,597,957\n",
            "Trainable params: 23,544,837\n",
            "Non-trainable params: 53,120\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://app.wandb.ai/authors/swav-tf\" target=\"_blank\">https://app.wandb.ai/authors/swav-tf</a><br/>\n",
              "                Run page: <a href=\"https://app.wandb.ai/authors/swav-tf/runs/resnet50-scratch\" target=\"_blank\">https://app.wandb.ai/authors/swav-tf/runs/resnet50-scratch</a><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/100\n",
            "25/25 [==============================] - 19s 740ms/step - loss: 2.1911 - acc: 0.4321 - val_loss: 1.4640 - val_acc: 0.5091\n",
            "Epoch 2/100\n",
            "25/25 [==============================] - 17s 664ms/step - loss: 1.1474 - acc: 0.5497 - val_loss: 1.1207 - val_acc: 0.5982\n",
            "Epoch 3/100\n",
            "25/25 [==============================] - 17s 677ms/step - loss: 0.9998 - acc: 0.6321 - val_loss: 0.9639 - val_acc: 0.6509\n",
            "Epoch 4/100\n",
            "25/25 [==============================] - 17s 673ms/step - loss: 0.8823 - acc: 0.6686 - val_loss: 0.9426 - val_acc: 0.6600\n",
            "Epoch 5/100\n",
            "25/25 [==============================] - 17s 668ms/step - loss: 0.8131 - acc: 0.7042 - val_loss: 0.9311 - val_acc: 0.6364\n",
            "Epoch 6/100\n",
            "25/25 [==============================] - 16s 621ms/step - loss: 0.7122 - acc: 0.7397 - val_loss: 0.9827 - val_acc: 0.6582\n",
            "Epoch 7/100\n",
            "25/25 [==============================] - ETA: 0s - loss: 0.6778 - acc: 0.7513Restoring model weights from the end of the best epoch.\n",
            "25/25 [==============================] - 16s 623ms/step - loss: 0.6778 - acc: 0.7513 - val_loss: 1.0428 - val_acc: 0.6509\n",
            "Epoch 00007: early stopping\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "QMUITZxpDaS8"
      },
      "source": [
        "#### Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "KNpchslXDPFR",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "a4c08942-0b57-426b-f374-a5cf70fcdc1d"
      },
      "source": [
        "loss, acc = model.evaluate(testing_ds)\n",
        "wandb.log({'Test Accuracy': round(acc*100, 2)})"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "5/5 [==============================] - 1s 177ms/step - loss: 0.9311 - acc: 0.6364\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "966mDKFhqsrk"
      },
      "source": [
        "# Training with Augmentation\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "iPlQ3JKtq6j-"
      },
      "source": [
        "#### Augmentation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "hACfOEyQnk4r",
        "colab": {}
      },
      "source": [
        "# Configs\n",
        "CROP_SIZE = 224\n",
        "MIN_SCALE = 0.5\n",
        "MAX_SCALE = 1.\n",
        "\n",
        "# Experimental options\n",
        "options = tf.data.Options()\n",
        "options.experimental_optimization.noop_elimination = True\n",
        "options.experimental_optimization.map_vectorization.enabled = True\n",
        "options.experimental_optimization.apply_default_optimizations = True\n",
        "options.experimental_deterministic = False\n",
        "options.experimental_threading.max_intra_op_parallelism = 1"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "hU2OZh1oTFaN",
        "colab": {}
      },
      "source": [
        "@tf.function\n",
        "def scale_image(image, label):\n",
        "\timage = tf.image.convert_image_dtype(image, tf.float32)\n",
        "\treturn (image, label)\n",
        "\n",
        "@tf.function\n",
        "def random_apply(func, x, p):\n",
        "\treturn tf.cond(\n",
        "\t\ttf.less(tf.random.uniform([], minval=0, maxval=1, dtype=tf.float32),\n",
        "\t\t\t\ttf.cast(p, tf.float32)),\n",
        "\t\tlambda: func(x),\n",
        "\t\tlambda: x)\n",
        " \n",
        "@tf.function\n",
        "def random_resize_crop(image, label):\n",
        "  # Conditional resizing\n",
        "  image = tf.image.resize(image, (260, 260))\n",
        "  # Get the crop size for given min and max scale\n",
        "  size = tf.random.uniform(shape=(1,), minval=MIN_SCALE*260,\n",
        "\t\t          maxval=MAX_SCALE*260, dtype=tf.float32)\n",
        "  size = tf.cast(size, tf.int32)[0]\n",
        "  # Get the crop from the image\n",
        "  crop = tf.image.random_crop(image, (size,size,3))\n",
        "  crop_resize = tf.image.resize(crop, (CROP_SIZE, CROP_SIZE))\n",
        "  \n",
        "  return crop_resize, label\n",
        "\n",
        "@tf.function\n",
        "def tie_together(image, label):\n",
        "  # Scale the pixel values\n",
        "  image, label = scale_image(image , label)\n",
        "  # random horizontal flip\n",
        "  image = random_apply(tf.image.random_flip_left_right, image, p=0.5)\n",
        "  # Random resized crops\n",
        "  image, label = random_resize_crop(image, label)\n",
        "  \n",
        "  return image, label"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "MLzlomx_d2iO",
        "colab": {}
      },
      "source": [
        "trainloader = (\n",
        "\ttrain_ds\n",
        "\t.shuffle(1024)\n",
        "\t.map(tie_together, num_parallel_calls=AUTO)\n",
        "\t.batch(BATCH_SIZE)\n",
        "\t.prefetch(AUTO)\n",
        ")\n",
        "\n",
        "trainloader = trainloader.with_options(options)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "y3CxMB95EVo9"
      },
      "source": [
        "#### Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Pat13QzdtsL3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 544
        },
        "outputId": "e75b9b7e-554f-4e34-c404-2975d68e3d1f"
      },
      "source": [
        "# get model and compile\n",
        "tf.keras.backend.clear_session()\n",
        "model = get_training_model()\n",
        "model.summary()\n",
        "\n",
        "model.compile(loss=\"sparse_categorical_crossentropy\", metrics=[\"acc\"],\n",
        "                     optimizer=\"adam\")\n",
        "\n",
        "# initialize wandb run\n",
        "wandb.init(entity='authors', project='swav-tf', id='resnet50-scratch-aug')\n",
        "\n",
        "# train \n",
        "history = model.fit(training_ds,\n",
        "                 validation_data=(testing_ds),\n",
        "                 epochs=100,\n",
        "                 callbacks=[WandbCallback(),\n",
        "                            early_stopper])"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
            "_________________________________________________________________\n",
            "resnet50 (Functional)        (None, 7, 7, 2048)        23587712  \n",
            "_________________________________________________________________\n",
            "global_average_pooling2d (Gl (None, 2048)              0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 5)                 10245     \n",
            "=================================================================\n",
            "Total params: 23,597,957\n",
            "Trainable params: 23,544,837\n",
            "Non-trainable params: 53,120\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://app.wandb.ai/authors/swav-tf\" target=\"_blank\">https://app.wandb.ai/authors/swav-tf</a><br/>\n",
              "                Run page: <a href=\"https://app.wandb.ai/authors/swav-tf/runs/resnet50-scratch-aug\" target=\"_blank\">https://app.wandb.ai/authors/swav-tf/runs/resnet50-scratch-aug</a><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/100\n",
            "25/25 [==============================] - 18s 706ms/step - loss: 2.2764 - acc: 0.4083 - val_loss: 1.4937 - val_acc: 0.5527\n",
            "Epoch 2/100\n",
            "25/25 [==============================] - 17s 683ms/step - loss: 1.2415 - acc: 0.5865 - val_loss: 1.2379 - val_acc: 0.6200\n",
            "Epoch 3/100\n",
            "25/25 [==============================] - 17s 675ms/step - loss: 1.0328 - acc: 0.6314 - val_loss: 0.9624 - val_acc: 0.6582\n",
            "Epoch 4/100\n",
            "25/25 [==============================] - 16s 622ms/step - loss: 0.9649 - acc: 0.6615 - val_loss: 0.9770 - val_acc: 0.6436\n",
            "Epoch 5/100\n",
            "25/25 [==============================] - ETA: 0s - loss: 0.8654 - acc: 0.6936Restoring model weights from the end of the best epoch.\n",
            "25/25 [==============================] - 16s 627ms/step - loss: 0.8654 - acc: 0.6936 - val_loss: 1.0189 - val_acc: 0.6418\n",
            "Epoch 00005: early stopping\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "LstPmZTHEiVQ"
      },
      "source": [
        "#### Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7M_IdQv9EiVd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "3190963a-8109-455c-ba1f-9c0fbc308b04"
      },
      "source": [
        "loss, acc = model.evaluate(testing_ds)\n",
        "wandb.log({'Test Accuracy': round(acc*100, 2)})"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "5/5 [==============================] - 1s 194ms/step - loss: 0.9624 - acc: 0.6582\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}