NNFS/05_nnfs_calculating_network_error_with_loss.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/theaveas/DeepLearning/blob/main/NNFS/05_nnfs_calculating_network_error_with_loss.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "1e00d438-847c-4f3f-8284-88d231385c1b",
      "metadata": {
        "id": "1e00d438-847c-4f3f-8284-88d231385c1b"
      },
      "source": [
        "# Calculating Network Error with Loss\n",
        "`Loss function`, also referred to as the `cost function`, is the algorithm that quantifies how wrong a model is.\\\n",
        "`Loss` is the measure of this metric. we ideally want it to be 0."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "id": "db664100-278e-42f9-b37c-571c29fa5056",
      "metadata": {
        "id": "db664100-278e-42f9-b37c-571c29fa5056",
        "outputId": "20544f1f-5279-46cb-9370-2d1d87d0ddf4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "3.7.12\n",
            "1.19.5\n",
            "3.2.2\n"
          ]
        }
      ],
      "source": [
        "import platform\n",
        "import numpy as np\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "print(platform.python_version())\n",
        "print(np.__version__)\n",
        "print(matplotlib. __version__)\n",
        "\n",
        "#python version 3.9.7\n",
        "#numpy version 1.21.2\n",
        "#matplotlib version 3.5.0"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# install library\n",
        "!pip install nnfs"
      ],
      "metadata": {
        "id": "RVEBJydKCr5x",
        "outputId": "9ff6d8f3-9f47-4df7-d584-423b3c7740b0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "id": "RVEBJydKCr5x",
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting nnfs\n",
            "  Downloading nnfs-0.5.1-py3-none-any.whl (9.1 kB)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from nnfs) (1.19.5)\n",
            "Installing collected packages: nnfs\n",
            "Successfully installed nnfs-0.5.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "id": "a6a246fe-2487-4f88-ac11-451452337a9b",
      "metadata": {
        "id": "a6a246fe-2487-4f88-ac11-451452337a9b"
      },
      "source": [
        "## Categorical Cross-Entropy Loss\n",
        "Is explicitly used to compare a `groud-truth` probability(y or labels) and some `predicted` distribution  (y-hat or predictions)\n",
        "\\\n",
        "`L = - sum(yi,j * log(yhat i,j)` "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "id": "393b90f4-e368-42dc-a655-1a9e0ab14a19",
      "metadata": {
        "id": "393b90f4-e368-42dc-a655-1a9e0ab14a19",
        "outputId": "468756c4-f0d5-4949-c36a-ca0fb88ccaba",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.35667494393873245\n"
          ]
        }
      ],
      "source": [
        "# log loss(correct_class_confidence)\n",
        "# Li = -log(yhat i,k)  # where k is an index of \"true\" probability\n",
        "softmax_output = [0.7, 0.1, 0.2]\n",
        "\n",
        "# targets of [1, 0, 0]\n",
        "m = len(softmax_output)\n",
        "loss = - np.sum(1 * np.log(0.7) + 0 * np.log(0.1) + 0 * np.log(0.2))\n",
        "print(loss)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "id": "06fb6f50-e696-4299-8f8c-23621ba3f572",
      "metadata": {
        "id": "06fb6f50-e696-4299-8f8c-23621ba3f572",
        "outputId": "d8de006b-8434-460f-9bd6-91ac75bd89ed",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-0.0\n",
            "0.05129329438755058\n",
            "0.10536051565782628\n",
            "0.2231435513142097\n",
            "1.6094379124341003\n",
            "2.3025850929940455\n",
            "2.995732273553991\n",
            "4.605170185988091\n"
          ]
        }
      ],
      "source": [
        "# the categorical cross entropy loss account for the larger the outputs the larger the confidence is\n",
        "output = [1., 0.95, 0.9, 0.8, 0.2, 0.1, 0.05, 0.01]\n",
        "\n",
        "for conf in output:\n",
        "    print(-np.log(conf))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "id": "50a332a6-5741-4d35-abf2-cdab8b7edf0f",
      "metadata": {
        "id": "50a332a6-5741-4d35-abf2-cdab8b7edf0f",
        "outputId": "8f6bcd7d-0bd9-4d45-ff0d-e336fd25cbb6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.6486586255873816\n",
            "e**1.6486586255873816 is  5.2\n"
          ]
        }
      ],
      "source": [
        "# log is short for logarithm\n",
        "# logarithm with e as its base is referred to as the \"natural logarithm\"\n",
        "# this equation solve the term e**x = b, e**x = 5.2 is solved by log(5.2)\n",
        "\n",
        "b = 5.2\n",
        "print(np.log(b))\n",
        "print('e**1.6486586255873816 is ', np.exp(1.6486586255873816))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "id": "29211dcf-dc77-4fad-a4a9-f8b41f640f17",
      "metadata": {
        "id": "29211dcf-dc77-4fad-a4a9-f8b41f640f17",
        "outputId": "96807db0-678d-4c58-c5b4-c0916c541c01",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.7\n",
            "0.5\n",
            "0.9\n"
          ]
        }
      ],
      "source": [
        "# calculate categorical cross-entropy loss\n",
        "# consider the output from softmax activation function is \n",
        "softmax_outputs = np.array([[0.7, 0.1, 0.2], \n",
        "                             [0.1, 0.5, 0.4], \n",
        "                             [0.02, 0.9, 0.08]])\n",
        "\n",
        "# with three training example with three prediction probability\n",
        "# the target is dog, cat, cat \n",
        "# 0 = dog, 1 = cat, 2 = human\n",
        "class_targets = np.array([0, 1, 1])\n",
        "\n",
        "for targ_idx, distribution in zip(class_targets, softmax_outputs):\n",
        "    print(distribution[targ_idx])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "id": "deb85a19-7614-4ff9-b18c-372ef95c6b22",
      "metadata": {
        "id": "deb85a19-7614-4ff9-b18c-372ef95c6b22",
        "outputId": "3924795a-5ba2-4a58-e681-dfa5fc641fec",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.7 0.5 0.9]\n",
            "softmax output probability distribution [0.35667494 0.69314718 0.10536052]\n"
          ]
        }
      ],
      "source": [
        "# print distribution using numpy array\n",
        "print(softmax_outputs[range(len(softmax_outputs)), class_targets]) # y i,j where i is the index of the output, j is the index of the class target\n",
        "\n",
        "# calculate the categorical cross-entropy loss\n",
        "print('softmax output probability distribution', -np.log(softmax_outputs[range(len(softmax_outputs)), class_targets]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "id": "14ab20f1-1135-41c6-ab6b-bf0701d49e5b",
      "metadata": {
        "id": "14ab20f1-1135-41c6-ab6b-bf0701d49e5b",
        "outputId": "948ad877-762f-48a2-fc8d-f8c7b1b53d53",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The average loss of this batch 0.38506088005216804\n"
          ]
        }
      ],
      "source": [
        "# What we need is the average loss per batch to have an idea how the model perform\n",
        "# average equation sum(iterable) / len(iterable)\n",
        "\n",
        "m = len(softmax_outputs)\n",
        "average_loss = 1/m * np.sum(-np.log(softmax_outputs[range(len(softmax_outputs)), class_targets]))\n",
        "print('The average loss of this batch', average_loss)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "id": "90577141-d6bf-42f9-a5ac-3351094a645d",
      "metadata": {
        "id": "90577141-d6bf-42f9-a5ac-3351094a645d",
        "outputId": "4703b81d-157b-4f32-e2c8-4d94f8c0536a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "len of sparse class: 1\n",
            "len of sparse class: 2\n"
          ]
        }
      ],
      "source": [
        "# different between sparse data and one hot encoded\n",
        "# same as our class target [dog, cat, cat]\n",
        "sparse_target = np.array([0, 1, 1])\n",
        "\n",
        "one_hot_encoded = np.array([[1,0,0],\n",
        "                            [0,1,0],\n",
        "                            [0,1,0]])\n",
        "print('len of sparse class:', len(sparse_target.shape))\n",
        "print('len of sparse class:', len(one_hot_encoded.shape))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "id": "8e6de411-932e-448f-8736-65a34ed5861a",
      "metadata": {
        "id": "8e6de411-932e-448f-8736-65a34ed5861a",
        "outputId": "537806cf-6572-4d59-c8a1-becf2a387fb0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.38506088005216804\n"
          ]
        }
      ],
      "source": [
        "# encounter different target class types (sparse, one hot encoded)\n",
        "\n",
        "# probabilities for target values only if categorical labes\n",
        "if len(class_targets.shape) == 1:\n",
        "    correct_conf = softmax_outputs[range(len(softmax_outputs)), class_targets]\n",
        "# one hot encoded\n",
        "elif len(class_targets.shape) == 2:\n",
        "    correct_conf = np.sum(softmax_outputs * class_targets, axis=1, keepdims=True)\n",
        "\n",
        "# compute loss\n",
        "m = len(softmax_outputs)\n",
        "average_loss = 1/m * np.sum(-np.log(softmax_outputs[range(len(softmax_outputs)), class_targets]))\n",
        "print(average_loss)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "id": "54bec2a4-0f07-4fa6-ab59-7262415c8f42",
      "metadata": {
        "id": "54bec2a4-0f07-4fa6-ab59-7262415c8f42",
        "outputId": "4f11a170-91d7-4e09-b9ea-c9d5f563c0cf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "16.11809565095832\n",
            "-9.999999505838704e-08\n"
          ]
        }
      ],
      "source": [
        "# -np.log(0) = inf\n",
        "# >>> __main__:1: RuntimeWarning: divide by zero encountered in log\n",
        "# >>> inf\n",
        "\n",
        "# the problem is np.mean([1, 2, 3, -np.log(0)])\n",
        "# >>> inf\n",
        "\n",
        "# we could add a very small value to the confidence to prevent it from being a zero 1e-7\n",
        "print(-np.log(1e-7))\n",
        "\n",
        "# but this could impact the result insignificantly, \n",
        "# and in case where the conf value is 1: conf in the correct label loss becomes a negative value instead of being 0\n",
        "print(-np.log(1 + 1e-7))\n",
        "\n",
        "# to prevent this from happening instead of being 1 + 1e-7, will become 1-1e-7(so slightly less than 1)\n",
        "def y_pred_clipped(y_pred):\n",
        "    return np.clip(y_pred, 1e-7, 1-1e-7)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "0bd84423-6917-47e9-abfa-0f201bb2960d",
      "metadata": {
        "id": "0bd84423-6917-47e9-abfa-0f201bb2960d"
      },
      "source": [
        "# The Categoriacal Cross-Entropy Loss Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "id": "c624bd63-46fe-426d-82b4-deba65676c2a",
      "metadata": {
        "id": "c624bd63-46fe-426d-82b4-deba65676c2a"
      },
      "outputs": [],
      "source": [
        "# common loss class\n",
        "class Loss:\n",
        "\n",
        "    def calc(self, output, y):\n",
        "        \"\"\" Calculate the data and regularization losses\n",
        "        Input: \n",
        "            output : The output of the activation function\n",
        "            y : Class targets\n",
        "            \n",
        "        Output:\n",
        "            data_loss : Average Loss\n",
        "        \"\"\"\n",
        "        data_loss = self.forward(output, y)\n",
        "        \n",
        "        return data_loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "id": "6f300694-d2d3-4348-90f9-76df2a313453",
      "metadata": {
        "id": "6f300694-d2d3-4348-90f9-76df2a313453"
      },
      "outputs": [],
      "source": [
        "# categorical cross entropy loss \n",
        "class Loss_CategoricalCrossentropy(Loss):\n",
        "    \n",
        "    def forward(self, y_pred, y):\n",
        "        \"\"\"Calculate Cross-entropy loss\n",
        "        Input : \n",
        "            y_pred : The output of the activation function\n",
        "            y : Class targets\n",
        "            \n",
        "        Output :\n",
        "            average_loss : Average Loss\n",
        "        \"\"\"\n",
        "        # len of training example\n",
        "        m = len(y_pred)\n",
        "        \n",
        "        # clip data to prevent divison by 0\n",
        "        # clip both sides to not drag mean toward any values\n",
        "        y_pred_clipped = np.clip(y_pred, 1e-7, 1-1e-7)\n",
        "        \n",
        "        # probabilities for target values only if categorical labels\n",
        "        if len(y.shape) == 1:\n",
        "            correct_conf = y_pred_clipped[range(len(y_pred_clipped)), y]\n",
        "        # one hot encoded\n",
        "        elif len(y.shape) == 2:\n",
        "            correct_conf = np.sum(y_pred_clipped * y, axis=1, keepdims=True)\n",
        "\n",
        "        # compute loss\n",
        "        average_loss = 1/m * np.sum(-np.log(y_pred_clipped[range(len(y_pred_clipped)), y]))\n",
        "        \n",
        "        return average_loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "id": "64403b00-9c62-4ce8-88ae-e3d780ab7ee9",
      "metadata": {
        "id": "64403b00-9c62-4ce8-88ae-e3d780ab7ee9",
        "outputId": "23c4c6f6-a453-4b48-a593-0ada8a91bce8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.38506088005216804\n"
          ]
        }
      ],
      "source": [
        "# testing\n",
        "loss_function = Loss_CategoricalCrossentropy()\n",
        "loss = loss_function.forward(softmax_outputs, class_targets)\n",
        "print(loss)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "f599b056-05fb-4b78-a909-bd4ea552ae60",
      "metadata": {
        "id": "f599b056-05fb-4b78-a909-bd4ea552ae60"
      },
      "source": [
        "## Accuracy Calculation\n",
        "Accuracy describes how often the largest confidence is the correct class in terms of a fraction"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "id": "1486c8fd-708a-4db5-8486-85534e61a0f3",
      "metadata": {
        "id": "1486c8fd-708a-4db5-8486-85534e61a0f3",
        "outputId": "54259d69-005c-4d16-c71c-b822b7e0fc8e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.0\n"
          ]
        }
      ],
      "source": [
        "# calculate values along second axis (axis of index 1)\n",
        "class_targets = np.array([0,1,1])\n",
        "preds = np.argmax(softmax_outputs, axis=1)\n",
        "# convert one-hot encoded\n",
        "if len(class_targets.shape) == 2:\n",
        "    class_targets = np.argmax(class_targets, axis=1)\n",
        "\n",
        "# true evaluates to 1; false to 0\n",
        "acc = np.mean(preds==class_targets)\n",
        "print(acc)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "6cab8276-3002-4309-91dd-d66998e824da",
      "metadata": {
        "id": "6cab8276-3002-4309-91dd-d66998e824da"
      },
      "source": [
        "---\n",
        "## Combining everything up to this point"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "id": "e0848b29-eb59-43ff-a67a-9fcd8d83c041",
      "metadata": {
        "id": "e0848b29-eb59-43ff-a67a-9fcd8d83c041"
      },
      "outputs": [],
      "source": [
        "# import dataset \n",
        "import nnfs\n",
        "from nnfs.datasets import spiral_data\n",
        "\n",
        "# set random seed to 0, create float32 dtype, overrides the original dot product from Numpy\n",
        "nnfs.init()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "id": "84c5f749-11df-4201-ac10-57e457da4562",
      "metadata": {
        "id": "84c5f749-11df-4201-ac10-57e457da4562"
      },
      "outputs": [],
      "source": [
        "class Dense:\n",
        "    def __init__(self, n_inputs, n_neurons):\n",
        "        \"\"\" Initialize the weights and biases of each neurons\n",
        "        n_inputs = number of input features\n",
        "        n_neurons = number of desired neurons\n",
        "        \"\"\"\n",
        "        # using np.random.randn and * 0.01 is to break the symetry of the neurons\n",
        "        self.weights = np.random.randn(n_inputs, n_neurons) * 0.01\n",
        "        # biases can be initialize as zeros\n",
        "        self.biases = np.zeros((1, n_neurons))\n",
        "    \n",
        "    def forward(self, inputs):\n",
        "        \"\"\" Calculate the output layer using The Dot product of input feature and weight plus bias\n",
        "        Input:\n",
        "        inputs = Training examples\n",
        "        \n",
        "        Output:\n",
        "        output = Output of the training example\n",
        "        \"\"\"\n",
        "        # calculate the output layer\n",
        "        output = np.dot(inputs, self.weights) + self.biases\n",
        "        \n",
        "        return output"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "id": "5614aad3-d40b-4669-ae33-41c46d43c81c",
      "metadata": {
        "id": "5614aad3-d40b-4669-ae33-41c46d43c81c"
      },
      "outputs": [],
      "source": [
        "# ReLU activation\n",
        "class Activation_ReLU:\n",
        "    def forward(self, inputs):\n",
        "        output = np.maximum(0, inputs)\n",
        "        \n",
        "        return output"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "id": "09706ee7-fc20-4c6a-8686-f66895f4e81c",
      "metadata": {
        "id": "09706ee7-fc20-4c6a-8686-f66895f4e81c"
      },
      "outputs": [],
      "source": [
        "# Sotfmax activation\n",
        "class Activation_Softmax:\n",
        "    def forward(self, inputs):\n",
        "        # input - np.max to prevent the exponential function from overflowing\n",
        "        exp_values = np.exp(inputs - np.max(inputs, axis=1, keepdims=True))\n",
        "        \n",
        "        softmax = exp_values / np.sum(exp_values, axis=1, keepdims=True)\n",
        "        return softmax"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "id": "23b99e80-3f2c-4526-be57-9b5fd2747c7a",
      "metadata": {
        "id": "23b99e80-3f2c-4526-be57-9b5fd2747c7a"
      },
      "outputs": [],
      "source": [
        "# common loss class\n",
        "class Loss:\n",
        "\n",
        "    def calc(self, output, y):\n",
        "        \"\"\" Calculate the data and regularization losses\n",
        "        Input: \n",
        "            output : The output of the activation function\n",
        "            y : Class targets\n",
        "            \n",
        "        Output:\n",
        "            data_loss : Average Loss\n",
        "        \"\"\"\n",
        "        data_loss = self.forward(output, y)\n",
        "        \n",
        "        return data_loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "id": "f9841d79-f622-4bd0-9f3f-473897313812",
      "metadata": {
        "id": "f9841d79-f622-4bd0-9f3f-473897313812"
      },
      "outputs": [],
      "source": [
        "# categorical cross entropy loss \n",
        "class Loss_CategoricalCrossentropy(Loss):\n",
        "    \n",
        "    def forward(self, y_pred, y):\n",
        "        \"\"\"Calculate Cross-entropy loss\n",
        "        Input : \n",
        "            y_pred : The output of the activation function\n",
        "            y : Class targets\n",
        "            \n",
        "        Output :\n",
        "            average_loss : Average Loss\n",
        "        \"\"\"\n",
        "        # len of training example\n",
        "        m = len(y_pred)\n",
        "        \n",
        "        # clip data to prevent divison by 0\n",
        "        # clip both sides to not drag mean toward any values\n",
        "        y_pred_clipped = np.clip(y_pred, 1e-7, 1-1e-7)\n",
        "        \n",
        "        # probabilities for target values only if categorical labels\n",
        "        if len(y.shape) == 1:\n",
        "            correct_conf = y_pred_clipped[range(len(y_pred_clipped)), y]\n",
        "        # one hot encoded\n",
        "        elif len(y.shape) == 2:\n",
        "            correct_conf = np.sum(y_pred_clipped * y, axis=1, keepdims=True)\n",
        "\n",
        "        # compute loss\n",
        "        average_loss = 1/m * np.sum(-np.log(y_pred_clipped[range(len(y_pred_clipped)), y]))\n",
        "        \n",
        "        return average_loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "id": "93b5a3e8-4685-4696-a032-085ed1ed9edc",
      "metadata": {
        "id": "93b5a3e8-4685-4696-a032-085ed1ed9edc",
        "outputId": "cf250c6b-293c-43ea-cf5c-2a89d9c223d9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[0.33333334 0.33333334 0.33333334]\n",
            " [0.33332068 0.33335868 0.33332068]\n",
            " [0.3332981  0.33340386 0.3332981 ]\n",
            " [0.3332748  0.3334504  0.3332748 ]\n",
            " [0.33325398 0.33349204 0.33325398]]\n",
            "1.098567097981771\n"
          ]
        }
      ],
      "source": [
        "# create dataset\n",
        "X, y = spiral_data(samples=100, classes=3)\n",
        "\n",
        "# create dense layer with 2 input features and 3 output values\n",
        "l1 = Dense(2, 3)\n",
        "a1 = Activation_ReLU()\n",
        "\n",
        "# create dense layer with 3 input features and 3 output values\n",
        "l2 = Dense(3, 3)\n",
        "a2 = Activation_Softmax()\n",
        "\n",
        "# forward pass through activation func\n",
        "yhat1 = a1.forward(l1.forward(X))\n",
        "yhat2 = a2.forward(yhat1)\n",
        "\n",
        "# compute loss\n",
        "loss_function = Loss_CategoricalCrossentropy()\n",
        "loss = loss_function.forward(yhat2, y)\n",
        "\n",
        "print(yhat2[:5])\n",
        "print(loss)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "id": "2c6647c9-ea60-46b1-b30a-5cf1ab8fd8bd",
      "metadata": {
        "id": "2c6647c9-ea60-46b1-b30a-5cf1ab8fd8bd",
        "outputId": "1ba22701-ce7d-44de-8021-eb843ef18697",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.3333333333333333\n"
          ]
        }
      ],
      "source": [
        "# calculate accuraccy\n",
        "preds = np.argmax(yhat2, axis=1)\n",
        "# convert one-hot encoded\n",
        "if len(y.shape) == 2:\n",
        "    y = np.argmax(y, axis=1)\n",
        "\n",
        "# true evaluates to 1; false to 0\n",
        "acc = np.mean(preds==y)\n",
        "print(acc)"
      ]
    }
  ],
  "metadata": {
    "interpreter": {
      "hash": "4bc200a3b7399c7a1efb5de92c3fe5e746477b0833a798de6c61fa3b81d0c1fb"
    },
    "kernelspec": {
      "display_name": "torch110",
      "language": "python",
      "name": "torch110"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    },
    "colab": {
      "name": "05_nnfs_calculating_network_error_with_loss.ipynb",
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}