lenet/keras/LeNet_Keras_v2_custom_Subsampling_layer_and_activation.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": null
      },
      "outputs": [],
      "source": [
        "# Copyright 2022 Google LLC\n",
        "#\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "#      http://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": null
      },
      "source": [
        "[![View on GitHub][github-badge]][github-keras-v2] [![Open In Colab][colab-badge]][colab-keras-v2] [![Open in Binder][binder-badge]][binder-keras-v2]\n",
        "\n",
        "[github-badge]: https://img.shields.io/badge/View-on%20GitHub-blue?logo=GitHub\n",
        "[colab-badge]: https://colab.research.google.com/assets/colab-badge.svg\n",
        "[binder-badge]: https://static.mybinder.org/badge_logo.svg\n",
        "\n",
        "[github-keras-v2]: https://github.com/mbrukman/reimplementing-ml-papers/blob/main/lenet/keras/LeNet_Keras_v2_custom_Subsampling_layer_and_activation.ipynb\n",
        "[colab-keras-v2]: https://colab.research.google.com/github/mbrukman/reimplementing-ml-papers/blob/main/lenet/keras/LeNet_Keras_v2_custom_Subsampling_layer_and_activation.ipynb\n",
        "[binder-keras-v2]: https://mybinder.org/v2/gh/mbrukman/reimplementing-ml-papers/main?filepath=lenet/keras/LeNet_Keras_v2_custom_Subsampling_layer_and_activation.ipynb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": null
      },
      "outputs": [],
      "source": [
        "%%bash\n",
        "\n",
        "readonly GH_USER=\"mbrukman\"\n",
        "readonly GH_REPO=\"reimplementing-ml-papers\"\n",
        "readonly GH_BRANCH=\"main\"\n",
        "\n",
        "# Download the library for processing MNIST dataset as well as our custom\n",
        "# Subsampling layer, activation function, and LeNet model.\n",
        "for path in datasets/mnist/mnist_keras.py \\\n",
        "            lenet/keras/{activations,lenet,subsampling}.py ; do\n",
        "  module=\"$(basename \"${path}\")\"\n",
        "  if ! [ -f \"${module}\" ]; then\n",
        "    curl -s -o \"${module}\" \"https://raw.githubusercontent.com/${GH_USER}/${GH_REPO}/${GH_BRANCH}/${path}\"\n",
        "  fi\n",
        "done"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": null
      },
      "outputs": [],
      "source": [
        "from tensorflow import keras\n",
        "\n",
        "# Local imports downloaded above.\n",
        "from activations import scaled_tanh\n",
        "from lenet import LeNet\n",
        "from subsampling import Subsampling\n",
        "\n",
        "from mnist_keras import MNIST"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": null
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"LeNet-5\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " C1 (Conv2D)                 (None, 28, 28, 6)         156       \n",
            "                                                                 \n",
            " S2 (Subsampling)            (None, 14, 14, 6)         12        \n",
            "                                                                 \n",
            " S2_act (Activation)         (None, 14, 14, 6)         0         \n",
            "                                                                 \n",
            " C3 (Conv2D)                 (None, 10, 10, 16)        2416      \n",
            "                                                                 \n",
            " S4 (Subsampling)            (None, 5, 5, 16)          32        \n",
            "                                                                 \n",
            " S4_act (Activation)         (None, 5, 5, 16)          0         \n",
            "                                                                 \n",
            " C5 (Conv2D)                 (None, 1, 1, 120)         48120     \n",
            "                                                                 \n",
            " Flatten (Flatten)           (None, 120)               0         \n",
            "                                                                 \n",
            " F6 (Dense)                  (None, 84)                10164     \n",
            "                                                                 \n",
            " Output (Dense)              (None, 10)                850       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 61750 (241.21 KB)\n",
            "Trainable params: 61750 (241.21 KB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# Define the model architecture.\n",
        "model = LeNet(subsampling=Subsampling, activation=scaled_tanh)\n",
        "\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": null
      },
      "outputs": [],
      "source": [
        "# Compile the model with optimizer and loss function.\n",
        "opt = keras.optimizers.Adam(learning_rate=0.001)\n",
        "loss_fn = keras.losses.CategoricalCrossentropy()\n",
        "model.compile(optimizer=opt, loss=loss_fn, metrics=['accuracy'])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": null
      },
      "source": [
        "For details on the MNIST dataset including a data exploration, see [MNIST directory in my repo](https://github.com/mbrukman/reimplementing-ml-papers/tree/main/datasets/mnist).\n",
        "\n",
        "Here, we will import a shared library to process the MNIST dataset into the format that we need to use below for model training and testing."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": null
      },
      "outputs": [],
      "source": [
        "%%capture --no-stderr\n",
        "\n",
        "# This will download the MNIST dataset via the Keras library which outputs data\n",
        "# to stdout, so we silence it above to avoid extraneous output.\n",
        "mnist_data = MNIST()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": null
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "1875/1875 [==============================] - 16s 4ms/step - loss: 0.2747 - accuracy: 0.9146\n",
            "Epoch 2/20\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0920 - accuracy: 0.9722\n",
            "Epoch 3/20\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0606 - accuracy: 0.9815\n",
            "Epoch 4/20\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0451 - accuracy: 0.9860\n",
            "Epoch 5/20\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0352 - accuracy: 0.9890\n",
            "Epoch 6/20\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0309 - accuracy: 0.9903\n",
            "Epoch 7/20\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0245 - accuracy: 0.9917\n",
            "Epoch 8/20\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0196 - accuracy: 0.9934\n",
            "Epoch 9/20\n",
            "1875/1875 [==============================] - 7s 3ms/step - loss: 0.0193 - accuracy: 0.9932\n",
            "Epoch 10/20\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0173 - accuracy: 0.9942\n",
            "Epoch 11/20\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0134 - accuracy: 0.9955\n",
            "Epoch 12/20\n",
            "1875/1875 [==============================] - 7s 3ms/step - loss: 0.0136 - accuracy: 0.9954\n",
            "Epoch 13/20\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0121 - accuracy: 0.9960\n",
            "Epoch 14/20\n",
            "1875/1875 [==============================] - 7s 3ms/step - loss: 0.0112 - accuracy: 0.9961\n",
            "Epoch 15/20\n",
            "1875/1875 [==============================] - 7s 3ms/step - loss: 0.0092 - accuracy: 0.9967\n",
            "Epoch 16/20\n",
            "1875/1875 [==============================] - 7s 3ms/step - loss: 0.0110 - accuracy: 0.9966\n",
            "Epoch 17/20\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0106 - accuracy: 0.9964\n",
            "Epoch 18/20\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0073 - accuracy: 0.9978\n",
            "Epoch 19/20\n",
            "1875/1875 [==============================] - 7s 4ms/step - loss: 0.0083 - accuracy: 0.9972\n",
            "Epoch 20/20\n",
            "1875/1875 [==============================] - 7s 3ms/step - loss: 0.0101 - accuracy: 0.9967\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fcbd001ee80>"
            ]
          },
          "execution_count": null,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Train the model.\n",
        "#\n",
        "# In this notebook, we scale the input into the range [0.0, 1.0] and convert the\n",
        "# labels y to a categorical (one-hot) encoding from the default numeric values.\n",
        "#\n",
        "# For consistency, we use the same transformations for the test dataset below.\n",
        "model.fit(mnist_data.x_train_scale_0_1(),\n",
        "          mnist_data.y_train_categorical(),\n",
        "          epochs=20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": null
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "313/313 [==============================] - 1s 3ms/step - loss: 0.0683 - accuracy: 0.9853\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[0.06825553625822067, 0.9853000044822693]"
            ]
          },
          "execution_count": null,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Evaluate the model.\n",
        "#\n",
        "# Note that we use the same input range scaling and label encoding as above.\n",
        "model.evaluate(mnist_data.x_test_scale_0_1(), mnist_data.y_test_categorical())"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "LeNet Keras v2: custom Subsampling layer and activation function"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}