CNNfinal.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CNNfinal.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Mr-nvk/convolutionalNeuralNetwork/blob/master/CNNfinal.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "zME0Y5l4WGeI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import time"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "KjDGwtGDmrcu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Code to read csv file into Colaboratory:\n",
        "\n",
        "!pip install -U -q PyDrive\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "# Authenticate and create the PyDrive client.\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WnDDXYQ5m0mS",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "trainlink = \"https://drive.google.com/open?id=1U4Yrb5z2ciLXr1YE2jGkMnV1losEYktG\"\n",
        "testlink = \"https://drive.google.com/open?id=11axtXwQjC3ig6TPiq101eWv76bw9qm-i\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1IEVCQ1Fm5jP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "fluff, id = trainlink.split('=')\n",
        "downloaded = drive.CreateFile({'id':id}) \n",
        "downloaded.GetContentFile('fashion-mnist_train.csv')  \n",
        "train = pd.read_csv('fashion-mnist_train.csv')\n",
        "\n",
        "\n",
        "fluff, id = testlink.split('=')\n",
        "downloaded = drive.CreateFile({'id':id}) \n",
        "downloaded.GetContentFile('fashion-mnist_test.csv')  \n",
        "test = pd.read_csv('fashion-mnist_test.csv')\n",
        "\n",
        "# Dataset is now stored in a Pandas Dataframe"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "a-R-ENbgnEFe",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "trainY = train[\"label\"]\n",
        "del train[\"label\"]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1auMurxYnU1Z",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def onehotencoding(y,num_classes):\n",
        "  y_true = []\n",
        "  for i in y:\n",
        "    temp_list = [0]*num_classes\n",
        "    temp_list[i] = 1\n",
        "    y_true.append(temp_list)\n",
        "  return y_true"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ygvnqbBsoOgo",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "y_true = onehotencoding(trainY,10)\n",
        "x = np.array(train)\n",
        "y_true = np.array(y_true)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZK_0a4vdh2C5",
        "colab_type": "code",
        "outputId": "dd31924e-ca8b-4378-ebb6-7df777ab5728",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "trainx = x\n",
        "trainy_true = y_true\n",
        "trainy_true.shape, trainx.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((60000, 10), (60000, 784))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 230
        }
      ]
    },
    {
      "metadata": {
        "id": "rc2nMsJuodV4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "testy = (test[\"label\"])\n",
        "testy_true = np.array(onehotencoding(trainY,10))\n",
        "\n",
        "del test[\"label\"]\n",
        "testx = np.array(test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4ya-GZcsZgE_",
        "colab_type": "code",
        "outputId": "ea4c2571-0c33-4509-87cb-912a1735d1c5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "testx.shape, testy_true.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((10000, 784), (60000, 10))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 232
        }
      ]
    },
    {
      "metadata": {
        "id": "4qCUrpDWpCTl",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "img_size = 28\n",
        "img_size_flat = 28*28\n",
        "img_shape = (img_size,img_size)\n",
        "num_classes = 10\n",
        "num_channels = 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vJrDcAUuqTdh",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Convolutional Layer 1.\n",
        "filter_size1 = 5          # Convolution filters are 5 x 5 pixels.\n",
        "num_filters1 = 16         # There are 16 of these filters.\n",
        "\n",
        "# Convolutional Layer 2.\n",
        "filter_size2 = 5          # Convolution filters are 5 x 5 pixels.\n",
        "num_filters2 = 36         # There are 36 of these filters.\n",
        "\n",
        "# Fully-connected layer.\n",
        "fc_size = 128             # Number of neurons in fully-connected layer."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "NAYz2mzhWQ6Q",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def new_weights(shape):\n",
        "  return tf.Variable(tf.truncated_normal(shape, stddev = 0.5))\n",
        "\n",
        "def new_bias(length):\n",
        "  return tf.Variable(tf.constant(0.05, shape=[length]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "I3-LIZH2XWR8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def new_convolutional_layer(input,\n",
        "                           number_of_input_channels,\n",
        "                           filter_size,\n",
        "                           num_filters,\n",
        "                           use_pooling=True):\n",
        "  shape = [filter_size,filter_size,number_of_input_channels,num_filters]\n",
        "  \n",
        "  weights = new_weights(shape=shape)\n",
        "  biases = new_bias(length=num_filters)\n",
        "  \n",
        "  layer = tf.nn.conv2d(input=input,\n",
        "                       filter=weights,\n",
        "                       strides=[1,1,1,1],\n",
        "                       padding='SAME') + biases\n",
        "  \n",
        "  if use_pooling:\n",
        "    layer = tf.nn.max_pool(value=layer,\n",
        "                           ksize=[1, 2, 2, 1],\n",
        "                           strides=[1, 2, 2, 1],\n",
        "                           padding='SAME')\n",
        "    \n",
        "  layer = tf.nn.relu(layer)\n",
        "  \n",
        "  return layer, weights"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "RjlYS0q3gg6E",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def flatten_layer(layer):\n",
        "  layer_shape = layer.get_shape()\n",
        "  \n",
        "  num_features = layer_shape[1:4].num_elements()\n",
        "  \n",
        "  layer_flat = tf.reshape(layer, [-1,num_features])\n",
        "  \n",
        "  return layer_flat,num_features"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Wetx3iozikx2",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def new_fc_layer(input,num_inputs,num_outputs,use_relu):\n",
        "  weights = new_weights(shape=[num_inputs,num_outputs])\n",
        "  bias = new_bias(length=num_outputs)\n",
        "  layer = (tf.matmul(input,weights) + bias)\n",
        "  if use_relu:\n",
        "    layer = tf.nn.relu(layer)\n",
        "  return layer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "obFwuTSWjd6C",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "x = tf.placeholder(tf.float32, shape=[None,img_size_flat])\n",
        "x_image = tf.reshape(x, [-1,img_size,img_size,num_channels])\n",
        "\n",
        "y_true = tf.placeholder(tf.float32, shape=[None,num_classes])\n",
        "y_true_cls = tf.argmax(y_true, axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "PPdnLGjNlxVC",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "layer_conv1, weights_conv1 = new_convolutional_layer(input=x_image,\n",
        "                                                     number_of_input_channels=num_channels,\n",
        "                                                     filter_size=filter_size1,\n",
        "                                                     num_filters=num_filters1,\n",
        "                                                     use_pooling=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WkchrMV3o8pY",
        "colab_type": "code",
        "outputId": "97b267d5-9e67-4be6-e3b3-17dc0fc30976",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "layer_conv1"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'Relu_12:0' shape=(?, 14, 14, 16) dtype=float32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 241
        }
      ]
    },
    {
      "metadata": {
        "id": "h6L8PMX8sCjR",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "layer_conv2, weights_conv2 = new_convolutional_layer(input=layer_conv1,\n",
        "                                                     number_of_input_channels=num_filters1,\n",
        "                                                     filter_size=filter_size2,\n",
        "                                                     num_filters=num_filters2,\n",
        "                                                     use_pooling=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7UNSYhnSstHt",
        "colab_type": "code",
        "outputId": "a6c7607a-68fa-4d04-fc9e-5a6a91a4f06a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "layer_conv2"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'Relu_13:0' shape=(?, 7, 7, 36) dtype=float32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 243
        }
      ]
    },
    {
      "metadata": {
        "id": "74tfTjDTsu1u",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "layer_flat,num_features = flatten_layer(layer_conv2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "uo2uesL5tCG1",
        "colab_type": "code",
        "outputId": "8761bfe0-d3fb-4096-bce3-f9504d462089",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "layer_flat"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'Reshape_9:0' shape=(?, 1764) dtype=float32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 245
        }
      ]
    },
    {
      "metadata": {
        "id": "EX_l43DPtDrj",
        "colab_type": "code",
        "outputId": "9a9a7a1e-09a8-4359-ab62-be8d34170936",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "num_features"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1764"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 246
        }
      ]
    },
    {
      "metadata": {
        "id": "4cuN8s1etJhH",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "layer_fc1 = new_fc_layer(input=layer_flat,\n",
        "                     num_inputs=num_features,\n",
        "                     num_outputs=fc_size,\n",
        "                     use_relu=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "M-5Aiv97t6d7",
        "colab_type": "code",
        "outputId": "fd68687c-74ad-4ae7-c1c9-b1bd3bf77a80",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "layer_fc1"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'Relu_14:0' shape=(?, 128) dtype=float32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 248
        }
      ]
    },
    {
      "metadata": {
        "id": "jgJY7KJ3uC6r",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "layer_fc2 = new_fc_layer(input=layer_fc1,\n",
        "                     num_inputs=fc_size,\n",
        "                     num_outputs=num_classes,\n",
        "                     use_relu=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3hbrcETbuQ7T",
        "colab_type": "code",
        "outputId": "bbd07938-6141-42f7-c141-0b12ff259d3c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "layer_fc2"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'add_19:0' shape=(?, 10) dtype=float32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 250
        }
      ]
    },
    {
      "metadata": {
        "id": "Q_z4Y359uVBB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "y_pred = tf.nn.softmax(layer_fc2)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "dZ_ZbWlbujHz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "y_pred_cls = tf.argmax(y_pred, axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "xW5nHJuFuou-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=layer_fc2,\n",
        "                                                       labels=y_true)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "d8zd0GervLbE",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "cost = tf.reduce_mean(cross_entropy)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "GrsMSk1BvdyM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "optimizer = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cost)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "oh4rNu8tvut8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "correct_prediction = tf.equal(y_pred_cls, y_true_cls)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "glt0Fd9Mv9Po",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "DMDZ9i3Fv_8R",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "session = tf.Session()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "eXHt2yC46tKw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "session.run(tf.global_variables_initializer())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "LAnyoi9R64tj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_batch_size = 60"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Sow2iVk_7QBI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "total_iteration = 0\n",
        "def optimize(num_iteration):\n",
        "  global total_iteration\n",
        "  \n",
        "  start_time = time.time()\n",
        "  for i in range(total_iteration, total_iteration + num_iteration):\n",
        "    #generate data in batch\n",
        "    random_number = np.random.randint(0,10000, size=[60])\n",
        "    x_batch,y_true_batch = trainx[random_number], trainy_true[random_number]\n",
        "    \n",
        "    feed_dict_train = {x: x_batch,\n",
        "                      y_true: y_true_batch}\n",
        "    \n",
        "    session.run(optimizer, feed_dict=feed_dict_train)\n",
        "    \n",
        "    if i%1000 == 0:\n",
        "      acc = session.run(accuracy, feed_dict=feed_dict_train)\n",
        "      msg = \"Optimization Iteration: {0:>6}, Training Accuracy: {1:>6.1%}\"\n",
        "      print(msg.format(i, acc))\n",
        "  total_iteration += num_iteration\n",
        "  end_time = time.time()\n",
        "  time_diff = end_time - start_time\n",
        "  \n",
        "  print(\"Time usage: \" + str(timedelta(seconds=int(round(time_diff)))))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "RSdb4-yP-RgA",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# test_batch_size = 200\n",
        "def print_test_accuracy(testx,testy_true,testy):\n",
        "  num_test = testx.shape[0]\n",
        "  feed_dict = {x: testx,\n",
        "               y_true: testy_true}\n",
        "  cls_pred = session.run(y_pred_cls, feed_dict=feed_dict)\n",
        "  \n",
        "  correct = (cls_pred == testy)\n",
        "  correct_sum = correct.sum()\n",
        "  \n",
        "  accuracy = float((correct_sum/testx.shape[0]))\n",
        "  msg = \"Accuracy on Test-Set: {0:.1%} ({1} / {2})\"\n",
        "  print(msg.format(accuracy, correct_sum, num_test))\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "f47-d1ykiKg1",
        "colab_type": "code",
        "outputId": "ac2688f2-0c1d-4101-b89f-ab7507e941ac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "print_test_accuracy(testx,testy_true,testy)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy on Test-Set: 9.2% (916 / 10000)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "_kRIMrETdp6o",
        "colab_type": "code",
        "outputId": "1accd437-ad6f-46c5-bfcf-d9d0e2f602f2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "cell_type": "code",
      "source": [
        "optimize(num_iteration=1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Optimization Iteration:      1, Training Accuracy:   3.3%\n",
            "0.13512396812438965\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "tlAFenCTh2pB",
        "colab_type": "code",
        "outputId": "d9653f12-6dd3-4f05-b820-a4f44a13265b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "print_test_accuracy(testx,testy_true,testy)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy on Test-Set: 9.2% (916 / 10000)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "udSFKm0ZicAM",
        "colab_type": "code",
        "outputId": "93870677-8075-4c75-cc96-b43d0a390f20",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "cell_type": "code",
      "source": [
        "optimize(num_iteration=100)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Optimization Iteration:    101, Training Accuracy:  25.0%\n",
            "8.628565549850464\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "bV41D5aIjFi7",
        "colab_type": "code",
        "outputId": "28f59c4e-29ab-47bc-c31d-d24612da608f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "print_test_accuracy(testx,testy_true,testy)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy on Test-Set: 28.6% (2857 / 10000)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "L1w2kXGynV_L",
        "colab_type": "code",
        "outputId": "ae2221e7-d4b4-438b-cdf9-c04ad3fb6fd1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208
        }
      },
      "cell_type": "code",
      "source": [
        "optimize(num_iteration=1000)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Optimization Iteration:      0, Training Accuracy:  21.7%\n",
            "Optimization Iteration:    100, Training Accuracy:  50.0%\n",
            "Optimization Iteration:    200, Training Accuracy:  55.0%\n",
            "Optimization Iteration:    300, Training Accuracy:  55.0%\n",
            "Optimization Iteration:    400, Training Accuracy:  56.7%\n",
            "Optimization Iteration:    500, Training Accuracy:  60.0%\n",
            "Optimization Iteration:    600, Training Accuracy:  76.7%\n",
            "Optimization Iteration:    700, Training Accuracy:  68.3%\n",
            "Optimization Iteration:    800, Training Accuracy:  76.7%\n",
            "Optimization Iteration:    900, Training Accuracy:  73.3%\n",
            "84.01229548454285\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "zPvSJdWxnl_z",
        "colab_type": "code",
        "outputId": "d731e781-8dea-4d55-eea7-70c97cce520d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "print_test_accuracy(testx,testy_true,testy)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy on Test-Set: 70.9% (7088 / 10000)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "exm4AUFOoUJU",
        "colab_type": "code",
        "outputId": "5bb85eed-7a07-43fa-db4f-bb78d8942094",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 428
        }
      },
      "cell_type": "code",
      "source": [
        "optimize(num_iteration=10000)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Optimization Iteration:      0, Training Accuracy:  83.3%\n",
            "Optimization Iteration:   1000, Training Accuracy:  73.3%\n",
            "Optimization Iteration:   2000, Training Accuracy:  85.0%\n",
            "Optimization Iteration:   3000, Training Accuracy:  85.0%\n",
            "Optimization Iteration:   4000, Training Accuracy:  80.0%\n",
            "Optimization Iteration:   5000, Training Accuracy:  85.0%\n",
            "Optimization Iteration:   6000, Training Accuracy:  81.7%\n",
            "Optimization Iteration:   7000, Training Accuracy:  78.3%\n",
            "Optimization Iteration:   8000, Training Accuracy:  91.7%\n",
            "Optimization Iteration:   9000, Training Accuracy:  91.7%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-291-a644cbe1d467>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_iteration\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-290-42d3ad504164>\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(num_iteration)\u001b[0m\n\u001b[1;32m     22\u001b[0m   \u001b[0mtime_diff\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mend_time\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m   \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Time usage: \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimedelta\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseconds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mround\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime_diff\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'timedelta' is not defined"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "GCeKGEAboljh",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "print_test_accuracy(testx,testy_true,testy)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "mlX3qwOzpLE-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}