Basics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyToch Deep Learning Basics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define a Tensor\n",
    "x = torch.Tensor(2,3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above line creates a tensor of shape (2,3) => (rows,cols) filled with zero float values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "1.00000e-08 *\n",
       "  1.7745  0.0000  0.0000\n",
       "  0.0000  1.0140  1.0140\n",
       "[torch.FloatTensor of size 2x3]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Tensors filled with random values\n",
    "y = torch.rand(2,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 0.5164  0.0844  0.9822\n",
       " 0.3529  0.9475  0.2728\n",
       "[torch.FloatTensor of size 2x3]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Multiplication and addition of tensors\n",
    "x1 = torch.ones(2,3)\n",
    "y1 = torch.ones(2,3) * 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 10  10  10\n",
       " 10  10  10\n",
       "[torch.FloatTensor of size 2x3]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1 + y1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1  1  1\n",
       " 1  1  1\n",
       "[torch.FloatTensor of size 2x3]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 9  9  9\n",
       " 9  9  9\n",
       "[torch.FloatTensor of size 2x3]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Numpy Slice Functionality is also available\n",
    "y1[:,1] = y1[:,1] + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "  9  10   9\n",
       "  9  10   9\n",
       "[torch.FloatTensor of size 2x3]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Autograd in PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import Dependencies\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define a Variable from a Tensor\n",
    "x = Variable(torch.ones(2,2) * 3, requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 3  3\n",
       " 3  3\n",
       "[torch.FloatTensor of size 2x2]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the above code line, we define a Variable using PyTorch. We give it the tensor of (2,2) Ones and multiply by 3. Then we set the \"requires_grad\" flag to True. Setting this flag to true means that this Variable is trainable and when we do a backpropagation on this Variable, we'll see the gradients being updated and the change in weights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# New Variable using Previous Variable\n",
    "z = 2 * (x * x) + 9 * x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 45  45\n",
       " 45  45\n",
       "[torch.FloatTensor of size 2x2]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "z = 2 x^2 + 9x\n",
    "\n",
    "dz/dx = 4x + 9\n",
    "\n",
    "For, x = 3, \n",
    "\n",
    "Gradient:   dz/dx = 12 + 9 = 21"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Compute the Gradients using Backpropagation\n",
    "# The backprop will be computed w.r.t some values, so we provide ones(2,2)\n",
    "# So, the calculation becomes: d/dx.\n",
    "z.backward(torch.ones(2,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 21  21\n",
       " 21  21\n",
       "[torch.FloatTensor of size 2x2]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print the Gradients of x\n",
    "x.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The computed gradient values match with the ones we derived above."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Neural Network in PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import Dependencies\n",
    "from torch import nn\n",
    "from torch.nn import functional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define the Neural Network Class\n",
    "class neuralNetwork(nn.Module):\n",
    "    # Define the Skeleton of our Neural Network\n",
    "    # Initialize Layers every time\n",
    "    def __init__(self):\n",
    "        \n",
    "        # Overwrite the inherited Function with the Instance of base \"nn.Module\" Class\n",
    "        super(neuralNetwork, self).__init__()\n",
    "        \n",
    "        # Define the Layers of Neural Network\n",
    "        self.input_dim = 28*28\n",
    "        self.h1 = 200\n",
    "        self.h2 = 200\n",
    "        self.out = 10\n",
    "        \n",
    "        self.fc1 = nn.Linear(self.input_dim, self.h1)\n",
    "        self.fc2 = nn.Linear(self.h1, self.h2)\n",
    "        self.fc3 = nn.Linear(self.h2, self.out)\n",
    "        \n",
    "    # Overwrite the Dummy \"Forward\" method in the Base Class \"nn.Model\"\n",
    "    # Function to do the Forward Pass in the Neural Network\n",
    "    # x: Input Data\n",
    "    def forward(self,x):\n",
    "        # First Layer with ReLU Activation\n",
    "        x = functional.relu(self.fc1(x))\n",
    "        # Update x with second Layer Values\n",
    "        x = functional.relu(self.fc2(x))\n",
    "        # Output Layer with Softmax Activation for 10 Ouput Classes\n",
    "        x = self.fc3(x)\n",
    "        return functional.log_softmax(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create an Instance of the Network\n",
    "model = neuralNetwork()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "neuralNetwork(\n",
       "  (fc1): Linear(in_features=784, out_features=200)\n",
       "  (fc2): Linear(in_features=200, out_features=200)\n",
       "  (fc3): Linear(in_features=200, out_features=10)\n",
       ")"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print the Model Summary\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Train the Model\n",
    "from torch import optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Learning Rate\n",
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define Optimizer: Stochastic Gradient Descent\n",
    "# model.parameters(): Passes all the parameters of network to optimizer\n",
    "optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)\n",
    "\n",
    "# Define Loss Function: Negative Log Likelihood\n",
    "loss = nn.NLLLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Dataset\n",
    "from torchvision import datasets, transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Batch Size\n",
    "batch_size = 200\n",
    "\n",
    "# Epochs\n",
    "epochs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Training Dataset\n",
    "train_loader = torch.utils.data.DataLoader(datasets.MNIST('./dataset/', train=True, download=True,\n",
    "                                           transform=transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])),\n",
    "                                           batch_size=batch_size, shuffle=True)\n",
    "    \n",
    "# Load Test Data\n",
    "test_loader = torch.utils.data.DataLoader(datasets.MNIST('./dataset/', train=False, \n",
    "                                                         transform=transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])),\n",
    "                                          batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Hare Krishna\\AppData\\Roaming\\Python\\Python36\\site-packages\\ipykernel_launcher.py:30: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.303520\n",
      "Train Epoch: 0 [2000/60000 (3%)]\tLoss: 2.218448\n",
      "Train Epoch: 0 [4000/60000 (7%)]\tLoss: 1.980745\n",
      "Train Epoch: 0 [6000/60000 (10%)]\tLoss: 1.415943\n",
      "Train Epoch: 0 [8000/60000 (13%)]\tLoss: 0.844651\n",
      "Train Epoch: 0 [10000/60000 (17%)]\tLoss: 0.652560\n",
      "Train Epoch: 0 [12000/60000 (20%)]\tLoss: 0.542761\n",
      "Train Epoch: 0 [14000/60000 (23%)]\tLoss: 0.498504\n",
      "Train Epoch: 0 [16000/60000 (27%)]\tLoss: 0.460597\n",
      "Train Epoch: 0 [18000/60000 (30%)]\tLoss: 0.381994\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.373688\n",
      "Train Epoch: 0 [22000/60000 (37%)]\tLoss: 0.445366\n",
      "Train Epoch: 0 [24000/60000 (40%)]\tLoss: 0.333127\n",
      "Train Epoch: 0 [26000/60000 (43%)]\tLoss: 0.403760\n",
      "Train Epoch: 0 [28000/60000 (47%)]\tLoss: 0.281598\n",
      "Train Epoch: 0 [30000/60000 (50%)]\tLoss: 0.340894\n",
      "Train Epoch: 0 [32000/60000 (53%)]\tLoss: 0.275561\n",
      "Train Epoch: 0 [34000/60000 (57%)]\tLoss: 0.352527\n",
      "Train Epoch: 0 [36000/60000 (60%)]\tLoss: 0.286704\n",
      "Train Epoch: 0 [38000/60000 (63%)]\tLoss: 0.388468\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.250552\n",
      "Train Epoch: 0 [42000/60000 (70%)]\tLoss: 0.241903\n",
      "Train Epoch: 0 [44000/60000 (73%)]\tLoss: 0.315438\n",
      "Train Epoch: 0 [46000/60000 (77%)]\tLoss: 0.216478\n",
      "Train Epoch: 0 [48000/60000 (80%)]\tLoss: 0.216098\n",
      "Train Epoch: 0 [50000/60000 (83%)]\tLoss: 0.169930\n",
      "Train Epoch: 0 [52000/60000 (87%)]\tLoss: 0.249781\n",
      "Train Epoch: 0 [54000/60000 (90%)]\tLoss: 0.391666\n",
      "Train Epoch: 0 [56000/60000 (93%)]\tLoss: 0.286169\n",
      "Train Epoch: 0 [58000/60000 (97%)]\tLoss: 0.293359\n",
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 0.211786\n",
      "Train Epoch: 1 [2000/60000 (3%)]\tLoss: 0.262004\n",
      "Train Epoch: 1 [4000/60000 (7%)]\tLoss: 0.224741\n",
      "Train Epoch: 1 [6000/60000 (10%)]\tLoss: 0.242064\n",
      "Train Epoch: 1 [8000/60000 (13%)]\tLoss: 0.210828\n",
      "Train Epoch: 1 [10000/60000 (17%)]\tLoss: 0.163035\n",
      "Train Epoch: 1 [12000/60000 (20%)]\tLoss: 0.302525\n",
      "Train Epoch: 1 [14000/60000 (23%)]\tLoss: 0.140421\n",
      "Train Epoch: 1 [16000/60000 (27%)]\tLoss: 0.202121\n",
      "Train Epoch: 1 [18000/60000 (30%)]\tLoss: 0.157380\n",
      "Train Epoch: 1 [20000/60000 (33%)]\tLoss: 0.135225\n",
      "Train Epoch: 1 [22000/60000 (37%)]\tLoss: 0.220576\n",
      "Train Epoch: 1 [24000/60000 (40%)]\tLoss: 0.239909\n",
      "Train Epoch: 1 [26000/60000 (43%)]\tLoss: 0.139490\n",
      "Train Epoch: 1 [28000/60000 (47%)]\tLoss: 0.131523\n",
      "Train Epoch: 1 [30000/60000 (50%)]\tLoss: 0.149486\n",
      "Train Epoch: 1 [32000/60000 (53%)]\tLoss: 0.153453\n",
      "Train Epoch: 1 [34000/60000 (57%)]\tLoss: 0.251109\n",
      "Train Epoch: 1 [36000/60000 (60%)]\tLoss: 0.164483\n",
      "Train Epoch: 1 [38000/60000 (63%)]\tLoss: 0.249405\n",
      "Train Epoch: 1 [40000/60000 (67%)]\tLoss: 0.113978\n",
      "Train Epoch: 1 [42000/60000 (70%)]\tLoss: 0.216571\n",
      "Train Epoch: 1 [44000/60000 (73%)]\tLoss: 0.219106\n",
      "Train Epoch: 1 [46000/60000 (77%)]\tLoss: 0.211145\n",
      "Train Epoch: 1 [48000/60000 (80%)]\tLoss: 0.092735\n",
      "Train Epoch: 1 [50000/60000 (83%)]\tLoss: 0.166621\n",
      "Train Epoch: 1 [52000/60000 (87%)]\tLoss: 0.113228\n",
      "Train Epoch: 1 [54000/60000 (90%)]\tLoss: 0.165292\n",
      "Train Epoch: 1 [56000/60000 (93%)]\tLoss: 0.183650\n",
      "Train Epoch: 1 [58000/60000 (97%)]\tLoss: 0.143198\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 0.148653\n",
      "Train Epoch: 2 [2000/60000 (3%)]\tLoss: 0.113439\n",
      "Train Epoch: 2 [4000/60000 (7%)]\tLoss: 0.205922\n",
      "Train Epoch: 2 [6000/60000 (10%)]\tLoss: 0.149997\n",
      "Train Epoch: 2 [8000/60000 (13%)]\tLoss: 0.101974\n",
      "Train Epoch: 2 [10000/60000 (17%)]\tLoss: 0.144743\n",
      "Train Epoch: 2 [12000/60000 (20%)]\tLoss: 0.155246\n",
      "Train Epoch: 2 [14000/60000 (23%)]\tLoss: 0.119555\n",
      "Train Epoch: 2 [16000/60000 (27%)]\tLoss: 0.101767\n",
      "Train Epoch: 2 [18000/60000 (30%)]\tLoss: 0.079482\n",
      "Train Epoch: 2 [20000/60000 (33%)]\tLoss: 0.178506\n",
      "Train Epoch: 2 [22000/60000 (37%)]\tLoss: 0.075378\n",
      "Train Epoch: 2 [24000/60000 (40%)]\tLoss: 0.090677\n",
      "Train Epoch: 2 [26000/60000 (43%)]\tLoss: 0.142638\n",
      "Train Epoch: 2 [28000/60000 (47%)]\tLoss: 0.055417\n",
      "Train Epoch: 2 [30000/60000 (50%)]\tLoss: 0.143851\n",
      "Train Epoch: 2 [32000/60000 (53%)]\tLoss: 0.143091\n",
      "Train Epoch: 2 [34000/60000 (57%)]\tLoss: 0.098493\n",
      "Train Epoch: 2 [36000/60000 (60%)]\tLoss: 0.181513\n",
      "Train Epoch: 2 [38000/60000 (63%)]\tLoss: 0.128863\n",
      "Train Epoch: 2 [40000/60000 (67%)]\tLoss: 0.097422\n",
      "Train Epoch: 2 [42000/60000 (70%)]\tLoss: 0.126111\n",
      "Train Epoch: 2 [44000/60000 (73%)]\tLoss: 0.115157\n",
      "Train Epoch: 2 [46000/60000 (77%)]\tLoss: 0.168063\n",
      "Train Epoch: 2 [48000/60000 (80%)]\tLoss: 0.146405\n",
      "Train Epoch: 2 [50000/60000 (83%)]\tLoss: 0.204473\n",
      "Train Epoch: 2 [52000/60000 (87%)]\tLoss: 0.101614\n",
      "Train Epoch: 2 [54000/60000 (90%)]\tLoss: 0.118280\n",
      "Train Epoch: 2 [56000/60000 (93%)]\tLoss: 0.085093\n",
      "Train Epoch: 2 [58000/60000 (97%)]\tLoss: 0.156680\n",
      "Train Epoch: 3 [0/60000 (0%)]\tLoss: 0.134343\n",
      "Train Epoch: 3 [2000/60000 (3%)]\tLoss: 0.079077\n",
      "Train Epoch: 3 [4000/60000 (7%)]\tLoss: 0.062341\n",
      "Train Epoch: 3 [6000/60000 (10%)]\tLoss: 0.120785\n",
      "Train Epoch: 3 [8000/60000 (13%)]\tLoss: 0.175781\n",
      "Train Epoch: 3 [10000/60000 (17%)]\tLoss: 0.152849\n",
      "Train Epoch: 3 [12000/60000 (20%)]\tLoss: 0.088044\n",
      "Train Epoch: 3 [14000/60000 (23%)]\tLoss: 0.080142\n",
      "Train Epoch: 3 [16000/60000 (27%)]\tLoss: 0.083923\n",
      "Train Epoch: 3 [18000/60000 (30%)]\tLoss: 0.184087\n",
      "Train Epoch: 3 [20000/60000 (33%)]\tLoss: 0.097506\n",
      "Train Epoch: 3 [22000/60000 (37%)]\tLoss: 0.070569\n",
      "Train Epoch: 3 [24000/60000 (40%)]\tLoss: 0.059380\n",
      "Train Epoch: 3 [26000/60000 (43%)]\tLoss: 0.088078\n",
      "Train Epoch: 3 [28000/60000 (47%)]\tLoss: 0.097983\n",
      "Train Epoch: 3 [30000/60000 (50%)]\tLoss: 0.142676\n",
      "Train Epoch: 3 [32000/60000 (53%)]\tLoss: 0.093789\n",
      "Train Epoch: 3 [34000/60000 (57%)]\tLoss: 0.049759\n",
      "Train Epoch: 3 [36000/60000 (60%)]\tLoss: 0.085657\n",
      "Train Epoch: 3 [38000/60000 (63%)]\tLoss: 0.077567\n",
      "Train Epoch: 3 [40000/60000 (67%)]\tLoss: 0.154374\n",
      "Train Epoch: 3 [42000/60000 (70%)]\tLoss: 0.103154\n",
      "Train Epoch: 3 [44000/60000 (73%)]\tLoss: 0.097235\n",
      "Train Epoch: 3 [46000/60000 (77%)]\tLoss: 0.112236\n",
      "Train Epoch: 3 [48000/60000 (80%)]\tLoss: 0.121613\n",
      "Train Epoch: 3 [50000/60000 (83%)]\tLoss: 0.065749\n",
      "Train Epoch: 3 [52000/60000 (87%)]\tLoss: 0.047998\n",
      "Train Epoch: 3 [54000/60000 (90%)]\tLoss: 0.122206\n",
      "Train Epoch: 3 [56000/60000 (93%)]\tLoss: 0.078127\n",
      "Train Epoch: 3 [58000/60000 (97%)]\tLoss: 0.078149\n",
      "Train Epoch: 4 [0/60000 (0%)]\tLoss: 0.100946\n",
      "Train Epoch: 4 [2000/60000 (3%)]\tLoss: 0.087813\n",
      "Train Epoch: 4 [4000/60000 (7%)]\tLoss: 0.093293\n",
      "Train Epoch: 4 [6000/60000 (10%)]\tLoss: 0.081129\n",
      "Train Epoch: 4 [8000/60000 (13%)]\tLoss: 0.142881\n",
      "Train Epoch: 4 [10000/60000 (17%)]\tLoss: 0.086005\n",
      "Train Epoch: 4 [12000/60000 (20%)]\tLoss: 0.084642\n",
      "Train Epoch: 4 [14000/60000 (23%)]\tLoss: 0.089820\n",
      "Train Epoch: 4 [16000/60000 (27%)]\tLoss: 0.163107\n",
      "Train Epoch: 4 [18000/60000 (30%)]\tLoss: 0.078533\n",
      "Train Epoch: 4 [20000/60000 (33%)]\tLoss: 0.082007\n",
      "Train Epoch: 4 [22000/60000 (37%)]\tLoss: 0.042416\n",
      "Train Epoch: 4 [24000/60000 (40%)]\tLoss: 0.070957\n",
      "Train Epoch: 4 [26000/60000 (43%)]\tLoss: 0.117465\n",
      "Train Epoch: 4 [28000/60000 (47%)]\tLoss: 0.059761\n",
      "Train Epoch: 4 [30000/60000 (50%)]\tLoss: 0.083823\n",
      "Train Epoch: 4 [32000/60000 (53%)]\tLoss: 0.123205\n",
      "Train Epoch: 4 [34000/60000 (57%)]\tLoss: 0.092471\n",
      "Train Epoch: 4 [36000/60000 (60%)]\tLoss: 0.082841\n",
      "Train Epoch: 4 [38000/60000 (63%)]\tLoss: 0.070377\n",
      "Train Epoch: 4 [40000/60000 (67%)]\tLoss: 0.065370\n",
      "Train Epoch: 4 [42000/60000 (70%)]\tLoss: 0.053301\n",
      "Train Epoch: 4 [44000/60000 (73%)]\tLoss: 0.047281\n",
      "Train Epoch: 4 [46000/60000 (77%)]\tLoss: 0.089627\n",
      "Train Epoch: 4 [48000/60000 (80%)]\tLoss: 0.099486\n",
      "Train Epoch: 4 [50000/60000 (83%)]\tLoss: 0.114095\n",
      "Train Epoch: 4 [52000/60000 (87%)]\tLoss: 0.075652\n",
      "Train Epoch: 4 [54000/60000 (90%)]\tLoss: 0.064313\n",
      "Train Epoch: 4 [56000/60000 (93%)]\tLoss: 0.123137\n",
      "Train Epoch: 4 [58000/60000 (97%)]\tLoss: 0.053603\n",
      "Train Epoch: 5 [0/60000 (0%)]\tLoss: 0.034720\n",
      "Train Epoch: 5 [2000/60000 (3%)]\tLoss: 0.041839\n",
      "Train Epoch: 5 [4000/60000 (7%)]\tLoss: 0.056410\n",
      "Train Epoch: 5 [6000/60000 (10%)]\tLoss: 0.036095\n",
      "Train Epoch: 5 [8000/60000 (13%)]\tLoss: 0.095897\n",
      "Train Epoch: 5 [10000/60000 (17%)]\tLoss: 0.048146\n",
      "Train Epoch: 5 [12000/60000 (20%)]\tLoss: 0.098028\n",
      "Train Epoch: 5 [14000/60000 (23%)]\tLoss: 0.069041\n",
      "Train Epoch: 5 [16000/60000 (27%)]\tLoss: 0.073149\n",
      "Train Epoch: 5 [18000/60000 (30%)]\tLoss: 0.041586\n",
      "Train Epoch: 5 [20000/60000 (33%)]\tLoss: 0.067233\n",
      "Train Epoch: 5 [22000/60000 (37%)]\tLoss: 0.126071\n",
      "Train Epoch: 5 [24000/60000 (40%)]\tLoss: 0.047118\n",
      "Train Epoch: 5 [26000/60000 (43%)]\tLoss: 0.050992\n",
      "Train Epoch: 5 [28000/60000 (47%)]\tLoss: 0.096568\n",
      "Train Epoch: 5 [30000/60000 (50%)]\tLoss: 0.042780\n",
      "Train Epoch: 5 [32000/60000 (53%)]\tLoss: 0.031190\n",
      "Train Epoch: 5 [34000/60000 (57%)]\tLoss: 0.031437\n",
      "Train Epoch: 5 [36000/60000 (60%)]\tLoss: 0.120644\n",
      "Train Epoch: 5 [38000/60000 (63%)]\tLoss: 0.063293\n",
      "Train Epoch: 5 [40000/60000 (67%)]\tLoss: 0.096384\n",
      "Train Epoch: 5 [42000/60000 (70%)]\tLoss: 0.098062\n",
      "Train Epoch: 5 [44000/60000 (73%)]\tLoss: 0.039660\n",
      "Train Epoch: 5 [46000/60000 (77%)]\tLoss: 0.091479\n",
      "Train Epoch: 5 [48000/60000 (80%)]\tLoss: 0.036459\n",
      "Train Epoch: 5 [50000/60000 (83%)]\tLoss: 0.062472\n",
      "Train Epoch: 5 [52000/60000 (87%)]\tLoss: 0.045425\n",
      "Train Epoch: 5 [54000/60000 (90%)]\tLoss: 0.055683\n",
      "Train Epoch: 5 [56000/60000 (93%)]\tLoss: 0.073853\n",
      "Train Epoch: 5 [58000/60000 (97%)]\tLoss: 0.090644\n",
      "Train Epoch: 6 [0/60000 (0%)]\tLoss: 0.026205\n",
      "Train Epoch: 6 [2000/60000 (3%)]\tLoss: 0.028947\n",
      "Train Epoch: 6 [4000/60000 (7%)]\tLoss: 0.064870\n",
      "Train Epoch: 6 [6000/60000 (10%)]\tLoss: 0.028743\n",
      "Train Epoch: 6 [8000/60000 (13%)]\tLoss: 0.067828\n",
      "Train Epoch: 6 [10000/60000 (17%)]\tLoss: 0.033762\n",
      "Train Epoch: 6 [12000/60000 (20%)]\tLoss: 0.090963\n",
      "Train Epoch: 6 [14000/60000 (23%)]\tLoss: 0.033564\n",
      "Train Epoch: 6 [16000/60000 (27%)]\tLoss: 0.023569\n",
      "Train Epoch: 6 [18000/60000 (30%)]\tLoss: 0.038972\n",
      "Train Epoch: 6 [20000/60000 (33%)]\tLoss: 0.045493\n",
      "Train Epoch: 6 [22000/60000 (37%)]\tLoss: 0.060854\n",
      "Train Epoch: 6 [24000/60000 (40%)]\tLoss: 0.040195\n",
      "Train Epoch: 6 [26000/60000 (43%)]\tLoss: 0.069349\n",
      "Train Epoch: 6 [28000/60000 (47%)]\tLoss: 0.017699\n",
      "Train Epoch: 6 [30000/60000 (50%)]\tLoss: 0.039327\n",
      "Train Epoch: 6 [32000/60000 (53%)]\tLoss: 0.086708\n",
      "Train Epoch: 6 [34000/60000 (57%)]\tLoss: 0.025381\n",
      "Train Epoch: 6 [36000/60000 (60%)]\tLoss: 0.060754\n",
      "Train Epoch: 6 [38000/60000 (63%)]\tLoss: 0.049895\n",
      "Train Epoch: 6 [40000/60000 (67%)]\tLoss: 0.052303\n",
      "Train Epoch: 6 [42000/60000 (70%)]\tLoss: 0.045241\n",
      "Train Epoch: 6 [44000/60000 (73%)]\tLoss: 0.057713\n",
      "Train Epoch: 6 [46000/60000 (77%)]\tLoss: 0.070442\n",
      "Train Epoch: 6 [48000/60000 (80%)]\tLoss: 0.044750\n",
      "Train Epoch: 6 [50000/60000 (83%)]\tLoss: 0.032241\n",
      "Train Epoch: 6 [52000/60000 (87%)]\tLoss: 0.062274\n",
      "Train Epoch: 6 [54000/60000 (90%)]\tLoss: 0.069143\n",
      "Train Epoch: 6 [56000/60000 (93%)]\tLoss: 0.037919\n",
      "Train Epoch: 6 [58000/60000 (97%)]\tLoss: 0.029972\n",
      "Train Epoch: 7 [0/60000 (0%)]\tLoss: 0.033992\n",
      "Train Epoch: 7 [2000/60000 (3%)]\tLoss: 0.037546\n",
      "Train Epoch: 7 [4000/60000 (7%)]\tLoss: 0.066979\n",
      "Train Epoch: 7 [6000/60000 (10%)]\tLoss: 0.038284\n",
      "Train Epoch: 7 [8000/60000 (13%)]\tLoss: 0.083367\n",
      "Train Epoch: 7 [10000/60000 (17%)]\tLoss: 0.037149\n",
      "Train Epoch: 7 [12000/60000 (20%)]\tLoss: 0.051343\n",
      "Train Epoch: 7 [14000/60000 (23%)]\tLoss: 0.039916\n",
      "Train Epoch: 7 [16000/60000 (27%)]\tLoss: 0.042984\n",
      "Train Epoch: 7 [18000/60000 (30%)]\tLoss: 0.028639\n",
      "Train Epoch: 7 [20000/60000 (33%)]\tLoss: 0.035569\n",
      "Train Epoch: 7 [22000/60000 (37%)]\tLoss: 0.029252\n",
      "Train Epoch: 7 [24000/60000 (40%)]\tLoss: 0.061154\n",
      "Train Epoch: 7 [26000/60000 (43%)]\tLoss: 0.062687\n",
      "Train Epoch: 7 [28000/60000 (47%)]\tLoss: 0.024625\n",
      "Train Epoch: 7 [30000/60000 (50%)]\tLoss: 0.041475\n",
      "Train Epoch: 7 [32000/60000 (53%)]\tLoss: 0.038457\n",
      "Train Epoch: 7 [34000/60000 (57%)]\tLoss: 0.022161\n",
      "Train Epoch: 7 [36000/60000 (60%)]\tLoss: 0.033792\n",
      "Train Epoch: 7 [38000/60000 (63%)]\tLoss: 0.033307\n",
      "Train Epoch: 7 [40000/60000 (67%)]\tLoss: 0.016663\n",
      "Train Epoch: 7 [42000/60000 (70%)]\tLoss: 0.036338\n",
      "Train Epoch: 7 [44000/60000 (73%)]\tLoss: 0.039313\n",
      "Train Epoch: 7 [46000/60000 (77%)]\tLoss: 0.049488\n",
      "Train Epoch: 7 [48000/60000 (80%)]\tLoss: 0.074534\n",
      "Train Epoch: 7 [50000/60000 (83%)]\tLoss: 0.021263\n",
      "Train Epoch: 7 [52000/60000 (87%)]\tLoss: 0.055766\n",
      "Train Epoch: 7 [54000/60000 (90%)]\tLoss: 0.076435\n",
      "Train Epoch: 7 [56000/60000 (93%)]\tLoss: 0.034980\n",
      "Train Epoch: 7 [58000/60000 (97%)]\tLoss: 0.086712\n",
      "Train Epoch: 8 [0/60000 (0%)]\tLoss: 0.071204\n",
      "Train Epoch: 8 [2000/60000 (3%)]\tLoss: 0.053727\n",
      "Train Epoch: 8 [4000/60000 (7%)]\tLoss: 0.063707\n",
      "Train Epoch: 8 [6000/60000 (10%)]\tLoss: 0.024227\n",
      "Train Epoch: 8 [8000/60000 (13%)]\tLoss: 0.011795\n",
      "Train Epoch: 8 [10000/60000 (17%)]\tLoss: 0.044598\n",
      "Train Epoch: 8 [12000/60000 (20%)]\tLoss: 0.027279\n",
      "Train Epoch: 8 [14000/60000 (23%)]\tLoss: 0.015110\n",
      "Train Epoch: 8 [16000/60000 (27%)]\tLoss: 0.016540\n",
      "Train Epoch: 8 [18000/60000 (30%)]\tLoss: 0.013324\n",
      "Train Epoch: 8 [20000/60000 (33%)]\tLoss: 0.065503\n",
      "Train Epoch: 8 [22000/60000 (37%)]\tLoss: 0.064152\n",
      "Train Epoch: 8 [24000/60000 (40%)]\tLoss: 0.048080\n",
      "Train Epoch: 8 [26000/60000 (43%)]\tLoss: 0.041943\n",
      "Train Epoch: 8 [28000/60000 (47%)]\tLoss: 0.078070\n",
      "Train Epoch: 8 [30000/60000 (50%)]\tLoss: 0.093281\n",
      "Train Epoch: 8 [32000/60000 (53%)]\tLoss: 0.092174\n",
      "Train Epoch: 8 [34000/60000 (57%)]\tLoss: 0.018510\n",
      "Train Epoch: 8 [36000/60000 (60%)]\tLoss: 0.070130\n",
      "Train Epoch: 8 [38000/60000 (63%)]\tLoss: 0.021712\n",
      "Train Epoch: 8 [40000/60000 (67%)]\tLoss: 0.052215\n",
      "Train Epoch: 8 [42000/60000 (70%)]\tLoss: 0.070911\n",
      "Train Epoch: 8 [44000/60000 (73%)]\tLoss: 0.024501\n",
      "Train Epoch: 8 [46000/60000 (77%)]\tLoss: 0.027818\n",
      "Train Epoch: 8 [48000/60000 (80%)]\tLoss: 0.016868\n",
      "Train Epoch: 8 [50000/60000 (83%)]\tLoss: 0.020474\n",
      "Train Epoch: 8 [52000/60000 (87%)]\tLoss: 0.037163\n",
      "Train Epoch: 8 [54000/60000 (90%)]\tLoss: 0.039801\n",
      "Train Epoch: 8 [56000/60000 (93%)]\tLoss: 0.029517\n",
      "Train Epoch: 8 [58000/60000 (97%)]\tLoss: 0.028524\n",
      "Train Epoch: 9 [0/60000 (0%)]\tLoss: 0.028080\n",
      "Train Epoch: 9 [2000/60000 (3%)]\tLoss: 0.029479\n",
      "Train Epoch: 9 [4000/60000 (7%)]\tLoss: 0.026847\n",
      "Train Epoch: 9 [6000/60000 (10%)]\tLoss: 0.036936\n",
      "Train Epoch: 9 [8000/60000 (13%)]\tLoss: 0.028442\n",
      "Train Epoch: 9 [10000/60000 (17%)]\tLoss: 0.025819\n",
      "Train Epoch: 9 [12000/60000 (20%)]\tLoss: 0.038962\n",
      "Train Epoch: 9 [14000/60000 (23%)]\tLoss: 0.028757\n",
      "Train Epoch: 9 [16000/60000 (27%)]\tLoss: 0.041404\n",
      "Train Epoch: 9 [18000/60000 (30%)]\tLoss: 0.055463\n",
      "Train Epoch: 9 [20000/60000 (33%)]\tLoss: 0.042308\n",
      "Train Epoch: 9 [22000/60000 (37%)]\tLoss: 0.041772\n",
      "Train Epoch: 9 [24000/60000 (40%)]\tLoss: 0.020819\n",
      "Train Epoch: 9 [26000/60000 (43%)]\tLoss: 0.026441\n",
      "Train Epoch: 9 [28000/60000 (47%)]\tLoss: 0.009361\n",
      "Train Epoch: 9 [30000/60000 (50%)]\tLoss: 0.021575\n",
      "Train Epoch: 9 [32000/60000 (53%)]\tLoss: 0.043641\n",
      "Train Epoch: 9 [34000/60000 (57%)]\tLoss: 0.020015\n",
      "Train Epoch: 9 [36000/60000 (60%)]\tLoss: 0.022726\n",
      "Train Epoch: 9 [38000/60000 (63%)]\tLoss: 0.044657\n",
      "Train Epoch: 9 [40000/60000 (67%)]\tLoss: 0.094416\n",
      "Train Epoch: 9 [42000/60000 (70%)]\tLoss: 0.028708\n",
      "Train Epoch: 9 [44000/60000 (73%)]\tLoss: 0.040107\n",
      "Train Epoch: 9 [46000/60000 (77%)]\tLoss: 0.022631\n",
      "Train Epoch: 9 [48000/60000 (80%)]\tLoss: 0.032547\n",
      "Train Epoch: 9 [50000/60000 (83%)]\tLoss: 0.016152\n",
      "Train Epoch: 9 [52000/60000 (87%)]\tLoss: 0.030437\n",
      "Train Epoch: 9 [54000/60000 (90%)]\tLoss: 0.013237\n",
      "Train Epoch: 9 [56000/60000 (93%)]\tLoss: 0.051484\n",
      "Train Epoch: 9 [58000/60000 (97%)]\tLoss: 0.033828\n"
     ]
    }
   ],
   "source": [
    "# Train the Model\n",
    "for epoch in range(epochs):\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        # Convert MNIST Images [data] and Labels [target] into PyTorch Variables\n",
    "        data, target = Variable(data), Variable(target)\n",
    "        # Flatten out / Reshape the data from (batch_size, 1, 28, 28) to (batch_size, 28*28)\n",
    "        data = data.view(-1,28*28)\n",
    "        # Initialize all Gradients with Zeros before running so that it is ready for the next backpropagation pass\n",
    "        optimizer.zero_grad()\n",
    "        # Get the Softmax Output from the Neural Network Class Forward() function\n",
    "        # data: batch of input data\n",
    "        output = model(data)\n",
    "        # Calculate the Loss at Output [Negative Log Likelihood Loss]\n",
    "        # Output: Output of the Model, Target: Original Label\n",
    "        loss_val = loss(output, target)\n",
    "        # Backpropagate the Loss throughout the Neural Network\n",
    "        loss_val.backward()\n",
    "        # Execute Gradient Descent Step based on the Gradients Calculated above\n",
    "        optimizer.step()\n",
    "        \n",
    "        # Print Training Results after Every 10 Iterations\n",
    "        if batch_idx % 10 == 0:\n",
    "                print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                    epoch, batch_idx * len(data), len(train_loader.dataset),\n",
    "                           100. * batch_idx / len(train_loader), loss_val.data[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the lines above, we run through the code using a nested for loop. On one hand the outer for loop takes all the images and goes through the training process for each epoch, the inner for loop loads the data and runs through the entire training set in batch_sizes that we have defined above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Test the Trained Model\n",
    "test_loss = 0\n",
    "correct_predictions = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Hare Krishna\\AppData\\Roaming\\Python\\Python36\\site-packages\\ipykernel_launcher.py:30: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    }
   ],
   "source": [
    "# Loop through the Test Data and Get the Trained Model Accuracy\n",
    "for data, target in test_loader:\n",
    "    data, target = Variable(data, volatile=True), Variable(target)\n",
    "    data = data.view(-1,28*28)\n",
    "    output = model(data)\n",
    "    # Batch Loss\n",
    "    test_loss += loss(output, target).data[0]\n",
    "    # Index of Max Log Probability\n",
    "    # .max(): returns index of max value in a certain dimension of tensor\n",
    "    # That index represents the digit label [0,1,2,3,4,5,6,7,8,9]\n",
    "    # .max(1): returns max value in 2nd dimension\n",
    "    # .max(0): returns both max value and it's index value\n",
    "    pred = output.data.max(1)[1]\n",
    "    # Get Number of Correct Predictions\n",
    "    correct_predictions += pred.eq(target.data).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test set: Average loss: 0.0004, Accuracy: 9774/10000 (98%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Average Loss\n",
    "test_loss /= len(test_loader.dataset)\n",
    "\n",
    "print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "        test_loss, correct_predictions, len(test_loader.dataset),\n",
    "        100. * correct_predictions / len(test_loader.dataset)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So, we see that this simple model gets us to an accuracy of 98% for the MNIST dataset."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}