Batch-Data-Loading.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**This notebook contains the example code of how you can load a dataset in PyTorch in Batches and train your model on these batches of data.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Dependencies\n",
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch.utils.data as Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Sample Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(numpy.ndarray, numpy.ndarray)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read Dataset\n",
    "df = pd.read_csv(\"./dataset/Insurance-dataset.csv\")\n",
    "\n",
    "# Features\n",
    "X = np.array(df[\"X\"], dtype=np.float64)\n",
    "\n",
    "# Labels\n",
    "y = np.array(df[\"Y\"], dtype=np.float64)\n",
    "\n",
    "type(X), type(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define Batch Size in which data must be loaded\n",
    "batch_size = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Tensor, torch.Tensor)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert Features and Labels to Torch tensors\n",
    "X = torch.from_numpy(X)\n",
    "y = torch.from_numpy(y)\n",
    "\n",
    "type(X), type(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Batch Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.utils.data.dataset.TensorDataset"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load Dataset as Tensor Dataset\n",
    "# Handles reading data into memory, preprocessing data and converting it to numerical values.\n",
    "dataset = Data.TensorDataset(X,y)\n",
    "\n",
    "type(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data Loader\n",
    "# Handles reading data into memory, preprocessing data and converting it to numerical values.\n",
    "# Random shuffle of data for training, num_workers: Number of subprocesses for loading the data\n",
    "data_loader = Data.DataLoader(\n",
    "              dataset = dataset,\n",
    "              batch_size = batch_size,\n",
    "              shuffle = True,\n",
    "              num_workers = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Data in Batches\n",
    "\n",
    "epochs = 5\n",
    "\n",
    "def loadBatchData():\n",
    "    # Train entire dataset 5 times\n",
    "    for epoch in range(epochs):\n",
    "        # Iterator for the data\n",
    "        # Handles Batching data and moving to GPU\n",
    "        for step, (x_batch, y_batch) in enumerate(data_loader):\n",
    "            print('Epoch: ', epoch, '| Step: ', step, '| x_batch: ',x_batch.numpy(), '| y_batch: ', y_batch.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | Step:  0 | x_batch:  [108.  17.  23.  19.  22.] | y_batch:  [392.5 142.1  39.6  46.2 161.5]\n",
      "Epoch:  0 | Step:  1 | x_batch:  [ 8.  2. 25.  7. 15.] | y_batch:  [55.6  6.6 69.2 77.5 32.1]\n",
      "Epoch:  0 | Step:  2 | x_batch:  [26. 23.  5. 41. 13.] | y_batch:  [187.5  56.9  40.3 181.3  31.9]\n",
      "Epoch:  0 | Step:  3 | x_batch:  [55.  6. 11.  0. 23.] | y_batch:  [162.8  14.6  21.3   0.  113. ]\n",
      "Epoch:  0 | Step:  4 | x_batch:  [ 7.  9. 20. 60.  3.] | y_batch:  [ 48.8  52.1  98.1 202.4   4.4]\n",
      "Epoch:  0 | Step:  5 | x_batch:  [ 3. 11. 24. 27. 13.] | y_batch:  [ 39.9  57.2 137.9  92.6  15.7]\n",
      "Epoch:  0 | Step:  6 | x_batch:  [ 10.  14.   9. 124.  31.] | y_batch:  [ 65.3  77.5  87.4 422.2 209.8]\n",
      "Epoch:  0 | Step:  7 | x_batch:  [41. 12. 13.  3. 29.] | y_batch:  [ 73.4  58.1  93.   13.2 103.9]\n",
      "Epoch:  0 | Step:  8 | x_batch:  [ 4.  5. 48. 24. 57.] | y_batch:  [ 12.6  20.9 248.1 134.9 170.9]\n",
      "Epoch:  0 | Step:  9 | x_batch:  [14. 45. 53. 37.  9.] | y_batch:  [ 95.5 214.  244.6 152.8  48.7]\n",
      "Epoch:  0 | Step:  10 | x_batch:  [29.  6.  7. 61. 11.] | y_batch:  [133.3  14.8  27.9 217.6  23.5]\n",
      "Epoch:  0 | Step:  11 | x_batch:  [ 6.  8.  4. 16. 40.] | y_batch:  [ 50.9  76.1  38.1  59.6 119.4]\n",
      "Epoch:  0 | Step:  12 | x_batch:  [ 4. 13. 30.] | y_batch:  [ 11.8  89.9 194.5]\n",
      "Epoch:  1 | Step:  0 | x_batch:  [ 8. 24.  9. 19. 24.] | y_batch:  [ 55.6 137.9  52.1  46.2 134.9]\n",
      "Epoch:  1 | Step:  1 | x_batch:  [  8.  17.  23. 124.   5.] | y_batch:  [ 76.1 142.1 113.  422.2  20.9]\n",
      "Epoch:  1 | Step:  2 | x_batch:  [11. 37. 41. 48.  0.] | y_batch:  [ 23.5 152.8  73.4 248.1   0. ]\n",
      "Epoch:  1 | Step:  3 | x_batch:  [13. 26. 41. 22. 12.] | y_batch:  [ 93.  187.5 181.3 161.5  58.1]\n",
      "Epoch:  1 | Step:  4 | x_batch:  [ 15. 108.  23.   6.  11.] | y_batch:  [ 32.1 392.5  56.9  50.9  21.3]\n",
      "Epoch:  1 | Step:  5 | x_batch:  [ 3. 53. 30.  7. 57.] | y_batch:  [ 13.2 244.6 194.5  27.9 170.9]\n",
      "Epoch:  1 | Step:  6 | x_batch:  [ 7. 29.  5.  4.  6.] | y_batch:  [ 77.5 133.3  40.3  11.8  14.6]\n",
      "Epoch:  1 | Step:  7 | x_batch:  [27.  4. 16. 61.  3.] | y_batch:  [ 92.6  12.6  59.6 217.6  39.9]\n",
      "Epoch:  1 | Step:  8 | x_batch:  [55. 13. 10.  7. 40.] | y_batch:  [162.8  15.7  65.3  48.8 119.4]\n",
      "Epoch:  1 | Step:  9 | x_batch:  [ 9. 13.  6. 60. 11.] | y_batch:  [ 48.7  89.9  14.8 202.4  57.2]\n",
      "Epoch:  1 | Step:  10 | x_batch:  [ 2. 25.  9. 45. 14.] | y_batch:  [  6.6  69.2  87.4 214.   77.5]\n",
      "Epoch:  1 | Step:  11 | x_batch:  [14. 20.  3.  4. 31.] | y_batch:  [ 95.5  98.1   4.4  38.1 209.8]\n",
      "Epoch:  1 | Step:  12 | x_batch:  [29. 13. 23.] | y_batch:  [103.9  31.9  39.6]\n",
      "Epoch:  2 | Step:  0 | x_batch:  [ 9. 11. 23.  5. 11.] | y_batch:  [87.4 21.3 39.6 40.3 23.5]\n",
      "Epoch:  2 | Step:  1 | x_batch:  [13. 41. 15.  4.  7.] | y_batch:  [ 15.7 181.3  32.1  38.1  77.5]\n",
      "Epoch:  2 | Step:  2 | x_batch:  [ 4. 10. 41. 17. 22.] | y_batch:  [ 12.6  65.3  73.4 142.1 161.5]\n",
      "Epoch:  2 | Step:  3 | x_batch:  [14. 60. 31.  2. 16.] | y_batch:  [ 95.5 202.4 209.8   6.6  59.6]\n",
      "Epoch:  2 | Step:  4 | x_batch:  [124.  11.  37.  20.  13.] | y_batch:  [422.2  57.2 152.8  98.1  31.9]\n",
      "Epoch:  2 | Step:  5 | x_batch:  [23.  7. 48. 29. 57.] | y_batch:  [ 56.9  27.9 248.1 133.3 170.9]\n",
      "Epoch:  2 | Step:  6 | x_batch:  [  4.  24.   3.  61. 108.] | y_batch:  [ 11.8 137.9   4.4 217.6 392.5]\n",
      "Epoch:  2 | Step:  7 | x_batch:  [53.  9.  3. 25. 26.] | y_batch:  [244.6  48.7  13.2  69.2 187.5]\n",
      "Epoch:  2 | Step:  8 | x_batch:  [30. 45. 13. 55.  8.] | y_batch:  [194.5 214.   89.9 162.8  55.6]\n",
      "Epoch:  2 | Step:  9 | x_batch:  [ 5. 27.  6.  7.  6.] | y_batch:  [20.9 92.6 14.8 48.8 50.9]\n",
      "Epoch:  2 | Step:  10 | x_batch:  [23. 40. 24.  8. 14.] | y_batch:  [113.  119.4 134.9  76.1  77.5]\n",
      "Epoch:  2 | Step:  11 | x_batch:  [ 9. 12. 13. 19.  0.] | y_batch:  [52.1 58.1 93.  46.2  0. ]\n",
      "Epoch:  2 | Step:  12 | x_batch:  [ 3. 29.  6.] | y_batch:  [ 39.9 103.9  14.6]\n",
      "Epoch:  3 | Step:  0 | x_batch:  [ 2. 27.  8. 61. 25.] | y_batch:  [  6.6  92.6  55.6 217.6  69.2]\n",
      "Epoch:  3 | Step:  1 | x_batch:  [  7. 124.  13.  29.   8.] | y_batch:  [ 27.9 422.2  15.7 133.3  76.1]\n",
      "Epoch:  3 | Step:  2 | x_batch:  [ 23. 108.  57.  19.  41.] | y_batch:  [ 39.6 392.5 170.9  46.2 181.3]\n",
      "Epoch:  3 | Step:  3 | x_batch:  [11. 23.  9. 29.  5.] | y_batch:  [ 23.5  56.9  52.1 103.9  20.9]\n",
      "Epoch:  3 | Step:  4 | x_batch:  [13. 20. 41.  4. 14.] | y_batch:  [93.  98.1 73.4 12.6 95.5]\n",
      "Epoch:  3 | Step:  5 | x_batch:  [22. 55.  3.  0. 37.] | y_batch:  [161.5 162.8  39.9   0.  152.8]\n",
      "Epoch:  3 | Step:  6 | x_batch:  [ 9.  6. 48.  6. 31.] | y_batch:  [ 48.7  50.9 248.1  14.8 209.8]\n",
      "Epoch:  3 | Step:  7 | x_batch:  [12. 10. 16. 17. 26.] | y_batch:  [ 58.1  65.3  59.6 142.1 187.5]\n",
      "Epoch:  3 | Step:  8 | x_batch:  [45. 13.  4.  3.  5.] | y_batch:  [214.   89.9  11.8   4.4  40.3]\n",
      "Epoch:  3 | Step:  9 | x_batch:  [11. 60. 11. 13. 53.] | y_batch:  [ 57.2 202.4  21.3  31.9 244.6]\n",
      "Epoch:  3 | Step:  10 | x_batch:  [23.  6. 15.  3.  9.] | y_batch:  [113.   14.6  32.1  13.2  87.4]\n",
      "Epoch:  3 | Step:  11 | x_batch:  [24. 14. 30. 24.  7.] | y_batch:  [137.9  77.5 194.5 134.9  48.8]\n",
      "Epoch:  3 | Step:  12 | x_batch:  [40.  7.  4.] | y_batch:  [119.4  77.5  38.1]\n",
      "Epoch:  4 | Step:  0 | x_batch:  [ 4. 40.  9. 61. 13.] | y_batch:  [ 11.8 119.4  87.4 217.6  89.9]\n",
      "Epoch:  4 | Step:  1 | x_batch:  [ 26.   7. 108.  14.  20.] | y_batch:  [187.5  77.5 392.5  95.5  98.1]\n",
      "Epoch:  4 | Step:  2 | x_batch:  [ 6. 13. 12.  6.  6.] | y_batch:  [50.9 93.  58.1 14.6 14.8]\n",
      "Epoch:  4 | Step:  3 | x_batch:  [13. 48.  7. 45.  5.] | y_batch:  [ 31.9 248.1  48.8 214.   40.3]\n",
      "Epoch:  4 | Step:  4 | x_batch:  [ 4. 11. 37. 16.  0.] | y_batch:  [ 38.1  23.5 152.8  59.6   0. ]\n",
      "Epoch:  4 | Step:  5 | x_batch:  [ 4. 10. 15. 19.  3.] | y_batch:  [12.6 65.3 32.1 46.2  4.4]\n",
      "Epoch:  4 | Step:  6 | x_batch:  [24.  5. 22. 14. 53.] | y_batch:  [137.9  20.9 161.5  77.5 244.6]\n",
      "Epoch:  4 | Step:  7 | x_batch:  [ 55.  13.   8.  23. 124.] | y_batch:  [162.8  15.7  55.6  39.6 422.2]\n",
      "Epoch:  4 | Step:  8 | x_batch:  [60. 11. 57.  9. 41.] | y_batch:  [202.4  21.3 170.9  52.1  73.4]\n",
      "Epoch:  4 | Step:  9 | x_batch:  [ 3. 29. 23. 31. 30.] | y_batch:  [ 39.9 133.3 113.  209.8 194.5]\n",
      "Epoch:  4 | Step:  10 | x_batch:  [23.  9. 29. 17. 24.] | y_batch:  [ 56.9  48.7 103.9 142.1 134.9]\n",
      "Epoch:  4 | Step:  11 | x_batch:  [ 8. 11.  3. 27. 41.] | y_batch:  [ 76.1  57.2  13.2  92.6 181.3]\n",
      "Epoch:  4 | Step:  12 | x_batch:  [25.  7.  2.] | y_batch:  [69.2 27.9  6.6]\n"
     ]
    }
   ],
   "source": [
    "# Test Load Batch logic\n",
    "loadBatchData()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}