baselines.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "baselines.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.5"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "NzqxHIh4OCdW"
      },
      "source": [
        "# Incremental learning on image classification"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "wBHSznCZxpNB"
      },
      "source": [
        "## Libraries and packages\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4eQ6O12jxMFf",
        "colab": {}
      },
      "source": [
        "!pip3 install 'torch==1.4.0'\n",
        "!pip3 install 'torchvision==0.5.0'\n",
        "!pip3 install 'Pillow-SIMD'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xAYXtIdpx0Yy",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import urllib\n",
        "import logging\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.init as init\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Dataset, Subset, DataLoader, ConcatDataset\n",
        "from torch.backends import cudnn\n",
        "\n",
        "import torchvision\n",
        "from torchvision import transforms\n",
        "from torchvision.models import resnet34\n",
        "\n",
        "from PIL import Image\n",
        "from copy import deepcopy\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "from sklearn.metrics import confusion_matrix"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "09iWc_oCotu2",
        "colab": {}
      },
      "source": [
        "# Download packages from repository\n",
        "!git clone https://github.com/manuelemacchia/incremental-learning-image-classification.git\n",
        "!mv -v incremental-learning-image-classification/* .\n",
        "!rm -rf incremental-learning-image-classification README.md\n",
        "\n",
        "from data.cifar100 import Cifar100\n",
        "from model.resnet_cifar import resnet32\n",
        "from model.manager import Manager\n",
        "from model.lwf import LWF\n",
        "from model.icarl import Exemplars\n",
        "from model.icarl import iCaRL\n",
        "from utils import plot"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "j12pgffMR6Qv"
      },
      "source": [
        "## Arguments"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "JwE0x8gkSisn",
        "colab": {}
      },
      "source": [
        "# Directories\n",
        "DATA_DIR = 'data'       # Directory where the dataset will be downloaded\n",
        "\n",
        "# Settings\n",
        "DEVICE = 'cuda'\n",
        "\n",
        "# Dataset\n",
        "RANDOM_STATES = [658, 423, 422]      # For reproducibility of results                        \n",
        "                                     # Note: different random states give very different\n",
        "                                     # splits and therefore very different results.\n",
        "\n",
        "NUM_CLASSES = 100       # Total number of classes\n",
        "\n",
        "VAL_SIZE = 0.1          # Proportion of validation set with respect to training set (between 0 and 1)\n",
        "\n",
        "# Training\n",
        "BATCH_SIZE = 64         # Batch size (iCaRL sets this to 128)\n",
        "LR = 2                  # Initial learning rate\n",
        "                       \n",
        "MOMENTUM = 0.9          # Momentum for stochastic gradient descent (SGD)\n",
        "WEIGHT_DECAY = 1e-5     # Weight decay from iCaRL\n",
        "\n",
        "NUM_RUNS = 3            # Number of runs of every method\n",
        "                        # Note: this should be at least 3 to have a fair benchmark\n",
        "\n",
        "NUM_EPOCHS = 70         # Total number of training epochs\n",
        "MILESTONES = [49, 63]   # Step down policy from iCaRL (MultiStepLR)\n",
        "                        # Decrease the learning rate by gamma at each milestone\n",
        "GAMMA = 0.2             # Gamma factor from iCaRL"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "QJqnljCV5gJ5"
      },
      "source": [
        "## Fine tuning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "HDDdumxRwbdQ"
      },
      "source": [
        "### Data preparation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "skknIP5Jwspm",
        "colab": {}
      },
      "source": [
        "# Define transformations for training\n",
        "train_transform = transforms.Compose([transforms.RandomCrop(32, padding=4),\n",
        "                                      transforms.RandomHorizontalFlip(),\n",
        "                                      transforms.ToTensor(), # Turn PIL Image to torch.Tensor\n",
        "                                      transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
        "])\n",
        "\n",
        "# Define transformations for evaluation\n",
        "test_transform = transforms.Compose([transforms.ToTensor(),\n",
        "                                     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))                                    \n",
        "])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fdp8D_27twFn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_dataloaders = [[] for i in range(NUM_RUNS)]\n",
        "val_dataloaders = [[] for i in range(NUM_RUNS)]\n",
        "test_dataloaders = [[] for i in range(NUM_RUNS)]\n",
        "\n",
        "for run_i in range(NUM_RUNS):\n",
        "    test_subsets = []\n",
        "    random_state = RANDOM_STATES[run_i]\n",
        "\n",
        "    for split_i in range(10):\n",
        "        # Download dataset only at first instantiation\n",
        "        if run_i+split_i == 0:\n",
        "            download = True\n",
        "        else:\n",
        "            download = False\n",
        "\n",
        "        # Create CIFAR100 dataset\n",
        "        train_dataset = Cifar100(DATA_DIR, train=True, download=download, random_state=random_state, transform=train_transform)\n",
        "        test_dataset = Cifar100(DATA_DIR, train=False, download=False, random_state=random_state, transform=test_transform)\n",
        "\n",
        "        # Subspace of CIFAR100 of 10 classes\n",
        "        train_dataset.set_classes_batch(train_dataset.batch_splits[split_i])\n",
        "        test_dataset.set_classes_batch([test_dataset.batch_splits[i] for i in range(0, split_i+1)])\n",
        "\n",
        "        # Define train and validation indices\n",
        "        train_indices, val_indices = train_dataset.train_val_split(VAL_SIZE, random_state)\n",
        "\n",
        "        train_dataloaders[run_i].append(DataLoader(Subset(train_dataset, train_indices),\n",
        "                                                   batch_size=BATCH_SIZE, shuffle=True, num_workers=4, drop_last=True))\n",
        "\n",
        "        val_dataloaders[run_i].append(DataLoader(Subset(train_dataset, val_indices),\n",
        "                                                 batch_size=BATCH_SIZE, shuffle=True, num_workers=4, drop_last=True))\n",
        "\n",
        "        # Dataset with all seen class\n",
        "        test_dataloaders[run_i].append(DataLoader(test_dataset,\n",
        "                                                  batch_size=BATCH_SIZE, shuffle=True, num_workers=4))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "mNwcf1fpsvm_",
        "colab": {}
      },
      "source": [
        "# Sanity check: visualize a batch of images\n",
        "dataiter = iter(test_dataloaders[0][0])\n",
        "images, labels = dataiter.next()\n",
        "plot.image_grid(images, one_channel=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7EmP5GRP1JoP",
        "colab_type": "text"
      },
      "source": [
        "### Execution"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4JqA6VD_VxTg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "logs = [[] for _ in range(NUM_RUNS)]\n",
        "\n",
        "for run_i in range(NUM_RUNS):\n",
        "    net = resnet32()\n",
        "    \n",
        "    criterion = nn.BCEWithLogitsLoss()\n",
        "    \n",
        "    for split_i in range(10):\n",
        "        print(f\"## Split {split_i} of run {run_i} ##\")\n",
        "\n",
        "        parameters_to_optimize = net.parameters()\n",
        "        optimizer = optim.SGD(parameters_to_optimize, lr=LR, momentum=MOMENTUM, weight_decay=WEIGHT_DECAY)\n",
        "        scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=MILESTONES, gamma=GAMMA)\n",
        "\n",
        "        manager = Manager(DEVICE, net, criterion, optimizer, scheduler,\n",
        "                          train_dataloaders[run_i][split_i],\n",
        "                          val_dataloaders[run_i][split_i],\n",
        "                          test_dataloaders[run_i][split_i])\n",
        "\n",
        "        scores = manager.train(NUM_EPOCHS)  # train the model\n",
        "\n",
        "        logs[run_i].append({})\n",
        "\n",
        "        # score[i] = dictionary with key:epoch, value: score\n",
        "        logs[run_i][split_i]['train_loss'] = scores[0]\n",
        "        logs[run_i][split_i]['train_accuracy'] = scores[1]\n",
        "        logs[run_i][split_i]['val_loss'] = scores[2]\n",
        "        logs[run_i][split_i]['val_accuracy'] = scores[3]\n",
        "\n",
        "        # Test the model on classes seen until now\n",
        "        test_accuracy, all_targets, all_preds = manager.test()\n",
        "\n",
        "        logs[run_i][split_i]['test_accuracy'] = test_accuracy\n",
        "        logs[run_i][split_i]['conf_mat'] = confusion_matrix(all_targets.to('cpu'), all_preds.to('cpu'))\n",
        "\n",
        "        # Add 10 nodes to last FC layer\n",
        "        manager.increment_classes(n=10)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "5ESzk5gF2c_c",
        "colab": {}
      },
      "source": [
        "train_loss = [[logs[run_i][i]['train_loss'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "train_accuracy = [[logs[run_i][i]['train_accuracy'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "val_loss = [[logs[run_i][i]['val_loss'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "val_accuracy = [[logs[run_i][i]['val_accuracy'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "test_accuracy = [[logs[run_i][i]['test_accuracy'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "\n",
        "train_loss = np.array(train_loss)\n",
        "train_accuracy = np.array(train_accuracy)\n",
        "val_loss = np.array(val_loss)\n",
        "val_accuracy = np.array(val_accuracy)\n",
        "test_accuracy = np.array(test_accuracy)\n",
        "\n",
        "train_loss_stats = np.array([train_loss.mean(0), train_loss.std(0)]).transpose()\n",
        "train_accuracy_stats = np.array([train_accuracy.mean(0), train_accuracy.std(0)]).transpose()\n",
        "val_loss_stats = np.array([val_loss.mean(0), val_loss.std(0)]).transpose()\n",
        "val_accuracy_stats = np.array([val_accuracy.mean(0), val_accuracy.std(0)]).transpose()\n",
        "test_accuracy_stats = np.array([test_accuracy.mean(0), test_accuracy.std(0)]).transpose()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7TAdCtXDEa5d",
        "colab": {}
      },
      "source": [
        "plot.train_val_scores(train_loss_stats, train_accuracy_stats, val_loss_stats, val_accuracy_stats)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "E1APlRtTpmkK",
        "colab": {}
      },
      "source": [
        "plot.test_scores(test_accuracy_stats)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bKJ_Z48QmQ2C",
        "colab_type": "text"
      },
      "source": [
        "## Learning Without Forgetting"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "EHqtSdwzm16h"
      },
      "source": [
        "### Data preparation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "373M_sOAm16i",
        "colab": {}
      },
      "source": [
        "# Transformations for Learning Without Forgetting\n",
        "train_transform = transforms.Compose([transforms.RandomCrop(32, padding=4),\n",
        "                                      transforms.RandomHorizontalFlip(),\n",
        "                                      transforms.ToTensor(), # Turn PIL Image to torch.Tensor\n",
        "                                      transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
        "])\n",
        "\n",
        "test_transform = transforms.Compose([transforms.ToTensor(),\n",
        "                                     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))                                    \n",
        "])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "97Gi3Sp8m16k",
        "colab": {}
      },
      "source": [
        "train_dataloaders = [[] for i in range(NUM_RUNS)]\n",
        "val_dataloaders = [[] for i in range(NUM_RUNS)]\n",
        "test_dataloaders = [[] for i in range(NUM_RUNS)]\n",
        "\n",
        "for run_i in range(NUM_RUNS):\n",
        "    test_subsets = []\n",
        "    random_state = RANDOM_STATES[run_i]\n",
        "\n",
        "    for split_i in range(10):\n",
        "        # Download dataset only at first instantiation\n",
        "        if run_i+split_i == 0:\n",
        "            download = True\n",
        "        else:\n",
        "            download = False\n",
        "\n",
        "        # Create CIFAR100 dataset\n",
        "        train_dataset = Cifar100(DATA_DIR, train=True, download=download, random_state=random_state, transform=train_transform)\n",
        "        test_dataset = Cifar100(DATA_DIR, train=False, download=False, random_state=random_state, transform=test_transform)\n",
        "\n",
        "        # Subspace of CIFAR100 of 10 classes\n",
        "        train_dataset.set_classes_batch(train_dataset.batch_splits[split_i])\n",
        "        test_dataset.set_classes_batch([test_dataset.batch_splits[i] for i in range(0, split_i+1)])\n",
        "\n",
        "        # Define train and validation indices\n",
        "        train_indices, val_indices = train_dataset.train_val_split(VAL_SIZE, random_state)\n",
        "\n",
        "        train_dataloaders[run_i].append(DataLoader(Subset(train_dataset, train_indices),\n",
        "                                                   batch_size=BATCH_SIZE, shuffle=True, num_workers=4, drop_last=True))\n",
        "\n",
        "        val_dataloaders[run_i].append(DataLoader(Subset(train_dataset, val_indices),\n",
        "                                                 batch_size=BATCH_SIZE, shuffle=True, num_workers=4, drop_last=True))\n",
        "\n",
        "        # Dataset with all seen class\n",
        "        test_dataloaders[run_i].append(DataLoader(test_dataset,\n",
        "                                                  batch_size=BATCH_SIZE, shuffle=True, num_workers=4))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Ksaz2qZ5m16n",
        "colab": {}
      },
      "source": [
        "# Sanity check: visualize a batch of images\n",
        "dataiter = iter(test_dataloaders[0][5])\n",
        "images, labels = dataiter.next()\n",
        "\n",
        "plot.image_grid(images, one_channel=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iYwMtMJuLyYe",
        "colab_type": "text"
      },
      "source": [
        "### Execution"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JpGuC_hSL0jN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Arguments for Learning without Forgetting\n",
        "BATCH_SIZE = 128\n",
        "LR = 2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MlThDLCvXJwS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "logs = [[] for _ in range(NUM_RUNS)]\n",
        "\n",
        "# Iterate over runs\n",
        "for run_i in range(NUM_RUNS):\n",
        "    net = resnet32()\n",
        "    \n",
        "    criterion = nn.BCEWithLogitsLoss()\n",
        "    \n",
        "    for split_i in range(10):\n",
        "        print(f\"## Split {split_i} of run {run_i} ##\")\n",
        "\n",
        "        # Redefine optimizer at each split (pass by reference issue)\n",
        "        parameters_to_optimize = net.parameters()\n",
        "        optimizer = optim.SGD(parameters_to_optimize, lr=LR,\n",
        "                                momentum=MOMENTUM, weight_decay=WEIGHT_DECAY)\n",
        "        scheduler = optim.lr_scheduler.MultiStepLR(optimizer, \n",
        "                                                    milestones=MILESTONES, gamma=GAMMA)\n",
        "\n",
        "        num_classes = 10*(split_i+1)\n",
        "\n",
        "        if num_classes == 10: # old network == None\n",
        "            lwf = LWF(DEVICE, net, None, criterion, optimizer, scheduler,\n",
        "                            train_dataloaders[run_i][split_i],\n",
        "                            val_dataloaders[run_i][split_i],\n",
        "                            test_dataloaders[run_i][split_i],\n",
        "                            num_classes)\n",
        "        else:\n",
        "            lwf = LWF(DEVICE, net, old_net, criterion, optimizer, scheduler,\n",
        "                            train_dataloaders[run_i][split_i],\n",
        "                            val_dataloaders[run_i][split_i],\n",
        "                            test_dataloaders[run_i][split_i],\n",
        "                            num_classes)\n",
        "\n",
        "        scores = lwf.train(NUM_EPOCHS)  # train the model\n",
        "\n",
        "        logs[run_i].append({})\n",
        "\n",
        "        # score[i] = dictionary with key:epoch, value: score\n",
        "        logs[run_i][split_i]['train_loss'] = scores[0]\n",
        "        logs[run_i][split_i]['train_accuracy'] = scores[1]\n",
        "        logs[run_i][split_i]['val_loss'] = scores[2]\n",
        "        logs[run_i][split_i]['val_accuracy'] = scores[3]\n",
        "\n",
        "        # Test the model on classes seen until now\n",
        "        test_accuracy, all_targets, all_preds = lwf.test()\n",
        "\n",
        "        logs[run_i][split_i]['test_accuracy'] = test_accuracy\n",
        "        logs[run_i][split_i]['conf_mat'] = confusion_matrix(all_targets.to('cpu'), all_preds.to('cpu'))\n",
        "\n",
        "        old_net = deepcopy(lwf.net)\n",
        "\n",
        "        lwf.increment_classes()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2xZbK6EGSaZN",
        "colab_type": "text"
      },
      "source": [
        "### Plots"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bUfgSq1xSbrD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_loss = [[logs[run_i][i]['train_loss'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "train_accuracy = [[logs[run_i][i]['train_accuracy'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "val_loss = [[logs[run_i][i]['val_loss'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "val_accuracy = [[logs[run_i][i]['val_accuracy'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "test_accuracy = [[logs[run_i][i]['test_accuracy'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "\n",
        "train_loss = np.array(train_loss)\n",
        "train_accuracy = np.array(train_accuracy)\n",
        "val_loss = np.array(val_loss)\n",
        "val_accuracy = np.array(val_accuracy)\n",
        "test_accuracy = np.array(test_accuracy)\n",
        "\n",
        "train_loss_stats = np.array([train_loss.mean(0), train_loss.std(0)]).transpose()\n",
        "train_accuracy_stats = np.array([train_accuracy.mean(0), train_accuracy.std(0)]).transpose()\n",
        "val_loss_stats = np.array([val_loss.mean(0), val_loss.std(0)]).transpose()\n",
        "val_accuracy_stats = np.array([val_accuracy.mean(0), val_accuracy.std(0)]).transpose()\n",
        "test_accuracy_stats = np.array([test_accuracy.mean(0), test_accuracy.std(0)]).transpose()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "1w3_YPJCSeli",
        "colab": {}
      },
      "source": [
        "plot.train_val_scores(train_loss_stats, train_accuracy_stats, val_loss_stats, val_accuracy_stats)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ZSt6-FJbSelp",
        "colab": {}
      },
      "source": [
        "plot.test_scores(test_accuracy_stats)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7B6CcDcDlMf_",
        "colab_type": "text"
      },
      "source": [
        "## iCaRL"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rF0ypxGognNR",
        "colab_type": "text"
      },
      "source": [
        "### Data preparation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4mf04UjEgmPG",
        "colab": {}
      },
      "source": [
        "# Transformations for Learning Without Forgetting\n",
        "train_transform = transforms.Compose([transforms.RandomCrop(32, padding=4),\n",
        "                                      transforms.RandomHorizontalFlip(),\n",
        "                                      transforms.ToTensor(), # Turn PIL Image to torch.Tensor\n",
        "                                      transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
        "])\n",
        "\n",
        "test_transform = transforms.Compose([transforms.ToTensor(),\n",
        "                                     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))                                    \n",
        "])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Y9Oq44dxgmPN",
        "colab": {}
      },
      "source": [
        "train_subsets = [[] for i in range(NUM_RUNS)]\n",
        "val_subsets = [[] for i in range(NUM_RUNS)]\n",
        "test_subsets = [[] for i in range(NUM_RUNS)]\n",
        "\n",
        "for run_i in range(NUM_RUNS):\n",
        "    for split_i in range(10):\n",
        "        if run_i+split_i == 0: # Download dataset only at first instantiation\n",
        "            download = True\n",
        "        else:\n",
        "            download = False\n",
        "\n",
        "        # Create CIFAR100 dataset\n",
        "        train_dataset = Cifar100(DATA_DIR, train=True, download=download, random_state=RANDOM_STATES[run_i], transform=train_transform)\n",
        "        test_dataset = Cifar100(DATA_DIR, train=False, download=False, random_state=RANDOM_STATES[run_i], transform=test_transform)\n",
        "    \n",
        "        # Subspace of CIFAR100 of 10 classes\n",
        "        train_dataset.set_classes_batch(train_dataset.batch_splits[split_i]) \n",
        "        test_dataset.set_classes_batch([test_dataset.batch_splits[i] for i in range(0, split_i+1)])\n",
        "\n",
        "        # Define train and validation indices\n",
        "        train_indices, val_indices = train_dataset.train_val_split(VAL_SIZE, RANDOM_STATES[run_i])\n",
        "\n",
        "        # Define subsets\n",
        "        train_subsets[run_i].append(Subset(train_dataset, train_indices))\n",
        "        val_subsets[run_i].append(Subset(train_dataset, val_indices))\n",
        "        test_subsets[run_i].append(test_dataset)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cmCpRMBKgvDB",
        "colab_type": "text"
      },
      "source": [
        "### Execution"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-nRU--zYjmZT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# iCaRL hyperparameters\n",
        "LR = 2\n",
        "MOMENTUM = 0.9\n",
        "WEIGHT_DECAY = 0.00001\n",
        "MILESTONES = [49, 63]\n",
        "GAMMA = 0.2\n",
        "NUM_EPOCHS = 30\n",
        "BATCH_SIZE = 64"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1mcTQUN7VLPF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Define what tests to run\n",
        "TEST_ICARL = True # Run test with iCaRL (exemplars + train dataset)\n",
        "TEST_HYBRID1 = False # Run test with hybrid1\n",
        "\n",
        "# Initialize logs\n",
        "logs_icarl = [[] for _ in range(NUM_RUNS)]\n",
        "logs_hybrid1 = [[] for _ in range(NUM_RUNS)]\n",
        "\n",
        "for run_i in range(NUM_RUNS):\n",
        "    net = resnet32()\n",
        "    icarl = iCaRL(DEVICE, net, LR, MOMENTUM, WEIGHT_DECAY, MILESTONES, GAMMA, NUM_EPOCHS, BATCH_SIZE, train_transform, test_transform)\n",
        "\n",
        "    for split_i in range(10):\n",
        "        print(f\"## Split {split_i} of run {run_i} ##\")\n",
        "        \n",
        "        train_logs = icarl.incremental_train(split_i, train_subsets[run_i][split_i], val_subsets[run_i][split_i])\n",
        "\n",
        "        if TEST_ICARL:\n",
        "            logs_icarl[run_i].append({})\n",
        "\n",
        "            acc, all_targets, all_preds = icarl.test(test_subsets[run_i][split_i], train_subsets[run_i][split_i])\n",
        "\n",
        "            logs_icarl[run_i][split_i]['accuracy'] = acc\n",
        "            logs_icarl[run_i][split_i]['conf_mat'] = confusion_matrix(all_targets.to('cpu'), all_preds.to('cpu'))\n",
        "\n",
        "            logs_icarl[run_i][split_i]['train_loss'] = train_logs[0]\n",
        "            logs_icarl[run_i][split_i]['train_accuracy'] = train_logs[1]\n",
        "            logs_icarl[run_i][split_i]['val_loss'] = train_logs[2]\n",
        "            logs_icarl[run_i][split_i]['val_accuracy'] = train_logs[3]\n",
        "\n",
        "        if TEST_HYBRID1:\n",
        "            logs_hybrid1[run_i].append({})\n",
        "\n",
        "            acc, all_targets, all_preds = icarl.test_without_classifier(test_subsets[run_i][split_i])\n",
        "\n",
        "            logs_hybrid1[run_i][split_i]['accuracy'] = acc\n",
        "            logs_hybrid1[run_i][split_i]['conf_mat'] = confusion_matrix(all_targets.to('cpu'), all_preds.to('cpu'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wuDM6Ydokv05",
        "colab_type": "text"
      },
      "source": [
        "### Plots"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nEn9P6usbwYa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_loss = [[logs_icarl[run_i][i]['train_loss'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "train_accuracy = [[logs_icarl[run_i][i]['train_accuracy'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "val_loss = [[logs_icarl[run_i][i]['val_loss'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "val_accuracy = [[logs_icarl[run_i][i]['val_accuracy'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "test_accuracy = [[logs_icarl[run_i][i]['accuracy'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "\n",
        "train_loss = np.array(train_loss)\n",
        "train_accuracy = np.array(train_accuracy)\n",
        "val_loss = np.array(val_loss)\n",
        "val_accuracy = np.array(val_accuracy)\n",
        "test_accuracy = np.array(test_accuracy)\n",
        "\n",
        "train_loss_stats = np.array([train_loss.mean(0), train_loss.std(0)]).transpose()\n",
        "train_accuracy_stats = np.array([train_accuracy.mean(0), train_accuracy.std(0)]).transpose()\n",
        "val_loss_stats = np.array([val_loss.mean(0), val_loss.std(0)]).transpose()\n",
        "val_accuracy_stats = np.array([val_accuracy.mean(0), val_accuracy.std(0)]).transpose()\n",
        "test_accuracy_stats = np.array([test_accuracy.mean(0), test_accuracy.std(0)]).transpose()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-GmiU4BfhVht",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "plot.train_val_scores(train_loss_stats, train_accuracy_stats, val_loss_stats, val_accuracy_stats)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PAWPfif3j7pK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "plot.test_scores(test_accuracy_stats)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}