wgan_alp_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "%run dihiggs_dataset.ipynb\n",
    "\n",
    "import argparse\n",
    "import os\n",
    "import numpy as np\n",
    "import math\n",
    "import sys\n",
    "\n",
    "import torchvision.transforms as transforms\n",
    "from torchvision.utils import save_image\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets\n",
    "from torch.autograd import Variable\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F   # NOTE: I don't think this is used\n",
    "import torch.autograd as autograd\n",
    "import torch\n",
    "\n",
    "from IPython import display\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "#\n",
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "os.makedirs(\"images\", exist_ok=True)\n",
    "\n",
    "#parser = argparse.ArgumentParser()\n",
    "\n",
    "\"\"\"\n",
    "parser.add_argument(\"--n_epochs\", type=int, default=200, help=\"number of epochs of training\")\n",
    "parser.add_argument(\"--batch_size\", type=int, default=64, help=\"size of the batches\")\n",
    "parser.add_argument(\"--lr\", type=float, default=0.0002, help=\"adam: learning rate\")\n",
    "parser.add_argument(\"--b1\", type=float, default=0.5, help=\"adam: decay of first order momentum of gradient\")\n",
    "parser.add_argument(\"--b2\", type=float, default=0.999, help=\"adam: decay of first order momentum of gradient\")\n",
    "parser.add_argument(\"--n_cpu\", type=int, default=8, help=\"number of cpu threads to use during batch generation\")\n",
    "parser.add_argument(\"--latent_dim\", type=int, default=100, help=\"dimensionality of the latent space\")\n",
    "parser.add_argument(\"--img_size\", type=int, default=28, help=\"size of each image dimension\")\n",
    "parser.add_argument(\"--channels\", type=int, default=1, help=\"number of image channels\")\n",
    "parser.add_argument(\"--n_critic\", type=int, default=5, help=\"number of training steps for discriminator per iter\")\n",
    "parser.add_argument(\"--clip_value\", type=float, default=0.01, help=\"lower and upper clip value for disc. weights\")\n",
    "parser.add_argument(\"--sample_interval\", type=int, default=400, help=\"interval betwen image samples\")\n",
    "\"\"\"\n",
    "#opt = parser.parse_args()\n",
    "#print(opt)\n",
    "class opt():   # Class used for optimizers in the future. Defines all variables and stuff needed.\n",
    "    n_epochs = 20000   # an epoch is the number of times it works through the entire training set\n",
    "    #batch_size = 1000   # the training set is broken up into batches, \n",
    "                        # and the average loss is used from a given batch for back propagation\n",
    "    lr =  0.0002 # 0.001   # learning rate (how much to change based on error)\n",
    "    b1 = 0     # 0.9 # Used for Adam. Exponential decay rate for the first moment. \n",
    "    b2 = 0.9   # 0.999 # Used for Adam. Exponential decay rate for the second moment estimates (gradient squared)\n",
    "    #NOTE: The default epsilon for torch.optim.adam is 1e-8, so I will just leave it that way\n",
    "    \n",
    "    #n_cpu = 2   # not used rn\n",
    "    latent_dim = 100 #size of noise input to generator (latent space) \n",
    "    #img_size = 28\n",
    "    # channels = 1   # Only used for img_shape right below, and img_shape isn't needed\n",
    "    n_critic = 5   # The generator is trained after this many critic steps\n",
    "    #   clip_value = 0.01   # No other usages rn. \n",
    "    sample_interval = 400   # Determines when a to save the image(s?) generated\n",
    "    \n",
    "    Xi = 10;   # multiplier for recursively finding r_adversarial\n",
    "    \n",
    "    # Loss weight for alp penalty\n",
    "    lambda_alp = 100\n",
    "\n",
    "# img_shape = (opt.channels, opt.img_size, opt.img_size)   # Not used rn\n",
    "\n",
    "cuda = True if torch.cuda.is_available() else False\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "class Generator(nn.Module):\n",
    "    \"\"\"\n",
    "    Create hidden layers. Apply normalization. Apply leaky relu. \n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        super(Generator, self).__init__()   \n",
    "\n",
    "        def block(in_feat, out_feat, normalize=True):   # This function creates the hidden layers\n",
    "            layers = [nn.Linear(in_feat, out_feat)]   # layer is a hidden layer. Takes input\n",
    "                                                      # (batch_size,in_feat) and give an output (batch_size,out_feat)\n",
    "            if normalize:\n",
    "                layers.append(nn.BatchNorm1d(out_feat, 0.8))   # adds normalization to what Layers does to input and comes out in\n",
    "                                                               # size (batch_size,out_feat). I think this does bn1d(linear(input))\n",
    "            layers.append(nn.LeakyReLU(0.2, inplace=True))   # inplace means just modify input, don't allocate more memory\n",
    "            return layers\n",
    "\n",
    "\n",
    "\n",
    "        \"\"\"\n",
    "        stores layers and functions applied to layers\n",
    "        \"\"\"       \n",
    "        self.model = nn.Sequential(   \n",
    "            *block(opt.latent_dim, 128, normalize=False),   # first layer\n",
    "            *block(128, 256),   # second layer\n",
    "            *block(256, 512),   # 3rd layer\n",
    "            *block(512, 1024),   # 4th layer\n",
    "            nn.Linear(1024, 784),   # final layer. Output is size 25\n",
    "            nn.Tanh()   # Using tanh for final output (why tanh vs leaky relu?)\n",
    "        )\n",
    "\n",
    "    def forward(self, z):\n",
    "        \"\"\"\n",
    "        applies layers to input to get img\n",
    "        \"\"\"\n",
    "        img = self.model(z)   # applies model (layers and functions on layers) to z\n",
    "        #img = img.view(img.shape[0], *img_shape)\n",
    "        return img\n",
    "\n",
    "\n",
    "class Discriminator(nn.Module):\n",
    "    \"\"\"\n",
    "    Discriminator/critic layers\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        super(Discriminator, self).__init__()   # Just uses the module constructor with name Discriminator \n",
    "\n",
    "        self.model = nn.Sequential(\n",
    "            nn.Linear(784, 512),   # first layer\n",
    "            nn.LeakyReLU(0.2, inplace=True),   # apply leaky relu to layer\n",
    "            nn.Linear(512, 256),   # 2nd layer\n",
    "            nn.LeakyReLU(0.2, inplace=True),   # apply leaky relu to layer\n",
    "            nn.Linear(256, 1),   # Final layer to give output. Output is size 1 (validity score)\n",
    "                                 # NOTE: weird to end with comma\n",
    "        )\n",
    "\n",
    "    def forward(self, img):\n",
    "        \"\"\"\n",
    "        applies model to image and gives validity score\n",
    "        \"\"\"\n",
    "        img_flat = img.view(img.shape[0], -1)   # TODO: Figure out what this does \n",
    "        validity = self.model(img_flat)   # calculates validity score\n",
    "        #print(\"forward validity from discriminator: \" + str((np.max(np.abs(validity.detach().numpy())))))\n",
    "        return validity\n",
    "\n",
    "\n",
    "# ******* OUT OF CLASSES NOW ************\n",
    "\n",
    "# Initialize generator and discriminator\n",
    "generator = Generator()\n",
    "discriminator = Discriminator()\n",
    "\n",
    "if cuda:\n",
    "    generator.cuda()\n",
    "    discriminator.cuda()\n",
    "    \n",
    "    \n",
    "training_dataset = datasets.MNIST(\"\", train = True, download = True, transform = transforms.Compose([transforms.ToTensor()]))\n",
    "testing_dataset = datasets.MNIST(\"\", train = False, download = True, transform = transforms.Compose([transforms.ToTensor()]))\n",
    "# testing_dataset -> out-of-sample testing data to run through nn.If you run in sample data only, won't be able to tell\n",
    "# whether The nn has overfitted\n",
    "train_set = torch.utils.data.DataLoader(training_dataset, batch_size=10, shuffle = True)\n",
    "test_set = torch.utils.data.DataLoader(testing_dataset, batch_size=10, shuffle = True) \n",
    "\n",
    "## Configure data loader - CHANGE\n",
    "#os.makedirs(\"./data/\", exist_ok=True)\n",
    "#dataloader = torch.utils.data.DataLoader(\n",
    " #   DiHiggsSignalMCDataset('./DiHiggs Data', generator_level = False),\n",
    "  #  batch_size=opt.batch_size,\n",
    "  #  shuffle=True,\n",
    "#)\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def mnist_data():\n",
    "    compose = transforms.Compose(\n",
    "        [transforms.ToTensor(),\n",
    "         transforms.Normalize([0.5], [0.5])\n",
    "        ])\n",
    "    out_dir = '{}/dataset'.format(DATA_FOLDER)\n",
    "    return datasets.MNIST(root=out_dir, train=True, transform=compose, download=True)\n",
    "\n",
    "# def mnist_data():\n",
    "#     compose = transforms.Compose(\n",
    "#         [transforms.ToTensor(),\n",
    "#          transforms.Normalize((.5,), (.5,))\n",
    "#         ])\n",
    "#     out_dir = '{}/dataset'.format(DATA_FOLDER)\n",
    "#     return datasets.MNIST(root=out_dir, train=True, transform=compose, download=True)\n",
    "\n",
    "def noise(size):\n",
    "    '''\n",
    "    Generates a 1-d vector of gaussian sampled random values\n",
    "    '''\n",
    "    n = Variable(torch.randn(size, 100))\n",
    "    if torch.cuda.is_available(): return n.cuda()\n",
    "    return n\n",
    "\n",
    "def images_to_vectors(images):\n",
    "    return images.view(images.size(0), 784)\n",
    "\n",
    "def vectors_to_images(vectors):\n",
    "    return vectors.view(vectors.size(0), 1, 28, 28)\n",
    "\n",
    "def ones_target(size):\n",
    "    '''\n",
    "    Tensor containing ones, with shape = size\n",
    "    '''\n",
    "    data = Variable(torch.ones(size, 1))\n",
    "    if torch.cuda.is_available(): return data.cuda()\n",
    "    return data\n",
    "\n",
    "def zeros_target(size):\n",
    "    '''\n",
    "    Tensor containing zeros, with shape = size\n",
    "    '''\n",
    "    data = Variable(torch.zeros(size, 1))\n",
    "    if torch.cuda.is_available(): return data.cuda()\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_ALP(D, real_samples, fake_samples):   # TODO: Find out why these are .data\n",
    "    \"\"\"\n",
    "    Calculates the gradient penalty loss for WGAN GP\n",
    "    D input will be discrimantor function\n",
    "    real_samples and fake_samples are from reality and generator. Both are sent in via memory location of buffer\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    # Random weight term for interpolation between real and fake samples (how much of each)\n",
    "    alpha = Tensor(np.random.random((real_samples.size(0),1)))   # This is a tensor designating which to use where\n",
    "    #print(alpha)\n",
    "  #  print(alpha.shape)\n",
    "    # Get random interpolation between real and fake samples\n",
    "   # print(real_samples.shape)\n",
    "    \n",
    "    # Gets some of real and some of fake samples for gradient penalty calculation\n",
    "    interpolates = (alpha * real_samples + ((1 - alpha) * fake_samples)).requires_grad_(True)\n",
    "    # .requires grad is something attached to all tensors and can be used to speed up (by making false I think)\n",
    "    # It is automatically false, but if you need gradient then set to be true\n",
    "    # TODO: Understand how this statement works\n",
    "    \n",
    "    \n",
    "    ################## CALCULATE R ADVERSARIAL ###############################################\n",
    "    # start with random unit vector r0\n",
    "    r0 = np.random.rand(interpolates.shape[0], interpolates.shape[1])\n",
    "    r0 = Tensor(r0/r0.max(axis = 0)).requires_grad_(True)\n",
    "    #print(r[0])\n",
    "    \n",
    "    #  add this initial r to our random data points\n",
    "    interpol_y0 = (interpolates + opt.Xi * r0).requires_grad_(True)   #.requires_grad_(True)\n",
    "    # run the discriminator on both of these\n",
    "    d_interpolates = D(interpolates)   # Run discriminator on interpolates to get validity scores\n",
    "    d_interpol_y0 = D(interpol_y0)   # do the same for the adjusted interpolates to find r adversarial\n",
    "\n",
    "    \n",
    "    # find gradient(d(f(x) - f(x+r)))\n",
    "    difference = (d_interpolates - d_interpol_y0).requires_grad_(True)  #.requires_grad_(True)\n",
    "    #print(\"d interpolates: \" + str(d_interpolates.shape) + \" \" + str(d_interpolates.type))\n",
    "    #print(\"difference: \" + str(difference.shape) + \" \" + str(difference.type))\n",
    "    fake = Variable(Tensor(real_samples.shape[0], 1).fill_(1.0), requires_grad=False) \n",
    "    gradient_r0 = autograd.grad(\n",
    "        outputs=difference,\n",
    "        inputs=r0,\n",
    "        grad_outputs=fake,\n",
    "        create_graph=True,\n",
    "        retain_graph=True,\n",
    "        only_inputs=True\n",
    "    )[0]\n",
    "    \n",
    "    # finally, find r_adversarial!\n",
    "    epsilon_r = np.random.uniform(0.1,10)\n",
    "    r_adv = epsilon_r * gradient_r0/np.linalg.norm(gradient_r0.cpu().detach().numpy())\n",
    "    #print(np.max(np.linalg.norm(r_adv.cpu().detach().numpy())))\n",
    "###########################################################################################################\n",
    "\n",
    "######### Now find the loss ###########################\n",
    "    \n",
    "    interpol_adversarial = (interpolates + r_adv).requires_grad_(True)\n",
    "    d_interpol_adv = D(interpol_adversarial)\n",
    "    abs_difference = np.abs((d_interpolates - d_interpol_adv).cpu().detach().numpy())/ \\\n",
    "    (np.linalg.norm(r_adv.cpu().detach().numpy())) - 1\n",
    "    squared = np.square(np.maximum(abs_difference,np.zeros(100)))\n",
    "    #print(\"Max of alp before mean: \" + str(np.max(np.abs(squared))))\n",
    "    \n",
    "    alp_penalty = squared.mean()\n",
    "   # print(\"ALP final: \" + str(alp_penalty))\n",
    "    \n",
    "    return alp_penalty\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.5955372  0.31434822 0.4211952  0.99794535]\n",
      " [0.30086308 0.91122678 0.0620271  0.3263242 ]\n",
      " [0.9490622  0.70296402 0.9985708  0.2520672 ]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9985707953440579"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = np.random.rand(3,4)\n",
    "print(y)\n",
    "np.max(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_FOLDER = './torch_data/VGAN/MNIST'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = np.array((1,2,-3,-4,5))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "600\n"
     ]
    }
   ],
   "source": [
    "# Load data\n",
    "data = mnist_data()\n",
    "# Create loader with data, so that we can iterate over it\n",
    "train_set = torch.utils.data.DataLoader(data, batch_size=100, shuffle=True)\n",
    "# Num batches\n",
    "num_batches = len(train_set)\n",
    "print(num_batches)\n",
    "\n",
    "\n",
    "num_test_samples = 16\n",
    "test_noise = noise(num_test_samples)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# lists to check things converge\n",
    "ALP_LIST = []\n",
    "G_LOSS_LIST = []\n",
    "D_LOSSL_LIST = []\n",
    "\n",
    "# Optimizers (Adam optimizers are an alternative to stochastic gradient descent. TODO learn more about them)\n",
    "optimizer_G = torch.optim.Adam(generator.parameters(), lr=opt.lr, betas=(opt.b1, opt.b2))   \n",
    "optimizer_D = torch.optim.Adam(discriminator.parameters(), lr=opt.lr, betas=(opt.b1, opt.b2))\n",
    "\n",
    "Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor\n",
    "\n",
    "dis_error = []\n",
    "gen_error = []\n",
    "\n",
    "# ----------\n",
    "#  Training\n",
    "# ----------\n",
    "\n",
    "batches_done = 0   # Counter for batches\n",
    "for epoch in range(opt.n_epochs):   # Loop through all epochs\n",
    "    for i, (x,_) in enumerate(train_set): # x is in dataloader (a batch I think). i\n",
    "                                       # is the index of x (number of times critic is trained this epoch)\n",
    "        #if cuda:\n",
    "         #   x = x.cuda()\n",
    "\n",
    "        # Configure input\n",
    "        #x = np.ndarray(x)\n",
    "        #print(type(x))\n",
    "        if cuda: real_imgs = Variable(images_to_vectors(x.cuda()))\n",
    "        else: real_imgs = Variable(images_to_vectors(x))\n",
    "        #print(real_imgs)\n",
    "        #real_imgs = Variable(images_to_vectors(x).type(Tensor))   # Variable is a wrapper for the Tensor x was just made into\n",
    "\n",
    "        # ---------------------\n",
    "        #  Train Discriminator\n",
    "        # ---------------------\n",
    "\n",
    "        optimizer_D.zero_grad()   # Make gradients zero so they don't accumulate\n",
    "\n",
    "        # Sample noise (latent space) to make generator input\n",
    "        if cuda: z = Variable(Tensor(np.random.normal(0, 1, (x.shape[0], opt.latent_dim))).cuda())   \n",
    "        else: z = Variable(Tensor(np.random.normal(0, 1, (x.shape[0], opt.latent_dim))) )\n",
    "        # Generate a batch of images from the latent space sampled\n",
    "        fake_imgs = generator(z)\n",
    "        #fake_imgs = generator(noise(x.size(0)))\n",
    "        #print(fake_imgs[0])\n",
    "\n",
    "        # Calculate validity score for real images\n",
    "        real_validity = discriminator(real_imgs)\n",
    "\n",
    "        # Calculate validity score for fake images\n",
    "        fake_validity = discriminator(fake_imgs)\n",
    "\n",
    "        # Calculate gradient penalty\n",
    "        alp_penalty = compute_ALP(discriminator, real_imgs.data, fake_imgs.data)\n",
    "        # TODO: figure out why .data is used\n",
    "\n",
    "        # Calculate loss for critic (Adversarial loss)\n",
    "        d_loss_noALP = -torch.mean(real_validity) + torch.mean(fake_validity)\n",
    "        d_loss = d_loss_noALP + opt.lambda_alp * alp_penalty\n",
    "        #print(-torch.mean(real_validity) + torch.mean(fake_validity))\n",
    "        \n",
    "        d_loss.backward()   # Do back propagation \n",
    "        optimizer_D.step()   # Update parameters based on gradients for individualss\n",
    "\n",
    "        optimizer_G.zero_grad()   # Resets gradients for generator to be zero to avoid accumulation\n",
    "\n",
    "        # Train the generator every n_critic steps\n",
    "        if i % opt.n_critic == 0:\n",
    "\n",
    "            # -----------------\n",
    "            #  Train Generator\n",
    "            # -----------------\n",
    "\n",
    "            # Generate a batch of images\n",
    "            fake_imgs = generator(z)\n",
    "\n",
    "            # Loss measures generator's ability to fool the discriminator\n",
    "            # Train on fake images\n",
    "            fake_validity = discriminator(fake_imgs)\n",
    "            g_loss = -torch.mean(fake_validity)\n",
    "\n",
    "            g_loss.backward()\n",
    "            optimizer_G.step()\n",
    "\n",
    "\n",
    "            # ----------------------------\n",
    "            # Save stuff when time is right\n",
    "            # ----------------------------\n",
    "            if batches_done % 10 == 0:\n",
    "                print(\n",
    "                    \"[Epoch %d/%d] [Batch %d/%d] [D loss: %f] [G loss: %f]\"\n",
    "                    % (epoch, opt.n_epochs, i, len(train_set), d_loss.item(), g_loss.item())\n",
    "                )\n",
    "                print(\"alp: \" + str(alp_penalty))\n",
    "\n",
    "            if batches_done % opt.sample_interval == 0:\n",
    "                save_image(fake_imgs.data[:728], \"images/%d.png\" % batches_done, nrow=5, normalize=True)\n",
    "        \n",
    "                test_images = vectors_to_images(fake_imgs).data.cpu()\n",
    "                #print(test_images.shape)\n",
    "                display.clear_output(True)\n",
    "                count = 0\n",
    "                for i in range(0,10):\n",
    "                    t_image = test_images[i]\n",
    "                    t = t_image.numpy()\n",
    "                 #   print(t.shape)\n",
    "                \n",
    "                    image1 = t[0]\n",
    "\n",
    "                    image = np.array((image1))\n",
    "                    plt.imshow(image, cmap='gray')\n",
    "                    plt.show()\n",
    "                # Display stuff\n",
    "#                 display.clear_output(True)\n",
    "\n",
    "#                 dis_error.append(d_loss.item())\n",
    "#                 gen_error.append(g_loss.item())\n",
    "#                 print(g_loss.data.cpu().numpy())\n",
    "#                 # Display Images\n",
    "#                 test_images = vectors_to_images(generator(test_noise)).data.cpu()\n",
    "#                 # Display status Logs\n",
    "\n",
    "#                 # Model Checkpoints\n",
    "#                 fig, axs = plt.subplots(2)\n",
    "#                 axs[0].plot(batches_done)\n",
    "#                 axs[1].plot(batches_done)\n",
    "#                 axs[0].set_title(\"Discriminator Error\")\n",
    "#                 axs[0].set_xlabel(\"Batch #\")\n",
    "#                 axs[0].set_ylabel(\"Error\")\n",
    "#                 axs[0].set_xscale('log')\n",
    "#                 axs[1].set_title(\"Generator Error\")\n",
    "#                 axs[1].set_xlabel(\"Batch #\")\n",
    "#                 axs[1].set_ylabel(\"Error\")\n",
    "#                 axs[1].set_xscale('log')\n",
    "#                 fig.tight_layout()\n",
    "#                 plt.show()\n",
    "\n",
    "            batches_done += opt.n_critic\n",
    "            \n",
    "\n",
    "            \n",
    "            \n",
    "            \n",
    "    if epoch % 10 == 0:\n",
    "        z = Variable(Tensor(np.random.normal(0, 1, (300000, opt.latent_dim))))\n",
    "        fake_data = generator(z)\n",
    "        np.save('./mnist_data/{num_batches}.npy'.format(num_batches=batches_done), fake_data.cpu().detach().numpy())\n",
    "    \n",
    "    #if epoch % 0 == 0 or epoch == 999:\n",
    "        \n",
    "\n",
    "#for i, x in enumerate(dataloader):\n",
    "    #print(x[0])\n",
    "#Evaluation KS metric\n",
    "# Generate ~300,000 samples and save (NumPy array)\n",
    "# Load ^, 5000 then\n",
    "# KS <-- Root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ALP_np = np.array(ALP_LIST)\n",
    "G_LOSS_np = np.array(G_LOSS_LIST)\n",
    "D_LOSS_np = np.array(D_LOSS_LIST)\n",
    "np.save('./mnist_data/ALP_LIST.npy')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (GANS)",
   "language": "python",
   "name": "gans"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}