DistTrain_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Keras + horovod + ipyparallel MNIST example\n",
    "\n",
    "In this notebook we will use ipyparallel to deploy a Keras + Horovod distributed training example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# System imports\n",
    "from __future__ import print_function\n",
    "from __future__ import division\n",
    "from __future__ import absolute_import\n",
    "\n",
    "# External imports\n",
    "import ipyparallel as ipp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Connect to ipyparallel cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)\n",
      "          24175753 interacti       sh sfarrell  R       1:35      8 nid00[163-170]\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "squeue -u sfarrell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Worker IDs: [0, 1, 2, 3, 4, 5, 6, 7]\n"
     ]
    }
   ],
   "source": [
    "# Cluster ID taken from job ID above\n",
    "job_id = 24175753\n",
    "cluster_id = 'cori_{}'.format(job_id)\n",
    "\n",
    "# Use default profile\n",
    "c = ipp.Client(timeout=60, cluster_id=cluster_id)\n",
    "print('Worker IDs:', c.ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize environment on the workers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[stderr:0] Using TensorFlow backend.\n",
      "[stderr:1] Using TensorFlow backend.\n",
      "[stderr:2] Using TensorFlow backend.\n",
      "[stderr:3] Using TensorFlow backend.\n",
      "[stderr:4] Using TensorFlow backend.\n",
      "[stderr:5] Using TensorFlow backend.\n",
      "[stderr:6] Using TensorFlow backend.\n",
      "[stderr:7] Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%%px\n",
    "\n",
    "from __future__ import print_function\n",
    "from __future__ import division\n",
    "\n",
    "import socket\n",
    "import math\n",
    "\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras import backend as K\n",
    "import tensorflow as tf\n",
    "\n",
    "# Horovod for MPI synchronization routines\n",
    "import horovod.keras as hvd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[stdout:0] MPI rank 0, local rank 0, host nid00163\n",
      "[stdout:1] MPI rank 6, local rank 0, host nid00169\n",
      "[stdout:2] MPI rank 4, local rank 0, host nid00167\n",
      "[stdout:3] MPI rank 1, local rank 0, host nid00164\n",
      "[stdout:4] MPI rank 7, local rank 0, host nid00170\n",
      "[stdout:5] MPI rank 5, local rank 0, host nid00168\n",
      "[stdout:6] MPI rank 3, local rank 0, host nid00166\n",
      "[stdout:7] MPI rank 2, local rank 0, host nid00165\n"
     ]
    }
   ],
   "source": [
    "%%px\n",
    "\n",
    "# Initialize horovod\n",
    "hvd.init()\n",
    "print('MPI rank %i, local rank %i, host %s' %\n",
    "      (hvd.rank(), hvd.local_rank(), socket.gethostname()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%px\n",
    "\n",
    "# Data config\n",
    "n_classes = 10\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "# Training config\n",
    "batch_size = 128\n",
    "n_epochs = 8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the data on each worker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[stdout:0] \n",
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "[stdout:1] \n",
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "[stdout:2] \n",
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "[stdout:3] \n",
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "[stdout:4] \n",
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "[stdout:5] \n",
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "[stdout:6] \n",
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "[stdout:7] \n",
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "%%px\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n",
    "    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "\n",
    "# Scale pixels to [0, 1]\n",
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# Convert class vectors to binary class matrices\n",
    "y_train = keras.utils.to_categorical(y_train, n_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, n_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[stdout:0] \n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 26, 26, 32)        320       \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 24, 24, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 12, 12, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 12, 12, 64)        0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 9216)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 128)               1179776   \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 1,199,882\n",
      "Trainable params: 1,199,882\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "%%px\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Conv2D(32, kernel_size=(3, 3),\n",
    "                 activation='relu',\n",
    "                 input_shape=input_shape))\n",
    "model.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128, activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(n_classes, activation='softmax'))\n",
    "\n",
    "# Adjust learning rate based on number of workers.\n",
    "opt = keras.optimizers.Adadelta(1.0 * hvd.size())\n",
    "\n",
    "# Add Horovod Distributed Optimizer.\n",
    "opt = hvd.DistributedOptimizer(opt)\n",
    "\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=opt,\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "if hvd.rank() == 0:\n",
    "    model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Distributed training\n",
    "\n",
    "Training with horovod + MPI allows for synchronous distributed batch updates.\n",
    "\n",
    "We need to register the model synchronization callback and restrict checkpoint writing to a single worker."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[stdout:0] \n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/8\n",
      " - 12s - loss: 0.1705 - acc: 0.9485 - val_loss: 0.0311 - val_acc: 0.9891\n",
      "Epoch 2/8\n",
      " - 11s - loss: 0.0374 - acc: 0.9884 - val_loss: 0.0306 - val_acc: 0.9911\n",
      "Epoch 3/8\n",
      " - 11s - loss: 0.0261 - acc: 0.9912 - val_loss: 0.0501 - val_acc: 0.9877\n",
      "Epoch 4/8\n",
      " - 11s - loss: 0.0191 - acc: 0.9935 - val_loss: 0.0358 - val_acc: 0.9911\n",
      "Epoch 5/8\n",
      " - 11s - loss: 0.0157 - acc: 0.9949 - val_loss: 0.0322 - val_acc: 0.9921\n",
      "Epoch 6/8\n",
      " - 11s - loss: 0.0128 - acc: 0.9956 - val_loss: 0.0301 - val_acc: 0.9929\n",
      "Epoch 7/8\n",
      " - 11s - loss: 0.0103 - acc: 0.9965 - val_loss: 0.0328 - val_acc: 0.9935\n",
      "Epoch 8/8\n",
      " - 11s - loss: 0.0091 - acc: 0.9969 - val_loss: 0.0336 - val_acc: 0.9932\n",
      "[stdout:1] \n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/8\n",
      " - 12s - loss: 0.1703 - acc: 0.9483 - val_loss: 0.0311 - val_acc: 0.9891\n",
      "Epoch 2/8\n",
      " - 11s - loss: 0.0371 - acc: 0.9879 - val_loss: 0.0306 - val_acc: 0.9911\n",
      "Epoch 3/8\n",
      " - 11s - loss: 0.0249 - acc: 0.9919 - val_loss: 0.0501 - val_acc: 0.9877\n",
      "Epoch 4/8\n",
      " - 11s - loss: 0.0194 - acc: 0.9937 - val_loss: 0.0358 - val_acc: 0.9911\n",
      "Epoch 5/8\n",
      " - 11s - loss: 0.0149 - acc: 0.9953 - val_loss: 0.0322 - val_acc: 0.9921\n",
      "Epoch 6/8\n",
      " - 11s - loss: 0.0123 - acc: 0.9959 - val_loss: 0.0301 - val_acc: 0.9929\n",
      "Epoch 7/8\n",
      " - 11s - loss: 0.0117 - acc: 0.9962 - val_loss: 0.0328 - val_acc: 0.9935\n",
      "Epoch 8/8\n",
      " - 11s - loss: 0.0104 - acc: 0.9966 - val_loss: 0.0336 - val_acc: 0.9932\n",
      "[stdout:2] \n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/8\n",
      " - 12s - loss: 0.1756 - acc: 0.9465 - val_loss: 0.0311 - val_acc: 0.9891\n",
      "Epoch 2/8\n",
      " - 11s - loss: 0.0371 - acc: 0.9883 - val_loss: 0.0306 - val_acc: 0.9911\n",
      "Epoch 3/8\n",
      " - 11s - loss: 0.0247 - acc: 0.9914 - val_loss: 0.0501 - val_acc: 0.9877\n",
      "Epoch 4/8\n",
      " - 11s - loss: 0.0194 - acc: 0.9935 - val_loss: 0.0358 - val_acc: 0.9911\n",
      "Epoch 5/8\n",
      " - 11s - loss: 0.0161 - acc: 0.9945 - val_loss: 0.0322 - val_acc: 0.9921\n",
      "Epoch 6/8\n",
      " - 11s - loss: 0.0133 - acc: 0.9956 - val_loss: 0.0301 - val_acc: 0.9929\n",
      "Epoch 7/8\n",
      " - 11s - loss: 0.0094 - acc: 0.9970 - val_loss: 0.0328 - val_acc: 0.9935\n",
      "Epoch 8/8\n",
      " - 11s - loss: 0.0106 - acc: 0.9968 - val_loss: 0.0336 - val_acc: 0.9932\n",
      "[stdout:3] \n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/8\n",
      " - 12s - loss: 0.1753 - acc: 0.9468 - val_loss: 0.0311 - val_acc: 0.9891\n",
      "Epoch 2/8\n",
      " - 11s - loss: 0.0382 - acc: 0.9884 - val_loss: 0.0306 - val_acc: 0.9911\n",
      "Epoch 3/8\n",
      " - 11s - loss: 0.0244 - acc: 0.9919 - val_loss: 0.0501 - val_acc: 0.9877\n",
      "Epoch 4/8\n",
      " - 11s - loss: 0.0192 - acc: 0.9935 - val_loss: 0.0358 - val_acc: 0.9911\n",
      "Epoch 5/8\n",
      " - 11s - loss: 0.0153 - acc: 0.9951 - val_loss: 0.0322 - val_acc: 0.9921\n",
      "Epoch 6/8\n",
      " - 11s - loss: 0.0124 - acc: 0.9960 - val_loss: 0.0301 - val_acc: 0.9929\n",
      "Epoch 7/8\n",
      " - 11s - loss: 0.0113 - acc: 0.9965 - val_loss: 0.0328 - val_acc: 0.9935\n",
      "Epoch 8/8\n",
      " - 11s - loss: 0.0109 - acc: 0.9965 - val_loss: 0.0336 - val_acc: 0.9932\n",
      "[stdout:4] \n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/8\n",
      " - 12s - loss: 0.1716 - acc: 0.9473 - val_loss: 0.0311 - val_acc: 0.9891\n",
      "Epoch 2/8\n",
      " - 11s - loss: 0.0377 - acc: 0.9879 - val_loss: 0.0306 - val_acc: 0.9911\n",
      "Epoch 3/8\n",
      " - 11s - loss: 0.0252 - acc: 0.9922 - val_loss: 0.0501 - val_acc: 0.9877\n",
      "Epoch 4/8\n",
      " - 11s - loss: 0.0182 - acc: 0.9937 - val_loss: 0.0358 - val_acc: 0.9911\n",
      "Epoch 5/8\n",
      " - 11s - loss: 0.0144 - acc: 0.9950 - val_loss: 0.0322 - val_acc: 0.9921\n",
      "Epoch 6/8\n",
      " - 11s - loss: 0.0130 - acc: 0.9959 - val_loss: 0.0301 - val_acc: 0.9929\n",
      "Epoch 7/8\n",
      " - 11s - loss: 0.0106 - acc: 0.9966 - val_loss: 0.0328 - val_acc: 0.9935\n",
      "Epoch 8/8\n",
      " - 11s - loss: 0.0098 - acc: 0.9968 - val_loss: 0.0336 - val_acc: 0.9932\n",
      "[stdout:5] \n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/8\n",
      " - 12s - loss: 0.1755 - acc: 0.9470 - val_loss: 0.0311 - val_acc: 0.9891\n",
      "Epoch 2/8\n",
      " - 11s - loss: 0.0397 - acc: 0.9872 - val_loss: 0.0306 - val_acc: 0.9911\n",
      "Epoch 3/8\n",
      " - 11s - loss: 0.0236 - acc: 0.9921 - val_loss: 0.0501 - val_acc: 0.9877\n",
      "Epoch 4/8\n",
      " - 11s - loss: 0.0204 - acc: 0.9936 - val_loss: 0.0358 - val_acc: 0.9911\n",
      "Epoch 5/8\n",
      " - 11s - loss: 0.0152 - acc: 0.9949 - val_loss: 0.0322 - val_acc: 0.9921\n",
      "Epoch 6/8\n",
      " - 11s - loss: 0.0120 - acc: 0.9959 - val_loss: 0.0301 - val_acc: 0.9929\n",
      "Epoch 7/8\n",
      " - 11s - loss: 0.0101 - acc: 0.9965 - val_loss: 0.0328 - val_acc: 0.9935\n",
      "Epoch 8/8\n",
      " - 11s - loss: 0.0102 - acc: 0.9966 - val_loss: 0.0336 - val_acc: 0.9932\n",
      "[stdout:6] \n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/8\n",
      " - 12s - loss: 0.1722 - acc: 0.9471 - val_loss: 0.0311 - val_acc: 0.9891\n",
      "Epoch 2/8\n",
      " - 11s - loss: 0.0385 - acc: 0.9881 - val_loss: 0.0306 - val_acc: 0.9911\n",
      "Epoch 3/8\n",
      " - 11s - loss: 0.0231 - acc: 0.9925 - val_loss: 0.0501 - val_acc: 0.9877\n",
      "Epoch 4/8\n",
      " - 11s - loss: 0.0187 - acc: 0.9937 - val_loss: 0.0358 - val_acc: 0.9911\n",
      "Epoch 5/8\n",
      " - 11s - loss: 0.0145 - acc: 0.9953 - val_loss: 0.0322 - val_acc: 0.9921\n",
      "Epoch 6/8\n",
      " - 11s - loss: 0.0126 - acc: 0.9957 - val_loss: 0.0301 - val_acc: 0.9929\n",
      "Epoch 7/8\n",
      " - 11s - loss: 0.0107 - acc: 0.9963 - val_loss: 0.0328 - val_acc: 0.9935\n",
      "Epoch 8/8\n",
      " - 11s - loss: 0.0103 - acc: 0.9967 - val_loss: 0.0336 - val_acc: 0.9932\n",
      "[stdout:7] \n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/8\n",
      " - 12s - loss: 0.1719 - acc: 0.9463 - val_loss: 0.0311 - val_acc: 0.9891\n",
      "Epoch 2/8\n",
      " - 11s - loss: 0.0384 - acc: 0.9876 - val_loss: 0.0306 - val_acc: 0.9911\n",
      "Epoch 3/8\n",
      " - 11s - loss: 0.0242 - acc: 0.9918 - val_loss: 0.0501 - val_acc: 0.9877\n",
      "Epoch 4/8\n",
      " - 11s - loss: 0.0185 - acc: 0.9938 - val_loss: 0.0358 - val_acc: 0.9911\n",
      "Epoch 5/8\n",
      " - 11s - loss: 0.0153 - acc: 0.9949 - val_loss: 0.0322 - val_acc: 0.9921\n",
      "Epoch 6/8\n",
      " - 11s - loss: 0.0131 - acc: 0.9958 - val_loss: 0.0301 - val_acc: 0.9929\n",
      "Epoch 7/8\n",
      " - 11s - loss: 0.0116 - acc: 0.9963 - val_loss: 0.0328 - val_acc: 0.9935\n",
      "Epoch 8/8\n",
      " - 11s - loss: 0.0098 - acc: 0.9970 - val_loss: 0.0336 - val_acc: 0.9932\n"
     ]
    }
   ],
   "source": [
    "%%px\n",
    "\n",
    "callbacks = [\n",
    "    # Horovod: broadcast initial variable states from rank 0 to all other processes.\n",
    "    # This is necessary to ensure consistent initialization of all workers when\n",
    "    # training is started with random weights or restored from a checkpoint.\n",
    "    hvd.callbacks.BroadcastGlobalVariablesCallback(0),\n",
    "]\n",
    "\n",
    "# Horovod: save checkpoints only on worker 0 to prevent other workers from corrupting them.\n",
    "#if hvd.rank() == 0:\n",
    "#    callbacks.append(keras.callbacks.ModelCheckpoint('./checkpoint-{epoch}.h5'))\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "                    batch_size=batch_size,\n",
    "                    callbacks=callbacks,\n",
    "                    epochs=n_epochs,\n",
    "                    verbose=2,\n",
    "                    validation_data=(x_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[stdout:0] \n",
      "Test loss: 0.033566167738372045\n",
      "Test accuracy: 0.9932\n",
      "[stdout:1] \n",
      "Test loss: 0.033566167738372045\n",
      "Test accuracy: 0.9932\n",
      "[stdout:2] \n",
      "Test loss: 0.033566167738372045\n",
      "Test accuracy: 0.9932\n",
      "[stdout:3] \n",
      "Test loss: 0.033566167738372045\n",
      "Test accuracy: 0.9932\n",
      "[stdout:4] \n",
      "Test loss: 0.033566167738372045\n",
      "Test accuracy: 0.9932\n",
      "[stdout:5] \n",
      "Test loss: 0.033566167738372045\n",
      "Test accuracy: 0.9932\n",
      "[stdout:6] \n",
      "Test loss: 0.033566167738372045\n",
      "Test accuracy: 0.9932\n",
      "[stdout:7] \n",
      "Test loss: 0.033566167738372045\n",
      "Test accuracy: 0.9932\n"
     ]
    }
   ],
   "source": [
    "%%px\n",
    "\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow-intel(cpu)/1.13.1-py36",
   "language": "python",
   "name": "tensorflow_intel_1.13.1_py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}