TF2_RecommenderSystem_CyclicLR.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TF2-Recommender-System-Cyclic-LR.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UCDRD4_WHwIE",
        "colab_type": "text"
      },
      "source": [
        "# Recommender Systems in Tensorflow 2\n",
        "\n",
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/googlecolab/colabtools/blob/master/notebooks/colab-github-demo.ipynb)\n",
        "\n",
        "- Implementation of Matrix Factorization in Tensorflow 2.0 using feature columns\n",
        "- Comparison to performance using fast.ai & bringing TensorFlow model to the same level (using cyclic learning rates (1-cycle policy))\n",
        "\n",
        "Dataset: https://grouplens.org/datasets/movielens/1m/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-1fJZMolCLZB",
        "colab_type": "code",
        "outputId": "115de6b7-2692-4b89-ff35-622bb67f713d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Use Tensorflow 2.0 \n",
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  %tensorflow_version 2.x\n",
        "  # Load the TensorBoard extension\n",
        "  %load_ext tensorboard\n",
        "except Exception:\n",
        "  pass"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 2.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OSPOpE18DNLs",
        "colab_type": "code",
        "outputId": "7445645a-2c93-4f7d-8621-f48004fefb7b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "from tensorflow import keras\n",
        "AUTOTUNE = tf.data.experimental.AUTOTUNE\n",
        "print(tf.__version__)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "poJ2TdqRH5DB",
        "colab_type": "text"
      },
      "source": [
        "### Download Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FarEB_qoDXog",
        "colab_type": "code",
        "outputId": "2613c04f-04c2-4703-9766-dfd539443484",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 444
        }
      },
      "source": [
        "!wget https://storage.googleapis.com/dl-workshop/datasets/ml-1m/ratings.dat\n",
        "!wget https://storage.googleapis.com/dl-workshop/datasets/ml-1m/movies.dat\n",
        "\n",
        "!mkdir -vp data\n",
        "!mv ratings.dat data/\n",
        "!mv movies.dat data/\n",
        "!ls data"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-11-21 17:06:28--  https://storage.googleapis.com/dl-workshop/datasets/ml-1m/ratings.dat\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 172.217.212.128, 2607:f8b0:4001:c05::80\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|172.217.212.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 24594131 (23M) [application/octet-stream]\n",
            "Saving to: ‘ratings.dat’\n",
            "\n",
            "ratings.dat         100%[===================>]  23.45M  17.1MB/s    in 1.4s    \n",
            "\n",
            "2019-11-21 17:06:30 (17.1 MB/s) - ‘ratings.dat’ saved [24594131/24594131]\n",
            "\n",
            "--2019-11-21 17:06:31--  https://storage.googleapis.com/dl-workshop/datasets/ml-1m/movies.dat\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 173.194.195.128, 2607:f8b0:4001:c07::80\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|173.194.195.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 171308 (167K) [application/octet-stream]\n",
            "Saving to: ‘movies.dat’\n",
            "\n",
            "movies.dat          100%[===================>] 167.29K  --.-KB/s    in 0.002s  \n",
            "\n",
            "2019-11-21 17:06:32 (83.5 MB/s) - ‘movies.dat’ saved [171308/171308]\n",
            "\n",
            "mkdir: created directory 'data'\n",
            "movies.dat  ratings.dat\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sjWt4q_Ea30B",
        "colab_type": "text"
      },
      "source": [
        "## Data Loading & Preprocessing\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8SkjANrsNhnS",
        "colab_type": "code",
        "outputId": "cc91d41e-8b2a-4eb0-dc15-19ba611ea86e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 248
        }
      },
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "\n",
        "dataset_path = 'data'\n",
        "\n",
        "ratings_path ='data/ratings.dat'\n",
        "movies_path = 'data/movies.dat'\n",
        "\n",
        "ratings_columns = 'UserID::MovieID::Rating::Timestamp'.split('::')\n",
        "movie_columns = 'MovieID::Title::Genres'.split('::')\n",
        "\n",
        "df_ratings = pd.read_csv(ratings_path, sep='::', names=ratings_columns)\n",
        "df_movies = pd.read_csv(os.path.join(dataset_path, 'movies.dat'), sep='::', names=movie_columns)\n",
        "df_ratings = df_ratings.sample(frac=1, random_state=0)\n",
        "\n",
        "df_ratings['UserID'] = df_ratings['UserID'] - 1\n",
        "df_ratings['MovieID'] = df_ratings['MovieID'] - 1\n",
        "\n",
        "df_ratings = df_ratings.join(df_movies.set_index('MovieID'), on='MovieID')\n",
        "\n",
        "# | check number of unique users & movies\n",
        "nr_users = len(df_ratings['UserID'].unique())\n",
        "nr_movies = len(df_ratings['MovieID'].unique())\n",
        "print('#Users: {}, #Movies: {}'.format(nr_users, nr_movies))\n",
        "\n",
        "df_ratings.head(3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:12: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
            "  if sys.path[0] == '':\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:13: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
            "  del sys.path[0]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "#Users: 6040, #Movies: 3706\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>UserID</th>\n",
              "      <th>MovieID</th>\n",
              "      <th>Rating</th>\n",
              "      <th>Timestamp</th>\n",
              "      <th>Title</th>\n",
              "      <th>Genres</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>324271</th>\n",
              "      <td>1921</td>\n",
              "      <td>2093</td>\n",
              "      <td>4</td>\n",
              "      <td>974694867</td>\n",
              "      <td>Return to Oz (1985)</td>\n",
              "      <td>Adventure|Children's|Fantasy|Sci-Fi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>818637</th>\n",
              "      <td>4917</td>\n",
              "      <td>2807</td>\n",
              "      <td>1</td>\n",
              "      <td>963084545</td>\n",
              "      <td>Universal Soldier: The Return (1999)</td>\n",
              "      <td>Action|Sci-Fi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>148677</th>\n",
              "      <td>956</td>\n",
              "      <td>1659</td>\n",
              "      <td>4</td>\n",
              "      <td>975130093</td>\n",
              "      <td>Hurricane Streets (1998)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        UserID  ...                               Genres\n",
              "324271    1921  ...  Adventure|Children's|Fantasy|Sci-Fi\n",
              "818637    4917  ...                        Action|Sci-Fi\n",
              "148677     956  ...                                Drama\n",
              "\n",
              "[3 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N3MzYc8RsjGF",
        "colab_type": "code",
        "outputId": "00c52c16-c09b-4d82-975f-54fd87c7c710",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 106
        }
      },
      "source": [
        "train_ratings_path = os.path.join(dataset_path, 'ratings-train.dat')\n",
        "val_ratings_path = os.path.join(dataset_path, 'ratings-val.dat')\n",
        "\n",
        "train_size = int(0.8*len(df_ratings))\n",
        "val_size = len(df_ratings) - train_size\n",
        "\n",
        "df_ratings.iloc[:train_size].to_csv(train_ratings_path, sep=';', index=False, header=True)\n",
        "df_ratings.iloc[train_size:].to_csv(val_ratings_path, sep=';', index=False, header=True)\n",
        "\n",
        "# | for fast.ai\n",
        "df_ratings['validation'] = False\n",
        "df_ratings['validation'].iloc[train_size:] = True"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:205: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  self._setitem_with_indexer(indexer, value)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wPSWv0ZIePxh",
        "colab_type": "text"
      },
      "source": [
        "# TensorFlow"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AwfGHks8m7KZ",
        "colab_type": "text"
      },
      "source": [
        "## Create data pipeline (tf.data.dataset)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EiZIH409nAzy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_csv_dataset_hl(csv_path, label_name, select_columns=None, epochs=1, batch_size=8, buffer_size=10000, shuffle=False):\n",
        "    return tf.data.experimental.make_csv_dataset(csv_path,\n",
        "                                               batch_size=batch_size,\n",
        "                                               field_delim=';',\n",
        "                                               label_name=label_name,\n",
        "                                               select_columns=select_columns,\n",
        "                                               num_epochs=epochs,\n",
        "                                               shuffle=shuffle,\n",
        "                                               shuffle_buffer_size=buffer_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wcqjyATLzIgn",
        "colab_type": "code",
        "outputId": "8b8d1bdb-a838-42d5-ffc7-53eb6dc460cb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "source": [
        "ds = create_csv_dataset_hl(train_ratings_path, 'Rating', epochs=-1, select_columns=[0,1,2], shuffle=False)\n",
        "\n",
        "for data, label in ds.take(1):\n",
        "    print(label)\n",
        "    print(data)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /tensorflow-2.0.0/python3.6/tensorflow_core/python/data/experimental/ops/readers.py:521: parallel_interleave (from tensorflow.python.data.experimental.ops.interleave_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.data.Dataset.interleave(map_func, cycle_length, block_length, num_parallel_calls=tf.data.experimental.AUTOTUNE)` instead. If sloppy execution is desired, use `tf.data.Options.experimental_determinstic`.\n",
            "tf.Tensor([4 1 4 5 1 5 3 4], shape=(8,), dtype=int32)\n",
            "OrderedDict([('UserID', <tf.Tensor: id=51, shape=(8,), dtype=int32, numpy=array([1921, 4917,  956, 4652, 3244, 5090,  703, 3939], dtype=int32)>), ('MovieID', <tf.Tensor: id=50, shape=(8,), dtype=int32, numpy=array([2093, 2807, 1659,  913, 3323,  587,  968, 3782], dtype=int32)>)])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "LqSYCkh1hr72"
      },
      "source": [
        "### Define the feature columns"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fuyqqfl1NzXt",
        "colab_type": "code",
        "outputId": "3c093213-5ac6-49dd-c78d-e5ad5c1452e7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "print(df_ratings['UserID'].max(), df_ratings['MovieID'].max())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "6039 3951\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SYvcZAhPXV-e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "feature_columns = []\n",
        "embeddeding_size = 50\n",
        "\n",
        "# user_id = tf.feature_column.categorical_column_with_hash_bucket('UserID', hash_bucket_size=1000, dtype=tf.dtypes.int32)\n",
        "# movie_id = tf.feature_column.categorical_column_with_hash_bucket('MovieID', hash_bucket_size=1000, dtype=tf.dtypes.int32)\n",
        "user_id = tf.feature_column.categorical_column_with_identity('UserID', num_buckets=6040)\n",
        "movie_id = tf.feature_column.categorical_column_with_identity('MovieID', num_buckets=3952)\n",
        "\n",
        "feature_columns.append(tf.feature_column.embedding_column(user_id, embeddeding_size))\n",
        "feature_columns.append(tf.feature_column.embedding_column(movie_id, embeddeding_size))\n",
        "\n",
        "bias_columns = []\n",
        "bias_columns.append(tf.feature_column.embedding_column(user_id, 1))\n",
        "bias_columns.append(tf.feature_column.embedding_column(movie_id, 1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "24pk0Ivp8Rcm",
        "colab_type": "text"
      },
      "source": [
        "## Modelling"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N4LnKEEAgx3e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class RecommenderModel(tf.keras.Model):\n",
        "  def __init__(self, feature_columns, bias_columns, use_bias=True, sigmoid_activation=True):\n",
        "    super(RecommenderModel, self).__init__()\n",
        "    self.feature_embedding_layer = tf.keras.layers.DenseFeatures(feature_columns)\n",
        "    self.bias_embedding_layer = tf.keras.layers.DenseFeatures(bias_columns)\n",
        "    self.use_bias = use_bias\n",
        "    self.sigmoid_activation = sigmoid_activation\n",
        "\n",
        "  @tf.function\n",
        "  def call(self, x):\n",
        "    feature_tensor = self.feature_embedding_layer(x)\n",
        "    bias_tensor = self.bias_embedding_layer(x)\n",
        " \n",
        "    # | Split into two equal parts, as hash_bucket_size of both feature_columns is equal\n",
        "    user_id_tensor, movie_id_tensor = tf.split(feature_tensor, num_or_size_splits=2, axis=1)\n",
        "    user_id_bias, movie_id_bias = tf.split(bias_tensor, num_or_size_splits=2, axis=1)\n",
        "    \n",
        "    # | Dot product of user vector with movie vector -> the more correlation between these vectors, the higher the ranking should be\n",
        "    predicted_ranking = tf.reduce_sum(tf.multiply(user_id_tensor, movie_id_tensor), axis=1)\n",
        "\n",
        "    # | Add biases\n",
        "    if self.use_bias:\n",
        "        predicted_ranking = tf.add(predicted_ranking, user_id_bias)\n",
        "        predicted_ranking = tf.add(predicted_ranking, movie_id_bias)\n",
        "\n",
        "    # | Add Sigmoid activation function (Ratings are between 1 and 5 --> design sigmoid accordingly)\n",
        "    if self.sigmoid_activation:\n",
        "        predicted_ranking = 5.5*tf.math.sigmoid(tf.add(1.6*predicted_ranking, -5))\n",
        "\n",
        "    return predicted_ranking\n",
        "   \n",
        "   \n",
        "def create_model(lr=3e-4, use_bias=False, sigmoid_activation=False):\n",
        "    model = RecommenderModel(feature_columns=feature_columns, bias_columns=bias_columns, use_bias=use_bias, sigmoid_activation=sigmoid_activation)\n",
        "\n",
        "    model.compile(optimizer=tf.keras.optimizers.Adam(lr=lr),\n",
        "                loss=tf.keras.losses.MeanSquaredError(),\n",
        "                metrics=[tf.metrics.MeanAbsoluteError(), tf.metrics.MeanSquaredError(), tf.metrics.RootMeanSquaredError()])\n",
        "    \n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YfZ-XkIB7ay-",
        "colab_type": "text"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yRtyKHFCOtJn",
        "colab_type": "code",
        "outputId": "d03b0e84-07f3-478c-ae12-a047e7609a71",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        }
      },
      "source": [
        "batch_size = 64\n",
        "\n",
        "train_dataset = create_csv_dataset_hl(train_ratings_path, 'Rating', batch_size = batch_size, epochs=-1, select_columns=[0,1,2], shuffle=True)\n",
        "val_dataset = create_csv_dataset_hl(val_ratings_path, 'Rating', batch_size = batch_size, epochs=-1, select_columns=[0,1,2])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /tensorflow-2.0.0/python3.6/tensorflow_core/python/data/experimental/ops/readers.py:215: shuffle_and_repeat (from tensorflow.python.data.experimental.ops.shuffle_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.data.Dataset.shuffle(buffer_size, seed)` followed by `tf.data.Dataset.repeat(count)`. Static tf.data optimizations will take care of using the fused implementation.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rk720A5H7OcK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.callbacks import *\n",
        "from tensorflow.keras import backend as K\n",
        "\n",
        "class MyCyclicLR(Callback):\n",
        "    \"\"\"\n",
        "    Code from https://github.com/bckenstler/CLR.git\n",
        "    Just needed to replace the imports at the beginning.. to make it compatible with TF 2.0\n",
        "    \"\"\"\n",
        "    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n",
        "                 gamma=1., scale_fn=None, scale_mode='cycle'):\n",
        "        super(MyCyclicLR, self).__init__()\n",
        "\n",
        "        self.base_lr = base_lr\n",
        "        self.max_lr = max_lr\n",
        "        self.step_size = step_size\n",
        "        self.mode = mode\n",
        "        self.gamma = gamma\n",
        "        if scale_fn == None:\n",
        "            if self.mode == 'triangular':\n",
        "                self.scale_fn = lambda x: 1.\n",
        "                self.scale_mode = 'cycle'\n",
        "            elif self.mode == 'triangular2':\n",
        "                self.scale_fn = lambda x: 1/(2.**(x-1))\n",
        "                self.scale_mode = 'cycle'\n",
        "            elif self.mode == 'exp_range':\n",
        "                self.scale_fn = lambda x: gamma**(x)\n",
        "                self.scale_mode = 'iterations'\n",
        "        else:\n",
        "            self.scale_fn = scale_fn\n",
        "            self.scale_mode = scale_mode\n",
        "        self.clr_iterations = 0.\n",
        "        self.trn_iterations = 0.\n",
        "        self.history = {}\n",
        "\n",
        "        self._reset()\n",
        "\n",
        "    def _reset(self, new_base_lr=None, new_max_lr=None,\n",
        "               new_step_size=None):\n",
        "        \"\"\"Resets cycle iterations.\n",
        "        Optional boundary/step size adjustment.\n",
        "        \"\"\"\n",
        "        if new_base_lr != None:\n",
        "            self.base_lr = new_base_lr\n",
        "        if new_max_lr != None:\n",
        "            self.max_lr = new_max_lr\n",
        "        if new_step_size != None:\n",
        "            self.step_size = new_step_size\n",
        "        self.clr_iterations = 0.\n",
        "        \n",
        "    def clr(self):\n",
        "        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n",
        "        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n",
        "        if self.scale_mode == 'cycle':\n",
        "            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n",
        "        else:\n",
        "            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n",
        "        \n",
        "    def on_train_begin(self, logs={}):\n",
        "        logs = logs or {}\n",
        "\n",
        "        if self.clr_iterations == 0:\n",
        "            K.set_value(self.model.optimizer.lr, self.base_lr)\n",
        "        else:\n",
        "            K.set_value(self.model.optimizer.lr, self.clr())        \n",
        "            \n",
        "    def on_batch_end(self, epoch, logs=None):\n",
        "        \n",
        "        logs = logs or {}\n",
        "        self.trn_iterations += 1\n",
        "        self.clr_iterations += 1\n",
        "\n",
        "        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n",
        "        self.history.setdefault('iterations', []).append(self.trn_iterations)\n",
        "\n",
        "        for k, v in logs.items():\n",
        "            self.history.setdefault(k, []).append(v)\n",
        "        \n",
        "        K.set_value(self.model.optimizer.lr, self.clr())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Je0Y_xCb2Two",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train(model, train_dataset, train_size, val_dataset=None, val_size=None, nr_epochs=10, batch_size=32, one_cycle=False, max_lr=5e-3):\n",
        "    steps_per_epoch = train_size // batch_size\n",
        "    validation_steps = val_size // batch_size\n",
        "    callbacks = []\n",
        "\n",
        "    from datetime import datetime\n",
        "    ts = str(int(datetime.now().timestamp()))\n",
        "\n",
        "    # | Tensorboard\n",
        "    log_dir = './log/' + ts\n",
        "    if not os.path.exists(log_dir):\n",
        "        os.makedirs(log_dir)\n",
        "    tb_callback = keras.callbacks.TensorBoard(log_dir=log_dir, update_freq='epoch')\n",
        "    callbacks.append(tb_callback)\n",
        "\n",
        "    # | Checkpoints\n",
        "    checkpoint_dir = './checkpoints/' + ts\n",
        "    if not os.path.exists(checkpoint_dir):\n",
        "        os.makedirs(checkpoint_dir)\n",
        "    checkpoint_callback = keras.callbacks.ModelCheckpoint(checkpoint_dir + '/weights.{epoch:02d}-{val_loss:.2f}.h5',\n",
        "                                                        save_weights_only=True,\n",
        "                                                        save_freq='epoch',\n",
        "                                                        monitor='val_loss',\n",
        "                                                        save_best_only=True)\n",
        "\n",
        "    if one_cycle:\n",
        "        # from clr_callback import CyclicLR\n",
        "        max_lr = 5e-3\n",
        "        step_size = (nr_epochs*steps_per_epoch)/2\n",
        "        callbacks.append(MyCyclicLR(base_lr=max_lr/25, max_lr=max_lr, mode='triangular', step_size=step_size))\n",
        "\n",
        "    else:\n",
        "        # | Early Stopping\n",
        "        earlystop_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=4, verbose=1, mode='auto', restore_best_weights=True)\n",
        "        callbacks.append(earlystop_callback)\n",
        "\n",
        "        # | Learning Rate Reducer\n",
        "        lr_reducer = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.9, patience=2, verbose=1)\n",
        "        callbacks.append(lr_reducer)\n",
        "        callbacks.append(checkpoint_callback)\n",
        "\n",
        "    # | Training\n",
        "    history = model.fit(train_dataset,\n",
        "                        epochs=nr_epochs,\n",
        "                        steps_per_epoch = steps_per_epoch,\n",
        "                        validation_data=val_dataset, \n",
        "                        validation_steps=validation_steps,\n",
        "                        callbacks=callbacks)\n",
        "    \n",
        "    return model, history"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OMsKSzx3O0Ao",
        "colab_type": "text"
      },
      "source": [
        "#### No Bias, No Sigmoid"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tjvCdbPMOzVC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = create_model(lr=5e-3)\n",
        "train(model, train_dataset, train_size, val_dataset, val_size, batch_size=batch_size, nr_epochs=10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RACaHljnO7xt",
        "colab_type": "text"
      },
      "source": [
        "#### With Bias, No Sigmoid"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yS5y3uyxO-qT",
        "colab_type": "code",
        "outputId": "13e31695-5132-4ddb-aa06-9c5d018a51c1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 339
        }
      },
      "source": [
        "model = create_model(lr=5e-3, use_bias=True, sigmoid_activation=False)\n",
        "train(model, train_dataset, train_size, val_dataset, val_size, batch_size=batch_size, nr_epochs=100)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train for 12502 steps, validate for 3125 steps\n",
            "Epoch 1/100\n",
            "12502/12502 [==============================] - 133s 11ms/step - loss: 2.0240 - mean_absolute_error: 1.0345 - mean_squared_error: 2.0240 - root_mean_squared_error: 1.4227 - val_loss: 0.9678 - val_mean_absolute_error: 0.7793 - val_mean_squared_error: 0.9678 - val_root_mean_squared_error: 0.9838\n",
            "Epoch 2/100\n",
            "12502/12502 [==============================] - 125s 10ms/step - loss: 0.9458 - mean_absolute_error: 0.7688 - mean_squared_error: 0.9458 - root_mean_squared_error: 0.9725 - val_loss: 0.9407 - val_mean_absolute_error: 0.7656 - val_mean_squared_error: 0.9407 - val_root_mean_squared_error: 0.9699\n",
            "Epoch 3/100\n",
            "12502/12502 [==============================] - 130s 10ms/step - loss: 0.9284 - mean_absolute_error: 0.7612 - mean_squared_error: 0.9284 - root_mean_squared_error: 0.9635 - val_loss: 0.9305 - val_mean_absolute_error: 0.7624 - val_mean_squared_error: 0.9305 - val_root_mean_squared_error: 0.9646\n",
            "Epoch 4/100\n",
            "12502/12502 [==============================] - 127s 10ms/step - loss: 0.9190 - mean_absolute_error: 0.7570 - mean_squared_error: 0.9190 - root_mean_squared_error: 0.9587 - val_loss: 0.9195 - val_mean_absolute_error: 0.7576 - val_mean_squared_error: 0.9195 - val_root_mean_squared_error: 0.9589\n",
            "Epoch 5/100\n",
            "12502/12502 [==============================] - 125s 10ms/step - loss: 0.9094 - mean_absolute_error: 0.7528 - mean_squared_error: 0.9094 - root_mean_squared_error: 0.9536 - val_loss: 0.9125 - val_mean_absolute_error: 0.7548 - val_mean_squared_error: 0.9125 - val_root_mean_squared_error: 0.9552\n",
            "Epoch 6/100\n",
            "12502/12502 [==============================] - 127s 10ms/step - loss: 0.9026 - mean_absolute_error: 0.7500 - mean_squared_error: 0.9026 - root_mean_squared_error: 0.9501 - val_loss: 0.9069 - val_mean_absolute_error: 0.7522 - val_mean_squared_error: 0.9069 - val_root_mean_squared_error: 0.9523\n",
            "Epoch 7/100\n",
            "12502/12502 [==============================] - 128s 10ms/step - loss: 0.8970 - mean_absolute_error: 0.7475 - mean_squared_error: 0.8970 - root_mean_squared_error: 0.9471 - val_loss: 0.9024 - val_mean_absolute_error: 0.7501 - val_mean_squared_error: 0.9024 - val_root_mean_squared_error: 0.9499\n",
            "Epoch 8/100\n",
            "12265/12502 [============================>.] - ETA: 2s - loss: 0.8926 - mean_absolute_error: 0.7456 - mean_squared_error: 0.8926 - root_mean_squared_error: 0.9448"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VC2HOb5LPMXX",
        "colab_type": "text"
      },
      "source": [
        "#### With Bias, With Sigmoid"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fDteCcfnPLpF",
        "colab_type": "code",
        "outputId": "bf42e31c-5a7a-4a82-b391-f1cbdff34de9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "model = create_model(lr=5e-3, use_bias=True, sigmoid_activation=True)\n",
        "train(model, train_dataset, train_size, val_dataset, val_size, batch_size=batch_size, nr_epochs=100)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train for 12502 steps, validate for 3125 steps\n",
            "Epoch 1/100\n",
            "12502/12502 [==============================] - 41s 3ms/step - loss: 1.6283 - mean_absolute_error: 0.9688 - root_mean_squared_error: 1.2761 - val_loss: 1.1294 - val_mean_absolute_error: 0.8468 - val_root_mean_squared_error: 1.0627\n",
            "Epoch 2/100\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.1201 - mean_absolute_error: 0.8472 - root_mean_squared_error: 1.0583 - val_loss: 1.1220 - val_mean_absolute_error: 0.8442 - val_root_mean_squared_error: 1.0593\n",
            "Epoch 3/100\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.1121 - mean_absolute_error: 0.8443 - root_mean_squared_error: 1.0545 - val_loss: 1.1161 - val_mean_absolute_error: 0.8474 - val_root_mean_squared_error: 1.0564\n",
            "Epoch 4/100\n",
            "12502/12502 [==============================] - 41s 3ms/step - loss: 1.1080 - mean_absolute_error: 0.8430 - root_mean_squared_error: 1.0526 - val_loss: 1.1113 - val_mean_absolute_error: 0.8455 - val_root_mean_squared_error: 1.0542\n",
            "Epoch 5/100\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.1048 - mean_absolute_error: 0.8419 - root_mean_squared_error: 1.0511 - val_loss: 1.1074 - val_mean_absolute_error: 0.8444 - val_root_mean_squared_error: 1.0523\n",
            "Epoch 6/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.1015 - mean_absolute_error: 0.8406 - root_mean_squared_error: 1.0495 - val_loss: 1.1088 - val_mean_absolute_error: 0.8439 - val_root_mean_squared_error: 1.0530\n",
            "Epoch 7/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.1010 - mean_absolute_error: 0.8405 - root_mean_squared_error: 1.0493 - val_loss: 1.1037 - val_mean_absolute_error: 0.8418 - val_root_mean_squared_error: 1.0506\n",
            "Epoch 8/100\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.1005 - mean_absolute_error: 0.8403 - root_mean_squared_error: 1.0491 - val_loss: 1.1070 - val_mean_absolute_error: 0.8443 - val_root_mean_squared_error: 1.0522\n",
            "Epoch 9/100\n",
            "12490/12502 [============================>.] - ETA: 0s - loss: 1.1000 - mean_absolute_error: 0.8402 - root_mean_squared_error: 1.0488\n",
            "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0044999998994171625.\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.1000 - mean_absolute_error: 0.8402 - root_mean_squared_error: 1.0488 - val_loss: 1.1084 - val_mean_absolute_error: 0.8464 - val_root_mean_squared_error: 1.0528\n",
            "Epoch 10/100\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.0941 - mean_absolute_error: 0.8381 - root_mean_squared_error: 1.0460 - val_loss: 1.0989 - val_mean_absolute_error: 0.8422 - val_root_mean_squared_error: 1.0483\n",
            "Epoch 11/100\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.0933 - mean_absolute_error: 0.8378 - root_mean_squared_error: 1.0456 - val_loss: 1.0971 - val_mean_absolute_error: 0.8401 - val_root_mean_squared_error: 1.0474\n",
            "Epoch 12/100\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.0926 - mean_absolute_error: 0.8376 - root_mean_squared_error: 1.0453 - val_loss: 1.0951 - val_mean_absolute_error: 0.8377 - val_root_mean_squared_error: 1.0465\n",
            "Epoch 13/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0913 - mean_absolute_error: 0.8372 - root_mean_squared_error: 1.0447 - val_loss: 1.0972 - val_mean_absolute_error: 0.8399 - val_root_mean_squared_error: 1.0475\n",
            "Epoch 14/100\n",
            "12501/12502 [============================>.] - ETA: 0s - loss: 1.0901 - mean_absolute_error: 0.8367 - root_mean_squared_error: 1.0441\n",
            "Epoch 00014: ReduceLROnPlateau reducing learning rate to 0.004049999825656414.\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.0901 - mean_absolute_error: 0.8367 - root_mean_squared_error: 1.0441 - val_loss: 1.0981 - val_mean_absolute_error: 0.8405 - val_root_mean_squared_error: 1.0479\n",
            "Epoch 15/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0855 - mean_absolute_error: 0.8352 - root_mean_squared_error: 1.0419 - val_loss: 1.0895 - val_mean_absolute_error: 0.8374 - val_root_mean_squared_error: 1.0438\n",
            "Epoch 16/100\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.0852 - mean_absolute_error: 0.8350 - root_mean_squared_error: 1.0417 - val_loss: 1.0903 - val_mean_absolute_error: 0.8361 - val_root_mean_squared_error: 1.0442\n",
            "Epoch 17/100\n",
            "12498/12502 [============================>.] - ETA: 0s - loss: 1.0852 - mean_absolute_error: 0.8350 - root_mean_squared_error: 1.0417\n",
            "Epoch 00017: ReduceLROnPlateau reducing learning rate to 0.0036449996754527093.\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.0852 - mean_absolute_error: 0.8350 - root_mean_squared_error: 1.0417 - val_loss: 1.0903 - val_mean_absolute_error: 0.8360 - val_root_mean_squared_error: 1.0442\n",
            "Epoch 18/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0817 - mean_absolute_error: 0.8339 - root_mean_squared_error: 1.0401 - val_loss: 1.0876 - val_mean_absolute_error: 0.8357 - val_root_mean_squared_error: 1.0429\n",
            "Epoch 19/100\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.0814 - mean_absolute_error: 0.8338 - root_mean_squared_error: 1.0399 - val_loss: 1.0872 - val_mean_absolute_error: 0.8369 - val_root_mean_squared_error: 1.0427\n",
            "Epoch 20/100\n",
            "12502/12502 [==============================] - 38s 3ms/step - loss: 1.0813 - mean_absolute_error: 0.8338 - root_mean_squared_error: 1.0399 - val_loss: 1.0865 - val_mean_absolute_error: 0.8371 - val_root_mean_squared_error: 1.0424\n",
            "Epoch 21/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0811 - mean_absolute_error: 0.8337 - root_mean_squared_error: 1.0397 - val_loss: 1.0868 - val_mean_absolute_error: 0.8341 - val_root_mean_squared_error: 1.0425\n",
            "Epoch 22/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0810 - mean_absolute_error: 0.8337 - root_mean_squared_error: 1.0397 - val_loss: 1.0855 - val_mean_absolute_error: 0.8360 - val_root_mean_squared_error: 1.0419\n",
            "Epoch 23/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0808 - mean_absolute_error: 0.8336 - root_mean_squared_error: 1.0396 - val_loss: 1.0870 - val_mean_absolute_error: 0.8386 - val_root_mean_squared_error: 1.0426\n",
            "Epoch 24/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0804 - mean_absolute_error: 0.8334 - root_mean_squared_error: 1.0394 - val_loss: 1.0850 - val_mean_absolute_error: 0.8365 - val_root_mean_squared_error: 1.0416\n",
            "Epoch 25/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0800 - mean_absolute_error: 0.8334 - root_mean_squared_error: 1.0392 - val_loss: 1.0858 - val_mean_absolute_error: 0.8374 - val_root_mean_squared_error: 1.0420\n",
            "Epoch 26/100\n",
            "12498/12502 [============================>.] - ETA: 0s - loss: 1.0794 - mean_absolute_error: 0.8332 - root_mean_squared_error: 1.0390\n",
            "Epoch 00026: ReduceLROnPlateau reducing learning rate to 0.0032804996240884065.\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0795 - mean_absolute_error: 0.8332 - root_mean_squared_error: 1.0390 - val_loss: 1.0850 - val_mean_absolute_error: 0.8354 - val_root_mean_squared_error: 1.0416\n",
            "Epoch 27/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0770 - mean_absolute_error: 0.8323 - root_mean_squared_error: 1.0378 - val_loss: 1.0820 - val_mean_absolute_error: 0.8341 - val_root_mean_squared_error: 1.0402\n",
            "Epoch 28/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0767 - mean_absolute_error: 0.8322 - root_mean_squared_error: 1.0376 - val_loss: 1.0819 - val_mean_absolute_error: 0.8336 - val_root_mean_squared_error: 1.0401\n",
            "Epoch 29/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0769 - mean_absolute_error: 0.8323 - root_mean_squared_error: 1.0377 - val_loss: 1.0814 - val_mean_absolute_error: 0.8346 - val_root_mean_squared_error: 1.0399\n",
            "Epoch 30/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0768 - mean_absolute_error: 0.8322 - root_mean_squared_error: 1.0377 - val_loss: 1.0831 - val_mean_absolute_error: 0.8338 - val_root_mean_squared_error: 1.0407\n",
            "Epoch 31/100\n",
            "12484/12502 [============================>.] - ETA: 0s - loss: 1.0767 - mean_absolute_error: 0.8322 - root_mean_squared_error: 1.0377\n",
            "Epoch 00031: ReduceLROnPlateau reducing learning rate to 0.0029524497454985975.\n",
            "12502/12502 [==============================] - 41s 3ms/step - loss: 1.0768 - mean_absolute_error: 0.8323 - root_mean_squared_error: 1.0377 - val_loss: 1.0829 - val_mean_absolute_error: 0.8361 - val_root_mean_squared_error: 1.0406\n",
            "Epoch 32/100\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.0745 - mean_absolute_error: 0.8315 - root_mean_squared_error: 1.0366 - val_loss: 1.0800 - val_mean_absolute_error: 0.8328 - val_root_mean_squared_error: 1.0392\n",
            "Epoch 33/100\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.0744 - mean_absolute_error: 0.8314 - root_mean_squared_error: 1.0365 - val_loss: 1.0806 - val_mean_absolute_error: 0.8333 - val_root_mean_squared_error: 1.0395\n",
            "Epoch 34/100\n",
            "12499/12502 [============================>.] - ETA: 0s - loss: 1.0743 - mean_absolute_error: 0.8315 - root_mean_squared_error: 1.0365\n",
            "Epoch 00034: ReduceLROnPlateau reducing learning rate to 0.00265720474999398.\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.0743 - mean_absolute_error: 0.8315 - root_mean_squared_error: 1.0365 - val_loss: 1.0799 - val_mean_absolute_error: 0.8356 - val_root_mean_squared_error: 1.0392\n",
            "Epoch 35/100\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.0725 - mean_absolute_error: 0.8308 - root_mean_squared_error: 1.0356 - val_loss: 1.0775 - val_mean_absolute_error: 0.8321 - val_root_mean_squared_error: 1.0380\n",
            "Epoch 36/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0722 - mean_absolute_error: 0.8307 - root_mean_squared_error: 1.0355 - val_loss: 1.0774 - val_mean_absolute_error: 0.8330 - val_root_mean_squared_error: 1.0380\n",
            "Epoch 37/100\n",
            "12485/12502 [============================>.] - ETA: 0s - loss: 1.0721 - mean_absolute_error: 0.8307 - root_mean_squared_error: 1.0354\n",
            "Epoch 00037: ReduceLROnPlateau reducing learning rate to 0.002391484379768372.\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.0723 - mean_absolute_error: 0.8307 - root_mean_squared_error: 1.0355 - val_loss: 1.0777 - val_mean_absolute_error: 0.8332 - val_root_mean_squared_error: 1.0381\n",
            "Epoch 38/100\n",
            "12502/12502 [==============================] - 38s 3ms/step - loss: 1.0705 - mean_absolute_error: 0.8302 - root_mean_squared_error: 1.0347 - val_loss: 1.0765 - val_mean_absolute_error: 0.8311 - val_root_mean_squared_error: 1.0376\n",
            "Epoch 39/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0703 - mean_absolute_error: 0.8300 - root_mean_squared_error: 1.0345 - val_loss: 1.0761 - val_mean_absolute_error: 0.8327 - val_root_mean_squared_error: 1.0374\n",
            "Epoch 40/100\n",
            "12502/12502 [==============================] - 38s 3ms/step - loss: 1.0705 - mean_absolute_error: 0.8302 - root_mean_squared_error: 1.0346 - val_loss: 1.0759 - val_mean_absolute_error: 0.8325 - val_root_mean_squared_error: 1.0373\n",
            "Epoch 41/100\n",
            "12502/12502 [==============================] - 38s 3ms/step - loss: 1.0703 - mean_absolute_error: 0.8301 - root_mean_squared_error: 1.0346 - val_loss: 1.0765 - val_mean_absolute_error: 0.8313 - val_root_mean_squared_error: 1.0376\n",
            "Epoch 42/100\n",
            "12498/12502 [============================>.] - ETA: 0s - loss: 1.0702 - mean_absolute_error: 0.8301 - root_mean_squared_error: 1.0345\n",
            "Epoch 00042: ReduceLROnPlateau reducing learning rate to 0.0021523358998820187.\n",
            "12502/12502 [==============================] - 38s 3ms/step - loss: 1.0703 - mean_absolute_error: 0.8301 - root_mean_squared_error: 1.0345 - val_loss: 1.0763 - val_mean_absolute_error: 0.8327 - val_root_mean_squared_error: 1.0374\n",
            "Epoch 43/100\n",
            "12502/12502 [==============================] - 38s 3ms/step - loss: 1.0689 - mean_absolute_error: 0.8296 - root_mean_squared_error: 1.0339 - val_loss: 1.0743 - val_mean_absolute_error: 0.8313 - val_root_mean_squared_error: 1.0365\n",
            "Epoch 44/100\n",
            "12502/12502 [==============================] - 38s 3ms/step - loss: 1.0686 - mean_absolute_error: 0.8295 - root_mean_squared_error: 1.0337 - val_loss: 1.0748 - val_mean_absolute_error: 0.8317 - val_root_mean_squared_error: 1.0367\n",
            "Epoch 45/100\n",
            "12486/12502 [============================>.] - ETA: 0s - loss: 1.0688 - mean_absolute_error: 0.8296 - root_mean_squared_error: 1.0338\n",
            "Epoch 00045: ReduceLROnPlateau reducing learning rate to 0.0019371022470295429.\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0688 - mean_absolute_error: 0.8296 - root_mean_squared_error: 1.0338 - val_loss: 1.0750 - val_mean_absolute_error: 0.8329 - val_root_mean_squared_error: 1.0368\n",
            "Epoch 46/100\n",
            "12502/12502 [==============================] - 41s 3ms/step - loss: 1.0676 - mean_absolute_error: 0.8292 - root_mean_squared_error: 1.0333 - val_loss: 1.0731 - val_mean_absolute_error: 0.8316 - val_root_mean_squared_error: 1.0359\n",
            "Epoch 47/100\n",
            "12502/12502 [==============================] - 38s 3ms/step - loss: 1.0671 - mean_absolute_error: 0.8290 - root_mean_squared_error: 1.0330 - val_loss: 1.0733 - val_mean_absolute_error: 0.8322 - val_root_mean_squared_error: 1.0360\n",
            "Epoch 48/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0673 - mean_absolute_error: 0.8291 - root_mean_squared_error: 1.0331 - val_loss: 1.0730 - val_mean_absolute_error: 0.8316 - val_root_mean_squared_error: 1.0359\n",
            "Epoch 49/100\n",
            "12502/12502 [==============================] - 39s 3ms/step - loss: 1.0673 - mean_absolute_error: 0.8290 - root_mean_squared_error: 1.0331 - val_loss: 1.0730 - val_mean_absolute_error: 0.8307 - val_root_mean_squared_error: 1.0358\n",
            "Epoch 50/100\n",
            "12501/12502 [============================>.] - ETA: 0s - loss: 1.0674 - mean_absolute_error: 0.8292 - root_mean_squared_error: 1.0331\n",
            "Epoch 00050: ReduceLROnPlateau reducing learning rate to 0.0017433920642361046.\n",
            "12502/12502 [==============================] - 41s 3ms/step - loss: 1.0674 - mean_absolute_error: 0.8292 - root_mean_squared_error: 1.0331 - val_loss: 1.0734 - val_mean_absolute_error: 0.8313 - val_root_mean_squared_error: 1.0360\n",
            "Epoch 51/100\n",
            "12502/12502 [==============================] - 40s 3ms/step - loss: 1.0660 - mean_absolute_error: 0.8286 - root_mean_squared_error: 1.0324 - val_loss: 1.0717 - val_mean_absolute_error: 0.8319 - val_root_mean_squared_error: 1.0352\n",
            "Epoch 52/100\n",
            "12502/12502 [==============================] - 41s 3ms/step - loss: 1.0659 - mean_absolute_error: 0.8286 - root_mean_squared_error: 1.0324 - val_loss: 1.0718 - val_mean_absolute_error: 0.8315 - val_root_mean_squared_error: 1.0353\n",
            "Epoch 53/100\n",
            "12496/12502 [============================>.] - ETA: 0s - loss: 1.0660 - mean_absolute_error: 0.8287 - root_mean_squared_error: 1.0325\n",
            "Epoch 00053: ReduceLROnPlateau reducing learning rate to 0.001569052878767252.\n",
            "12502/12502 [==============================] - 41s 3ms/step - loss: 1.0660 - mean_absolute_error: 0.8287 - root_mean_squared_error: 1.0325 - val_loss: 1.0719 - val_mean_absolute_error: 0.8306 - val_root_mean_squared_error: 1.0353\n",
            "Epoch 54/100\n",
            "12502/12502 [==============================] - 41s 3ms/step - loss: 1.0648 - mean_absolute_error: 0.8282 - root_mean_squared_error: 1.0319 - val_loss: 1.0708 - val_mean_absolute_error: 0.8305 - val_root_mean_squared_error: 1.0348\n",
            "Epoch 55/100\n",
            "12502/12502 [==============================] - 50s 4ms/step - loss: 1.0648 - mean_absolute_error: 0.8282 - root_mean_squared_error: 1.0319 - val_loss: 1.0710 - val_mean_absolute_error: 0.8303 - val_root_mean_squared_error: 1.0349\n",
            "Epoch 56/100\n",
            "12493/12502 [============================>.] - ETA: 0s - loss: 1.0650 - mean_absolute_error: 0.8283 - root_mean_squared_error: 1.0320\n",
            "Epoch 00056: ReduceLROnPlateau reducing learning rate to 0.0014121476328000427.\n",
            "12502/12502 [==============================] - 46s 4ms/step - loss: 1.0649 - mean_absolute_error: 0.8283 - root_mean_squared_error: 1.0320 - val_loss: 1.0709 - val_mean_absolute_error: 0.8314 - val_root_mean_squared_error: 1.0348\n",
            "Epoch 57/100\n",
            "11178/12502 [=========================>....] - ETA: 4s - loss: 1.0641 - mean_absolute_error: 0.8281 - root_mean_squared_error: 1.0316"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_sWpG2LjYtkI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.save_weights('./final-model/checkpoint')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0gNliOgYYyCW",
        "colab_type": "code",
        "outputId": "4a056f04-16f2-41a1-8ca5-104f01d25808",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Create a new model instance\n",
        "model = create_model()\n",
        "\n",
        "# Restore the weights\n",
        "model.load_weights('./final-model/checkpoint')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7f406ee9eda0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a-SO0p9yxkLD",
        "colab_type": "text"
      },
      "source": [
        "### With 1-cycle policy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nYmMY-kVHglh",
        "colab_type": "text"
      },
      "source": [
        "... Now we finally get comparable results (mean squared error) to collab_learner() in fast.ai (see below)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_nFnTfZ4ycWw",
        "colab_type": "code",
        "outputId": "fe0c9773-4a7f-4d63-8879-dd482a971549",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 232
        }
      },
      "source": [
        "model = create_model(lr=5e-3, use_bias=True, sigmoid_activation=False)\n",
        "model, history = train(model, train_dataset, train_size, val_dataset, val_size, batch_size=batch_size, nr_epochs=5, one_cycle=True, max_lr=5e-3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train for 12502 steps, validate for 3125 steps\n",
            "Epoch 1/5\n",
            "12502/12502 [==============================] - 107s 9ms/step - loss: 6.7297 - mean_absolute_error: 2.0421 - mean_squared_error: 6.7297 - root_mean_squared_error: 2.5942 - val_loss: 1.2162 - val_mean_absolute_error: 0.8737 - val_mean_squared_error: 1.2162 - val_root_mean_squared_error: 1.1028\n",
            "Epoch 2/5\n",
            "12502/12502 [==============================] - 114s 9ms/step - loss: 0.9786 - mean_absolute_error: 0.7819 - mean_squared_error: 0.9786 - root_mean_squared_error: 0.9893 - val_loss: 0.9395 - val_mean_absolute_error: 0.7657 - val_mean_squared_error: 0.9395 - val_root_mean_squared_error: 0.9693\n",
            "Epoch 3/5\n",
            "12502/12502 [==============================] - 107s 9ms/step - loss: 0.9382 - mean_absolute_error: 0.7661 - mean_squared_error: 0.9382 - root_mean_squared_error: 0.9686 - val_loss: 0.9178 - val_mean_absolute_error: 0.7570 - val_mean_squared_error: 0.9178 - val_root_mean_squared_error: 0.9580\n",
            "Epoch 4/5\n",
            "12502/12502 [==============================] - 105s 8ms/step - loss: 0.8777 - mean_absolute_error: 0.7399 - mean_squared_error: 0.8777 - root_mean_squared_error: 0.9368 - val_loss: 0.8614 - val_mean_absolute_error: 0.7320 - val_mean_squared_error: 0.8614 - val_root_mean_squared_error: 0.9281\n",
            "Epoch 5/5\n",
            "12502/12502 [==============================] - 105s 8ms/step - loss: 0.8328 - mean_absolute_error: 0.7201 - mean_squared_error: 0.8328 - root_mean_squared_error: 0.9126 - val_loss: 0.8375 - val_mean_absolute_error: 0.7213 - val_mean_squared_error: 0.8375 - val_root_mean_squared_error: 0.9152\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jOhveFIu2Ght",
        "colab_type": "code",
        "outputId": "a59f8660-c230-4780-ffcb-aee6bdb12179",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.plot(history.history['lr'])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fde3cb7a6d8>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU9bnH8c9DEsIeIIQ1gYTdsIUk\nIi51wQVwIWhRQei1vdxaEeqCVUGvtmLdq7gB1lustCAQcYtWQRSsa4EkhCVAIAJCWEPYt0DCc/+Y\nQ5vEQAZI5szyvF+vvJg553d+85yjM9/knJlnRFUxxhhjTqrldgHGGGP8iwWDMcaYciwYjDHGlGPB\nYIwxphwLBmOMMeWEu11AdWjWrJnGx8e7XYYxxgSUrKysXaoaU3F5UARDfHw8mZmZbpdhjDEBRUR+\nrGy5nUoyxhhTjgWDMcaYciwYjDHGlGPBYIwxphwLBmOMMeVYMBhjjCnHgsEYY0w5Xn2OQUQGAC8D\nYcBfVPWZCusjgb8BKUARcKuqbnTWjQdGAqXA3ao6r8x2YUAmsEVVr3eWJQCzgGggC/iFqh47h300\n1aRgz2HezdpC6YkTbpcSMJo3qsOwPm0JqyVul2KM16oMBufFexJwNVAALBGRDFVdVWbYSGCPqnYU\nkaHAs8CtIpIIDAW6Aa2Bz0Wks6qWOtvdA6wGGpWZ61lgoqrOEpHXnbmnnNNemnN2sLiE/3pzMesL\nDyH2GueVk191sutgMfde1dndYow5A978xdAHyFfV9QAiMgtIA8oGQxrwB+f2HOA1ERFn+SxVLQY2\niEi+M9/3IhILXAc8CYx15hagH3CbM9c0Z14LBhepKg+9u5yNuw4x89d9ubBDtNslBQRV5f70Zbz8\nxTp6t23CZZ1/0nnAGL/kzTWGNsDmMvcLnGWVjlHVEmAfnlNBp9v2JeBBoOx5iWhgrzPHqR4LABG5\nQ0QyRSSzsLDQi90wZ+ut7zbyj+Xb+F3/LhYKZ0BEePLGHnRu3pB7Zy1ly94jbpdkjFdcufgsItcD\nO1U162znUNU3VDVVVVNjYuw3sZqS9eMenvzHaq46rzl3XtrB7XICTt3aYUwZkczxUmX0jGyOldj1\nGeP/vAmGLUBcmfuxzrJKx4hIOBCF5yL0qba9GBgkIhvxXGjuJyLTnW0aO3Oc6rGMjxQdLGbM29m0\nalyHF25OopZdQD0r7WMa8NyQnuRs3suT/1hV9QbGuMybYFgCdBKRBBGpjedickaFMRnA7c7tIcAC\nVVVn+VARiXTebdQJWKyq41U1VlXjnfkWqOoIZ5uFzhw4c354DvtnzlLpCeWeWTkUHTrGlOEpRNWL\ncLukgHZtj1aMvCSBad//SMayrW6XY8xpVRkMzvn+McA8PO8gSlfVXBGZICKDnGFTgWjn4vJYYJyz\nbS6QjudC9VxgdJl3JJ3KQ8BYZ65oZ27jYy9/vpZv8nfxRFo3ureJcrucoDBuYFdS2zVh3LvLWbfj\ngNvlGHNKoiffUxfAUlNT1b6PofosXLOTX721hJtTYnn+5l5ulxNUduw/ynWvfE1U3Qg+HHMJDSKD\n4itRTIASkSxVTa243D75bMrZvPsw987O4bxWjXhicHe3ywk6LRrV4ZVhvdmw6xDj3l1OMPxiZoKP\nBYP5t+KSUka/nc2JE8qU4cnUiQhzu6SgdFGHZtx/TRc+Xr6Nad9tdLscY37CgsH824SPVrG8YB9/\nuqUX8c3qu11OUBt1WQeu7NqcJz9ZTfamPW6XY0w5FgwGgPeXFjBj0SZ+c2l7+ndr6XY5Qa9WLeHF\nW5JoGVWH0TOyKTpY7HZJxvybBYMhb/sBxr+3gj4JTXmgfxe3ywkZUfUimDI8haJDx7h3dg6lJ+x6\ng/EPFgwh7sDR44yankWDyAheG9ab8DD7X8KXureJ4vFB3fh63S5e/mKd2+UYA1gwhLSTzfF+3H2Y\n127rTfNGddwuKSQNPT+OnyfH8uqCdXyZt9PtcoyxYAhlb367kU9WbOeB/l3o296a47lFRPjj4O50\nadGQe2fnULDnsNslmRBnwRCiMjfu5ulPVnN1Ygt+c2l7t8sJeZ5meymUOs32ikuqahBgTM2xYAhB\nuw4WM/rtbNo0qcufbu6F2Dfv+IWEZvV5/uaeLCvYxx8/Xu12OSaEWTCEGE9zvKXsPXycycOTiapr\nzfH8yYDurbjj0vb8/V8/8mGONRY27rBgCDET56/l2/winhjcnW6trTmeP3qwfxf6xDdl3LsrWGvN\n9owLLBhCyBerd/DawnxuTY3jltS4qjcwrggPq8Vrt/WmfmQ4d07P4mBxSdUbGVONLBhCxObdh7lv\ndg6JrRrxeFo3t8sxVWjeqA6vDuvNxl2HeGiONdszvmXBEAKOHi9l1IwsFHh9RIo1xwsQF3aI5oH+\nXfnHim389duNbpdjQogFQwh4/KNVrNyynxdvSaJtdD23yzFn4M7L2nPVeS146pPVZP242+1yTIiw\nYAhy72YVMHPxJu68rANXJ7ZwuxxzhkSEF27pRevGdRk9Yym7rNme8QGvgkFEBohInojki8i4StZH\nishsZ/0iEYkvs268szxPRPo7y+qIyGIRWSYiuSLyeJnxb4nIBhHJcX6Szn03Q9Oa7ft55IMV9G3f\nlN9d09ntcsxZiqobweThyew+fIx7Zi21ZnumxlUZDCISBkwCBgKJwDARSawwbCSwR1U7AhOBZ51t\nE4GhQDdgADDZma8Y6KeqvYAkYICI9C0z3wOqmuT85JzTHoao/UePM2p6No3qRPCKNccLeN3bRPFE\nWje+zS/ipc/Xul2OCXLevFr0AfJVdb2qHgNmAWkVxqQB05zbc4ArxfNx2jRglqoWq+oGIB/oox4H\nnfERzo/9GlRNVJUH31nOpt2Hee22ZJo3tOZ4weDW89tyc0osry7IZ+Eaa7Znao43wdAG2FzmfoGz\nrNIxqloC7AOiT7etiISJSA6wE5ivqovKjHtSRJaLyEQRiaysKBG5Q0QyRSSzsLDQi90IHVO/2cDc\n3O08NKALfRKaul2OqUZPDO7Oea0ace/sHDbvtmZ7pma4dn5BVUtVNQmIBfqIyMlvnh8PdAXOB5oC\nD51i+zdUNVVVU2NiYnxScyBYsnE3T3+6hv7dWvDrn1lzvGBTJyKMKcOTOXFCGf22NdszNcObYNgC\nlP2YbKyzrNIxIhIORAFF3myrqnuBhXiuQaCq25xTTcXAX/GcyjJeKDxQzOgZ2cQ1qcvz1hwvaMU3\nq88Lt/RiecE+Jny0yu1yTBDyJhiWAJ1EJEFEauO5mJxRYUwGcLtzewiwQD0f1cwAhjrvWkoAOgGL\nRSRGRBoDiEhd4GpgjXO/lfOvAIOBleeyg6GipPQEd89cyv6jx5kyIoVGdaw5XjC7pltLfnNZe2Ys\n2sR72QVul2OCTHhVA1S1RETGAPOAMOBNVc0VkQlApqpmAFOBv4tIPrAbT3jgjEsHVgElwGhVLXVe\n/Kc571CqBaSr6sfOQ84QkRhAgBzgzurc4WD1wvy1fL++iD/d3IvzWjVyuxzjAw9c04WcTXt5+P0V\nJLZuRNeW9t/dVA8Jhh4sqampmpmZ6XYZrpm/age//lsmw/rE8fRNPd0ux/jQzgNHue6Vb2gQGU7G\nmItpaH8pmjMgIlmqmlpxub25PcBtKjrM2PQcurdpxO9vsOZ4oaZ5wzq8Nqw3m3Yf5kFrtmeqiQVD\nADvZHE+AKcOtOV6ouqB9NA/278KnK7cz9ZsNbpdjgoAFQwD7Q0YuuVv3M/HWJOKaWnO8UHbHpe25\nJrEFz3y6hsyN1mzPnBsLhgD1TuZmZi3ZzF2Xd+DK86w5XqgTEZ6/uRdtmtRl9NvZ1mzPnBMLhgC0\naut+/veDlVzYPpqxV1tzPOMRVTeCKcNT2Hv4OHfPtGZ75uxZMASY/UePc9eMLKLqWnM881OJrRvx\nxODufPdDES/Oz3O7HBOg7FUlgKgqv0tfxuY9R5g0PJmYhpW2kTIh7pbUOG5NjWPSwh/4YvUOt8sx\nAciCIYD839fr+WzVDsYP7Mr58dYcz5za42ndSGzViPus2Z45CxYMAWLR+iKenZvHwO4tGXlJgtvl\nGD9XJyKM10ekADBqRhZHj1uzPeM9C4YAsPPAUcbMXEq7pvV4bkhPa45nvNI2uh4v3pLEyi37edya\n7ZkzYMHg50pKTzDm7aUcOHqcySOSreWBOSNXJbZg1OUdmLl4E3OyrNme8Y4Fg597/rM8Fm/YzVM3\n9rAmaeas3H91Zy5sH80j769g9bb9bpdjAoAFgx/7LHc7f/7nem67oC03Jce6XY4JUOFhtXhlWG+i\n6kYwanoW+48ed7sk4+csGPzUj0WHuP+dZfRoE8Vj1ye6XY4JcDENI3nttmQ27znCg+9Ysz1zehYM\nfujo8VLunJ5NLREmD0+25nimWvRJaMq4AV2Zm7udv3xtzfbMqVkw+KHHPlzJ6m37mXhrL2uOZ6rV\n//wsgQHdWvLM3DUs3mDN9kzlLBj8TPqSzaRnFjDmio7062rN8Uz1EhGeu7kncU3qMubtbHYeOOp2\nScYPeRUMIjJARPJEJF9ExlWyPlJEZjvrF4lIfJl1453leSLS31lWR0QWi8gyEckVkcfLjE9w5sh3\n5qx97rsZGHK37uPRD1dyccdo7rPmeKaGNKoTwZQRKew/6mm2V1J6wu2SjJ+pMhic72WeBAwEEoFh\nIlLxauhIYI+qdgQmAs862ybi+f7nbsAAYLIzXzHQT1V7AUnAABHp68z1LDDRmWuPM3fQ23fkOKOm\nZ9OkXm1eHtqbsFr2ITZTc85r1Yg/Du7Bv9bv5oX5a90ux/gZb/5i6APkq+p6VT0GzALSKoxJA6Y5\nt+cAV4rn47lpwCxVLVbVDUA+0Ec9DjrjI5wfdbbp58yBM+fgs9y3gKGq/O6dZWzde4RJw3vTrIE1\nxzM1b0hKLMP6xDHlyx+Yv8qa7Zn/8CYY2gCby9wvcJZVOkZVS4B9QPTpthWRMBHJAXYC81V1kbPN\nXmeOUz0WzvZ3iEimiGQWFhZ6sRv+689frWf+qh08fO15pLSz5njGd35/Qze6t2nE2PQcNhVZsz3j\n4drFZ1UtVdUkIBboIyLdz3D7N1Q1VVVTY2JiaqZIH/j+hyKem7uG63q24lcXx7tdjgkxdSLCmDI8\nhVoi1mzP/Js3wbAFiCtzP9ZZVukYEQkHooAib7ZV1b3AQjzXIIqAxs4cp3qsoLFz/1F+O3Mp8c3q\n8+zPrTmecUdc03pMvLUXuVv38/sPc90ux/gBb4JhCdDJebdQbTwXkzMqjMkAbnduDwEWqOejlRnA\nUOddSwlAJ2CxiMSISGMAEakLXA2scbZZ6MyBM+eHZ797/uu40xzvUHEJr49IoUFkeNUbGVND+nVt\nwegrOjA7czPpmZur3sAEtSqDwTnfPwaYB6wG0lU1V0QmiMggZ9hUIFpE8oGxwDhn21wgHVgFzAVG\nq2op0ApYKCLL8QTPfFX92JnrIWCsM1e0M3fQeX5eHos37ubpm3rQuUVDt8sxhrFXd+GiDtE8+sFK\ncrfuc7sc4yIJhp4pqampmpmZ6XYZXpu7cjt3Ts9iRN+2/HFwD7fLMebfdh0s5rpXvqZORBgZYy4h\nqq61eQ9mIpKlqqkVl9snn31sw65DPPDOMnrFRvGoNcczfqZZg0gm3ZbMlj1HeOCdZdZsL0RZMPjQ\nkWOljJqeRViYMGl4MpHh1hzP+J/U+KaMG9iVz1bt4I2v1rtdjnGBBYOPqCqPfriSvB0HmHhrErFN\nrDme8V8jL0ng2h4teW5eHovWF7ldjvExCwYfmb1kM3OyCvjtFR25oktzt8sx5rREhGd/3pN2Tesx\nZuZSdu63ZnuhxILBB1Zu2cdjGbn8rFMz7rnKmuOZwNCwTgSTRyRz4OhxxlizvZBiwVDD9h0+zqgZ\nWUTXr81LtyZZczwTULq2bMRTN/Zg8YbdPP9ZntvlGB+xYKhBJ04o97+Tw/Z9R5k0PJloa45nAtBN\nybEMv6Atf/7nej7L3e52OcYHLBhq0Otf/cDnq3fyyLXnkdy2idvlGHPWHrshkZ6xUdz/zjI27jrk\ndjmmhlkw1JDvftjFn+blcUOv1tx+Ubzb5RhzTiLDw5h0W7LTbC/bmu0FOQuGGrB931HunrmUhGb1\neeamHtYczwSFuKb1eOnWJFZv28+jH6x0uxxTgywYqpmnOV42h4+V8vqIFOpbczwTRK7o2pzf9uvI\nO1kFzF6yye1yTA2xYKhmz366hswf9/D0TT3oZM3xTBC696rOXNKxGY9+mMvKLdZsLxhZMFSjT1ds\n4y/fbOC/LmxHWlKlXzxnTMALqyW8PDSJpvVqc9eMbPYdOe52SaaaWTBUk/WFB3lgznJ6xTXmkevO\nc7scY2pUdINIJg1PZuveI9yfvowTJ6zZXjCxYKgGR46VcteMbCLChMnWHM+EiJR2TXj42vP4fPUO\n/mzN9oKKBcM5UlUe+WAFeTsO8NLQ3rRpXNftkozxmV9dHM91PVvx/Lw1fP+DNdsLFl4Fg4gMEJE8\nEckXkXGVrI8UkdnO+kUiEl9m3XhneZ6I9HeWxYnIQhFZJSK5InJPmfF/EJEtIpLj/Fx77rtZc2Yu\n3sx72Vu4u18nLusc43Y5xvjUyWZ78c3q81trthc0qgwGEQkDJgEDgURgmIhU/IaZkcAeVe0ITASe\ndbZNxPMd0d2AAcBkZ74S4H5VTQT6AqMrzDlRVZOcn0/OaQ9r0IqCffzBaY5395Wd3C7HGFc0iAzn\n9REpHCouYczbSzluzfYCnjd/MfQB8lV1vaoeA2YBaRXGpAHTnNtzgCvF86muNGCWqhar6gYgH+ij\nqttUNRtAVQ/g+S7pgHobz97Dxxg1I4tmDWrz8tDe1hzPhLTOLRry9E09WLxxN8/Ps2Z7gc6bYGgD\nbC5zv4Cfvoj/e4yqlgD7gGhvtnVOO/UGFpVZPEZElovImyLid02GTpxQxqYvY8f+o0wekULT+rXd\nLskY1w3u3YZf9G3HG1+tZ+7KbW6XY86BqxefRaQB8C5wr6rudxZPAToAScA24IVTbHuHiGSKSGZh\nYaFP6j1p8pf5LFizk0evTyQprrFPH9sYf/a/159Hr7jGPPDOcjZYs72A5U0wbAHiytyPdZZVOkZE\nwoEooOh024pIBJ5QmKGq750coKo7VLVUVU8A/4fnVNZPqOobqpqqqqkxMb676Ptt/i5enL+WQb1a\n84u+7Xz2uMYEAk+zvd6EhQmjpmdx5Jg12wtE3gTDEqCTiCSISG08F5MzKozJAG53bg8BFqiqOsuH\nOu9aSgA6AYud6w9TgdWq+mLZiUSkVZm7NwJ+063rZHO89jENeNqa4xlTqdgmnmZ7eTsO8L8frMTz\nUmACSZXB4FwzGAPMw3OROF1Vc0VkgogMcoZNBaJFJB8YC4xzts0F0oFVwFxgtKqWAhcDvwD6VfK2\n1OdEZIWILAeuAO6rrp09F8dLTzD67WyOHC/l9RHJ1hzPmNO4vEtzftuvE+9mFzBryeaqNzB+RYIh\nzVNTUzUzM7NGH2PCR6t489sNvDqsNzf0al2jj2VMMCg9ofzyr4tZtGE37426iO5totwuyVQgIlmq\nmlpxuX3y2Qv/WL6NN7/dwC8virdQMMZLnmZ7vYmuX5s7p2ex77A12wsUFgxV+KHwIA/OWUbvto15\n+FprjmfMmWhavzaThiezY/9RxqbnWLO9AGHBcBqHj5UwanoWkRGerzWsHW6Hy5gzldy2CY9cex5f\nrNnJlH/+4HY5xgv2SncKqsoj769k3c6DvDw0idbWHM+Ys3a7cxr2hc/y+O6HXW6XY6pgwXAKMxZt\n4v2lW7j3ys78rJM1xzPmXIgIz9zUg/YxDbh75lK277Nme/7MgqESywv2MuGjVVzeJYbf9uvodjnG\nBIX6keG8PiKZw8dKGfN2tjXb82MWDBXsOXSMUdOziWkYycRbkqhlzfGMqTYdmzfkmZ/3JPPHPTzz\n6Rq3yzGnYMFQxokTyn3pORQeKGby8GSaWHM8Y6rdoF6tuf3Cdkz9ZgOfrLBme/7IgqGM1xbm82Ve\nIY/ekEgva45nTI155DpPA8oH5yxnfeFBt8sxFVgwOL5eV8jEz9cyOKk1Iy5o63Y5xgS12uG1mDQ8\nmYgwYdT0bA4fK3G7JFOGBQOwde8R7pmVQ6fmDXjKmuMZ4xNtGtfl5aG9WbvzAP/7vjXb8ychHwzH\nSjzN8YqPlzJlRAr1altzPGN85dLOMdxzZSfeW7qFtxdvcrsc4wj5YHjqk9Us3bSX54b0okNMA7fL\nMSbk3N2vE5d2juHxjFUsL9jrdjmGEA+Gj5Zt5a3vNvKri+O5rmerqjcwxlS7WrWEl25NolmD2oya\nns3ew8fcLinkhXQwbNp9mNR2TRg/0JrjGeOmpvVrM3lECjsPHOW+2dZsz20hHQyjr+jIzDv6WnM8\nY/xAUlxjHr0+kYV5hUz+Mt/tckJayL8iRoSF/CEwxm/8om870pJa8+L8tXybb8323OLVq6KIDBCR\nPBHJF5FxlayPFJHZzvpFIhJfZt14Z3meiPR3lsWJyEIRWSUiuSJyT5nxTUVkvoisc/5tcu67aYwJ\nBCLC0zf1oIM123NVlcEgImHAJGAgkAgME5HECsNGAntUtSMwEXjW2TYRGAp0AwYAk535SoD7VTUR\n6AuMLjPnOOALVe0EfOHcN8aEiHq1w5kyIoWjx0u5a0YWx0qs2Z6vefMXQx8gX1XXq+oxYBaQVmFM\nGjDNuT0HuFI8nxJLA2aparGqbgDygT6quk1VswFU9QCwGmhTyVzTgMFnt2vGmEDVsXkDnh3Sk+xN\ne3n609VulxNyvAmGNsDmMvcL+M+L+E/GqGoJsA+I9mZb57RTb2CRs6iFqp7srLUdaFFZUSJyh4hk\nikhmYWGhF7thjAkk1/dszS8viuev327k4+Vb3S4npLh65VVEGgDvAveq6v6K69XzGflK37emqm+o\naqqqpsbE2BfpGBOMHr72PJLbNuahOcvJ32nN9nzFm2DYAsSVuR/rLKt0jIiEA1FA0em2FZEIPKEw\nQ1XfKzNmh4i0csa0AnZ6uzPGmOBystleZEQYd83IsmZ7PuJNMCwBOolIgojUxnMxOaPCmAzgduf2\nEGCB89t+BjDUeddSAtAJWOxcf5gKrFbVF08z1+3Ah2e6U8aY4NEqqi4vD01i3c6DPPzeCmu25wNV\nBoNzzWAMMA/PReJ0Vc0VkQkiMsgZNhWIFpF8YCzOO4lUNRdIB1YBc4HRqloKXAz8AugnIjnOz7XO\nXM8AV4vIOuAq574xJoT9rFMM913VmQ9ytjJ9kTXbq2kSDOmbmpqqmZmZbpdhjKlBJ04o/z1tCd/l\nF/HOnRfal2lVAxHJUtXUisvtY7/GmIBQq5Yw8ZYkYhpGcteMbPYcsmZ7NcWCwRgTMJrUr83k4ckU\nHijmvnRrtldTLBiMMQGlV1xjHrshkS/zCnltoTXbqwkWDMaYgDP8grbc2LsNEz9fy9fr7AOu1c2C\nwRgTcESEJ2/sTqfmDbhnVg5b9x5xu6SgYsFgjAlIJ5vtnfzedmu2V30sGIwxAatDTAOeG9KTpZv2\n8tQn1myvulgwGGMC2rU9WvHfFyfw1ncbyVhmzfaqgwWDMSbgjb+2KyntmjDu3eXk7zzgdjkBz4LB\nGBPwIsJqMem2ZOpGhHHn9GwOFVuzvXNhwWCMCQoto+rwyrDerC88yHhrtndOLBiMMUHj4o7NGHt1\nZzKWbeXv//rR7XIClgWDMSao3HV5R/p1bc4TH69i6aY9bpcTkCwYjDFBpVYt4cVbetGiUR1Gz8hm\ntzXbO2MWDMaYoNO4Xm2mDE9h18Fj3Ds7h1JrtndGLBiMMUGpR2wUfxjUja/WFvLqgnVulxNQLBiM\nMUFrWJ84bkpuw8tfrOOfa63Znre8CgYRGSAieSKSLyLjKlkfKSKznfWLRCS+zLrxzvI8EelfZvmb\nIrJTRFZWmOsPIrKlkq/8NMaYMyIiPDm4B11aNOTeWUvZYs32vFJlMIhIGDAJGAgkAsNEJLHCsJHA\nHlXtCEwEnnW2TQSGAt2AAcBkZz6At5xllZmoqknOzydntkvGGPMfdWuHMXl4MsdLlbtmZFNcUup2\nSX7Pm78Y+gD5qrpeVY8Bs4C0CmPSgGnO7TnAlSIizvJZqlqsqhuAfGc+VPUrYHc17IMxxpxW+5gG\n/OnmnizbvJcn/2HN9qriTTC0ATaXuV/gLKt0jKqWAPuAaC+3rcwYEVnunG5qUtkAEblDRDJFJLOw\n0M4dGmNOb0D3VvzPJQn87fsf+TBni9vl+DV/vPg8BegAJAHbgBcqG6Sqb6hqqqqmxsTE+LI+Y0yA\nemhgV86Pb8K4d1ewboc12zsVb4JhCxBX5n6ss6zSMSISDkQBRV5uW46q7lDVUlU9AfwfzqknY4w5\nVxFhtXjttmTqR4Zx5/QsDlqzvUp5EwxLgE4ikiAitfFcTM6oMCYDuN25PQRYoJ4OVhnAUOddSwlA\nJ2Dx6R5MRFqVuXsjsPJUY40x5ky1aORptrdh1yHGvbvcmu1VospgcK4ZjAHmAauBdFXNFZEJIjLI\nGTYViBaRfGAsMM7ZNhdIB1YBc4HRqloKICIzge+BLiJSICIjnbmeE5EVIrIcuAK4r5r21RhjALio\nQzPuv6YLHy/fxrTvNrpdjt+RYEjL1NRUzczMdLsMY0wAOXFC+fXfMvlqXSGzf3MhyW0rfZ9LUBOR\nLFVNrbjcHy8+G2NMjfM020uiZZSn2V7RwWK3S/IbFgzGmJAVVS+CKcNTKDpkzfbKsmAwxoS07m2i\nmDCoG1+v28XLX1izPbBgMMYYbj0/jiEpsby6YB1f5u10uxzXWTAYY0KeiPBEWndPs73ZORTsOex2\nSa6yYDDGGDzN9l4fkUJpqTI6xJvtWTAYY4wjvll9nr+5F8sK9vHEx6vcLsc1FgzGGFPGgO4tuePS\n9kz/1yY+WBqazfYsGIwxpoIH+3ehT3xTxr+3grUh2GzPgsEYYyoID6vFa7f1pn5keEg227NgMMaY\nSjRvVIdXh/Vm465DPDQntJrtWTAYY8wpXNghmgf6d+UfK7bx1283ul2Oz1gwGGPMadx5WXuuTmzB\nU5+sJuvH0Pg2YgsGY4w5DbHMFNEAAAwYSURBVBHhTzf3ok2TuoyesZRdIdBsz4LBGGOqEFU3gsnD\nk9lz+Bj3zFoa9M32LBiMMcYL3VpH8URad77NL+Klz9e6XU6NsmAwxhgv3XJ+HLekxvLqgnwWrgne\nZnteBYOIDBCRPBHJF5FxlayPFJHZzvpFIhJfZt14Z3meiPQvs/xNEdkpIisrzNVUROaLyDrn39D7\nWiVjjN+akNadxFaNuHd2Dpt3B2ezvSqDQUTCgEnAQCARGCYiiRWGjQT2qGpHYCLwrLNtIjAU6AYM\nACY78wG85SyraBzwhap2Ar5w7htjjF+oExHGlBHJnFDlrhnZHD0efM32vPmLoQ+Qr6rrVfUYMAtI\nqzAmDZjm3J4DXCki4iyfparFqroByHfmQ1W/Aip771fZuaYBg89gf4wxpsa1i67PCzf3YsWWfUwI\nwmZ73gRDG2BzmfsFzrJKx6hqCbAPiPZy24paqOo25/Z2oEVlg0TkDhHJFJHMwsJCL3bDGGOqzzXd\nWvKby9rz9qJNvJdd4HY51cqvLz6r5zPolb4vTFXfUNVUVU2NiYnxcWXGGAMPXNOFCxKa8vD7K1iz\nfb/b5VQbb4JhCxBX5n6ss6zSMSISDkQBRV5uW9EOEWnlzNUKCN5L/8aYgBYeVotXb+tNwzoRjJqe\nzYGjx90uqVp4EwxLgE4ikiAitfFcTM6oMCYDuN25PQRY4Py2nwEMdd61lAB0AhZX8Xhl57od+NCL\nGo0xxhXNG9bhtWG92bT7MA8GSbO9KoPBuWYwBpgHrAbSVTVXRCaIyCBn2FQgWkTygbE47yRS1Vwg\nHVgFzAVGq2opgIjMBL4HuohIgYiMdOZ6BrhaRNYBVzn3jTHGb13QPpqHBnTh05XbmfrNBrfLOWcS\nDOmWmpqqmZmZbpdhjAlhqsqd07P4YvVOZt3Rl9T4pm6XVCURyVLV1IrL/friszHGBAoR4fmbexHb\npC6j384O6GZ7FgzGGFNNGtWJYPLwFPYePs7dMwO32Z4FgzHGVKPE1o344+DufPdDES/Oz3O7nLNi\nwWCMMdXs5tQ4hp4fx6SFP/DF6h1ul3PGLBiMMaYG/GFQN7q1bsR9Adhsz4LBGGNqQJ2IMKYMTwFg\n1IysgGq2Z8FgjDE1pG10PV68JYmVW/bz+Ee5bpfjNQsGY4ypQVcltmDU5R2YuXgzc7ICo9meBYMx\nxtSw+6/uzIXto3nk/RWs3ub/zfYsGIwxpoaFh9XilWG9iaobwajpWez382Z7FgzGGOMDMQ0jmTQ8\nmc17jvDgO/7dbM+CwRhjfOT8+KaMH9iVubnb+cvX/ttsz4LBGGN8aOQlCQzs3pJn5q5h8YbKvt3Y\nfRYMxhjjQyLCc0N60rZpPca8nc3OA0fdLuknLBiMMcbHGtaJYMqIZPYf9TTbKyk94XZJ5VgwGGOM\nC7q2bMSTg3vwr/W7eWH+WrfLKcerYBCRASKSJyL5IjKukvWRIjLbWb9IROLLrBvvLM8Tkf5VzSki\nb4nIBhHJcX6Szm0XjTHGP/08JZZhfdoy5csfmL/Kf5rtVRkMIhIGTAIGAonAMBFJrDBsJLBHVTsC\nE4FnnW0T8XxHdDdgADBZRMK8mPMBVU1yfnLOaQ+NMcaP/f6GRLq3acTY9Bw2FflHsz1v/mLoA+Sr\n6npVPQbMAtIqjEkDpjm35wBXiog4y2eparGqbgDynfm8mdMYY4LeyWZ7tUT8ptmeN8HQBthc5n6B\ns6zSMapaAuwDok+zbVVzPikiy0VkoohEVlaUiNwhIpkikllYWOjFbhhjjH+Ka1qPibf2Infrfn7/\nofvN9vzx4vN4oCtwPtAUeKiyQar6hqqmqmpqTEyML+szxphq169rC0Zf0YHZmZtJz9xc9QY1yJtg\n2ALElbkf6yyrdIyIhANRQNFptj3lnKq6TT2Kgb/iOe1kjDFBb+zVXbioQzSPfrCS3K37XKvDm2BY\nAnQSkQQRqY3nYnJGhTEZwO3O7SHAAvU0AskAhjrvWkoAOgGLTzeniLRy/hVgMLDyXHbQGGMCRVgt\n4ZVhvWlcL4K7ZmSz74g7zfaqDAbnmsEYYB6wGkhX1VwRmSAig5xhU4FoEckHxgLjnG1zgXRgFTAX\nGK2qpaea05lrhoisAFYAzYA/Vs+uGmOM/2vWIJLJw5PZsucID7yzzJVme+LPHf68lZqaqpmZmW6X\nYYwx1WbqNxt44uNVjB/Yld9c1qFGHkNEslQ1teJyf7z4bIwxIe+/L47nuh6teG5eHovWF/n0sS0Y\njDHGD4kIz/y8B+2a1mPMzKXs3O+7ZnsWDMYY46c8zfZSOHi0hDE+bLZnwWCMMX6sS8uGPHVTdxZv\n2M3zn+X55DEtGIwxxs/d2DuW4Re05c//XM9nudtr/PEsGIwxJgA8dkMiPWOjuP+dZfxYdKhGH8uC\nwRhjAkBkeBiTbkumlgh3Ts+u0WZ7FgzGGBMg4prW46Vbk1i9bT+PfVhzTSEsGIwxJoBc0bU5v+3X\nkfTMAmYv2VQjj2HBYIwxAebeqzpzScdmPPphLiu3VH+zPQsGY4wJMGG1hJeHJnFBQlMiw6v/ZTy8\n2mc0xhhT46IbRPL3kRfUyNz2F4MxxphyLBiMMcaUY8FgjDGmHAsGY4wx5VgwGGOMKceCwRhjTDkW\nDMYYY8qxYDDGGFOOqKrbNZwzESkEfjzLzZsBu6qxnOpidZ0Zq+vMWF1nxl/rgnOrrZ2qxlRcGBTB\ncC5EJFNVU92uoyKr68xYXWfG6joz/loX1ExtdirJGGNMORYMxhhjyrFggDfcLuAUrK4zY3WdGavr\nzPhrXVADtYX8NQZjjDHl2V8MxhhjyrFgMMYYU07IBIOIDBCRPBHJF5FxlayPFJHZzvpFIhLvJ3X9\nUkQKRSTH+fkfH9T0pojsFJFKv21cPF5xal4uIsk1XZOXdV0uIvvKHKvHfFRXnIgsFJFVIpIrIvdU\nMsbnx8zLunx+zESkjogsFpFlTl2PVzLG589HL+vy+fOxzGOHichSEfm4knXVe7xUNeh/gDDgB6A9\nUBtYBiRWGHMX8Lpzeygw20/q+iXwmo+P16VAMrDyFOuvBT4FBOgLLPKTui4HPnbh/69WQLJzuyGw\ntpL/jj4/Zl7W5fNj5hyDBs7tCGAR0LfCGDeej97U5fPnY5nHHgu8Xdl/r+o+XqHyF0MfIF9V16vq\nMWAWkFZhTBowzbk9B7hSRMQP6vI5Vf0K2H2aIWnA39TjX0BjEWnlB3W5QlW3qWq2c/sAsBpoU2GY\nz4+Zl3X5nHMMDjp3I5yfiu+C8fnz0cu6XCEiscB1wF9OMaRaj1eoBEMbYHOZ+wX89Any7zGqWgLs\nA6L9oC6AnzunH+aISFwN1+QNb+t2w4XOqYBPRaSbrx/c+RO+N57fNsty9Zidpi5w4Zg5p0VygJ3A\nfFU95fHy4fPRm7rAnefjS8CDwIlTrK/W4xUqwRDIPgLiVbUnMJ///FZgfiobT++XXsCrwAe+fHAR\naQC8C9yrqvt9+dinU0VdrhwzVS1V1SQgFugjIt198bhV8aIunz8fReR6YKeqZtX0Y50UKsGwBSib\n7LHOskrHiEg4EAUUuV2XqhaparFz9y9ASg3X5A1vjqfPqer+k6cCVPUTIEJEmvnisUUkAs+L7wxV\nfa+SIa4cs6rqcvOYOY+5F1gIDKiwyo3nY5V1ufR8vBgYJCIb8Zxu7ici0yuMqdbjFSrBsAToJCIJ\nIlIbz8WZjApjMoDbndtDgAXqXMlxs64K56EH4TlP7LYM4L+cd9r0Bfap6ja3ixKRlifPq4pIHzz/\nf9f4i4nzmFOB1ar64imG+fyYeVOXG8dMRGJEpLFzuy5wNbCmwjCfPx+9qcuN56OqjlfVWFWNx/Ma\nsUBVR1QYVq3HK/xsNwwkqloiImOAeXjeCfSmquaKyAQgU1Uz8DyB/i4i+XgucA71k7ruFpFBQIlT\n1y9rui4RmYnn3SrNRKQA+D2eC3Go6uvAJ3jeZZMPHAZ+VdM1eVnXEGCUiJQAR4ChPgh38PxG9wtg\nhXN+GuBhoG2Z2tw4Zt7U5cYxawVME5EwPEGUrqofu/189LIunz8fT6Umj5e1xDDGGFNOqJxKMsYY\n4yULBmOMMeVYMBhjjCnHgsEYY0w5FgzGGGPKsWAwxhhTjgWDMcaYcv4fPDiJg/Uh8OwAAAAASUVO\nRK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WVlQRtsDL92P",
        "colab_type": "text"
      },
      "source": [
        "## Analysis of predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FG8iTzzqNcvE",
        "colab_type": "code",
        "outputId": "01d9eb2f-55cd-4a9b-c42b-87b9212b067d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        }
      },
      "source": [
        "movie_columns = 'MovieID::Title::Genres'.split('::')\n",
        "movies_path = 'data/movies.dat'\n",
        "df_movies = pd.read_csv(os.path.join(dataset_path, 'movies.dat'), sep='::', names=movie_columns)\n",
        "df_movies['MovieID'] = df_movies['MovieID'] - 1\n",
        "df_movies = df_movies.set_index('MovieID')\n",
        "df_ratings_val = pd.read_csv(val_ratings_path, sep=';')\n",
        "\n",
        "df_movies.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Title</th>\n",
              "      <th>Genres</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MovieID</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Toy Story (1995)</td>\n",
              "      <td>Animation|Children's|Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Jumanji (1995)</td>\n",
              "      <td>Adventure|Children's|Fantasy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Grumpier Old Men (1995)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Waiting to Exhale (1995)</td>\n",
              "      <td>Comedy|Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Father of the Bride Part II (1995)</td>\n",
              "      <td>Comedy</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                      Title                        Genres\n",
              "MovieID                                                                  \n",
              "0                          Toy Story (1995)   Animation|Children's|Comedy\n",
              "1                            Jumanji (1995)  Adventure|Children's|Fantasy\n",
              "2                   Grumpier Old Men (1995)                Comedy|Romance\n",
              "3                  Waiting to Exhale (1995)                  Comedy|Drama\n",
              "4        Father of the Bride Part II (1995)                        Comedy"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jrHAj7S_L88T",
        "colab_type": "code",
        "outputId": "7fb89353-6449-4482-e6b6-64bcf2cb42e8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "batch_size = 1\n",
        "val_dataset = create_csv_dataset_hl(val_ratings_path, 'Rating', batch_size = 1, epochs=1, select_columns=[0,1,2])\n",
        "predictions = model.predict(val_dataset, steps=len(df_ratings_val), verbose=1, workers=4)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "200042/200042 [==============================] - 251s 1ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "er5nupFMTqNd",
        "colab_type": "code",
        "outputId": "4bcbadfe-9775-426f-cd26-c59aece76df7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# | round predictions\n",
        "y_pred = predictions.round().astype('int32')\n",
        "\n",
        "# | ensure that results are in interval [1,5]\n",
        "y_pred[np.where(y_pred < 1)] = 1\n",
        "y_pred[np.where(y_pred > 5)] = 5\n",
        "\n",
        "y_pred.min(), y_pred.max()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1, 5)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JdLK3fxqMsMJ",
        "colab_type": "code",
        "outputId": "411dd658-33c8-4b32-d88b-42ef1c9424b4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "from sklearn.metrics import mean_absolute_error\n",
        "\n",
        "df_ratings_val = df_ratings_val.iloc[:len(y_pred)]\n",
        "mean_absolute_error(df_ratings_val['Rating'], y_pred)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9603883184531249"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IWVW7OvgUUx1",
        "colab_type": "code",
        "outputId": "ec0321f3-f51a-455b-d71e-67f2f72e6561",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "df_comparison = pd.DataFrame({'pred': np.squeeze(y_pred), 'label': df_ratings_val['Rating'].values}, index = df_ratings_val['MovieID'])\n",
        "df_comparison = df_comparison.sort_values(by='pred', axis=0, ascending=False)\n",
        "\n",
        "df_comparison = pd.concat([df_movies.loc[df_comparison.index], df_comparison], axis=1)\n",
        "\n",
        "df_comparison.head(100)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Title</th>\n",
              "      <th>Genres</th>\n",
              "      <th>pred</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MovieID</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2897</th>\n",
              "      <td>Dark Half, The (1993)</td>\n",
              "      <td>Horror|Mystery</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>918</th>\n",
              "      <td>Wizard of Oz, The (1939)</td>\n",
              "      <td>Adventure|Children's|Drama|Musical</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2079</th>\n",
              "      <td>Lady and the Tramp (1955)</td>\n",
              "      <td>Animation|Children's|Comedy|Musical|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>332</th>\n",
              "      <td>Tommy Boy (1995)</td>\n",
              "      <td>Comedy</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1911</th>\n",
              "      <td>Out of Sight (1998)</td>\n",
              "      <td>Action|Crime|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1072</th>\n",
              "      <td>Willy Wonka and the Chocolate Factory (1971)</td>\n",
              "      <td>Adventure|Children's|Comedy|Fantasy</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>550</th>\n",
              "      <td>Nightmare Before Christmas, The (1993)</td>\n",
              "      <td>Children's|Comedy|Musical</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1913</th>\n",
              "      <td>Smoke Signals (1998)</td>\n",
              "      <td>Comedy|Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>355</th>\n",
              "      <td>Forrest Gump (1994)</td>\n",
              "      <td>Comedy|Romance|War</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3354</th>\n",
              "      <td>Ninth Gate, The (2000)</td>\n",
              "      <td>Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1222</th>\n",
              "      <td>Grand Day Out, A (1992)</td>\n",
              "      <td>Animation|Comedy</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2115</th>\n",
              "      <td>Lord of the Rings, The (1978)</td>\n",
              "      <td>Adventure|Animation|Children's|Sci-Fi</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>857</th>\n",
              "      <td>Godfather, The (1972)</td>\n",
              "      <td>Action|Crime|Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2086</th>\n",
              "      <td>Peter Pan (1953)</td>\n",
              "      <td>Animation|Children's|Fantasy|Musical</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2723</th>\n",
              "      <td>Runaway Bride (1999)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2691</th>\n",
              "      <td>Run Lola Run (Lola rennt) (1998)</td>\n",
              "      <td>Action|Crime|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3509</th>\n",
              "      <td>Frequency (2000)</td>\n",
              "      <td>Drama|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>912</th>\n",
              "      <td>Maltese Falcon, The (1941)</td>\n",
              "      <td>Film-Noir|Mystery</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1249</th>\n",
              "      <td>Bridge on the River Kwai, The (1957)</td>\n",
              "      <td>Drama|War</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1951</th>\n",
              "      <td>Midnight Cowboy (1969)</td>\n",
              "      <td>Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2598</th>\n",
              "      <td>Election (1999)</td>\n",
              "      <td>Comedy</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>355</th>\n",
              "      <td>Forrest Gump (1994)</td>\n",
              "      <td>Comedy|Romance|War</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3762</th>\n",
              "      <td>F/X (1986)</td>\n",
              "      <td>Action|Crime|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1207</th>\n",
              "      <td>Apocalypse Now (1979)</td>\n",
              "      <td>Drama|War</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>224</th>\n",
              "      <td>Disclosure (1994)</td>\n",
              "      <td>Drama|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>352</th>\n",
              "      <td>Crow, The (1994)</td>\n",
              "      <td>Action|Romance|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1483</th>\n",
              "      <td>Daytrippers, The (1996)</td>\n",
              "      <td>Mystery</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2027</th>\n",
              "      <td>Saving Private Ryan (1998)</td>\n",
              "      <td>Action|Drama|War</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2301</th>\n",
              "      <td>My Cousin Vinny (1992)</td>\n",
              "      <td>Comedy</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1640</th>\n",
              "      <td>Full Monty, The (1997)</td>\n",
              "      <td>Comedy</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2924</th>\n",
              "      <td>Conformist, The (Il Conformista) (1970)</td>\n",
              "      <td>Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1170</th>\n",
              "      <td>Bob Roberts (1992)</td>\n",
              "      <td>Comedy</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1094</th>\n",
              "      <td>Glengarry Glen Ross (1992)</td>\n",
              "      <td>Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>288</th>\n",
              "      <td>Only You (1994)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2570</th>\n",
              "      <td>Matrix, The (1999)</td>\n",
              "      <td>Action|Sci-Fi|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1981</th>\n",
              "      <td>Halloween (1978)</td>\n",
              "      <td>Horror</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>918</th>\n",
              "      <td>Wizard of Oz, The (1939)</td>\n",
              "      <td>Adventure|Children's|Drama|Musical</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>317</th>\n",
              "      <td>Shawshank Redemption, The (1994)</td>\n",
              "      <td>Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3535</th>\n",
              "      <td>Keeping the Faith (2000)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2857</th>\n",
              "      <td>American Beauty (1999)</td>\n",
              "      <td>Comedy|Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>470</th>\n",
              "      <td>Hudsucker Proxy, The (1994)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3754</th>\n",
              "      <td>Perfect Storm, The (2000)</td>\n",
              "      <td>Action|Adventure|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1239</th>\n",
              "      <td>Terminator, The (1984)</td>\n",
              "      <td>Action|Sci-Fi|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>Usual Suspects, The (1995)</td>\n",
              "      <td>Crime|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2398</th>\n",
              "      <td>Santa Claus: The Movie (1985)</td>\n",
              "      <td>Adventure|Children's|Fantasy</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1096</th>\n",
              "      <td>E.T. the Extra-Terrestrial (1982)</td>\n",
              "      <td>Children's|Drama|Fantasy|Sci-Fi</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3684</th>\n",
              "      <td>Prizzi's Honor (1985)</td>\n",
              "      <td>Comedy|Drama|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3066</th>\n",
              "      <td>Women on the Verge of a Nervous Breakdown (1988)</td>\n",
              "      <td>Comedy|Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2857</th>\n",
              "      <td>American Beauty (1999)</td>\n",
              "      <td>Comedy|Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2570</th>\n",
              "      <td>Matrix, The (1999)</td>\n",
              "      <td>Action|Sci-Fi|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>288</th>\n",
              "      <td>Only You (1994)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3392</th>\n",
              "      <td>Date with an Angel (1987)</td>\n",
              "      <td>Comedy|Fantasy</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3107</th>\n",
              "      <td>Fisher King, The (1991)</td>\n",
              "      <td>Comedy|Drama|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2013</th>\n",
              "      <td>Freaky Friday (1977)</td>\n",
              "      <td>Children's|Comedy</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2395</th>\n",
              "      <td>Shakespeare in Love (1998)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2005</th>\n",
              "      <td>Mask of Zorro, The (1998)</td>\n",
              "      <td>Action|Adventure|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2996</th>\n",
              "      <td>Being John Malkovich (1999)</td>\n",
              "      <td>Comedy</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>Clueless (1995)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>336</th>\n",
              "      <td>What's Eating Gilbert Grape (1993)</td>\n",
              "      <td>Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1458</th>\n",
              "      <td>Absolute Power (1997)</td>\n",
              "      <td>Mystery|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1218</th>\n",
              "      <td>Psycho (1960)</td>\n",
              "      <td>Horror|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1220</th>\n",
              "      <td>Godfather: Part II, The (1974)</td>\n",
              "      <td>Action|Crime|Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2160</th>\n",
              "      <td>NeverEnding Story, The (1984)</td>\n",
              "      <td>Adventure|Children's|Fantasy</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2801</th>\n",
              "      <td>Tequila Sunrise (1988)</td>\n",
              "      <td>Action|Romance|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3090</th>\n",
              "      <td>Kagemusha (1980)</td>\n",
              "      <td>Drama|War</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1096</th>\n",
              "      <td>E.T. the Extra-Terrestrial (1982)</td>\n",
              "      <td>Children's|Drama|Fantasy|Sci-Fi</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3910</th>\n",
              "      <td>Best in Show (2000)</td>\n",
              "      <td>Comedy</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>799</th>\n",
              "      <td>Lone Star (1996)</td>\n",
              "      <td>Drama|Mystery</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2996</th>\n",
              "      <td>Being John Malkovich (1999)</td>\n",
              "      <td>Comedy</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>292</th>\n",
              "      <td>Professional, The (a.k.a. Leon: The Profession...</td>\n",
              "      <td>Crime|Drama|Romance|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>479</th>\n",
              "      <td>Jurassic Park (1993)</td>\n",
              "      <td>Action|Adventure|Sci-Fi</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>919</th>\n",
              "      <td>Gone with the Wind (1939)</td>\n",
              "      <td>Drama|Romance|War</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>411</th>\n",
              "      <td>Age of Innocence, The (1993)</td>\n",
              "      <td>Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1267</th>\n",
              "      <td>Pump Up the Volume (1990)</td>\n",
              "      <td>Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2958</th>\n",
              "      <td>Fight Club (1999)</td>\n",
              "      <td>Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1303</th>\n",
              "      <td>Butch Cassidy and the Sundance Kid (1969)</td>\n",
              "      <td>Action|Comedy|Western</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1277</th>\n",
              "      <td>Young Frankenstein (1974)</td>\n",
              "      <td>Comedy|Horror</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2019</th>\n",
              "      <td>Dangerous Liaisons (1988)</td>\n",
              "      <td>Drama|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1239</th>\n",
              "      <td>Terminator, The (1984)</td>\n",
              "      <td>Action|Sci-Fi|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>514</th>\n",
              "      <td>Remains of the Day, The (1993)</td>\n",
              "      <td>Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3535</th>\n",
              "      <td>Keeping the Faith (2000)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2287</th>\n",
              "      <td>Thing, The (1982)</td>\n",
              "      <td>Action|Horror|Sci-Fi|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1269</th>\n",
              "      <td>Back to the Future (1985)</td>\n",
              "      <td>Comedy|Sci-Fi</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2027</th>\n",
              "      <td>Saving Private Ryan (1998)</td>\n",
              "      <td>Action|Drama|War</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2715</th>\n",
              "      <td>Ghostbusters (1984)</td>\n",
              "      <td>Comedy|Horror</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>526</th>\n",
              "      <td>Schindler's List (1993)</td>\n",
              "      <td>Drama|War</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2996</th>\n",
              "      <td>Being John Malkovich (1999)</td>\n",
              "      <td>Comedy</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>Usual Suspects, The (1995)</td>\n",
              "      <td>Crime|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3246</th>\n",
              "      <td>Sister Act (1992)</td>\n",
              "      <td>Comedy|Crime</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>709</th>\n",
              "      <td>Celtic Pride (1996)</td>\n",
              "      <td>Comedy</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>914</th>\n",
              "      <td>Sabrina (1954)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1207</th>\n",
              "      <td>Apocalypse Now (1979)</td>\n",
              "      <td>Drama|War</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69</th>\n",
              "      <td>From Dusk Till Dawn (1996)</td>\n",
              "      <td>Action|Comedy|Crime|Horror|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1609</th>\n",
              "      <td>Hunt for Red October, The (1990)</td>\n",
              "      <td>Action|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>770</th>\n",
              "      <td>Vie est belle, La (Life is Rosey) (1987)</td>\n",
              "      <td>Comedy|Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>667</th>\n",
              "      <td>Pather Panchali (1955)</td>\n",
              "      <td>Drama</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2454</th>\n",
              "      <td>Fly, The (1986)</td>\n",
              "      <td>Horror|Sci-Fi</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>Sense and Sensibility (1995)</td>\n",
              "      <td>Drama|Romance</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2117</th>\n",
              "      <td>Dead Zone, The (1983)</td>\n",
              "      <td>Horror|Thriller</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2579</th>\n",
              "      <td>Go (1999)</td>\n",
              "      <td>Crime</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                     Title  ... label\n",
              "MovieID                                                     ...      \n",
              "2897                                 Dark Half, The (1993)  ...     3\n",
              "918                               Wizard of Oz, The (1939)  ...     5\n",
              "2079                             Lady and the Tramp (1955)  ...     4\n",
              "332                                       Tommy Boy (1995)  ...     4\n",
              "1911                                   Out of Sight (1998)  ...     5\n",
              "1072          Willy Wonka and the Chocolate Factory (1971)  ...     5\n",
              "550                 Nightmare Before Christmas, The (1993)  ...     5\n",
              "1913                                  Smoke Signals (1998)  ...     2\n",
              "355                                    Forrest Gump (1994)  ...     3\n",
              "3354                                Ninth Gate, The (2000)  ...     3\n",
              "1222                               Grand Day Out, A (1992)  ...     5\n",
              "2115                         Lord of the Rings, The (1978)  ...     2\n",
              "857                                  Godfather, The (1972)  ...     5\n",
              "2086                                      Peter Pan (1953)  ...     4\n",
              "2723                                  Runaway Bride (1999)  ...     3\n",
              "2691                      Run Lola Run (Lola rennt) (1998)  ...     5\n",
              "3509                                      Frequency (2000)  ...     5\n",
              "912                             Maltese Falcon, The (1941)  ...     4\n",
              "1249                  Bridge on the River Kwai, The (1957)  ...     5\n",
              "1951                                Midnight Cowboy (1969)  ...     5\n",
              "2598                                       Election (1999)  ...     4\n",
              "355                                    Forrest Gump (1994)  ...     3\n",
              "3762                                            F/X (1986)  ...     3\n",
              "1207                                 Apocalypse Now (1979)  ...     4\n",
              "224                                      Disclosure (1994)  ...     5\n",
              "352                                       Crow, The (1994)  ...     1\n",
              "1483                               Daytrippers, The (1996)  ...     3\n",
              "2027                            Saving Private Ryan (1998)  ...     4\n",
              "2301                                My Cousin Vinny (1992)  ...     3\n",
              "1640                                Full Monty, The (1997)  ...     3\n",
              "2924               Conformist, The (Il Conformista) (1970)  ...     5\n",
              "1170                                    Bob Roberts (1992)  ...     4\n",
              "1094                            Glengarry Glen Ross (1992)  ...     4\n",
              "288                                        Only You (1994)  ...     3\n",
              "2570                                    Matrix, The (1999)  ...     2\n",
              "1981                                      Halloween (1978)  ...     4\n",
              "918                               Wizard of Oz, The (1939)  ...     4\n",
              "317                       Shawshank Redemption, The (1994)  ...     5\n",
              "3535                              Keeping the Faith (2000)  ...     4\n",
              "2857                                American Beauty (1999)  ...     5\n",
              "470                            Hudsucker Proxy, The (1994)  ...     4\n",
              "3754                             Perfect Storm, The (2000)  ...     4\n",
              "1239                                Terminator, The (1984)  ...     4\n",
              "49                              Usual Suspects, The (1995)  ...     5\n",
              "2398                         Santa Claus: The Movie (1985)  ...     2\n",
              "1096                     E.T. the Extra-Terrestrial (1982)  ...     2\n",
              "3684                                 Prizzi's Honor (1985)  ...     4\n",
              "3066      Women on the Verge of a Nervous Breakdown (1988)  ...     4\n",
              "2857                                American Beauty (1999)  ...     5\n",
              "2570                                    Matrix, The (1999)  ...     5\n",
              "288                                        Only You (1994)  ...     2\n",
              "3392                             Date with an Angel (1987)  ...     3\n",
              "3107                               Fisher King, The (1991)  ...     4\n",
              "2013                                  Freaky Friday (1977)  ...     2\n",
              "2395                            Shakespeare in Love (1998)  ...     2\n",
              "2005                             Mask of Zorro, The (1998)  ...     4\n",
              "2996                           Being John Malkovich (1999)  ...     4\n",
              "38                                         Clueless (1995)  ...     4\n",
              "336                     What's Eating Gilbert Grape (1993)  ...     4\n",
              "1458                                 Absolute Power (1997)  ...     4\n",
              "1218                                         Psycho (1960)  ...     4\n",
              "1220                        Godfather: Part II, The (1974)  ...     5\n",
              "2160                         NeverEnding Story, The (1984)  ...     3\n",
              "2801                                Tequila Sunrise (1988)  ...     4\n",
              "3090                                      Kagemusha (1980)  ...     5\n",
              "1096                     E.T. the Extra-Terrestrial (1982)  ...     4\n",
              "3910                                   Best in Show (2000)  ...     5\n",
              "799                                       Lone Star (1996)  ...     5\n",
              "2996                           Being John Malkovich (1999)  ...     4\n",
              "292      Professional, The (a.k.a. Leon: The Profession...  ...     4\n",
              "479                                   Jurassic Park (1993)  ...     4\n",
              "919                              Gone with the Wind (1939)  ...     4\n",
              "411                           Age of Innocence, The (1993)  ...     3\n",
              "1267                             Pump Up the Volume (1990)  ...     2\n",
              "2958                                     Fight Club (1999)  ...     5\n",
              "1303             Butch Cassidy and the Sundance Kid (1969)  ...     5\n",
              "1277                             Young Frankenstein (1974)  ...     5\n",
              "2019                             Dangerous Liaisons (1988)  ...     5\n",
              "1239                                Terminator, The (1984)  ...     2\n",
              "514                         Remains of the Day, The (1993)  ...     4\n",
              "3535                              Keeping the Faith (2000)  ...     4\n",
              "2287                                     Thing, The (1982)  ...     3\n",
              "1269                             Back to the Future (1985)  ...     5\n",
              "2027                            Saving Private Ryan (1998)  ...     4\n",
              "2715                                   Ghostbusters (1984)  ...     4\n",
              "526                                Schindler's List (1993)  ...     5\n",
              "2996                           Being John Malkovich (1999)  ...     3\n",
              "49                              Usual Suspects, The (1995)  ...     5\n",
              "3246                                     Sister Act (1992)  ...     2\n",
              "709                                    Celtic Pride (1996)  ...     3\n",
              "914                                         Sabrina (1954)  ...     5\n",
              "1207                                 Apocalypse Now (1979)  ...     5\n",
              "69                              From Dusk Till Dawn (1996)  ...     4\n",
              "1609                      Hunt for Red October, The (1990)  ...     4\n",
              "770               Vie est belle, La (Life is Rosey) (1987)  ...     3\n",
              "667                                 Pather Panchali (1955)  ...     5\n",
              "2454                                       Fly, The (1986)  ...     3\n",
              "16                            Sense and Sensibility (1995)  ...     4\n",
              "2117                                 Dead Zone, The (1983)  ...     4\n",
              "2579                                             Go (1999)  ...     4\n",
              "\n",
              "[100 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4yLAJNiyFTBy",
        "colab_type": "text"
      },
      "source": [
        "# Fast.AI approach\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pOW59hlscE1j",
        "colab_type": "text"
      },
      "source": [
        "Does work much better than TF out of the box.\n",
        "\n",
        "What configuration does fast.ai use?:\n",
        "\n",
        "- doesn't use sigmoid by default.. (only when defining y_range argument in collab_learner, however this implementation actually seems to be wrong)\n",
        "```python \n",
        "torch.sigmoid(res) * (self.y_range[1]-self.y_range[0]) + self.y_range[0] \n",
        "```\n",
        "- uses one hot encoding (no hash_bucket) of users/movies\n",
        "- uses bias (for both #user & #movie)\n",
        "- in the fast.ai code there is a rule of thumb for calculating embedding sizes (however here we just use an embedding size of 50): \n",
        "```python \n",
        "def emb_sz_rule(n_cat:int)->int: return min(600, round(1.6 * n_cat**0.56))\n",
        "# | n_cat is the number of categories, e.g. #Users, #Movies, ...\n",
        "```\n",
        "\n",
        "So far.. nothing really special.\n",
        "However CollabLearner inherits from the Learner baseclass (whithout changing any of the default arguments), which has some interesting options activated:\n",
        "\n",
        "- Weight Decay:\n",
        "(defaults.wd = 1e-2, true_wd=True)\n",
        "By default in Fastai library, the weight decay is 1e-2 It is less than it should because in rare cases too big weight decay is causing that model doesn’t learn and that might cause hard to recognize problem for beginners. Jeremy recommends using 1e-1 instead of the default because now when you understand that if parameters become zero then the weight decay is too high. Having too small weight decay is just going to overfit earlier so it doesn’t break the model right away.\n",
        "\n",
        "The default weight decay will be wd, which will be handled using the method from [Fixing Weight Decay Regularization](https://arxiv.org/abs/1711.05101) in Adam if true_wd is set (otherwise it's L2 regularization). If true_wd is set it will affect all optimizers, not only Adam.\n",
        "\n",
        "- Optimizer is AdamW (same as Adam but small changes for implementing (true) weight decay?)\n",
        "\n",
        "**Results**:\n",
        "- Using collab_learner(wd=0) without weight decay: Training is slower but end results are still significantly better than in TensorFlow... \n",
        "\n",
        "- When using embeddingsize=50 & no sigmoid activation in TF we get better results (than with bigger embedding size, such as 200), however, still not comparable to fast.ai\n",
        "\n",
        "- When training fast.ai with learner.fit() instead of using 1-cycle policy, we finally get same results as in tensorflow --> so it's mainly a learning rate issue\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3pOynp3oFVQI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from fastai.collab import *\n",
        "from fastai.tabular import *"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M3OBFDp1Gi8k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# data = CollabDataBunch.from_df(df_ratings, seed=42, valid_pct=0.2, item_name='Title')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0NgEdlh-bEYy",
        "colab_type": "code",
        "outputId": "55876163-8b7d-46c2-a95e-db883a4c805d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "batch_size = 64\n",
        "embedding_size = 50\n",
        "rating_name = 'Rating'\n",
        "user_name = 'UserID'\n",
        "item_name = 'MovieID'\n",
        "cat_names = [user_name, item_name]\n",
        "\n",
        "data = CollabList.from_df(df_ratings, cat_names=cat_names, procs=Categorify) \\\n",
        "                 .split_from_df(col='validation') \\\n",
        "                 .label_from_df(cols=rating_name) \\\n",
        "                 .databunch(bs=batch_size, val_bs=batch_size)\n",
        "\n",
        "data.show_batch(3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>UserID</th>\n",
              "      <th>MovieID</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>2819</td>\n",
              "      <td>2656</td>\n",
              "      <td>4.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2902</td>\n",
              "      <td>1643</td>\n",
              "      <td>4.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4032</td>\n",
              "      <td>2114</td>\n",
              "      <td>4.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jDdEZQ_5G698",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# learn.lr_find(num_it=1000)\n",
        "# learn.recorder.plot(skip_end=10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-31teXV2ifCS",
        "colab_type": "text"
      },
      "source": [
        "### With bias, weight decay & sigmoid"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LzALekFyG9Tu",
        "colab_type": "code",
        "outputId": "0cc873dc-a9a4-4d2a-907d-8ebcb7d8381d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 200
        }
      },
      "source": [
        "learn = collab_learner(data, n_factors=embedding_size, y_range=[0, 5.5])\n",
        "learn.fit_one_cycle(5, 5e-3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.877579</td>\n",
              "      <td>0.853606</td>\n",
              "      <td>02:06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.833464</td>\n",
              "      <td>0.859048</td>\n",
              "      <td>02:19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.775253</td>\n",
              "      <td>0.815083</td>\n",
              "      <td>02:18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.612501</td>\n",
              "      <td>0.782485</td>\n",
              "      <td>02:14</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.447126</td>\n",
              "      <td>0.787592</td>\n",
              "      <td>02:06</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iMYyKzrziliB",
        "colab_type": "text"
      },
      "source": [
        "### With bias, weight decay"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6aIeQw-ssozh",
        "colab_type": "code",
        "outputId": "1337e436-2a28-4183-a2b4-58f29be393ab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 200
        }
      },
      "source": [
        "learn = collab_learner(data, n_factors=embedding_size)\n",
        "learn.fit_one_cycle(5, 5e-3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>1.135364</td>\n",
              "      <td>1.130077</td>\n",
              "      <td>02:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.079947</td>\n",
              "      <td>1.078794</td>\n",
              "      <td>02:15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.861280</td>\n",
              "      <td>0.907813</td>\n",
              "      <td>02:15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.707603</td>\n",
              "      <td>0.774266</td>\n",
              "      <td>02:09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.601839</td>\n",
              "      <td>0.741988</td>\n",
              "      <td>02:03</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z9fqjTwAisbQ",
        "colab_type": "text"
      },
      "source": [
        "### With bias"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q8oebnmgt_hv",
        "colab_type": "code",
        "outputId": "42d2fafa-aa14-4f50-fda7-fc3763e38217",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 200
        }
      },
      "source": [
        "learn = collab_learner(data, n_factors=embedding_size, wd=0, true_wd=False)\n",
        "learn.fit_one_cycle(5, 5e-3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>1.080151</td>\n",
              "      <td>1.128480</td>\n",
              "      <td>01:59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.016672</td>\n",
              "      <td>1.034428</td>\n",
              "      <td>02:12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.832345</td>\n",
              "      <td>0.882976</td>\n",
              "      <td>02:13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.643231</td>\n",
              "      <td>0.811179</td>\n",
              "      <td>02:08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.512417</td>\n",
              "      <td>0.799990</td>\n",
              "      <td>02:00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qV-nEnKS0qJA",
        "colab_type": "code",
        "outputId": "a9f534dc-e872-4cad-f216-ea3c018027cd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "learn.recorder.plot_lr()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3xV9f3H8dcnO4EQIIQACRBGGGFD\n2A4URVxQJ7jXr+5d24K21dpfq7b+HK1atWq1LqAoigMZQqUORpAwEgiEGSCDHSBkf35/3INNYwI3\nkJtzb/J5Ph73wb1n5X3uJfeTc77nfL+iqhhjjDHeCnI7gDHGmMBihcMYY0ydWOEwxhhTJ1Y4jDHG\n1IkVDmOMMXUS4naAhtCmTRtNSkpyO4YxxgSMFStW7FHVuJrmNYnCkZSURFpamtsxjDEmYIjIttrm\n2akqY4wxdWKFwxhjTJ1Y4TDGGFMnVjiMMcbUiRUOY4wxdeLTwiEi40UkS0SyRWRKDfPDRWS6M3+p\niCRVmTfVmZ4lIudVmb5VRNaISLqI2KVSxhjTwHx2Oa6IBAMvAucCO4DlIjJbVTOrLHYLsF9Vu4vI\nZOApYJKIpACTgT5AB2CBiPRQ1QpnvbNUdY+vshtjjKmdL+/jGAZkq+pmABGZBkwEqhaOicBjzvOZ\nwAsiIs70aapaAmwRkWxne9/5MK9pYKrKtr1FrMzZT35hCcVlFUSEBhMZGkxkWDBto8NpFxNB+xaR\nxESFuh3XGOPwZeFIAHKqvN4BDK9tGVUtF5GDQKwzfUm1dROc5wrMExEFXlHVV2v64SJyK3ArQKdO\nnU5tT0y9Kq+o5J8rdvCP77axLrfQq3XaNA+jZ7toesa3YEDHGIZ1aU37mEgfJzXG1CQQ7xw/TVV3\nikhbYL6IrFfVxdUXcgrKqwCpqak2WpWfWL51H1M/XEN2wWH6dGjBoxenMLJbLJ1bNyMsJIiS8gqK\nyyo5UlJOfmExeYXF5B4oZmPBIbLyDvHesm288U0lAJ1aRzGqWyznpsQzunsbIkKDXd47Y5oGXxaO\nnUDHKq8TnWk1LbNDREKAGGDv8dZV1WP/FojILDynsH5UOIx/UVVeWJjNsws2kNAqklevG8K5KfF4\nzkz+R1RYCFFh0LpZGB1bR/1oOxWVyrrcQpZu2cfSzXv5bHUu05bnEBUWzFk923LJoATG9IwjJNgu\nGDTGV3xZOJYDySLSBc+X/mTg6mrLzAZuwNN2cTmwUFVVRGYD74nIM3gax5OBZSLSDAhS1UPO83HA\n4z7cB1MPyisqmfLhGmau2MHEgR3435/0JTri5NosgoOEvgkx9E2I4ZbTulBaXsl3m/cyNyOPeRl5\nfLYml7jocC4bnMjVwzrRKfbHxccYc2rEl2OOi8gFwHNAMPCGqv5eRB4H0lR1tohEAG8Dg4B9wOQq\njemPADcD5cD9qjpHRLoCs5zNhwDvqervT5QjNTVVrZNDd5RXVHL3eyv5IiOP+89J5r6xyT86yqgv\nZRWVLFpfwIy0HSzKKkBVGd+3HT89vSuDOrXyyc80prESkRWqmlrjPF8WDn9hhcMdqsrUD9cwbXkO\nv7qwN/9zetcG+9n5hcW89e1W3lmyjcLicoYltebBcT0Y0TW2wTIYE8iscFjhcMVfvtzI/83fwN1n\ndeeh83q6kuFISTnTl+fw6uLN5BUWc3pyG35+Xk/6J7Z0JY8xgeJ4hcNaEI1PfLVhN88s2MBPBnbg\nZ+N6uJajWXgIN5/WhX/9fAy/urA3a3ceZMIL33Dv+yvJO1jsWi5jApkdcZh6t/PAUS7887+Jj45g\n1l2jiArzn6u+DxWX8bfFm3l58WZCgoR7xyZz8+guhIXY31DGVGVHHKbBVFYqP//nKsrKK3n5uiF+\nVTQAoiNCeXBcTxY8cCaju7fhyTnrGf/8YpZv3ed2NGMChhUOU6/eXbqNbzft5VcXpdClTTO349Sq\nU2wUf7s+lb/fNJTS8kqufOU7Hv8kk6OlFSde2ZgmzgqHqTfb9xbxxJz1nJ7chslDO554BT9wVs+2\nzL3/DK4d3pk3vtnC+Xb0YcwJWeEw9ebR2WsJEuGpy/r77F4NX2gWHsLvftKX9346nApVJr3yHc8v\n2EhFZeNv/zPmZFjhMPXiy3X5LMrazf3nJNOhZWB2PjiqWxvm3HcGEwcm8OyCDVz9tyV25ZUxNbDC\nYU5ZSXkFj3+aSbe4Zlw/MsntOKekeXgIz04ayNNXDGD1joOc//xiFmUVuB3LGL9ihcOcste/3sK2\nvUU8NqFPo7ms9fIhiXx672m0i4nk5jeX89K/smkKl64b443G8VtuXHOgqJS//msTY3u15fTkOLfj\n1Ktucc358I5RXNy/A3/8Iou731tJUWm527GMcZ0VDnNKXlm8mcMl5a51KeJrkWHBPD95IA9f0Is5\na3O59KVvydlX5HYsY1xlhcOctIJDxfz9my1MGNCB3u1buB3HZ0SEW8/oxt9vGsauA0e55KVvWJVz\nwO1YxrjGCoc5aS8uzKa8QnngHPf6ompIZ/aIY9Zdo4kMC2byq0tYkJnvdiRjXGGFw5yU3INHeW/Z\ndq5I7UiSH98hXt887R6jSY5vzq1vp/H2km1uRzKmwVnhMCfltX9voVLhzjHd3I7S4OKiw5l26wjO\n6tmWX3+0lj9+sd6uuDJNihUOU2f7j5Ty/rLtTBjQocZxwZuCqLAQXrluCFcN68RL/9rEo7MzqLQ7\nzU0T4V9dl5qA8I/vtlFUWsHtZza9o42qQoKD+MMlfWkREfLD1WV/vKw/IcH295hp3KxwmDopKi3n\nzW+3cE7vtvRsF+12HNeJCFPO70Xz8BD+b/4GikoqeP6qgYSHBLsdzRifsT+NTJ1MX57D/qIy7hjT\n3e0ofkNEuGdsMr++KIUvMvK49R8rKC6z7tlN42WFw3itslJ589utDOnciiGdW7kdx+/ccloXnrqs\nH4s37ua2t1dQUm7FwzROVjiM1/61oYBte4u4cVSS21H81qShnXjikn58tWE3d7zzvRUP0yhZ4TBe\ne/PbbcS3CGd833ZuR/Frk4d14veX9GXh+gLufm8lpeWVbkcypl5Z4TBe2bT7MIs37Oaa4Z0JtauG\nTuia4Z15fGIf5mfmc+/7KymrsOJhGg/7BjBeefu7bYQFB3HVsE5uRwkY149M4jdOg/lD/1xl93mY\nRsMuxzUndKi4jH+m5XBR//bERYe7HSeg3HxaF46WVfCnuVm0igrj0YtTAmpYXWNqYoXDnNBH6bs4\nUlrB9dYoflLuHNON/UdKee3rLbRuFsa9Y5PdjmTMKbHCYU5o+vLtpLRvwYDEGLejBCQR4eELerO/\nqIxn5m+gVVQo1wX4ELumabM2DnNca3ceZO3OQiYP62inWE5BUJDw1GX9OKd3PL+ZncHsVbvcjmTM\nSbPCYY5r+vIcwkOCmDggwe0oAS8kOIgXrh7E0KTWPDg9nW+y97gdyZiTYoXD1OpoaQUfpe/kgn7t\niYkKdTtOoxARGsxrN6TSLa45t7+9gqy8Q25HMqbOfFo4RGS8iGSJSLaITKlhfriITHfmLxWRpCrz\npjrTs0TkvGrrBYvIShH51Jf5m7o5a3M5VFzOpKEd3Y7SqLSICOWNm4YSGRbMzW8up6Cw2O1IxtSJ\nzwqHiAQDLwLnAynAVSKSUm2xW4D9qtodeBZ4ylk3BZgM9AHGAy852zvmPmCdr7Ibj2nLc0iKjWJ4\nl9ZuR2l0ElpG8saNQ9lfVMrNby3nSEm525GM8ZovjziGAdmqullVS4FpwMRqy0wE3nKezwTGiqcF\ndiIwTVVLVHULkO1sDxFJBC4EXvNh9iZv8+7DLNuyj0lDO1mjuI/0TYjhxasHk7mrkHveX0m53V1u\nAoQvC0cCkFPl9Q5nWo3LqGo5cBCIPcG6zwG/AI77WyYit4pImoik7d69+2T3ocmauWIHQQKXDbZG\ncV86q1dbHp/o6dfqt59k2hC0JiAEVOO4iFwEFKjqihMtq6qvqmqqqqbGxcU1QLrGo7JS+Th9F6cn\nx9G2RYTbcRq9a0d05rYzuvL2km289e1Wt+MYc0K+LBw7gaqtqonOtBqXEZEQIAbYe5x1RwMTRGQr\nnlNfZ4vIO74I35Qt27qPnQeOcqkdbTSYX47vxbkp8fzus3V8vdEu0zX+zZeFYzmQLCJdRCQMT2P3\n7GrLzAZucJ5fDixUz7H6bGCyc9VVFyAZWKaqU1U1UVWTnO0tVNVrfbgPTdJHK3cSFRbMuSnxbkdp\nMoKChGcnDaR7XHPueu97tuw54nYkY2rls8LhtFncDczFcwXUDFXNEJHHRWSCs9jrQKyIZAMPAlOc\ndTOAGUAm8AVwl6raiDgNoLisgs/W5DK+bzuiwqxHmobUPDyE125IJUjgp/9Io7C4zO1IxtRImkJj\nXGpqqqalpbkdIyB8tjqXu977nrdvGcbpydY25IbvNu3luteXcnpyG167YSjBQXZVm2l4IrJCVVNr\nmhdQjePG92at3El8i3BGdWvjdpQma2S3WB6b0IdFWbv549z1bscx5kescJgf7DtSyr+yCpg4MMH+\nynXZtSM6c+2ITrzy1WZmrdzhdhxj/osVDvODz1bvorxS+clAu5rKHzx6cR+Gd2nN1A/XkLmr0O04\nxvzACof5wayVO+kZH01KhxZuRzFAaHAQL1w9mJjIUG5/ZwUHi6yx3PgHKxwGgJ0HjvL99gNMGNjB\n7SimirjocF66Zgi5B4/ywIx0G7fc+AUrHAaAz1fnAnBhv/YuJzHVDencil9flMLC9QW8sCjb7TjG\nWOEwHp+uyaVvQguS2jRzO4qpwXUjOnPpoASeXbCBRVkFbscxTZwVDkPOviJW5Rzgov52mspfiQi/\nv6QfPeOjuX9aOjn7ityOZJowKxyGz9bYaapAEBkWzCvXDUFVuf2dFRSXWWcKxh1WOAyfrc5lQGIM\nHVtHuR3FnEDn2GY8N3kgGbsK+e0nmW7HMU2UFY4mbuueI6zZedBOUwWQs3vFc8eYbry/bDsfp1fv\ncNoY37PC0cQdO011QX87TRVIfnZuD1I7t+LhD9ewefdht+OYJsYKRxP32epcBndqSULLSLejmDoI\nCQ7iL1cPIiwkiLveW2ntHaZBWeFowjbvPkxmbiEX2mmqgNQ+JpJnJg1kXW4hj39q7R2m4VjhaMI+\ns5v+At5ZPdty+5ndeG/pdmav2uV2HNNEWOFowj5fm0dq51a0i7FxxQPZz8b1YEjnVkz9YLWNHGga\nhBWOJipnXxHrcgsZ37ed21HMKQoNDuIvVw0iNCSIO9/93to7jM9Z4Wii5mXmA9i44o1Eh5aRPHPl\nANblFvI7a+8wPmaFo4mal5FHr3bRdI61vqkai7N7xXPrGV15d+l25mbkuR3HNGJWOJqg/UdKWb51\nnx1tNEIPjetJv4QYfvnBanIPHnU7jmmkvCocInKaiNzkPI8TkS6+jWV86cv1BVQqjEux9o3GJiwk\niOcnD6S0vJIHpqdTYeN3GB84YeEQkUeBXwJTnUmhwDu+DGV8a15GHu1jIuibYCP9NUZd45rz2IQ+\nLNm8j5e/2uR2HNMIeXPEcQkwATgCoKq7gGhfhjK+c7S0gsUbdzMuJR4RcTuO8ZErhiRyUf/2PDN/\nAyu373c7jmlkvCkcpaqqgAKIiLWmBrCvs/dQXFbJuXaaqlE7Nn5HuxYR3DctnUPFNl65qT/eFI4Z\nIvIK0FJEfgosAF7zbSzjK/My8oiOCGF419ZuRzE+FhMZyvOTB7JjfxG/+TjD7TimETlh4VDVp4GZ\nwAdAT+A3qvpnXwcz9a+8opIF6/IZ26stocF2QV1TkJrUmvvG9mDWyp3MWrnD7TimkfCmcfwpVZ2v\nqj9X1YdUdb6IPNUQ4Uz9WrFtP/uLyhjXx05TNSV3ndWNoUmt+PVHGWzfa0POmlPnzZ+d59Yw7fz6\nDmJ8b15mPmHBQZzRI87tKKYBhQQH8dzkQYjAvdNWUlZR6XYkE+BqLRwicoeIrAF6isjqKo8twOqG\ni2jqg6oyPzOf0d1jaR4e4nYc08ASWkby5KX9Sc85wF8WZrsdxwS4432DvAfMAZ4AplSZfkhV9/k0\nlal3WfmH2L6viDvGdHM7inHJhf3b8+X6BF5clM2YnnEM7tTK7UgmQNV6xKGqB1V1q6peparbgKN4\nLsltLiKdvNm4iIwXkSwRyRaRKTXMDxeR6c78pSKSVGXeVGd6loic50yLEJFlIrJKRDJE5Ld13N8m\na15GPiIwtndbt6MYFz02oQ/tWkTwwPR0jpSUux3HBChvGscvFpGNwBbgK2ArniORE60XDLyIpz0k\nBbhKRFKqLXYLsF9VuwPPAk8566YAk4E+wHjgJWd7JcDZqjoAGAiMF5ERXuxnkzcvM49BHVvSNtrG\n3mjKWkSE8uykgWzfV8T/fma96JqT403j+P8CI4ANqtoFGAss8WK9YUC2qm5W1VJgGjCx2jITgbec\n5zOBseK5nXkiME1VS1R1C5ANDFOPw87yoc7DOuM5gV0HjrJ2Z6FdTWUAGNalNbed0Y33l+Uw3+le\n35i68KZwlKnqXiBIRIJUdRGQ6sV6CUBOldc7nGk1LqOq5cBBIPZ464pIsIikAwXAfFVdWtMPF5Fb\nRSRNRNJ2797tRdzG69iXwzjrDdc4Hjg3md7tWzDlg9XsOVzidhwTYLwpHAdEpDmwGHhXRJ7H6bfK\nDapaoaoDgURgmIj0rWW5V1U1VVVT4+Ka9uWn8zLz6BbXjK5xzd2OYvxEeEgwz08eyKGScqZ8sBpP\nr0LGeMebwjERKAIeAL4ANgEXe7HeTqBjldeJzrQalxGRECAG2OvNuqp6AFiEpw3E1OJgURlLNu+z\n01TmR3rER/PL8b1YsK6AactzTryCMQ5vuhw5oqqVqlquqm8BL+Ddl/VyIFlEuohIGJ7G7tnVlpkN\n3OA8vxxY6HSoOBuY7Fx11QVIBpY5Y4G0BBCRSDw3J673IkuTtSirgIpKtdNUpkY3jUpidPdYfvdp\nJlv3uHYiwQSY490A2MK5JPYFERknHncDm4ErT7Rhp83ibmAusA6YoaoZIvK4iExwFnsdiBWRbOBB\nnPtFVDUDmAFk4jnKuUtVK4D2wCIRWY2nMM1X1U9PbtebhnmZebSNDmdAYku3oxg/FBQkPH3FAEKC\nhPunp1Nud5UbL0ht5zZF5GNgP/Adniup2gIC3Keq6Q2WsB6kpqZqWlqa2zEaXHFZBYN/N5+fDErg\nD5f0czuO8WOfrNrFPe+v5IFzenDfOcluxzF+QERWqGqNF0Id787xrqraz9nAa0Au0ElVi32Q0fjA\nt5v2UFRaYaepzAldPKADX67L588LN3JmzzgGdrQjVFO747Vx/DDyi3OaaIcVjcAyPzOf5uEhjOwW\n63YUEwB+O7Ev8dHhPDA9naJSu6vc1O54hWOAiBQ6j0NA/2PPRaSwoQKak1NR6enUcEzPOMJDgt2O\nYwJATGQoT185gK17j/CHz9e5Hcf4seP1VRWsqi2cR7SqhlR53qIhQ5q6S8/Zz57DpZxrp6lMHYzq\n1ob/Oa0L7yzZzqKsArfjGD9lw8A1UvMy8gkNFs7qZZ0amrr52bie9IyP5hczV7P/SKnbcYwfssLR\nCKkq8zLzGdE1lhYRoW7HMQEmIjSYZycN5EBRKQ/PWmN3lZsfscLRCG3afZgte47Y1VTmpKV0aMGD\n5/Zkzto8Zq2s3uGDaeqscDRCczM8nRqeY4XDnIJbz+jK0KRWPPpxBjsPHHU7jvEj3ozHcajK1VXH\nHjkiMktEujZESFM38zLzGZAYQ/uYSLejmAAWHCQ8c+VAKlX52Yx0KivtlJXx8OaI4zng53i6NU8E\nHsIzrOw04A3fRTMnI7+wmFU5B6xTQ1MvOraO4tGL+7Bk8z7e+GaL23GMn/CmcExQ1VdU9ZCqFqrq\nq8B5qjodsEGL/cyxsTfsMlxTX65ITeTclHj++EUWWXmH3I5j/IA3haNIRK4UkSDncSVw7A5yO3b1\nM/My80mKjSK5rY29YeqHiPDEpf1oERnC/dPTKSmvcDuScZk3heMa4Do8I+7lO8+vdbo1v9uH2Uwd\nFRaX8d2mPYzr0w7PCLzG1I82zcN58tL+rMst5LkFG92OY1x2vE4OAVDVzdQ+cNPX9RvHnIqvsnZT\nVmFjbxjfOCclnslDO/LyV5s4u1dbhia1djuScYk3V1XFicjDIvKqiLxx7NEQ4UzdzMvMJ7ZZGIM6\nWdOT8Y1fXZRCx1ZRPDgjncMl1hFiU+XNqaqP8QzpugD4rMrD+JGS8goWrS/gnN7xBAfZaSrjG83D\nQ3jmygHs3H+U332S6XYc45ITnqoColT1lz5PYk7Jks37OFxSzrg+dprK+FZqUmvuGNONFxdtYmzv\ntnbpdxPkzRHHpyJygc+TmFMyLyOPqLBgRndv43YU0wTcN7YHfTq0YOqHa9h9qMTtOKaBeVM47sNT\nPI7aeBz+qbJSWbAunzOS44gItbE3jO+FhQTx7KSBHCopZ+qHq60jxCbmhIXDGX8jSFUjbTwO/7R6\n50HyC0vsNJVpUD3io/nl+F4sWFfAjLQct+OYBlRrG4eI9FLV9SIyuKb5qvq972KZupiXkUdwkHC2\njb1hGthNo5L4cl0+v/0kkxFdY+kc28ztSKYBHO+I40Hn3/+r4fG0j3OZOpiXmc/wLq1pGRXmdhTT\nxAQFCX+6YgDBQcKDM1ZRYR0hNgnHGzr2Vuffs2p4nN1wEc3xbNlzhOyCw9Y3lXFNQstIfjexLyu2\n7eflrza5Hcc0AG8ux0VERgFJVZdX1X/4KJOpg7kZeYB1amjcNXFgB+avy+e5BRs4s0ccfRNi3I5k\nfMibO8ffxnNq6jRgqPNI9XEu46V5GXn0TWhBYqsot6OYJkxE+P1P+tIqKowHpqdTXGYdITZm3lyO\nmwqMVtU7VfUe53Gvr4OZEysoLOb77QcYl2I3YBn3tYwK409XDGBjwWH+NDfL7TjGh7wpHGsB+2by\nQ/PXecbeOM/u3DV+4swecVw/sjOvf72Fb7P3uB3H+Ig3haMNkCkic0Vk9rGHr4OZE5ubkU/n2Ch6\nxNvYG8Z/TD2/N13bNOOhf67i4NEyt+MYH/CmcfwxX4cwdXds7I2bRnexsTeMX4kMC+bZSQO59K/f\n8tjsDJ6dNNDtSKaeHbdwiEgw8JiqntVAeYyXFq0vsLE3jN8a0LEl95zdnecWbGRs77Zc1L+D25FM\nPTruqSpVrQAqRcSurfMz8zLzadPcxt4w/uuus7ozoGNLHpm1lvzC4hOvYAKGN20ch4E1IvK6iPz5\n2MObjYvIeBHJEpFsEZlSw/xwEZnuzF8qIklV5k11pmeJyHnOtI4iskhEMkUkQ0Tu8243G5eS8gr+\ntb6Ac1Ns7A3jv0KDg3j2ygGUlFfw85nWEWJj4k3h+BD4NbAYWFHlcVzOaa4XgfOBFOAqEUmpttgt\nwH5V7Q48CzzlrJsCTAb6AOOBl5ztlQM/U9UUYARwVw3bbPS+zd7LkdIKuwzX+L2ucc155ILeLN6w\nm3eWbHM7jqkn3ow5/tZJbnsYkO2MWY6ITAMmAlWHDZvIfxrfZwIviKeldyIwTVVLgC0ikg0MU9Xv\ngFwn1yERWQckVNtmozcvM49mYcGM6h7rdhRjTujaEZ2Zv66A33++jlHd29Atzq4CDHTe3DmeLCIz\nndNDm489vNh2AlC1r+UdzrQal1HVcuAgEOvNus5prUHA0lpy3yoiaSKStnv3bi/iBoaKSmV+Zj5j\nerUlPMTG3jD+T0T40+X9iQgN5sHp6ZRVVLodyZwib05V/R34K57TRGcB/wDe8WWoExGR5sAHwP2q\nWuOgUqr6qqqmqmpqXFxcwwb0oZXb97PncKnd9GcCSnyLCH7/k36s2nGQFxdlux3HnCJvCkekqn4J\niKpuU9XHgAu9WG8n0LHK60RnWo3LiEgIEAPsPd66IhKKp2i8q6ofepGjUZmbkUdosDCmZ+MphqZp\nuLB/ey4ZlMBfFmaTnnPA7TjmFHhTOEpEJAjYKCJ3i8glgDcnKZcDySLSRUTC8DR2V7/jfDZwg/P8\ncmChei69mA1Mdq666gIkA8uc9o/XgXWq+owXGRoVVWVeZj6jurWhRUSo23GMqbPHJvQhPjqcB6an\nU1Ra7nYcc5K8HXM8CrgXGAJcy3++7GvltFncDcwF1gEzVDVDRB4XkQnOYq8DsU7j94PAFGfdDGAG\nnkbvL4C7nHtKRgPXAWeLSLrzuMDrvQ1wWfmH2La3yIaINQErJjKUp68cwJY9R3ji8/VuxzEnyZur\nqpYDiEilqt5Ul42r6ufA59Wm/abK82LgilrW/T3w+2rTvgaa7I0Lc9bkIWJjb5jANqpbG245rQuv\nf72Fsb3bMqanDXkcaLy5qmqkiGQC653XA0TkJZ8nMz8yZ20uQ5Na0zY6wu0oxpySn5/Xk+S2zfnF\nzNXsP1LqdhxTR96cqnoOOA9PozWqugo4w5ehzI9lFxxiQ/5hLuzX3u0oxpyyiFBPR4j7i0p55KM1\ndld5gPGmcKCqOdUm2fBeDezzNZ4hYsf3tctwTePQNyGGB87twedr8vgovfoFl8afeVM4cpwxx1VE\nQkXkITyN3aYBfb4ml9TOrYhvYaepTONx2xndSO3cit98lMH2vUVuxzFe8qZw3A7chefO7Z3AQOBO\nX4Yy/23T7sOszzvEBXaayjQywUHiGa9D4J5pK+2u8gBxwsKhqntU9RpVjVfVtqp6LXB9A2Qzji/W\nek5Tnd/PTlOZxqdj6yievLQ/q3IO8PQ8G6s8EHjVxlGDB+s1hTmuz1bnMrhTS9rHRLodxRifuLB/\ne64e3olXvtrM4g2Np2+5xupkC0eTvZeioW3dc4TM3EI7TWUavd9clEKP+OY8OCOdgkM28JM/O9nC\nYdfONZDP1+YCdjWVafwiQoN54erBHC4p52czVlFZaV8z/qrWwiEih0SksIbHIcAGEG4gc9bkMaBj\nSxJbRbkdxRif6xEfzW8u6sO/N+7hlcXejN5g3FBr4VDVaFVtUcMjWlVP2FWJOXU5+4pYs/MgF9jR\nhmlCrhrWkQv7tef/5mXx/fb9bscxNTjZU1WmAXyyeheAtW+YJkVE+MOl/YhvEcG976/k4NEytyOZ\naqxw+LHZ6bsY0rkVHVvbaVukvtQAABTpSURBVCrTtMREhvKXqweRe7CYhz+0Lkn8jRUOP5WVd4j1\neYeYMMCak0zTNLhTK342rgefrcnlnaXb3Y5jqrDC4admr9pJcJDYaSrTpN1+RjfG9Izjd59ksnqH\njRroL6xw+CFVZfaqXYzqFktcdLjbcYxxTVCQ8OyVA4mLDueOd77nQJF1we4PrHD4oZU5B8jZd5SJ\nAxPcjmKM61o1C+OFqwdRcKjY7u/wE1Y4/NDs9F2EhQRxng0RawwAgzq14lcXpvDl+gJeXrzJ7ThN\nnhUOP1NeUcmnq3MZ26st0RGhbscxxm9cP7IzF/Vvz9Nzs/hu01634zRpVjj8zHeb97LncAkTB9rV\nVMZUJSI8eVl/kto04573V1JQaP1ZucUKh5/5OH0X0eEhjOnZ1u0oxvid5uEh/PWaIRwuKeOe91dS\nbuN3uMIKhx85UlLO52tyOb9fOyJCg92OY4xf6tkumj9c0o+lW/bxp7k2focbrHD4kTlr8ygqreCK\n1I5uRzHGr106OJFrhnfilcWb+WTVLrfjNDlWOPzIzBU5JMVGkdq5ldtRjPF7j17ch9TOrfj5zFVk\n7ip0O06TYoXDT+TsK2LJ5n1cPiQRERsny5gTCQsJ4qVrB9MyMoxb305j/xG7ObChWOHwEx98vwMR\nuGRwottRjAkYbaMjePm6IRQcKuHu97+3xvIGYoXDD1RWKh98v4PR3dqQ0NLGFTemLgZ2bMn//qQv\n32Tv5ck5692O0yRY4fADy7buI2ffUS4fYkcbxpyMK1M7cuOoJF77eguzVu5wO06jZ4XDD/wzbQfN\nw0M4r4+N9GfMyXrkwt4M79KaKR+ssZ50fcwKh8sOFpXx2ZpdXDygA5Fhdu+GMScrNDiIF68ZTJvm\n4fzPW2nkHjzqdqRGy6eFQ0TGi0iWiGSLyJQa5oeLyHRn/lIRSaoyb6ozPUtEzqsy/Q0RKRCRtb7M\n3lA+XLmD4rJKrhneye0oxgS8Ns3DeePGoRSVVnDLm2kcKSl3O1Kj5LPCISLBwIvA+UAKcJWIpFRb\n7BZgv6p2B54FnnLWTQEmA32A8cBLzvYA3nSmBTxV5d2l2xnYsSV9E2LcjmNMo9CzXTQvXD2I9XmF\n3DdtJRXWDXu98+URxzAgW1U3q2opMA2YWG2ZicBbzvOZwFjx3MQwEZimqiWqugXIdraHqi4G9vkw\nd4NZtmUf2QWHudqONoypV2N6tuWxCX1YsK6AJz5f53acRseXhSMByKnyeoczrcZlVLUcOAjEernu\ncYnIrSKSJiJpu3fvrmP0hvHu0u1ER4RwcX/rCdeY+nb9yKQfrrR6z8Ysr1eNtnFcVV9V1VRVTY2L\ni3M7zo/sOVzCnLW5XDY40RrFjfGRX13YmzE94/j1x2v590b//AMyEPmycOwEqvbWl+hMq3EZEQkB\nYoC9Xq4b0Gak5VBWodYobowPhQQH8ZerBpHctjl3vPM9a3cedDtSo+DLwrEcSBaRLiIShqexe3a1\nZWYDNzjPLwcWqqo60yc7V111AZKBZT7M2qDKKir5x7fbGN09luT4aLfjGNOoRUeE8uZNw4iJDOXG\nvy9n+94ityMFPJ8VDqfN4m5gLrAOmKGqGSLyuIhMcBZ7HYgVkWzgQWCKs24GMAPIBL4A7lLVCgAR\neR/4DugpIjtE5BZf7YOvfL4ml7zCYm45rYvbUYxpEtrFRPDWzUMpr6zk+jeWsudwiduRApp4/sBv\n3FJTUzUtLc3tGIDnEtyJL37D4ZJyFjxwJkFB1hOuMQ1lxbb9XPPaEnrGR/PeT0fQLDzE7Uh+S0RW\nqGpqTfMabeO4v1q+dT+rdxzkptFdrGgY08CGdG7FC1cNZu2uQu5493tKy6033ZNhhaOBvf71ZlpG\nhXLZ4DpdXWyMqSfnpMTzh0v6snjDbh765yq7QfAk2HFaA9q0+zDzMvO548xuRIXZW2+MWyYN7cS+\nI2U89cV6IkODeeLSfnYGoA7s26sBvbgom/CQIG62RnFjXHfHmG4cLS3nzwuziQgN4rEJfWz0TS9Z\n4Wgg2/cW8XH6Lm4YmUSb5uFuxzHGAA+c24OjZRX87d9biAgNZsr5vax4eMEKRwP561fZBItw25ld\n3Y5ijHGICA9f0JujZRW8sngzkWHB3H9OD7dj+T0rHA1g14GjzFyxg0lDOxLfIsLtOMaYKkSExyf0\npbiskucWbCRIhHvHJrsdy69Z4WgAzy/YCMDtZ3ZzOYkxpiZBQcJTl/WnUpVn5m+gtLySn43rYaet\namGFw8c25h/inytyuGFUEomtotyOY4ypRXCQ8PTlAwgLDuKFRdmUlFfw8AW9rXjUwAqHj/1xbhZR\nYSHcc7Yd+hrj74KChD9c0o/wkCD+9u8tlJRX8tjFfexS3WqscPhQ2tZ9zM/M56FxPWjdLMztOMYY\nLwQFCY9N6EOYUzyKyyr4wyX9CAm2+6WPscLhIxWVymOfZBDfItzu2zAmwBy72ioyLIQ/f7mRfUdK\n+ctVg23sHIeVUB95d+k21u4s5JELU+wucWMCkIjw4Lk9+N3EPny5voBrXlvC/iOlbsfyC1Y4fGD3\noRL+NDeLUd1iubh/e7fjGGNOwXUjk3jpak/HiJe//C079tt4HlY4fODxTzMpLqvg8Yl97YoMYxqB\n8/u15+2bh1FwqIRLX/qWVTkH3I7kKisc9eyTVbv4ZNUu7jk7me5tm7sdxxhTT4Z3jWXm7aMICwni\nyle+4+P0RjWadZ1Y4ahH+YXF/PrjtQzo2JI7x9jNfsY0Nj3bRfPxXaMZkNiS+6al86e566lsgt2y\nW+GoJ2UVldz7/kqKyyp45soBdumeMY1UbPNw3vmf4Uwe2pEXF23itndWUFhc5nasBmXfbvXkic/X\ns3TLPp64tB/d4uwUlTGNWVhIEE9c2o9HL05h4foCLv7L16zdedDtWA3GCkc9mLZsO298s4UbRyVx\nyaBEt+MYYxqAiHDT6C7MuG0EpeWVXPrSt7y9ZBuqjf/UlRWOUzRnTS4Pz1rDmT3ieOTC3m7HMcY0\nsCGdW/PZvaczqnssv/5oLfe8v5KDRY371JUVjlPwcfpO7p22ksGdWvHytUMItXYNY5qk1s3CeOOG\nofxifE++WJvHuOe+4l9ZBW7H8hn7pjsJlZXKCws3ct+0dIZ0bsUbNw21rgiMaeKCgoQ7x3Rn1p2j\niYkM5ca/L2fqh2s4XFLudrR6Z4WjjrbsOcJ1byzl6XkbmDCgA2/dPIwWEaFuxzLG+Il+iTHMvvs0\nbjujK9OWb+e8ZxczLyOvUbV9SGPamdqkpqZqWlraSa9fUams3nGA6ctzmLliB2EhQfzmohQmDe1o\nd4YbY2q1Yts+Hv5wLVn5hxjbqy2PTehDx9aBMS6PiKxQ1dQa51nhqJmqcuPfl7P3SAnb9hZxqLic\niNAgLh2cyP3nJNM22oaANcacWFlFJW9+s5XnFmygvFK57Yyu/PSMrkT7+ZmK4xUO67a1FiJCpSpt\noyPon9iSoUmtGNs73k5LGWPqJDQ4iJ+e0ZWLBrTnD5+v588Ls3ln6XbuObs7Vw/vRHhI4LWP2hGH\nMcY0oNU7DvDknPV8u2kvia0iue2MrlyR2pGIUP8qIHaqygqHMcaPqCr/3riHZxdsYOX2A8Q2C+PG\nUUlcO6IzrfxktFArHFY4jDF+SFVZtmUfL3+1iUVZuwkLDmJ833ZMHtaRkV1jXb34xto4jDHGD4kI\nw7vGMrxrLFl5h3hv6TZmrdzJ7FW76NQ6igv6tef8vu3onxjjV1dw+vSIQ0TGA88DwcBrqvpktfnh\nwD+AIcBeYJKqbnXmTQVuASqAe1V1rjfbrIkdcRhjAkVxWQVz1uby4fc7+W7TXsorlYSWkZye3IaR\n3WIZ2TWWti18f1WnK6eqRCQY2ACcC+wAlgNXqWpmlWXuBPqr6u0iMhm4RFUniUgK8D4wDOgALAB6\nOKsdd5s1scJhjAlEB4pKWbCugLkZeSzZvJdDxZ670BNbRdK7fQt6t29Bj/jmJLSMpEPLSOKahxMU\nVD9HJm6dqhoGZKvqZifENGAiUPVLfiLwmPN8JvCCeI7HJgLTVLUE2CIi2c728GKbxhjTKLSMCuPy\nIYlcPiSRikolc1ch323ew+odB1mXW8iX6/KpOo5UcJAQFRbsPEKIax7OjNtH1nsuXxaOBCCnyusd\nwPDallHVchE5CMQ605dUWzfBeX6ibQIgIrcCtwJ06tTp5PbAGGP8RHCQ0C8xhn6JMT9MO1pawZY9\nR8g9eJRdB4vJP1jM4ZJyjpZWcKS0nObhvvmKb7SN46r6KvAqeE5VuRzHGGPqXWRYMCkdWpDSoUWD\n/lxfdnK4E+hY5XWiM63GZUQkBIjB00he27rebNMYY4wP+bJwLAeSRaSLiIQBk4HZ1ZaZDdzgPL8c\nWKie1vrZwGQRCReRLkAysMzLbRpjjPEhn52qctos7gbm4rl09g1VzRCRx4E0VZ0NvA687TR+78NT\nCHCWm4Gn0bscuEtVKwBq2qav9sEYY8yP2Z3jxhhjfuR4l+PaQE7GGGPqxAqHMcaYOrHCYYwxpk6s\ncBhjjKmTJtE4LiK7gW0nuXobYE89xmlogZ4fbB/8RaDvQ6Dnh4bdh86qGlfTjCZROE6FiKTVdmVB\nIAj0/GD74C8CfR8CPT/4zz7YqSpjjDF1YoXDGGNMnVjhOLFX3Q5wigI9P9g++ItA34dAzw9+sg/W\nxmGMMaZO7IjDGGNMnVjhMMYYUydWOGohIuNFJEtEskVkitt5qhKRjiKySEQyRSRDRO5zprcWkfki\nstH5t5UzXUTkz86+rBaRwVW2dYOz/EYRuaG2n+mj/QgWkZUi8qnzuouILHVyTne6zsfpXn+6M32p\niCRV2cZUZ3qWiJzXwPlbishMEVkvIutEZGQAfgYPOP+H1orI+yIS4e+fg4i8ISIFIrK2yrR6e99F\nZIiIrHHW+bOI1M8g3sfP/yfn/9FqEZklIi2rzKvxva3tO6q2z69eqao9qj3wdNm+CegKhAGrgBS3\nc1XJ1x4Y7DyPBjYAKcAfgSnO9CnAU87zC4A5gAAjgKXO9NbAZuffVs7zVg24Hw8C7wGfOq9nAJOd\n5y8DdzjP7wRedp5PBqY7z1OczyYc6OJ8ZsENmP8t4H+c52FAy0D6DPAMx7wFiKzy/t/o758DcAYw\nGFhbZVq9ve94xv4Z4awzBzi/AfKPA0Kc509VyV/je8txvqNq+/zqdR8a4j9ooD2AkcDcKq+nAlPd\nznWcvB8D5wJZQHtnWnsgy3n+CnBVleWznPlXAa9Umf5fy/k4cyLwJXA28KnzS7qnyi/PD58BnvFX\nRjrPQ5zlpPrnUnW5Bsgfg+dLV6pND6TPIAHIcb48Q5zP4bxA+ByApGpfvPXyvjvz1leZ/l/L+Sp/\ntXmXAO86z2t8b6nlO+p4v0f1+bBTVTU79gt1zA5nmt9xThcMApYC8aqa68zKA+Kd57Xtj5v7+Rzw\nC6DSeR0LHFDV8hqy/JDTmX/QWd7N/F2A3cDfndNtr4lIMwLoM1DVncDTwHYgF8/7uoLA+hyOqa/3\nPcF5Xn16Q7oZz5EO1D3/8X6P6o0VjgAmIs2BD4D7VbWw6jz1/Lnhl9dai8hFQIGqrnA7yykIwXO6\n4a+qOgg4gucUyQ/8+TMAcNoBJuIpgh2AZsB4V0PVA39/349HRB7BM+rpu25nOR4rHDXbCXSs8jrR\nmeY3RCQUT9F4V1U/dCbni0h7Z357oMCZXtv+uLWfo4EJIrIVmIbndNXzQEsROTaccdUsP+R05scA\ne3H3c9oB7FDVpc7rmXgKSaB8BgDnAFtUdbeqlgEf4vlsAulzOKa+3vedzvPq031ORG4ELgKucYof\n1D3/Xmr//OqNFY6aLQeSnasTwvA0BM52OdMPnKs8XgfWqeozVWbNBo5dHXIDnraPY9Ovd64wGQEc\ndA7r5wLjRKSV89fnOGeaT6nqVFVNVNUkPO/tQlW9BlgEXF5L/mP7dbmzvDrTJztX+3QBkvE0bPqc\nquYBOSLS05k0FsgkQD4Dx3ZghIhEOf+nju1DwHwOVdTL++7MKxSREc57cn2VbfmMiIzHc+p2gqoW\nVduvmt7bGr+jnM+jts+v/viyASuQH3iuxtiA58qFR9zOUy3baXgOxVcD6c7jAjznN78ENgILgNbO\n8gK86OzLGiC1yrZuBrKdx00u7MsY/nNVVVfnlyIb+CcQ7kyPcF5nO/O7Vln/EWe/sqjnq1+8yD4Q\nSHM+h4/wXJ0TUJ8B8FtgPbAWeBvP1Tt+/TkA7+NpkynDc+R3S32+70Cq835sAl6g2gUQPsqfjafN\n4tjv88snem+p5Tuqts+vPh/W5Ygxxpg6sVNVxhhj6sQKhzHGmDqxwmGMMaZOrHAYY4ypEyscxhhj\n6sQKhzEnICKHnX+TROTqet72w9Vef1uf2zfGF6xwGOO9JKBOhaPKHby1+a/Coaqj6pjJmAZnhcMY\n7z0JnC4i6eIZxyLYGUdhuTOOwm0AIjJGRP4tIrPx3ImNiHwkIivEM/bFrc60J4FIZ3vvOtOOHd2I\ns+21ztgQk6ps+1/yn3FA3j02XoSIPCmeMVpWi8jTDf7umCbjRH8NGWP+YwrwkKpeBOAUgIOqOlRE\nwoFvRGSes+xgoK+qbnFe36yq+0QkElguIh+o6hQRuVtVB9bwsy7Fc2f6AKCNs85iZ94goA+wC/gG\nGC0i6/B0x91LVVWqDARkTH2zIw5jTt44PP0gpePp1j4WT19CAMuqFA2Ae0VkFbAET+d0yRzfacD7\nqlqhqvnAV8DQKtveoaqVeLqnSMLTxXkx8LqIXAoU1bBNY+qFFQ5jTp4A96jqQOfRRVWPHXEc+WEh\nkTF4eqIdqaoDgJV4+n06WSVVnlfgGbSnHBiGp5fei4AvTmH7xhyXFQ5jvHcIz1C9x8wF7nC6uEdE\nejiDOVUXA+xX1SIR6YVnWNJjyo6tX82/gUlOO0ocnuFGa+1x1hmbJUZVPwcewHOKyxifsDYOY7y3\nGqhwTjm9iWcMkSTge6eBejfwkxrW+wK43WmHyMJzuuqYV4HVIvK9erqWP2YWnmE/V+HpCfkXqprn\nFJ6aRAMfi0gEniOhB09uF405Mesd1xhjTJ3YqSpjjDF1YoXDGGNMnVjhMMYYUydWOIwxxtSJFQ5j\njDF1YoXDGGNMnVjhMMYYUyf/D6ubQpQIyIknAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qIFjGvb-qv6i",
        "colab_type": "text"
      },
      "source": [
        "### With bias, No 1-cycle policy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BpJkwyvDIUv4",
        "colab_type": "text"
      },
      "source": [
        "This gives us comparable results to TensorFlow without cycling learning rates... so te reason for the performance boost in fast.ai seems to depend mainly on the training/learning rate schedule "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9u7EiWi8qt6f",
        "colab_type": "code",
        "outputId": "929ac7f5-ec0b-417e-a58d-9b97b3d58992",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 200
        }
      },
      "source": [
        "learn = collab_learner(data, n_factors=embedding_size, wd=0, true_wd=False)\n",
        "learn.fit(5, 5e-3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.975569</td>\n",
              "      <td>1.024511</td>\n",
              "      <td>02:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.937285</td>\n",
              "      <td>1.019601</td>\n",
              "      <td>02:05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.895879</td>\n",
              "      <td>1.043847</td>\n",
              "      <td>02:06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.853240</td>\n",
              "      <td>1.053256</td>\n",
              "      <td>02:05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.845312</td>\n",
              "      <td>1.071324</td>\n",
              "      <td>02:05</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}