feature_selection_sequential.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this example we will see how to **select features** through **sequential selection**.\n",
    "\n",
    "As a toy example, we will use data from 'Titanic: Machine Learning for Disaster', one of the most popular Kaggle competitions. However, we will not use the original data set. We will use a modified data set, which results from a [Kaggle kernel](https://www.kaggle.com/pmarcelino/data-analysis-and-feature-extraction-with-python) that I did. \n",
    "\n",
    "[Here](https://github.com/pmarcelino/blog/data/titanic_modified.csv) you can access to the data set used in this exercise."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from mlxtend.feature_selection import SequentialFeatureSelector as SFS\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's load the data set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Fare</th>\n",
       "      <th>FamilySize</th>\n",
       "      <th>Imputed</th>\n",
       "      <th>Pclass_2</th>\n",
       "      <th>Pclass_3</th>\n",
       "      <th>Sex_male</th>\n",
       "      <th>Age_Child</th>\n",
       "      <th>Age_Elder</th>\n",
       "      <th>Embarked_Q</th>\n",
       "      <th>Embarked_S</th>\n",
       "      <th>Title_Miss</th>\n",
       "      <th>Title_Mr</th>\n",
       "      <th>Title_Mrs</th>\n",
       "      <th>Title_Other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived     Fare  FamilySize  Imputed  Pclass_2  Pclass_3  Sex_male  \\\n",
       "0         0   7.2500           1        0         0         1         1   \n",
       "1         1  71.2833           1        0         0         0         0   \n",
       "2         1   7.9250           0        0         0         1         0   \n",
       "3         1  53.1000           1        0         0         0         0   \n",
       "4         0   8.0500           0        0         0         1         1   \n",
       "\n",
       "   Age_Child  Age_Elder  Embarked_Q  Embarked_S  Title_Miss  Title_Mr  \\\n",
       "0          0          0           0           1           0         1   \n",
       "1          0          0           0           0           0         0   \n",
       "2          0          0           0           1           1         0   \n",
       "3          0          0           0           1           0         0   \n",
       "4          0          0           0           1           0         1   \n",
       "\n",
       "   Title_Mrs  Title_Other  \n",
       "0          0            0  \n",
       "1          1            0  \n",
       "2          0            0  \n",
       "3          1            0  \n",
       "4          0            0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/titanic_modified.csv', index_col=0)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As I mentioned, this data set results from a [kernel](https://www.kaggle.com/pmarcelino/data-analysis-and-feature-extraction-with-python) that I already did for the Titanic competition. I'll summarize each of the features to give you some context:\n",
    "\n",
    "* **Survived**. Target variable. It's 1 if the passenger survived and 0 if it didn't.\n",
    "* **Fare**. Passenger fare. It keeps the same properties as the original feature.\n",
    "* **FamilySize**. It's the sum of the original features SipSp and Parch. SipSp refers to the # of siblings / spouses aboard the Titanic. Parch refers to the # of parents / children aboard the Titanic.\n",
    "* **Imputed**. Identifies instances where some missing data imputation was made.\n",
    "* **Pclass**. Ticket class. The feature was [encoded](http://scikit-learn.org/stable/modules/preprocessing.html#preprocessing-categorical-features), so we just have the features corresponding to the second and third class. The one corresponding to the first class was deleted to avoid the [dummy trap](http://www.algosome.com/articles/dummy-variable-trap-regression.html).\n",
    "* **Sex**. It was also encoded. It's 1 if the instance corresponds to a male, and 0 if it corresponds to a female.\n",
    "* **Age**. Encoded to the following classes: Children, Adult, and Elder. It's an Adult if Age_Child = 0 and Age_Elder = 0.\n",
    "* **Embarked**. Port of embarkation. Originally, we had three possible ports: C = Cherbourg, Q = Queenstown, S = Southampton.\n",
    "* **Title**. Results from the name of the passenger. Guess what? It's also an encoded feature."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train and test data sets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In feature selection, our goal is to distinguish features that are useful for prediction from features that just add noise to the prediction model.\n",
    "\n",
    "To test the model's performance on unseen data, we need a train and a test data set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create train and test set\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = df.drop('Survived', axis=1)  # Keep all features except 'Survived'\n",
    "y = df['Survived']  # Just keep 'Survived'\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The solution we propose uses [mlxtend](http://rasbt.github.io/mlxtend/). Mlxtend is a Python library of useful tools for the day-to-day data science tasks. It was written by [Sebastian Raschka](https://sebastianraschka.com/), one of the data scientists that I follow.\n",
    "\n",
    "Sequential feature selection recursively removes or adds features based on the classifier performance. The recursion proceeds until we reach a feature subset of the desired size. We say that the recursion works 'backward' if we remove features, and we say that the recursion works 'forward' if we add features.\n",
    "\n",
    "For example, let's consider that we have 14 features and we want to reduce the feature subset to 10 features. Assume that we want to use sequential forward selection. In this case, we will start without features and, step by step, we will add one feature to the subset. This feature will be the one that brings the greatest increase to the classifier performance. We keep adding features until our feature subset has 10 features. \n",
    "\n",
    "Sequential backward selection works in the opposite way. We start with all features and, step by step, we remove the feature that least influences the classifier performance.\n",
    "\n",
    "In addition to the 'backward' and 'forward', mlxtend has a 'floating' option. Floating allow us to have an additional exclusion or inclusion step to remove features once they were included (or excluded). This step is conditional and only occurs if the resulting feature subset is assessed as \"better\" by the criterion function after removal (or addition) of a particular feature. Floating allow us to consider a wider range of combinations, which can provide us better results.\n",
    "\n",
    "Mlxtend offers four different ways to perform sequential feature selection:\n",
    "\n",
    "* Sequential forward selection.\n",
    "* Sequential backward selection.\n",
    "* Sequential forward floating selection.\n",
    "* Sequential backward floating selection.\n",
    "\n",
    "In this example we will test all of them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "lr = LogisticRegression()\n",
    "max_features = np.shape(X_train)[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Here we define the base classifier *lr* that will be used to solve the problem. Logistic regression was chosen to make this notebook coherent with previous notebooks ([x](), [x]()).\n",
    "* *max_features* is the number of features that we want to select."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential forward selection (12 features): \n",
      "(0, 1, 2, 4, 5, 6, 7, 8, 9, 11, 12, 13)\n",
      "CV score: \n",
      "0.8385295103957077\n"
     ]
    }
   ],
   "source": [
    "# Sequential forward selection\n",
    "sfs = SFS(lr,\n",
    "          k_features = (1,max_features), # Select the best feature combination in the range 1 to max_features\n",
    "          forward = True,\n",
    "          floating = False,\n",
    "          scoring = 'accuracy',\n",
    "          cv = 10,  # Performs stratified k-fold by default.\n",
    "          n_jobs = -1)\n",
    "sfs = sfs.fit(X_train.values, y_train) # Use .values because the method can't read DataFrame data structures.\n",
    "\n",
    "print('Sequential forward selection (%i features): ' % np.size(sfs.k_feature_idx_))\n",
    "print(sfs.k_feature_idx_)\n",
    "print('CV score: ')\n",
    "print(sfs.k_score_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* We create a sequential feature selection structure *SFS* that uses *lr* as an estimator, has *7 features*, does *forward* selection, *excludes* the conditional floating option, uses *accuracy* as the performance metrics, performs *cross-validation*, and uses all CPUs to evaluate the different subsests (n_jobs = -1).\n",
    "* We perform feature selection and learn the model using the training data.\n",
    "* We print the selected features and the performance score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential backward selection (4 features): \n",
      "(0, 1, 11, 13)\n",
      "CV score: \n",
      "0.8455130784708249\n"
     ]
    }
   ],
   "source": [
    "# Sequential backward selection\n",
    "sbs = SFS(lr, \n",
    "          k_features = (1,max_features),\n",
    "          forward = False, \n",
    "          floating = False, \n",
    "          scoring = 'accuracy',\n",
    "          cv = 10, \n",
    "          n_jobs = -1)\n",
    "sbs = sbs.fit(X_train.values, y_train)\n",
    "\n",
    "print('Sequential backward selection (%i features): ' % np.size(sbs.k_feature_idx_))\n",
    "print(sbs.k_feature_idx_)\n",
    "print('CV score: ')\n",
    "print(sbs.k_score_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* In this case we perform a backward selection, so we define that *forward = False*. Everything else remains equal."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential forward floating selection (10 features): \n",
      "(1, 2, 3, 4, 6, 8, 9, 11, 12, 13)\n",
      "CV score: \n",
      "0.8483501006036217\n"
     ]
    }
   ],
   "source": [
    "# Sequential forward floating selection\n",
    "sffs = SFS(lr, \n",
    "           k_features = (1,max_features), \n",
    "           forward = True, \n",
    "           floating = True, \n",
    "           scoring = 'accuracy', \n",
    "           cv = 10, \n",
    "           n_jobs = -1)\n",
    "sffs = sffs.fit(X_train.values, y_train)\n",
    "\n",
    "print('Sequential forward floating selection (%i features): ' % np.size(sffs.k_feature_idx_))\n",
    "print(sffs.k_feature_idx_)\n",
    "print('CV score: ')\n",
    "print(sffs.k_score_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* The main difference here is that we activate the *floating* option."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential backward floating selection (10 features): \n",
      "(1, 2, 3, 4, 6, 8, 9, 11, 12, 13)\n",
      "CV score: \n",
      "0.8483501006036217\n"
     ]
    }
   ],
   "source": [
    "# Sequential backward floating selection\n",
    "sbfs = SFS(lr, \n",
    "           k_features = (1,max_features), \n",
    "           forward = False,\n",
    "           floating = True,\n",
    "           scoring = 'accuracy', \n",
    "           cv = 10, \n",
    "           n_jobs = -1)\n",
    "sbfs = sbfs.fit(X_train.values, y_train)\n",
    "\n",
    "print('Sequential backward floating selection (%i features): ' % np.size(sbfs.k_feature_idx_))\n",
    "print(sbfs.k_feature_idx_)\n",
    "print('CV score: ')\n",
    "print(sbfs.k_score_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see, the classifier performs better when we use backward or forward floating selection.\n",
    "\n",
    "Let's visualize the results of the best performing solution in DataFrames and plots."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize results with DataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>avg_score</th>\n",
       "      <th>ci_bound</th>\n",
       "      <th>cv_scores</th>\n",
       "      <th>feature_idx</th>\n",
       "      <th>std_dev</th>\n",
       "      <th>std_err</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.801928</td>\n",
       "      <td>0.0347337</td>\n",
       "      <td>[0.7916666666666666, 0.7361111111111112, 0.875...</td>\n",
       "      <td>(5,)</td>\n",
       "      <td>0.046766</td>\n",
       "      <td>0.0155887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.806153</td>\n",
       "      <td>0.0328688</td>\n",
       "      <td>[0.7916666666666666, 0.7361111111111112, 0.875...</td>\n",
       "      <td>(1, 5)</td>\n",
       "      <td>0.0442551</td>\n",
       "      <td>0.0147517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.810359</td>\n",
       "      <td>0.0313243</td>\n",
       "      <td>[0.7777777777777778, 0.75, 0.8888888888888888,...</td>\n",
       "      <td>(1, 4, 5)</td>\n",
       "      <td>0.0421755</td>\n",
       "      <td>0.0140585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.82865</td>\n",
       "      <td>0.03601</td>\n",
       "      <td>[0.7777777777777778, 0.7777777777777778, 0.888...</td>\n",
       "      <td>(1, 4, 5, 6)</td>\n",
       "      <td>0.0484845</td>\n",
       "      <td>0.0161615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.831486</td>\n",
       "      <td>0.039996</td>\n",
       "      <td>[0.7638888888888888, 0.7777777777777778, 0.888...</td>\n",
       "      <td>(1, 4, 5, 6, 12)</td>\n",
       "      <td>0.0538512</td>\n",
       "      <td>0.0179504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.832875</td>\n",
       "      <td>0.038803</td>\n",
       "      <td>[0.7777777777777778, 0.7777777777777778, 0.888...</td>\n",
       "      <td>(1, 2, 4, 5, 6, 12)</td>\n",
       "      <td>0.052245</td>\n",
       "      <td>0.017415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.832915</td>\n",
       "      <td>0.0392131</td>\n",
       "      <td>[0.7777777777777778, 0.7638888888888888, 0.888...</td>\n",
       "      <td>(1, 2, 4, 5, 6, 8, 12)</td>\n",
       "      <td>0.0527972</td>\n",
       "      <td>0.0175991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.832915</td>\n",
       "      <td>0.0392131</td>\n",
       "      <td>[0.7777777777777778, 0.7638888888888888, 0.888...</td>\n",
       "      <td>(1, 2, 4, 5, 6, 7, 8, 12)</td>\n",
       "      <td>0.0527972</td>\n",
       "      <td>0.0175991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.832915</td>\n",
       "      <td>0.0397522</td>\n",
       "      <td>[0.7916666666666666, 0.75, 0.8888888888888888,...</td>\n",
       "      <td>(1, 2, 4, 5, 6, 7, 8, 9, 12)</td>\n",
       "      <td>0.0535229</td>\n",
       "      <td>0.017841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.84835</td>\n",
       "      <td>0.0321055</td>\n",
       "      <td>[0.8194444444444444, 0.7916666666666666, 0.888...</td>\n",
       "      <td>(1, 2, 3, 4, 6, 8, 9, 11, 12, 13)</td>\n",
       "      <td>0.0432273</td>\n",
       "      <td>0.0144091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.848331</td>\n",
       "      <td>0.03183</td>\n",
       "      <td>[0.8333333333333334, 0.7916666666666666, 0.888...</td>\n",
       "      <td>(1, 2, 3, 4, 6, 7, 8, 9, 11, 12, 13)</td>\n",
       "      <td>0.0428564</td>\n",
       "      <td>0.0142855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.845533</td>\n",
       "      <td>0.0360445</td>\n",
       "      <td>[0.8194444444444444, 0.7916666666666666, 0.888...</td>\n",
       "      <td>(0, 1, 2, 3, 4, 6, 7, 8, 9, 11, 12, 13)</td>\n",
       "      <td>0.0485308</td>\n",
       "      <td>0.0161769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.837081</td>\n",
       "      <td>0.0373458</td>\n",
       "      <td>[0.7916666666666666, 0.7916666666666666, 0.888...</td>\n",
       "      <td>(0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13)</td>\n",
       "      <td>0.0502829</td>\n",
       "      <td>0.016761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.831447</td>\n",
       "      <td>0.0403506</td>\n",
       "      <td>[0.7916666666666666, 0.7638888888888888, 0.902...</td>\n",
       "      <td>(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13)</td>\n",
       "      <td>0.0543287</td>\n",
       "      <td>0.0181096</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   avg_score   ci_bound                                          cv_scores  \\\n",
       "1   0.801928  0.0347337  [0.7916666666666666, 0.7361111111111112, 0.875...   \n",
       "2   0.806153  0.0328688  [0.7916666666666666, 0.7361111111111112, 0.875...   \n",
       "3   0.810359  0.0313243  [0.7777777777777778, 0.75, 0.8888888888888888,...   \n",
       "4    0.82865    0.03601  [0.7777777777777778, 0.7777777777777778, 0.888...   \n",
       "5   0.831486   0.039996  [0.7638888888888888, 0.7777777777777778, 0.888...   \n",
       "6   0.832875   0.038803  [0.7777777777777778, 0.7777777777777778, 0.888...   \n",
       "7   0.832915  0.0392131  [0.7777777777777778, 0.7638888888888888, 0.888...   \n",
       "8   0.832915  0.0392131  [0.7777777777777778, 0.7638888888888888, 0.888...   \n",
       "9   0.832915  0.0397522  [0.7916666666666666, 0.75, 0.8888888888888888,...   \n",
       "10   0.84835  0.0321055  [0.8194444444444444, 0.7916666666666666, 0.888...   \n",
       "11  0.848331    0.03183  [0.8333333333333334, 0.7916666666666666, 0.888...   \n",
       "12  0.845533  0.0360445  [0.8194444444444444, 0.7916666666666666, 0.888...   \n",
       "13  0.837081  0.0373458  [0.7916666666666666, 0.7916666666666666, 0.888...   \n",
       "14  0.831447  0.0403506  [0.7916666666666666, 0.7638888888888888, 0.902...   \n",
       "\n",
       "                                       feature_idx    std_dev    std_err  \n",
       "1                                             (5,)   0.046766  0.0155887  \n",
       "2                                           (1, 5)  0.0442551  0.0147517  \n",
       "3                                        (1, 4, 5)  0.0421755  0.0140585  \n",
       "4                                     (1, 4, 5, 6)  0.0484845  0.0161615  \n",
       "5                                 (1, 4, 5, 6, 12)  0.0538512  0.0179504  \n",
       "6                              (1, 2, 4, 5, 6, 12)   0.052245   0.017415  \n",
       "7                           (1, 2, 4, 5, 6, 8, 12)  0.0527972  0.0175991  \n",
       "8                        (1, 2, 4, 5, 6, 7, 8, 12)  0.0527972  0.0175991  \n",
       "9                     (1, 2, 4, 5, 6, 7, 8, 9, 12)  0.0535229   0.017841  \n",
       "10               (1, 2, 3, 4, 6, 8, 9, 11, 12, 13)  0.0432273  0.0144091  \n",
       "11            (1, 2, 3, 4, 6, 7, 8, 9, 11, 12, 13)  0.0428564  0.0142855  \n",
       "12         (0, 1, 2, 3, 4, 6, 7, 8, 9, 11, 12, 13)  0.0485308  0.0161769  \n",
       "13     (0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13)  0.0502829   0.016761  \n",
       "14  (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13)  0.0543287  0.0181096  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sequential forward floating selection (dataframe)\n",
    "pd.DataFrame.from_dict(sffs.get_metric_dict()).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize results with plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEWCAYAAAB1xKBvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcHGWd+PHPt4/puZJJIMkQcnEfAQRNAF0REw4NKqCI\nyyUuKEZ2xVtXDn8e673oqiu4yCJEBYwXSASWQ0kARSUgIQdnCDEHkANyTeboo76/P56nZ2p6eqZr\nZrpmMsn3/Xp10nV96+me6vpWPfXUU6KqGGOMMZUkhrsAxhhjRgZLGMYYYyKxhGGMMSYSSxjGGGMi\nsYRhjDEmEksYxhhjIrGEsRsRkRYROSDCfPuJiIpIqpfpq0XklCqXbZaIrKtmzAGWY56IfK2P6V8T\nkc0i8kql72mQ5XiLiDxb7biD4T/rQTHEjbRdDjD2N0Xkk3HEHigROUVEVg/RuiaKyFMiUjMU67OE\nAYjICSLyiIhsE5HXROTPInLscJerLyKySEQuCY9T1UZVXTVcZRrpRGQq8BlguqruU+XY3XbGqvqw\nqh5azXXsCoZyuxSR8cAHgB9XOe5RInK/iGzxr8dE5O1+Wr+TgYisE5E2nzi3+v3LXBGRwZZVVV8G\n/gR8aLCxotjjE4aIjAbuBH4I7AVMAr4CdAxnuczgDPCsYCrwqqpurHZ5TCwuAu5W1bZqBfQ78TuB\nu4EJwD7Ap4CWQYY+TVUbgf2Aq4ErgesHGbPoFuAjVYrVN1Xdo1/ATGBrhXk+CDwNbAHuBaaFpp0K\nPANsA64BHgQu8dO+DNwcmnc/QIGUH24CfgK8DKwHvgYk/bSLcEcO3/HrfRG30QF8HSgA7bgN+Ro/\nXoGD/Pt3Ak8A24G1wJd7K0eZz7sauAJ4yq/7JqDWTxuL+0Ft8tPuBCaHlt3Lz/+Sn/47P34WsC40\n38d9/Mn+O3uvH/9mX7Z3+uGTgSX+/YHAA8CrwGbcD2VMSbk/DyzFJfwU8Hrg78AO4JfAfOBrZT7z\nKUAbEPjvdF6Zv9e+wALgNWAl8OHQ8scBfwG2+r/nNUCNn/aQj7PTxz6nzPexGvisL/s2X9ba0PR/\n93FfAi4J/63LfJaLgFX+M78IXBBxWw5vPxnctrcG2ABcB9SF5j0TWILbvl4A5hBtu2wCfobbfv4B\nfAFIVNrme/mcDwDvDw1H2o4q/Nb38cs1lpnWVLKNtOCSSj3wc1/mFbhtcHVouXXArJJYb/JxDvPD\ntcB/4X6rG4Af0fWbex6YE1q2BrcNvi403A5Min1/GfcKdvUXMBq3A/opcBowtmT6mbidw+G4HdAX\ngEf8tHH+R3k2kMYdieSJnjBux51ON/gN71HgI37aRUAO+DCQBP4Vt7MQP31RcT2h+OEf5izgKNxZ\n5Ov8RvjucuUo852sBpYDU3AJ4M/4nSywN/Be/yMZBfwanxT89LtwO7ux/jt5a6g86/z7L+J24uP9\n8H8AP/Tvr8TtgL4dmvYD//4gXILOAONxO+Lvl5R7iS93nf8h/cP/XdL+75SjTMIoLWMvf6+H8D9k\n4BjcTu8kP20G8EbcNrIfbqf8yXJ/m17Wtdr//ff13/nTwKV+2hzgFeAI/73fXBovFKcBtxM/1A9P\nBI6otC2X2X6+h0uOe/m/8++Bb/ppx+GS2qm47WsSXTu+RfS9Xf4MuMPH3A94DvhQlG2+zGfdBBwb\nGo60HVXYHyT8cnf472tCyfRTCCUDP+47/nOPBabhDoRWh6b3SBh+/Ev4gw5cDcftPsZo3BnOV0Nl\n/2nJPml5SayngHfEvr+MewUj4eV/QPP8HzbvfyjNftr/FTfo0AbV6jeMDwB/DU0TH6NiwgCacUfB\n4aO284CF/v1FwMrQtHq/7D5+uM8fZpnP+H3ge6Xl6GXe1fidlR9+B/BCL/MeA2zx7yfijprGlplv\nFu4s6r9wR5FNoWknA0v9+3twR9B/9cMPAmf1su53A0+UlPuDoeETKdnhAI8wgISBS0IFYFRo+jeB\neb3E+iRwe29/mzLrWk33o+X/BK7z72/E76z98EG9/a1xCWMrLqnXlUzrdVsOlxG3He8EDgzN+ybg\nRf/+x8Vtqcz6e90ucUkgi7tGVJz2EWBRlG2+zLpy+EQ1mO2oTNwpuAODVf5vvrD4XVA+YawBTgkN\n/xvREsZjuLORBO4MYVpo2luA5/37w3AJunjG8UvgypJYfwPOj/L5BvPa469hAKjq06p6kapOBo7E\nHeV930+eBvzAX6zaijsVFNxR1b64U8hiHA0PVzANd9T7cij2j3FnGkWvhGK3+reNUYKLyPEislBE\nNonINuBS3BlRVOHP8Q/cZ0VE6kXkxyLyDxHZjjvqHiMiSdwP7TVV3dJLzDHAXNzOb1to/F+AQ0Sk\nGZeAfgZMEZFxuKPZh/y6m0Vkvois9+u+ucxnCpd7X2C9/7uEP8tA7Os/246SWJN82Q4RkTt966rt\nwDfKlK2SV0LvW+n6W3fbzuhjG1PVnbgqr0tx29ZdInKYn9zXthw2Hrezfjw07z1+PLi/8wv9/Gzg\nvo803f8Gnd+h159tfgvuTKUo0nZUiaquVdV/U9UDgP1xiWleH4tMpOfvJYpJuL/BPriz5idD3/ed\n+H2Bqj6D+77fKSKNwLuAW0tijcIdKMTKEkYJ/8eZh0sc4DaEj6jqmNCrTlUfwdUpTyku6y+YTQmF\n24n74RWFW96sxZ1hjAvFHa2qR0QtaoXpt+LOlKaoahOuDro/rTLCn2Mq7kgdXCuiQ4HjVXU07ige\nH3stsJeIjOkl5hbcxn6TiLy584O4HcPjwCdwp9pZ3JnAp3FnNpv9rN/Afe6j/LrfX+Yzhb+Xl4FJ\nJa1Rpvb5qXv3Eu6zhXdQU3FnTQD/g7uWdbAv25VlyjZQL+Ou9RRN6W1GAFW9V1VPxe3IngH+10/q\na1sO24yrqz8iNF+Tuou2xTgH9rb6Poq2GbfznRYaF/4O+2spcEjniqNvR5Gp6hrc2UZxf1Du871C\nz99Ln0Tkjbhahj/hqouzuGrE8PfdFFrkF7gaiPfgrsWsDsWqAQ4Anoz6uQZqj08YInKYiHxGRCb7\n4Sm4P8xf/SzXAVeIyBF+epOIvM9Puws4QkTO8q1yPk73pLAEOFFEpopIE+5CMtDZHO4+4LsiMlpE\nEiJyoIi8NWLRN+A2kt6Mwh0Rt4vIccD5EeMWfVREJovIXsBVuNPgYtw2YKuf9qWSz/R/wI9EZKyI\npEXkxHBQVV0EXADc5stV9CBwmf8fXNVGeLi47hZgm4hMAj5X4TP8BVfF+HFflrNwR5r9pqprcTuf\nb4pIrYi8DteU8eZQ2bYDLf6I/l9LQlT6e/XlV8DFInK4iNQD/6+3Gf1Z2Jki0oA7IGnBVRNC39ty\n+LMGuCTzPRGZ4OedVGxaimuocbGInOy320mhs5heP6eqFvxn+bqIjBKRabid+c3l5o/gbqD09xJl\nO+qViIwTkS+JyAHijAcupmt/sAEYV3Lg8CvgShEZ45tmX9ZH/CYROQN3QDfP124UgBuA74vIeL/e\nySLyttCiv8BdY51Lz7OLNwLPqepAE290cdd57eov3Gnhr3BHOTv9/z8GRofmuRBYRleLoxtD0+bg\nLtz1aCXlp1+LO1VcibuYF76I2oQ7Ml3nl38CONdPuwj4U0lZwxcP3+TXuwX47zLTz8adGu/And5e\ng7+eQv9aSW3FNQio99P2xf0IW/z6P1Lymfby82/wZbvNj59F9zr7d/p53uCH3+7jvNUPH+mHzwkt\ncwTuCLIFl4w/Q8/rAKeUfJaZ/nsttpL6JQO/6D3Zf5ev4aoIwtd5TsQdzbcAD+MuVP4pNP1S3JnC\nVuCfy6yrW9npef3rCtyR7Eu4ZKS4s8fSzzARtw1u8+taRPdrBn1ty+HtpxZ3RrfKz/s08PHQvO/B\nHeHvwG3bb4+4XY7FJYhNfv1fpKSVVG/bfJnPOg732wlfB4yyHc2il5aRuMT/M9xvZ6f/zm8BJobm\n+SmuocxWXLVRo59nK723kmrz28Y23IHHvxY/d+j7/hauZdh23G/voyVlexB3hja+ZPyPgX8biv1l\nscWNqRIRWYT7od8w3GUxuycRORzXii2jqvnhLs9wEpFvABtV9fsVZ94NichE4I/AMeqq4GJV9S4P\njDHVJyLvwVXB1APfBn6/pycLAFW9crjLMJzUVQNPH6r1xXYNQ0RuFJGNIrK8l+kiIv8tIitFZKmI\nvCE0bY6IPOunXR5XGY0ZQT4CbMRVhRXoeY3EmNjFViXlL3a2AD9T1SPLTH8H8DFcG//jcTfVHO+b\nZz6HuyloHbAYOE9Vn4qloMYYYyKJ7QxDVR/CXRzszZm4ZKKq+ldcW/6JuFYsK1V1la+Tm+/nNcYY\nM4yG8xrGJLrf7LLOjys3/vjegojIXFxTM+rq6mZMmdJnE/VeBUFAIhFP/owzdtzxR2rsuONb2Ycn\n/kiNHXf8wcR+7rnnNqvq+MpzEm+zWlyzxOW9TLsTOCE0/EdcE8izgRtKmgFeE2V9M2bM0IFauHDh\ngJcdzthxxx+pseOOb2UfnvgjNXbc8QcTG3hMI+7Th/MMYz3d746c7MelexlvjDFmGA3nnd4LgA/4\n1lJvBLapayK2GDhYRPb3t7yf6+c1xhgzjGI7wxCRX+DuqBwn7tGcX8KdPaCq1+HalL8Dd5doK+72\ne1Q1LyKX4frqT+LuRF0RVzmNMcZEE1vCUNXzKkxX4KO9TLsbl1CMMcbsIvb4zgeNMcZEYwnDGGNM\nJJYwjDHGRGIJwxhjTCSWMIwxxkRiCcMYY0wkljCMMcZEYgnDGGNMJJYwjDHGRGIJwxhjTCSWMIwx\nxkRiCcMYY0wkljCMMcZEYgnDGGNMJJYwjDHGRGIJwxhjTCSWMIwxxkRiCcMYY0wksSYMEZkjIs+K\nyEoRubzM9LEicruILBWRR0XkyNC01SKyTESWiMhjcZbTGGNMZbE901tEksC1wKnAOmCxiCxQ1adC\ns10JLFHV94jIYX7+k0PTZ6vq5rjKaIwxJro4zzCOA1aq6ipVzQLzgTNL5pkOPACgqs8A+4lIc4xl\nMsYYM0BxJoxJwNrQ8Do/LuxJ4CwAETkOmAZM9tMU+IOIPC4ic2MspzHGmAhEVeMJLHI2MEdVL/HD\nFwLHq+ploXlGAz8AXg8sAw4DPqyqS0RkkqquF5EJwP3Ax1T1oTLrmQvMBWhubp4xf/78AZW3paWF\nxsbGAS07nLHjjj9SY8cd38o+PPFHauy44w8m9uzZsx9X1ZmRZlbVWF7Am4B7Q8NXAFf0Mb8Aq4HR\nZaZ9GfhspXXOmDFDB2rhwoUDXnY4Y8cdf6TGjju+lX144o/U2HHHH0xs4DGNuF+Ps0pqMXCwiOwv\nIjXAucCC8AwiMsZPA7gEeEhVt4tIg4iM8vM0AG8DlsdYVmOMMRXE1kpKVfMichlwL5AEblTVFSJy\nqZ9+HXA48FMRUWAF8CG/eDNwu4gUy3irqt4TV1mNMcZUFlvCAFDVu4G7S8ZdF3r/F+CQMsutAo6O\ns2zGGGP6x+70NsYYE4klDGOMMZFYwjDGGBOJJQxjjDGRWMIwxhgTiSUMY4wxkVjCMMYYE4klDGOM\nMZFYwjDGGBOJJQxjjDGRWMIwxhgTiSUMY4wxkVjCMMYYE4klDGOMMZFYwjDGGBOJJQxjjDGRWMIw\nxhgTiSUMY4wxkVjCMMYYE0msCUNE5ojIsyKyUkQuLzN9rIjcLiJLReRRETky6rLGGGOGVmwJQ0SS\nwLXAacB04DwRmV4y25XAElV9HfAB4Af9WNYYY8wQivMM4zhgpaquUtUsMB84s2Se6cADAKr6DLCf\niDRHXNYYY8wQElWNJ7DI2cAcVb3ED18IHK+ql4Xm+QZQp6qfEpHjgEeA44H9Ky0bijEXmAvQ3Nw8\nY/78+QMqb0tLC42NjQNadjhjxx1/pMaOO76VfXjij9TYcccfTOzZs2c/rqozI82sqrG8gLOBG0LD\nFwLXlMwzGrgJWAL8HFgMHBNl2XKvGTNm6EAtXLhwwMsOZ+y444/U2HHHt7IPT/yRGjvu+IOJDTym\nEffrqQGlpGjWA1NCw5P9uE6quh24GEBEBHgRWAXUVVrWGGPM0IozYSwGDhaR/XE7+3OB88MziMgY\noFXddYpLgIdUdbuIVFzWGGNKBaoEgVIItMf7QqDkCgG5fIF8oOTyAfmCkisUyBcCWjvy/H3VJhoy\nKeoyKeprUmTSKWpSCdKpBAmR4f54wy62hKGqeRG5DLgXSAI3quoKEbnUT78OOBz4qYgosAL4UF/L\nxlVWY0z/FQIlXwjIFwJyhYBsPqA9lyebD3hx4w4EEIHO3ax0/Sci7lWcJO4Ffpy4d8XpiYR0rvOV\nra09dvr5wJUhCJRiFEU7ly9eqU0m3HoTIv49ZNJJamtSbE4IDZk0uUJA644ONhTauhYWqKtJUV+T\npL42TW06SU0qSU0qQSq559zOFucZBqp6N3B3ybjrQu//AhwSdVljzNDpSgYuMXTk87RlC7RnC3Tk\nCuQKQeeOGVwSSCWEQqBsb812i6W+cU1pE5vubW60zLiuZVSVjlyB9a/uJJHovtOvSSXIpJODPgtI\nJIRMIkkmnexR/nygtHTk2bIzS6AuGSmQTiaoy6RoyKSoz6Q6E0k6mUB2s7OSWBOGMWbXpOqqZ/I+\nGWTzBdpz/pUt0JHLE5TsuJMJIZlIkEoKtTUpGhLld4YJgfpMPLuW1xLC6PqaWGL3RURIJ8Ulh5LV\nF/yZzqaOdvKFoNsZVX1NmnqfSIqJ1J3h0O1/KRneVau/LGGYYVEIFFVoac8BoWqLIikd7BpR+lvq\nuawbowrt2bx774eLR8No+Mi153gNHeZq5yTtfJ8PlM3b2yp9zAHLF5SN29pc6xS/XlVfR+8KjILf\nqSsaKAGgAQQUpytB4OIFxZYuQFs2z2MvbOo8Qi5KJRMkE0IqkaCxrmaX3Wntalwi7eWspKBsa83y\n6o52OnIFnntpa7GGq+z/YYmEO4NK+uq7ZEK6xvkzrIRAMpEgX3DbRdx/M0sYJnb5QkCHP3rd2Z5j\ne1uO9myBtmyeZ9Zt6TZvpbuCKv0cwsu3ZfMsW/NatyO+4g6/tG67OK63H2+pbK7A6k0tFeYauGy+\nwNrNLn5n3X7owxevA3RN7/pMPceLvwYgncs11dfsdtUluxoRIZ0S0il3jWNzQmhqyERe3iX54q0P\nbriQV7I+8RfHqUJH3l24r0klK8YdDEsYpqqy+QIduYC2bJ6W9hwt7Tk6coXOHXE6maAmlWR0fZpX\n+/kD6q/XEsKYmOK/lnA73bi8GmPVSzjZmF1XorPFQOW/1Suxl8axhGEGRFXpyAdkcwVas3l2tLrk\nkPd1IK7ON0FNKkFdjW1mxuwO7JdsKioE6s8cCrS059jZnqOlPd/ZhDGREJcYMimSvVwINcaMfJYw\nTK92tOVozxX4+6quC6TJhDtraKxL20VRY/YwljBMD+3ZPOte28mrO9pRxS6QGmMASxgmJF8I2LC1\nlfVbWkknE4xtyLBV7AKpMcaxhGEIVHltRztrNrcQBEpTXU1nVwzGGFNkCWMPt70ty5pNLbR25Gis\nqyG9B/WLY4zpH0sYe6jwdYqGTJqxjbXDXSRjzC7OEsYeptx1CrtGYYyJwhLGHsKuUxhjBssSxh7A\nrlMYY6rBEsZuzK5TGGOqyRLGbsiuUxhj4mAJYzdi1ymMMXGKtTJbROaIyLMislJELi8zvUlEfi8i\nT4rIChG5ODRttYgsE5ElIvJYnOXcHWxvy/LU2i2s2rCd2poUTQ0ZSxbGmKqKnDBEpE5EDu3H/Eng\nWuA0YDpwnohML5nto8BTqno0MAv4roiEHwIwW1WPUdWZUde7O1FVCkHoEZrZPK0d7jkT29uybN3Z\nwZaWDjpyBZ72DyIa21hrF7XNsFhwW5pZx47itLefxKxjR7HgtvSIiG2ii1QlJSKnA98BaoD9ReQY\n4D9U9Yw+FjsOWKmqq3yM+cCZwFOheRQYJa6CvRF4Dcj3+1PsAoqPwAz8E7K6/nev7a3ZzvdB4B4o\nXwiUQiGgEATkC1BQ/4zlICDw04sEQVH3FLWup4l2rnsvu6BthtGC29J84XN1tLe5s9qX1gtf+Fwd\nAGecldtlYxfj/9c3a3n5pZOYuK/y6SvaqxJ3dyThZxf3OpPI48BJwCJVfb0ft0xVj+pjmbOBOap6\niR++EDheVS8LzTMKWAAcBowCzlHVu/y0F4FtQAH4sape38t65gJzAZqbm2fMnz+/4ucpZ8eOHdQ3\nNALFx3h2PQbR0dCznbvPU+l5nvmONlKZujJlh+LTtEofrNWfyqSO9lYytfX9WGL3jx13fCt7l0JB\nOP/cE9i2rdwTApXa2kK3MZXaX5ROb2tLotpzoZqaAv/05k00NuRpHJWjsTFPQ2OeUY05GhrzNIbe\nNzTkSZQ58X7gj8384PuH09HR9WjTTKbAJz75NCedvKHvgvZTrH/TtlZGjR7Vr/1G0ezZsx+PWosT\n9aJ3TlW3lbS0qZxpKns7sASXjA4E7heRh1V1O3CCqq4XkQl+/DOq+lBpAJ9IrgeYOXOmzpo1q9+F\naM/meWDhIkZPPQJwvbMWH2OZ8L21irhHJrqnJvr/Q+P7smr5YxxwZHy1anHGH6mx446/p5ddFZYu\nSfL729LcdUeabdt6rwY9/1+6EkalnUa549d515d/TnU2m+DF1ePZtlXYsV0oFHr/HYoojaOgqUkZ\nHXr9aVGKjo7uy3V0JLn55ulc8okpFUrbP3H+TZ9fupg3n/CWXeaZ3itE5HwgKSIHAx8HHqmwzHog\n/I1P9uPCLga+pe40Z6U/qzgMeFRV1wOo6kYRuR1XxdUjYVRTXM9/NmZ3sWZ1ggW3pVlwW5rVq5LU\nZJSTTs3zt7/Alld7Jo19JymXf6l9UOu87640L63vmQz2naTc/+cWwCWanTth+1Zh2zaXQIqJZNs2\nYftWYXvxvX+tWpmgtbX8Ol9aL1zxqTqmH1XgiKMKHHZEgfr4TihHjKgJ42PAVUAHcCtwL/C1Csss\nBg4Wkf1xieJc4PySedYAJwMPi0gzcCiwSkQagISq7vDv3wb8R8SyGmOq6LVXhbsXuCSx5PEUIspx\nbyrw4X9r5e3vzDG6qed1BoDaOnc9YLA+fUV7xdgi0NgIjY3KvpOjV37MOnZU2WSUycDCP6T47S9d\nNVsioRxwUNCZQI44qsD0Iws0jhrEBxuBIiUMVW3FJYyrogZW1byIXIZLLkngRlVdISKX+unXAV8F\n5onIMly1/edVdbOIHADc7qvAUsCtqnpPPz6XMWYQ2lrhgfvTLPhtmocXpcjnhUMPL/C5q9p417tz\nTJzUfadcvEjsLh5LVS8exxm7t2T0tavbOP09OTa8Ijy1LMnypUmeWpbkb4+kWPDbrms1+x3gEscR\nRxV8MgkYM7b7d7M7XVSP2krqfuB9qrrVD48F5qvq2/taTlXvBu4uGXdd6P1LuLOH0uVWAUdHKZsx\npjoKBfjrn5MsuK2G++5Ks3On0Dwx4OK5WU4/K8th04M+lz/jrBxnnJWLpa4+rtiVktE+E5V9JuY5\n6W1djTc3bxJWLHMJZMWyJEufSHH3gq4kMnlK15nIjh1w840Z2tvjaeE11KJWSY0rJgsAVd3iL0Yb\ns8uJ84gu7qPFoS776e/J8fTyBHfcVsNdv0uzcUOCxlHKaafnOOO9WY59Y4FkvNdRh11/k9G48cpb\nT8rz1pO6ksjWLcKKZYnOJLJiWZL77i5/r0h7m/D1L9Yybb+AKdMCxu6lFVuO7SqiJoxARKaq6hoA\nEZlGdVpJmT1Q3DvFkXw/wFCW/fJP1vGfX6tl4ysJ0mnlxJPynPneNmadnKe2Zytw04cxY5U3n1jg\nzSd2tQjbsR1mHD4ayjQJ3vJagve9yzXjr69XpkwLmDwl6P7/VPe+ro+L7V2/pZOZMgW+8Q244IKq\nf7xOURPGVcCfRORB3LWGt+DvfTCmP6LuFPN56OiAjnahox06OsQNd/jh9q7h9nbI+uk//E6mW300\nuCO6L11ex1PLug6VS492ejTn1J7TfzO/ptfYy5YM/jA8zvjlYufzwtbX4CvfamPOu3KM3cuOAatp\n1GjYd18te1F9/ISAr/5nG2vXJFi3JsHafyRYuybBIw+naCv5O40bHzBlqksg4YTyzFMJvvvNrt/S\nmjUw1++V40oaUS963yMibwDe6Ed9UlU3x1MksysYyFlAoQA7trsmi11NGel8v22bcMu88jv0f/94\nHV//Ym1nIuirTf1A7GyB+T/vfmNZj2qAkuHS6Ttbeo9926/K3bTWP3HG7y12LgfnfSA7qNimd71d\nVP/8F9u7XRcpUnWt0tb+wycS/1q3JsETj6W46w4hCHr/bbS2wlVXDXPC8DK4rjtSwHQRodyNdGbk\nK3cWcMWn6vjDvSkmTdJubdnDbd13bO97J59OK7leck4QwGmn58jUQiajZDJQW6vUZPxwbcmwn56p\nhZrQ8BmnjOLll8rfD7Bo8Y5BfS+9NcGsRuy44/cWe+K+dlYRp/628BKBvccpe48rcMyMQo/puRy8\n/JJLKBef20C5PiHWrKnqR+gmaiupbwPnACuAYlMJJeYb6czQU4VvfaW2x1lALifc8/saMrXa7W7Z\nfSYGHHK4Mnq0G24a4/9vUkaP8f+Pdu/r6mD2cb3vFL/8zcG32f/MlZXb7A9UlPsBdtX4cZfd9K6a\nLbzSaZg6TZk6rcC+k8pXd02dOqhV9CnqGca7gUNVtSO+opjh1NYKd/4uzc9vzLB5U/luHkSUZau2\nD2o9ce+4Rur9AHHHj7vsZuiV+y3V18PXvx7fOqMmjFVAGnent9mNrFsr3Dovw2/mp9m6JcGhhxdo\nGhOwbWvPpFGN6ouh2HGNxPsBhiJ+3GU3Q6v0t+RaScku0UqqFVgiIn8klDRU9eOxlMrEShUeeTjJ\nzTdmeOD+FIkEnHpangs/2MrM4wv8/vb4unkA23EZUy3F39LzSxdzysmzd5nOBxf4lxnBdu6E3/26\nhptvquGF55OM3SvgIx/r4LwLs926erDqC2NMOVGb1f407oKY+Lz4QoJb5tVw269qaNkhHHl0nm//\noJV3+FZ74MPFAAAdhklEQVRJ5dhZgDGmVNRWUgcD38Q9arVzF6OqB8RULjNIQQAPPpDi5ptqeHhh\nmnTadfdw4QezvO71hRHTFYExZtcRtUrqJuBLwPeA2bjnWNiDo4dZuZvrZp2c47e/rOGWeTWsWZ1k\nQnPAxz/bzjnvzzJ+grW5N8YMXNSEUaeqfxQRUdV/AF/2j239YoxlM30od3Pd5z9RRyJRRy4nvOHY\nPJ/6fCunnpajZvA3IRtjTOSE0SEiCeB5/4yL9UBjfMUyfcnn4eqv97y5rlAQMhnl13fuYPpRfXdF\nbYwx/RU1YXwCqMc9mvWruGdw/0tchdpd9Lc/po522LRJ2LQhwcaN7v9NG3sOv/Zq7/3JtLVhycIY\nE4uoraQW+7ctuOsXpoJyVUZXfqaOJx5LMnW/oGdS2Chlb5ZLJJRx45XxE5TmiQFHHq1MmBBw87ya\n2G6uM8aYcqK2kpqJ6+J8WngZVX1dTOUaMrfcAldcmWTd2pP7db+BKmzb6nb0GzcImzYm2LRB2Lgh\nwcYNwh/vTZPNdj8LyHa43lrBdZg3YYIybkLA/gcGHPemgAnNyvgJAeObXVIY36zstbeWfYDN/gcF\n1jeQMWZIRa2SugX4HLCMrs4HKxKROcAPcM/0vkFVv1UyvQm4GZjqy/IdVb0pyrLVcMstrv/41tbu\nz2bYsR1eP7PQmQQ2bUx0JoWNG/xZwSYh29GzWqihQRnfHJDtpcdoEeXRFdsZ3VSme+1+sJvrzK4m\nUKWlLUchULbu7N6LkAg9nzFSnNbtvfSYI7xYOHYmnaQmlSCZsAabQyVqwtikqv2601tEksC1wKnA\nOmCxiCxQ1adCs30UeEpVTxeR8cCzInILUIiw7KBddZXrPz6svU34ypU9H3HVNCZg/ARXNTTj+DwT\nJrjEMKFZmdDspzUHNDS4+fvqTrppTHXKbzfXmV1FWzZPWzbPPmPq2VaTZPqUsZ3TikmgeIDU7Vfh\nR0rPUaHhrhF/WpfisEljacvmaWnLsr09Ry6f60w6NSmXRFJJSyJxiJowviQiNwClfUnd1scyxwEr\nVXUVgIjMB84Ewjt9BUaJ2yIacc/byAPHR1h20HrvN1754f+2Mn6CMmGfgPHjtdc7ontj3UmbPUGu\nENDSlmNUXZoDp+5FQybNiyI0ZMo/z3qwBBhVl2ZUXZoJTXWdZejIFWjP5tnRnmNHW46W9q4z7ZpU\nknQqQdqSyKCJ9ng2ZZmZRG4GDqPkeRiq+sE+ljkbmKOql/jhC4HjVfWy0DyjcH1UHQaMAs5R1bui\nLBuKMRf/uNjm5uYZ8+fPr/ypvXPPfSMbNvTMBBMmtPGzmx+JHKc3D/yxmXk3HcimTbWMH9/ORRe/\nwEknbxh03FId7a1kavt48O8eGDvu+FZ2Vz2EuB1yKtF1YNTS0kJjYzyt7vsTO1AlUAgCpRAo4X2d\niDtzCZ/MjOi/aVsro0aPKvM4pcpmz579uKpGqqKIeoZxrKoeOoCyVPJ2YAmume6BwP0i8nB/Aqjq\n9cD1ADNnztRZs2ZFXva73y1ew+gaV1un/PsXtSpVPAccCZd8Isuq5Y/4eFP8q7rirJIaqbHjjr8n\nl721I09HLs/EsQ3sM7a+x5H7okWL6M/vsD8GE7sQBLTnCrRnC+xsz7G9LUd7Lg/qqjo2r1rKlMPe\nQDoVz5lInH/T55cu5s0nvGWX6a32ERGZ3s9rCOvpvnec7MeFXQx8S13qXykiL+LONqIsO2jFfuOv\nuFJZtxa7cGxMHzqrn+rTHDRxr9iqneKSTCRoyCRoyKTZe5SrWSgEAR05V6W1ZXWCQhCwc2fOzy9k\n0knSyUS36yh7sqgJ442452G8iLuGIbgqqb6a1S4GDhaR/XE7+3OB80vmWQOcDDwsIs3AobiHNW2N\nsGxVXHABvPd9BR5YuIiDX3dsHKswZkQLVNnRmiWZFA7aZzRjGzO7zQ40mUhQn0lQn0lRk0pw1LS9\nyRUC2rJ5drbn2Lozy/a2LKquCiuTSlCTTpLYTT5/f0VNGHP6G1hV874bkXtxTWNvVNUVInKpn34d\n7q7xeSKyDJeEPq+qmwHKLdvfMhhjBsdVPxWYuFc9E8fU7xGtj9LJBOm6GkbX1TBxbIOrysoWaOlw\nCaSlLUfgr4fUpJJk0kmSiT0jgVRMGL557L2qelh/g6vq3cDdJeOuC71/CXhb1GWNMUMjlw/Y0Z5j\ndH2agyc2UZ+Jemy5+0kmEjTUJmioTdPcVE+gSnu2QGtHju2tWba15sgXAhCXbDLp5G6bWCtuBapa\nEJFnRWSqqvbaENUYM/KFq58O3s2qn6olIUJ9JkV9JsW40XWoKh35gPZsnu2tWbbszLpmvQrJZIJM\nOhH7xeihEvWwYSywQkQeBXYWR6rqGbGUyhgz5PbE6qdqEBFq00lq00nGNGSYOh6y+QJt2QI72rJs\n3Zll684OCoGyvTVLzQi+DhI1Yfy/WEthjBk2uXzAjrYsYxoyHLJvE3U1e271U7W4O86TNNXXMHlv\nyBcCHlqbZPLeDWxt9ddB/H0sI6kaK2pvtQ/6VkzFZkSPqurG+IpljIlbECg72rKkkgkO2beJMQ1W\n/RSXVDJBQoTmMfU0j6l31Vi50FlIa1c1VqLYnDeV2OXOQqL2VvvPwNXAIlxrph+KyOdU9Tcxls3s\nAhR3R29C2KV3JkGg/s7e4l29vjM830VEseThzxD+OJWmu3mkc0bFHTXGxX3vQXGtFFddrb/BzvYc\n2XzApL3cDmwkHN3uTkSE2poUtTUpxjZmmAqdzXlbO3JsK9saa/g7Wox67nkV7m7vjQC+o8A/AJYw\ndjNBoLTnCnTkCsWbbWjP5ckXAgIFUbq6Fy3TSVwxsST8+0RCug33tcPTUFcOxZ1/8X0hcD+c7ksL\niiICSUmQSgrpVKLzZquNyQTjR9eiqq7HU39HryooCqr+vX8FbnxxXnDlKfYooaibxy8Xa8JQ6MgV\nupfLfz/liEhn1xfF7vz8J+nWU2zxAKA+k+LQSY1W/bQLCTfn3WdMQ2drrLZsnu1tWba1ZsnlciDD\nd1Nh1K0lUVIF9SpghyS7iY5cgfZcAQ2URELcEc+4RuozKf6yPsUx+40D6LYDDwKloEoQdI0vBAH5\nQMkXAvIFJR8EFAoBuYKSDQoUfNIBt1Mr7QZbxJ26pxIJ0qkENUkhmUy43kcTic7T+mRCSCTc/8mE\nS0blfjQvJhNM3juePo1efTHJUdP2jiV2Mf7R/nsvp5gE1Se2YuIrTgO6xnWbF/66NsXBE5t26TNG\n0701VvHO9GzedW3S0p5jW2uW7a25rgOiIRA1YdwjIvcCv/DD52D3SIxYhSCgLVsgn3dHyA11aSbv\n3cCoujR1Nale600TIiSSg9vJhJPOn9emOGra3iTEtXXfU25+qobOjvO6/olsV69eNL0rXkwfXV/D\nvns1UAhcDcD2tUNz0bzPhCEiGVXtUNXPichZwAl+0vWqenvspTNVUbzA1pErEAA1yQR7N2ZoashQ\nn0kNabfP4aQjArXp3aN9ujHDIZlwXckXz7TjVukM4y/AG0Tk56p6IdDX8y/MLiRXcDcSFQJXxz+q\nroZ9xtbTWJumNp20I0xjTL9VShg1InI+8E/+DKObCg9QMkOoeLE6my8A7si9uamO0fUZ6jPJYW9d\nYYwZ+SoljEuBC4AxwOkl0xQ74xgWhSAgl++6wFwIXPPRsQ01jGlooKE2TcaqeowxVdZnwlDVP4nI\nI8A6Vf36EJXJ4FoQ5QsBuULQ2Xyz2JI1nUrQUJOmLpOkPpNix9okrz9g3C53k48xZvcSpfPBwD8y\n1RJGlQWBdiaEXCHw9z3Q2V1AXSbF6Po09TUpatKudUQ62bM1UUKG5oKXMWbPFrVZ7R9F5L3AbRrl\nIeCmU+DbSLd25MkVAlS1874318Ihxei6jH+AS5KalLsHwa45GGN2NVETxkeATwMFEWmj64l7o2Mr\n2QhTvK6QKyiF0B3ACX820FRfQ30mRSbtk0IyYd0xGGNGlKidD46KuyAjgaqSL3RVIwWhs4XidYW9\nR6Woq0l1nimkkwkeXJtkvwn2FRpjRraonQ8KrrXU/qr6VRGZAkxU1UdjLd0wKV5bCF9wdh3PKbXp\nrusKtT4x1FgVkjFmDxC1SupHQACchHsOdwtwLV3dnZclInOAH+Cey32Dqn6rZPrncImoWJbDgfGq\n+pqIrAZ2AAUgr6ozI5Z1QBQ6+zXqvLZQn6G+JkUm3XXGYBeXjTF7qqgJ43hVfYOIPAGgqltEpKav\nBfyzwK8FTgXWAYtFZIGqPlWcR1WvxnWbjoicDnxKVV8LhZmtqpujf5yBqfFPy5o+ZWxnSyRjjDHd\nRd0z5nwCUOjs3rxS387HAStVdZWqZoH5wJl9zH8eXZ0bDqliD6gNmbQlC2OM6YVEaSUrIhfgeqh9\nA/BT4GzgC6r66z6WORuYo6qX+OELcWcql5WZtx53FnJQ8QxDRF4EtuGqpH6sqtf3sp65wFyA5ubm\nGfPnz6/4ecppaWmhsTGerrDjjB13/JEaO+74VvbhiT9SY8cdfzCxZ8+e/XjkKn/3MJjKL+Aw4KPA\nZcDhEeY/G3fdojh8IXBNL/OeA/y+ZNwk//8E4EngxErrnDFjhg7UwoULB7zscMaOO/5IjR13fCv7\n8MQfqbHjjj+Y2MBjGjEPVOrevBbXn9RBwDLckX4+YuJaD0wJDU/248o5l5LqKFVd7//fKCK346q4\nHoq4bmOMMVVWqcL+p8BMXLI4DfhOP2IvBg4Wkf39BfJzgQWlM4lIE/BW4I7QuAYRGVV8D7wNWN6P\ndRtjjKmySq2kpqvqUQAi8hMg8n0XqpoXkcuAe3HNam9U1RUicqmffp2f9T3Afaq6M7R4M3C7f2ZD\nCrhVVe+Jum5jjDHVVylh5IpvfALoV3BVvZuSR7mGEkVxeB4wr2TcKuDofq3MGGNMrColjKNFZLt/\nL0CdH7a+pIwxZg9T6XkY9hQeY4wxQPQb94wxxuzhLGEYY4yJxBKGMcaYSCxhGGOMicQShjHGmEgs\nYRhjjInEEoYxxphILGEYY4yJxBKGMcaYSCxhGGOMicQShjHGmEgsYRhjjInEEoYxxphILGEYY4yJ\nxBKGMcaYSCxhGGOMiSTWhCEic0TkWRFZKSKXl5n+ORFZ4l/LRaQgIntFWdYYY8zQii1hiEgSuBY4\nDZgOnCci08PzqOrVqnqMqh4DXAE8qKqvRVnWGGPM0IrzDOM4YKWqrlLVLDAfOLOP+c8DfjHAZY0x\nxsRMVDWewCJnA3NU9RI/fCFwvKpeVmbeemAdcJA/w+jPsnOBuQDNzc0z5s+fP6DytrS00NjYOKBl\nhzN23PFHauy441vZhyf+SI0dd/zBxJ49e/bjqjoz0syqGssLOBu4ITR8IXBNL/OeA/x+IMuGXzNm\nzNCBWrhw4YCXHc7YcccfqbHjjm9lH574IzV23PEHExt4TCPu1+OskloPTAkNT/bjyjmXruqo/i5r\njDFmCMSZMBYDB4vI/iJSg0sKC0pnEpEm4K3AHf1d1hhjzNBJxRVYVfMichlwL5AEblTVFSJyqZ9+\nnZ/1PcB9qrqz0rJxldUYY0xlsSUMAFW9G7i7ZNx1JcPzgHlRljXGGDN87E5vY4wxkVjCMMYYE4kl\nDGOMMZFYwjDGGBOJJQxjjDGRWMIwxhgTiSUMY4wxkVjCMMYYE4klDGOMMZFYwjDGGBOJJQxjjDGR\nWMIwxhgTiSUMY4wxkVjCMMYYE4klDGOMMZFYwjDGGBOJJQxjjDGRWMIwxhgTSawJQ0TmiMizIrJS\nRC7vZZ5ZIrJERFaIyIOh8atFZJmf9lic5TTGGFNZbM/0FpEkcC1wKrAOWCwiC1T1qdA8Y4AfAXNU\ndY2ITCgJM1tVN8dVRmOMMdHFeYZxHLBSVVepahaYD5xZMs/5wG2qugZAVTfGWB5jjDGDEGfCmASs\nDQ2v8+PCDgHGisgiEXlcRD4QmqbAH/z4uTGW0xhjTASiqvEEFjkbV9V0iR++EDheVS8LzXMNMBM4\nGagD/gK8U1WfE5FJqrreV1PdD3xMVR8qs565wFyA5ubmGfPnzx9QeVtaWmhsbBzQssMZO+74IzV2\n3PGt7MMTf6TGjjv+YGLPnj37cVWdGWlmVY3lBbwJuDc0fAVwRck8lwNfCQ3/BHhfmVhfBj5baZ0z\nZszQgVq4cOGAlx3O2HHHH6mx445vZR+e+CM1dtzxBxMbeEwj7tfjrJJaDBwsIvuLSA1wLrCgZJ47\ngBNEJCUi9cDxwNMi0iAiowBEpAF4G7A8xrIaY4ypILZWUqqaF5HLgHuBJHCjqq4QkUv99OtU9WkR\nuQdYCgTADaq6XEQOAG4XkWIZb1XVe+IqqzHGmMpiSxgAqno3cHfJuOtKhq8Gri4Ztwo4Os6yGWOM\n6R+709sYY0wkljCMMcZEYgnDGGNMJJYwjDHGRGIJwxhjTCSWMIwxxkRiCcMYY0wkljCMMcZEYgnD\nGGNMJJYwjDHGRGIJwxhjTCSWMIwxxkRiCcMYY0wkljCMMcZEYgnDGGNMJJYwjDHGRGIJwxhjTCSW\nMIwxxkRiCcMYY0wksSYMEZkjIs+KyEoRubyXeWaJyBIRWSEiD/ZnWWOMMUMnFVdgEUkC1wKnAuuA\nxSKyQFWfCs0zBvgRMEdV14jIhKjLGmOMGVpxnmEcB6xU1VWqmgXmA2eWzHM+cJuqrgFQ1Y39WNYY\nY8wQiu0MA5gErA0NrwOOL5nnECAtIouAUcAPVPVnEZcFQETmAnP9YIuIPDvA8o4DNg9w2eGMHXf8\nkRo77vhW9uGJP1Jjxx1/MLGnRZ0xzoQRdf0zgJOBOuAvIvLX/gRQ1euB6wdbEBF5TFVnDjbOUMeO\nO/5IjR13fCv78MQfqbHjjh932YviTBjrgSmh4cl+XNg64FVV3QnsFJGHgKP9+ErLGmOMGUJxXsNY\nDBwsIvuLSA1wLrCgZJ47gBNEJCUi9bhqp6cjLmuMMWYIxXaGoap5EbkMuBdIAjeq6goRudRPv05V\nnxaRe4ClQADcoKrLAcotG1dZvUFXaw1T7Ljjj9TYcce3sg9P/JEaO+74cZcdAFHVoViPMcaYEc7u\n9DbGGBOJJQxjjDGR7PEJQ0RuFJGNIrI8hthTRGShiDzluz75RBVj14rIoyLypI/9lWrFDq0jKSJP\niMidMcReLSLLfLcwj1U59hgR+Y2IPCMiT4vIm6oY+1Bf5uJru4h8sorxP+X/nstF5BciUlut2D7+\nJ3zsFYMtd7nfjojsJSL3i8jz/v+xVY7/Pl/2QEQG3Iy0l9hX+21mqYjc7nuiqFbsr/q4S0TkPhHZ\nt5plD037jIioiIwbaPw+qeoe/QJOBN4ALI8h9kTgDf79KOA5YHqVYgvQ6N+ngb8Bb6xy+T8N3Arc\nGcN3sxoYF9Pf9KfAJf59DTAmpvUkgVeAaVWKNwl4Eajzw78CLqpieY8ElgP1uAYvfwAOGkS8Hr8d\n4D+By/37y4FvVzn+4cChwCJgZpVjvw1I+fffHmjZe4k9OvT+48B11Sy7Hz8F11DoH3H9tvb4MwxV\nfQh4LabYL6vq3/37Hbgmw5OqFFtVtcUPpv2rai0YRGQy8E7ghmrFHAoi0oT7Qf0EQFWzqro1ptWd\nDLygqv+oYswUUCciKdyO/aUqxj4c+JuqtqpqHngQOGugwXr57ZyJS9j4/99dzfiq+rSqDrQ3h0qx\n7/PfC8Bfcfd/VSv29tBgA4P4rfaxz/oe8O+DiV3JHp8whoqI7Ae8HncmUK2YSRFZAmwE7lfVqsUG\nvo/b+IIqxgxT4A8i8rjv3qVa9gc2ATf56rQbRKShivHDzgV+Ua1gqroe+A6wBngZ2Kaq91UrPu7s\n4i0isre/7+kddL9BthqaVfVl//4VoLnK8YfKB4H/q2ZAEfm6iKwFLgC+WOXYZwLrVfXJasYtZQlj\nCIhII/Bb4JMlRxqDoqoFVT0GdyR0nIgcWY24IvIuYKOqPl6NeL04wZf9NOCjInJileKmcKfr/6Oq\nrwd24qpGqsrfUHoG8OsqxhyLO0LfH9gXaBCR91crvqo+jatquQ+4B1gCFKoVv8z6lBiPduMiIlcB\neeCWasZV1atUdYqPe1m14vrkfyVVTkLlWMKImYikccniFlW9LY51+CqXhcCcKoV8M3CGiKzG9RR8\nkojcXKXYQOfRNOp6KL4d10NxNawD1oXOtn6DSyDVdhrwd1XdUMWYpwAvquomVc0BtwH/VMX4qOpP\nVHWGqp4IbMFdV6umDSIyEcD/v7HC/LsUEbkIeBdwgU94cbgFeG8V4x2IO8h40v9mJwN/F5F9qrgO\nwBJGrEREcHXpT6vqf1U59vhiKw4RqcM9O+SZasRW1StUdbKq7oerdnlAVat2pCsiDSIyqvged7Gx\nKq3UVPUVYK2IHOpHnQzE8RyV86hidZS3BnijiNT7bedk3HWvqpGuZ85MxV2/uLWa8XFd+PyLf/8v\nuO5/RgQRmYOrhj1DVVurHPvg0OCZVOm3CqCqy1R1gqru53+z63CNbV6p1jrCK9ujX7gf/ctAzn/R\nH6pi7BNwp+RLcaf/S4B3VCn264AnfOzlwBdj+n5mUeVWUsABwJP+tQK4qsrxjwEe89/N74CxVY7f\nALwKNMXwfX8FtzNZDvwcyFQ5/sO4BPokcPIgY/X47QB7A38Ense1wtqryvHf4993ABuAe6sYeyXu\nsQrF3+qAWjL1Evu3/m+6FPg9MKma30vJ9NXE1ErKugYxxhgTiVVJGWOMicQShjHGmEgsYRhjjInE\nEoYxxphILGEYY4yJxBKG2eX53je/Gxr+rIh8uUqx54nI2dWIVWE97/M95y4sGb+fiLSV9IBbM4D4\n+4nI+dUrsTE9WcIwI0EHcFZsXTYPkO8gMKoPAR9W1dllpr2gqseEXtkBFGc/oN8JQ0SSA1iX2UNZ\nwjAjQR73zOJPlU4oPUMQkRb//ywReVBE7hCRVSLyLRG5QNwzRJaJyIGhMKeIyGMi8pzvR6vYsePV\nIrLYP8fgI6G4D4vIAsrcQS4i5/n4y0Xk237cF3E3cf5ERK6O8oH93fA3+vI+4TuXK55JPCwif/ev\nYtch38J1LLhE3DM1LhKRa0Lx7hSRWcXvSES+KyJPAm8SkRn+u3pcRO4Nde3xcXHPclkqIvOjlNvs\n5uK4G9Be9qrmC2gBRuPuYG0CPgt82U+bB5wdntf/PwvYinsmSQZYD3zFT/sE8P3Q8vfgDp4Oxt05\nWwvMBb7g58ng7hzf38fdCexfppz74rr3GI/rBPEB4N1+2iLKPL8Bd2bQRtfdxdf68d8A3u/fj8H1\n+dSA6/K81o8/GHgs9HnvDMW9CLgmNHwnMMu/V+Cf/fs08Agw3g+fA9zo37+Ev9OcmJ4pYq+R9erP\nKbUxw0ZVt4vIz3APn2mLuNhi9V1ti8gLuF5aAZYB4aqhX6lqADwvIquAw3D9W70udPbShNtBZ4FH\nVfXFMus7Flikqpv8Om/BPZvjdxXK+YK6nnvD3obrAPKzfrgWmIrbiV8jIsfgepo9pELscgq4rirA\nPYzoSOB+130VSVy3E+C6sbhFRH4X4TOYPYAlDDOSfB/4O3BTaFweX7UqIgncE/aKOkLvg9BwQPdt\nv7R/HMU90fBjqnpveIKv1tk5sOL3iwDv1ZKHBfmL/RuAo3Gfu72X5Tu/Fy/8qNd2VS12ay7AClUt\n9xjbd+IS3unAVSJylHY9YMjsgewahhkxVPU13GNLPxQavRqY4d+fgati6a/3iUjCX9c4AHgW96jL\nf/Xd0yMih0jlBzE9CrxVRMb5i8nn4Z5qNxD3Ah/zvdYiIq/345uAl/0Z0YW4MwKAHbjHABetBo7x\nn2sKvXcf/ywwXvxzz0UkLSJH+OQ7RVUXAp/3620c4Gcxuwk7wzAjzXfp/vCZ/wXu8Bdw72FgR/9r\ncDv70cClqtouIjfgri/83e+0N1HhcaOq+rKIXI57NokAd6nqQLv3/irujGqp33m/iHtOw4+A34rI\nB+j+eZcCBf89zPPLvoi7MP807sysXJmzvtrtv8U93jbll30OuNmPE+C/Nb5H3ZoRwnqrNcYYE4lV\nSRljjInEEoYxxphILGEYY4yJxBKGMcaYSCxhGGOMicQShjHGmEgsYRhjjInk/wPm84YUIXQEKgAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11c3558d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Sequential forward floating selection (plot)\n",
    "from mlxtend.plotting import plot_sequential_feature_selection as plot_sfs\n",
    "\n",
    "fig = plot_sfs(sffs.get_metric_dict(), kind='std_dev')\n",
    "plt.ylim([0.6,1])\n",
    "plt.title('Sequential backward floating selection (w. StdDev)')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conclusion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this example we applied sequential feature selection. This technique recursively adds (or removes) features to the feature subset based on classifier's performance. \n",
    "\n",
    "The best performing cases were the ones considering the 'floating' option. This is true for both backward and forward methods. Comparing the results with other solutions ([univariate feature selection]() and [model-based selection]()), we can say that sequential feature selection is the best perfoming solution."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Complete solution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To conclude, we will define a complete solution that automatically chooses the best performing model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential forward selection (12 features): \n",
      "(0, 1, 2, 4, 5, 6, 7, 8, 9, 11, 12, 13)\n",
      "CV score: \n",
      "0.8385295103957077\n",
      "Sequential backward selection (4 features): \n",
      "(0, 1, 11, 13)\n",
      "CV score: \n",
      "0.8455130784708249\n",
      "Sequential forward floating selection (10 features): \n",
      "(1, 2, 3, 4, 6, 8, 9, 11, 12, 13)\n",
      "CV score: \n",
      "0.8483501006036217\n",
      "Sequential backward floating selection (10 features): \n",
      "(1, 2, 3, 4, 6, 8, 9, 11, 12, 13)\n",
      "CV score: \n",
      "0.8483501006036217\n",
      "Best model: SequentialFeatureSelector(clone_estimator=True, cv=10,\n",
      "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
      "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
      "          verbose=0, warm_start=False),\n",
      "             floating=True, forward=False, k_features=(1, 14), n_jobs=-1,\n",
      "             pre_dispatch='2*n_jobs', scoring='accuracy', verbose=0)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcXGWZ9//PVUvvnQUSmiUJm2EJImgiizJKZNQ4jjAq\nPGwybjwZfi9Rxxl5BFFHRx31UecZ+YGD/BBxQfObUZCIDIgSBBSHBA0hAZEYIBuQhCy913au549z\nKl3pVKdPd9fpTqW/79erXnXWq+4+XXVf5z7LfczdERERGU5qogsgIiL1QQlDRERiUcIQEZFYlDBE\nRCQWJQwREYlFCUNERGJJLGGY2S1mtsXMVg8x38zsOjNba2arzOw1FfMWmdnT0byrkyqjiIjEl2QL\n41Zg0T7mvw2YG70WA/8OYGZp4IZo/jzgYjObl2A5RUQkhsQShrs/CGzfxyLnAd/z0O+AaWZ2GHAa\nsNbd17l7HlgSLSsiIhMoM4GffQSwoWJ8YzSt2vTThwpiZosJWyg0NzfPnz179qgKEwQBqVQy+TPJ\n2EnHr9fYScdX2Scmfr3GTjr+WGL/6U9/2ubuM2Mt7O6JvYCjgNVDzLsLOKti/FfAAuB84OaK6ZcB\n18f5vPnz5/toLVu2bNTrTmTspOPXa+yk46vsExO/XmMnHX8ssYEVHrNOn8gWxiagsjkwK5qWHWK6\niIhMoIm8rHYp8LfR1VJnALvc/QVgOTDXzI42swbgomhZERGZQIm1MMzsR8DZwAwz2wj8E2HrAXe/\nEbgb+CtgLdALvD+aVzSzK4F7gTRwi7uvSaqcIiIST2IJw90vHma+Ax8aYt7dhAlFRET2E7rTW0RE\nYlHCEBGRWJQwREQkFiUMERGJRQlDRERiUcIQEZFYlDBERCQWJQwREYlFCUNERGJRwhARkViUMERE\nJBYlDBERiUUJQ0REYlHCEBGRWJQwREQkFiUMERGJRQlDRERiUcIQEZFYEk0YZrbIzJ42s7VmdnWV\n+dPN7A4zW2Vmj5rZKyvmPWdmT5jZSjNbkWQ5RURkeIk909vM0sANwJuBjcByM1vq7k9WLPZJYKW7\nv9PMToiWP6di/kJ335ZUGUVEJL4kWxinAWvdfZ2754ElwHmDlpkH3A/g7n8EjjKzjgTLJCIio5Rk\nwjgC2FAxvjGaVulx4F0AZnYacCQwK5rnwC/N7DEzW5xgOUVEJAZz92QCm50PLHL3y6Pxy4DT3f3K\nimWmAN8AXg08AZwA/E93X2lmR7j7JjM7BLgP+LC7P1jlcxYDiwE6OjrmL1myZFTl7e7upq2tbVTr\nTmTspOPXa+yk46vsExO/XmMnHX8ssRcuXPiYuy+ItbC7J/ICzgTurRi/BrhmH8sb8Bwwpcq8zwIf\nH+4z58+f76O1bNmyUa87kbGTjl+vsZOOr7JPTPx6jZ10/LHEBlZ4zHo9yUNSy4G5Zna0mTUAFwFL\nKxcws2nRPIDLgQfdvdPMWs2sPVqmFXgLsDrBsoqIyDASu0rK3YtmdiVwL5AGbnH3NWZ2RTT/RuBE\n4Ltm5sAa4IPR6h3AHWZWLuMP3f2epMoqIiLDSyxhALj73cDdg6bdWDH8CHBclfXWAackWTYRERkZ\n3ektIiKxKGGIiEgsShgiIhKLEoaIiMSihCEiIrEoYYiISCxKGCIiEosShoiIxKKEISIisShhiIhI\nLEoYIiISixKGiIjEooQhIiKxKGGIiEgsShgiIhKLEoaIiMSihCEiIrEoYYiISCxKGCIiEkuiCcPM\nFpnZ02a21syurjJ/upndYWarzOxRM3tl3HVFRGR8JZYwzCwN3AC8DZgHXGxm8wYt9klgpbu/Cvhb\n4BsjWFdERMZRki2M04C17r7O3fPAEuC8QcvMA+4HcPc/AkeZWUfMdUVEZByZuycT2Ox8YJG7Xx6N\nXwac7u5XVizzL0Czu3/MzE4DfgucDhw93LoVMRYDiwE6OjrmL1myZFTl7e7upq2tbVTrTmTspOPX\na+yk46vsExO/XmMnHX8ssRcuXPiYuy+ItbC7J/ICzgdurhi/DLh+0DJTgO8AK4HvA8uBU+OsW+01\nf/58H61ly5aNet2JjJ10/HqNnXR8lX1i4tdr7KTjjyU2sMJj1uuZUaWkeDYBsyvGZ0XTdnP3TuD9\nAGZmwLPAOqB5uHVFRGR8JZkwlgNzzexowsr+IuCSygXMbBrQ6+F5isuBB92908yGXVekrBQ4hWKJ\nfDGgv1CiN1cMX/kiv1+3lZRZuGD0Vh6PRrFofPf03cvtXiAcx8DC0VyxxIZtXTQ3ZGjIpmlIp8hm\nUqRTulJ9IgXuBIFTCnyv4VLgFEoBhWKJYuAUigHFklMolSiWAnpz4feltTFDc2OGloYMjdkMDZnw\nf7v7ezSJJZYw3L1oZlcC9wJp4BZ3X2NmV0TzbwROBL5rZg6sAT64r3WTKqvUh0IpIF8sUSgG9OWL\n9ESJIVcoAVECMCObTpFJG2kzWhuzAAycqQuHyqfufM/JOL57XuDl5SrWjmYGAWzrylEs9e1Rxmw6\nRXNDmuaGzO5kkk2naMikyKTHL5kE7hRLAUHgFINwuFgKyBfD17NbOhP77DB+F0aYXHdXsxV528zC\nV3mWlfNyNM3CofL8VJS9S4Hz4s7evSr9YhBQiP7echTHd69f/g+mU+HnpsyiYWjMpmlqyLAtFX5f\nCqWA3q4cL5X/tx6Wp7khQ0tDmpamLE3ZNA2Z9Lj/Xydaki0M3P1u4O5B026sGH4EOC7uujJ+3J3+\nQolS4Gzv7icV/cjMjFQK0ruHLZo3sKc+FoF7VKmVyEethZ6otVAKBiqAdMrIpFNRBT301ziVGlym\n2uwlpgzamrJ7TS/vxfZ15ygG/biHZfaoLM3ZDC2NGZob0zRm0mQz4d+QTadibb9SMFBBlqL38vbK\nFQIK0XChFOz+a8uVZbmiLAVOV1+hJtuhehmdzt78HtPKiXbwJTZ7XnPjVaYNrOPu5AolNr3cs/t7\nV670GzIpGrPpMbcCUimjMZWmMZveq/zFwOnOFdnRkyeo+L9m0ymaGzO0Nob/23Iiifs/rSeJJgyp\nL4E7vbkiu3pybO3qJ18MyBVK/PnFvfdGywd1vKIKSKXCH3AmVd7DT5HJhHv6mXRqdyWfihJN4M7O\nnlx4GKm/QE+uSK4Q7BEzGx3qaWvMVqn89z/plJGuUuHAwF7/rt48L3cFBBU1oxk0ZcM92ObGDMXA\n2by9h1yUOHNRIiivUrnnXK44w1eYQFv3sa22RnvLSUkZtDQmE397ypjS0pBI7H0xM7Jpi3ZQ9pxX\nilo6W3P9FEvBHi2qloYsLVEiKSdSq9jBKr/boPH99fCXEsYkVywF9OSKbO/uZ3tXjiBw0mkLK53G\nLDtSxrTWxlixguhKisCjH1FQxAsQBB4d3vE99sz68yWeeWEXKTOyUdN+SjZ9wO2VlaXMor3Pvee5\nO8VSuAe7szdPvlDipV19pKM96Ww6RdMBvG3q2VA7CeX/abiD0E+uUOJPm3eWj3BVfa9U3gErt+bT\nKRuYVtGyT6dSFEvhbyvpRKOEMQnlCiW6+wu83NXPrt48OOFefNPY9uJT0YHovfetq9s+gmR0oDMz\nshkjG91LuzVlVQ95Sf3Y/T/NhP/TbSlj6gi+7+EOWPnWh3C8VHTyHrbBy9Pcw4swiqWAhkzcX9/o\nKGFMAu5OX75EZ1+elzv76c0VwaApm2ZqS4P2WkX2Q6ndVwwM//t8MfHShJQwDlClIDwfsbM3x8td\n/RQKAZYymhvSTGvTXr2IjJwSxgGkUAro6S+wvTvHju7y+YjwMs/y5aUiIqOlhFHHSkF4mWExcJ7e\nvIPO3vBSyYZMesznI0REBlPCqBPl5JArlOjqz9PdF9605jj5Qol8IdD5CBFJlBLGfmhwcujsLdCX\nL+6en4nuHJ7SksXM2J4ymhK8rl5EBJQwJlw5OfQXinT3FejsC5ND+brscnJQ60FEJpoSxjgqBWHn\neLlCaY/kUL5zJ5NO0ZjRpa4isn9SwkhYfyHs12f1+u0DyYHwRrkGJQcRqSNKGAnpyRV4cWcf2zv7\nKRQDUqa7mkWkvilh1JC709Vf4IUdvezqyZPNpJja2sD21ED3ACIi9UoJowYCd3b15Ni0vZfeXIGm\nhgzTdTe1iBxglDDGoBQEbO/OsWl7D4VCQHNjhultTRNdLBGRRChhjEK+WGJ7dz+bXu6lFDhtTVl1\nvSEiBzwljBHoL5TY2tnHizt6MaCtOatnOIvIpJFowjCzRcA3CJ/LfbO7f3nQ/KnAD4A5UVm+5u7f\nieY9B3QBJaDo7guSLOu+9OQKbNnZx9bOftIpY0pzg/ppEpFJJ/busZk1m9nxI1g+DdwAvA2YB1xs\nZvMGLfYh4El3PwU4G/i6mVU+AHGhu586EcnCPXzu8dObd7Jm/Q529uaZ1trAlBYli8ls6e1Zzn5t\nO29765s4+7XtLL29fg5FJl32JOPX83Y/kMRKGGb2DmAlcE80fqqZLR1mtdOAte6+zt3zwBLgvEHL\nONBu4Z1rbcB2oMgECtzZ0d3Pkxt28NTGHeQKJaa3NdLWlJ1UN9jpx7+3pbdn+dRVzWzelMLd2Lwp\nxaeuaq6LbZN02ZOMPx5lr8fv40Qw98FPkq2ykNljwJuAB9z91dG0J9z95H2scz6wyN0vj8YvA053\n9ysrlmkHlgInAO3Ahe7+82jes8AuwkNS33L3m4b4nMXAYoCOjo75S5YsGfbvqaa7u5umllYKxRLu\n4dOuapUfcv29NDa11CbYOMS//1cdfOPfTiSXG3jcY2NjiY/+/VO86ZyX9tvYlZ9x63eOZevWJmbO\n7Od97/9z7NilEnR2NrB9ewPbX24M37c3smNHA/fec/ge5S7LZkucceY22tqKtLUVovdBw+0FWlvD\n4Uym+m9uJNumVILengzdPRm6u7J0dWfo7s7S052huztDV1eW7u4MPd3hvFWPT6dYrLZ/6DQ1lWJt\nm33p709T/clwe8cf7nc1eH5fXxr3vVdqaCjxutdvpa013L5tbUVa24q0txVojbZ7ebi1tUi1043j\n8X0sS7IeyPX10j6lPcaz+fa2cOHCx+IexYl7DqPg7rsG7WEPn2mG91bClsubgGOB+8zsIXfvBM5y\n901mdkg0/Y/u/uDgAFEiuQlgwYIFfvbZZ4+4EIVSwC9/tYwps0+irSlb85vs1q1ewTGvTO6oWq3j\nf+D97eRye26DXC7Nt741j5Zpx1b9wVetBMz3WuZb32qqGvvmm+dx4qlH09DkNDZCY2PFexM0ROOZ\nzL4rnKW3Z7nuumb6+8KFtmxp5rrrTuKgQ4/htNcV2bYlxZYtxtaXUmzdYmx5KcW2rQPjL28zSqW9\nP2DKVCeXq/6ZhUKKjZtm0NVp7NplFPL7/tm2tDhTpla8pjhTpjn33Z0ll9tz3VwuzXXfOIkHHz6B\nzl1G567wM7q7qFqJlmUyYeypU8PYxeLQy17y3rEnjFu+NfSzpCvjD1dpVNt/vfWm6rHz+RTPPjeT\nXTuNrs7q/7cyM6etnXB7VLwefiBTdZv/4AfzuPyjs4cp7cgkWQ88s2o5rz/rL/abZ3qvMbNLgLSZ\nzQU+Avx2mHU2AZVbfFY0rdL7gS972MxZG7UqTgAedfdNAO6+xczuIDzEtVfCqIVSKcDdJ/XNdrl+\n+MNjaR55OMPmTdV/eLt2pvjsNc2JfP72l1O8/+LWYZdLpaJE0uQ0NEDDoOSyZnWa/KAKoL/PuPpj\ne+/ZpVLOwTOcmYc4MzsCTjwpfC+Pz5w58N7UDGe/tr3qtjn8COfeh7qBsMLL9cOuXWEltmun0dlp\nuyv7zl3ReMX0FzanePopo6en+t/c3w/5PHQcGjD3hCjBTHWmThtIOOXhqVOd9qlOS8ueiXVfZb/6\nn/qH3e7DueeubGLxf/HzoWPf95uB7d7TA507bY9tX07i5e29q+L/sG5tit7e6p+5eZNxzceamXdy\niZNOLnHCSSVakjtIUDfiJowPA9cCOeCHwL3AF4ZZZzkw18yOJkwUFwGXDFpmPXAO8JCZdQDHA+vM\nrBVIuXtXNPwW4J9jllViKJVg9ao0v3s4wyMPp3lseYZcv5FOO9kGKOT3XqfjsIDb/6t7r+nV9goH\nTyuPX/D2Nl56ce8W3IxDAq77Vi+5HORzRj4PuZyR64/eo+m5HOT6o/douXzFtPwQrQBwvvi1PmZ2\nOIccEjCzwznoYCc9gh2yf7imn09dNdB6AWhqdv7hmoEK0QyamsPpHYeOrBG+r0p9yZ1DZJOY4pR9\nf40fd7u3tUFbm3P4rPjbfaht3tgIy36Z4Sf/f3gNTirlHPOKYHcCOenkEvNeWaKtfQx/WB2KlTDc\nvZcwYVwbN7C7F83sSsLkkgZucfc1ZnZFNP9G4PPArWb2BOEB0E+4+zYzOwa4IzoElgF+6O73jODv\nkkHc4c/PpPjtQxkeeTjDo49k6OoMfyjHn1jiovfkOfOsIqedWeT++7JVf6BXXdvPzEPGdiTyqk9V\n//Ff/Zl+Fpw+9kMj+6p0L7ikMKbY574rXP9fv9TEC5uNww4PK63y9LFKstJNuuxJxk8y9lDb/Atf\n7eMd7yzw0ovGk0+kWb0qzZNPpPnv32ZY+pOBCzmPOiZMHCedXIqSScC06Xv+Rpbeno3K/qaab/fx\nFvek933ABe6+MxqfDixx97cmXL4RWbBgga9YsWLE6/Xni9y/7AHmvuq1CZQquWOXA1/E6j+izRuN\nRx4OE8TvfpNhy0vhnv2sOQFnnlXkzLOKnPH6IjNm7v0dGC52kuUea+yhKoBa/kgn6n9aC/V2Ti3p\n2CPd5tu2GmueCBNI+X3jhoFW86zZAy2Rri74wS2N9Pcn+318ZtVy/vKchaM6h2FmNT/pPaOcLADc\nfUd0MlomyOCKcfMm41Mfb+bx36coFMJE8fyz4Zfn4BkBZ7w+TBBn/kWR2XOG30k4910Fzn1XIZEf\naNKxIbk96aQluW2kupFu8xkznTe+qcgb3zRwB8DOHcaaJ1K7k8iaJ9L84u7ql+f29xlf/EwTRx4V\nMPvIgOkHec2uyExa3IQRmNkcd18PYGZHUpurpGSU/vVLTXvsRQP09xvfv6WJ1jbntDOKXPq+8DDT\ncScEdfOFrAVVujLepk13Xv+GEq9/w8Bh1a5OmH/iFKhyNduO7Sku+Os2ILxqbvaRAbNmB3u+zwmH\nm/dxsn2gdXQOs2fDv/wLXHppzf+83eImjGuBh83s14TnGv6C6N4HGV/usGplesgrmcyc5U92klEv\nYSITqn0KHH64V/2tzjwk4PP/u48N61NsXJ9iw/MpNqwPzzH2DdoRnDEzYPacMIFUJpQ/Ppni618a\nOMqwfj0sjmrlpJJG3JPe95jZa4Azokl/7+7bkimSVLP+eWPpTxpYenuW59alGaqBd9jhrmQhsp8Y\n6qT6Jz7Tz5vesnenFu6w/WVjw/NRIoleG9en+MOKDD+/0wiCoQ8X9PbCtddOcMKINBJ23ZEB5pkZ\n1W6kk9rZsd34r59lWfqTLL9fkcHMOf11JRZ/qJcggC98JrnLJEVk7EZ6Ts0MDp7hHDyjxKnz975q\nsFCAFzaHCeX9F7VS7e769etr+ifsIVbCMLOvABcCa4AgmuwkdCPdZJbrD6//vvMnDTx4f4ZCwZh7\nfImPfzK8zO+wIwZaFk3N9XtyV2SyqOU5tWwW5hzpzDmyxOFHVD/cNWfOmD5in+K2MP4GON7dh7wt\nSkYvCOCxR9P89McN3HNXlq5O45COgMs+kOfcd+c58aTqJ611cldk8qp2uKulBb74xeQ+M27CWAdk\nCe/0lhpZ+0yKO3+c5We3N7B5U4qWFuctby9w3rsKnHFWcUR3IYvI5DL4cFd4lZTtF1dJ9QIrzexX\nVCQNd/9IIqU6QFS7w/PMs4rc9dMsS3/SwJon0qTTzuvfWOQfr+nnnEUF9VcjIrGVjzKM5ca9kYib\nMJZGL4mp2o11/+sjzQQBgPHKU4pc+899vP28QtU7rUVE9jdxL6v9btIFOdBUu7EuCIy29oD/uKub\nV8wNhlhTRGT/FPcqqbnAlwgftdpUnu7uxyRUrroWBAx5Y11PtylZiEhdivukoO8A/074+NSFwPeA\nHyRVqHq2amWaC8+tfn00hDfWiYjUo7gJo9ndf0XYu+3z7v5Z4O3JFav+vLzN+OQ/NnPB21vZtCHF\nhe/J0dS8Z3LQjXUiUs/invTOmVkKeCZ6xsUmoC25YtWPQgF++N0GrvtaE3298IG/y/Ohj/XT1g6v\nPaOkG+tE5IARN2F8FGghfDTr5wmfwf3epApVLx55OM0XPt3MM0+nOeuNBa79536OrTg/oRvrRORA\nEvcqqeXRYDfhc7gntU0bja/8czP33JVl1pyAb97SwzlvLU6qLsRFZPKJe5XUAsIuzo+sXMfdX5VQ\nufZL/X1w8783ctP1jQB89Kp+PnhFjqbmCS6YiMg4iHtI6jbgKuAJBjofHJaZLQK+QfhM75vd/cuD\n5k8lvNpqTlSWr7n7d+KsO57c4b57Mnz5s81s3JDibe/I84lP94/oYfMiY+HuuIPjBB6Oh9MhKM9z\nx6NpA8N7f0dLgbOrJ7lefkqBs3NQfDP26pG/PGp7DNteS1SuVhm7MZumIZMinYp77Y6MVdyEsdXd\nR3Snt5mlgRuANwMbgeVmttTdn6xY7EPAk+7+DjObCTxtZrcBpRjrjou1z6T44qeb+M2DWY47ocT3\n/rObM16/d7fDk0lYQTlBEFVgQVg5DX539z0qispKwfGBH79DOp2iMZsim05h++mxPXenUArIF4Pd\nle7g6rhc8qGmM+T8PdccXOmmDFKWIpWClBnpVIp0CsyMdMpIpVKkDVKpaDyabhYOmxG9G10b0pw4\na/roN8QwOjekmTd7IH75b6v8t+4ejCZWbp9q//7yd+LhjRlOOGI6ffki3X15OvsLFIqF3UmnIRMm\nkUxaSSQJcRPGP5nZzcDgvqRu38c6pwFr3X0dgJktAc4DKit9B9ot/Da0ET5vowicHmPdRHV1wvX/\np4nvf7uBllb41Of7uOS9+UnxcKJCMSBXLFEsVa8Y01GllE6lyKSNhnSaTMZIm5FJp0iljEzKSEUV\n1+53s7Diiyqy32zM8KojD6a/UKKnv8Cu3jydvQXCNDSxP/7K5FAslaJKz2hpyjBzShPbs2mOO2Ia\nsHeFOFAZMsz8PSvLyoryNxszvOaYGUC4zWqZRFNmtDZVf950zeI3JhPfgPbmLO3NWQ6ZGh4LLpQC\ncoUS/fkiXf0FuvoKdPcPXI3YkEmTzYQ7IzI2Vq3JutdCZj8ATmDQ8zDc/QP7WOd8YJG7Xx6NXwac\n7u5XVizTTthH1QlAO3Chu/88zroVMRYTPS62o6Nj/pIlS4b/qwdxh66uLhqbWwgC+OUvD+M73z6W\nnTsbeOuizbzv/X9m2rTRXw6b6++lsSm5XgXHEr/yEEZZea81nTJ6e3poa28b4jbEsenu7qatbe+r\ns8uHXEpB+BpctlTMwoxku5RbRJU/h3KyK++xV9bZQ5W9VpKMP1nKHnh4+C6o+j0Kv0uVX6X9+Xc6\nbOy+XtqntI/qd7pw4cLH3D3WZZxx95df6+7Hj6Isw3krsJLwMt1jgfvM7KGRBHD3m4CbABYsWOBn\nn332iApw221wzSedjRvg4JlOc7Oz4fk0p84vcvMXejj5lDbglBHFHCzpy2rjxi/vNecKJUolB4NM\nKsWUliztzQ20NGZoyqb32KN/4IEHGOk2jStu7EIpoC9fpDdXpLM3T1dfgVJ4QJ9sJkVjtAc52FDb\npVAKKBQD8sWBQ4vNDRmmNGdpa87SlM3QmE2T3kdmSnK7JB1/spa9FAT0F0r058MWbWdfgf5CEcKv\nEtvWrWL2Ca+p+l2qhSTrgWdWLef1Z/3FftNb7W/NbN4IzyFsAmZXjM+KplV6P/BlD1P/WjN7lrC1\nEWfdMbvttvCh6b29YcWwbUt4JPTCy3J87kv91Pu5tGJ0SCVfLJUPitDalOHQaS20NmVpik4a7q/n\nDMqy6RTZ5gamNDdw6LQW3D08BFEo0dkXHsaqPN4fngwNfzjlbVAolnYfVmvKppna0kB7c5amhjRN\n2bROnE4C6VSK1sYUrY1ZDm4Pu8QrBQG5QrgTteO5FKUgoKenEC1vNGbT+/V5tfEWN2GcQfg8jGcJ\nz2EY4SGpfV1WuxyYa2ZHE1b2FwGXDFpmPXAO8JCZdQDHEz6saWeMdcfs2mvDh6bvyXjo/iypVH11\n4RG4ky+UyBcDgqjp3ZAZqBibG4bfa64XZkZTQ4amhgzTWsNLnEtBQH++RG++ohUShC2qKS1Z2pta\ndm8DnRCVsnQqRUtjipbGDA2ZFCcfefDuFm1Pf4GdPXk6+/K4h9+7xkyKhmya1CRNIHETxqKRBnb3\nYtSNyL2El8be4u5rzOyKaP6NhHeN32pmTxAmoU+4+zaAauuOtAzDGeph6S9sro8vQ+BOd1QxdvcV\naG/OMnNqMy2NGZobMpPqJF86laK1KUVrU5aZU5pxd3Y+H55UFxmJyhbtYdNbd++MdOfCBNLdV9hj\np+xA2RGLY9iEEV0ee6+7nzDS4O5+N3D3oGk3VgxvBt4Sd91amzMHnn9+7+n7e4+y7k53f5FiEHDY\ntBY6G8IratRsHqBtIbVSuTPSMbWFwD1szeYKdPbm2dVboFgKwMJkcyC3YodNGO5eMrOnzWyOuw+x\nT16fvvjF8jmMgWn7c4+y7k5PrkihGHDI1CYOnd5KUzbNn2t82aWIDC1lRktjhpbGDDOi1myuGNAf\nHQ7d0ZMPL+utuL8o6ZPR4yXuIanpwBozexToKU9093MTKdU4KT8svXyV1P7co2xvrkiuUOKg9kaO\nOKiV5oZJcEOISB0wM5qy4cUT01obmTMT8sUSffkSXX15dvbk2dmToxQ4nb15Gur4PEjcWufTiZZi\nAl16Kbz7ghL3L3uAua967UQXZy99+SJ9+SLTWxt5xWFTErshSkRqJ7zpNLzoZNbB4dV6D25IM+vg\nVnb2RudBAq+7w1hxe6v9dXQVU7lGfdTdtyRXLMkVwmOk7S0NHH3IQbQ3K1GI1KtMOkXKjI5pLXRU\nXBq+uxXJ6NFKAAAUMUlEQVTSO3AYK1W+nDeT2u9aIXF7q/0fwFeBBwivZvp/zewqd/9xgmWblPLF\nEt39RVobwz5z2puzOj8hcoCpvDR8elsjc6i8QbXArqpXY018R4txD0ldS3i39xaAqKPAXwJKGDVS\nKAZ09+dpbshw3GFTmNbaqEQhMonseYNq6+6rsfryRTr78uzqzVMoFMAm7qbCuAkjNegQ1MvEfx64\n7EOxFNDdXyCbSXHsoVOZ3ta43zVDRWT8VV6NVb4zPV8Muzbpruiss9yV/XiImzDuMbN7gR9F4xeS\n8D0SB7pSENDVlyeTTnHUIe0c1NY0aW7+EZHRKZ9Mn9LSwOEHtVIKnP5Ckc4N43PSfJ8Jw8wa3T3n\n7leZ2buAs6JZN7n7HYmX7gAU3pWdJ5Uy5sxoZ8aUpgk/Liki9SmdCruSL/eonLThWhiPAK8xs++7\n+2XAvp5/IfsQBE5Xf/ish8MPbmXmlOZJ1XWHiNS/4RJGg5ldArwuamHsYZgHKEmk5OENO4ce1ELH\n1OYD5q5PEZlchksYVwCXAtOAdwya56jFsU+lIGBXT550ynjVUQfTmFWiEJH6tc+E4e4Pm9lvgY3u\n/sVxKtMBoTdXJF8sceyhU1i9Oa1kISJ1b9iD6O4eAOePQ1kOCIE7O7pzZNPGyXMOYsaU5okukohI\nTcQ96/orM3u36U6yfcoXS+zqyXPEQS0cf8R0mtRBoIgcQOLWaH8H/ANQMrM+Bp64NyWxktWZzt7w\nXMUJs6YxpblhoosjIlJzcTsfbE+6IPWqfGL74ClNHDmzXZfKisgBK1btZqH3mNmno/HZZnZaskXb\n//XminT1FTj20Ckc2zFFyUJEDmhxa7hvAmcCl0Tj3cANw61kZouip/WtNbOrq8y/ysxWRq/VZlYy\ns4Oiec+Z2RPRvBUxyzkuqp3Y1ukdETnQxT2Hcbq7v8bM/gDg7jvMbJ8H6qNngd8AvBnYCCw3s6Xu\n/mR5GXf/KmG36ZjZO4CPufv2ijAL3X1b/D8nefliiZ7+Ikcc1MKh01vV/5OITBpxE0YhSgAOu7s3\nD4ZZ5zRgrbuvi9ZZApwHPDnE8hcz0LnhfkkntkVkMjP34TvGNbNLCXuofQ3wXcL7Mj7l7v+5j3XO\nBxa5++XR+GWELZUrqyzbQtgKeUW5hWFmzwK7gBLwLXe/aYjPWQwsBujo6Ji/ZMmSYf+ewdyhq6uL\nxuaW6vMJOw3MpI2GTJqRtim6u7tpa2sbcbn2h/j1Gjvp+Cr7xMSv19hJxx9L7IULFz7m7gtiLezu\nsV7ACcCHgCuBE2Msfz5wc8X4ZcD1Qyx7IfCzQdOOiN4PAR4H3jDcZ86fP99Hoy9X8J/fc5//afPO\nvV4rn93my9du8a27ej0IglHFX7Zs2ajW2x/i12vspOOr7BMTv15jJx1/LLGBFR4zDwzXvXkTYX9S\nrwCeINzTL8ZMXJuA2RXjs6Jp1VzEoMNR7r4pet9iZncQHuJ6MOZnj1ngzq6ePG1NGY4/XDfhiYgM\nd5XUd4EFhMnibcDXRhB7OTDXzI6OTpBfBCwdvJCZTQXeCNxZMa3VzNrLw8BbgNUj+Owx0R3bIiJ7\nG64mnOfuJwOY2beBR+MGdveimV0J3AukgVvcfY2ZXRHNvzFa9J3AL9y9p2L1DuCO6FLVDPBDd78n\n7mePhU5si4hUN1zCKJQHogQwouDufjeDHuVakSjK47cCtw6atg44ZUQfNkYObO/q1x3bIiJDGC5h\nnGJmndGwAc3R+IHVl5QZKYNjD53Cwe1NuglPRKSK4Z6HMSke4tCUTdPUkFFX5CIi+6DjLhG1KURE\n9k0JQ0REYlHCEBGRWJQwREQkFiUMERGJRQlDRERiUcIQEZFYlDBERCQWJQwREYlFCUNERGJRwhAR\nkViUMEREJBYlDBERiUUJQ0REYlHCEBGRWJQwREQklkQThpktMrOnzWytmV1dZf5VZrYyeq02s5KZ\nHRRnXRERGV+JJQwzSwM3AG8D5gEXm9m8ymXc/avufqq7nwpcA/za3bfHWVdERMZXki2M04C17r7O\n3fPAEuC8fSx/MfCjUa4rIiIJM3dPJrDZ+cAid788Gr8MON3dr6yybAuwEXhF1MIYybqLgcUAHR0d\n85csWTKq8nZ3d9PW1jaqdScydtLx6zV20vFV9omJX6+xk44/ltgLFy58zN0XxFrY3RN5AecDN1eM\nXwZcP8SyFwI/G826la/58+f7aC1btmzU605k7KTj12vspOOr7BMTv15jJx1/LLGBFR6zXk/ykNQm\nYHbF+KxoWjUXMXA4aqTriojIOEgyYSwH5prZ0WbWQJgUlg5eyMymAm8E7hzpuiIiMn4ySQV296KZ\nXQncC6SBW9x9jZldEc2/MVr0ncAv3L1nuHWTKquIiAwvsYQB4O53A3cPmnbjoPFbgVvjrCsiIhNH\nd3qLiEgsShgiIhKLEoaIiMSihCEiIrEoYYiISCxKGCIiEosShoiIxKKEISIisShhiIhILEoYIiIS\nixKGiIjEooQhIiKxKGGIiEgsShgiIhKLEoaIiMSihCEiIrEoYYiISCxKGCIiEkuiCcPMFpnZ02a2\n1syuHmKZs81spZmtMbNfV0x/zsyeiOatSLKcIiIyvMSe6W1maeAG4M3ARmC5mS119ycrlpkGfBNY\n5O7rzeyQQWEWuvu2pMooIiLxJdnCOA1Y6+7r3D0PLAHOG7TMJcDt7r4ewN23JFgeEREZgyQTxhHA\nhorxjdG0SscB083sATN7zMz+tmKeA7+Mpi9OsJwiIhKDuXsygc3OJzzUdHk0fhlwurtfWbHM9cAC\n4BygGXgEeLu7/8nMjnD3TdFhqvuAD7v7g1U+ZzGwGKCjo2P+kiVLRlXe7u5u2traRrXuRMZOOn69\nxk46vso+MfHrNXbS8ccSe+HChY+5+4JYC7t7Ii/gTODeivFrgGsGLXM18LmK8W8DF1SJ9Vng48N9\n5vz58320li1bNup1JzJ20vHrNXbS8VX2iYlfr7GTjj+W2MAKj1mvJ3lIajkw18yONrMG4CJg6aBl\n7gTOMrOMmbUApwNPmVmrmbUDmFkr8BZgdYJlFRGRYSR2lZS7F83sSuBeIA3c4u5rzOyKaP6N7v6U\nmd0DrAIC4GZ3X21mxwB3mFm5jD9093uSKquIiAwvsYQB4O53A3cPmnbjoPGvAl8dNG0dcEqSZRMR\nkZHRnd4iIhKLEoaIiMSihCEiIrEoYYiISCxKGCIiEosShoiIxKKEISIisShhiIhILEoYIiISixKG\niIjEooQhIiKxKGGIiEgsShgiIhKLEoaIiMSihCEiIrEoYYiISCxKGCIiEosShoiIxKKEISIisSSa\nMMxskZk9bWZrzezqIZY528xWmtkaM/v1SNYVEZHxk0kqsJmlgRuANwMbgeVmttTdn6xYZhrwTWCR\nu683s0PirisiIuMryRbGacBad1/n7nlgCXDeoGUuAW539/UA7r5lBOuKiMg4SqyFARwBbKgY3wic\nPmiZ44CsmT0AtAPfcPfvxVwXADNbDCyORrvN7OlRlncGsG2U605k7KTj12vspOOr7BMTv15jJx1/\nLLGPjLtgkgkj7ufPB84BmoFHzOx3Iwng7jcBN421IGa2wt0XjDXOeMdOOn69xk46vso+MfHrNXbS\n8ZMue1mSCWMTMLtifFY0rdJG4GV37wF6zOxB4JRo+nDriojIOEryHMZyYK6ZHW1mDcBFwNJBy9wJ\nnGVmGTNrITzs9FTMdUVEZBwl1sJw96KZXQncC6SBW9x9jZldEc2/0d2fMrN7gFVAANzs7qsBqq2b\nVFkjYz6sNUGxk45fr7GTjq+yT0z8eo2ddPykyw6Auft4fI6IiNQ53ektIiKxKGGIiEgskz5hmNkt\nZrbFzFYnEHu2mS0zsyejrk8+WsPYTWb2qJk9HsX+XK1iV3xG2sz+YGZ3JRD7OTN7IuoWZkWNY08z\nsx+b2R/N7CkzO7OGsY+Pylx+dZrZ39cw/sei/+dqM/uRmTXVKnYU/6NR7DVjLXe1346ZHWRm95nZ\nM9H79BrHvyAqe2Bmo76MdIjYX42+M6vM7I6oJ4paxf58FHelmf3CzA6vZdkr5v2jmbmZzRht/H1y\n90n9At4AvAZYnUDsw4DXRMPtwJ+AeTWKbUBbNJwF/hs4o8bl/wfgh8BdCWyb54AZCf1PvwtcHg03\nANMS+pw08CJwZI3iHQE8CzRH4/8BvK+G5X0lsBpoIbzg5ZfAK8YQb6/fDvC/gauj4auBr9Q4/onA\n8cADwIIax34LkImGvzLasg8Re0rF8EeAG2tZ9mj6bMILhZ5P6rc16VsY7v4gsD2h2C+4+++j4S7C\nS4aPqFFsd/fuaDQbvWp2BYOZzQLeDtxcq5jjwcymEv6gvg3g7nl335nQx50D/Nndn69hzAzQbGYZ\nwop9cw1jnwj8t7v3unsR+DXwrtEGG+K3cx5hwiZ6/5taxnf3p9x9tL05DBf7F9F2Afgd4f1ftYrd\nWTHayhh+q/uos/4P8L/GEns4kz5hjBczOwp4NWFLoFYx02a2EtgC3OfuNYsN/Bvhly+oYcxKDvzS\nzB6LuneplaOBrcB3osNpN5tZaw3jV7oI+FGtgrn7JuBrwHrgBWCXu/+iVvEJWxd/YWYHR/c9/RV7\n3iBbCx3u/kI0/CLQUeP44+UDwH/VMqCZfdHMNgCXAp+pcezzgE3u/ngt4w6mhDEOzKwN+Anw94P2\nNMbE3UvufirhntBpZvbKWsQ1s78Gtrj7Y7WIN4SzorK/DfiQmb2hRnEzhM31f3f3VwM9hIdGaiq6\nofRc4D9rGHM64R760cDhQKuZvadW8d39KcJDLb8A7gFWAqVaxa/yeU6Ce7tJMbNrgSJwWy3juvu1\n7j47intlreJGyf+T1DgJVaOEkTAzyxImi9vc/fYkPiM65LIMWFSjkK8HzjWz5wh7Cn6Tmf2gRrGB\n3XvTeNhD8R2EPRTXwkZgY0Vr68eECaTW3gb83t1fqmHMvwSedfet7l4AbgdeV8P4uPu33X2+u78B\n2EF4Xq2WXjKzwwCi9y3DLL9fMbP3AX8NXBolvCTcBry7hvGOJdzJeDz6zc4Cfm9mh9bwMwAljESZ\nmREeS3/K3f+1xrFnlq/iMLNmwmeH/LEWsd39Gnef5e5HER52ud/da7ana2atZtZeHiY82ViTq9Tc\n/UVgg5kdH006B0jiOSoXU8PDUZH1wBlm1hJ9d84hPO9VMzbwzJk5hOcvfljL+IRd+Lw3Gn4vYfc/\ndcHMFhEehj3X3XtrHHtuxeh51Oi3CuDuT7j7Ie5+VPSb3Uh4sc2LtfqMyg+b1C/CH/0LQCHa0B+s\nYeyzCJvkqwib/yuBv6pR7FcBf4hirwY+k9D2OZsaXyUFHAM8Hr3WANfWOP6pwIpo2/wUmF7j+K3A\ny8DUBLb35wgrk9XA94HGGsd/iDCBPg6cM8ZYe/12gIOBXwHPEF6FdVCN478zGs4BLwH31jD2WsLH\nKpR/q6O6kmmI2D+J/qergJ8BR9Ryuwya/xwJXSWlrkFERCQWHZISEZFYlDBERCQWJQwREYlFCUNE\nRGJRwhARkViUMGS/F/W++fWK8Y+b2WdrFPtWMzu/FrGG+ZwLop5zlw2afpSZ9Q3qAbdhFPGPMrNL\naldikb0pYUg9yAHvSqzL5lGKOgiM64PA/3T3hVXm/dndT6145UdRnKOAEScMM0uP4rNkklLCkHpQ\nJHxm8ccGzxjcQjCz7uj9bDP7tZndaWbrzOzLZnaphc8QecLMjq0I85dmtsLM/hT1o1Xu2PGrZrY8\neo7B31XEfcjMllLlDnIzuziKv9rMvhJN+wzhTZzfNrOvxvmDo7vhb4nK+4eoc7lyS+IhM/t99Cp3\nHfJlwo4FV1r4TI33mdn1FfHuMrOzy9vIzL5uZo8DZ5rZ/GhbPWZm91Z07fERC5/lssrMlsQptxzg\nkrgbUC+9avkCuoEphHewTgU+Dnw2mncrcH7lstH72cBOwmeSNAKbgM9F8z4K/FvF+vcQ7jzNJbxz\ntglYDHwqWqaR8M7xo6O4PcDRVcp5OGH3HjMJO0G8H/ibaN4DVHl+A2HLoI+Bu4tviKb/C/CeaHga\nYZ9PrYRdnjdF0+cCKyr+3rsq4r4PuL5i/C7g7GjYgf8RDWeB3wIzo/ELgVui4c1Ed5qT0DNF9Kqv\n10ia1CITxt07zex7hA+f6Yu52nKPuto2sz8T9tIK8ARQeWjoP9w9AJ4xs3XACYT9W72qovUylbCC\nzgOPuvuzVT7vtcAD7r41+szbCJ/N8dNhyvlnD3vurfQWwg4gPx6NNwFzCCvx683sVMKeZo8bJnY1\nJcKuKiB8GNErgfvC7qtIE3Y7AWE3FreZ2U9j/A0yCShhSD35N+D3wHcqphWJDq2aWYrwCXtluYrh\noGI8YM/v/uD+cZzwiYYfdvd7K2dEh3V6Rlf8ETHg3T7oYUHRyf6XgFMI/+7+IdbfvV0ilY967Xf3\ncrfmBqxx92qPsX07YcJ7B3CtmZ3sAw8YkklI5zCkbrj7dsLHln6wYvJzwPxo+FzCQywjdYGZpaLz\nGscATxM+6vL/ibqnx8yOs+EfxPQo8EYzmxGdTL6Y8Kl2o3Ev8OGo11rM7NXR9KnAC1GL6DLCFgFA\nF+FjgMueA06N/q7ZDN19/NPATIuee25mWTM7KUq+s919GfCJ6HPbRvm3yAFCLQypN19nz4fP/H/A\nndEJ3HsY3d7/esLKfgpwhbv3m9nNhOcXfh9V2lsZ5nGj7v6CmV1N+GwSA37u7qPt3vvzhC2qVVHl\n/Szhcxq+CfzEzP6WPf/eVUAp2g63Rus+S3hi/inCllm1Muejw27XWfh420y07p+AH0TTDLjOk3vU\nrdQJ9VYrIiKx6JCUiIjEooQhIiKxKGGIiEgsShgiIhKLEoaIiMSihCEiIrEoYYiISCz/F6Er2AyL\nq5W5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11c4415c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from mlxtend.feature_selection import SequentialFeatureSelector as SFS\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "%matplotlib inline\n",
    "\n",
    "# Load data\n",
    "df = pd.read_csv('data/titanic_modified.csv', index_col=0)\n",
    "df.head()\n",
    "\n",
    "# Create train and test set\n",
    "X = df.drop('Survived', axis=1)  # Keep all features except 'Survived'\n",
    "y = df['Survived']  # Just keep 'Survived'\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=7)\n",
    "\n",
    "# Create classifier\n",
    "lr = LogisticRegression()\n",
    "\n",
    "# Start variables\n",
    "max_features = np.shape(X_train)[1]\n",
    "best_score = 0\n",
    "best_model = ''\n",
    "\n",
    "# Sequential forward selection\n",
    "sfs = SFS(lr,\n",
    "          k_features = (1,max_features), # Select the best feature combination in the range 1 to max_features\n",
    "          forward = True,\n",
    "          floating = False,\n",
    "          scoring = 'accuracy',\n",
    "          cv = 10,  # Performs stratified k-fold by default.\n",
    "          n_jobs = -1)\n",
    "sfs = sfs.fit(X_train.values, y_train) # Use .values because the method can't read DataFrame data structures.\n",
    "\n",
    "if sfs.k_score_ > best_score:  # To select the best model and visualize the results (DataFrame and plot)\n",
    "    best_model = sfs\n",
    "    \n",
    "print('Sequential forward selection (%i features): ' % np.size(sfs.k_feature_idx_))\n",
    "print(sfs.k_feature_idx_)\n",
    "print('CV score: ')\n",
    "print(sfs.k_score_)\n",
    "\n",
    "# Sequential backward selection\n",
    "sbs = SFS(lr, \n",
    "          k_features = (1,max_features),\n",
    "          forward = False, \n",
    "          floating = False, \n",
    "          scoring = 'accuracy',\n",
    "          cv = 10, \n",
    "          n_jobs = -1)\n",
    "sbs = sbs.fit(X_train.values, y_train)\n",
    "\n",
    "if sbs.k_score_ > best_score:\n",
    "    best_model = sbs\n",
    "\n",
    "print('Sequential backward selection (%i features): ' % np.size(sbs.k_feature_idx_))\n",
    "print(sbs.k_feature_idx_)\n",
    "print('CV score: ')\n",
    "print(sbs.k_score_)\n",
    "\n",
    "# Sequential forward floating selection\n",
    "sffs = SFS(lr, \n",
    "           k_features = (1,max_features), \n",
    "           forward = True, \n",
    "           floating = True, \n",
    "           scoring = 'accuracy', \n",
    "           cv = 10, \n",
    "           n_jobs = -1)\n",
    "sffs = sffs.fit(X_train.values, y_train)\n",
    "\n",
    "if sffs.k_score_ > best_score:\n",
    "    best_model = sffs\n",
    "\n",
    "print('Sequential forward floating selection (%i features): ' % np.size(sffs.k_feature_idx_))\n",
    "print(sffs.k_feature_idx_)\n",
    "print('CV score: ')\n",
    "print(sffs.k_score_)\n",
    "\n",
    "# Sequential backward floating selection\n",
    "sbfs = SFS(lr, \n",
    "           k_features = (1,max_features), \n",
    "           forward = False,\n",
    "           floating = True,\n",
    "           scoring = 'accuracy', \n",
    "           cv = 10, \n",
    "           n_jobs = -1)\n",
    "sbfs = sbfs.fit(X_train.values, y_train)\n",
    "\n",
    "if sbfs.k_score_ > best_score:\n",
    "    best_model = sbfs\n",
    "\n",
    "print('Sequential backward floating selection (%i features): ' % np.size(sbfs.k_feature_idx_))\n",
    "print(sbfs.k_feature_idx_)\n",
    "print('CV score: ')\n",
    "print(sbfs.k_score_)\n",
    "\n",
    "# Visualize results\n",
    "print('Best model: %s' % best_model)\n",
    "## Sequential forward floating selection (dataframe)\n",
    "pd.DataFrame.from_dict(best_model.get_metric_dict()).T\n",
    "\n",
    "## Sequential forward floating selection (plot)\n",
    "from mlxtend.plotting import plot_sequential_feature_selection as plot_sfs\n",
    "\n",
    "fig = plot_sfs(best_model.get_metric_dict(), kind='std_dev')\n",
    "plt.ylim([0.6,1])\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}