H2O_AutoML.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "H2O AutoML.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/srivatsan88/YouTubeLI/blob/master/H2O_AutoML.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ogrpiMMOU9Dw",
        "colab_type": "text"
      },
      "source": [
        "Link to supporting video for below notebook walkhthrough - https://www.youtube.com/watch?v=91QljBnvM7s"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BiTtGL6IcA8J",
        "colab_type": "code",
        "outputId": "bb894ecc-022e-4b0a-e352-dd7e4a54d9a4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "source": [
        "#this command might be relevant ony on GPU nodes. But the notebook will run on CPU as well\n",
        "#If running on CPU remove the command wherever it appears\n",
        "!nvidia-smi"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mon Jan  6 01:16:37 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 440.44       Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   33C    P0    26W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0GA_-zrlpEa5",
        "colab_type": "code",
        "outputId": "638955f9-9f84-4549-958d-0e7b0c935c98",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "!apt-get install default-jre\n",
        "!java -version"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "default-jre is already the newest version (2:1.11-68ubuntu1~18.04.1).\n",
            "default-jre set to manually installed.\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-430\n",
            "Use 'apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 7 not upgraded.\n",
            "openjdk version \"11.0.5\" 2019-10-15\n",
            "OpenJDK Runtime Environment (build 11.0.5+10-post-Ubuntu-0ubuntu1.118.04)\n",
            "OpenJDK 64-Bit Server VM (build 11.0.5+10-post-Ubuntu-0ubuntu1.118.04, mixed mode, sharing)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eHZ8z6iGzZew",
        "colab_type": "code",
        "outputId": "3d5f2fff-3dc9-44ee-d78a-87b657d293c9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 340
        }
      },
      "source": [
        "!pip install h2o"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting h2o\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6b/d1/aaa74df4716b2454d6034261807ef3855d014862c801f0e9b803b568006d/h2o-3.28.0.1.tar.gz (123.4MB)\n",
            "\u001b[K     |████████████████████████████████| 123.4MB 71kB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from h2o) (2.21.0)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.6/dist-packages (from h2o) (0.8.6)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from h2o) (0.16.0)\n",
            "Collecting colorama>=0.3.8\n",
            "  Downloading https://files.pythonhosted.org/packages/c9/dc/45cdef1b4d119eb96316b3117e6d5708a08029992b2fee2c143c7a0a5cc5/colorama-0.4.3-py2.py3-none-any.whl\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->h2o) (1.24.3)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->h2o) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->h2o) (2019.11.28)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->h2o) (2.8)\n",
            "Building wheels for collected packages: h2o\n",
            "  Building wheel for h2o (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for h2o: filename=h2o-3.28.0.1-py2.py3-none-any.whl size=123410554 sha256=168ca8e2cbf809df8675b09983d1f1374c69d2043856b7abad946d0a207736eb\n",
            "  Stored in directory: /root/.cache/pip/wheels/ff/82/f4/3fe9c895b93c53b25ae44dc44aa54dad571c711f48bc485fe9\n",
            "Successfully built h2o\n",
            "Installing collected packages: colorama, h2o\n",
            "Successfully installed colorama-0.4.3 h2o-3.28.0.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m0g8oq01zcgM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        " import h2o"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z2WAtRd4zixl",
        "colab_type": "code",
        "outputId": "838e4362-b62d-4457-d880-de3f8ee35999",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 562
        }
      },
      "source": [
        "h2o.init()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Checking whether there is an H2O instance running at http://localhost:54321 ..... not found.\n",
            "Attempting to start a local H2O server...\n",
            "  Java Version: openjdk version \"11.0.5\" 2019-10-15; OpenJDK Runtime Environment (build 11.0.5+10-post-Ubuntu-0ubuntu1.118.04); OpenJDK 64-Bit Server VM (build 11.0.5+10-post-Ubuntu-0ubuntu1.118.04, mixed mode, sharing)\n",
            "  Starting server from /usr/local/lib/python3.6/dist-packages/h2o/backend/bin/h2o.jar\n",
            "  Ice root: /tmp/tmp0ikb58_p\n",
            "  JVM stdout: /tmp/tmp0ikb58_p/h2o_unknownUser_started_from_python.out\n",
            "  JVM stderr: /tmp/tmp0ikb58_p/h2o_unknownUser_started_from_python.err\n",
            "  Server is running at http://127.0.0.1:54321\n",
            "Connecting to H2O server at http://127.0.0.1:54321 ... successful.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
              "<td>02 secs</td></tr>\n",
              "<tr><td>H2O cluster timezone:</td>\n",
              "<td>Etc/UTC</td></tr>\n",
              "<tr><td>H2O data parsing timezone:</td>\n",
              "<td>UTC</td></tr>\n",
              "<tr><td>H2O cluster version:</td>\n",
              "<td>3.28.0.1</td></tr>\n",
              "<tr><td>H2O cluster version age:</td>\n",
              "<td>20 days </td></tr>\n",
              "<tr><td>H2O cluster name:</td>\n",
              "<td>H2O_from_python_unknownUser_w5qqra</td></tr>\n",
              "<tr><td>H2O cluster total nodes:</td>\n",
              "<td>1</td></tr>\n",
              "<tr><td>H2O cluster free memory:</td>\n",
              "<td>2.938 Gb</td></tr>\n",
              "<tr><td>H2O cluster total cores:</td>\n",
              "<td>2</td></tr>\n",
              "<tr><td>H2O cluster allowed cores:</td>\n",
              "<td>2</td></tr>\n",
              "<tr><td>H2O cluster status:</td>\n",
              "<td>accepting new members, healthy</td></tr>\n",
              "<tr><td>H2O connection url:</td>\n",
              "<td>http://127.0.0.1:54321</td></tr>\n",
              "<tr><td>H2O connection proxy:</td>\n",
              "<td>{'http': None, 'https': None}</td></tr>\n",
              "<tr><td>H2O internal security:</td>\n",
              "<td>False</td></tr>\n",
              "<tr><td>H2O API Extensions:</td>\n",
              "<td>Amazon S3, XGBoost, Algos, AutoML, Core V3, TargetEncoder, Core V4</td></tr>\n",
              "<tr><td>Python version:</td>\n",
              "<td>3.6.9 final</td></tr></table></div>"
            ],
            "text/plain": [
              "--------------------------  ------------------------------------------------------------------\n",
              "H2O cluster uptime:         02 secs\n",
              "H2O cluster timezone:       Etc/UTC\n",
              "H2O data parsing timezone:  UTC\n",
              "H2O cluster version:        3.28.0.1\n",
              "H2O cluster version age:    20 days\n",
              "H2O cluster name:           H2O_from_python_unknownUser_w5qqra\n",
              "H2O cluster total nodes:    1\n",
              "H2O cluster free memory:    2.938 Gb\n",
              "H2O cluster total cores:    2\n",
              "H2O cluster allowed cores:  2\n",
              "H2O cluster status:         accepting new members, healthy\n",
              "H2O connection url:         http://127.0.0.1:54321\n",
              "H2O connection proxy:       {'http': None, 'https': None}\n",
              "H2O internal security:      False\n",
              "H2O API Extensions:         Amazon S3, XGBoost, Algos, AutoML, Core V3, TargetEncoder, Core V4\n",
              "Python version:             3.6.9 final\n",
              "--------------------------  ------------------------------------------------------------------"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9kc8CZBJzt6b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from h2o.automl import H2OAutoML"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j-ZCVNqg5DMo",
        "colab_type": "code",
        "outputId": "d8ca5600-03f6-402d-b7aa-6a64a46495ad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "churn_df = h2o.import_file('https://raw.githubusercontent.com/srivatsan88/YouTubeLI/master/dataset/WA_Fn-UseC_-Telco-Customer-Churn.csv')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aRFEg-iE1LbK",
        "colab_type": "code",
        "outputId": "9961cd33-049e-465a-e90c-adc6c5890318",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        }
      },
      "source": [
        "churn_df.types"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'Churn': 'enum',\n",
              " 'Contract': 'enum',\n",
              " 'Dependents': 'enum',\n",
              " 'DeviceProtection': 'enum',\n",
              " 'InternetService': 'enum',\n",
              " 'MonthlyCharges': 'real',\n",
              " 'MultipleLines': 'enum',\n",
              " 'OnlineBackup': 'enum',\n",
              " 'OnlineSecurity': 'enum',\n",
              " 'PaperlessBilling': 'enum',\n",
              " 'Partner': 'enum',\n",
              " 'PaymentMethod': 'enum',\n",
              " 'PhoneService': 'enum',\n",
              " 'SeniorCitizen': 'int',\n",
              " 'StreamingMovies': 'enum',\n",
              " 'StreamingTV': 'enum',\n",
              " 'TechSupport': 'enum',\n",
              " 'TotalCharges': 'real',\n",
              " 'customerID': 'string',\n",
              " 'gender': 'enum',\n",
              " 'tenure': 'int'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R2nAk2Pf5fF1",
        "colab_type": "code",
        "outputId": "bf54929a-e782-4020-9d0a-9296b51883d4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 655
        }
      },
      "source": [
        "churn_df.describe()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Rows:7043\n",
            "Cols:21\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table>\n",
              "<thead>\n",
              "<tr><th>       </th><th>customerID  </th><th>gender  </th><th>SeniorCitizen     </th><th>Partner  </th><th>Dependents  </th><th>tenure            </th><th>PhoneService  </th><th>MultipleLines   </th><th>InternetService  </th><th>OnlineSecurity  </th><th>OnlineBackup  </th><th>DeviceProtection  </th><th>TechSupport  </th><th>StreamingTV  </th><th>StreamingMovies  </th><th>Contract      </th><th>PaperlessBilling  </th><th>PaymentMethod            </th><th>MonthlyCharges    </th><th>TotalCharges     </th><th>Churn  </th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "<tr><td>type   </td><td>string      </td><td>enum    </td><td>int               </td><td>enum     </td><td>enum        </td><td>int               </td><td>enum          </td><td>enum            </td><td>enum             </td><td>enum            </td><td>enum          </td><td>enum              </td><td>enum         </td><td>enum         </td><td>enum             </td><td>enum          </td><td>enum              </td><td>enum                     </td><td>real              </td><td>real             </td><td>enum   </td></tr>\n",
              "<tr><td>mins   </td><td>NaN         </td><td>        </td><td>0.0               </td><td>         </td><td>            </td><td>0.0               </td><td>              </td><td>                </td><td>                 </td><td>                </td><td>              </td><td>                  </td><td>             </td><td>             </td><td>                 </td><td>              </td><td>                  </td><td>                         </td><td>18.25             </td><td>18.8             </td><td>       </td></tr>\n",
              "<tr><td>mean   </td><td>NaN         </td><td>        </td><td>0.1621468124378816</td><td>         </td><td>            </td><td>32.37114865824223 </td><td>              </td><td>                </td><td>                 </td><td>                </td><td>              </td><td>                  </td><td>             </td><td>             </td><td>                 </td><td>              </td><td>                  </td><td>                         </td><td>64.76169246059916 </td><td>2283.300440841865</td><td>       </td></tr>\n",
              "<tr><td>maxs   </td><td>NaN         </td><td>        </td><td>1.0               </td><td>         </td><td>            </td><td>72.0              </td><td>              </td><td>                </td><td>                 </td><td>                </td><td>              </td><td>                  </td><td>             </td><td>             </td><td>                 </td><td>              </td><td>                  </td><td>                         </td><td>118.75            </td><td>8684.8           </td><td>       </td></tr>\n",
              "<tr><td>sigma  </td><td>NaN         </td><td>        </td><td>0.3686116056100131</td><td>         </td><td>            </td><td>24.559481023094456</td><td>              </td><td>                </td><td>                 </td><td>                </td><td>              </td><td>                  </td><td>             </td><td>             </td><td>                 </td><td>              </td><td>                  </td><td>                         </td><td>30.090047097678482</td><td>2266.771361883145</td><td>       </td></tr>\n",
              "<tr><td>zeros  </td><td>0           </td><td>        </td><td>5901              </td><td>         </td><td>            </td><td>11                </td><td>              </td><td>                </td><td>                 </td><td>                </td><td>              </td><td>                  </td><td>             </td><td>             </td><td>                 </td><td>              </td><td>                  </td><td>                         </td><td>0                 </td><td>0                </td><td>       </td></tr>\n",
              "<tr><td>missing</td><td>0           </td><td>0       </td><td>0                 </td><td>0        </td><td>0           </td><td>0                 </td><td>0             </td><td>0               </td><td>0                </td><td>0               </td><td>0             </td><td>0                 </td><td>0            </td><td>0            </td><td>0                </td><td>0             </td><td>0                 </td><td>0                        </td><td>0                 </td><td>11               </td><td>0      </td></tr>\n",
              "<tr><td>0      </td><td>7590-VHVEG  </td><td>Female  </td><td>0.0               </td><td>Yes      </td><td>No          </td><td>1.0               </td><td>No            </td><td>No phone service</td><td>DSL              </td><td>No              </td><td>Yes           </td><td>No                </td><td>No           </td><td>No           </td><td>No               </td><td>Month-to-month</td><td>Yes               </td><td>Electronic check         </td><td>29.85             </td><td>29.85            </td><td>No     </td></tr>\n",
              "<tr><td>1      </td><td>5575-GNVDE  </td><td>Male    </td><td>0.0               </td><td>No       </td><td>No          </td><td>34.0              </td><td>Yes           </td><td>No              </td><td>DSL              </td><td>Yes             </td><td>No            </td><td>Yes               </td><td>No           </td><td>No           </td><td>No               </td><td>One year      </td><td>No                </td><td>Mailed check             </td><td>56.95             </td><td>1889.5           </td><td>No     </td></tr>\n",
              "<tr><td>2      </td><td>3668-QPYBK  </td><td>Male    </td><td>0.0               </td><td>No       </td><td>No          </td><td>2.0               </td><td>Yes           </td><td>No              </td><td>DSL              </td><td>Yes             </td><td>Yes           </td><td>No                </td><td>No           </td><td>No           </td><td>No               </td><td>Month-to-month</td><td>Yes               </td><td>Mailed check             </td><td>53.85             </td><td>108.15           </td><td>Yes    </td></tr>\n",
              "<tr><td>3      </td><td>7795-CFOCW  </td><td>Male    </td><td>0.0               </td><td>No       </td><td>No          </td><td>45.0              </td><td>No            </td><td>No phone service</td><td>DSL              </td><td>Yes             </td><td>No            </td><td>Yes               </td><td>Yes          </td><td>No           </td><td>No               </td><td>One year      </td><td>No                </td><td>Bank transfer (automatic)</td><td>42.3              </td><td>1840.75          </td><td>No     </td></tr>\n",
              "<tr><td>4      </td><td>9237-HQITU  </td><td>Female  </td><td>0.0               </td><td>No       </td><td>No          </td><td>2.0               </td><td>Yes           </td><td>No              </td><td>Fiber optic      </td><td>No              </td><td>No            </td><td>No                </td><td>No           </td><td>No           </td><td>No               </td><td>Month-to-month</td><td>Yes               </td><td>Electronic check         </td><td>70.7              </td><td>151.65           </td><td>Yes    </td></tr>\n",
              "<tr><td>5      </td><td>9305-CDSKC  </td><td>Female  </td><td>0.0               </td><td>No       </td><td>No          </td><td>8.0               </td><td>Yes           </td><td>Yes             </td><td>Fiber optic      </td><td>No              </td><td>No            </td><td>Yes               </td><td>No           </td><td>Yes          </td><td>Yes              </td><td>Month-to-month</td><td>Yes               </td><td>Electronic check         </td><td>99.65             </td><td>820.5            </td><td>Yes    </td></tr>\n",
              "<tr><td>6      </td><td>1452-KIOVK  </td><td>Male    </td><td>0.0               </td><td>No       </td><td>Yes         </td><td>22.0              </td><td>Yes           </td><td>Yes             </td><td>Fiber optic      </td><td>No              </td><td>Yes           </td><td>No                </td><td>No           </td><td>Yes          </td><td>No               </td><td>Month-to-month</td><td>Yes               </td><td>Credit card (automatic)  </td><td>89.1              </td><td>1949.4           </td><td>No     </td></tr>\n",
              "<tr><td>7      </td><td>6713-OKOMC  </td><td>Female  </td><td>0.0               </td><td>No       </td><td>No          </td><td>10.0              </td><td>No            </td><td>No phone service</td><td>DSL              </td><td>Yes             </td><td>No            </td><td>No                </td><td>No           </td><td>No           </td><td>No               </td><td>Month-to-month</td><td>No                </td><td>Mailed check             </td><td>29.75             </td><td>301.9            </td><td>No     </td></tr>\n",
              "<tr><td>8      </td><td>7892-POOKP  </td><td>Female  </td><td>0.0               </td><td>Yes      </td><td>No          </td><td>28.0              </td><td>Yes           </td><td>Yes             </td><td>Fiber optic      </td><td>No              </td><td>No            </td><td>Yes               </td><td>Yes          </td><td>Yes          </td><td>Yes              </td><td>Month-to-month</td><td>Yes               </td><td>Electronic check         </td><td>104.8             </td><td>3046.05          </td><td>Yes    </td></tr>\n",
              "<tr><td>9      </td><td>6388-TABGU  </td><td>Male    </td><td>0.0               </td><td>No       </td><td>Yes         </td><td>62.0              </td><td>Yes           </td><td>No              </td><td>DSL              </td><td>Yes             </td><td>Yes           </td><td>No                </td><td>No           </td><td>No           </td><td>No               </td><td>One year      </td><td>No                </td><td>Bank transfer (automatic)</td><td>56.15             </td><td>3487.95          </td><td>No     </td></tr>\n",
              "</tbody>\n",
              "</table>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fzAValVj0Uh0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "churn_train,churn_test,churn_valid = churn_df.split_frame(ratios=[.7, .15])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SHHiJBsU21_U",
        "colab_type": "code",
        "outputId": "6d8be0a5-9222-4f73-fd56-406e19e58614",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 440
        }
      },
      "source": [
        "churn_train"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table>\n",
              "<thead>\n",
              "<tr><th>customerID  </th><th>gender  </th><th style=\"text-align: right;\">  SeniorCitizen</th><th>Partner  </th><th>Dependents  </th><th style=\"text-align: right;\">  tenure</th><th>PhoneService  </th><th>MultipleLines   </th><th>InternetService  </th><th>OnlineSecurity     </th><th>OnlineBackup       </th><th>DeviceProtection   </th><th>TechSupport        </th><th>StreamingTV        </th><th>StreamingMovies    </th><th>Contract      </th><th>PaperlessBilling  </th><th>PaymentMethod            </th><th style=\"text-align: right;\">  MonthlyCharges</th><th style=\"text-align: right;\">  TotalCharges</th><th>Churn  </th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "<tr><td>7590-VHVEG  </td><td>Female  </td><td style=\"text-align: right;\">              0</td><td>Yes      </td><td>No          </td><td style=\"text-align: right;\">       1</td><td>No            </td><td>No phone service</td><td>DSL              </td><td>No                 </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>Month-to-month</td><td>Yes               </td><td>Electronic check         </td><td style=\"text-align: right;\">           29.85</td><td style=\"text-align: right;\">         29.85</td><td>No     </td></tr>\n",
              "<tr><td>5575-GNVDE  </td><td>Male    </td><td style=\"text-align: right;\">              0</td><td>No       </td><td>No          </td><td style=\"text-align: right;\">      34</td><td>Yes           </td><td>No              </td><td>DSL              </td><td>Yes                </td><td>No                 </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>No                 </td><td>One year      </td><td>No                </td><td>Mailed check             </td><td style=\"text-align: right;\">           56.95</td><td style=\"text-align: right;\">       1889.5 </td><td>No     </td></tr>\n",
              "<tr><td>3668-QPYBK  </td><td>Male    </td><td style=\"text-align: right;\">              0</td><td>No       </td><td>No          </td><td style=\"text-align: right;\">       2</td><td>Yes           </td><td>No              </td><td>DSL              </td><td>Yes                </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>Month-to-month</td><td>Yes               </td><td>Mailed check             </td><td style=\"text-align: right;\">           53.85</td><td style=\"text-align: right;\">        108.15</td><td>Yes    </td></tr>\n",
              "<tr><td>7795-CFOCW  </td><td>Male    </td><td style=\"text-align: right;\">              0</td><td>No       </td><td>No          </td><td style=\"text-align: right;\">      45</td><td>No            </td><td>No phone service</td><td>DSL              </td><td>Yes                </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>One year      </td><td>No                </td><td>Bank transfer (automatic)</td><td style=\"text-align: right;\">           42.3 </td><td style=\"text-align: right;\">       1840.75</td><td>No     </td></tr>\n",
              "<tr><td>9237-HQITU  </td><td>Female  </td><td style=\"text-align: right;\">              0</td><td>No       </td><td>No          </td><td style=\"text-align: right;\">       2</td><td>Yes           </td><td>No              </td><td>Fiber optic      </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>Month-to-month</td><td>Yes               </td><td>Electronic check         </td><td style=\"text-align: right;\">           70.7 </td><td style=\"text-align: right;\">        151.65</td><td>Yes    </td></tr>\n",
              "<tr><td>1452-KIOVK  </td><td>Male    </td><td style=\"text-align: right;\">              0</td><td>No       </td><td>Yes         </td><td style=\"text-align: right;\">      22</td><td>Yes           </td><td>Yes             </td><td>Fiber optic      </td><td>No                 </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>No                 </td><td>Month-to-month</td><td>Yes               </td><td>Credit card (automatic)  </td><td style=\"text-align: right;\">           89.1 </td><td style=\"text-align: right;\">       1949.4 </td><td>No     </td></tr>\n",
              "<tr><td>6388-TABGU  </td><td>Male    </td><td style=\"text-align: right;\">              0</td><td>No       </td><td>Yes         </td><td style=\"text-align: right;\">      62</td><td>Yes           </td><td>No              </td><td>DSL              </td><td>Yes                </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>One year      </td><td>No                </td><td>Bank transfer (automatic)</td><td style=\"text-align: right;\">           56.15</td><td style=\"text-align: right;\">       3487.95</td><td>No     </td></tr>\n",
              "<tr><td>9763-GRSKD  </td><td>Male    </td><td style=\"text-align: right;\">              0</td><td>Yes      </td><td>Yes         </td><td style=\"text-align: right;\">      13</td><td>Yes           </td><td>No              </td><td>DSL              </td><td>Yes                </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>No                 </td><td>Month-to-month</td><td>Yes               </td><td>Mailed check             </td><td style=\"text-align: right;\">           49.95</td><td style=\"text-align: right;\">        587.45</td><td>No     </td></tr>\n",
              "<tr><td>7469-LKBCI  </td><td>Male    </td><td style=\"text-align: right;\">              0</td><td>No       </td><td>No          </td><td style=\"text-align: right;\">      16</td><td>Yes           </td><td>No              </td><td>No               </td><td>No internet service</td><td>No internet service</td><td>No internet service</td><td>No internet service</td><td>No internet service</td><td>No internet service</td><td>Two year      </td><td>No                </td><td>Credit card (automatic)  </td><td style=\"text-align: right;\">           18.95</td><td style=\"text-align: right;\">        326.8 </td><td>No     </td></tr>\n",
              "<tr><td>8091-TTVAX  </td><td>Male    </td><td style=\"text-align: right;\">              0</td><td>Yes      </td><td>No          </td><td style=\"text-align: right;\">      58</td><td>Yes           </td><td>Yes             </td><td>Fiber optic      </td><td>No                 </td><td>No                 </td><td>Yes                </td><td>No                 </td><td>Yes                </td><td>Yes                </td><td>One year      </td><td>No                </td><td>Credit card (automatic)  </td><td style=\"text-align: right;\">          100.35</td><td style=\"text-align: right;\">       5681.1 </td><td>No     </td></tr>\n",
              "</tbody>\n",
              "</table>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e6QvFazO5tzP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = \"Churn\"\n",
        "x = churn_df.columns\n",
        "x.remove(y)\n",
        "x.remove(\"customerID\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ESplmnWT570d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "aml = H2OAutoML(max_models = 10, seed = 10, exclude_algos = [\"StackedEnsemble\", \"DeepLearning\"], verbosity=\"info\", nfolds=0)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1uYx3BrXeHtu",
        "colab_type": "code",
        "outputId": "454a0e2e-93b3-4046-e53f-6174d0e209c5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mon Jan  6 01:17:16 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 440.44       Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   33C    P0    26W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fBEls6JqeH0p",
        "colab_type": "code",
        "outputId": "9319154e-e8ef-41d9-dec6-292eca038812",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 989
        }
      },
      "source": [
        "aml.train(x = x, y = y, training_frame = churn_train, validation_frame=churn_valid)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "AutoML progress: |\n",
            "01:17:17.297: Project: AutoML_20200106_11717292\n",
            "01:17:17.299: Cross validation disabled by user: no fold column nor nfolds > 1.\n",
            "01:17:17.299: Setting stopping tolerance adaptively based on the training frame: 0.014236408049319776\n",
            "01:17:17.299: Build control seed: 10\n",
            "01:17:17.299: Since cross-validation is disabled, and no leaderboard frame was provided, automatically split the training data into training and leaderboard frames in the ratio 90/10\n",
            "01:17:17.399: training frame: Frame key: automl_training_py_3_sid_87de    cols: 21    rows: 4454  chunks: 8    size: 294934  checksum: 5310680771849131343\n",
            "01:17:17.399: validation frame: Frame key: py_5_sid_87de    cols: 21    rows: 1069  chunks: 8    size: 139382  checksum: -5299405090363570025\n",
            "01:17:17.406: leaderboard frame: Frame key: automl_leaderboard_py_3_sid_87de    cols: 21    rows: 480  chunks: 8    size: 124576  checksum: 2229072054065022190\n",
            "01:17:17.406: blending frame: NULL\n",
            "01:17:17.406: response column: Churn\n",
            "01:17:17.406: fold column: null\n",
            "01:17:17.406: weights column: null\n",
            "01:17:17.428: Loading execution steps: [{XGBoost : defaults}, {GLM : defaults}, {DRF : [def_1]}, {GBM : defaults}, {DeepLearning : defaults}, {DRF : [XRT]}, {XGBoost : grids}, {GBM : grids}, {DeepLearning : grids}, {StackedEnsemble : defaults}]\n",
            "01:17:17.439: Disabling Algo: DeepLearning as requested by the user.\n",
            "01:17:17.439: Disabling Algo: StackedEnsemble as requested by the user.\n",
            "01:17:17.440: AutoML job created: 2020.01.06 01:17:17.292\n",
            "01:17:17.441: AutoML build started: 2020.01.06 01:17:17.441\n",
            "01:17:17.454: AutoML: starting XGBoost_1_AutoML_20200106_011717 model training\n",
            "\n",
            "█\n",
            "01:17:25.751: New leader: XGBoost_1_AutoML_20200106_011717, auc: 0.8329799686388835\n",
            "01:17:25.752: AutoML: starting XGBoost_2_AutoML_20200106_011717 model training\n",
            "\n",
            "█\n",
            "01:17:30.771: New leader: XGBoost_2_AutoML_20200106_011717, auc: 0.837871861127675\n",
            "01:17:30.772: AutoML: starting XGBoost_3_AutoML_20200106_011717 model training\n",
            "\n",
            "██\n",
            "01:17:33.798: AutoML: starting GLM_1_AutoML_20200106_011717 model training\n",
            "01:17:34.815: AutoML: starting DRF_1_AutoML_20200106_011717 model training\n",
            "\n",
            "███\n",
            "01:17:38.871: AutoML: starting GBM_1_AutoML_20200106_011717 model training\n",
            "\n",
            "██\n",
            "01:17:40.897: AutoML: starting GBM_2_AutoML_20200106_011717 model training\n",
            "\n",
            "█\n",
            "01:17:42.915: AutoML: starting GBM_3_AutoML_20200106_011717 model training\n",
            "\n",
            "██\n",
            "01:17:44.938: AutoML: starting GBM_4_AutoML_20200106_011717 model training\n",
            "\n",
            "██\n",
            "01:17:46.955: AutoML: starting GBM_5_AutoML_20200106_011717 model training\n",
            "\n",
            "██████████████████████████████████████████| 100%\n",
            "\n",
            "01:17:48.975: StackedEnsemble builds skipped due to the exclude_algos option.\n",
            "01:17:48.975: StackedEnsemble builds skipped due to the exclude_algos option.\n",
            "01:17:48.976: Actual modeling steps: [{XGBoost : [def_1 (10), def_2 (10), def_3 (10)]}, {GLM : [def_1 (10)]}, {DRF : [def_1 (10)]}, {GBM : [def_1 (10), def_2 (10), def_3 (10), def_4 (10), def_5 (10)]}]\n",
            "01:17:48.976: AutoML build stopped: 2020.01.06 01:17:48.976\n",
            "01:17:48.976: AutoML build done: built 10 models\n",
            "01:17:48.976: AutoML duration: 31.535 sec\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5fHgZtj_eNq9",
        "colab_type": "code",
        "outputId": "7cbb8cac-29cc-4029-ed96-d11926fea16e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        }
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mon Jan  6 01:17:50 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 440.44       Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   34C    P0    32W / 250W |    331MiB / 16280MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ALlZEBNd68Nm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lb = aml.leaderboard"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x-bfxNwL6-2l",
        "colab_type": "code",
        "outputId": "1d9d485f-f211-4c54-d9a2-27e2fad60b2d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        }
      },
      "source": [
        "lb.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table>\n",
              "<thead>\n",
              "<tr><th>model_id                        </th><th style=\"text-align: right;\">     auc</th><th style=\"text-align: right;\">  logloss</th><th style=\"text-align: right;\">   aucpr</th><th style=\"text-align: right;\">  mean_per_class_error</th><th style=\"text-align: right;\">    rmse</th><th style=\"text-align: right;\">     mse</th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "<tr><td>XGBoost_2_AutoML_20200106_011717</td><td style=\"text-align: right;\">0.837872</td><td style=\"text-align: right;\"> 0.42428 </td><td style=\"text-align: right;\">0.646227</td><td style=\"text-align: right;\">              0.258862</td><td style=\"text-align: right;\">0.371571</td><td style=\"text-align: right;\">0.138065</td></tr>\n",
              "<tr><td>XGBoost_3_AutoML_20200106_011717</td><td style=\"text-align: right;\">0.837309</td><td style=\"text-align: right;\"> 0.425259</td><td style=\"text-align: right;\">0.643129</td><td style=\"text-align: right;\">              0.230902</td><td style=\"text-align: right;\">0.372439</td><td style=\"text-align: right;\">0.138711</td></tr>\n",
              "<tr><td>GBM_5_AutoML_20200106_011717    </td><td style=\"text-align: right;\">0.836359</td><td style=\"text-align: right;\"> 0.427113</td><td style=\"text-align: right;\">0.642654</td><td style=\"text-align: right;\">              0.242762</td><td style=\"text-align: right;\">0.374129</td><td style=\"text-align: right;\">0.139972</td></tr>\n",
              "<tr><td>XGBoost_1_AutoML_20200106_011717</td><td style=\"text-align: right;\">0.83298 </td><td style=\"text-align: right;\"> 0.430737</td><td style=\"text-align: right;\">0.632764</td><td style=\"text-align: right;\">              0.244882</td><td style=\"text-align: right;\">0.375162</td><td style=\"text-align: right;\">0.140747</td></tr>\n",
              "<tr><td>GBM_1_AutoML_20200106_011717    </td><td style=\"text-align: right;\">0.831865</td><td style=\"text-align: right;\"> 0.432989</td><td style=\"text-align: right;\">0.636245</td><td style=\"text-align: right;\">              0.261379</td><td style=\"text-align: right;\">0.375055</td><td style=\"text-align: right;\">0.140666</td></tr>\n",
              "<tr><td>GLM_1_AutoML_20200106_011717    </td><td style=\"text-align: right;\">0.831257</td><td style=\"text-align: right;\"> 0.430245</td><td style=\"text-align: right;\">0.646667</td><td style=\"text-align: right;\">              0.24243 </td><td style=\"text-align: right;\">0.376352</td><td style=\"text-align: right;\">0.141641</td></tr>\n",
              "<tr><td>GBM_4_AutoML_20200106_011717    </td><td style=\"text-align: right;\">0.826354</td><td style=\"text-align: right;\"> 0.449559</td><td style=\"text-align: right;\">0.610091</td><td style=\"text-align: right;\">              0.238554</td><td style=\"text-align: right;\">0.381647</td><td style=\"text-align: right;\">0.145654</td></tr>\n",
              "<tr><td>GBM_3_AutoML_20200106_011717    </td><td style=\"text-align: right;\">0.826167</td><td style=\"text-align: right;\"> 0.440873</td><td style=\"text-align: right;\">0.642437</td><td style=\"text-align: right;\">              0.248758</td><td style=\"text-align: right;\">0.377807</td><td style=\"text-align: right;\">0.142738</td></tr>\n",
              "<tr><td>GBM_2_AutoML_20200106_011717    </td><td style=\"text-align: right;\">0.825725</td><td style=\"text-align: right;\"> 0.442284</td><td style=\"text-align: right;\">0.610157</td><td style=\"text-align: right;\">              0.251143</td><td style=\"text-align: right;\">0.380009</td><td style=\"text-align: right;\">0.144407</td></tr>\n",
              "<tr><td>DRF_1_AutoML_20200106_011717    </td><td style=\"text-align: right;\">0.817631</td><td style=\"text-align: right;\"> 0.515702</td><td style=\"text-align: right;\">0.607515</td><td style=\"text-align: right;\">              0.273239</td><td style=\"text-align: right;\">0.382128</td><td style=\"text-align: right;\">0.146022</td></tr>\n",
              "</tbody>\n",
              "</table>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lob8HRvc4bwq",
        "colab_type": "code",
        "outputId": "77e032ab-8162-4020-e43f-1d3f1da02dbd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "churn_pred=aml.leader.predict(churn_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "xgboost prediction progress: |████████████████████████████████████████████| 100%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jX50NC-L5NJR",
        "colab_type": "code",
        "outputId": "85521a6d-0621-4dcc-c288-750d7d1817a1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        }
      },
      "source": [
        "churn_pred.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table>\n",
              "<thead>\n",
              "<tr><th>predict  </th><th style=\"text-align: right;\">      No</th><th style=\"text-align: right;\">      Yes</th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "<tr><td>Yes      </td><td style=\"text-align: right;\">0.130665</td><td style=\"text-align: right;\">0.869335 </td></tr>\n",
              "<tr><td>Yes      </td><td style=\"text-align: right;\">0.486019</td><td style=\"text-align: right;\">0.513981 </td></tr>\n",
              "<tr><td>No       </td><td style=\"text-align: right;\">0.961202</td><td style=\"text-align: right;\">0.0387979</td></tr>\n",
              "<tr><td>No       </td><td style=\"text-align: right;\">0.826175</td><td style=\"text-align: right;\">0.173825 </td></tr>\n",
              "<tr><td>Yes      </td><td style=\"text-align: right;\">0.480061</td><td style=\"text-align: right;\">0.519939 </td></tr>\n",
              "<tr><td>No       </td><td style=\"text-align: right;\">0.82035 </td><td style=\"text-align: right;\">0.17965  </td></tr>\n",
              "<tr><td>Yes      </td><td style=\"text-align: right;\">0.159316</td><td style=\"text-align: right;\">0.840684 </td></tr>\n",
              "<tr><td>No       </td><td style=\"text-align: right;\">0.9772  </td><td style=\"text-align: right;\">0.0228002</td></tr>\n",
              "<tr><td>No       </td><td style=\"text-align: right;\">0.87115 </td><td style=\"text-align: right;\">0.12885  </td></tr>\n",
              "<tr><td>No       </td><td style=\"text-align: right;\">0.947038</td><td style=\"text-align: right;\">0.0529623</td></tr>\n",
              "</tbody>\n",
              "</table>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GVTyHEfe6whc",
        "colab_type": "code",
        "outputId": "cb279bf2-988d-4a64-c106-ee8e577a46f6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "aml.leader.model_performance(churn_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "ModelMetricsBinomial: xgboost\n",
            "** Reported on test data. **\n",
            "\n",
            "MSE: 0.12989202891170706\n",
            "RMSE: 0.3604053674845965\n",
            "LogLoss: 0.4034449942607744\n",
            "Mean Per-Class Error: 0.21751434948979598\n",
            "AUC: 0.8459995814732143\n",
            "AUCPR: 0.6333245605498026\n",
            "Gini: 0.6919991629464286\n",
            "\n",
            "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.2768990397453308: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>Yes</th>\n",
              "      <th>Error</th>\n",
              "      <th>Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>No</td>\n",
              "      <td>593.0</td>\n",
              "      <td>191.0</td>\n",
              "      <td>0.2436</td>\n",
              "      <td>(191.0/784.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Yes</td>\n",
              "      <td>49.0</td>\n",
              "      <td>207.0</td>\n",
              "      <td>0.1914</td>\n",
              "      <td>(49.0/256.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Total</td>\n",
              "      <td>642.0</td>\n",
              "      <td>398.0</td>\n",
              "      <td>0.2308</td>\n",
              "      <td>(240.0/1040.0)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             No    Yes   Error             Rate\n",
              "0     No  593.0  191.0  0.2436    (191.0/784.0)\n",
              "1    Yes   49.0  207.0  0.1914     (49.0/256.0)\n",
              "2  Total  642.0  398.0  0.2308   (240.0/1040.0)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Maximum Metrics: Maximum metrics at their respective thresholds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>threshold</th>\n",
              "      <th>value</th>\n",
              "      <th>idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>max f1</td>\n",
              "      <td>0.276899</td>\n",
              "      <td>0.633028</td>\n",
              "      <td>222.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>max f2</td>\n",
              "      <td>0.200497</td>\n",
              "      <td>0.742838</td>\n",
              "      <td>261.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>max f0point5</td>\n",
              "      <td>0.543980</td>\n",
              "      <td>0.625000</td>\n",
              "      <td>103.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>max accuracy</td>\n",
              "      <td>0.543980</td>\n",
              "      <td>0.815385</td>\n",
              "      <td>103.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>max precision</td>\n",
              "      <td>0.917811</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>max recall</td>\n",
              "      <td>0.024428</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>389.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>max specificity</td>\n",
              "      <td>0.917811</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>max absolute_mcc</td>\n",
              "      <td>0.276899</td>\n",
              "      <td>0.500720</td>\n",
              "      <td>222.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>max min_per_class_accuracy</td>\n",
              "      <td>0.299446</td>\n",
              "      <td>0.767857</td>\n",
              "      <td>211.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>max mean_per_class_accuracy</td>\n",
              "      <td>0.276899</td>\n",
              "      <td>0.782486</td>\n",
              "      <td>222.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>max tns</td>\n",
              "      <td>0.917811</td>\n",
              "      <td>784.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>max fns</td>\n",
              "      <td>0.917811</td>\n",
              "      <td>255.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>max fps</td>\n",
              "      <td>0.018266</td>\n",
              "      <td>784.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>max tps</td>\n",
              "      <td>0.024428</td>\n",
              "      <td>256.000000</td>\n",
              "      <td>389.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>max tnr</td>\n",
              "      <td>0.917811</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>max fnr</td>\n",
              "      <td>0.917811</td>\n",
              "      <td>0.996094</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>max fpr</td>\n",
              "      <td>0.018266</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>max tpr</td>\n",
              "      <td>0.024428</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>389.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         metric  threshold       value    idx\n",
              "0                        max f1   0.276899    0.633028  222.0\n",
              "1                        max f2   0.200497    0.742838  261.0\n",
              "2                  max f0point5   0.543980    0.625000  103.0\n",
              "3                  max accuracy   0.543980    0.815385  103.0\n",
              "4                 max precision   0.917811    1.000000    0.0\n",
              "5                    max recall   0.024428    1.000000  389.0\n",
              "6               max specificity   0.917811    1.000000    0.0\n",
              "7              max absolute_mcc   0.276899    0.500720  222.0\n",
              "8    max min_per_class_accuracy   0.299446    0.767857  211.0\n",
              "9   max mean_per_class_accuracy   0.276899    0.782486  222.0\n",
              "10                      max tns   0.917811  784.000000    0.0\n",
              "11                      max fns   0.917811  255.000000    0.0\n",
              "12                      max fps   0.018266  784.000000  399.0\n",
              "13                      max tps   0.024428  256.000000  389.0\n",
              "14                      max tnr   0.917811    1.000000    0.0\n",
              "15                      max fnr   0.917811    0.996094    0.0\n",
              "16                      max fpr   0.018266    1.000000  399.0\n",
              "17                      max tpr   0.024428    1.000000  389.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Gains/Lift Table: Avg response rate: 24.62 %, avg score: 26.00 %\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>group</th>\n",
              "      <th>cumulative_data_fraction</th>\n",
              "      <th>lower_threshold</th>\n",
              "      <th>lift</th>\n",
              "      <th>cumulative_lift</th>\n",
              "      <th>response_rate</th>\n",
              "      <th>score</th>\n",
              "      <th>cumulative_response_rate</th>\n",
              "      <th>cumulative_score</th>\n",
              "      <th>capture_rate</th>\n",
              "      <th>cumulative_capture_rate</th>\n",
              "      <th>gain</th>\n",
              "      <th>cumulative_gain</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "      <td>1</td>\n",
              "      <td>0.010577</td>\n",
              "      <td>0.880649</td>\n",
              "      <td>3.323864</td>\n",
              "      <td>3.323864</td>\n",
              "      <td>0.818182</td>\n",
              "      <td>0.903886</td>\n",
              "      <td>0.818182</td>\n",
              "      <td>0.903886</td>\n",
              "      <td>0.035156</td>\n",
              "      <td>0.035156</td>\n",
              "      <td>232.386364</td>\n",
              "      <td>232.386364</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td></td>\n",
              "      <td>2</td>\n",
              "      <td>0.020192</td>\n",
              "      <td>0.855068</td>\n",
              "      <td>3.656250</td>\n",
              "      <td>3.482143</td>\n",
              "      <td>0.900000</td>\n",
              "      <td>0.864420</td>\n",
              "      <td>0.857143</td>\n",
              "      <td>0.885093</td>\n",
              "      <td>0.035156</td>\n",
              "      <td>0.070312</td>\n",
              "      <td>265.625000</td>\n",
              "      <td>248.214286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td></td>\n",
              "      <td>3</td>\n",
              "      <td>0.030769</td>\n",
              "      <td>0.825610</td>\n",
              "      <td>2.954545</td>\n",
              "      <td>3.300781</td>\n",
              "      <td>0.727273</td>\n",
              "      <td>0.841739</td>\n",
              "      <td>0.812500</td>\n",
              "      <td>0.870190</td>\n",
              "      <td>0.031250</td>\n",
              "      <td>0.101562</td>\n",
              "      <td>195.454545</td>\n",
              "      <td>230.078125</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td></td>\n",
              "      <td>4</td>\n",
              "      <td>0.040385</td>\n",
              "      <td>0.809157</td>\n",
              "      <td>3.250000</td>\n",
              "      <td>3.288690</td>\n",
              "      <td>0.800000</td>\n",
              "      <td>0.817316</td>\n",
              "      <td>0.809524</td>\n",
              "      <td>0.857601</td>\n",
              "      <td>0.031250</td>\n",
              "      <td>0.132812</td>\n",
              "      <td>225.000000</td>\n",
              "      <td>228.869048</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td></td>\n",
              "      <td>5</td>\n",
              "      <td>0.050000</td>\n",
              "      <td>0.779834</td>\n",
              "      <td>2.437500</td>\n",
              "      <td>3.125000</td>\n",
              "      <td>0.600000</td>\n",
              "      <td>0.793376</td>\n",
              "      <td>0.769231</td>\n",
              "      <td>0.845250</td>\n",
              "      <td>0.023438</td>\n",
              "      <td>0.156250</td>\n",
              "      <td>143.750000</td>\n",
              "      <td>212.500000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td></td>\n",
              "      <td>6</td>\n",
              "      <td>0.100000</td>\n",
              "      <td>0.658079</td>\n",
              "      <td>2.812500</td>\n",
              "      <td>2.968750</td>\n",
              "      <td>0.692308</td>\n",
              "      <td>0.721111</td>\n",
              "      <td>0.730769</td>\n",
              "      <td>0.783180</td>\n",
              "      <td>0.140625</td>\n",
              "      <td>0.296875</td>\n",
              "      <td>181.250000</td>\n",
              "      <td>196.875000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td></td>\n",
              "      <td>7</td>\n",
              "      <td>0.150000</td>\n",
              "      <td>0.553653</td>\n",
              "      <td>2.578125</td>\n",
              "      <td>2.838542</td>\n",
              "      <td>0.634615</td>\n",
              "      <td>0.603153</td>\n",
              "      <td>0.698718</td>\n",
              "      <td>0.723171</td>\n",
              "      <td>0.128906</td>\n",
              "      <td>0.425781</td>\n",
              "      <td>157.812500</td>\n",
              "      <td>183.854167</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td></td>\n",
              "      <td>8</td>\n",
              "      <td>0.200000</td>\n",
              "      <td>0.489896</td>\n",
              "      <td>1.875000</td>\n",
              "      <td>2.597656</td>\n",
              "      <td>0.461538</td>\n",
              "      <td>0.516815</td>\n",
              "      <td>0.639423</td>\n",
              "      <td>0.671582</td>\n",
              "      <td>0.093750</td>\n",
              "      <td>0.519531</td>\n",
              "      <td>87.500000</td>\n",
              "      <td>159.765625</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td></td>\n",
              "      <td>9</td>\n",
              "      <td>0.300000</td>\n",
              "      <td>0.352833</td>\n",
              "      <td>1.406250</td>\n",
              "      <td>2.200521</td>\n",
              "      <td>0.346154</td>\n",
              "      <td>0.417750</td>\n",
              "      <td>0.541667</td>\n",
              "      <td>0.586971</td>\n",
              "      <td>0.140625</td>\n",
              "      <td>0.660156</td>\n",
              "      <td>40.625000</td>\n",
              "      <td>120.052083</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td></td>\n",
              "      <td>10</td>\n",
              "      <td>0.400000</td>\n",
              "      <td>0.258981</td>\n",
              "      <td>1.640625</td>\n",
              "      <td>2.060547</td>\n",
              "      <td>0.403846</td>\n",
              "      <td>0.308348</td>\n",
              "      <td>0.507212</td>\n",
              "      <td>0.517315</td>\n",
              "      <td>0.164062</td>\n",
              "      <td>0.824219</td>\n",
              "      <td>64.062500</td>\n",
              "      <td>106.054688</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td></td>\n",
              "      <td>11</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.164331</td>\n",
              "      <td>0.625000</td>\n",
              "      <td>1.773438</td>\n",
              "      <td>0.153846</td>\n",
              "      <td>0.209604</td>\n",
              "      <td>0.436538</td>\n",
              "      <td>0.455773</td>\n",
              "      <td>0.062500</td>\n",
              "      <td>0.886719</td>\n",
              "      <td>-37.500000</td>\n",
              "      <td>77.343750</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td></td>\n",
              "      <td>12</td>\n",
              "      <td>0.600000</td>\n",
              "      <td>0.108377</td>\n",
              "      <td>0.468750</td>\n",
              "      <td>1.555990</td>\n",
              "      <td>0.115385</td>\n",
              "      <td>0.135695</td>\n",
              "      <td>0.383013</td>\n",
              "      <td>0.402427</td>\n",
              "      <td>0.046875</td>\n",
              "      <td>0.933594</td>\n",
              "      <td>-53.125000</td>\n",
              "      <td>55.598958</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td></td>\n",
              "      <td>13</td>\n",
              "      <td>0.700000</td>\n",
              "      <td>0.059353</td>\n",
              "      <td>0.351562</td>\n",
              "      <td>1.383929</td>\n",
              "      <td>0.086538</td>\n",
              "      <td>0.082764</td>\n",
              "      <td>0.340659</td>\n",
              "      <td>0.356761</td>\n",
              "      <td>0.035156</td>\n",
              "      <td>0.968750</td>\n",
              "      <td>-64.843750</td>\n",
              "      <td>38.392857</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td></td>\n",
              "      <td>14</td>\n",
              "      <td>0.800000</td>\n",
              "      <td>0.037975</td>\n",
              "      <td>0.156250</td>\n",
              "      <td>1.230469</td>\n",
              "      <td>0.038462</td>\n",
              "      <td>0.048225</td>\n",
              "      <td>0.302885</td>\n",
              "      <td>0.318194</td>\n",
              "      <td>0.015625</td>\n",
              "      <td>0.984375</td>\n",
              "      <td>-84.375000</td>\n",
              "      <td>23.046875</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td></td>\n",
              "      <td>15</td>\n",
              "      <td>0.900000</td>\n",
              "      <td>0.026525</td>\n",
              "      <td>0.078125</td>\n",
              "      <td>1.102431</td>\n",
              "      <td>0.019231</td>\n",
              "      <td>0.031353</td>\n",
              "      <td>0.271368</td>\n",
              "      <td>0.286322</td>\n",
              "      <td>0.007812</td>\n",
              "      <td>0.992188</td>\n",
              "      <td>-92.187500</td>\n",
              "      <td>10.243056</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td></td>\n",
              "      <td>16</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.018266</td>\n",
              "      <td>0.078125</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.019231</td>\n",
              "      <td>0.022614</td>\n",
              "      <td>0.246154</td>\n",
              "      <td>0.259952</td>\n",
              "      <td>0.007812</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-92.187500</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      group  ...        gain  cumulative_gain\n",
              "0         1  ...  232.386364       232.386364\n",
              "1         2  ...  265.625000       248.214286\n",
              "2         3  ...  195.454545       230.078125\n",
              "3         4  ...  225.000000       228.869048\n",
              "4         5  ...  143.750000       212.500000\n",
              "5         6  ...  181.250000       196.875000\n",
              "6         7  ...  157.812500       183.854167\n",
              "7         8  ...   87.500000       159.765625\n",
              "8         9  ...   40.625000       120.052083\n",
              "9        10  ...   64.062500       106.054688\n",
              "10       11  ...  -37.500000        77.343750\n",
              "11       12  ...  -53.125000        55.598958\n",
              "12       13  ...  -64.843750        38.392857\n",
              "13       14  ...  -84.375000        23.046875\n",
              "14       15  ...  -92.187500        10.243056\n",
              "15       16  ...  -92.187500         0.000000\n",
              "\n",
              "[16 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kl3m9BrA8whW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_ids = list(aml.leaderboard['model_id'].as_data_frame().iloc[:,0])\n",
        "#se = h2o.get_model([mid for mid in model_ids if \"StackedEnsemble_AllModels\" in mid][0])\n",
        "#metalearner = h2o.get_model(se.metalearner()['name'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S_7RYNenT527",
        "colab_type": "code",
        "outputId": "6666cd70-fbf6-4439-d85d-be31d782a83b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "model_ids"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['XGBoost_2_AutoML_20200106_011717',\n",
              " 'XGBoost_3_AutoML_20200106_011717',\n",
              " 'GBM_5_AutoML_20200106_011717',\n",
              " 'XGBoost_1_AutoML_20200106_011717',\n",
              " 'GBM_1_AutoML_20200106_011717',\n",
              " 'GLM_1_AutoML_20200106_011717',\n",
              " 'GBM_4_AutoML_20200106_011717',\n",
              " 'GBM_3_AutoML_20200106_011717',\n",
              " 'GBM_2_AutoML_20200106_011717',\n",
              " 'DRF_1_AutoML_20200106_011717']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vOxeivTyUCeq",
        "colab_type": "code",
        "outputId": "c04fb827-fe68-42a7-b646-f4309b32ec76",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "h2o.get_model([mid for mid in model_ids if \"XGBoost\" in mid][0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model Details\n",
            "=============\n",
            "H2OXGBoostEstimator :  XGBoost\n",
            "Model Key:  XGBoost_2_AutoML_20200106_011717\n",
            "\n",
            "\n",
            "Model Summary: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>number_of_trees</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "      <td>85.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     number_of_trees\n",
              "0               85.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "ModelMetricsBinomial: xgboost\n",
            "** Reported on train data. **\n",
            "\n",
            "MSE: 0.11463685881524834\n",
            "RMSE: 0.33858065333868137\n",
            "LogLoss: 0.36311309011473725\n",
            "Mean Per-Class Error: 0.18470830772382718\n",
            "AUC: 0.8936131171891006\n",
            "AUCPR: 0.7615483662521837\n",
            "Gini: 0.7872262343782013\n",
            "\n",
            "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.3606812246143818: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>Yes</th>\n",
              "      <th>Error</th>\n",
              "      <th>Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>No</td>\n",
              "      <td>2752.0</td>\n",
              "      <td>502.0</td>\n",
              "      <td>0.1543</td>\n",
              "      <td>(502.0/3254.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Yes</td>\n",
              "      <td>284.0</td>\n",
              "      <td>916.0</td>\n",
              "      <td>0.2367</td>\n",
              "      <td>(284.0/1200.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Total</td>\n",
              "      <td>3036.0</td>\n",
              "      <td>1418.0</td>\n",
              "      <td>0.1765</td>\n",
              "      <td>(786.0/4454.0)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              No     Yes   Error             Rate\n",
              "0     No  2752.0   502.0  0.1543   (502.0/3254.0)\n",
              "1    Yes   284.0   916.0  0.2367   (284.0/1200.0)\n",
              "2  Total  3036.0  1418.0  0.1765   (786.0/4454.0)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Maximum Metrics: Maximum metrics at their respective thresholds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>threshold</th>\n",
              "      <th>value</th>\n",
              "      <th>idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>max f1</td>\n",
              "      <td>0.360681</td>\n",
              "      <td>0.699771</td>\n",
              "      <td>206.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>max f2</td>\n",
              "      <td>0.235889</td>\n",
              "      <td>0.794122</td>\n",
              "      <td>265.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>max f0point5</td>\n",
              "      <td>0.583763</td>\n",
              "      <td>0.720965</td>\n",
              "      <td>116.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>max accuracy</td>\n",
              "      <td>0.495243</td>\n",
              "      <td>0.838348</td>\n",
              "      <td>151.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>max precision</td>\n",
              "      <td>0.923532</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>max recall</td>\n",
              "      <td>0.022820</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>394.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>max specificity</td>\n",
              "      <td>0.923532</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>max absolute_mcc</td>\n",
              "      <td>0.379353</td>\n",
              "      <td>0.581268</td>\n",
              "      <td>197.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>max min_per_class_accuracy</td>\n",
              "      <td>0.315295</td>\n",
              "      <td>0.807007</td>\n",
              "      <td>226.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>max mean_per_class_accuracy</td>\n",
              "      <td>0.288407</td>\n",
              "      <td>0.815292</td>\n",
              "      <td>239.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>max tns</td>\n",
              "      <td>0.923532</td>\n",
              "      <td>3254.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>max fns</td>\n",
              "      <td>0.923532</td>\n",
              "      <td>1197.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>max fps</td>\n",
              "      <td>0.018585</td>\n",
              "      <td>3254.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>max tps</td>\n",
              "      <td>0.022820</td>\n",
              "      <td>1200.000000</td>\n",
              "      <td>394.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>max tnr</td>\n",
              "      <td>0.923532</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>max fnr</td>\n",
              "      <td>0.923532</td>\n",
              "      <td>0.997500</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>max fpr</td>\n",
              "      <td>0.018585</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>max tpr</td>\n",
              "      <td>0.022820</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>394.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         metric  threshold        value    idx\n",
              "0                        max f1   0.360681     0.699771  206.0\n",
              "1                        max f2   0.235889     0.794122  265.0\n",
              "2                  max f0point5   0.583763     0.720965  116.0\n",
              "3                  max accuracy   0.495243     0.838348  151.0\n",
              "4                 max precision   0.923532     1.000000    0.0\n",
              "5                    max recall   0.022820     1.000000  394.0\n",
              "6               max specificity   0.923532     1.000000    0.0\n",
              "7              max absolute_mcc   0.379353     0.581268  197.0\n",
              "8    max min_per_class_accuracy   0.315295     0.807007  226.0\n",
              "9   max mean_per_class_accuracy   0.288407     0.815292  239.0\n",
              "10                      max tns   0.923532  3254.000000    0.0\n",
              "11                      max fns   0.923532  1197.000000    0.0\n",
              "12                      max fps   0.018585  3254.000000  399.0\n",
              "13                      max tps   0.022820  1200.000000  394.0\n",
              "14                      max tnr   0.923532     1.000000    0.0\n",
              "15                      max fnr   0.923532     0.997500    0.0\n",
              "16                      max fpr   0.018585     1.000000  399.0\n",
              "17                      max tpr   0.022820     1.000000  394.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Gains/Lift Table: Avg response rate: 26.94 %, avg score: 27.35 %\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>group</th>\n",
              "      <th>cumulative_data_fraction</th>\n",
              "      <th>lower_threshold</th>\n",
              "      <th>lift</th>\n",
              "      <th>cumulative_lift</th>\n",
              "      <th>response_rate</th>\n",
              "      <th>score</th>\n",
              "      <th>cumulative_response_rate</th>\n",
              "      <th>cumulative_score</th>\n",
              "      <th>capture_rate</th>\n",
              "      <th>cumulative_capture_rate</th>\n",
              "      <th>gain</th>\n",
              "      <th>cumulative_gain</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "      <td>1</td>\n",
              "      <td>0.010103</td>\n",
              "      <td>0.882697</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>0.901923</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>0.901923</td>\n",
              "      <td>0.035000</td>\n",
              "      <td>0.035000</td>\n",
              "      <td>246.422222</td>\n",
              "      <td>246.422222</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td></td>\n",
              "      <td>2</td>\n",
              "      <td>0.020207</td>\n",
              "      <td>0.855541</td>\n",
              "      <td>3.711667</td>\n",
              "      <td>3.587944</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.867949</td>\n",
              "      <td>0.966667</td>\n",
              "      <td>0.884936</td>\n",
              "      <td>0.037500</td>\n",
              "      <td>0.072500</td>\n",
              "      <td>271.166667</td>\n",
              "      <td>258.794444</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td></td>\n",
              "      <td>3</td>\n",
              "      <td>0.030085</td>\n",
              "      <td>0.829599</td>\n",
              "      <td>3.458598</td>\n",
              "      <td>3.545473</td>\n",
              "      <td>0.931818</td>\n",
              "      <td>0.843338</td>\n",
              "      <td>0.955224</td>\n",
              "      <td>0.871277</td>\n",
              "      <td>0.034167</td>\n",
              "      <td>0.106667</td>\n",
              "      <td>245.859848</td>\n",
              "      <td>254.547264</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td></td>\n",
              "      <td>4</td>\n",
              "      <td>0.040189</td>\n",
              "      <td>0.813237</td>\n",
              "      <td>3.134296</td>\n",
              "      <td>3.442104</td>\n",
              "      <td>0.844444</td>\n",
              "      <td>0.820567</td>\n",
              "      <td>0.927374</td>\n",
              "      <td>0.858529</td>\n",
              "      <td>0.031667</td>\n",
              "      <td>0.138333</td>\n",
              "      <td>213.429630</td>\n",
              "      <td>244.210428</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td></td>\n",
              "      <td>5</td>\n",
              "      <td>0.050067</td>\n",
              "      <td>0.791322</td>\n",
              "      <td>3.458598</td>\n",
              "      <td>3.445359</td>\n",
              "      <td>0.931818</td>\n",
              "      <td>0.801340</td>\n",
              "      <td>0.928251</td>\n",
              "      <td>0.847245</td>\n",
              "      <td>0.034167</td>\n",
              "      <td>0.172500</td>\n",
              "      <td>245.859848</td>\n",
              "      <td>244.535874</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td></td>\n",
              "      <td>6</td>\n",
              "      <td>0.100135</td>\n",
              "      <td>0.686146</td>\n",
              "      <td>2.962676</td>\n",
              "      <td>3.204017</td>\n",
              "      <td>0.798206</td>\n",
              "      <td>0.738535</td>\n",
              "      <td>0.863229</td>\n",
              "      <td>0.792890</td>\n",
              "      <td>0.148333</td>\n",
              "      <td>0.320833</td>\n",
              "      <td>196.267564</td>\n",
              "      <td>220.401719</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td></td>\n",
              "      <td>7</td>\n",
              "      <td>0.149978</td>\n",
              "      <td>0.599372</td>\n",
              "      <td>2.808829</td>\n",
              "      <td>3.072682</td>\n",
              "      <td>0.756757</td>\n",
              "      <td>0.639037</td>\n",
              "      <td>0.827844</td>\n",
              "      <td>0.741759</td>\n",
              "      <td>0.140000</td>\n",
              "      <td>0.460833</td>\n",
              "      <td>180.882883</td>\n",
              "      <td>207.268214</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td></td>\n",
              "      <td>8</td>\n",
              "      <td>0.200045</td>\n",
              "      <td>0.519963</td>\n",
              "      <td>2.097175</td>\n",
              "      <td>2.828532</td>\n",
              "      <td>0.565022</td>\n",
              "      <td>0.559085</td>\n",
              "      <td>0.762065</td>\n",
              "      <td>0.696039</td>\n",
              "      <td>0.105000</td>\n",
              "      <td>0.565833</td>\n",
              "      <td>109.717489</td>\n",
              "      <td>182.853161</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td></td>\n",
              "      <td>9</td>\n",
              "      <td>0.299955</td>\n",
              "      <td>0.383621</td>\n",
              "      <td>1.709869</td>\n",
              "      <td>2.455923</td>\n",
              "      <td>0.460674</td>\n",
              "      <td>0.449520</td>\n",
              "      <td>0.661677</td>\n",
              "      <td>0.613928</td>\n",
              "      <td>0.170833</td>\n",
              "      <td>0.736667</td>\n",
              "      <td>70.986891</td>\n",
              "      <td>145.592315</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td></td>\n",
              "      <td>10</td>\n",
              "      <td>0.400090</td>\n",
              "      <td>0.268587</td>\n",
              "      <td>1.198386</td>\n",
              "      <td>2.141186</td>\n",
              "      <td>0.322870</td>\n",
              "      <td>0.325506</td>\n",
              "      <td>0.576880</td>\n",
              "      <td>0.541741</td>\n",
              "      <td>0.120000</td>\n",
              "      <td>0.856667</td>\n",
              "      <td>19.838565</td>\n",
              "      <td>114.118593</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td></td>\n",
              "      <td>11</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.178341</td>\n",
              "      <td>0.708970</td>\n",
              "      <td>1.855000</td>\n",
              "      <td>0.191011</td>\n",
              "      <td>0.221391</td>\n",
              "      <td>0.499775</td>\n",
              "      <td>0.477729</td>\n",
              "      <td>0.070833</td>\n",
              "      <td>0.927500</td>\n",
              "      <td>-29.102996</td>\n",
              "      <td>85.500000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td></td>\n",
              "      <td>12</td>\n",
              "      <td>0.599910</td>\n",
              "      <td>0.115601</td>\n",
              "      <td>0.333633</td>\n",
              "      <td>1.601629</td>\n",
              "      <td>0.089888</td>\n",
              "      <td>0.144576</td>\n",
              "      <td>0.431512</td>\n",
              "      <td>0.422245</td>\n",
              "      <td>0.033333</td>\n",
              "      <td>0.960833</td>\n",
              "      <td>-66.636704</td>\n",
              "      <td>60.162862</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td></td>\n",
              "      <td>13</td>\n",
              "      <td>0.700045</td>\n",
              "      <td>0.069408</td>\n",
              "      <td>0.233019</td>\n",
              "      <td>1.405862</td>\n",
              "      <td>0.062780</td>\n",
              "      <td>0.092118</td>\n",
              "      <td>0.378768</td>\n",
              "      <td>0.375023</td>\n",
              "      <td>0.023333</td>\n",
              "      <td>0.984167</td>\n",
              "      <td>-76.698057</td>\n",
              "      <td>40.586220</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td></td>\n",
              "      <td>14</td>\n",
              "      <td>0.799955</td>\n",
              "      <td>0.041702</td>\n",
              "      <td>0.091749</td>\n",
              "      <td>1.241736</td>\n",
              "      <td>0.024719</td>\n",
              "      <td>0.054122</td>\n",
              "      <td>0.334550</td>\n",
              "      <td>0.334944</td>\n",
              "      <td>0.009167</td>\n",
              "      <td>0.993333</td>\n",
              "      <td>-90.825094</td>\n",
              "      <td>24.173636</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td></td>\n",
              "      <td>15</td>\n",
              "      <td>0.899865</td>\n",
              "      <td>0.026047</td>\n",
              "      <td>0.050045</td>\n",
              "      <td>1.109425</td>\n",
              "      <td>0.013483</td>\n",
              "      <td>0.032791</td>\n",
              "      <td>0.298902</td>\n",
              "      <td>0.301397</td>\n",
              "      <td>0.005000</td>\n",
              "      <td>0.998333</td>\n",
              "      <td>-94.995506</td>\n",
              "      <td>10.942532</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td></td>\n",
              "      <td>16</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.016862</td>\n",
              "      <td>0.016644</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.004484</td>\n",
              "      <td>0.022601</td>\n",
              "      <td>0.269421</td>\n",
              "      <td>0.273480</td>\n",
              "      <td>0.001667</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-98.335575</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      group  ...        gain  cumulative_gain\n",
              "0         1  ...  246.422222       246.422222\n",
              "1         2  ...  271.166667       258.794444\n",
              "2         3  ...  245.859848       254.547264\n",
              "3         4  ...  213.429630       244.210428\n",
              "4         5  ...  245.859848       244.535874\n",
              "5         6  ...  196.267564       220.401719\n",
              "6         7  ...  180.882883       207.268214\n",
              "7         8  ...  109.717489       182.853161\n",
              "8         9  ...   70.986891       145.592315\n",
              "9        10  ...   19.838565       114.118593\n",
              "10       11  ...  -29.102996        85.500000\n",
              "11       12  ...  -66.636704        60.162862\n",
              "12       13  ...  -76.698057        40.586220\n",
              "13       14  ...  -90.825094        24.173636\n",
              "14       15  ...  -94.995506        10.942532\n",
              "15       16  ...  -98.335575         0.000000\n",
              "\n",
              "[16 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "ModelMetricsBinomial: xgboost\n",
            "** Reported on validation data. **\n",
            "\n",
            "MSE: 0.13200930891094706\n",
            "RMSE: 0.36333085323289993\n",
            "LogLoss: 0.4065191866668133\n",
            "Mean Per-Class Error: 0.21935274064770793\n",
            "AUC: 0.8530052929039204\n",
            "AUCPR: 0.6761107367846269\n",
            "Gini: 0.7060105858078407\n",
            "\n",
            "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.38777639468510944: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>Yes</th>\n",
              "      <th>Error</th>\n",
              "      <th>Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>No</td>\n",
              "      <td>654.0</td>\n",
              "      <td>131.0</td>\n",
              "      <td>0.1669</td>\n",
              "      <td>(131.0/785.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Yes</td>\n",
              "      <td>84.0</td>\n",
              "      <td>200.0</td>\n",
              "      <td>0.2958</td>\n",
              "      <td>(84.0/284.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Total</td>\n",
              "      <td>738.0</td>\n",
              "      <td>331.0</td>\n",
              "      <td>0.2011</td>\n",
              "      <td>(215.0/1069.0)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             No    Yes   Error             Rate\n",
              "0     No  654.0  131.0  0.1669    (131.0/785.0)\n",
              "1    Yes   84.0  200.0  0.2958     (84.0/284.0)\n",
              "2  Total  738.0  331.0  0.2011   (215.0/1069.0)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Maximum Metrics: Maximum metrics at their respective thresholds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>threshold</th>\n",
              "      <th>value</th>\n",
              "      <th>idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>max f1</td>\n",
              "      <td>0.387776</td>\n",
              "      <td>0.650407</td>\n",
              "      <td>181.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>max f2</td>\n",
              "      <td>0.167549</td>\n",
              "      <td>0.754773</td>\n",
              "      <td>281.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>max f0point5</td>\n",
              "      <td>0.471985</td>\n",
              "      <td>0.644410</td>\n",
              "      <td>144.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>max accuracy</td>\n",
              "      <td>0.471985</td>\n",
              "      <td>0.810103</td>\n",
              "      <td>144.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>max precision</td>\n",
              "      <td>0.920657</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>max recall</td>\n",
              "      <td>0.024334</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>391.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>max specificity</td>\n",
              "      <td>0.920657</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>max absolute_mcc</td>\n",
              "      <td>0.389219</td>\n",
              "      <td>0.513706</td>\n",
              "      <td>180.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>max min_per_class_accuracy</td>\n",
              "      <td>0.310758</td>\n",
              "      <td>0.764085</td>\n",
              "      <td>217.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>max mean_per_class_accuracy</td>\n",
              "      <td>0.248484</td>\n",
              "      <td>0.780647</td>\n",
              "      <td>244.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>max tns</td>\n",
              "      <td>0.920657</td>\n",
              "      <td>785.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>max fns</td>\n",
              "      <td>0.920657</td>\n",
              "      <td>283.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>max fps</td>\n",
              "      <td>0.018399</td>\n",
              "      <td>785.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>max tps</td>\n",
              "      <td>0.024334</td>\n",
              "      <td>284.000000</td>\n",
              "      <td>391.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>max tnr</td>\n",
              "      <td>0.920657</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>max fnr</td>\n",
              "      <td>0.920657</td>\n",
              "      <td>0.996479</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>max fpr</td>\n",
              "      <td>0.018399</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>max tpr</td>\n",
              "      <td>0.024334</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>391.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         metric  threshold       value    idx\n",
              "0                        max f1   0.387776    0.650407  181.0\n",
              "1                        max f2   0.167549    0.754773  281.0\n",
              "2                  max f0point5   0.471985    0.644410  144.0\n",
              "3                  max accuracy   0.471985    0.810103  144.0\n",
              "4                 max precision   0.920657    1.000000    0.0\n",
              "5                    max recall   0.024334    1.000000  391.0\n",
              "6               max specificity   0.920657    1.000000    0.0\n",
              "7              max absolute_mcc   0.389219    0.513706  180.0\n",
              "8    max min_per_class_accuracy   0.310758    0.764085  217.0\n",
              "9   max mean_per_class_accuracy   0.248484    0.780647  244.0\n",
              "10                      max tns   0.920657  785.000000    0.0\n",
              "11                      max fns   0.920657  283.000000    0.0\n",
              "12                      max fps   0.018399  785.000000  399.0\n",
              "13                      max tps   0.024334  284.000000  391.0\n",
              "14                      max tnr   0.920657    1.000000    0.0\n",
              "15                      max fnr   0.920657    0.996479    0.0\n",
              "16                      max fpr   0.018399    1.000000  399.0\n",
              "17                      max tpr   0.024334    1.000000  391.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Gains/Lift Table: Avg response rate: 26.57 %, avg score: 27.18 %\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>group</th>\n",
              "      <th>cumulative_data_fraction</th>\n",
              "      <th>lower_threshold</th>\n",
              "      <th>lift</th>\n",
              "      <th>cumulative_lift</th>\n",
              "      <th>response_rate</th>\n",
              "      <th>score</th>\n",
              "      <th>cumulative_response_rate</th>\n",
              "      <th>cumulative_score</th>\n",
              "      <th>capture_rate</th>\n",
              "      <th>cumulative_capture_rate</th>\n",
              "      <th>gain</th>\n",
              "      <th>cumulative_gain</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "      <td>1</td>\n",
              "      <td>0.010290</td>\n",
              "      <td>0.898883</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.910360</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.910360</td>\n",
              "      <td>0.038732</td>\n",
              "      <td>0.038732</td>\n",
              "      <td>276.408451</td>\n",
              "      <td>276.408451</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td></td>\n",
              "      <td>2</td>\n",
              "      <td>0.020580</td>\n",
              "      <td>0.874588</td>\n",
              "      <td>3.079706</td>\n",
              "      <td>3.421895</td>\n",
              "      <td>0.818182</td>\n",
              "      <td>0.887892</td>\n",
              "      <td>0.909091</td>\n",
              "      <td>0.899126</td>\n",
              "      <td>0.031690</td>\n",
              "      <td>0.070423</td>\n",
              "      <td>207.970551</td>\n",
              "      <td>242.189501</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td></td>\n",
              "      <td>3</td>\n",
              "      <td>0.030870</td>\n",
              "      <td>0.844898</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>3.535958</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.858250</td>\n",
              "      <td>0.939394</td>\n",
              "      <td>0.885500</td>\n",
              "      <td>0.038732</td>\n",
              "      <td>0.109155</td>\n",
              "      <td>276.408451</td>\n",
              "      <td>253.595817</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td></td>\n",
              "      <td>4</td>\n",
              "      <td>0.040225</td>\n",
              "      <td>0.821185</td>\n",
              "      <td>3.387676</td>\n",
              "      <td>3.501474</td>\n",
              "      <td>0.900000</td>\n",
              "      <td>0.830053</td>\n",
              "      <td>0.930233</td>\n",
              "      <td>0.872606</td>\n",
              "      <td>0.031690</td>\n",
              "      <td>0.140845</td>\n",
              "      <td>238.767606</td>\n",
              "      <td>250.147396</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td></td>\n",
              "      <td>5</td>\n",
              "      <td>0.050514</td>\n",
              "      <td>0.791687</td>\n",
              "      <td>3.421895</td>\n",
              "      <td>3.485263</td>\n",
              "      <td>0.909091</td>\n",
              "      <td>0.808192</td>\n",
              "      <td>0.925926</td>\n",
              "      <td>0.859484</td>\n",
              "      <td>0.035211</td>\n",
              "      <td>0.176056</td>\n",
              "      <td>242.189501</td>\n",
              "      <td>248.526343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td></td>\n",
              "      <td>6</td>\n",
              "      <td>0.100094</td>\n",
              "      <td>0.672021</td>\n",
              "      <td>2.130614</td>\n",
              "      <td>2.814269</td>\n",
              "      <td>0.566038</td>\n",
              "      <td>0.715380</td>\n",
              "      <td>0.747664</td>\n",
              "      <td>0.788105</td>\n",
              "      <td>0.105634</td>\n",
              "      <td>0.281690</td>\n",
              "      <td>113.061387</td>\n",
              "      <td>181.426879</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td></td>\n",
              "      <td>7</td>\n",
              "      <td>0.150608</td>\n",
              "      <td>0.590287</td>\n",
              "      <td>2.230569</td>\n",
              "      <td>2.618494</td>\n",
              "      <td>0.592593</td>\n",
              "      <td>0.632172</td>\n",
              "      <td>0.695652</td>\n",
              "      <td>0.735805</td>\n",
              "      <td>0.112676</td>\n",
              "      <td>0.394366</td>\n",
              "      <td>123.056860</td>\n",
              "      <td>161.849357</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td></td>\n",
              "      <td>8</td>\n",
              "      <td>0.200187</td>\n",
              "      <td>0.526436</td>\n",
              "      <td>2.343675</td>\n",
              "      <td>2.550431</td>\n",
              "      <td>0.622642</td>\n",
              "      <td>0.558755</td>\n",
              "      <td>0.677570</td>\n",
              "      <td>0.691956</td>\n",
              "      <td>0.116197</td>\n",
              "      <td>0.510563</td>\n",
              "      <td>134.367526</td>\n",
              "      <td>155.043109</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td></td>\n",
              "      <td>9</td>\n",
              "      <td>0.300281</td>\n",
              "      <td>0.393655</td>\n",
              "      <td>1.758918</td>\n",
              "      <td>2.286593</td>\n",
              "      <td>0.467290</td>\n",
              "      <td>0.449514</td>\n",
              "      <td>0.607477</td>\n",
              "      <td>0.611142</td>\n",
              "      <td>0.176056</td>\n",
              "      <td>0.686620</td>\n",
              "      <td>75.891799</td>\n",
              "      <td>128.659339</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td></td>\n",
              "      <td>10</td>\n",
              "      <td>0.400374</td>\n",
              "      <td>0.281592</td>\n",
              "      <td>1.196064</td>\n",
              "      <td>2.013961</td>\n",
              "      <td>0.317757</td>\n",
              "      <td>0.337617</td>\n",
              "      <td>0.535047</td>\n",
              "      <td>0.542761</td>\n",
              "      <td>0.119718</td>\n",
              "      <td>0.806338</td>\n",
              "      <td>19.606424</td>\n",
              "      <td>101.396110</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td></td>\n",
              "      <td>11</td>\n",
              "      <td>0.500468</td>\n",
              "      <td>0.169858</td>\n",
              "      <td>0.809102</td>\n",
              "      <td>1.772989</td>\n",
              "      <td>0.214953</td>\n",
              "      <td>0.222529</td>\n",
              "      <td>0.471028</td>\n",
              "      <td>0.478714</td>\n",
              "      <td>0.080986</td>\n",
              "      <td>0.887324</td>\n",
              "      <td>-19.089772</td>\n",
              "      <td>77.298934</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td></td>\n",
              "      <td>12</td>\n",
              "      <td>0.599626</td>\n",
              "      <td>0.107140</td>\n",
              "      <td>0.532653</td>\n",
              "      <td>1.567879</td>\n",
              "      <td>0.141509</td>\n",
              "      <td>0.136238</td>\n",
              "      <td>0.416537</td>\n",
              "      <td>0.422080</td>\n",
              "      <td>0.052817</td>\n",
              "      <td>0.940141</td>\n",
              "      <td>-46.734653</td>\n",
              "      <td>56.787919</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td></td>\n",
              "      <td>13</td>\n",
              "      <td>0.699719</td>\n",
              "      <td>0.062433</td>\n",
              "      <td>0.351784</td>\n",
              "      <td>1.393919</td>\n",
              "      <td>0.093458</td>\n",
              "      <td>0.083757</td>\n",
              "      <td>0.370321</td>\n",
              "      <td>0.373684</td>\n",
              "      <td>0.035211</td>\n",
              "      <td>0.975352</td>\n",
              "      <td>-64.821640</td>\n",
              "      <td>39.391900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td></td>\n",
              "      <td>14</td>\n",
              "      <td>0.799813</td>\n",
              "      <td>0.037689</td>\n",
              "      <td>0.140713</td>\n",
              "      <td>1.237085</td>\n",
              "      <td>0.037383</td>\n",
              "      <td>0.049346</td>\n",
              "      <td>0.328655</td>\n",
              "      <td>0.333094</td>\n",
              "      <td>0.014085</td>\n",
              "      <td>0.989437</td>\n",
              "      <td>-85.928656</td>\n",
              "      <td>23.708508</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td></td>\n",
              "      <td>15</td>\n",
              "      <td>0.899906</td>\n",
              "      <td>0.025419</td>\n",
              "      <td>0.070357</td>\n",
              "      <td>1.107314</td>\n",
              "      <td>0.018692</td>\n",
              "      <td>0.030728</td>\n",
              "      <td>0.294179</td>\n",
              "      <td>0.299463</td>\n",
              "      <td>0.007042</td>\n",
              "      <td>0.996479</td>\n",
              "      <td>-92.964328</td>\n",
              "      <td>10.731384</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td></td>\n",
              "      <td>16</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.018266</td>\n",
              "      <td>0.035178</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.009346</td>\n",
              "      <td>0.022617</td>\n",
              "      <td>0.265669</td>\n",
              "      <td>0.271752</td>\n",
              "      <td>0.003521</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-96.482164</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      group  ...        gain  cumulative_gain\n",
              "0         1  ...  276.408451       276.408451\n",
              "1         2  ...  207.970551       242.189501\n",
              "2         3  ...  276.408451       253.595817\n",
              "3         4  ...  238.767606       250.147396\n",
              "4         5  ...  242.189501       248.526343\n",
              "5         6  ...  113.061387       181.426879\n",
              "6         7  ...  123.056860       161.849357\n",
              "7         8  ...  134.367526       155.043109\n",
              "8         9  ...   75.891799       128.659339\n",
              "9        10  ...   19.606424       101.396110\n",
              "10       11  ...  -19.089772        77.298934\n",
              "11       12  ...  -46.734653        56.787919\n",
              "12       13  ...  -64.821640        39.391900\n",
              "13       14  ...  -85.928656        23.708508\n",
              "14       15  ...  -92.964328        10.731384\n",
              "15       16  ...  -96.482164         0.000000\n",
              "\n",
              "[16 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Scoring History: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>timestamp</th>\n",
              "      <th>duration</th>\n",
              "      <th>number_of_trees</th>\n",
              "      <th>training_rmse</th>\n",
              "      <th>training_logloss</th>\n",
              "      <th>training_auc</th>\n",
              "      <th>training_pr_auc</th>\n",
              "      <th>training_lift</th>\n",
              "      <th>training_classification_error</th>\n",
              "      <th>validation_rmse</th>\n",
              "      <th>validation_logloss</th>\n",
              "      <th>validation_auc</th>\n",
              "      <th>validation_pr_auc</th>\n",
              "      <th>validation_lift</th>\n",
              "      <th>validation_classification_error</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:25</td>\n",
              "      <td>0.002 sec</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.693147</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.730579</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.693147</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.734331</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:26</td>\n",
              "      <td>0.322 sec</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.451306</td>\n",
              "      <td>0.599465</td>\n",
              "      <td>0.864095</td>\n",
              "      <td>0.650674</td>\n",
              "      <td>3.486717</td>\n",
              "      <td>0.225415</td>\n",
              "      <td>0.451369</td>\n",
              "      <td>0.599555</td>\n",
              "      <td>0.855019</td>\n",
              "      <td>0.612469</td>\n",
              "      <td>3.450411</td>\n",
              "      <td>0.197381</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:26</td>\n",
              "      <td>0.543 sec</td>\n",
              "      <td>10.0</td>\n",
              "      <td>0.417875</td>\n",
              "      <td>0.537171</td>\n",
              "      <td>0.867374</td>\n",
              "      <td>0.700870</td>\n",
              "      <td>3.484422</td>\n",
              "      <td>0.204311</td>\n",
              "      <td>0.418241</td>\n",
              "      <td>0.537659</td>\n",
              "      <td>0.858810</td>\n",
              "      <td>0.677673</td>\n",
              "      <td>3.513146</td>\n",
              "      <td>0.201123</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:26</td>\n",
              "      <td>0.788 sec</td>\n",
              "      <td>15.0</td>\n",
              "      <td>0.395782</td>\n",
              "      <td>0.494959</td>\n",
              "      <td>0.870006</td>\n",
              "      <td>0.707311</td>\n",
              "      <td>3.474752</td>\n",
              "      <td>0.210597</td>\n",
              "      <td>0.397467</td>\n",
              "      <td>0.497480</td>\n",
              "      <td>0.858590</td>\n",
              "      <td>0.688202</td>\n",
              "      <td>3.450411</td>\n",
              "      <td>0.207671</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:26</td>\n",
              "      <td>1.005 sec</td>\n",
              "      <td>20.0</td>\n",
              "      <td>0.381068</td>\n",
              "      <td>0.465116</td>\n",
              "      <td>0.871990</td>\n",
              "      <td>0.713065</td>\n",
              "      <td>3.469601</td>\n",
              "      <td>0.210597</td>\n",
              "      <td>0.383858</td>\n",
              "      <td>0.469220</td>\n",
              "      <td>0.860119</td>\n",
              "      <td>0.682457</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.217961</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:26</td>\n",
              "      <td>1.244 sec</td>\n",
              "      <td>25.0</td>\n",
              "      <td>0.370788</td>\n",
              "      <td>0.442708</td>\n",
              "      <td>0.873594</td>\n",
              "      <td>0.720191</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.209699</td>\n",
              "      <td>0.374983</td>\n",
              "      <td>0.448909</td>\n",
              "      <td>0.860153</td>\n",
              "      <td>0.685266</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.208606</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:27</td>\n",
              "      <td>1.433 sec</td>\n",
              "      <td>30.0</td>\n",
              "      <td>0.363515</td>\n",
              "      <td>0.425682</td>\n",
              "      <td>0.874957</td>\n",
              "      <td>0.720912</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.207678</td>\n",
              "      <td>0.369251</td>\n",
              "      <td>0.434260</td>\n",
              "      <td>0.859561</td>\n",
              "      <td>0.683473</td>\n",
              "      <td>3.474540</td>\n",
              "      <td>0.208606</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:27</td>\n",
              "      <td>1.600 sec</td>\n",
              "      <td>35.0</td>\n",
              "      <td>0.358498</td>\n",
              "      <td>0.413048</td>\n",
              "      <td>0.876798</td>\n",
              "      <td>0.728192</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.198249</td>\n",
              "      <td>0.366187</td>\n",
              "      <td>0.424738</td>\n",
              "      <td>0.859056</td>\n",
              "      <td>0.682699</td>\n",
              "      <td>3.474540</td>\n",
              "      <td>0.199252</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:27</td>\n",
              "      <td>1.855 sec</td>\n",
              "      <td>40.0</td>\n",
              "      <td>0.354522</td>\n",
              "      <td>0.402953</td>\n",
              "      <td>0.879462</td>\n",
              "      <td>0.732853</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.201617</td>\n",
              "      <td>0.364337</td>\n",
              "      <td>0.418383</td>\n",
              "      <td>0.858242</td>\n",
              "      <td>0.684977</td>\n",
              "      <td>3.450411</td>\n",
              "      <td>0.192703</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:27</td>\n",
              "      <td>2.064 sec</td>\n",
              "      <td>45.0</td>\n",
              "      <td>0.351564</td>\n",
              "      <td>0.394884</td>\n",
              "      <td>0.880912</td>\n",
              "      <td>0.735681</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.175573</td>\n",
              "      <td>0.362666</td>\n",
              "      <td>0.412550</td>\n",
              "      <td>0.858552</td>\n",
              "      <td>0.686469</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.202058</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:28</td>\n",
              "      <td>2.382 sec</td>\n",
              "      <td>50.0</td>\n",
              "      <td>0.349213</td>\n",
              "      <td>0.388562</td>\n",
              "      <td>0.882465</td>\n",
              "      <td>0.740370</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.198024</td>\n",
              "      <td>0.362049</td>\n",
              "      <td>0.409335</td>\n",
              "      <td>0.857778</td>\n",
              "      <td>0.683861</td>\n",
              "      <td>3.450411</td>\n",
              "      <td>0.198316</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:28</td>\n",
              "      <td>2.656 sec</td>\n",
              "      <td>55.0</td>\n",
              "      <td>0.347289</td>\n",
              "      <td>0.383445</td>\n",
              "      <td>0.883859</td>\n",
              "      <td>0.744340</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.192187</td>\n",
              "      <td>0.362073</td>\n",
              "      <td>0.407784</td>\n",
              "      <td>0.856865</td>\n",
              "      <td>0.682152</td>\n",
              "      <td>3.474540</td>\n",
              "      <td>0.205800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:28</td>\n",
              "      <td>2.882 sec</td>\n",
              "      <td>60.0</td>\n",
              "      <td>0.345407</td>\n",
              "      <td>0.378674</td>\n",
              "      <td>0.885603</td>\n",
              "      <td>0.747830</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.196004</td>\n",
              "      <td>0.362200</td>\n",
              "      <td>0.406650</td>\n",
              "      <td>0.856094</td>\n",
              "      <td>0.681473</td>\n",
              "      <td>3.474540</td>\n",
              "      <td>0.201123</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:28</td>\n",
              "      <td>3.146 sec</td>\n",
              "      <td>65.0</td>\n",
              "      <td>0.343767</td>\n",
              "      <td>0.374715</td>\n",
              "      <td>0.887414</td>\n",
              "      <td>0.750432</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.186349</td>\n",
              "      <td>0.362427</td>\n",
              "      <td>0.406161</td>\n",
              "      <td>0.855504</td>\n",
              "      <td>0.681001</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.202058</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:29</td>\n",
              "      <td>3.385 sec</td>\n",
              "      <td>70.0</td>\n",
              "      <td>0.342284</td>\n",
              "      <td>0.371238</td>\n",
              "      <td>0.889134</td>\n",
              "      <td>0.754232</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.182084</td>\n",
              "      <td>0.362367</td>\n",
              "      <td>0.405651</td>\n",
              "      <td>0.854918</td>\n",
              "      <td>0.680593</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.199252</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:29</td>\n",
              "      <td>3.592 sec</td>\n",
              "      <td>75.0</td>\n",
              "      <td>0.341016</td>\n",
              "      <td>0.368250</td>\n",
              "      <td>0.890739</td>\n",
              "      <td>0.756042</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.180736</td>\n",
              "      <td>0.362761</td>\n",
              "      <td>0.405939</td>\n",
              "      <td>0.854082</td>\n",
              "      <td>0.679601</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.200187</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:29</td>\n",
              "      <td>3.871 sec</td>\n",
              "      <td>80.0</td>\n",
              "      <td>0.339823</td>\n",
              "      <td>0.365632</td>\n",
              "      <td>0.892047</td>\n",
              "      <td>0.759011</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.182308</td>\n",
              "      <td>0.362969</td>\n",
              "      <td>0.406050</td>\n",
              "      <td>0.853528</td>\n",
              "      <td>0.679006</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.202993</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:29</td>\n",
              "      <td>4.077 sec</td>\n",
              "      <td>85.0</td>\n",
              "      <td>0.338581</td>\n",
              "      <td>0.363113</td>\n",
              "      <td>0.893613</td>\n",
              "      <td>0.761548</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.176471</td>\n",
              "      <td>0.363331</td>\n",
              "      <td>0.406519</td>\n",
              "      <td>0.853005</td>\n",
              "      <td>0.676111</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.201123</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                timestamp  ... validation_lift  validation_classification_error\n",
              "0     2020-01-06 01:17:25  ...        1.000000                         0.734331\n",
              "1     2020-01-06 01:17:26  ...        3.450411                         0.197381\n",
              "2     2020-01-06 01:17:26  ...        3.513146                         0.201123\n",
              "3     2020-01-06 01:17:26  ...        3.450411                         0.207671\n",
              "4     2020-01-06 01:17:26  ...        3.764085                         0.217961\n",
              "5     2020-01-06 01:17:26  ...        3.764085                         0.208606\n",
              "6     2020-01-06 01:17:27  ...        3.474540                         0.208606\n",
              "7     2020-01-06 01:17:27  ...        3.474540                         0.199252\n",
              "8     2020-01-06 01:17:27  ...        3.450411                         0.192703\n",
              "9     2020-01-06 01:17:27  ...        3.764085                         0.202058\n",
              "10    2020-01-06 01:17:28  ...        3.450411                         0.198316\n",
              "11    2020-01-06 01:17:28  ...        3.474540                         0.205800\n",
              "12    2020-01-06 01:17:28  ...        3.474540                         0.201123\n",
              "13    2020-01-06 01:17:28  ...        3.764085                         0.202058\n",
              "14    2020-01-06 01:17:29  ...        3.764085                         0.199252\n",
              "15    2020-01-06 01:17:29  ...        3.764085                         0.200187\n",
              "16    2020-01-06 01:17:29  ...        3.764085                         0.202993\n",
              "17    2020-01-06 01:17:29  ...        3.764085                         0.201123\n",
              "\n",
              "[18 rows x 16 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Variable Importances: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>variable</th>\n",
              "      <th>relative_importance</th>\n",
              "      <th>scaled_importance</th>\n",
              "      <th>percentage</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Contract.Month-to-month</td>\n",
              "      <td>3655.777832</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.256163</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>TotalCharges</td>\n",
              "      <td>2007.098267</td>\n",
              "      <td>0.549021</td>\n",
              "      <td>0.140639</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>tenure</td>\n",
              "      <td>1818.509888</td>\n",
              "      <td>0.497434</td>\n",
              "      <td>0.127424</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>MonthlyCharges</td>\n",
              "      <td>1745.274536</td>\n",
              "      <td>0.477402</td>\n",
              "      <td>0.122292</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>TechSupport.No</td>\n",
              "      <td>732.165771</td>\n",
              "      <td>0.200276</td>\n",
              "      <td>0.051303</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>OnlineSecurity.No</td>\n",
              "      <td>715.071289</td>\n",
              "      <td>0.195600</td>\n",
              "      <td>0.050105</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>InternetService.Fiberoptic</td>\n",
              "      <td>570.084229</td>\n",
              "      <td>0.155941</td>\n",
              "      <td>0.039946</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Contract.Twoyear</td>\n",
              "      <td>481.123566</td>\n",
              "      <td>0.131606</td>\n",
              "      <td>0.033713</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>PaymentMethod.Electroniccheck</td>\n",
              "      <td>284.173859</td>\n",
              "      <td>0.077733</td>\n",
              "      <td>0.019912</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>PaperlessBilling.No</td>\n",
              "      <td>220.357941</td>\n",
              "      <td>0.060277</td>\n",
              "      <td>0.015441</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>gender.Female</td>\n",
              "      <td>209.769211</td>\n",
              "      <td>0.057380</td>\n",
              "      <td>0.014699</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>InternetService.DSL</td>\n",
              "      <td>130.192383</td>\n",
              "      <td>0.035613</td>\n",
              "      <td>0.009123</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Contract.Oneyear</td>\n",
              "      <td>121.914719</td>\n",
              "      <td>0.033349</td>\n",
              "      <td>0.008543</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>SeniorCitizen</td>\n",
              "      <td>119.388542</td>\n",
              "      <td>0.032657</td>\n",
              "      <td>0.008366</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>DeviceProtection.No</td>\n",
              "      <td>110.401329</td>\n",
              "      <td>0.030199</td>\n",
              "      <td>0.007736</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>StreamingMovies.Yes</td>\n",
              "      <td>105.824234</td>\n",
              "      <td>0.028947</td>\n",
              "      <td>0.007415</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>PaperlessBilling.Yes</td>\n",
              "      <td>102.785744</td>\n",
              "      <td>0.028116</td>\n",
              "      <td>0.007202</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>StreamingMovies.No</td>\n",
              "      <td>94.509766</td>\n",
              "      <td>0.025852</td>\n",
              "      <td>0.006622</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>MultipleLines.No</td>\n",
              "      <td>84.426620</td>\n",
              "      <td>0.023094</td>\n",
              "      <td>0.005916</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>gender.Male</td>\n",
              "      <td>82.005692</td>\n",
              "      <td>0.022432</td>\n",
              "      <td>0.005746</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         variable  ...  percentage\n",
              "0         Contract.Month-to-month  ...    0.256163\n",
              "1                    TotalCharges  ...    0.140639\n",
              "2                          tenure  ...    0.127424\n",
              "3                  MonthlyCharges  ...    0.122292\n",
              "4                  TechSupport.No  ...    0.051303\n",
              "5               OnlineSecurity.No  ...    0.050105\n",
              "6      InternetService.Fiberoptic  ...    0.039946\n",
              "7                Contract.Twoyear  ...    0.033713\n",
              "8   PaymentMethod.Electroniccheck  ...    0.019912\n",
              "9             PaperlessBilling.No  ...    0.015441\n",
              "10                  gender.Female  ...    0.014699\n",
              "11            InternetService.DSL  ...    0.009123\n",
              "12               Contract.Oneyear  ...    0.008543\n",
              "13                  SeniorCitizen  ...    0.008366\n",
              "14            DeviceProtection.No  ...    0.007736\n",
              "15            StreamingMovies.Yes  ...    0.007415\n",
              "16           PaperlessBilling.Yes  ...    0.007202\n",
              "17             StreamingMovies.No  ...    0.006622\n",
              "18               MultipleLines.No  ...    0.005916\n",
              "19                    gender.Male  ...    0.005746\n",
              "\n",
              "[20 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "See the whole table with table.as_data_frame()\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "85ppw0hgMxxv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "out = h2o.get_model([mid for mid in model_ids if \"XGBoost\" in mid][0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b4f7YNB6hqbq",
        "colab_type": "code",
        "outputId": "619f2c0c-7c2d-49be-bff6-7a343b13d797",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "out.params"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'backend': {'actual': 'auto', 'default': 'auto'},\n",
              " 'booster': {'actual': 'gbtree', 'default': 'gbtree'},\n",
              " 'calibrate_model': {'actual': False, 'default': False},\n",
              " 'calibration_frame': {'actual': None, 'default': None},\n",
              " 'categorical_encoding': {'actual': 'AUTO', 'default': 'AUTO'},\n",
              " 'checkpoint': {'actual': None, 'default': None},\n",
              " 'col_sample_rate': {'actual': 0.8, 'default': 1.0},\n",
              " 'col_sample_rate_per_tree': {'actual': 0.8, 'default': 1.0},\n",
              " 'colsample_bylevel': {'actual': 1.0, 'default': 1.0},\n",
              " 'colsample_bytree': {'actual': 1.0, 'default': 1.0},\n",
              " 'distribution': {'actual': 'bernoulli', 'default': 'AUTO'},\n",
              " 'dmatrix_type': {'actual': 'auto', 'default': 'auto'},\n",
              " 'eta': {'actual': 0.3, 'default': 0.3},\n",
              " 'export_checkpoints_dir': {'actual': None, 'default': None},\n",
              " 'fold_assignment': {'actual': 'AUTO', 'default': 'AUTO'},\n",
              " 'fold_column': {'actual': None, 'default': None},\n",
              " 'gamma': {'actual': 0.0, 'default': 0.0},\n",
              " 'gpu_id': {'actual': 0, 'default': 0},\n",
              " 'grow_policy': {'actual': 'depthwise', 'default': 'depthwise'},\n",
              " 'ignore_const_cols': {'actual': True, 'default': True},\n",
              " 'ignored_columns': {'actual': ['customerID'], 'default': None},\n",
              " 'keep_cross_validation_fold_assignment': {'actual': False, 'default': False},\n",
              " 'keep_cross_validation_models': {'actual': False, 'default': True},\n",
              " 'keep_cross_validation_predictions': {'actual': True, 'default': False},\n",
              " 'learn_rate': {'actual': 0.05, 'default': 0.3},\n",
              " 'max_abs_leafnode_pred': {'actual': 0.0, 'default': 0.0},\n",
              " 'max_bins': {'actual': 256, 'default': 256},\n",
              " 'max_delta_step': {'actual': 0.0, 'default': 0.0},\n",
              " 'max_depth': {'actual': 20, 'default': 6},\n",
              " 'max_leaves': {'actual': 0, 'default': 0},\n",
              " 'max_runtime_secs': {'actual': 9223372036854776.0, 'default': 0.0},\n",
              " 'min_child_weight': {'actual': 1.0, 'default': 1.0},\n",
              " 'min_data_in_leaf': {'actual': 0.0, 'default': 0.0},\n",
              " 'min_rows': {'actual': 10.0, 'default': 1.0},\n",
              " 'min_split_improvement': {'actual': 0.0, 'default': 0.0},\n",
              " 'min_sum_hessian_in_leaf': {'actual': 100.0, 'default': 100.0},\n",
              " 'model_id': {'actual': {'URL': '/3/Models/XGBoost_2_AutoML_20200106_011717',\n",
              "   '__meta': {'schema_name': 'ModelKeyV3',\n",
              "    'schema_type': 'Key<Model>',\n",
              "    'schema_version': 3},\n",
              "   'name': 'XGBoost_2_AutoML_20200106_011717',\n",
              "   'type': 'Key<Model>'},\n",
              "  'default': None},\n",
              " 'monotone_constraints': {'actual': None, 'default': None},\n",
              " 'nfolds': {'actual': 0, 'default': 0},\n",
              " 'normalize_type': {'actual': 'tree', 'default': 'tree'},\n",
              " 'nthread': {'actual': -1, 'default': -1},\n",
              " 'ntrees': {'actual': 10000, 'default': 50},\n",
              " 'offset_column': {'actual': None, 'default': None},\n",
              " 'one_drop': {'actual': False, 'default': False},\n",
              " 'quiet_mode': {'actual': True, 'default': True},\n",
              " 'rate_drop': {'actual': 0.0, 'default': 0.0},\n",
              " 'reg_alpha': {'actual': 0.0, 'default': 0.0},\n",
              " 'reg_lambda': {'actual': 1.0, 'default': 1.0},\n",
              " 'response_column': {'actual': {'__meta': {'schema_name': 'ColSpecifierV3',\n",
              "    'schema_type': 'VecSpecifier',\n",
              "    'schema_version': 3},\n",
              "   'column_name': 'Churn',\n",
              "   'is_member_of_frames': None},\n",
              "  'default': None},\n",
              " 'sample_rate': {'actual': 0.6, 'default': 1.0},\n",
              " 'sample_type': {'actual': 'uniform', 'default': 'uniform'},\n",
              " 'save_matrix_directory': {'actual': None, 'default': None},\n",
              " 'score_each_iteration': {'actual': False, 'default': False},\n",
              " 'score_tree_interval': {'actual': 5, 'default': 0},\n",
              " 'seed': {'actual': 11, 'default': -1},\n",
              " 'skip_drop': {'actual': 0.0, 'default': 0.0},\n",
              " 'stopping_metric': {'actual': 'logloss', 'default': 'AUTO'},\n",
              " 'stopping_rounds': {'actual': 5, 'default': 0},\n",
              " 'stopping_tolerance': {'actual': 0.014236408049319776, 'default': 0.001},\n",
              " 'subsample': {'actual': 1.0, 'default': 1.0},\n",
              " 'training_frame': {'actual': {'URL': '/3/Frames/automl_training_py_3_sid_87de',\n",
              "   '__meta': {'schema_name': 'FrameKeyV3',\n",
              "    'schema_type': 'Key<Frame>',\n",
              "    'schema_version': 3},\n",
              "   'name': 'automl_training_py_3_sid_87de',\n",
              "   'type': 'Key<Frame>'},\n",
              "  'default': None},\n",
              " 'tree_method': {'actual': 'auto', 'default': 'auto'},\n",
              " 'tweedie_power': {'actual': 1.5, 'default': 1.5},\n",
              " 'validation_frame': {'actual': {'URL': '/3/Frames/py_5_sid_87de',\n",
              "   '__meta': {'schema_name': 'FrameKeyV3',\n",
              "    'schema_type': 'Key<Frame>',\n",
              "    'schema_version': 3},\n",
              "   'name': 'py_5_sid_87de',\n",
              "   'type': 'Key<Frame>'},\n",
              "  'default': None},\n",
              " 'weights_column': {'actual': None, 'default': None}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mxY-GjDVRxVl",
        "colab_type": "code",
        "outputId": "5f97c99c-d060-4614-d469-70189f33e71f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 340
        }
      },
      "source": [
        "out.convert_H2OXGBoostParams_2_XGBoostParams()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'alpha': 0.0,\n",
              "  'booster': 'gbtree',\n",
              "  'colsample_bylevel': 0.8,\n",
              "  'colsample_bytree': 0.8,\n",
              "  'eta': 0.05,\n",
              "  'gamma': 0.0,\n",
              "  'grow_policy': 'depthwise',\n",
              "  'lambda': 1.0,\n",
              "  'max_delta_step': 0.0,\n",
              "  'max_depth': 20,\n",
              "  'min_child_weight': 10.0,\n",
              "  'nround': 10000,\n",
              "  'nthread': 2,\n",
              "  'objective': 'binary:logistic',\n",
              "  'seed': 11,\n",
              "  'silent': True,\n",
              "  'subsample': 0.6,\n",
              "  'tree_method': 'auto'},\n",
              " 10000)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wl4mI1K9lteZ",
        "colab_type": "code",
        "outputId": "1e179cb2-d45b-4f62-f2dc-e19414053685",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "out"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model Details\n",
            "=============\n",
            "H2OXGBoostEstimator :  XGBoost\n",
            "Model Key:  XGBoost_2_AutoML_20200106_011717\n",
            "\n",
            "\n",
            "Model Summary: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>number_of_trees</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "      <td>85.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     number_of_trees\n",
              "0               85.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "ModelMetricsBinomial: xgboost\n",
            "** Reported on train data. **\n",
            "\n",
            "MSE: 0.11463685881524834\n",
            "RMSE: 0.33858065333868137\n",
            "LogLoss: 0.36311309011473725\n",
            "Mean Per-Class Error: 0.18470830772382718\n",
            "AUC: 0.8936131171891006\n",
            "AUCPR: 0.7615483662521837\n",
            "Gini: 0.7872262343782013\n",
            "\n",
            "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.3606812246143818: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>Yes</th>\n",
              "      <th>Error</th>\n",
              "      <th>Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>No</td>\n",
              "      <td>2752.0</td>\n",
              "      <td>502.0</td>\n",
              "      <td>0.1543</td>\n",
              "      <td>(502.0/3254.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Yes</td>\n",
              "      <td>284.0</td>\n",
              "      <td>916.0</td>\n",
              "      <td>0.2367</td>\n",
              "      <td>(284.0/1200.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Total</td>\n",
              "      <td>3036.0</td>\n",
              "      <td>1418.0</td>\n",
              "      <td>0.1765</td>\n",
              "      <td>(786.0/4454.0)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              No     Yes   Error             Rate\n",
              "0     No  2752.0   502.0  0.1543   (502.0/3254.0)\n",
              "1    Yes   284.0   916.0  0.2367   (284.0/1200.0)\n",
              "2  Total  3036.0  1418.0  0.1765   (786.0/4454.0)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Maximum Metrics: Maximum metrics at their respective thresholds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>threshold</th>\n",
              "      <th>value</th>\n",
              "      <th>idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>max f1</td>\n",
              "      <td>0.360681</td>\n",
              "      <td>0.699771</td>\n",
              "      <td>206.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>max f2</td>\n",
              "      <td>0.235889</td>\n",
              "      <td>0.794122</td>\n",
              "      <td>265.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>max f0point5</td>\n",
              "      <td>0.583763</td>\n",
              "      <td>0.720965</td>\n",
              "      <td>116.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>max accuracy</td>\n",
              "      <td>0.495243</td>\n",
              "      <td>0.838348</td>\n",
              "      <td>151.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>max precision</td>\n",
              "      <td>0.923532</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>max recall</td>\n",
              "      <td>0.022820</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>394.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>max specificity</td>\n",
              "      <td>0.923532</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>max absolute_mcc</td>\n",
              "      <td>0.379353</td>\n",
              "      <td>0.581268</td>\n",
              "      <td>197.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>max min_per_class_accuracy</td>\n",
              "      <td>0.315295</td>\n",
              "      <td>0.807007</td>\n",
              "      <td>226.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>max mean_per_class_accuracy</td>\n",
              "      <td>0.288407</td>\n",
              "      <td>0.815292</td>\n",
              "      <td>239.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>max tns</td>\n",
              "      <td>0.923532</td>\n",
              "      <td>3254.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>max fns</td>\n",
              "      <td>0.923532</td>\n",
              "      <td>1197.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>max fps</td>\n",
              "      <td>0.018585</td>\n",
              "      <td>3254.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>max tps</td>\n",
              "      <td>0.022820</td>\n",
              "      <td>1200.000000</td>\n",
              "      <td>394.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>max tnr</td>\n",
              "      <td>0.923532</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>max fnr</td>\n",
              "      <td>0.923532</td>\n",
              "      <td>0.997500</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>max fpr</td>\n",
              "      <td>0.018585</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>max tpr</td>\n",
              "      <td>0.022820</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>394.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         metric  threshold        value    idx\n",
              "0                        max f1   0.360681     0.699771  206.0\n",
              "1                        max f2   0.235889     0.794122  265.0\n",
              "2                  max f0point5   0.583763     0.720965  116.0\n",
              "3                  max accuracy   0.495243     0.838348  151.0\n",
              "4                 max precision   0.923532     1.000000    0.0\n",
              "5                    max recall   0.022820     1.000000  394.0\n",
              "6               max specificity   0.923532     1.000000    0.0\n",
              "7              max absolute_mcc   0.379353     0.581268  197.0\n",
              "8    max min_per_class_accuracy   0.315295     0.807007  226.0\n",
              "9   max mean_per_class_accuracy   0.288407     0.815292  239.0\n",
              "10                      max tns   0.923532  3254.000000    0.0\n",
              "11                      max fns   0.923532  1197.000000    0.0\n",
              "12                      max fps   0.018585  3254.000000  399.0\n",
              "13                      max tps   0.022820  1200.000000  394.0\n",
              "14                      max tnr   0.923532     1.000000    0.0\n",
              "15                      max fnr   0.923532     0.997500    0.0\n",
              "16                      max fpr   0.018585     1.000000  399.0\n",
              "17                      max tpr   0.022820     1.000000  394.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Gains/Lift Table: Avg response rate: 26.94 %, avg score: 27.35 %\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>group</th>\n",
              "      <th>cumulative_data_fraction</th>\n",
              "      <th>lower_threshold</th>\n",
              "      <th>lift</th>\n",
              "      <th>cumulative_lift</th>\n",
              "      <th>response_rate</th>\n",
              "      <th>score</th>\n",
              "      <th>cumulative_response_rate</th>\n",
              "      <th>cumulative_score</th>\n",
              "      <th>capture_rate</th>\n",
              "      <th>cumulative_capture_rate</th>\n",
              "      <th>gain</th>\n",
              "      <th>cumulative_gain</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "      <td>1</td>\n",
              "      <td>0.010103</td>\n",
              "      <td>0.882697</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>0.901923</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>0.901923</td>\n",
              "      <td>0.035000</td>\n",
              "      <td>0.035000</td>\n",
              "      <td>246.422222</td>\n",
              "      <td>246.422222</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td></td>\n",
              "      <td>2</td>\n",
              "      <td>0.020207</td>\n",
              "      <td>0.855541</td>\n",
              "      <td>3.711667</td>\n",
              "      <td>3.587944</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.867949</td>\n",
              "      <td>0.966667</td>\n",
              "      <td>0.884936</td>\n",
              "      <td>0.037500</td>\n",
              "      <td>0.072500</td>\n",
              "      <td>271.166667</td>\n",
              "      <td>258.794444</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td></td>\n",
              "      <td>3</td>\n",
              "      <td>0.030085</td>\n",
              "      <td>0.829599</td>\n",
              "      <td>3.458598</td>\n",
              "      <td>3.545473</td>\n",
              "      <td>0.931818</td>\n",
              "      <td>0.843338</td>\n",
              "      <td>0.955224</td>\n",
              "      <td>0.871277</td>\n",
              "      <td>0.034167</td>\n",
              "      <td>0.106667</td>\n",
              "      <td>245.859848</td>\n",
              "      <td>254.547264</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td></td>\n",
              "      <td>4</td>\n",
              "      <td>0.040189</td>\n",
              "      <td>0.813237</td>\n",
              "      <td>3.134296</td>\n",
              "      <td>3.442104</td>\n",
              "      <td>0.844444</td>\n",
              "      <td>0.820567</td>\n",
              "      <td>0.927374</td>\n",
              "      <td>0.858529</td>\n",
              "      <td>0.031667</td>\n",
              "      <td>0.138333</td>\n",
              "      <td>213.429630</td>\n",
              "      <td>244.210428</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td></td>\n",
              "      <td>5</td>\n",
              "      <td>0.050067</td>\n",
              "      <td>0.791322</td>\n",
              "      <td>3.458598</td>\n",
              "      <td>3.445359</td>\n",
              "      <td>0.931818</td>\n",
              "      <td>0.801340</td>\n",
              "      <td>0.928251</td>\n",
              "      <td>0.847245</td>\n",
              "      <td>0.034167</td>\n",
              "      <td>0.172500</td>\n",
              "      <td>245.859848</td>\n",
              "      <td>244.535874</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td></td>\n",
              "      <td>6</td>\n",
              "      <td>0.100135</td>\n",
              "      <td>0.686146</td>\n",
              "      <td>2.962676</td>\n",
              "      <td>3.204017</td>\n",
              "      <td>0.798206</td>\n",
              "      <td>0.738535</td>\n",
              "      <td>0.863229</td>\n",
              "      <td>0.792890</td>\n",
              "      <td>0.148333</td>\n",
              "      <td>0.320833</td>\n",
              "      <td>196.267564</td>\n",
              "      <td>220.401719</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td></td>\n",
              "      <td>7</td>\n",
              "      <td>0.149978</td>\n",
              "      <td>0.599372</td>\n",
              "      <td>2.808829</td>\n",
              "      <td>3.072682</td>\n",
              "      <td>0.756757</td>\n",
              "      <td>0.639037</td>\n",
              "      <td>0.827844</td>\n",
              "      <td>0.741759</td>\n",
              "      <td>0.140000</td>\n",
              "      <td>0.460833</td>\n",
              "      <td>180.882883</td>\n",
              "      <td>207.268214</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td></td>\n",
              "      <td>8</td>\n",
              "      <td>0.200045</td>\n",
              "      <td>0.519963</td>\n",
              "      <td>2.097175</td>\n",
              "      <td>2.828532</td>\n",
              "      <td>0.565022</td>\n",
              "      <td>0.559085</td>\n",
              "      <td>0.762065</td>\n",
              "      <td>0.696039</td>\n",
              "      <td>0.105000</td>\n",
              "      <td>0.565833</td>\n",
              "      <td>109.717489</td>\n",
              "      <td>182.853161</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td></td>\n",
              "      <td>9</td>\n",
              "      <td>0.299955</td>\n",
              "      <td>0.383621</td>\n",
              "      <td>1.709869</td>\n",
              "      <td>2.455923</td>\n",
              "      <td>0.460674</td>\n",
              "      <td>0.449520</td>\n",
              "      <td>0.661677</td>\n",
              "      <td>0.613928</td>\n",
              "      <td>0.170833</td>\n",
              "      <td>0.736667</td>\n",
              "      <td>70.986891</td>\n",
              "      <td>145.592315</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td></td>\n",
              "      <td>10</td>\n",
              "      <td>0.400090</td>\n",
              "      <td>0.268587</td>\n",
              "      <td>1.198386</td>\n",
              "      <td>2.141186</td>\n",
              "      <td>0.322870</td>\n",
              "      <td>0.325506</td>\n",
              "      <td>0.576880</td>\n",
              "      <td>0.541741</td>\n",
              "      <td>0.120000</td>\n",
              "      <td>0.856667</td>\n",
              "      <td>19.838565</td>\n",
              "      <td>114.118593</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td></td>\n",
              "      <td>11</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.178341</td>\n",
              "      <td>0.708970</td>\n",
              "      <td>1.855000</td>\n",
              "      <td>0.191011</td>\n",
              "      <td>0.221391</td>\n",
              "      <td>0.499775</td>\n",
              "      <td>0.477729</td>\n",
              "      <td>0.070833</td>\n",
              "      <td>0.927500</td>\n",
              "      <td>-29.102996</td>\n",
              "      <td>85.500000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td></td>\n",
              "      <td>12</td>\n",
              "      <td>0.599910</td>\n",
              "      <td>0.115601</td>\n",
              "      <td>0.333633</td>\n",
              "      <td>1.601629</td>\n",
              "      <td>0.089888</td>\n",
              "      <td>0.144576</td>\n",
              "      <td>0.431512</td>\n",
              "      <td>0.422245</td>\n",
              "      <td>0.033333</td>\n",
              "      <td>0.960833</td>\n",
              "      <td>-66.636704</td>\n",
              "      <td>60.162862</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td></td>\n",
              "      <td>13</td>\n",
              "      <td>0.700045</td>\n",
              "      <td>0.069408</td>\n",
              "      <td>0.233019</td>\n",
              "      <td>1.405862</td>\n",
              "      <td>0.062780</td>\n",
              "      <td>0.092118</td>\n",
              "      <td>0.378768</td>\n",
              "      <td>0.375023</td>\n",
              "      <td>0.023333</td>\n",
              "      <td>0.984167</td>\n",
              "      <td>-76.698057</td>\n",
              "      <td>40.586220</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td></td>\n",
              "      <td>14</td>\n",
              "      <td>0.799955</td>\n",
              "      <td>0.041702</td>\n",
              "      <td>0.091749</td>\n",
              "      <td>1.241736</td>\n",
              "      <td>0.024719</td>\n",
              "      <td>0.054122</td>\n",
              "      <td>0.334550</td>\n",
              "      <td>0.334944</td>\n",
              "      <td>0.009167</td>\n",
              "      <td>0.993333</td>\n",
              "      <td>-90.825094</td>\n",
              "      <td>24.173636</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td></td>\n",
              "      <td>15</td>\n",
              "      <td>0.899865</td>\n",
              "      <td>0.026047</td>\n",
              "      <td>0.050045</td>\n",
              "      <td>1.109425</td>\n",
              "      <td>0.013483</td>\n",
              "      <td>0.032791</td>\n",
              "      <td>0.298902</td>\n",
              "      <td>0.301397</td>\n",
              "      <td>0.005000</td>\n",
              "      <td>0.998333</td>\n",
              "      <td>-94.995506</td>\n",
              "      <td>10.942532</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td></td>\n",
              "      <td>16</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.016862</td>\n",
              "      <td>0.016644</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.004484</td>\n",
              "      <td>0.022601</td>\n",
              "      <td>0.269421</td>\n",
              "      <td>0.273480</td>\n",
              "      <td>0.001667</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-98.335575</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      group  ...        gain  cumulative_gain\n",
              "0         1  ...  246.422222       246.422222\n",
              "1         2  ...  271.166667       258.794444\n",
              "2         3  ...  245.859848       254.547264\n",
              "3         4  ...  213.429630       244.210428\n",
              "4         5  ...  245.859848       244.535874\n",
              "5         6  ...  196.267564       220.401719\n",
              "6         7  ...  180.882883       207.268214\n",
              "7         8  ...  109.717489       182.853161\n",
              "8         9  ...   70.986891       145.592315\n",
              "9        10  ...   19.838565       114.118593\n",
              "10       11  ...  -29.102996        85.500000\n",
              "11       12  ...  -66.636704        60.162862\n",
              "12       13  ...  -76.698057        40.586220\n",
              "13       14  ...  -90.825094        24.173636\n",
              "14       15  ...  -94.995506        10.942532\n",
              "15       16  ...  -98.335575         0.000000\n",
              "\n",
              "[16 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "ModelMetricsBinomial: xgboost\n",
            "** Reported on validation data. **\n",
            "\n",
            "MSE: 0.13200930891094706\n",
            "RMSE: 0.36333085323289993\n",
            "LogLoss: 0.4065191866668133\n",
            "Mean Per-Class Error: 0.21935274064770793\n",
            "AUC: 0.8530052929039204\n",
            "AUCPR: 0.6761107367846269\n",
            "Gini: 0.7060105858078407\n",
            "\n",
            "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.38777639468510944: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>Yes</th>\n",
              "      <th>Error</th>\n",
              "      <th>Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>No</td>\n",
              "      <td>654.0</td>\n",
              "      <td>131.0</td>\n",
              "      <td>0.1669</td>\n",
              "      <td>(131.0/785.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Yes</td>\n",
              "      <td>84.0</td>\n",
              "      <td>200.0</td>\n",
              "      <td>0.2958</td>\n",
              "      <td>(84.0/284.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Total</td>\n",
              "      <td>738.0</td>\n",
              "      <td>331.0</td>\n",
              "      <td>0.2011</td>\n",
              "      <td>(215.0/1069.0)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             No    Yes   Error             Rate\n",
              "0     No  654.0  131.0  0.1669    (131.0/785.0)\n",
              "1    Yes   84.0  200.0  0.2958     (84.0/284.0)\n",
              "2  Total  738.0  331.0  0.2011   (215.0/1069.0)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Maximum Metrics: Maximum metrics at their respective thresholds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>metric</th>\n",
              "      <th>threshold</th>\n",
              "      <th>value</th>\n",
              "      <th>idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>max f1</td>\n",
              "      <td>0.387776</td>\n",
              "      <td>0.650407</td>\n",
              "      <td>181.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>max f2</td>\n",
              "      <td>0.167549</td>\n",
              "      <td>0.754773</td>\n",
              "      <td>281.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>max f0point5</td>\n",
              "      <td>0.471985</td>\n",
              "      <td>0.644410</td>\n",
              "      <td>144.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>max accuracy</td>\n",
              "      <td>0.471985</td>\n",
              "      <td>0.810103</td>\n",
              "      <td>144.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>max precision</td>\n",
              "      <td>0.920657</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>max recall</td>\n",
              "      <td>0.024334</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>391.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>max specificity</td>\n",
              "      <td>0.920657</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>max absolute_mcc</td>\n",
              "      <td>0.389219</td>\n",
              "      <td>0.513706</td>\n",
              "      <td>180.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>max min_per_class_accuracy</td>\n",
              "      <td>0.310758</td>\n",
              "      <td>0.764085</td>\n",
              "      <td>217.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>max mean_per_class_accuracy</td>\n",
              "      <td>0.248484</td>\n",
              "      <td>0.780647</td>\n",
              "      <td>244.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>max tns</td>\n",
              "      <td>0.920657</td>\n",
              "      <td>785.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>max fns</td>\n",
              "      <td>0.920657</td>\n",
              "      <td>283.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>max fps</td>\n",
              "      <td>0.018399</td>\n",
              "      <td>785.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>max tps</td>\n",
              "      <td>0.024334</td>\n",
              "      <td>284.000000</td>\n",
              "      <td>391.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>max tnr</td>\n",
              "      <td>0.920657</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>max fnr</td>\n",
              "      <td>0.920657</td>\n",
              "      <td>0.996479</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>max fpr</td>\n",
              "      <td>0.018399</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>399.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>max tpr</td>\n",
              "      <td>0.024334</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>391.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         metric  threshold       value    idx\n",
              "0                        max f1   0.387776    0.650407  181.0\n",
              "1                        max f2   0.167549    0.754773  281.0\n",
              "2                  max f0point5   0.471985    0.644410  144.0\n",
              "3                  max accuracy   0.471985    0.810103  144.0\n",
              "4                 max precision   0.920657    1.000000    0.0\n",
              "5                    max recall   0.024334    1.000000  391.0\n",
              "6               max specificity   0.920657    1.000000    0.0\n",
              "7              max absolute_mcc   0.389219    0.513706  180.0\n",
              "8    max min_per_class_accuracy   0.310758    0.764085  217.0\n",
              "9   max mean_per_class_accuracy   0.248484    0.780647  244.0\n",
              "10                      max tns   0.920657  785.000000    0.0\n",
              "11                      max fns   0.920657  283.000000    0.0\n",
              "12                      max fps   0.018399  785.000000  399.0\n",
              "13                      max tps   0.024334  284.000000  391.0\n",
              "14                      max tnr   0.920657    1.000000    0.0\n",
              "15                      max fnr   0.920657    0.996479    0.0\n",
              "16                      max fpr   0.018399    1.000000  399.0\n",
              "17                      max tpr   0.024334    1.000000  391.0"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Gains/Lift Table: Avg response rate: 26.57 %, avg score: 27.18 %\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>group</th>\n",
              "      <th>cumulative_data_fraction</th>\n",
              "      <th>lower_threshold</th>\n",
              "      <th>lift</th>\n",
              "      <th>cumulative_lift</th>\n",
              "      <th>response_rate</th>\n",
              "      <th>score</th>\n",
              "      <th>cumulative_response_rate</th>\n",
              "      <th>cumulative_score</th>\n",
              "      <th>capture_rate</th>\n",
              "      <th>cumulative_capture_rate</th>\n",
              "      <th>gain</th>\n",
              "      <th>cumulative_gain</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "      <td>1</td>\n",
              "      <td>0.010290</td>\n",
              "      <td>0.898883</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.910360</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.910360</td>\n",
              "      <td>0.038732</td>\n",
              "      <td>0.038732</td>\n",
              "      <td>276.408451</td>\n",
              "      <td>276.408451</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td></td>\n",
              "      <td>2</td>\n",
              "      <td>0.020580</td>\n",
              "      <td>0.874588</td>\n",
              "      <td>3.079706</td>\n",
              "      <td>3.421895</td>\n",
              "      <td>0.818182</td>\n",
              "      <td>0.887892</td>\n",
              "      <td>0.909091</td>\n",
              "      <td>0.899126</td>\n",
              "      <td>0.031690</td>\n",
              "      <td>0.070423</td>\n",
              "      <td>207.970551</td>\n",
              "      <td>242.189501</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td></td>\n",
              "      <td>3</td>\n",
              "      <td>0.030870</td>\n",
              "      <td>0.844898</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>3.535958</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.858250</td>\n",
              "      <td>0.939394</td>\n",
              "      <td>0.885500</td>\n",
              "      <td>0.038732</td>\n",
              "      <td>0.109155</td>\n",
              "      <td>276.408451</td>\n",
              "      <td>253.595817</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td></td>\n",
              "      <td>4</td>\n",
              "      <td>0.040225</td>\n",
              "      <td>0.821185</td>\n",
              "      <td>3.387676</td>\n",
              "      <td>3.501474</td>\n",
              "      <td>0.900000</td>\n",
              "      <td>0.830053</td>\n",
              "      <td>0.930233</td>\n",
              "      <td>0.872606</td>\n",
              "      <td>0.031690</td>\n",
              "      <td>0.140845</td>\n",
              "      <td>238.767606</td>\n",
              "      <td>250.147396</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td></td>\n",
              "      <td>5</td>\n",
              "      <td>0.050514</td>\n",
              "      <td>0.791687</td>\n",
              "      <td>3.421895</td>\n",
              "      <td>3.485263</td>\n",
              "      <td>0.909091</td>\n",
              "      <td>0.808192</td>\n",
              "      <td>0.925926</td>\n",
              "      <td>0.859484</td>\n",
              "      <td>0.035211</td>\n",
              "      <td>0.176056</td>\n",
              "      <td>242.189501</td>\n",
              "      <td>248.526343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td></td>\n",
              "      <td>6</td>\n",
              "      <td>0.100094</td>\n",
              "      <td>0.672021</td>\n",
              "      <td>2.130614</td>\n",
              "      <td>2.814269</td>\n",
              "      <td>0.566038</td>\n",
              "      <td>0.715380</td>\n",
              "      <td>0.747664</td>\n",
              "      <td>0.788105</td>\n",
              "      <td>0.105634</td>\n",
              "      <td>0.281690</td>\n",
              "      <td>113.061387</td>\n",
              "      <td>181.426879</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td></td>\n",
              "      <td>7</td>\n",
              "      <td>0.150608</td>\n",
              "      <td>0.590287</td>\n",
              "      <td>2.230569</td>\n",
              "      <td>2.618494</td>\n",
              "      <td>0.592593</td>\n",
              "      <td>0.632172</td>\n",
              "      <td>0.695652</td>\n",
              "      <td>0.735805</td>\n",
              "      <td>0.112676</td>\n",
              "      <td>0.394366</td>\n",
              "      <td>123.056860</td>\n",
              "      <td>161.849357</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td></td>\n",
              "      <td>8</td>\n",
              "      <td>0.200187</td>\n",
              "      <td>0.526436</td>\n",
              "      <td>2.343675</td>\n",
              "      <td>2.550431</td>\n",
              "      <td>0.622642</td>\n",
              "      <td>0.558755</td>\n",
              "      <td>0.677570</td>\n",
              "      <td>0.691956</td>\n",
              "      <td>0.116197</td>\n",
              "      <td>0.510563</td>\n",
              "      <td>134.367526</td>\n",
              "      <td>155.043109</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td></td>\n",
              "      <td>9</td>\n",
              "      <td>0.300281</td>\n",
              "      <td>0.393655</td>\n",
              "      <td>1.758918</td>\n",
              "      <td>2.286593</td>\n",
              "      <td>0.467290</td>\n",
              "      <td>0.449514</td>\n",
              "      <td>0.607477</td>\n",
              "      <td>0.611142</td>\n",
              "      <td>0.176056</td>\n",
              "      <td>0.686620</td>\n",
              "      <td>75.891799</td>\n",
              "      <td>128.659339</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td></td>\n",
              "      <td>10</td>\n",
              "      <td>0.400374</td>\n",
              "      <td>0.281592</td>\n",
              "      <td>1.196064</td>\n",
              "      <td>2.013961</td>\n",
              "      <td>0.317757</td>\n",
              "      <td>0.337617</td>\n",
              "      <td>0.535047</td>\n",
              "      <td>0.542761</td>\n",
              "      <td>0.119718</td>\n",
              "      <td>0.806338</td>\n",
              "      <td>19.606424</td>\n",
              "      <td>101.396110</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td></td>\n",
              "      <td>11</td>\n",
              "      <td>0.500468</td>\n",
              "      <td>0.169858</td>\n",
              "      <td>0.809102</td>\n",
              "      <td>1.772989</td>\n",
              "      <td>0.214953</td>\n",
              "      <td>0.222529</td>\n",
              "      <td>0.471028</td>\n",
              "      <td>0.478714</td>\n",
              "      <td>0.080986</td>\n",
              "      <td>0.887324</td>\n",
              "      <td>-19.089772</td>\n",
              "      <td>77.298934</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td></td>\n",
              "      <td>12</td>\n",
              "      <td>0.599626</td>\n",
              "      <td>0.107140</td>\n",
              "      <td>0.532653</td>\n",
              "      <td>1.567879</td>\n",
              "      <td>0.141509</td>\n",
              "      <td>0.136238</td>\n",
              "      <td>0.416537</td>\n",
              "      <td>0.422080</td>\n",
              "      <td>0.052817</td>\n",
              "      <td>0.940141</td>\n",
              "      <td>-46.734653</td>\n",
              "      <td>56.787919</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td></td>\n",
              "      <td>13</td>\n",
              "      <td>0.699719</td>\n",
              "      <td>0.062433</td>\n",
              "      <td>0.351784</td>\n",
              "      <td>1.393919</td>\n",
              "      <td>0.093458</td>\n",
              "      <td>0.083757</td>\n",
              "      <td>0.370321</td>\n",
              "      <td>0.373684</td>\n",
              "      <td>0.035211</td>\n",
              "      <td>0.975352</td>\n",
              "      <td>-64.821640</td>\n",
              "      <td>39.391900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td></td>\n",
              "      <td>14</td>\n",
              "      <td>0.799813</td>\n",
              "      <td>0.037689</td>\n",
              "      <td>0.140713</td>\n",
              "      <td>1.237085</td>\n",
              "      <td>0.037383</td>\n",
              "      <td>0.049346</td>\n",
              "      <td>0.328655</td>\n",
              "      <td>0.333094</td>\n",
              "      <td>0.014085</td>\n",
              "      <td>0.989437</td>\n",
              "      <td>-85.928656</td>\n",
              "      <td>23.708508</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td></td>\n",
              "      <td>15</td>\n",
              "      <td>0.899906</td>\n",
              "      <td>0.025419</td>\n",
              "      <td>0.070357</td>\n",
              "      <td>1.107314</td>\n",
              "      <td>0.018692</td>\n",
              "      <td>0.030728</td>\n",
              "      <td>0.294179</td>\n",
              "      <td>0.299463</td>\n",
              "      <td>0.007042</td>\n",
              "      <td>0.996479</td>\n",
              "      <td>-92.964328</td>\n",
              "      <td>10.731384</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td></td>\n",
              "      <td>16</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.018266</td>\n",
              "      <td>0.035178</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.009346</td>\n",
              "      <td>0.022617</td>\n",
              "      <td>0.265669</td>\n",
              "      <td>0.271752</td>\n",
              "      <td>0.003521</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-96.482164</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      group  ...        gain  cumulative_gain\n",
              "0         1  ...  276.408451       276.408451\n",
              "1         2  ...  207.970551       242.189501\n",
              "2         3  ...  276.408451       253.595817\n",
              "3         4  ...  238.767606       250.147396\n",
              "4         5  ...  242.189501       248.526343\n",
              "5         6  ...  113.061387       181.426879\n",
              "6         7  ...  123.056860       161.849357\n",
              "7         8  ...  134.367526       155.043109\n",
              "8         9  ...   75.891799       128.659339\n",
              "9        10  ...   19.606424       101.396110\n",
              "10       11  ...  -19.089772        77.298934\n",
              "11       12  ...  -46.734653        56.787919\n",
              "12       13  ...  -64.821640        39.391900\n",
              "13       14  ...  -85.928656        23.708508\n",
              "14       15  ...  -92.964328        10.731384\n",
              "15       16  ...  -96.482164         0.000000\n",
              "\n",
              "[16 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Scoring History: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>timestamp</th>\n",
              "      <th>duration</th>\n",
              "      <th>number_of_trees</th>\n",
              "      <th>training_rmse</th>\n",
              "      <th>training_logloss</th>\n",
              "      <th>training_auc</th>\n",
              "      <th>training_pr_auc</th>\n",
              "      <th>training_lift</th>\n",
              "      <th>training_classification_error</th>\n",
              "      <th>validation_rmse</th>\n",
              "      <th>validation_logloss</th>\n",
              "      <th>validation_auc</th>\n",
              "      <th>validation_pr_auc</th>\n",
              "      <th>validation_lift</th>\n",
              "      <th>validation_classification_error</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:25</td>\n",
              "      <td>0.002 sec</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.693147</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.730579</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.693147</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.734331</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:26</td>\n",
              "      <td>0.322 sec</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.451306</td>\n",
              "      <td>0.599465</td>\n",
              "      <td>0.864095</td>\n",
              "      <td>0.650674</td>\n",
              "      <td>3.486717</td>\n",
              "      <td>0.225415</td>\n",
              "      <td>0.451369</td>\n",
              "      <td>0.599555</td>\n",
              "      <td>0.855019</td>\n",
              "      <td>0.612469</td>\n",
              "      <td>3.450411</td>\n",
              "      <td>0.197381</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:26</td>\n",
              "      <td>0.543 sec</td>\n",
              "      <td>10.0</td>\n",
              "      <td>0.417875</td>\n",
              "      <td>0.537171</td>\n",
              "      <td>0.867374</td>\n",
              "      <td>0.700870</td>\n",
              "      <td>3.484422</td>\n",
              "      <td>0.204311</td>\n",
              "      <td>0.418241</td>\n",
              "      <td>0.537659</td>\n",
              "      <td>0.858810</td>\n",
              "      <td>0.677673</td>\n",
              "      <td>3.513146</td>\n",
              "      <td>0.201123</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:26</td>\n",
              "      <td>0.788 sec</td>\n",
              "      <td>15.0</td>\n",
              "      <td>0.395782</td>\n",
              "      <td>0.494959</td>\n",
              "      <td>0.870006</td>\n",
              "      <td>0.707311</td>\n",
              "      <td>3.474752</td>\n",
              "      <td>0.210597</td>\n",
              "      <td>0.397467</td>\n",
              "      <td>0.497480</td>\n",
              "      <td>0.858590</td>\n",
              "      <td>0.688202</td>\n",
              "      <td>3.450411</td>\n",
              "      <td>0.207671</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:26</td>\n",
              "      <td>1.005 sec</td>\n",
              "      <td>20.0</td>\n",
              "      <td>0.381068</td>\n",
              "      <td>0.465116</td>\n",
              "      <td>0.871990</td>\n",
              "      <td>0.713065</td>\n",
              "      <td>3.469601</td>\n",
              "      <td>0.210597</td>\n",
              "      <td>0.383858</td>\n",
              "      <td>0.469220</td>\n",
              "      <td>0.860119</td>\n",
              "      <td>0.682457</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.217961</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:26</td>\n",
              "      <td>1.244 sec</td>\n",
              "      <td>25.0</td>\n",
              "      <td>0.370788</td>\n",
              "      <td>0.442708</td>\n",
              "      <td>0.873594</td>\n",
              "      <td>0.720191</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.209699</td>\n",
              "      <td>0.374983</td>\n",
              "      <td>0.448909</td>\n",
              "      <td>0.860153</td>\n",
              "      <td>0.685266</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.208606</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:27</td>\n",
              "      <td>1.433 sec</td>\n",
              "      <td>30.0</td>\n",
              "      <td>0.363515</td>\n",
              "      <td>0.425682</td>\n",
              "      <td>0.874957</td>\n",
              "      <td>0.720912</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.207678</td>\n",
              "      <td>0.369251</td>\n",
              "      <td>0.434260</td>\n",
              "      <td>0.859561</td>\n",
              "      <td>0.683473</td>\n",
              "      <td>3.474540</td>\n",
              "      <td>0.208606</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:27</td>\n",
              "      <td>1.600 sec</td>\n",
              "      <td>35.0</td>\n",
              "      <td>0.358498</td>\n",
              "      <td>0.413048</td>\n",
              "      <td>0.876798</td>\n",
              "      <td>0.728192</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.198249</td>\n",
              "      <td>0.366187</td>\n",
              "      <td>0.424738</td>\n",
              "      <td>0.859056</td>\n",
              "      <td>0.682699</td>\n",
              "      <td>3.474540</td>\n",
              "      <td>0.199252</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:27</td>\n",
              "      <td>1.855 sec</td>\n",
              "      <td>40.0</td>\n",
              "      <td>0.354522</td>\n",
              "      <td>0.402953</td>\n",
              "      <td>0.879462</td>\n",
              "      <td>0.732853</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.201617</td>\n",
              "      <td>0.364337</td>\n",
              "      <td>0.418383</td>\n",
              "      <td>0.858242</td>\n",
              "      <td>0.684977</td>\n",
              "      <td>3.450411</td>\n",
              "      <td>0.192703</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:27</td>\n",
              "      <td>2.064 sec</td>\n",
              "      <td>45.0</td>\n",
              "      <td>0.351564</td>\n",
              "      <td>0.394884</td>\n",
              "      <td>0.880912</td>\n",
              "      <td>0.735681</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.175573</td>\n",
              "      <td>0.362666</td>\n",
              "      <td>0.412550</td>\n",
              "      <td>0.858552</td>\n",
              "      <td>0.686469</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.202058</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:28</td>\n",
              "      <td>2.382 sec</td>\n",
              "      <td>50.0</td>\n",
              "      <td>0.349213</td>\n",
              "      <td>0.388562</td>\n",
              "      <td>0.882465</td>\n",
              "      <td>0.740370</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.198024</td>\n",
              "      <td>0.362049</td>\n",
              "      <td>0.409335</td>\n",
              "      <td>0.857778</td>\n",
              "      <td>0.683861</td>\n",
              "      <td>3.450411</td>\n",
              "      <td>0.198316</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:28</td>\n",
              "      <td>2.656 sec</td>\n",
              "      <td>55.0</td>\n",
              "      <td>0.347289</td>\n",
              "      <td>0.383445</td>\n",
              "      <td>0.883859</td>\n",
              "      <td>0.744340</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.192187</td>\n",
              "      <td>0.362073</td>\n",
              "      <td>0.407784</td>\n",
              "      <td>0.856865</td>\n",
              "      <td>0.682152</td>\n",
              "      <td>3.474540</td>\n",
              "      <td>0.205800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:28</td>\n",
              "      <td>2.882 sec</td>\n",
              "      <td>60.0</td>\n",
              "      <td>0.345407</td>\n",
              "      <td>0.378674</td>\n",
              "      <td>0.885603</td>\n",
              "      <td>0.747830</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.196004</td>\n",
              "      <td>0.362200</td>\n",
              "      <td>0.406650</td>\n",
              "      <td>0.856094</td>\n",
              "      <td>0.681473</td>\n",
              "      <td>3.474540</td>\n",
              "      <td>0.201123</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:28</td>\n",
              "      <td>3.146 sec</td>\n",
              "      <td>65.0</td>\n",
              "      <td>0.343767</td>\n",
              "      <td>0.374715</td>\n",
              "      <td>0.887414</td>\n",
              "      <td>0.750432</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.186349</td>\n",
              "      <td>0.362427</td>\n",
              "      <td>0.406161</td>\n",
              "      <td>0.855504</td>\n",
              "      <td>0.681001</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.202058</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:29</td>\n",
              "      <td>3.385 sec</td>\n",
              "      <td>70.0</td>\n",
              "      <td>0.342284</td>\n",
              "      <td>0.371238</td>\n",
              "      <td>0.889134</td>\n",
              "      <td>0.754232</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.182084</td>\n",
              "      <td>0.362367</td>\n",
              "      <td>0.405651</td>\n",
              "      <td>0.854918</td>\n",
              "      <td>0.680593</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.199252</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:29</td>\n",
              "      <td>3.592 sec</td>\n",
              "      <td>75.0</td>\n",
              "      <td>0.341016</td>\n",
              "      <td>0.368250</td>\n",
              "      <td>0.890739</td>\n",
              "      <td>0.756042</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.180736</td>\n",
              "      <td>0.362761</td>\n",
              "      <td>0.405939</td>\n",
              "      <td>0.854082</td>\n",
              "      <td>0.679601</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.200187</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:29</td>\n",
              "      <td>3.871 sec</td>\n",
              "      <td>80.0</td>\n",
              "      <td>0.339823</td>\n",
              "      <td>0.365632</td>\n",
              "      <td>0.892047</td>\n",
              "      <td>0.759011</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.182308</td>\n",
              "      <td>0.362969</td>\n",
              "      <td>0.406050</td>\n",
              "      <td>0.853528</td>\n",
              "      <td>0.679006</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.202993</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td></td>\n",
              "      <td>2020-01-06 01:17:29</td>\n",
              "      <td>4.077 sec</td>\n",
              "      <td>85.0</td>\n",
              "      <td>0.338581</td>\n",
              "      <td>0.363113</td>\n",
              "      <td>0.893613</td>\n",
              "      <td>0.761548</td>\n",
              "      <td>3.464222</td>\n",
              "      <td>0.176471</td>\n",
              "      <td>0.363331</td>\n",
              "      <td>0.406519</td>\n",
              "      <td>0.853005</td>\n",
              "      <td>0.676111</td>\n",
              "      <td>3.764085</td>\n",
              "      <td>0.201123</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                timestamp  ... validation_lift  validation_classification_error\n",
              "0     2020-01-06 01:17:25  ...        1.000000                         0.734331\n",
              "1     2020-01-06 01:17:26  ...        3.450411                         0.197381\n",
              "2     2020-01-06 01:17:26  ...        3.513146                         0.201123\n",
              "3     2020-01-06 01:17:26  ...        3.450411                         0.207671\n",
              "4     2020-01-06 01:17:26  ...        3.764085                         0.217961\n",
              "5     2020-01-06 01:17:26  ...        3.764085                         0.208606\n",
              "6     2020-01-06 01:17:27  ...        3.474540                         0.208606\n",
              "7     2020-01-06 01:17:27  ...        3.474540                         0.199252\n",
              "8     2020-01-06 01:17:27  ...        3.450411                         0.192703\n",
              "9     2020-01-06 01:17:27  ...        3.764085                         0.202058\n",
              "10    2020-01-06 01:17:28  ...        3.450411                         0.198316\n",
              "11    2020-01-06 01:17:28  ...        3.474540                         0.205800\n",
              "12    2020-01-06 01:17:28  ...        3.474540                         0.201123\n",
              "13    2020-01-06 01:17:28  ...        3.764085                         0.202058\n",
              "14    2020-01-06 01:17:29  ...        3.764085                         0.199252\n",
              "15    2020-01-06 01:17:29  ...        3.764085                         0.200187\n",
              "16    2020-01-06 01:17:29  ...        3.764085                         0.202993\n",
              "17    2020-01-06 01:17:29  ...        3.764085                         0.201123\n",
              "\n",
              "[18 rows x 16 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Variable Importances: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>variable</th>\n",
              "      <th>relative_importance</th>\n",
              "      <th>scaled_importance</th>\n",
              "      <th>percentage</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Contract.Month-to-month</td>\n",
              "      <td>3655.777832</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.256163</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>TotalCharges</td>\n",
              "      <td>2007.098267</td>\n",
              "      <td>0.549021</td>\n",
              "      <td>0.140639</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>tenure</td>\n",
              "      <td>1818.509888</td>\n",
              "      <td>0.497434</td>\n",
              "      <td>0.127424</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>MonthlyCharges</td>\n",
              "      <td>1745.274536</td>\n",
              "      <td>0.477402</td>\n",
              "      <td>0.122292</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>TechSupport.No</td>\n",
              "      <td>732.165771</td>\n",
              "      <td>0.200276</td>\n",
              "      <td>0.051303</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>OnlineSecurity.No</td>\n",
              "      <td>715.071289</td>\n",
              "      <td>0.195600</td>\n",
              "      <td>0.050105</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>InternetService.Fiberoptic</td>\n",
              "      <td>570.084229</td>\n",
              "      <td>0.155941</td>\n",
              "      <td>0.039946</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Contract.Twoyear</td>\n",
              "      <td>481.123566</td>\n",
              "      <td>0.131606</td>\n",
              "      <td>0.033713</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>PaymentMethod.Electroniccheck</td>\n",
              "      <td>284.173859</td>\n",
              "      <td>0.077733</td>\n",
              "      <td>0.019912</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>PaperlessBilling.No</td>\n",
              "      <td>220.357941</td>\n",
              "      <td>0.060277</td>\n",
              "      <td>0.015441</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>gender.Female</td>\n",
              "      <td>209.769211</td>\n",
              "      <td>0.057380</td>\n",
              "      <td>0.014699</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>InternetService.DSL</td>\n",
              "      <td>130.192383</td>\n",
              "      <td>0.035613</td>\n",
              "      <td>0.009123</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Contract.Oneyear</td>\n",
              "      <td>121.914719</td>\n",
              "      <td>0.033349</td>\n",
              "      <td>0.008543</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>SeniorCitizen</td>\n",
              "      <td>119.388542</td>\n",
              "      <td>0.032657</td>\n",
              "      <td>0.008366</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>DeviceProtection.No</td>\n",
              "      <td>110.401329</td>\n",
              "      <td>0.030199</td>\n",
              "      <td>0.007736</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>StreamingMovies.Yes</td>\n",
              "      <td>105.824234</td>\n",
              "      <td>0.028947</td>\n",
              "      <td>0.007415</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>PaperlessBilling.Yes</td>\n",
              "      <td>102.785744</td>\n",
              "      <td>0.028116</td>\n",
              "      <td>0.007202</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>StreamingMovies.No</td>\n",
              "      <td>94.509766</td>\n",
              "      <td>0.025852</td>\n",
              "      <td>0.006622</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>MultipleLines.No</td>\n",
              "      <td>84.426620</td>\n",
              "      <td>0.023094</td>\n",
              "      <td>0.005916</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>gender.Male</td>\n",
              "      <td>82.005692</td>\n",
              "      <td>0.022432</td>\n",
              "      <td>0.005746</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         variable  ...  percentage\n",
              "0         Contract.Month-to-month  ...    0.256163\n",
              "1                    TotalCharges  ...    0.140639\n",
              "2                          tenure  ...    0.127424\n",
              "3                  MonthlyCharges  ...    0.122292\n",
              "4                  TechSupport.No  ...    0.051303\n",
              "5               OnlineSecurity.No  ...    0.050105\n",
              "6      InternetService.Fiberoptic  ...    0.039946\n",
              "7                Contract.Twoyear  ...    0.033713\n",
              "8   PaymentMethod.Electroniccheck  ...    0.019912\n",
              "9             PaperlessBilling.No  ...    0.015441\n",
              "10                  gender.Female  ...    0.014699\n",
              "11            InternetService.DSL  ...    0.009123\n",
              "12               Contract.Oneyear  ...    0.008543\n",
              "13                  SeniorCitizen  ...    0.008366\n",
              "14            DeviceProtection.No  ...    0.007736\n",
              "15            StreamingMovies.Yes  ...    0.007415\n",
              "16           PaperlessBilling.Yes  ...    0.007202\n",
              "17             StreamingMovies.No  ...    0.006622\n",
              "18               MultipleLines.No  ...    0.005916\n",
              "19                    gender.Male  ...    0.005746\n",
              "\n",
              "[20 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "See the whole table with table.as_data_frame()\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JGqA2fD7fJS7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "out_gbm = h2o.get_model([mid for mid in model_ids if \"GBM\" in mid][0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zGGLvJlrM0r1",
        "colab_type": "code",
        "outputId": "b88febf0-6969-434e-dbd1-ab206c77f0f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176
        }
      },
      "source": [
        "out.confusion_matrix()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.3606812246143818: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>Yes</th>\n",
              "      <th>Error</th>\n",
              "      <th>Rate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>No</td>\n",
              "      <td>2752.0</td>\n",
              "      <td>502.0</td>\n",
              "      <td>0.1543</td>\n",
              "      <td>(502.0/3254.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Yes</td>\n",
              "      <td>284.0</td>\n",
              "      <td>916.0</td>\n",
              "      <td>0.2367</td>\n",
              "      <td>(284.0/1200.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Total</td>\n",
              "      <td>3036.0</td>\n",
              "      <td>1418.0</td>\n",
              "      <td>0.1765</td>\n",
              "      <td>(786.0/4454.0)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              No     Yes   Error             Rate\n",
              "0     No  2752.0   502.0  0.1543   (502.0/3254.0)\n",
              "1    Yes   284.0   916.0  0.2367   (284.0/1200.0)\n",
              "2  Total  3036.0  1418.0  0.1765   (786.0/4454.0)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WzYCf-itPIf_",
        "colab_type": "code",
        "outputId": "91affbb5-4e3f-434a-dc7c-0f8dd21000a6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 612
        }
      },
      "source": [
        "out.varimp_plot()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA8UAAAJTCAYAAAAherSGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdebgkVX3/8fdHRlkVBMEFl4mAggKy\njIgiBBSNP8cYF4xrIhJFxV2jQY0R8yOCwYg7igY3cF9RVgExgMo+MCAat/EXcEUjsgvD9/dHnZae\npvsuw2UuQ71fz9NPd586depU9b33uZ8+p6pSVUiSJEmS1Ed3mu8OSJIkSZI0XwzFkiRJkqTeMhRL\nkiRJknrLUCxJkiRJ6i1DsSRJkiSptwzFkiRJkqTeMhRLkm73kpye5MY5aOfSJD+eRf3Nk1SSj97a\nbUuSpNsnQ7Ek6RaSHNXC4H4zqHtiq/vUVdG3O5oW+CvJo+e7L7e12X4p0QdJ9myf/0lT1Bl8OfPj\nkfL7JnlVkuOTLEtyfZLftd/Jp0yz3TsneWGSbyb5TZI/tecTk+yTZMEs9+MzrY+vn7B8oyS/SHJd\nkm3HLL9Lkr2TfK39nFyX5JokP0vy5dandUbWWdC2Ofq4vh2PjyfZcjb7saolObAvv//S7dms/uBJ\nknrjI8BzgBcCH5xUKclCYE/gl8DXb8P+PAdY+zZsX1odvRp4HfBT4BTg18BC4KnA45IcUlVvGF0p\nyf2Bo4GHAb8CvtGe7wU8EXgcsF+SJ1fVL2bYl/2AXYH/m+SEqrpwZPnhwL2BfxxdluQhwJeALYH/\nbfvyM2A5cF9gt7ZP/9baGFXAvw693wB4BPB84OlJdhnTH0n6M0OxJOkWqurUJP8NbJ9kh6o6b0LV\nfwACfKyqbvX05in68/9uq7al1dj3gN2q6rThwiRbA98BXp/kqKq6YGjZesDxwFbAEcDLq+raoeXr\nAh8Cngccm+SRw8snqar/TfIC4ATgyCQPr6rrW5t7A08DTgXeNdLXTYGT6QL5ocBbqurqkToBHg8c\nNGHzN1XVAaOFSQ4DXgK8ku4LPkkay+nTkqRJPtKeXzRuYZI1gBfQjdJ8dKh80yRvTfKdJL9q0zIv\na1OybzGVcfi83SQPTvKFJL9NctNgSuG4c4qTrJnkFUmOS/LzNmXy92066F9NtWNJNkjywaHpnBcn\neVn753tGkqyb5E1JLkhydZKr2j4/c6ZtTNP+pUl+nORuSd7T3l+b5PwkT251FiR5S5Iftf34cZKX\njmlrMEX3n5PskuTkJH9sj+OS7DChDxskecdQ+79PN1X3MdNsY+ckx7b6leR5SQrYFNhsZKrr8M/O\n09rPyY+Gjuk5SV6e5Bb/syQ5srVxvyT7Jbmo9fNXST6U5G4T9ut+Sd43tF+/S3JWkjdPqPvBJD/N\nzdOTv5Zkx6k+v1Whqr44Gohb+UXAF9vb3UcW/yNdIP4v4IWjgbcF0r2BM+lGkl85i/58E/gAsA3d\nqO5gNsl7gSuA51dVjax2EF0g/mRVvXY0ELd2q6pOAHaaaV+aE9vzxqMLkqzVfn8var9Xf0zyX0n2\nmtRYkmclOa3VvTbJhUn+KcldxtTdLsnncvO09t8mOTfJoen+dpLkUmDwM3fa0O/EbfYFo6TxHCmW\nJE3yCbp/bJ+d5HVVdc3I8v9DF3K+WVU/GyrfA3gD8C3gfOBqYAvgb4G/TvKo9k/7qAcBZwHfB44E\n1gGunKJ/GwPvphsR+ybwW7qplU8GjkuyT1V9fMx6a9JNz1wP+HR7/wzg/a0Pr5pimwAkuXvbv4cB\n59KNuN0JeALw2SRbjRu5WglrAicBdwO+2t4/G/hykj2B1wA7AMcBN7T9+GCS31TVl8a09yjgX+jC\nwvvpPpenAX+ZZM+q+s7QPm4InEE3pfUsuumtG9N9jicl2beqxl2A7NFtG/8F/CewCfAj4G3Aa4Eb\n6ULSwPAshH8HrqcbAb0MWB94LPA+YEe6L2HG+Q+6Kb/foBupfCzwYmCzVv5nSR5Bd7zuTjdy+WVg\nXeAhrd//NlR3UWvv7nSjq4Nj8FTgCUn+uqpOHKq/gO5zWF5V8/0/1g3teTRgDb7kOnBMQAWgqpYn\neTvwNWBf4B2z2O4b6E6peE2SY+k+97sCzxud8ZFu1PpZ7e3bpmt4JWaj7NmezxnZ7pp0fzMeTff3\n5v10PwPPAL6Q5P9W1b+MrPPvwOvp/s4cSfd3bTFwMPD4JE+oqhta3e2B79JN/z4aWEb3O7wF8DLg\njW3Zu4Cn0E07/xgwOD43zXI/Jd1aVeXDhw8fPnyMfQCfoxsJ3nvMsq+1ZXuNlN8TWG9M/e3p/pH8\n+kj55q2dAv51Qj9OB24cKVsL2HRM3Q2AS+j+eV1zZNmlbTvfBu4yVH4PunMYC3jUmL59dKSdI1v5\na0fK16b7Z/smYJsZHuPTW1uPntDXrw7vB92XDgX8ni48rj+0bAu6MHT2SFt7Dh3jl4wse3or/wGQ\nofL/bOUfHKm/Jd2XFdcB95uwjX+YsK+XAj+e4lhsNqbsTsBRrd0dJ3wOPwPuO1R+Z7ovSwrYYah8\nTbrgUcDfjtnWaBs/Ba4d89ncl+48+ktHfo4WtLZvnLSPY7Y5OG4/BQ6Y8HhvqzPx2I35HfgtXfDa\nYqj8L1o7f2Lkd2NMG+u19Qu410z3p627qP0cXtfW/9yEeo8ZfH6zaX/M8b5p5Hi9i+736ia6v1Pr\njaz3lrbe0cCCofJ7Af/T1nvEUPmurf4yYJOR7R/blr1hqPw9rWzxmD5vyIq/Zwcy5vffhw8fq/Yx\n7x3w4cOHDx+33wfdiFsBp4+U37v90/tr4M6zaO9Y4BpgjaGyQfC8bDhgjKx3i1A8zXbewEjAbeWD\noPnIMeu8sC37yJi+fXSobJMWFr47Yds7tnXePsO+TheKHzBmnUGw223MstPoRluH//EeBK9LhsvH\n9GGX9n5NujB4BbDBmPoHtfpvGrONs6fY1ylD8RTr7TS6vVY+CMV7j1nnRYx8CQA8s5V9aQbbHHxZ\ncNCE5a9ryx8/Ur4l8OBZ7NvwlwnTPaY9dnTn+H+51X/PyLJHtfJLZ9i3yxn5YmEW+zX4bP4I3H1C\nnecw5u/L0PJ9uOUXBNsOLR+E4kmPpcCzxrT7M7rf4c3HLHtxW/fwobKPtbJ9xtTfii5E//dQ2SAU\nP2YGx8lQ7MPH7eAx31N7JEm3b6cAPwF2aVOCL2nlL6D7h/Tj1aYMDmvnvL6YLiBuxC1P19mQbiRr\n2JKq+tNsOpdkG7opjY8G7kMX5oZtOma1wfTcUae25+2n2exOdKOXSXLAmOWDPmw1TTszcXlV/XxM\n+S+A+7Hi1OOBy4C70IX3X48sO62qasw6pwK70O37GXRTidcCzqyqP4ypfwqwP+OP1VljymYkyT3o\nPs8n0o1qrjtSZdznCSPTY5v/ac93HyrbuT0fN4PuPLI9/8WEz/nB7Xkrbj53lar6wQzaHufkqtpz\n3IIkm9NNQZ+J99BN7z6V7liucuku9DU4N/eudKcVfGYlmtqH7udy2I+B0StJrzBdvU3LfijdtO/P\ntL9db23L7k53he6fV9W424Od0p6Hf7Z3GFn2Z1V1SZJfAlskWa+qrgI+C7wc+HqSL9KdAvGdqvrJ\ndDssaX4YiiVJE1XV4EJIB9GNpL4uSeiuOl3cfDGuP0vyOuCddNN7TwJ+TjfqWHTnr27DLcMrdLeE\nmbEku7T270R39dqv0U3rvYnun9i/nrCd304IhoPtrz/Npjdqz49oj0nWm6admbhiQvmNdEHgqgnL\noJv+O2o0JA+M7vvg+ZcT6g/KN5iirVlp5zCfAzyA7iJPn6T7GbqR7kuUVzD+8wQYF9wHx2GNobJB\nfy+bQZcGn/N0F06bi895TiR5F91x+hbwpDFfMg0+m42TrFnt6tAT2lqXm79QmOltmWgXnTqS7rN6\nOV0w/UCSb9ctb+806M99xrVVVX++d2+Sg4F/mkkf2u/FmUmeRjc74Y1JPty2vzI/2zNZ5z6t3lVV\n9d0kuwFvojsH/+/bPvwAOKCqPjeT/ZC06hiKJUnT+RjdPUD/Pskb6c6veyBwyuhIS5I7A2+l+yd6\nh6r69cjyXafYzrigOpW30I1m7lpVp49s5y10oXicjZNkTDC+V3ueFEQZWT72HrC3c/ecUD6671eM\nlI+690i9YbP9HAf2pQvEb6mqA4cXtJ+bV6xku8MG4XnSiPOwwb4trqpj52Dbt5n2RdV76ULoScCT\na8xtlKrqp21U89509/795hTNPobuC6efVtVsvug4kO4CdB+sqg8kuYnuXuf/SXdxvmFn0Z2G8RdJ\n/qJWvGDfrVZVv0/yI2BbupHfX7ByP9vD64ybuXGLdarqDGBxu6jXIrp9fzndyPWvq+rU2e2NpNuS\nt2SSJE2pBduj6S5G9RRuvt/n4WOq35NuuuTpYwLx3Zh+avJsbA78ZjQQN385xXprcvM02mG7t+fz\np9numXTBb6qAf3u1awtQo3Zvz4N9v4Rumvn2E25rtEd7nnT/6kmWs+LI7bDN2/O4q2ZP9XnOxmDa\n/Gg4m6ru7fpzbp/nh+gC1/HAX48LxEMGVwx/04SfBdLd/upN7e243/NJfdmN7lzr/6ZN3a6qw+iu\n4P2EJC8Zrt9GdAejpitc7XkODUa779S2+b90wfb+SR44pv64n+3B78Xuo5WTPJguFP9o3MyNqrq+\nqs6oqn+mu1p8gL8ZqrK8PU/6vZC0ChiKJUkzMZgm/Tq68xUvB74ypt4v6cLUw9v0S+DPUyrfx4rn\nd95ay+hGfR86XJjkxXQXCJvKwcP3Fm3nsg7uF/qxqVasql/SnTO4c5I3Du45OtKHzZM8YPpdWOW2\npDvX+8+SPJ3uvM0f0l2xmaq6ju52VevTzRIYrr8FXQD7E9002dn4HbBJGz0btaw97z6yvUXMcNrs\nDHyV7lzjpyV5xujCJPcdevuV1qdXZsJ9r5M8KslaI2VbtqB0m2vh9T/pRtm/ATylfXZTOYTus94d\n+PCY/q9Dd4uxnYELWPH2WVP15W50t3G7ie72S8O3cNsH+F/gnUk2G1n1jXTTqPdO8s62/XGmO61h\nXJ/2ojv3fvQ6AoNbqB2SoftfJ9mEm/8OHDFSH+AtSTYaqr+A7nZgofscBuW3+LloBjM1ho/N79rz\n/We4W5JuA06fliTNxIl0AWGn9v794y6KVd39Td8H/COwNMnRdCOzj6H7p/bbzN2o36F04fc7ST5P\nd5XbnegukPQluqsHj3Mp3Wj2RUP924tuauR7a+hevVN4Kd3I5tvp/pk/nZvvk/wQuumSz2D8VMv5\ndBzw3iSL6a7MO7hP8bV0t1Eanvo8uIDZq5LsRPfZDe5TvB7w0hq57+wMnEw3W+D4JKfRBevzq+oY\n4ON0X7q8r92D+cd0941+Et3nOd25vdOqqutbGD4e+HySb9FN4V2b7oJZu9FNyR/UfVqre3ySM4Al\ndMfq/sDD6S4GtjHdrYcGIekSutG/VfE/1tvoLnp3Dd3Fp944ZvD3vKo6evCmqq5M8gS62R8vAp6U\n5Di6YHovunvv3pNupHS6Uedh76O7gNUBVXX28IKq+kWSl9F90fKJJLtV1U1t2aVJHkt3xezXAfsk\nOZnuCtE3tT7tQvf79mu6QD/qTiMXQ1sX2BoYfJmxf1UNX9jvHXQX/3oacEHb/8F9ijemu3L8n0N0\nVf1XO1/7tcDF7eJZ17Rj9RC63413DbX/JrpZGae1/bi69ef/0J0nP3wthlPoZp68I8nD6Kb431RV\nbx+zn5JuK/N9+WsfPnz48LF6POhGUAa3Opl4yxm6MPB6unBwLd3o8SfpRmwGt2kZvh/s2HsBj7Q5\n9pZMwJPppjNfSTcSdQJdkBvcXul5I/UvpQtbGwCH0Z1jeD3wfbrRz4zUn9g3ujD9SuC7dOcSXkcX\ngk8CXgVsOMPjOtUtmcbegmfS8WjLxh3jwW1//pkuYJzSjtmVdKFvxwlt3Z1uZPHH7TgNjvGeY+r+\neRtT7Ot6wIfpLnR14+ixpQsO36D7guFqugtv7TPpcxi3rzPpD925yx+i+6LnerqZD9+jC0+jde9J\nF6IupgtCV9FdCfoLwHNZ8fZit+Y+xSdNUWew/z8eKR/s/1SPsb9XdFco35fui4rf0p3b+1u684z/\ngaH7985gH57WtnXmVOvRzbCoCcf5LnQB/+vt5+N6ur8fy+hG+PcB1h3zt2bcPt9A93fnq8BjJ/Rl\nbbrfh4vpfnevpLud2S3uXz20znPprs4+uE/3RXQj3aP3Q38C3Zc8l9D9bbiK7j7g7wHuP6bd59ON\nyg8uSDjjnx8fPnzMzSNVK3s9DEmStDpoI6/fZMxFrCRJ6jvPKZYkSZIk9ZahWJIkSZLUW4ZiSZIk\nSVJveU6xJEmSJKm3vCWT7hA+8YlP1POf//z57oYkSZKk269b3LcOnD6tO4irr756vrsgSZIkaTVk\nKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJv\nGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLU\nW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk\n9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIk\nSb21YL47IM2FpZddwcL9j5nvbkiSJEkClh28eL67MGOOFEuSJEmSestQLEmSJEnqLUOxJEmSJKm3\nDMWSJEmSpN4yFEuSJEmSestQLEmSJEnqLUOxJEmSJKm3DMWSJEmSpN4yFEuSJEmSestQLEmSJEnq\nLUOxJEmSJKm3DMWSJEmSpN4yFEuSJEmSestQLEmSJEnqLUOxJEmSJKm3DMWSJEmSpN5aLUNxknsl\n+WySnyQ5N8mxSR60Eu28Osk6c9SnhUmeM8XySnLk0PsFSX6b5Bsrub0Nkuw39H73mbQ1XT9XtSR7\nJ7nP0PtlSe4xn32SJEmS1B+rXShOEuArwKlVtVlV7Qi8EbjnSjT3amBsKE6yxizbWghMFTavBrZO\nsnZ7/zjgslluY9gGwH7T1rqlhUzdz1Vtb+A+01WSJEmSpNvCaheKgT2AG6rqQ4OCqroAOD3JIUku\nSrI0yTPhzyOopyb5YpIfJDkqnVfShbFvJflWq3tVkv9IcgHwyCT/kuTs1ubhLZCTZPMkJyW5IMl5\nSTYDDgZ2TbIkyWsm9P1YYHF7/WzgM4MFSTZM8tUkFyb5XpJtW/kBSY5o+/DT1m/a9jZr2zukla03\nup9j+rBCP5OsleRj7Zidn2SPcR1v2z80yTlJLkny8CRfTvKjJAcO1XttO14XJXl1K1vY1vlIkouT\nnJhk7SR7AYuAo1p/Bl8YvKId16VJtpxwLCVJkiTpVlsdQ/HWwLljyp8GbAc8DNgTOCTJvduy7elG\nhR8CPBDYpareC/wC2KOqBkFwXeDMqnpYVZ0OvL+qHl5VWwNrA09q9Y4CPlBVDwMeBfwS2B84raq2\nq6pDJ/T9s8CzkqwFbAucObTsbcD5VbUt8Cbgk0PLtgT+CtgJeGuSO7ft/aRt7/WT9nNMH0b7+TKg\nqmobuqD+ida/cf5UVYuADwFfa+tuDeydZKMkOwIvAB4B7Ay8KMn2bd0t2jF7KPAH4OlV9UXgHOC5\nrT/XtrqXV9UOwGHAP07oC0n2bSH9nOXXXDGpmiRJkiRNtDqG4kkeDXymqpZX1a+BbwMPb8vOqqpL\nq+omYAndFOJxlgNfGnq/R5IzkywFHgM8NMldgU2r6isAVXVdVV0zkw5W1YVt28+mGzUe7f+nWr1T\ngI2S3K0tO6aqrq+qy4HfMHmq+Ez3c3S7R7bt/gD4OTDp/Oyj2/NS4OKq+mVVXQ/8FLhfa+srVXV1\nVV0FfBnYta3zs6pa0l6fO03fvjyTelV1eFUtqqpFa6yz/hTNSZIkSdJ4q2MovhjYcZbrXD/0ejmw\nYEK966pqOUAbLf0gsFcbRf0IMGkEdTaOBt7J0NTpGZhp/29RL8kj2tTkJUmePNMNtinVS5IMh/dB\n+zeNbOumKfo0sW8zqDtdPUmSJEm6VVbHUHwKsGaSfQcF7fzbPwDPTLJGko2B3YCzpmnrSuCuE5YN\nAvDlSdYD9gKoqiuBS5M8pW17zXRXsJ6qrWFHAG+rqqUj5acBz21t7k43hfiPK9n3P6uqM9vU5O2q\n6ugx6w1v90HA/YEfVtUL2jpPnME+Dbf1lCTrJFkXeGorm8pMj5skSZIkzbnVLhRXVdGFrT3T3ZLp\nYuAg4NPAhcAFdMH5DVX1q2maOxw4fnChrZHt/IFudPgi4ATg7KHFfwe8MsmFwHeAe7VtL28X33pN\nkvuMjLIO2r20nc886gBgx9bmwcDzp+p4Vf0OOKNd0OqQqeqOWKGfdKPhd2pTxD8H7N2mRM9aVZ0H\nfJzuy4gzgY9W1fnTrPZx4EMjF9qSJEmSpFUiXcaUVm8vffNBddzybee7G5IkSZKAZQcvnr7Sqjfu\n7jyr30ixJEmSJElzxVAsSZIkSeotQ7EkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6y1AsSZIkSeotQ7Ek\nSZIkqbcMxZIkSZKk3jIUS5IkSZJ6y1AsSZIkSeotQ7EkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6a8F8\nd0CaC9tsuj6H7bd4vrshSZIkaTXjSLEkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6y1AsSZIkSeotQ7Ek\nSZIkqbcMxZIkSZKk3jIUS5IkSZJ6y1AsSZIkSeotQ7EkSZIkqbcWzHcHpLmw9LIrWLj/MfPdDUnS\nHcyygxfPdxckSbcxR4olSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYk\nSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYol\nSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYpv55JslGRJe/wqyWVD7+8ypv6GSV4yg3YX\nJPnD0PstkxyX5EdJzkvy2SSbJHlhknfP9X5JkiRJ0u3BgvnugKZWVb8DtgNIcgBwVVW9c4pVNgRe\nAnxopttIsg5wDPCKqjq2lT0W2Ggluz3c9oKquvHWtiNJkiRJtwVHildjSd6Q5KL2eEUrPhh4cBtJ\nPjjJ3ZKc0kZ/L0zypDFNPQ/49iAQA1TVyVV1SXt73yQntFHkg4a2f3iSc5JcnORfhsovbds+H3hq\nkp3btpckeWeSJa3egiTvSnJWW/7CVr5pktNb/YuSPGpuj5wkSZIkdRwpXk0leQTwXODhdJ/jWUlO\nBfYHNq+qwejynYGnVNUfk2wCnAF8Y6S5rYFzp9jcw4AdgRuA/07yvqr6BbB/Vf0+yQLgW0m+WFXf\nb+v8pqq2b324BHh+VZ2VZHiUe99Wb6ckawLfS3Ii8Gzg61X1jiRrAGtPOAb7tjZ40av/Cdac5qBJ\nkiRJ0ghHildfjwa+VFXXVtWVwFeBXcfUC3BwkguBE4H7JbnHLLd1UlX9saquBX4A3L+VPzvJecB5\nwFbAQ4bW+RxA29ZdquqsVv7poTqPB17QRo7PBDYAtgDOBl6Y5K3A1lV11bhOVdXhVbWoqhatsc76\ns9wlSZIkSXKkuA/+Hlgf2KGqbkxyKbDWSJ2LgUdM0cb1Q6+XAwuSbAG8Ctipqv6Q5MiRdq+eQd8C\n7FdVJ99iQbI7sBj4ZJJ/r6qjZtCeJEmSJM2KI8Wrr9PoztddO8l6wN+0siuBuw7VW59uivKNSR4H\nbDqmrU8Buyd5wqAgyR5Jtppi+3dr2/pjknsDfzWuUlVdDtyQZFEretbQ4hOA/dr0a5I8uO3PA4Bf\nVdXhwMeA7afohyRJkiStNEeKV1Pt/NzP0E01BjisqpYCJDk3yVK6K0q/C/h6e38W8KMxbV3TLsB1\naJL30Z07vIRuJHiS84Dv002n/jnducqT7AN8LMmNdMH9ilb+Ybqp2EuSAPyGLtw/Fnhtkhvogvff\nTXUsJEmSJGllparmuw+6g0uy3uC84CRvBjasqtfN5TZe+uaD6rjl285lk5IksezgxfPdBUnS3Mm4\nQkeKtSo8Ockb6H7elgF7z2tvJEmSJKkxFOs2V1WfZsWrTkuSJEnS7YIX2pIkSZIk9ZahWJIkSZLU\nW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk\n9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb21YL47IM2FbTZdn8P2Wzzf3ZAkSZK0mnGkWJIkSZLU\nW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk\n9ZahWJIkSZLUWwvmuwPSXFh62RUs3P+Y+e6GJGklLTt48Xx3QZLUU44US5IkSZJ6y1AsSZIkSeot\nQ7EkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6y1AsSZIkSeotQ7EkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6\ny1AsSZIkSeotQ7EkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6y1AsSZIkSeotQ7EkSZIkqbcMxZqRJBsk\n2W+++yFJkiRJc8lQrJnaALhNQ3GSBbdl+5IkSZI0ylCsmToY2CzJkiSHJHl9krOTXJjkbQBJFia5\nJMlHklyc5MQka7dlpyZZ1F7fI8my9nrvJEcnOQU4uZXdom1JkiRJui0YijVT+wM/qartgG8CWwA7\nAdsBOybZrdXbAvhAVT0U+APw9Bm0vQOwV1X9ZZLHT9H2CpLsm+ScJOcsv+aKW7NvkiRJknrKUKyV\n8fj2OB84D9iSLsgC/KyqlrTX5wILZ9DeN6vq9zNoewVVdXhVLaqqRWuss/7K7IckSZKknvMcTq2M\nAAdV1YdXKEwWAtcPFS0H1m6vb+TmL2HWGmnv6unaliRJkqTbgiPFmqkrgbu21ycA+yRZDyDJpkk2\nmWb9ZcCO7fVeU9RbmbYlSWwjDgwAACAASURBVJIkaaU4UqwZqarfJTkjyUXAccCnge8mAbgKeB7d\nyPAk7wQ+n2Rf4JgptnNikq3GtP2bOdkRSZIkSRqSqprvPki32kvffFAdt3zb+e6GJGklLTt48Xx3\nQZJ0x5dxhU6fliRJkiT1lqFYkiRJktRbhmJJkiRJUm8ZiiVJkiRJvWUoliRJkiT1lqFYkiRJktRb\nhmJJkiRJUm8ZiiVJkiRJvWUoliRJkiT1lqFYkiRJktRbhmJJkiRJUm8ZiiVJkiRJvWUoliRJkiT1\nlqFYkiRJktRbC+a7A9Jc2GbT9Tlsv8Xz3Q1JkiRJqxlHiiVJkiRJvWUoliRJkiT1lqFYkiRJktRb\nhmJJkiRJUm8ZiiVJkiRJvWUoliRJkiT1lqFYkiRJktRbhmJJkiRJUm8ZiiVJkiRJvbVgvjsgzYWl\nl13Bwv2Pme9uSFKvLTt48Xx3QZKkWXOkWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9Zah\nWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1l\nKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZaheBVJUkmOHHq/IMlvk3xj\nJdvbIMl+Q+93n9RWklOTLJqmvXsl+WySnyQ5N8mxSR40VbuSJEmStLozFK86VwNbJ1m7vX8ccNmt\naG8DYL9pa81AkgBfAU6tqs2qakfgjcA956DtBbe2DUmSJEm6rRiKV61jgcXt9bOBzwwWJNkwyVeT\nXJjke0m2beUHJDmijfb+NMkr2yoHA5slWZLkkFa2XpIvJvlBkqNa2GVoG/skeffQ+xclORTYA7ih\nqj40WFZVF1TVaVO1m+Rfkpyd5KIkhw+Vn5rk3UnOAV6VZLO2T0uTHJjkqqE+vL61cWGSt7WydZMc\nk+SC1vYzb91hlyRJkqTxDMWr1meBZyVZC9gWOHNo2duA86tqW+BNwCeHlm0J/BWwE/DWJHcG9gd+\nUlXbVdXrW73tgVcDDwEeCOwysv3PA3/d1gd4AXAEsDVw7hT9ntTu+6vq4VW1NbA28KShde5SVYuq\n6j+A9wDvqaptgEsHFZI8Htii7dd2wI5JdgOeAPyiqh7W2j5+XKeS7JvknCTnLL/miim6L0mSJEnj\nGYpXoaq6EFhIN0p87MjiRwOfavVOATZKcre27Jiqur6qLgd+w+RpzWdV1aVVdROwpG1rePtXAacA\nT0qyJXDnqlo6g65PanePJGcmWQo8Bnjo0DqfG3r9SOAL7fWnh8of3x7nA+fRhf8tgKXA45K8I8mu\nVTU28VbV4S14L1pjnfVnsBuSJEmStCLP91z1jgbeCewObDTDda4fer2cyZ/bTOp9lG4k+gfAx1rZ\nxcBes9l+G+3+ILCoqv4nyQHAWkP1rp6ivYEAB1XVh2+xINkBeCJwYJKTq+pfZ9CeJEmSJM2KI8Wr\n3hHA28aM0J4GPBe6K0kDl1fVH6do50rgrrPdeFWdCdwPeA43n9N8CrBmkn0H9ZJsm2TXKZoaBODL\nk6zH1KH6e8DT2+tnDZWfAOzT1ifJpkk2SXIf4JqqOhI4BNhhZnsnSZIkSbPjSPEqVlWXAu8ds+gA\n4IgkFwLXAM+fpp3fJTkjyUXAccAxs+jG54Htqup/W1uV5KnAu5P8E3AdsIzuPOJNJ2z/D0k+AlwE\n/Ao4e4rtvRo4Msmb6c4PvqK1cWKSrYDvtmt0XQU8D9gcOCTJTcANwEtnsW+SJEmSNGOpqvnug1ax\ndt/hQ6vq5FW0vXWAa1v4fhbw7Kr6m7ncxkvffFAdt3zbuWxSkjRLyw5ePH0lSZLmT8YVOlLcI0k2\nAM4CLlhVgbjZEXh/u2XTH4B9VuG2JUmSJGkiQ3GPVNUfgAfNw3ZPAx62qrcrSZIkSdPxQluSJEmS\npN4yFEuSJEmSestQLEmSJEnqLUOxJEmSJKm3DMWSJEmSpN4yFEuSJEmSestQLEmSJEnqLUOxJEmS\nJKm3DMWSJEmSpN4yFEuSJEmSestQLEmSJEnqLUOxJEmSJKm3Fsx3B6S5sM2m63PYfovnuxuSJEmS\nVjOOFEuSJEmSestQLEmSJEnqLUOxJEmSJKm3DMWSJEmSpN4yFEuSJEmSestQLEmSJEnqLUOxJEmS\nJKm3DMWSJEmSpN4yFEuSJEmSemvBfHdAmgtLL7uChfsfM9/d0B3IsoMXz3cXJEmStAo4UixJkiRJ\n6i1DsSRJkiSptwzFkiRJkqTeMhRLkiRJknrLUCxJkiRJ6i1DsSRJkiSptwzFkiRJkqTeMhRLkiRJ\nknrLUCxJkiRJ6i1DsSRJkiSptwzFkiRJkqTeMhRLkiRJknrLUCxJkiRJ6i1DsSRJkiSptwzFkiRJ\nkqTeMhSvQkk2SrKkPX6V5LKh93eZRTsHJnn1hGX/kuTiJBcmOT/Jw+duD2YnyWuTrDVh2ZFJ/mew\n30nuleTHq7aHkiRJkvpuwXx3oE+q6nfAdgBJDgCuqqp3zlX7SXYFHg9sX1V/SrIx8/QZJ1kDeC1w\nBHDdhGoFPB/4yKrqlyRJkiQNc6T4diLJ85Oc1UaNP5jkTq18cZLzklyQ5MShVbZJ8u0kP03yslZ2\nb+C3VfUngKr6bVX9srVzaZIN2uudk5zUXh+Y5BNJvpfkR0n2aeV7JvlWkuOS/DDJB5KkLXtekqVJ\nLkry9la2IMkfkrw7yYXAG4BNgNMG2xrjUOAfW4AePhZ3SvKu1v7SJHvd2uMrSZIkSeMYim8HkmwN\nPBV4VFVtRze6+6wk9wIOA55aVQ8DnjW02oOAxwE7A//aguXxwGZDIXbXGXZhG2B3YJfW1j1b+SOA\nlwIPAbYC/ibJfYEDgT2A7YFdkjyp1V8f+K+q2raqDgJ+A+xaVXtO2O7PgDOB54yUP6Nt72FtHw9N\nssnoykn2TXJOknOWX3PFDHdVkiRJkm5mKL592BN4OHBOkiXAXwKbAY8EvlVVPweoqt8PrfONqvpT\nVf0G+D2wcVX9EdgBeAnwO+CLSf5uBtv/alVd19r6r9YXgO9V1bKqWg58Fng0XVA+paour6obgE8D\nu7X6fwK+Mst9fzvwT6z4s/ho4DNVtbyqfgWcDiwaXbGqDq+qRVW1aI111p/lZiVJkiTJc4pvLwIc\nUVVvWaEweeoU61w/9Ho57bOsqhuBbwHfSvJ94JnAp4AbuTl4jl78qia8n1Q+ybVVNV2dFRus+kHr\n59Nms54kSZIkzQVHim8fTgL+Nsk94M9Xqb4/8B1gjyQPaOUbTtVIkq2SbD5UtB3w8/Z6GbBje/30\nkVWfkmTNdmGuXYFzWvnOSe7fpmb/Ld2I7ZmtTxslWUA3pfvbE7p0JXDXqfrc/Bvw+qH3p9FNH79T\nm8q9y1CfJEmSJGnOGIpvB6pqKfA24KR2kaoTgXtW1a/pzun9WpILgKOmaWo94FPtlkxLgc2Bf23L\nDgA+mORsumnOwy6iC7bfAd7atgtwFvAh4PvAD4Gjq+pS4C3AqcASuinWx0zoz+FtnwYX9fpYku3G\n7P8FwAVDRV8EfgBcSPeFwWvb1G5JkiRJmlOZ5WxX3cEkORC4vKrePVK+J/DyqnrK/PRsdl765oPq\nuOXbznc3dAey7ODF890FSZIkza2MK3SkWJIkSZLUW15oq+eq6p8nlJ9EN3VZkiRJku6wHCmWJEmS\nJPWWoViSJEmS1FuGYkmSJElSbxmKJUmSJEm9ZSiWJEmSJPWWoViSJEmS1FuGYkmSJElSbxmKJUmS\nJEm9ZSiWJEmSJPWWoViSJEmS1FuGYkmSJElSbxmKJUmSJEm9tWC+OyDNhW02XZ/D9ls8392QJEmS\ntJpxpFiSJEmS1FuGYkmSJElSbxmKJUmSJEm9ZSiWJEmSJPWWoViSJEmS1FuGYkmSJElSbxmKJUmS\nJEm9ZSiWJEmSJPWWoViSJEmS1FsL5rsD0lxYetkVLNz/mPnuhlZjyw5ePN9dkCRJ0jxwpFiSJEmS\n1FuGYkmSJElSbxmKJUmSJEm9ZSiWJEmSJPWWoViSJEmS1FuGYkmSJElSbxmKJUmSJEm9ZSiWJEmS\nJPWWoViSJEmS1FuGYkmSJElSbxmKJUmSJEm9ZSiWJEmSJPWWoViSJEmS1FuGYkmSJElSbxmKJUmS\nJEm9ZSiWJEmSJPWWoXiOJLlvkq8l+VGSnyR5T5K7TLPOsiT3aK+/cyu2vXOSM5MsSXJJkgNWtq0Z\nbm9Rkve217snedQs139hkpuSPHSo7AdJ7jvXfZUkSZKkqRiK50CSAF8GvlpVWwAPAtYD/m2mbVTV\nrILliE8A+1bVdsDWwOdvRVtTSrKgqs6pqle2ot2Blen7pcCb5qxjkiRJkrQSDMVz4zHAdVX1MYCq\nWg68BtgnyX5Jvpzk+DaK/O/jGkhyVXvePcmpSb7YRk+PaqGbJDsm+XaSc5OckOTebfVNgF8Otl1V\n32/1101yRJKzkpyf5G9a+RpJ3pnkoiQXJnlFKx8euV6U5NT2+oAkn0pyBvCp1sdvJFkIvAR4TRul\n3jXJz5Lcua13t+H3I74G7JBk8zHH4nlJlrb+vX1Wn4QkSZIkzYKheG48FDh3uKCq/gj8P2ABsB3w\nTGAb4JlJ7jdNe9sDrwYeAjwQ2KUFy/cBe1XVjsAR3DwSfSjwwyRfSfLiJGu18jcDp1TVTsAewCFJ\n1gX2BRYC21XVtsBRM9jHhwB7VtWzh/ZxGfAh4NCq2q6qTgNOBRa3Ks8CvlxVN4xpbzlwCPDG4cI2\nhfrA1t/t274/aVyHkuyb5Jwk5yy/5ooZ7IIkSZIkrchQvGqcXFVXVNV1wPeBB0xT/6yqurSqbgKW\n0AXYB9NNjf5mkiXAPwP3BaiqfwUWAScCzwGOb+08Hti/1T8VWAu4P7An8OGqurGt//sZ7MPRVXXt\nDOp9FHhBe/0C4GNT1P0UsFuS+w+VPYIuyF/ewvSngd3GrVxVh1fVoqpatMY668+ga5IkSZK0ogXz\n3YE7iO8Dew0XJLkbXQC9Ebh+aNFypj/u4+oHuLiqHjluhar6CXBYko8Av02yUVvn6VX1w5G+Tdru\njdz8RclaI8uunqbPg36ckWRhkt2BNarqoinq3pDkUOANM2lbkiRJkuaaI8Vz42RgnSR/D905u8B/\nAB8HrpmjbfwQ2DjJI9s27jy4enOSxbk56W5BF6T/AJwAvGLonOTtW51vAi9OsqCVb9jKlwE7ttdP\nn2G/rgTuOlL2SboR3qlGiQf+E/g/wKAPZwJ7JNmo9e9ZwLdn2BdJkiRJmhVD8RyoqgKeCjwjyY+A\n/wauYw6vrlxVf6IbjX5HkgvoplUPrvr8d3TnFC+hm5L83Haxr/8L3Bm4MMnF7T10U5z/Xyu/gG7K\nNcDbgPckOYcuWM/E14GnDi601cqOAu4OfGZQKcnLkrxwzH5dD3wA2Li9vxR4C9107yXA96rqmBn2\nRZIkSZJmJV2ek+ZOkr2Av6mqv1tV23zpmw+q45Zvu6o2pzugZQcvnr6SJEmSVmdjzyP1nGLNqSTv\no5sO/cT57oskSZIkTcdQrDlVVa+Y7z5IkiRJ0kx5TrEkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6y1As\nSZIkSeotQ7EkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6y1AsSZIkSeotQ7EkSZIkqbcMxZIkSZKk3jIU\nS5IkSZJ6a8F8d0CaC9tsuj6H7bd4vrshSZIkaTXjSLEkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6y1As\nSZIkSeotQ7EkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6y1AsSZIkSeotQ7EkSZIkqbcWzHcHpLmw9LIr\nWLj/MfPdDc2RZQcvnu8uSJIkqSccKZYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYk\nSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYol\nSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb212oTiJFfNoM6rk6yzCvqyMMlz\nht6vk+SoJEuTXJTk9CTrzdG2jk2ywa1sY2GSa5MsGXrcJcmTk+zf6nw8yV5z0edZ9GuFz2su9lWS\nJEmSZmPBfHdgjr0aOBK4ZqYrJFmjqpbPcjsLgecAn27vXwX8uqq2aW0+GLhhLvpQVU+cZd8m+UlV\nbTdSdnR73CpJFlTVjSux6gqf1xzuqyRJkiTNyGozUjyQZPckpyb5YpIftBHaJHklcB/gW0m+1eo+\nPsl3k5yX5AuD0dsky5K8I8l5wDNae+9IclaS/06ya6u3RpJDkpyd5MIkL27dOBjYtY24vga4N3DZ\noI9V9cOqur618bzW7pIkH06yRiu/Ksl/JLkAeGOSL4zs4zeG+nqP9vrvWz8uSPKpVrZxki+1Pp6d\nZJdZHMu9k7x/qGjPJOe0Y/CkqY5B6+NpSY4Gvt/KXttGyi9K8upWtnDoc7qkfW7rTPi8ptxXSZIk\nSZprq10obranG2V8CPBAYJeqei/wC2CPqtqjhat/Bvasqh2Ac4DXDrXxu6raoao+294vqKqdWrtv\nbWX/AFxRVQ8HHg68KMlfAPsDp1XVdlV1KHAE8E8tgB+YZAuAJFsBz2z92w5YDjy3tb0ucGZVPYwu\nZD8iybpt2TOBQb9obT207c9j2jqvaoveAxza+vh04KMTjtlmQ1OnPzChzkJgJ2Ax8KEka01xDAB2\nAF5VVQ9KsiPwAuARwM6t3vat3oOBD1bVVsAfgf1GP68Z7isj9fZtIf6c5ddcMWGXJEmSJGmy1TUU\nn1VVl1bVTcASujA3ame60HxGkiXA84EHDC3/3Ej9L7fnc4faezzw9239M4GNgC1GN1RVS+jC+SHA\nhsDZLRA/FtixvV/S3j+wrbYc+FJb/0bgeOCvkyygC6VfG9nMY4AvVNXlbZ3ft/I9gfe39o8G7pbx\n5zP/pIX47arqZWOWA3y+qm6qqh8BPwW2nOYYnFVVP2uvHw18paqurqqr6I7nrm3Z/1TVGe31ka3u\nVCbt6wqq6vCqWlRVi9ZYZ/1pmpQkSZKkW1pdzym+fuj1csbvR4BvVtWzJ7Rx9YQ2h9sL8IqqOmGF\nhpPdRxsbCoJfTnIT8ETgT8AnquqNY7Z/3ch5xJ8FXg78Hjinqq6c0O9RdwJ2rqrrZlh/KjXm/VTH\nYPQYzqZdSZIkSZp3q+tI8SRXAndtr78H7JJkc4Ak6yZ50CzbOwF4aZI7tzYe1KY4D2+HJLskuXt7\nfRe6EeqfAycDeyXZpC3bMMkDGO/bdNORX8TI1OnmFLrznzcatNXKTwReMdSX0YtpzcYzktwpyWZ0\nI9o/ZPIxGHUa8JR2vvC6wFNbGcD9kzyyvX4OcHp7vcJxHDJpXyVJkiRpTt3RQvHhwPFJ/n97dx5l\nSVmnefz7SKGAINhg24pLOYiDyCoFKotCS6tYNmiLDe6gBxQXRNsZsXGh1RlKaWWGUVFERMcNUfFw\nREBlLbFZygKqAJe2tUah2wUbSllELH7zR7wp1yQz61ZyK5Os+H7OyVNx3xvxvr+4+Z6sevKNiLqw\nqn4NHAJ8Icky4F/oLgdeE6fQPURqaZJrgY/TrSIvA1a1h0C9GdgKuDjJcuAquvuXv1JV19PdG/vN\nVsO36B7KdS9t1fjrwH7tz/HvXwf8jzbONcCH2ltHAgvaQ6muB14LkGRBksnuL57Mz4ArgHOA17bV\n58k+g/H1LQVOa8dfDpxSVVe1t38IvD7J94GHAie19j99v4Y8V0mSJEkaqVR5JavWniTzga9X1XZr\nc5wjjjmuzlm1w9ocQjNoxaKFs12CJEmS1j2ZqHFdWymWJEmSJGloc/VBW5ojqmoFsFZXiSVJkiRp\nulwpliRJkiT1lqFYkiRJktRbhmJJkiRJUm8ZiiVJkiRJvWUoliRJkiT1lqFYkiRJktRbhmJJkiRJ\nUm8ZiiVJkiRJvWUoliRJkiT1lqFYkiRJktRbhmJJkiRJUm8ZiiVJkiRJvTVvtguQRmH7LTflpNct\nnO0yJEmSJM0xrhRLkiRJknrLUCxJkiRJ6i1DsSRJkiSptwzFkiRJkqTeMhRLkiRJknrLUCxJkiRJ\n6i1DsSRJkiSptwzFkiRJkqTeMhRLkiRJknpr3mwXII3C8htXMv/os2e7jF5bsWjhbJcgSZIkrTFX\niiVJkiRJvWUoliRJkiT1lqFYkiRJktRbhmJJkiRJUm8ZiiVJkiRJvWUoliRJkiT1lqFYkiRJktRb\nhmJJkiRJUm8ZiiVJkiRJvWUoliRJkiT1lqFYkiRJktRbhmJJkiRJUm8ZiiVJkiRJvWUoliRJkiT1\nlqFYkiRJktRbhmJJkiRJUm8ZikcsyV8l+WKSf0vyvSTfSPKEafRzVJKNRlTT/CQvmaB98yRXt69f\nJLlx4PUDRzG2JEmSJN2fGYpHKEmAM4GLqmqrqtoFeDvw8Gl0dxQwYShOst4a9jUfuFcorqrfVNVO\nVbUT8DHghLHXVfWHNS14bUoyb7ZrkCRJkrTuMRSP1j7AXVX1sbGGqroG+E6S45Ncm2R5koMAkuyd\n5KIkX07ygySfS+dI4JHAhUkubPvemuSDSa4BnpbkXUmubH2e3AI5SR6f5NtJrkmyNMlWwCJgr7YC\n/OZhTiTJ25O8rm3/nyTfbNvPSvLptv2ydj7XJvmfre3wJP880M8RSY5v269MckWr46NJHtDaT06y\nJMl1Sd41cOwNSRYluQp4wXS+IZIkSZI0FUPxaG0HfG+C9r8DdgJ2BPYFjk/yiPbeznSrwtsC/wXY\no6pOBP4d2Keq9mn7PRi4vKp2rKrvAB+uql2rajtgQ+B5bb/PAR+pqh2B3YH/AI4GFrcV4BOGPJfF\nwF5t+8nAZm2Fei/gkiSPAt5H94uAnYE9kjwP+CLwgoGV3UOBU5NsRxdsd28r0/OAg9s+R1fVgvb5\n/E2SbQfq+FVV7VxVZ4wvsAXwJUmWrLp95ZCnJUmSJEn3MBTPjD2BL1TVqqr6JXAxsGt774qquqGq\n7gauprvUeSKrgK8MvN4nyeVJlgN/DTwpySbAllV1JkBV/b6qbp9mzVcCuybZDLi1vX4yXSheDDwF\nuKCqbqqqu4DPA0+vqt8ClwD7JXkSsKqqvk/3y4BdgSVJrgaeAWzVxnpxkqXAUuCJdL8gGHP6ZAVW\n1clVtaCqFqy30abTPE1JkiRJfeZ9mqN1HXDgGh5z58D2Kib/nvy+qlYBJNkA+CiwoKp+nuRYYIM1\nHHdKVXVnkhuBVwCXAj8Cngk8tqp+lGT7KQ4/BXgLsAL4VGsLcGpVvXNwxyRbA28CdquqW5J8dty5\n3DaK85EkSZKkibhSPFoXAA9KcvhYQ5IdgFuAg5Ksl+RhwNOBK1bT1++ATSZ5byw03pRkY1oQr6rf\nATckeX4b+0HtCdZT9TWVxcBb6VZ+FwOvB5a09y6nW63evF0qfTDdCjhVdSndKvCLuGel99vA3yfZ\notW2eZLHAA9p9f22XVL+7GnUKUmSJEnTYigeoaoquvtm923/JdN1wHF0lxYvA66hC87/vap+sZru\nTgbOHXvQ1rhxbgE+AVwLnEd3afOYlwNHJlkGfBf4qzb2qvbwrTcneWSSbwxxSovpnpx9WVXdCNzV\n2qiqG4B3AhfRXfZ9WVWdPXDsl4FLqmpl23858E/At1tt32x9LwWuB34AfIZuVVqSJEmSZkS6HCeN\nVpJzgeOq6uKZGO+IY46rc1btMBNDaRIrFi2c7RIkSZKkqWSiRleKNVLtsuh/BW6eqUAsSZIkSdPl\ng7Y0UlX1G2Dr2a5DkiRJkobhSrEkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6y1AsSZIkSeotQ7EkSZIk\nqbcMxZIkSZKk3jIUS5IkSZJ6y1AsSZIkSeotQ7EkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6a95sFyCN\nwvZbbspJr1s422VIkiRJmmNcKZYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk\n9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9da82S5AGoXlN65k/tFnz3YZa9WKRQtn\nuwRJkiRpneNKsSRJkiSptwzFkiRJkqTeMhRLkiRJknrLUCxJkiRJ6i1DsSRJkiSptwzFkiRJkqTe\nMhRLkiRJknrLUCxJkiRJ6i1DsSRJkiSptwzFkiRJkqTeMhRLkiRJknrLUCxJkiRJ6i1DsSRJkiSp\ntwzFkiRJkqTeMhRLkiRJknrLUCxJkiRJ6q3VhuIkq5JcneTaJGck2WgmChtWkn8c97qSfHbg9bwk\nv07y9dX0s1OS5w68PjbJW+9DXRMen+SQVs/VA1/bJpmf5NppjnVIkkdOt9YJ+luQ5MRpHHefPrNR\n9yNJkiRJqzPMSvEdVbVTVW0H/AF47VquaU3947jXtwHbJdmwvf4b4MYh+tkJeO5q9xqN09tnOvZ1\n/X3s7xBgwlCcZL017ayqllTVkfexJkmSJEm631vTy6cXA48HSPK1JN9Lcl2Sw1vbq5L8r7GdkxyW\n5IS2CvqDJKcl+VGSzyXZN8mlSf41yW5t/wcnOTXJFUmuSnJAaz8kyVeTnNv2/0BrXwRs2FZbPzdQ\n5zeAhW37xcAXBmq61xhJHgi8Bzio9XVQ233bJBcl+UmSIwf6eEtbOb82yVED7ce08/sO8F/X8LP9\nkyTrJTk+yZVJliV5zcB7b0uyPMk1SRYlORBYAHyu1b5hkhVJ3p9kKfCitgp+WevrzCQPbX1d1Pa7\notW9V2vfe2xlPcnGST7VxlyW5IWt/TlJlrY6zh8of7LP7GVtnKuTfHwsrE/Rz9hxhyU5Z+CXHJIk\nSZI0MkOH4iTzgP2A5a3pVVW1C10gOzLJ5sCXgL9Nsn7b51Dg1Lb9eOCDwDbt6yXAnsBbuWe19xjg\ngqraDdgHOD7Jg9t7lZOROQAAD5JJREFUOwEHAdvThddHV9XR3LOS/dKBcr8IHJxkA2AH4PKB9+41\nBrA+8C7uWcE9ve27DfBsYDfg3UnWT7JLO6+nAE8FDkuyc2s/mHtWnHed4uMcC99jX+MD36uBlVW1\na+vnsCSPS7IfcADwlKraEfhAVX0ZWAK8tNV+R+vjN1X15Kr6IvAZ4G1VtQPd9+/dA2PNa5/FUePa\nx7yz1bJ9O/6CJA8DPgG8sNXxooH9J/rMnkj3vdujqnYCVgEvXU0/JHkD8Dzg+QPnNfj+4UmWJFmy\n6vaVE3/SkiRJkjSFeUPss2GSq9v2YuCTbfvIJC9o248Gtq6qy5JcADwvyfeB9atqeZL5wE+rajlA\nkuuA86uqkiwH5rd+ngXsn3vuJ90AeEzbPr+qVrbjrwceC/x8ooKralkb88V0q8aDphpjvLOr6k7g\nziS/Ah5OF+TPrKrbWi1fBfai+wXDmVV1e2s/a5I+oQvfbxhsSDK+xh3aKjDApsDWwL7Ap8bGqKr/\nnGqM1u+mwGZVdXFr/zRwxsB+X21/fo97vg+D9qUL+7Qxb07yt8AlVfXTCeqY6DN7JrALcGU7zw2B\nX9H9UmGyfl5B9/19flXdNdEJVtXJwMkARxxzXLFq0s9CkiRJkiY0TCi+o63u/UmSvenC0tOq6vYk\nF9GFS4BT6FZ+fwB8auCwOwe27x54ffdAHaFbNfzhuPGeMu74VUPUfhbwz8DewOaD3U0xxnhrOuao\nBHhjVZ33Z43Js9egj9uG3G/sHEd1fhN9ZgE+XVVvH9yxhevJLKdbdX8U8NMR1CVJkiRJ9zLd/5Jp\nU+DmFoi3oVvxA6CqLqdbOX4JA/fyDuk84I1py4lJdh7imLsGLtcedCrwT2Or00OM8TtgkyHGWww8\nP8lG7dLuF7S2S1r7hkk2AaYKfKtzHnDE2HkleUIb61vAoWlPAE/yF6urva2u3zx2vzDwcuDiifad\nxLeA14+9aPcjXwY8PcnjxtUxmfOBA5P85dj+SR67mn6uAl4DnJURPllbkiRJkgZNNxSfC8xrl0gv\nogs3g74EXFpVN69hv++lu793WbvE+r1DHHNy23/wQVtU1Q1VNdF/KzTZGBfSPSRq8EFb91JVS4HT\ngCvo7lU+paquau2nA9cA5wBXjh2T5LVJBp/aPf6e4t3HDXMKcD2wNN1/0/Rxunt/z6VbAV/SLmkf\nuwT8NOBjk9yfDPBKuvuzl9Gtvr5nsvObwPuAh6Z7qNg1wD5V9WvgcOCrre30qTpoT9d+B/DNVsO3\ngEesrp+q+k47x7OTbLEGNUuSJEnSUFJVo++0e3LxCVV1r6cJS2vDEcccV+es2mG2y1irVixauPqd\nJEmSJE0mEzVOd6V44hGSzZL8iO4+ZAOxJEmSJOl+baQPjqqqW4AnjLJPSZIkSZLWlpGuFEuSJEmS\nNJcYiiVJkiRJvWUoliRJkiT1lqFYkiRJktRbhmJJkiRJUm8ZiiVJkiRJvWUoliRJkiT1lqFYkiRJ\nktRbhmJJkiRJUm8ZiiVJkiRJvWUoliRJkiT1lqFYkiRJktRb82a7AGkUtt9yU0563cLZLkOSJEnS\nHONKsSRJkiSptwzFkiRJkqTeMhRLkiRJknrLUCxJkiRJ6i1DsSRJkiSptwzFkiRJkqTeMhRLkiRJ\nknrLUCxJkiRJ6i1DsSRJkiSpt+bNdgHSKCy/cSXzjz57tssY2opFC2e7BEmSJEm4UixJkiRJ6jFD\nsSRJkiSptwzFkiRJkqTeMhRLkiRJknrLUCxJkiRJ6i1DsSRJkiSptwzFkiRJkqTeMhRLkiRJknrL\nUCxJkiRJ6i1DsSRJkiSptwzFkiRJkqTeMhRLkiRJknrLUCxJkiRJ6i1DsSRJkiSptwzFkiRJkqTe\nMhSvoSSrklyd5NokZyTZaC2Pd0iSD4+wvxVJlrdzWJ7kgIH3vtv+nJ/k2ra9d5Kvt+39kxw9ojqO\nTXJ7kr8caLt1FH1LkiRJ0rAMxWvujqraqaq2A/4AvHZtDZRk3lrqep+q2gk4EDhxrLGqdp/qoKo6\nq6oWjbCOm4B/GGF/kiRJkrRGDMX3zWLg8QBJvpbke0muS3L42A5Jbk1yQms/P8nDWvtWSc5txyxO\nsk1rPy3Jx5JcDnxgcLAkD0vylSRXtq89Wvsz2srv1UmuSrJJkkckuWRgVXuvCep/CHDzYK1Tnezg\nqnWr88Qk303ykyQHtvYHJPlokh8k+VaSb4y9N4FTgYOS/MUEY72l1X1tkqOmqkuSJEmSpstQPE1t\nFXc/YHlrelVV7QIsAI5MsnlrfzCwpKqeBFwMvLu1nwy8sR3zVuCjA90/Cti9qt4ybtj/DZxQVbsC\nLwROae1vBV7fVn/3Au4AXgKc19p2BK4e6OfCdnn0xcA7pvsZAI8A9gSeB4ytIP8dMB/YFng58LQp\njr+VLhi/abAxyS7AocBTgKcChyXZefzBSQ5PsiTJklW3r7wPpyFJkiSprwzFa27DJFcDS4CfAZ9s\n7UcmuQa4DHg0sHVrvxs4vW1/FtgzycbA7sAZra+P0wXMMWdU1aoJxt4X+HA75izgIa2vS4EPJTkS\n2Kyq/ghcCRya5Fhg+6r63UA/+7TLv7dv/W08zc/ia1V1d1VdDzy8te3Z6r+7qn4BXLiaPk4EXplk\nk4G2PYEzq+q2qroV+Cpd2P8zVXVyVS2oqgXrbbTpNE9BkiRJUp+trXtW12V3tNXXP0myN11gfVpV\n3Z7kImCDSY4vul9G3DK+nwG3TdL+AOCpVfX7ce2LkpwNPBe4NMmzq+qSJE8HFgKnJflQVX3mzwqp\n+rckv6Rb1b1ikjGncufAdqZxPFV1S5LPA6+fzvGSJEmSdF+4UjwamwI3t0C8Dd0lv2MeQPdAK+gu\naf5OVf0W+GmSFwGks+MQ43wTeOPYiyQ7tT+3qqrlVfV+uhXibZI8FvhlVX2C7jLrJ4/vrD35+XHA\n/1uz053SpcAL273FDwf2HuKYDwGv4Z5f0iwGnp9koyQPBl7Q2iRJkiRppAzFo3EuMC/J9+nurb1s\n4L3bgN3aPbx/Dbyntb8UeHW75Po64ABW70hgQZJlSa7nnidfH9UeSLUMuAs4hy6MXpPkKuAguvuR\nx1zYLsG+EDi6qn65xmc8ua8ANwDX010uvhRYCZDkPUn2H39AVd0EnAk8qL1eCpxGt3p9OXBKVV01\nwholSZIkCYBU1WzXsE5LcmtVTfee3TkpycZVdWt72NgVwB7t/uK15ohjjqtzVu2wNocYqRWLFs52\nCZIkSVLfTHjLp/cUa234epLNgAcC713bgViSJEmSpstQvJb1bZUYoKr2nu0aJEmSJGkY3lMsSZIk\nSeotQ7EkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6y1AsSZIkSeotQ7EkSZIkqbcMxZIkSZKk3jIUS5Ik\nSZJ6y1AsSZIkSeotQ7EkSZIkqbcMxZIkSZKk3jIUS5IkSZJ6a95sFyCNwvZbbspJr1s422VIkiRJ\nmmNcKZYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIk\nSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIk\nSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYk\nSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYolSZIkSb1lKJYkSZIk9ZahWJIkSZLUW4ZiSZIkSVJvGYol\nSZIkSb1lKJYkSZIk9VaqarZrkO6zt73tbb9bf/31fzjbdWjdcOutt26x8cYb3zTbdWjd4ZzSqDmn\nNGrOKY3a/XRO3fS+973vOeMbDcVaJyRZUlULZrsOrRucTxo155RGzTmlUXNOadTm0pzy8mlJkiRJ\nUm8ZiiVJkiRJvWUo1rri5NkuQOsU55NGzTmlUXNOadScUxq1OTOnvKdYkiRJktRbrhRLkiRJknrL\nUCxJkiRJ6i1DseaMJM9J8sMkP05y9ATvPyjJ6e39y5PMn/kqNZcMMafekuT6JMuSnJ/ksbNRp+aO\n1c2pgf1emKSSzIn/qkKzZ5g5leTv28+q65J8fqZr1NwyxN99j0lyYZKr2t9/z52NOjU3JDk1ya+S\nXDvJ+0lyYptvy5I8eaZrHIahWHNCkvWAjwD7AdsCL06y7bjdXg3cXFWPB04A3j+zVWouGXJOXQUs\nqKodgC8DH5jZKjWXDDmnSLIJ8Cbg8pmtUHPNMHMqydbA24E9qupJwFEzXqjmjCF/Tr0D+FJV7Qwc\nDHx0ZqvUHHMa8Jwp3t8P2Lp9HQ6cNAM1rTFDseaK3YAfV9VPquoPwBeBA8btcwDw6bb9ZeCZSTKD\nNWpuWe2cqqoLq+r29vIy4FEzXKPmlmF+TgG8l+6Xdr+fyeI0Jw0zpw4DPlJVNwNU1a9muEbNLcPM\nqQIe0rY3Bf59BuvTHFNVlwD/OcUuBwCfqc5lwGZJHjEz1Q3PUKy5Ykvg5wOvb2htE+5TVX8EVgKb\nz0h1mouGmVODXg2cs1Yr0ly32jnVLht7dFWdPZOFac4a5ufUE4AnJLk0yWVJplqxkYaZU8cCL0ty\nA/AN4I0zU5rWUWv6761ZMW+2C5Ck+7skLwMWAM+Y7Vo0dyV5APAh4JBZLkXrlnl0lyXuTXc1yyVJ\ntq+qW2a1Ks1lLwZOq6oPJnka8H+TbFdVd892YdLa4kqx5oobgUcPvH5Ua5twnyTz6C75+c2MVKe5\naJg5RZJ9gWOA/avqzhmqTXPT6ubUJsB2wEVJVgBPBc7yYVuawjA/p24Azqqqu6rqp8CP6EKyNJFh\n5tSrgS8BVNW/ABsAW8xIdVoXDfXvrdlmKNZccSWwdZLHJXkg3YMfzhq3z1nAK9v2gcAFVVUzWKPm\nltXOqSQ7Ax+nC8Tep6fVmXJOVdXKqtqiquZX1Xy6+9T3r6ols1Ou5oBh/u77Gt0qMUm2oLuc+icz\nWaTmlGHm1M+AZwIkeSJdKP71jFapdclZwCvaU6ifCqysqv+Y7aLG8/JpzQlV9cckbwDOA9YDTq2q\n65K8B1hSVWcBn6S7xOfHdDf8Hzx7Fev+bsg5dTywMXBGe2bbz6pq/1krWvdrQ84paWhDzqnzgGcl\nuR5YBfy3qvIqKU1oyDn1D8AnkryZ7qFbh7jIoMkk+QLdL+a2aPehvxtYH6CqPkZ3X/pzgR8DtwOH\nzk6lU4tzXJIkSZLUV14+LUmSJEnqLUOxJEmSJKm3DMWSJEmSpN4yFEuSJEmSestQLEmSJEnqLUOx\nJEmSJKm3DMWSJEmSpN76/+azKS7kY7KhAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 1008x720 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mzfOwj0vwmW4",
        "colab_type": "code",
        "outputId": "9c60e683-570b-4cf4-febe-c9f3457cdb4a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "aml.leader.download_mojo(path = \"./\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/XGBoost_2_AutoML_20200106_011717.zip'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    }
  ]
}