TCGA-KIRC 3 Filter to marker genes, run ComBat, prep Stan dicts.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "**Continuation of TCGA-KIRC 1 and 2 notebooks. Based on `prepare bladder data for infino v2.ipynb`**\n",
    "\n",
    "In `rcc make standata.ipynb`, we describe the procedure for creating a new list of marker gene transcripts (which also involved `R data prep.ipynb`). That list was saved as: `infino-rcc/bindea_and_our_own_marker_genes_lists_combined.pkl`\n",
    "\n",
    "**Strategy:**\n",
    "\n",
    "1. Extract counts and TPM from kallisto output --> done in `TCGA-KIRC 1` and `2`\n",
    "1. Filter single origin sample **TPM** and bladder **TPM** to marker genes\n",
    "2. Run ComBat and create training and testing datasets as in `rcc make standata.ipynb` (except we've already dropped problematic rows)\n",
    "3. Run PCA on bladder and single origin together post-ComBat with different colors to make sure ComBat did its job.\n",
    "4. Launch\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Filter to marker genes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:03:37.846863Z",
     "start_time": "2017-11-04T04:03:35.320564Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:stancache.seed:Setting seed to 1245502385\n",
      "INFO:root:Setting CACHE_DIR = /home/jovyan/modelcache/mz\n",
      "INFO:stancache.seed:Setting seed to 1245502385\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/jovyan/work/model-single-origin-samples\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib as mpl\n",
    "\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "\n",
    "import data\n",
    "import models\n",
    "import cache\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import pystan\n",
    "from time import time\n",
    "from datetime import timedelta\n",
    "import pickle\n",
    "import dill\n",
    "\n",
    "import os\n",
    "if 'infino-rcc' in os.getcwd():\n",
    "    os.chdir('..')\n",
    "    print(os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:03:40.593601Z",
     "start_time": "2017-11-04T04:03:37.850320Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sample_df = pickle.load(open('/modelcache/mz/prep_sample_df.cached.sample_n_82429761595.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:03:40.673740Z",
     "start_time": "2017-11-04T04:03:40.598968Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>sample_id</th>\n",
       "      <th>filename</th>\n",
       "      <th>gene_name</th>\n",
       "      <th>est_counts</th>\n",
       "      <th>tpm</th>\n",
       "      <th>log1p_tpm</th>\n",
       "      <th>log1p_counts</th>\n",
       "      <th>CCR6+</th>\n",
       "      <th>CCR7+</th>\n",
       "      <th>...</th>\n",
       "      <th>log1p_tpm_rescaled_subset</th>\n",
       "      <th>log1p_tpm_rescaled</th>\n",
       "      <th>gene_cat</th>\n",
       "      <th>gene_id</th>\n",
       "      <th>B_cell</th>\n",
       "      <th>T_cell</th>\n",
       "      <th>new_gene_cat</th>\n",
       "      <th>new_gene_id</th>\n",
       "      <th>new_sample_cat</th>\n",
       "      <th>new_sample_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ERR431566</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>56.74329</td>\n",
       "      <td>6.931783</td>\n",
       "      <td>2.070878</td>\n",
       "      <td>4.056007</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-3.517738</td>\n",
       "      <td>-5.478180</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>ERR431567</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>71.32171</td>\n",
       "      <td>9.090650</td>\n",
       "      <td>2.311609</td>\n",
       "      <td>4.281124</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-5.052464</td>\n",
       "      <td>-3.287291</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>ERR431568</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>113.22460</td>\n",
       "      <td>8.845730</td>\n",
       "      <td>2.287038</td>\n",
       "      <td>4.738167</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-3.303694</td>\n",
       "      <td>-3.510914</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>ERR431569</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>131.51760</td>\n",
       "      <td>10.714400</td>\n",
       "      <td>2.460819</td>\n",
       "      <td>4.886715</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-4.017633</td>\n",
       "      <td>-1.929339</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>ERR431570</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>57.29375</td>\n",
       "      <td>6.973180</td>\n",
       "      <td>2.076083</td>\n",
       "      <td>4.065495</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-4.708581</td>\n",
       "      <td>-5.430805</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  sample_id   filename gene_name  est_counts        tpm  log1p_tpm  \\\n",
       "0      0          1  ERR431566      A1BG    56.74329   6.931783   2.070878   \n",
       "1      1          2  ERR431567      A1BG    71.32171   9.090650   2.311609   \n",
       "2      2          3  ERR431568      A1BG   113.22460   8.845730   2.287038   \n",
       "3      3          4  ERR431569      A1BG   131.51760  10.714400   2.460819   \n",
       "4      4          5  ERR431570      A1BG    57.29375   6.973180   2.076083   \n",
       "\n",
       "   log1p_counts  CCR6+  CCR7+      ...        log1p_tpm_rescaled_subset  \\\n",
       "0      4.056007    0.0    0.0      ...                        -3.517738   \n",
       "1      4.281124    0.0    0.0      ...                        -5.052464   \n",
       "2      4.738167    0.0    0.0      ...                        -3.303694   \n",
       "3      4.886715    0.0    0.0      ...                        -4.017633   \n",
       "4      4.065495    0.0    0.0      ...                        -4.708581   \n",
       "\n",
       "   log1p_tpm_rescaled  gene_cat  gene_id  B_cell  T_cell  new_gene_cat  \\\n",
       "0           -5.478180      A1BG        1       0       1          A1BG   \n",
       "1           -3.287291      A1BG        1       0       1          A1BG   \n",
       "2           -3.510914      A1BG        1       1       0          A1BG   \n",
       "3           -1.929339      A1BG        1       1       0          A1BG   \n",
       "4           -5.430805      A1BG        1       0       1          A1BG   \n",
       "\n",
       "   new_gene_id  new_sample_cat  new_sample_id  \n",
       "0            1               1              1  \n",
       "1            1               2              2  \n",
       "2            1               3              3  \n",
       "3            1               4              4  \n",
       "4            1               5              5  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:03:41.941062Z",
     "start_time": "2017-11-04T04:03:41.930197Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "817"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_marker_genes = pickle.load(open('infino-rcc/bindea_and_our_own_marker_genes_lists_combined.pkl', 'rb'))\n",
    "len(combined_marker_genes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:05:25.665347Z",
     "start_time": "2017-11-04T04:05:24.935059Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(49878, 80)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gene_symbols</th>\n",
       "      <th>X953473f4.9927.4fd4.bfee.8f5908638cc7</th>\n",
       "      <th>df66c814.1407.4899.a6e3.9c2a928e6ea0</th>\n",
       "      <th>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</th>\n",
       "      <th>X87c1d789.9482.442a.8b42.97bb479060e3</th>\n",
       "      <th>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</th>\n",
       "      <th>X383ba5ca.1706.4959.b543.ea87fbd367bd</th>\n",
       "      <th>d9f44b98.58b7.4ebd.a6a6.8de472bf95a8</th>\n",
       "      <th>X550af27b.de50.402f.8a81.c508e3dc6781</th>\n",
       "      <th>c81f9d5b.f58d.411d.9e1a.f919a0ef5636</th>\n",
       "      <th>...</th>\n",
       "      <th>X14b8d86c.2fef.4c50.8633.a84e12d81b51</th>\n",
       "      <th>X0cc91a13.32d9.4262.a33d.264096b4102c</th>\n",
       "      <th>X9bae55b1.5a27.4a9f.b376.ad1ab4fc4ee2</th>\n",
       "      <th>X6b124b2e.dce9.48ef.bedb.7a2da9cd2ff8</th>\n",
       "      <th>X3b4be095.1d76.49de.8357.f932ba5d55bb</th>\n",
       "      <th>e644640b.78d3.460a.83fa.ea475620a3f0</th>\n",
       "      <th>X01f8736a.85bc.498d.91c6.1c053c479512</th>\n",
       "      <th>e40e21de.f116.4eb1.87af.3b16797ec3b2</th>\n",
       "      <th>X19de7074.f195.47f6.81fa.6d3bf01c1685</th>\n",
       "      <th>X0153168a.39ce.47d6.b7f5.ac1e0f312245</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5S_rRNA</td>\n",
       "      <td>9.204280</td>\n",
       "      <td>11.249630</td>\n",
       "      <td>10.73302</td>\n",
       "      <td>8.732760</td>\n",
       "      <td>1.49641</td>\n",
       "      <td>6.319660</td>\n",
       "      <td>4.41094</td>\n",
       "      <td>9.926649</td>\n",
       "      <td>4.398960</td>\n",
       "      <td>...</td>\n",
       "      <td>5.938090</td>\n",
       "      <td>9.418420</td>\n",
       "      <td>6.698460</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>8.251770</td>\n",
       "      <td>2.50000</td>\n",
       "      <td>3.544530</td>\n",
       "      <td>4.645700</td>\n",
       "      <td>2.095861</td>\n",
       "      <td>8.884160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7SK</td>\n",
       "      <td>1.435663</td>\n",
       "      <td>0.666666</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.666666</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.333334</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.726206</td>\n",
       "      <td>0.666666</td>\n",
       "      <td>...</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>1.333334</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.666666</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.89499</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.333334</td>\n",
       "      <td>3.123140</td>\n",
       "      <td>0.658972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A1BG</td>\n",
       "      <td>128.138700</td>\n",
       "      <td>205.199900</td>\n",
       "      <td>215.17890</td>\n",
       "      <td>140.744100</td>\n",
       "      <td>7.04211</td>\n",
       "      <td>157.791700</td>\n",
       "      <td>104.51000</td>\n",
       "      <td>107.278100</td>\n",
       "      <td>64.783480</td>\n",
       "      <td>...</td>\n",
       "      <td>630.209400</td>\n",
       "      <td>292.625300</td>\n",
       "      <td>183.565400</td>\n",
       "      <td>97.333850</td>\n",
       "      <td>96.643000</td>\n",
       "      <td>133.19810</td>\n",
       "      <td>95.820200</td>\n",
       "      <td>88.380000</td>\n",
       "      <td>86.603100</td>\n",
       "      <td>218.628800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A1BG-AS1</td>\n",
       "      <td>137.141800</td>\n",
       "      <td>207.946790</td>\n",
       "      <td>187.30330</td>\n",
       "      <td>202.679170</td>\n",
       "      <td>2.32233</td>\n",
       "      <td>183.595830</td>\n",
       "      <td>41.12993</td>\n",
       "      <td>143.327220</td>\n",
       "      <td>76.198220</td>\n",
       "      <td>...</td>\n",
       "      <td>826.231000</td>\n",
       "      <td>249.736070</td>\n",
       "      <td>126.324510</td>\n",
       "      <td>53.022270</td>\n",
       "      <td>78.511700</td>\n",
       "      <td>137.71936</td>\n",
       "      <td>70.129900</td>\n",
       "      <td>103.813740</td>\n",
       "      <td>81.016800</td>\n",
       "      <td>306.343700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A1CF</td>\n",
       "      <td>6100.216000</td>\n",
       "      <td>12536.641166</td>\n",
       "      <td>1201.95240</td>\n",
       "      <td>2628.853730</td>\n",
       "      <td>1221.00020</td>\n",
       "      <td>2039.842842</td>\n",
       "      <td>2279.93500</td>\n",
       "      <td>4234.261301</td>\n",
       "      <td>4380.631751</td>\n",
       "      <td>...</td>\n",
       "      <td>21.391444</td>\n",
       "      <td>2514.811700</td>\n",
       "      <td>2348.325003</td>\n",
       "      <td>293.498800</td>\n",
       "      <td>2560.211761</td>\n",
       "      <td>4128.04890</td>\n",
       "      <td>3055.639354</td>\n",
       "      <td>798.055720</td>\n",
       "      <td>630.081400</td>\n",
       "      <td>454.589871</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 80 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Gene_symbols  X953473f4.9927.4fd4.bfee.8f5908638cc7  \\\n",
       "0      5S_rRNA                               9.204280   \n",
       "1          7SK                               1.435663   \n",
       "2         A1BG                             128.138700   \n",
       "3     A1BG-AS1                             137.141800   \n",
       "4         A1CF                            6100.216000   \n",
       "\n",
       "   df66c814.1407.4899.a6e3.9c2a928e6ea0  \\\n",
       "0                             11.249630   \n",
       "1                              0.666666   \n",
       "2                            205.199900   \n",
       "3                            207.946790   \n",
       "4                          12536.641166   \n",
       "\n",
       "   X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d  \\\n",
       "0                               10.73302   \n",
       "1                                0.00000   \n",
       "2                              215.17890   \n",
       "3                              187.30330   \n",
       "4                             1201.95240   \n",
       "\n",
       "   X87c1d789.9482.442a.8b42.97bb479060e3  \\\n",
       "0                               8.732760   \n",
       "1                               0.666666   \n",
       "2                             140.744100   \n",
       "3                             202.679170   \n",
       "4                            2628.853730   \n",
       "\n",
       "   X4ddfd589.5a62.4149.9ccc.6a539f0b37b7  \\\n",
       "0                                1.49641   \n",
       "1                                0.00000   \n",
       "2                                7.04211   \n",
       "3                                2.32233   \n",
       "4                             1221.00020   \n",
       "\n",
       "   X383ba5ca.1706.4959.b543.ea87fbd367bd  \\\n",
       "0                               6.319660   \n",
       "1                               1.333334   \n",
       "2                             157.791700   \n",
       "3                             183.595830   \n",
       "4                            2039.842842   \n",
       "\n",
       "   d9f44b98.58b7.4ebd.a6a6.8de472bf95a8  \\\n",
       "0                               4.41094   \n",
       "1                               0.00000   \n",
       "2                             104.51000   \n",
       "3                              41.12993   \n",
       "4                            2279.93500   \n",
       "\n",
       "   X550af27b.de50.402f.8a81.c508e3dc6781  \\\n",
       "0                               9.926649   \n",
       "1                               1.726206   \n",
       "2                             107.278100   \n",
       "3                             143.327220   \n",
       "4                            4234.261301   \n",
       "\n",
       "   c81f9d5b.f58d.411d.9e1a.f919a0ef5636  \\\n",
       "0                              4.398960   \n",
       "1                              0.666666   \n",
       "2                             64.783480   \n",
       "3                             76.198220   \n",
       "4                           4380.631751   \n",
       "\n",
       "                   ...                    \\\n",
       "0                  ...                     \n",
       "1                  ...                     \n",
       "2                  ...                     \n",
       "3                  ...                     \n",
       "4                  ...                     \n",
       "\n",
       "   X14b8d86c.2fef.4c50.8633.a84e12d81b51  \\\n",
       "0                               5.938090   \n",
       "1                               0.750000   \n",
       "2                             630.209400   \n",
       "3                             826.231000   \n",
       "4                              21.391444   \n",
       "\n",
       "   X0cc91a13.32d9.4262.a33d.264096b4102c  \\\n",
       "0                               9.418420   \n",
       "1                               1.333334   \n",
       "2                             292.625300   \n",
       "3                             249.736070   \n",
       "4                            2514.811700   \n",
       "\n",
       "   X9bae55b1.5a27.4a9f.b376.ad1ab4fc4ee2  \\\n",
       "0                               6.698460   \n",
       "1                               0.000000   \n",
       "2                             183.565400   \n",
       "3                             126.324510   \n",
       "4                            2348.325003   \n",
       "\n",
       "   X6b124b2e.dce9.48ef.bedb.7a2da9cd2ff8  \\\n",
       "0                               3.000000   \n",
       "1                               0.666666   \n",
       "2                              97.333850   \n",
       "3                              53.022270   \n",
       "4                             293.498800   \n",
       "\n",
       "   X3b4be095.1d76.49de.8357.f932ba5d55bb  \\\n",
       "0                               8.251770   \n",
       "1                               2.000000   \n",
       "2                              96.643000   \n",
       "3                              78.511700   \n",
       "4                            2560.211761   \n",
       "\n",
       "   e644640b.78d3.460a.83fa.ea475620a3f0  \\\n",
       "0                               2.50000   \n",
       "1                               1.89499   \n",
       "2                             133.19810   \n",
       "3                             137.71936   \n",
       "4                            4128.04890   \n",
       "\n",
       "   X01f8736a.85bc.498d.91c6.1c053c479512  \\\n",
       "0                               3.544530   \n",
       "1                               0.000000   \n",
       "2                              95.820200   \n",
       "3                              70.129900   \n",
       "4                            3055.639354   \n",
       "\n",
       "   e40e21de.f116.4eb1.87af.3b16797ec3b2  \\\n",
       "0                              4.645700   \n",
       "1                              1.333334   \n",
       "2                             88.380000   \n",
       "3                            103.813740   \n",
       "4                            798.055720   \n",
       "\n",
       "   X19de7074.f195.47f6.81fa.6d3bf01c1685  \\\n",
       "0                               2.095861   \n",
       "1                               3.123140   \n",
       "2                              86.603100   \n",
       "3                              81.016800   \n",
       "4                             630.081400   \n",
       "\n",
       "   X0153168a.39ce.47d6.b7f5.ac1e0f312245  \n",
       "0                               8.884160  \n",
       "1                               0.658972  \n",
       "2                             218.628800  \n",
       "3                             306.343700  \n",
       "4                             454.589871  \n",
       "\n",
       "[5 rows x 80 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_counts = pd.read_csv('infino-rcc/tcga_kirc_data/tcga_kirc.2.subset.counts.filtered.tsv', sep='\\t')\n",
    "print(test_df_counts.shape)\n",
    "test_df_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:05:28.666263Z",
     "start_time": "2017-11-04T04:05:27.833000Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(49878, 80)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gene_symbols</th>\n",
       "      <th>X953473f4.9927.4fd4.bfee.8f5908638cc7</th>\n",
       "      <th>df66c814.1407.4899.a6e3.9c2a928e6ea0</th>\n",
       "      <th>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</th>\n",
       "      <th>X87c1d789.9482.442a.8b42.97bb479060e3</th>\n",
       "      <th>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</th>\n",
       "      <th>X383ba5ca.1706.4959.b543.ea87fbd367bd</th>\n",
       "      <th>d9f44b98.58b7.4ebd.a6a6.8de472bf95a8</th>\n",
       "      <th>X550af27b.de50.402f.8a81.c508e3dc6781</th>\n",
       "      <th>c81f9d5b.f58d.411d.9e1a.f919a0ef5636</th>\n",
       "      <th>...</th>\n",
       "      <th>X14b8d86c.2fef.4c50.8633.a84e12d81b51</th>\n",
       "      <th>X0cc91a13.32d9.4262.a33d.264096b4102c</th>\n",
       "      <th>X9bae55b1.5a27.4a9f.b376.ad1ab4fc4ee2</th>\n",
       "      <th>X6b124b2e.dce9.48ef.bedb.7a2da9cd2ff8</th>\n",
       "      <th>X3b4be095.1d76.49de.8357.f932ba5d55bb</th>\n",
       "      <th>e644640b.78d3.460a.83fa.ea475620a3f0</th>\n",
       "      <th>X01f8736a.85bc.498d.91c6.1c053c479512</th>\n",
       "      <th>e40e21de.f116.4eb1.87af.3b16797ec3b2</th>\n",
       "      <th>X19de7074.f195.47f6.81fa.6d3bf01c1685</th>\n",
       "      <th>X0153168a.39ce.47d6.b7f5.ac1e0f312245</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5S_rRNA</td>\n",
       "      <td>9.914770</td>\n",
       "      <td>15.130672</td>\n",
       "      <td>25.363480</td>\n",
       "      <td>13.224230</td>\n",
       "      <td>3.869380</td>\n",
       "      <td>11.317250</td>\n",
       "      <td>4.362510</td>\n",
       "      <td>13.665296</td>\n",
       "      <td>4.233722</td>\n",
       "      <td>...</td>\n",
       "      <td>6.857678</td>\n",
       "      <td>13.437248</td>\n",
       "      <td>9.344670</td>\n",
       "      <td>5.197250</td>\n",
       "      <td>13.068880</td>\n",
       "      <td>4.319850</td>\n",
       "      <td>6.512180</td>\n",
       "      <td>9.392700</td>\n",
       "      <td>2.772839</td>\n",
       "      <td>17.498520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7SK</td>\n",
       "      <td>0.411977</td>\n",
       "      <td>0.117796</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.150124</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.344334</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.394435</td>\n",
       "      <td>0.237672</td>\n",
       "      <td>...</td>\n",
       "      <td>0.157959</td>\n",
       "      <td>0.280088</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.171198</td>\n",
       "      <td>0.489391</td>\n",
       "      <td>0.523055</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.431138</td>\n",
       "      <td>1.700330</td>\n",
       "      <td>0.199897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A1BG</td>\n",
       "      <td>6.611734</td>\n",
       "      <td>7.657621</td>\n",
       "      <td>18.736413</td>\n",
       "      <td>5.590750</td>\n",
       "      <td>0.309164</td>\n",
       "      <td>8.832981</td>\n",
       "      <td>3.619899</td>\n",
       "      <td>4.672390</td>\n",
       "      <td>3.752960</td>\n",
       "      <td>...</td>\n",
       "      <td>30.057532</td>\n",
       "      <td>14.985862</td>\n",
       "      <td>7.511637</td>\n",
       "      <td>4.539710</td>\n",
       "      <td>4.106327</td>\n",
       "      <td>5.725944</td>\n",
       "      <td>4.635009</td>\n",
       "      <td>5.133269</td>\n",
       "      <td>7.756796</td>\n",
       "      <td>13.081544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A1BG-AS1</td>\n",
       "      <td>1.769861</td>\n",
       "      <td>2.154506</td>\n",
       "      <td>4.555689</td>\n",
       "      <td>2.742114</td>\n",
       "      <td>0.044404</td>\n",
       "      <td>2.860883</td>\n",
       "      <td>0.557519</td>\n",
       "      <td>1.780005</td>\n",
       "      <td>1.579688</td>\n",
       "      <td>...</td>\n",
       "      <td>9.587063</td>\n",
       "      <td>3.340660</td>\n",
       "      <td>2.063594</td>\n",
       "      <td>0.769476</td>\n",
       "      <td>1.055890</td>\n",
       "      <td>2.040633</td>\n",
       "      <td>1.381748</td>\n",
       "      <td>1.964373</td>\n",
       "      <td>1.997219</td>\n",
       "      <td>6.476828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A1CF</td>\n",
       "      <td>16.914616</td>\n",
       "      <td>27.108711</td>\n",
       "      <td>5.592854</td>\n",
       "      <td>7.241174</td>\n",
       "      <td>5.153873</td>\n",
       "      <td>6.403223</td>\n",
       "      <td>5.224765</td>\n",
       "      <td>11.907584</td>\n",
       "      <td>13.552498</td>\n",
       "      <td>...</td>\n",
       "      <td>0.053868</td>\n",
       "      <td>7.079731</td>\n",
       "      <td>6.363790</td>\n",
       "      <td>1.224351</td>\n",
       "      <td>6.665892</td>\n",
       "      <td>11.964710</td>\n",
       "      <td>10.849198</td>\n",
       "      <td>2.790016</td>\n",
       "      <td>3.062978</td>\n",
       "      <td>1.571386</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 80 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Gene_symbols  X953473f4.9927.4fd4.bfee.8f5908638cc7  \\\n",
       "0      5S_rRNA                               9.914770   \n",
       "1          7SK                               0.411977   \n",
       "2         A1BG                               6.611734   \n",
       "3     A1BG-AS1                               1.769861   \n",
       "4         A1CF                              16.914616   \n",
       "\n",
       "   df66c814.1407.4899.a6e3.9c2a928e6ea0  \\\n",
       "0                             15.130672   \n",
       "1                              0.117796   \n",
       "2                              7.657621   \n",
       "3                              2.154506   \n",
       "4                             27.108711   \n",
       "\n",
       "   X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d  \\\n",
       "0                              25.363480   \n",
       "1                               0.000000   \n",
       "2                              18.736413   \n",
       "3                               4.555689   \n",
       "4                               5.592854   \n",
       "\n",
       "   X87c1d789.9482.442a.8b42.97bb479060e3  \\\n",
       "0                              13.224230   \n",
       "1                               0.150124   \n",
       "2                               5.590750   \n",
       "3                               2.742114   \n",
       "4                               7.241174   \n",
       "\n",
       "   X4ddfd589.5a62.4149.9ccc.6a539f0b37b7  \\\n",
       "0                               3.869380   \n",
       "1                               0.000000   \n",
       "2                               0.309164   \n",
       "3                               0.044404   \n",
       "4                               5.153873   \n",
       "\n",
       "   X383ba5ca.1706.4959.b543.ea87fbd367bd  \\\n",
       "0                              11.317250   \n",
       "1                               0.344334   \n",
       "2                               8.832981   \n",
       "3                               2.860883   \n",
       "4                               6.403223   \n",
       "\n",
       "   d9f44b98.58b7.4ebd.a6a6.8de472bf95a8  \\\n",
       "0                              4.362510   \n",
       "1                              0.000000   \n",
       "2                              3.619899   \n",
       "3                              0.557519   \n",
       "4                              5.224765   \n",
       "\n",
       "   X550af27b.de50.402f.8a81.c508e3dc6781  \\\n",
       "0                              13.665296   \n",
       "1                               0.394435   \n",
       "2                               4.672390   \n",
       "3                               1.780005   \n",
       "4                              11.907584   \n",
       "\n",
       "   c81f9d5b.f58d.411d.9e1a.f919a0ef5636  \\\n",
       "0                              4.233722   \n",
       "1                              0.237672   \n",
       "2                              3.752960   \n",
       "3                              1.579688   \n",
       "4                             13.552498   \n",
       "\n",
       "                   ...                    \\\n",
       "0                  ...                     \n",
       "1                  ...                     \n",
       "2                  ...                     \n",
       "3                  ...                     \n",
       "4                  ...                     \n",
       "\n",
       "   X14b8d86c.2fef.4c50.8633.a84e12d81b51  \\\n",
       "0                               6.857678   \n",
       "1                               0.157959   \n",
       "2                              30.057532   \n",
       "3                               9.587063   \n",
       "4                               0.053868   \n",
       "\n",
       "   X0cc91a13.32d9.4262.a33d.264096b4102c  \\\n",
       "0                              13.437248   \n",
       "1                               0.280088   \n",
       "2                              14.985862   \n",
       "3                               3.340660   \n",
       "4                               7.079731   \n",
       "\n",
       "   X9bae55b1.5a27.4a9f.b376.ad1ab4fc4ee2  \\\n",
       "0                               9.344670   \n",
       "1                               0.000000   \n",
       "2                               7.511637   \n",
       "3                               2.063594   \n",
       "4                               6.363790   \n",
       "\n",
       "   X6b124b2e.dce9.48ef.bedb.7a2da9cd2ff8  \\\n",
       "0                               5.197250   \n",
       "1                               0.171198   \n",
       "2                               4.539710   \n",
       "3                               0.769476   \n",
       "4                               1.224351   \n",
       "\n",
       "   X3b4be095.1d76.49de.8357.f932ba5d55bb  \\\n",
       "0                              13.068880   \n",
       "1                               0.489391   \n",
       "2                               4.106327   \n",
       "3                               1.055890   \n",
       "4                               6.665892   \n",
       "\n",
       "   e644640b.78d3.460a.83fa.ea475620a3f0  \\\n",
       "0                              4.319850   \n",
       "1                              0.523055   \n",
       "2                              5.725944   \n",
       "3                              2.040633   \n",
       "4                             11.964710   \n",
       "\n",
       "   X01f8736a.85bc.498d.91c6.1c053c479512  \\\n",
       "0                               6.512180   \n",
       "1                               0.000000   \n",
       "2                               4.635009   \n",
       "3                               1.381748   \n",
       "4                              10.849198   \n",
       "\n",
       "   e40e21de.f116.4eb1.87af.3b16797ec3b2  \\\n",
       "0                              9.392700   \n",
       "1                              0.431138   \n",
       "2                              5.133269   \n",
       "3                              1.964373   \n",
       "4                              2.790016   \n",
       "\n",
       "   X19de7074.f195.47f6.81fa.6d3bf01c1685  \\\n",
       "0                               2.772839   \n",
       "1                               1.700330   \n",
       "2                               7.756796   \n",
       "3                               1.997219   \n",
       "4                               3.062978   \n",
       "\n",
       "   X0153168a.39ce.47d6.b7f5.ac1e0f312245  \n",
       "0                              17.498520  \n",
       "1                               0.199897  \n",
       "2                              13.081544  \n",
       "3                               6.476828  \n",
       "4                               1.571386  \n",
       "\n",
       "[5 rows x 80 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_tpm = pd.read_csv('infino-rcc/tcga_kirc_data/tcga_kirc.2.subset.tpm.filtered.tsv', sep='\\t')\n",
    "print(test_df_tpm.shape)\n",
    "test_df_tpm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:05:33.307933Z",
     "start_time": "2017-11-04T04:05:33.235559Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# confirm gene lists identical (by design)\n",
    "assert len(set(test_df_counts.Gene_symbols.unique()).symmetric_difference(set(test_df_tpm.Gene_symbols.unique()))) == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:05:35.172634Z",
     "start_time": "2017-11-04T04:05:35.050449Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(29733, 34832, 49878)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_genes_train = sample_df.gene_name.unique()\n",
    "all_genes_test = test_df_counts.Gene_symbols.unique()\n",
    "all_genes_intersect = list(set(all_genes_train) & set(all_genes_test))\n",
    "len(all_genes_intersect), len(all_genes_train), len(all_genes_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:05:38.264643Z",
     "start_time": "2017-11-04T04:05:38.237585Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(29733, 740)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove any marker transcripts that aren't present in our training+testing data\n",
    "marker_genes_adjusted = list(set(combined_marker_genes) - (set(combined_marker_genes) - set(all_genes_intersect)))\n",
    "len(all_genes_intersect), len(marker_genes_adjusted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:05:41.496903Z",
     "start_time": "2017-11-04T04:05:41.451214Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total number of genes we want: 900\n",
      "desired number of housekeeping genes: 160\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "160"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genes_to_sample_from = list(set(all_genes_intersect) - set(marker_genes_adjusted))\n",
    "total_num_genes=900\n",
    "n_genes_sample = total_num_genes-len(marker_genes_adjusted)\n",
    "print('total number of genes we want:', total_num_genes)\n",
    "print('desired number of housekeeping genes:', n_genes_sample)\n",
    "sampled_genes = np.random.choice(genes_to_sample_from, size=n_genes_sample)\n",
    "len(sampled_genes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:05:43.762985Z",
     "start_time": "2017-11-04T04:05:43.755363Z"
    },
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "selected_genes = np.concatenate([marker_genes_adjusted, sampled_genes]).tolist()\n",
    "assert len(selected_genes) == total_num_genes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:05:58.166663Z",
     "start_time": "2017-11-04T04:05:58.162644Z"
    },
    "collapsed": true,
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "outputs": [],
   "source": [
    "# pickle out selected gene list\n",
    "pickle.dump(selected_genes, open('infino-rcc/tcgakirc.selected_genes.pkl', 'wb'), pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:05:58.793277Z",
     "start_time": "2017-11-04T04:05:58.789949Z"
    },
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "selected_genes = pickle.load(open('infino-rcc/tcgakirc.selected_genes.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:06:05.786548Z",
     "start_time": "2017-11-04T04:06:05.476970Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(56700, 41)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_df = sample_df[sample_df['gene_name'].isin(selected_genes)].copy()\n",
    "training_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:06:11.465397Z",
     "start_time": "2017-11-04T04:06:11.417887Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    56700.000000\n",
       "mean       450.500000\n",
       "std        259.809752\n",
       "min          1.000000\n",
       "25%        225.750000\n",
       "50%        450.500000\n",
       "75%        675.250000\n",
       "max        900.000000\n",
       "Name: new_gene_id, dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reset ids\n",
    "training_df['new_gene_cat'] = training_df['gene_name'].astype('category')\n",
    "training_df['new_gene_id'] = training_df['new_gene_cat'].cat.codes+1\n",
    "training_df['new_sample_cat'] = training_df['sample_id'].astype('category')\n",
    "training_df['new_sample_id'] = training_df['new_sample_cat'].cat.codes+1\n",
    "training_df['new_gene_id'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:06:15.639813Z",
     "start_time": "2017-11-04T04:06:15.602197Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((900, 80), (900, 80))"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_counts = test_df_counts.loc[test_df_counts['Gene_symbols'].isin(selected_genes)]\n",
    "test_df_tpm = test_df_tpm.loc[test_df_tpm['Gene_symbols'].isin(selected_genes)]\n",
    "test_df_counts.shape, test_df_tpm.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Prepare to run ComBat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:06:27.611616Z",
     "start_time": "2017-11-04T04:06:27.524186Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X953473f4.9927.4fd4.bfee.8f5908638cc7</th>\n",
       "      <th>df66c814.1407.4899.a6e3.9c2a928e6ea0</th>\n",
       "      <th>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</th>\n",
       "      <th>X87c1d789.9482.442a.8b42.97bb479060e3</th>\n",
       "      <th>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</th>\n",
       "      <th>X383ba5ca.1706.4959.b543.ea87fbd367bd</th>\n",
       "      <th>d9f44b98.58b7.4ebd.a6a6.8de472bf95a8</th>\n",
       "      <th>X550af27b.de50.402f.8a81.c508e3dc6781</th>\n",
       "      <th>c81f9d5b.f58d.411d.9e1a.f919a0ef5636</th>\n",
       "      <th>cf0052ae.8440.49df.a20c.f260980c9922</th>\n",
       "      <th>...</th>\n",
       "      <th>X14b8d86c.2fef.4c50.8633.a84e12d81b51</th>\n",
       "      <th>X0cc91a13.32d9.4262.a33d.264096b4102c</th>\n",
       "      <th>X9bae55b1.5a27.4a9f.b376.ad1ab4fc4ee2</th>\n",
       "      <th>X6b124b2e.dce9.48ef.bedb.7a2da9cd2ff8</th>\n",
       "      <th>X3b4be095.1d76.49de.8357.f932ba5d55bb</th>\n",
       "      <th>e644640b.78d3.460a.83fa.ea475620a3f0</th>\n",
       "      <th>X01f8736a.85bc.498d.91c6.1c053c479512</th>\n",
       "      <th>e40e21de.f116.4eb1.87af.3b16797ec3b2</th>\n",
       "      <th>X19de7074.f195.47f6.81fa.6d3bf01c1685</th>\n",
       "      <th>X0153168a.39ce.47d6.b7f5.ac1e0f312245</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gene_symbols</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ABCB4</th>\n",
       "      <td>6.561507</td>\n",
       "      <td>6.104510</td>\n",
       "      <td>6.439970</td>\n",
       "      <td>5.885868</td>\n",
       "      <td>4.780072</td>\n",
       "      <td>6.648333</td>\n",
       "      <td>5.362536</td>\n",
       "      <td>6.329941</td>\n",
       "      <td>6.264012</td>\n",
       "      <td>5.965838</td>\n",
       "      <td>...</td>\n",
       "      <td>4.247332</td>\n",
       "      <td>5.312767</td>\n",
       "      <td>5.174255</td>\n",
       "      <td>4.715096</td>\n",
       "      <td>5.970627</td>\n",
       "      <td>5.919402</td>\n",
       "      <td>6.916850</td>\n",
       "      <td>5.997696</td>\n",
       "      <td>4.225654</td>\n",
       "      <td>4.939907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCC4</th>\n",
       "      <td>8.413711</td>\n",
       "      <td>9.095101</td>\n",
       "      <td>7.469923</td>\n",
       "      <td>8.347997</td>\n",
       "      <td>8.195988</td>\n",
       "      <td>8.481975</td>\n",
       "      <td>7.288892</td>\n",
       "      <td>8.980409</td>\n",
       "      <td>9.063691</td>\n",
       "      <td>8.632075</td>\n",
       "      <td>...</td>\n",
       "      <td>9.709272</td>\n",
       "      <td>8.689139</td>\n",
       "      <td>7.750338</td>\n",
       "      <td>8.324295</td>\n",
       "      <td>8.464965</td>\n",
       "      <td>7.820013</td>\n",
       "      <td>7.593914</td>\n",
       "      <td>7.647078</td>\n",
       "      <td>7.749776</td>\n",
       "      <td>7.222652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCG2</th>\n",
       "      <td>7.782389</td>\n",
       "      <td>8.377472</td>\n",
       "      <td>6.287859</td>\n",
       "      <td>7.632401</td>\n",
       "      <td>6.747587</td>\n",
       "      <td>7.241366</td>\n",
       "      <td>7.850105</td>\n",
       "      <td>6.456769</td>\n",
       "      <td>7.680635</td>\n",
       "      <td>5.081402</td>\n",
       "      <td>...</td>\n",
       "      <td>3.931826</td>\n",
       "      <td>6.428106</td>\n",
       "      <td>6.941189</td>\n",
       "      <td>5.262690</td>\n",
       "      <td>5.988962</td>\n",
       "      <td>8.043342</td>\n",
       "      <td>6.742881</td>\n",
       "      <td>6.579252</td>\n",
       "      <td>5.159055</td>\n",
       "      <td>7.412160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD2</th>\n",
       "      <td>9.248136</td>\n",
       "      <td>9.730278</td>\n",
       "      <td>8.497598</td>\n",
       "      <td>9.209793</td>\n",
       "      <td>8.009982</td>\n",
       "      <td>9.039929</td>\n",
       "      <td>8.152356</td>\n",
       "      <td>9.553096</td>\n",
       "      <td>9.588735</td>\n",
       "      <td>9.443778</td>\n",
       "      <td>...</td>\n",
       "      <td>10.515211</td>\n",
       "      <td>8.849146</td>\n",
       "      <td>8.731848</td>\n",
       "      <td>8.734675</td>\n",
       "      <td>8.103957</td>\n",
       "      <td>8.759076</td>\n",
       "      <td>8.610271</td>\n",
       "      <td>7.867093</td>\n",
       "      <td>7.846059</td>\n",
       "      <td>8.823254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD5</th>\n",
       "      <td>6.418562</td>\n",
       "      <td>6.963494</td>\n",
       "      <td>5.755742</td>\n",
       "      <td>6.629616</td>\n",
       "      <td>6.272879</td>\n",
       "      <td>6.818446</td>\n",
       "      <td>5.843869</td>\n",
       "      <td>6.260461</td>\n",
       "      <td>6.382567</td>\n",
       "      <td>6.234410</td>\n",
       "      <td>...</td>\n",
       "      <td>7.221956</td>\n",
       "      <td>6.985644</td>\n",
       "      <td>6.077690</td>\n",
       "      <td>6.235607</td>\n",
       "      <td>5.571532</td>\n",
       "      <td>5.870457</td>\n",
       "      <td>5.967761</td>\n",
       "      <td>5.908083</td>\n",
       "      <td>5.328380</td>\n",
       "      <td>5.902674</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 79 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              X953473f4.9927.4fd4.bfee.8f5908638cc7  \\\n",
       "Gene_symbols                                          \n",
       "ABCB4                                      6.561507   \n",
       "ABCC4                                      8.413711   \n",
       "ABCG2                                      7.782389   \n",
       "ABHD2                                      9.248136   \n",
       "ABHD5                                      6.418562   \n",
       "\n",
       "              df66c814.1407.4899.a6e3.9c2a928e6ea0  \\\n",
       "Gene_symbols                                         \n",
       "ABCB4                                     6.104510   \n",
       "ABCC4                                     9.095101   \n",
       "ABCG2                                     8.377472   \n",
       "ABHD2                                     9.730278   \n",
       "ABHD5                                     6.963494   \n",
       "\n",
       "              X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d  \\\n",
       "Gene_symbols                                          \n",
       "ABCB4                                      6.439970   \n",
       "ABCC4                                      7.469923   \n",
       "ABCG2                                      6.287859   \n",
       "ABHD2                                      8.497598   \n",
       "ABHD5                                      5.755742   \n",
       "\n",
       "              X87c1d789.9482.442a.8b42.97bb479060e3  \\\n",
       "Gene_symbols                                          \n",
       "ABCB4                                      5.885868   \n",
       "ABCC4                                      8.347997   \n",
       "ABCG2                                      7.632401   \n",
       "ABHD2                                      9.209793   \n",
       "ABHD5                                      6.629616   \n",
       "\n",
       "              X4ddfd589.5a62.4149.9ccc.6a539f0b37b7  \\\n",
       "Gene_symbols                                          \n",
       "ABCB4                                      4.780072   \n",
       "ABCC4                                      8.195988   \n",
       "ABCG2                                      6.747587   \n",
       "ABHD2                                      8.009982   \n",
       "ABHD5                                      6.272879   \n",
       "\n",
       "              X383ba5ca.1706.4959.b543.ea87fbd367bd  \\\n",
       "Gene_symbols                                          \n",
       "ABCB4                                      6.648333   \n",
       "ABCC4                                      8.481975   \n",
       "ABCG2                                      7.241366   \n",
       "ABHD2                                      9.039929   \n",
       "ABHD5                                      6.818446   \n",
       "\n",
       "              d9f44b98.58b7.4ebd.a6a6.8de472bf95a8  \\\n",
       "Gene_symbols                                         \n",
       "ABCB4                                     5.362536   \n",
       "ABCC4                                     7.288892   \n",
       "ABCG2                                     7.850105   \n",
       "ABHD2                                     8.152356   \n",
       "ABHD5                                     5.843869   \n",
       "\n",
       "              X550af27b.de50.402f.8a81.c508e3dc6781  \\\n",
       "Gene_symbols                                          \n",
       "ABCB4                                      6.329941   \n",
       "ABCC4                                      8.980409   \n",
       "ABCG2                                      6.456769   \n",
       "ABHD2                                      9.553096   \n",
       "ABHD5                                      6.260461   \n",
       "\n",
       "              c81f9d5b.f58d.411d.9e1a.f919a0ef5636  \\\n",
       "Gene_symbols                                         \n",
       "ABCB4                                     6.264012   \n",
       "ABCC4                                     9.063691   \n",
       "ABCG2                                     7.680635   \n",
       "ABHD2                                     9.588735   \n",
       "ABHD5                                     6.382567   \n",
       "\n",
       "              cf0052ae.8440.49df.a20c.f260980c9922  \\\n",
       "Gene_symbols                                         \n",
       "ABCB4                                     5.965838   \n",
       "ABCC4                                     8.632075   \n",
       "ABCG2                                     5.081402   \n",
       "ABHD2                                     9.443778   \n",
       "ABHD5                                     6.234410   \n",
       "\n",
       "                              ...                    \\\n",
       "Gene_symbols                  ...                     \n",
       "ABCB4                         ...                     \n",
       "ABCC4                         ...                     \n",
       "ABCG2                         ...                     \n",
       "ABHD2                         ...                     \n",
       "ABHD5                         ...                     \n",
       "\n",
       "              X14b8d86c.2fef.4c50.8633.a84e12d81b51  \\\n",
       "Gene_symbols                                          \n",
       "ABCB4                                      4.247332   \n",
       "ABCC4                                      9.709272   \n",
       "ABCG2                                      3.931826   \n",
       "ABHD2                                     10.515211   \n",
       "ABHD5                                      7.221956   \n",
       "\n",
       "              X0cc91a13.32d9.4262.a33d.264096b4102c  \\\n",
       "Gene_symbols                                          \n",
       "ABCB4                                      5.312767   \n",
       "ABCC4                                      8.689139   \n",
       "ABCG2                                      6.428106   \n",
       "ABHD2                                      8.849146   \n",
       "ABHD5                                      6.985644   \n",
       "\n",
       "              X9bae55b1.5a27.4a9f.b376.ad1ab4fc4ee2  \\\n",
       "Gene_symbols                                          \n",
       "ABCB4                                      5.174255   \n",
       "ABCC4                                      7.750338   \n",
       "ABCG2                                      6.941189   \n",
       "ABHD2                                      8.731848   \n",
       "ABHD5                                      6.077690   \n",
       "\n",
       "              X6b124b2e.dce9.48ef.bedb.7a2da9cd2ff8  \\\n",
       "Gene_symbols                                          \n",
       "ABCB4                                      4.715096   \n",
       "ABCC4                                      8.324295   \n",
       "ABCG2                                      5.262690   \n",
       "ABHD2                                      8.734675   \n",
       "ABHD5                                      6.235607   \n",
       "\n",
       "              X3b4be095.1d76.49de.8357.f932ba5d55bb  \\\n",
       "Gene_symbols                                          \n",
       "ABCB4                                      5.970627   \n",
       "ABCC4                                      8.464965   \n",
       "ABCG2                                      5.988962   \n",
       "ABHD2                                      8.103957   \n",
       "ABHD5                                      5.571532   \n",
       "\n",
       "              e644640b.78d3.460a.83fa.ea475620a3f0  \\\n",
       "Gene_symbols                                         \n",
       "ABCB4                                     5.919402   \n",
       "ABCC4                                     7.820013   \n",
       "ABCG2                                     8.043342   \n",
       "ABHD2                                     8.759076   \n",
       "ABHD5                                     5.870457   \n",
       "\n",
       "              X01f8736a.85bc.498d.91c6.1c053c479512  \\\n",
       "Gene_symbols                                          \n",
       "ABCB4                                      6.916850   \n",
       "ABCC4                                      7.593914   \n",
       "ABCG2                                      6.742881   \n",
       "ABHD2                                      8.610271   \n",
       "ABHD5                                      5.967761   \n",
       "\n",
       "              e40e21de.f116.4eb1.87af.3b16797ec3b2  \\\n",
       "Gene_symbols                                         \n",
       "ABCB4                                     5.997696   \n",
       "ABCC4                                     7.647078   \n",
       "ABCG2                                     6.579252   \n",
       "ABHD2                                     7.867093   \n",
       "ABHD5                                     5.908083   \n",
       "\n",
       "              X19de7074.f195.47f6.81fa.6d3bf01c1685  \\\n",
       "Gene_symbols                                          \n",
       "ABCB4                                      4.225654   \n",
       "ABCC4                                      7.749776   \n",
       "ABCG2                                      5.159055   \n",
       "ABHD2                                      7.846059   \n",
       "ABHD5                                      5.328380   \n",
       "\n",
       "              X0153168a.39ce.47d6.b7f5.ac1e0f312245  \n",
       "Gene_symbols                                         \n",
       "ABCB4                                      4.939907  \n",
       "ABCC4                                      7.222652  \n",
       "ABCG2                                      7.412160  \n",
       "ABHD2                                      8.823254  \n",
       "ABHD5                                      5.902674  \n",
       "\n",
       "[5 rows x 79 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_counts_log1p = test_df_counts.set_index('Gene_symbols').apply(np.log1p)\n",
    "test_df_tpm_log1p = test_df_tpm.set_index('Gene_symbols').apply(np.log1p)\n",
    "test_df_counts_log1p.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:06:30.437434Z",
     "start_time": "2017-11-04T04:06:30.362940Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>sample_id</th>\n",
       "      <th>filename</th>\n",
       "      <th>gene_name</th>\n",
       "      <th>est_counts</th>\n",
       "      <th>tpm</th>\n",
       "      <th>log1p_tpm</th>\n",
       "      <th>log1p_counts</th>\n",
       "      <th>CCR6+</th>\n",
       "      <th>CCR7+</th>\n",
       "      <th>...</th>\n",
       "      <th>log1p_tpm_rescaled_subset</th>\n",
       "      <th>log1p_tpm_rescaled</th>\n",
       "      <th>gene_cat</th>\n",
       "      <th>gene_id</th>\n",
       "      <th>B_cell</th>\n",
       "      <th>T_cell</th>\n",
       "      <th>new_gene_cat</th>\n",
       "      <th>new_gene_id</th>\n",
       "      <th>new_sample_cat</th>\n",
       "      <th>new_sample_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4347</th>\n",
       "      <td>4347</td>\n",
       "      <td>1</td>\n",
       "      <td>ERR431566</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.851882</td>\n",
       "      <td>0.616202</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.998440</td>\n",
       "      <td>-0.188314</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4348</th>\n",
       "      <td>4348</td>\n",
       "      <td>2</td>\n",
       "      <td>ERR431567</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.397441</td>\n",
       "      <td>0.334642</td>\n",
       "      <td>1.098612</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.115750</td>\n",
       "      <td>-0.636778</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4349</th>\n",
       "      <td>4349</td>\n",
       "      <td>3</td>\n",
       "      <td>ERR431568</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>782.000170</td>\n",
       "      <td>28.925644</td>\n",
       "      <td>3.398716</td>\n",
       "      <td>6.663133</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-6.838006</td>\n",
       "      <td>4.243620</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4350</th>\n",
       "      <td>4350</td>\n",
       "      <td>4</td>\n",
       "      <td>ERR431569</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>139.999971</td>\n",
       "      <td>5.299960</td>\n",
       "      <td>1.840543</td>\n",
       "      <td>4.948760</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-3.955303</td>\n",
       "      <td>1.761793</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4351</th>\n",
       "      <td>4351</td>\n",
       "      <td>5</td>\n",
       "      <td>ERR431570</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>3.007070</td>\n",
       "      <td>0.197082</td>\n",
       "      <td>0.179887</td>\n",
       "      <td>1.388060</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.390764</td>\n",
       "      <td>-0.883270</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index  sample_id   filename gene_name  est_counts        tpm  log1p_tpm  \\\n",
       "4347   4347          1  ERR431566     ABCB4    1.000000   0.851882   0.616202   \n",
       "4348   4348          2  ERR431567     ABCB4    2.000000   0.397441   0.334642   \n",
       "4349   4349          3  ERR431568     ABCB4  782.000170  28.925644   3.398716   \n",
       "4350   4350          4  ERR431569     ABCB4  139.999971   5.299960   1.840543   \n",
       "4351   4351          5  ERR431570     ABCB4    3.007070   0.197082   0.179887   \n",
       "\n",
       "      log1p_counts  CCR6+  CCR7+      ...        log1p_tpm_rescaled_subset  \\\n",
       "4347      0.693147    0.0    0.0      ...                         1.998440   \n",
       "4348      1.098612    0.0    0.0      ...                        -0.115750   \n",
       "4349      6.663133    0.0    0.0      ...                        -6.838006   \n",
       "4350      4.948760    0.0    0.0      ...                        -3.955303   \n",
       "4351      1.388060    0.0    0.0      ...                        -2.390764   \n",
       "\n",
       "      log1p_tpm_rescaled  gene_cat  gene_id  B_cell  T_cell  new_gene_cat  \\\n",
       "4347           -0.188314     ABCB4       70       0       1         ABCB4   \n",
       "4348           -0.636778     ABCB4       70       0       1         ABCB4   \n",
       "4349            4.243620     ABCB4       70       1       0         ABCB4   \n",
       "4350            1.761793     ABCB4       70       1       0         ABCB4   \n",
       "4351           -0.883270     ABCB4       70       0       1         ABCB4   \n",
       "\n",
       "      new_gene_id  new_sample_cat  new_sample_id  \n",
       "4347            1               1              1  \n",
       "4348            1               2              2  \n",
       "4349            1               3              3  \n",
       "4350            1               4              4  \n",
       "4351            1               5              5  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:06:33.226370Z",
     "start_time": "2017-11-04T04:06:32.777231Z"
    },
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "training_df['SubSet_and_sample'] = training_df['SubSet'].astype(str) + '.sampleid.' + training_df['new_sample_id'].astype(str)\n",
    "pivoted_train_log1p_counts = pd.pivot_table(training_df,\n",
    "                              index='gene_name',\n",
    "                              columns='SubSet_and_sample',\n",
    "                              values='log1p_counts'\n",
    "                             )\n",
    "pivoted_train_log1p_tpm = pd.pivot_table(training_df,\n",
    "                              index='gene_name',\n",
    "                              columns='SubSet_and_sample',\n",
    "                              values='log1p_tpm'\n",
    "                             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:06:43.232637Z",
     "start_time": "2017-11-04T04:06:43.214061Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Index(['B_CD5.sampleid.26', 'B_CD5.sampleid.3', 'B_CD5.sampleid.31',\n",
       "        'B_CD5.sampleid.34', 'B_Memory.sampleid.29', 'B_Memory.sampleid.4',\n",
       "        'B_Memory.sampleid.43', 'B_Memory.sampleid.48', 'B_Memory.sampleid.52',\n",
       "        'B_Naive.sampleid.21', 'B_Naive.sampleid.46', 'B_Naive.sampleid.54',\n",
       "        'B_Naive.sampleid.59', 'B_Naive.sampleid.7',\n",
       "        'CD4_Central_Memory.sampleid.17', 'CD4_Central_Memory.sampleid.33',\n",
       "        'CD4_Central_Memory.sampleid.49', 'CD4_Central_Memory.sampleid.55',\n",
       "        'CD4_Central_Memory.sampleid.61', 'CD4_Effector_Memory.sampleid.10',\n",
       "        'CD4_Effector_Memory.sampleid.11', 'CD4_Effector_Memory.sampleid.23',\n",
       "        'CD4_Effector_Memory.sampleid.38', 'CD4_Effector_Memory.sampleid.51',\n",
       "        'CD4_Naive.sampleid.15', 'CD4_Naive.sampleid.16',\n",
       "        'CD4_Naive.sampleid.45', 'CD4_Naive.sampleid.6', 'CD4_Naive.sampleid.9',\n",
       "        'CD4_Th1.sampleid.12', 'CD4_Th1.sampleid.19', 'CD4_Th1.sampleid.22',\n",
       "        'CD4_Th1.sampleid.5', 'CD4_Th1.sampleid.57', 'CD4_Th17.sampleid.30',\n",
       "        'CD4_Th17.sampleid.32', 'CD4_Th17.sampleid.39', 'CD4_Th17.sampleid.53',\n",
       "        'CD4_Th17.sampleid.60', 'CD4_Th2.sampleid.1', 'CD4_Th2.sampleid.14',\n",
       "        'CD4_Th2.sampleid.35', 'CD4_Th2.sampleid.50', 'CD4_Th2.sampleid.63',\n",
       "        'CD4_Treg.sampleid.18', 'CD4_Treg.sampleid.24', 'CD4_Treg.sampleid.36',\n",
       "        'CD4_Treg.sampleid.41', 'CD4_Treg.sampleid.44',\n",
       "        'CD8_Central_Memory.sampleid.13', 'CD8_Central_Memory.sampleid.28',\n",
       "        'CD8_Central_Memory.sampleid.37', 'CD8_Central_Memory.sampleid.42',\n",
       "        'CD8_Effector.sampleid.2', 'CD8_Effector.sampleid.20',\n",
       "        'CD8_Effector.sampleid.25', 'CD8_Effector.sampleid.56',\n",
       "        'CD8_Effector.sampleid.8', 'CD8_Naive.sampleid.27',\n",
       "        'CD8_Naive.sampleid.40', 'CD8_Naive.sampleid.47',\n",
       "        'CD8_Naive.sampleid.58', 'CD8_Naive.sampleid.62'],\n",
       "       dtype='object', name='SubSet_and_sample'),\n",
       " Index(['X953473f4.9927.4fd4.bfee.8f5908638cc7',\n",
       "        'df66c814.1407.4899.a6e3.9c2a928e6ea0',\n",
       "        'X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d',\n",
       "        'X87c1d789.9482.442a.8b42.97bb479060e3',\n",
       "        'X4ddfd589.5a62.4149.9ccc.6a539f0b37b7',\n",
       "        'X383ba5ca.1706.4959.b543.ea87fbd367bd',\n",
       "        'd9f44b98.58b7.4ebd.a6a6.8de472bf95a8',\n",
       "        'X550af27b.de50.402f.8a81.c508e3dc6781',\n",
       "        'c81f9d5b.f58d.411d.9e1a.f919a0ef5636',\n",
       "        'cf0052ae.8440.49df.a20c.f260980c9922',\n",
       "        'b9f6eddc.8830.4295.af93.47e68923c443',\n",
       "        'X5c1ef93f.78a5.40e1.8a7b.a17a05d683e9',\n",
       "        'X94008843.2c57.4464.9afa.b25429ad013c',\n",
       "        'X861afb24.028f.415c.ba15.cfc3db24c485',\n",
       "        'X8da0a7b4.11bc.44e3.bc80.bf41bbefcc15',\n",
       "        'd6af7bdb.52a9.4b8f.9af2.5ada584bc71d',\n",
       "        'X38e69c1a.8c5a.4dff.9cd8.e5813a615c43',\n",
       "        'e8166ef6.1f7e.476d.b013.6ec358587db4',\n",
       "        'b5858ec5.99c5.452b.b728.45e10e772856',\n",
       "        'f3fb6c23.5e68.491e.a596.ff2e76545d6b',\n",
       "        'X813d0e86.0b45.4e7a.b3f9.2bbd7b02be3d',\n",
       "        'X39ecb705.8b3d.48e4.8eb2.381e1d90941f',\n",
       "        'X23537b11.62a1.4e04.b9f9.51f51b24ab4a',\n",
       "        'X9aa664b0.e261.4dc6.95f2.550cd304622b',\n",
       "        'X6b88c83a.6d9a.4e08.ac90.06722b861b87',\n",
       "        'b1edf3f1.92f0.4d44.b1b8.e182c8a3ddc0',\n",
       "        'X538ea4a0.f304.400d.9e8e.d3a24849520d',\n",
       "        'X00b8988a.2995.444b.854c.4cd15c2c49a5',\n",
       "        'X4bfb178c.af2e.44fb.854b.2cdf998de348',\n",
       "        'X8cb60f95.e81c.4094.9032.9ef8fc6f3d75',\n",
       "        'ea604a55.0bf8.490d.9e4c.8aff9ac53c5e',\n",
       "        'X7bcc7295.676d.4262.9dd0.d85f2d60b84c',\n",
       "        'X3abb0bc9.3124.4538.add6.8ab9c1b3cc85',\n",
       "        'fddcd052.d62e.4f8a.90fe.cbcbf4d9db29',\n",
       "        'e8c79042.aa3d.4844.9c4e.0ca4dac89318',\n",
       "        'X0344d786.dcf5.402c.8460.16d3f08c014e',\n",
       "        'X44e05bdf.5ae3.4b28.88fa.0ee9e7908205',\n",
       "        'X4ab74f72.d4e0.4954.b88d.e65f995085c2',\n",
       "        'X018e9b8d.f4d5.479f.8e59.977c2e1c51d6',\n",
       "        'a0f62f75.1a15.413e.9960.3cfc09f52b9e',\n",
       "        'X116f7723.de02.4a81.bec1.cc068019ff38',\n",
       "        'd5aa55f8.95e6.49a8.928b.449062b6fb76',\n",
       "        'X37bbf767.210a.4087.bebb.6e78518dc13f',\n",
       "        'X41a85abf.47a4.40c2.8c91.09280cbab02a',\n",
       "        'b90a85d4.70dc.4f6a.9624.13dce30e7893',\n",
       "        'eb963d28.7788.447f.9446.dd507f17b978',\n",
       "        'ca4160b0.4f82.4fcd.8a5a.878b213dcd6b',\n",
       "        'X58f32c34.1979.4c3a.a3ee.ffbbc3c4ed8f',\n",
       "        'X16436896.712a.4929.a7b6.75927376d2f8',\n",
       "        'X945726c2.bc90.4564.899e.ea8083ff3df3',\n",
       "        'X4c1b2d3d.99ce.400e.b659.14a0d20ab16f',\n",
       "        'X845008b9.7a8e.4c0a.a706.69b002710e24',\n",
       "        'X7d5a7607.0439.4aac.a206.4096a2908f5b',\n",
       "        'e5894cd9.91f4.4175.aa84.fa183252de79',\n",
       "        'e8805ccf.6696.40e7.aae8.2301f974eb23',\n",
       "        'X4ba29a9a.30a1.41af.9290.1efea3b0f107',\n",
       "        'X8337428b.7c4f.4d82.a41a.cd03c5129c4f',\n",
       "        'X5a547f07.77ee.4a01.8df6.b9c0ae55c906',\n",
       "        'X831bfb31.ff56.44aa.88ce.abfc4d1874dd',\n",
       "        'edf8a7d1.705c.4142.b67d.edfbfb76f3d1',\n",
       "        'df14133a.71bd.43ad.8109.f3f0acf0e967',\n",
       "        'X8fe6b2e3.5fba.4757.9d88.8057fa5f5aa2',\n",
       "        'X05908623.0112.4998.a601.9e870b2665a8',\n",
       "        'X9af223c5.a6be.41b6.bfdb.ef2ecf885c9e',\n",
       "        'X5274e1ee.63cb.4b7d.9c44.f210d351979b',\n",
       "        'X77e31d23.9a91.417c.9fae.be2ec67b8546',\n",
       "        'eb3d0125.0eef.4593.ac46.b2c5ecbd0799',\n",
       "        'X65e536c8.0ea5.4206.9a52.c13be085467c',\n",
       "        'c872d7ed.2aa1.44ff.b3ea.a1c45887506a',\n",
       "        'X14b8d86c.2fef.4c50.8633.a84e12d81b51',\n",
       "        'X0cc91a13.32d9.4262.a33d.264096b4102c',\n",
       "        'X9bae55b1.5a27.4a9f.b376.ad1ab4fc4ee2',\n",
       "        'X6b124b2e.dce9.48ef.bedb.7a2da9cd2ff8',\n",
       "        'X3b4be095.1d76.49de.8357.f932ba5d55bb',\n",
       "        'e644640b.78d3.460a.83fa.ea475620a3f0',\n",
       "        'X01f8736a.85bc.498d.91c6.1c053c479512',\n",
       "        'e40e21de.f116.4eb1.87af.3b16797ec3b2',\n",
       "        'X19de7074.f195.47f6.81fa.6d3bf01c1685',\n",
       "        'X0153168a.39ce.47d6.b7f5.ac1e0f312245'],\n",
       "       dtype='object'))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract the column names for each batch\n",
    "single_origin_cols = pivoted_train_log1p_tpm.columns\n",
    "cohort_cols = test_df_tpm_log1p.columns\n",
    "single_origin_cols, cohort_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:06:59.511468Z",
     "start_time": "2017-11-04T04:06:59.439788Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(900, 79) (900, 63) (900, 142)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X953473f4.9927.4fd4.bfee.8f5908638cc7</th>\n",
       "      <th>df66c814.1407.4899.a6e3.9c2a928e6ea0</th>\n",
       "      <th>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</th>\n",
       "      <th>X87c1d789.9482.442a.8b42.97bb479060e3</th>\n",
       "      <th>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</th>\n",
       "      <th>X383ba5ca.1706.4959.b543.ea87fbd367bd</th>\n",
       "      <th>d9f44b98.58b7.4ebd.a6a6.8de472bf95a8</th>\n",
       "      <th>X550af27b.de50.402f.8a81.c508e3dc6781</th>\n",
       "      <th>c81f9d5b.f58d.411d.9e1a.f919a0ef5636</th>\n",
       "      <th>cf0052ae.8440.49df.a20c.f260980c9922</th>\n",
       "      <th>...</th>\n",
       "      <th>CD8_Effector.sampleid.2</th>\n",
       "      <th>CD8_Effector.sampleid.20</th>\n",
       "      <th>CD8_Effector.sampleid.25</th>\n",
       "      <th>CD8_Effector.sampleid.56</th>\n",
       "      <th>CD8_Effector.sampleid.8</th>\n",
       "      <th>CD8_Naive.sampleid.27</th>\n",
       "      <th>CD8_Naive.sampleid.40</th>\n",
       "      <th>CD8_Naive.sampleid.47</th>\n",
       "      <th>CD8_Naive.sampleid.58</th>\n",
       "      <th>CD8_Naive.sampleid.62</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProbeName</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ABCB4</th>\n",
       "      <td>6.561507</td>\n",
       "      <td>6.104510</td>\n",
       "      <td>6.439970</td>\n",
       "      <td>5.885868</td>\n",
       "      <td>4.780072</td>\n",
       "      <td>6.648333</td>\n",
       "      <td>5.362536</td>\n",
       "      <td>6.329941</td>\n",
       "      <td>6.264012</td>\n",
       "      <td>5.965838</td>\n",
       "      <td>...</td>\n",
       "      <td>1.098612</td>\n",
       "      <td>2.079442</td>\n",
       "      <td>1.947962</td>\n",
       "      <td>2.302585</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>2.198396</td>\n",
       "      <td>2.565462</td>\n",
       "      <td>2.080656</td>\n",
       "      <td>1.610132</td>\n",
       "      <td>2.079442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCC4</th>\n",
       "      <td>8.413711</td>\n",
       "      <td>9.095101</td>\n",
       "      <td>7.469923</td>\n",
       "      <td>8.347997</td>\n",
       "      <td>8.195988</td>\n",
       "      <td>8.481975</td>\n",
       "      <td>7.288892</td>\n",
       "      <td>8.980409</td>\n",
       "      <td>9.063691</td>\n",
       "      <td>8.632075</td>\n",
       "      <td>...</td>\n",
       "      <td>4.282047</td>\n",
       "      <td>3.903095</td>\n",
       "      <td>5.008213</td>\n",
       "      <td>4.290876</td>\n",
       "      <td>4.958242</td>\n",
       "      <td>4.785497</td>\n",
       "      <td>3.976639</td>\n",
       "      <td>3.833354</td>\n",
       "      <td>4.486932</td>\n",
       "      <td>5.096698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCG2</th>\n",
       "      <td>7.782389</td>\n",
       "      <td>8.377472</td>\n",
       "      <td>6.287859</td>\n",
       "      <td>7.632401</td>\n",
       "      <td>6.747587</td>\n",
       "      <td>7.241366</td>\n",
       "      <td>7.850105</td>\n",
       "      <td>6.456769</td>\n",
       "      <td>7.680635</td>\n",
       "      <td>5.081402</td>\n",
       "      <td>...</td>\n",
       "      <td>1.098612</td>\n",
       "      <td>3.970292</td>\n",
       "      <td>3.610918</td>\n",
       "      <td>3.135493</td>\n",
       "      <td>2.833213</td>\n",
       "      <td>3.583519</td>\n",
       "      <td>4.700480</td>\n",
       "      <td>4.127135</td>\n",
       "      <td>3.295838</td>\n",
       "      <td>2.944439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD2</th>\n",
       "      <td>9.248136</td>\n",
       "      <td>9.730278</td>\n",
       "      <td>8.497598</td>\n",
       "      <td>9.209793</td>\n",
       "      <td>8.009982</td>\n",
       "      <td>9.039929</td>\n",
       "      <td>8.152356</td>\n",
       "      <td>9.553096</td>\n",
       "      <td>9.588735</td>\n",
       "      <td>9.443778</td>\n",
       "      <td>...</td>\n",
       "      <td>6.460766</td>\n",
       "      <td>7.344138</td>\n",
       "      <td>6.955704</td>\n",
       "      <td>7.094800</td>\n",
       "      <td>6.673489</td>\n",
       "      <td>6.546612</td>\n",
       "      <td>7.468739</td>\n",
       "      <td>6.721209</td>\n",
       "      <td>6.847852</td>\n",
       "      <td>6.439608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD5</th>\n",
       "      <td>6.418562</td>\n",
       "      <td>6.963494</td>\n",
       "      <td>5.755742</td>\n",
       "      <td>6.629616</td>\n",
       "      <td>6.272879</td>\n",
       "      <td>6.818446</td>\n",
       "      <td>5.843869</td>\n",
       "      <td>6.260461</td>\n",
       "      <td>6.382567</td>\n",
       "      <td>6.234410</td>\n",
       "      <td>...</td>\n",
       "      <td>4.330733</td>\n",
       "      <td>4.442652</td>\n",
       "      <td>4.110873</td>\n",
       "      <td>4.264087</td>\n",
       "      <td>4.094344</td>\n",
       "      <td>4.094345</td>\n",
       "      <td>3.807695</td>\n",
       "      <td>3.178054</td>\n",
       "      <td>3.931826</td>\n",
       "      <td>3.914728</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 142 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           X953473f4.9927.4fd4.bfee.8f5908638cc7  \\\n",
       "ProbeName                                          \n",
       "ABCB4                                   6.561507   \n",
       "ABCC4                                   8.413711   \n",
       "ABCG2                                   7.782389   \n",
       "ABHD2                                   9.248136   \n",
       "ABHD5                                   6.418562   \n",
       "\n",
       "           df66c814.1407.4899.a6e3.9c2a928e6ea0  \\\n",
       "ProbeName                                         \n",
       "ABCB4                                  6.104510   \n",
       "ABCC4                                  9.095101   \n",
       "ABCG2                                  8.377472   \n",
       "ABHD2                                  9.730278   \n",
       "ABHD5                                  6.963494   \n",
       "\n",
       "           X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d  \\\n",
       "ProbeName                                          \n",
       "ABCB4                                   6.439970   \n",
       "ABCC4                                   7.469923   \n",
       "ABCG2                                   6.287859   \n",
       "ABHD2                                   8.497598   \n",
       "ABHD5                                   5.755742   \n",
       "\n",
       "           X87c1d789.9482.442a.8b42.97bb479060e3  \\\n",
       "ProbeName                                          \n",
       "ABCB4                                   5.885868   \n",
       "ABCC4                                   8.347997   \n",
       "ABCG2                                   7.632401   \n",
       "ABHD2                                   9.209793   \n",
       "ABHD5                                   6.629616   \n",
       "\n",
       "           X4ddfd589.5a62.4149.9ccc.6a539f0b37b7  \\\n",
       "ProbeName                                          \n",
       "ABCB4                                   4.780072   \n",
       "ABCC4                                   8.195988   \n",
       "ABCG2                                   6.747587   \n",
       "ABHD2                                   8.009982   \n",
       "ABHD5                                   6.272879   \n",
       "\n",
       "           X383ba5ca.1706.4959.b543.ea87fbd367bd  \\\n",
       "ProbeName                                          \n",
       "ABCB4                                   6.648333   \n",
       "ABCC4                                   8.481975   \n",
       "ABCG2                                   7.241366   \n",
       "ABHD2                                   9.039929   \n",
       "ABHD5                                   6.818446   \n",
       "\n",
       "           d9f44b98.58b7.4ebd.a6a6.8de472bf95a8  \\\n",
       "ProbeName                                         \n",
       "ABCB4                                  5.362536   \n",
       "ABCC4                                  7.288892   \n",
       "ABCG2                                  7.850105   \n",
       "ABHD2                                  8.152356   \n",
       "ABHD5                                  5.843869   \n",
       "\n",
       "           X550af27b.de50.402f.8a81.c508e3dc6781  \\\n",
       "ProbeName                                          \n",
       "ABCB4                                   6.329941   \n",
       "ABCC4                                   8.980409   \n",
       "ABCG2                                   6.456769   \n",
       "ABHD2                                   9.553096   \n",
       "ABHD5                                   6.260461   \n",
       "\n",
       "           c81f9d5b.f58d.411d.9e1a.f919a0ef5636  \\\n",
       "ProbeName                                         \n",
       "ABCB4                                  6.264012   \n",
       "ABCC4                                  9.063691   \n",
       "ABCG2                                  7.680635   \n",
       "ABHD2                                  9.588735   \n",
       "ABHD5                                  6.382567   \n",
       "\n",
       "           cf0052ae.8440.49df.a20c.f260980c9922          ...            \\\n",
       "ProbeName                                                ...             \n",
       "ABCB4                                  5.965838          ...             \n",
       "ABCC4                                  8.632075          ...             \n",
       "ABCG2                                  5.081402          ...             \n",
       "ABHD2                                  9.443778          ...             \n",
       "ABHD5                                  6.234410          ...             \n",
       "\n",
       "           CD8_Effector.sampleid.2  CD8_Effector.sampleid.20  \\\n",
       "ProbeName                                                      \n",
       "ABCB4                     1.098612                  2.079442   \n",
       "ABCC4                     4.282047                  3.903095   \n",
       "ABCG2                     1.098612                  3.970292   \n",
       "ABHD2                     6.460766                  7.344138   \n",
       "ABHD5                     4.330733                  4.442652   \n",
       "\n",
       "           CD8_Effector.sampleid.25  CD8_Effector.sampleid.56  \\\n",
       "ProbeName                                                       \n",
       "ABCB4                      1.947962                  2.302585   \n",
       "ABCC4                      5.008213                  4.290876   \n",
       "ABCG2                      3.610918                  3.135493   \n",
       "ABHD2                      6.955704                  7.094800   \n",
       "ABHD5                      4.110873                  4.264087   \n",
       "\n",
       "           CD8_Effector.sampleid.8  CD8_Naive.sampleid.27  \\\n",
       "ProbeName                                                   \n",
       "ABCB4                     0.693147               2.198396   \n",
       "ABCC4                     4.958242               4.785497   \n",
       "ABCG2                     2.833213               3.583519   \n",
       "ABHD2                     6.673489               6.546612   \n",
       "ABHD5                     4.094344               4.094345   \n",
       "\n",
       "           CD8_Naive.sampleid.40  CD8_Naive.sampleid.47  \\\n",
       "ProbeName                                                 \n",
       "ABCB4                   2.565462               2.080656   \n",
       "ABCC4                   3.976639               3.833354   \n",
       "ABCG2                   4.700480               4.127135   \n",
       "ABHD2                   7.468739               6.721209   \n",
       "ABHD5                   3.807695               3.178054   \n",
       "\n",
       "           CD8_Naive.sampleid.58  CD8_Naive.sampleid.62  \n",
       "ProbeName                                                \n",
       "ABCB4                   1.610132               2.079442  \n",
       "ABCC4                   4.486932               5.096698  \n",
       "ABCG2                   3.295838               2.944439  \n",
       "ABHD2                   6.847852               6.439608  \n",
       "ABHD5                   3.931826               3.914728  \n",
       "\n",
       "[5 rows x 142 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_raw_counts = pd.merge(test_df_counts_log1p, pivoted_train_log1p_counts, left_index=True, right_index=True)\n",
    "print(test_df_counts_log1p.shape, pivoted_train_log1p_counts.shape, combined_raw_counts.shape)\n",
    "combined_raw_counts.index.name = 'ProbeName'\n",
    "combined_raw_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:07:01.242628Z",
     "start_time": "2017-11-04T04:07:01.159281Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(900, 79) (900, 63) (900, 142)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X953473f4.9927.4fd4.bfee.8f5908638cc7</th>\n",
       "      <th>df66c814.1407.4899.a6e3.9c2a928e6ea0</th>\n",
       "      <th>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</th>\n",
       "      <th>X87c1d789.9482.442a.8b42.97bb479060e3</th>\n",
       "      <th>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</th>\n",
       "      <th>X383ba5ca.1706.4959.b543.ea87fbd367bd</th>\n",
       "      <th>d9f44b98.58b7.4ebd.a6a6.8de472bf95a8</th>\n",
       "      <th>X550af27b.de50.402f.8a81.c508e3dc6781</th>\n",
       "      <th>c81f9d5b.f58d.411d.9e1a.f919a0ef5636</th>\n",
       "      <th>cf0052ae.8440.49df.a20c.f260980c9922</th>\n",
       "      <th>...</th>\n",
       "      <th>CD8_Effector.sampleid.2</th>\n",
       "      <th>CD8_Effector.sampleid.20</th>\n",
       "      <th>CD8_Effector.sampleid.25</th>\n",
       "      <th>CD8_Effector.sampleid.56</th>\n",
       "      <th>CD8_Effector.sampleid.8</th>\n",
       "      <th>CD8_Naive.sampleid.27</th>\n",
       "      <th>CD8_Naive.sampleid.40</th>\n",
       "      <th>CD8_Naive.sampleid.47</th>\n",
       "      <th>CD8_Naive.sampleid.58</th>\n",
       "      <th>CD8_Naive.sampleid.62</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProbeName</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ABCB4</th>\n",
       "      <td>1.786577</td>\n",
       "      <td>1.215977</td>\n",
       "      <td>1.960582</td>\n",
       "      <td>1.278563</td>\n",
       "      <td>1.026254</td>\n",
       "      <td>2.062886</td>\n",
       "      <td>1.382510</td>\n",
       "      <td>1.729957</td>\n",
       "      <td>1.589234</td>\n",
       "      <td>1.432772</td>\n",
       "      <td>...</td>\n",
       "      <td>0.334642</td>\n",
       "      <td>0.409428</td>\n",
       "      <td>0.183653</td>\n",
       "      <td>0.734390</td>\n",
       "      <td>0.171621</td>\n",
       "      <td>0.267101</td>\n",
       "      <td>0.409472</td>\n",
       "      <td>0.283302</td>\n",
       "      <td>0.097145</td>\n",
       "      <td>0.662777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCC4</th>\n",
       "      <td>3.052037</td>\n",
       "      <td>3.404190</td>\n",
       "      <td>2.542866</td>\n",
       "      <td>2.944869</td>\n",
       "      <td>3.268303</td>\n",
       "      <td>3.195574</td>\n",
       "      <td>1.894603</td>\n",
       "      <td>3.481405</td>\n",
       "      <td>3.755063</td>\n",
       "      <td>3.088673</td>\n",
       "      <td>...</td>\n",
       "      <td>1.740690</td>\n",
       "      <td>1.182309</td>\n",
       "      <td>1.748660</td>\n",
       "      <td>1.749194</td>\n",
       "      <td>1.815492</td>\n",
       "      <td>1.583284</td>\n",
       "      <td>1.066921</td>\n",
       "      <td>0.810437</td>\n",
       "      <td>1.432758</td>\n",
       "      <td>1.523973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCG2</th>\n",
       "      <td>2.938073</td>\n",
       "      <td>3.135846</td>\n",
       "      <td>1.851018</td>\n",
       "      <td>2.716802</td>\n",
       "      <td>2.220262</td>\n",
       "      <td>2.397592</td>\n",
       "      <td>2.665559</td>\n",
       "      <td>1.655075</td>\n",
       "      <td>2.826349</td>\n",
       "      <td>0.655081</td>\n",
       "      <td>...</td>\n",
       "      <td>0.096339</td>\n",
       "      <td>0.834348</td>\n",
       "      <td>1.149255</td>\n",
       "      <td>0.455862</td>\n",
       "      <td>0.363216</td>\n",
       "      <td>0.592473</td>\n",
       "      <td>1.235624</td>\n",
       "      <td>0.969733</td>\n",
       "      <td>0.483663</td>\n",
       "      <td>0.424996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD2</th>\n",
       "      <td>3.365501</td>\n",
       "      <td>3.568185</td>\n",
       "      <td>3.046352</td>\n",
       "      <td>3.292031</td>\n",
       "      <td>2.663061</td>\n",
       "      <td>3.247669</td>\n",
       "      <td>2.177218</td>\n",
       "      <td>3.570614</td>\n",
       "      <td>3.833753</td>\n",
       "      <td>3.476872</td>\n",
       "      <td>...</td>\n",
       "      <td>3.785061</td>\n",
       "      <td>3.303301</td>\n",
       "      <td>2.875261</td>\n",
       "      <td>3.323798</td>\n",
       "      <td>2.652309</td>\n",
       "      <td>2.879672</td>\n",
       "      <td>3.365206</td>\n",
       "      <td>2.906016</td>\n",
       "      <td>2.907709</td>\n",
       "      <td>2.697088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD5</th>\n",
       "      <td>2.427891</td>\n",
       "      <td>2.638941</td>\n",
       "      <td>2.156326</td>\n",
       "      <td>2.594299</td>\n",
       "      <td>2.692320</td>\n",
       "      <td>2.861580</td>\n",
       "      <td>1.739094</td>\n",
       "      <td>2.180282</td>\n",
       "      <td>2.518814</td>\n",
       "      <td>2.119694</td>\n",
       "      <td>...</td>\n",
       "      <td>2.670204</td>\n",
       "      <td>2.110597</td>\n",
       "      <td>1.902689</td>\n",
       "      <td>2.109573</td>\n",
       "      <td>2.179342</td>\n",
       "      <td>1.839330</td>\n",
       "      <td>1.414390</td>\n",
       "      <td>1.096676</td>\n",
       "      <td>1.861320</td>\n",
       "      <td>1.976365</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 142 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           X953473f4.9927.4fd4.bfee.8f5908638cc7  \\\n",
       "ProbeName                                          \n",
       "ABCB4                                   1.786577   \n",
       "ABCC4                                   3.052037   \n",
       "ABCG2                                   2.938073   \n",
       "ABHD2                                   3.365501   \n",
       "ABHD5                                   2.427891   \n",
       "\n",
       "           df66c814.1407.4899.a6e3.9c2a928e6ea0  \\\n",
       "ProbeName                                         \n",
       "ABCB4                                  1.215977   \n",
       "ABCC4                                  3.404190   \n",
       "ABCG2                                  3.135846   \n",
       "ABHD2                                  3.568185   \n",
       "ABHD5                                  2.638941   \n",
       "\n",
       "           X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d  \\\n",
       "ProbeName                                          \n",
       "ABCB4                                   1.960582   \n",
       "ABCC4                                   2.542866   \n",
       "ABCG2                                   1.851018   \n",
       "ABHD2                                   3.046352   \n",
       "ABHD5                                   2.156326   \n",
       "\n",
       "           X87c1d789.9482.442a.8b42.97bb479060e3  \\\n",
       "ProbeName                                          \n",
       "ABCB4                                   1.278563   \n",
       "ABCC4                                   2.944869   \n",
       "ABCG2                                   2.716802   \n",
       "ABHD2                                   3.292031   \n",
       "ABHD5                                   2.594299   \n",
       "\n",
       "           X4ddfd589.5a62.4149.9ccc.6a539f0b37b7  \\\n",
       "ProbeName                                          \n",
       "ABCB4                                   1.026254   \n",
       "ABCC4                                   3.268303   \n",
       "ABCG2                                   2.220262   \n",
       "ABHD2                                   2.663061   \n",
       "ABHD5                                   2.692320   \n",
       "\n",
       "           X383ba5ca.1706.4959.b543.ea87fbd367bd  \\\n",
       "ProbeName                                          \n",
       "ABCB4                                   2.062886   \n",
       "ABCC4                                   3.195574   \n",
       "ABCG2                                   2.397592   \n",
       "ABHD2                                   3.247669   \n",
       "ABHD5                                   2.861580   \n",
       "\n",
       "           d9f44b98.58b7.4ebd.a6a6.8de472bf95a8  \\\n",
       "ProbeName                                         \n",
       "ABCB4                                  1.382510   \n",
       "ABCC4                                  1.894603   \n",
       "ABCG2                                  2.665559   \n",
       "ABHD2                                  2.177218   \n",
       "ABHD5                                  1.739094   \n",
       "\n",
       "           X550af27b.de50.402f.8a81.c508e3dc6781  \\\n",
       "ProbeName                                          \n",
       "ABCB4                                   1.729957   \n",
       "ABCC4                                   3.481405   \n",
       "ABCG2                                   1.655075   \n",
       "ABHD2                                   3.570614   \n",
       "ABHD5                                   2.180282   \n",
       "\n",
       "           c81f9d5b.f58d.411d.9e1a.f919a0ef5636  \\\n",
       "ProbeName                                         \n",
       "ABCB4                                  1.589234   \n",
       "ABCC4                                  3.755063   \n",
       "ABCG2                                  2.826349   \n",
       "ABHD2                                  3.833753   \n",
       "ABHD5                                  2.518814   \n",
       "\n",
       "           cf0052ae.8440.49df.a20c.f260980c9922          ...            \\\n",
       "ProbeName                                                ...             \n",
       "ABCB4                                  1.432772          ...             \n",
       "ABCC4                                  3.088673          ...             \n",
       "ABCG2                                  0.655081          ...             \n",
       "ABHD2                                  3.476872          ...             \n",
       "ABHD5                                  2.119694          ...             \n",
       "\n",
       "           CD8_Effector.sampleid.2  CD8_Effector.sampleid.20  \\\n",
       "ProbeName                                                      \n",
       "ABCB4                     0.334642                  0.409428   \n",
       "ABCC4                     1.740690                  1.182309   \n",
       "ABCG2                     0.096339                  0.834348   \n",
       "ABHD2                     3.785061                  3.303301   \n",
       "ABHD5                     2.670204                  2.110597   \n",
       "\n",
       "           CD8_Effector.sampleid.25  CD8_Effector.sampleid.56  \\\n",
       "ProbeName                                                       \n",
       "ABCB4                      0.183653                  0.734390   \n",
       "ABCC4                      1.748660                  1.749194   \n",
       "ABCG2                      1.149255                  0.455862   \n",
       "ABHD2                      2.875261                  3.323798   \n",
       "ABHD5                      1.902689                  2.109573   \n",
       "\n",
       "           CD8_Effector.sampleid.8  CD8_Naive.sampleid.27  \\\n",
       "ProbeName                                                   \n",
       "ABCB4                     0.171621               0.267101   \n",
       "ABCC4                     1.815492               1.583284   \n",
       "ABCG2                     0.363216               0.592473   \n",
       "ABHD2                     2.652309               2.879672   \n",
       "ABHD5                     2.179342               1.839330   \n",
       "\n",
       "           CD8_Naive.sampleid.40  CD8_Naive.sampleid.47  \\\n",
       "ProbeName                                                 \n",
       "ABCB4                   0.409472               0.283302   \n",
       "ABCC4                   1.066921               0.810437   \n",
       "ABCG2                   1.235624               0.969733   \n",
       "ABHD2                   3.365206               2.906016   \n",
       "ABHD5                   1.414390               1.096676   \n",
       "\n",
       "           CD8_Naive.sampleid.58  CD8_Naive.sampleid.62  \n",
       "ProbeName                                                \n",
       "ABCB4                   0.097145               0.662777  \n",
       "ABCC4                   1.432758               1.523973  \n",
       "ABCG2                   0.483663               0.424996  \n",
       "ABHD2                   2.907709               2.697088  \n",
       "ABHD5                   1.861320               1.976365  \n",
       "\n",
       "[5 rows x 142 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_raw_tpm = pd.merge(test_df_tpm_log1p, pivoted_train_log1p_tpm, left_index=True, right_index=True)\n",
    "print(test_df_tpm_log1p.shape, pivoted_train_log1p_tpm.shape, combined_raw_tpm.shape)\n",
    "combined_raw_tpm.index.name = 'ProbeName'\n",
    "combined_raw_tpm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:07:07.553813Z",
     "start_time": "2017-11-04T04:07:07.525929Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# confirm no nulls -- remove assert and len prefix to extract implicated column names if assertion fails\n",
    "assert len(combined_raw_counts.columns[pd.isnull(combined_raw_counts).sum() > 0]) == 0\n",
    "assert len(combined_raw_tpm.columns[pd.isnull(combined_raw_tpm).sum() > 0]) == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:07:47.091004Z",
     "start_time": "2017-11-04T04:07:46.481056Z"
    },
    "collapsed": true,
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "outputs": [],
   "source": [
    "# file 1 for ComBat\n",
    "combined_raw_counts.to_csv('infino-rcc/tcgakirc.combined.raw.combat_input.counts.tsv', sep='\\t')\n",
    "combined_raw_tpm.to_csv('infino-rcc/tcgakirc.combined.raw.combat_input.tpm.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## make sample information file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:08:14.459999Z",
     "start_time": "2017-11-04T04:08:14.428700Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Array name</th>\n",
       "      <th>Batch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>df66c814.1407.4899.a6e3.9c2a928e6ea0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>X87c1d789.9482.442a.8b42.97bb479060e3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Array name  Batch\n",
       "0  X953473f4.9927.4fd4.bfee.8f5908638cc7      2\n",
       "1   df66c814.1407.4899.a6e3.9c2a928e6ea0      2\n",
       "2  X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d      2\n",
       "3  X87c1d789.9482.442a.8b42.97bb479060e3      2\n",
       "4  X4ddfd589.5a62.4149.9ccc.6a539f0b37b7      2"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cohort_batch = pd.DataFrame({'Array name': cohort_cols})\n",
    "cohort_batch['Batch'] = 2\n",
    "cohort_batch.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:08:17.989732Z",
     "start_time": "2017-11-04T04:08:17.964795Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Array name</th>\n",
       "      <th>Batch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B_CD5.sampleid.26</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B_CD5.sampleid.3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B_CD5.sampleid.31</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B_CD5.sampleid.34</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B_Memory.sampleid.29</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Array name  Batch\n",
       "0     B_CD5.sampleid.26      1\n",
       "1      B_CD5.sampleid.3      1\n",
       "2     B_CD5.sampleid.31      1\n",
       "3     B_CD5.sampleid.34      1\n",
       "4  B_Memory.sampleid.29      1"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "single_origin_batch = pd.DataFrame({'Array name': single_origin_cols})\n",
    "single_origin_batch['Batch'] = 1\n",
    "single_origin_batch.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:08:22.511300Z",
     "start_time": "2017-11-04T04:08:22.470267Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(142, 2)\n",
      "2    79\n",
      "1    63\n",
      "Name: Batch, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Array name</th>\n",
       "      <th>Sample name</th>\n",
       "      <th>Batch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>df66c814.1407.4899.a6e3.9c2a928e6ea0</td>\n",
       "      <td>df66c814.1407.4899.a6e3.9c2a928e6ea0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</td>\n",
       "      <td>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>X87c1d789.9482.442a.8b42.97bb479060e3</td>\n",
       "      <td>X87c1d789.9482.442a.8b42.97bb479060e3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</td>\n",
       "      <td>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Array name  \\\n",
       "0  X953473f4.9927.4fd4.bfee.8f5908638cc7   \n",
       "1   df66c814.1407.4899.a6e3.9c2a928e6ea0   \n",
       "2  X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d   \n",
       "3  X87c1d789.9482.442a.8b42.97bb479060e3   \n",
       "4  X4ddfd589.5a62.4149.9ccc.6a539f0b37b7   \n",
       "\n",
       "                             Sample name  Batch  \n",
       "0  X953473f4.9927.4fd4.bfee.8f5908638cc7      2  \n",
       "1   df66c814.1407.4899.a6e3.9c2a928e6ea0      2  \n",
       "2  X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d      2  \n",
       "3  X87c1d789.9482.442a.8b42.97bb479060e3      2  \n",
       "4  X4ddfd589.5a62.4149.9ccc.6a539f0b37b7      2  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_sample_info = pd.concat([cohort_batch, single_origin_batch])\n",
    "print(all_sample_info.shape)\n",
    "print(all_sample_info.Batch.value_counts())\n",
    "all_sample_info['Sample name'] = all_sample_info['Array name']\n",
    "all_sample_info = all_sample_info.loc[:, ['Array name', 'Sample name', 'Batch']] # rearrange just in case\n",
    "all_sample_info.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:08:40.039143Z",
     "start_time": "2017-11-04T04:08:40.030256Z"
    },
    "collapsed": true,
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "outputs": [],
   "source": [
    "all_sample_info.to_csv('infino-rcc/tcgakirc.sample_info_batches.combat_input.tsv', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run ComBat\n",
    "\n",
    "Done in `TCGA-KIRC 4 Run ComBat.ipynb` notebook. PCA before and after is shown there, too."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Load corrected values and exponentiate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:26:31.092517Z",
     "start_time": "2017-11-04T04:26:30.862087Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X953473f4.9927.4fd4.bfee.8f5908638cc7</th>\n",
       "      <th>df66c814.1407.4899.a6e3.9c2a928e6ea0</th>\n",
       "      <th>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</th>\n",
       "      <th>X87c1d789.9482.442a.8b42.97bb479060e3</th>\n",
       "      <th>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</th>\n",
       "      <th>X383ba5ca.1706.4959.b543.ea87fbd367bd</th>\n",
       "      <th>d9f44b98.58b7.4ebd.a6a6.8de472bf95a8</th>\n",
       "      <th>X550af27b.de50.402f.8a81.c508e3dc6781</th>\n",
       "      <th>c81f9d5b.f58d.411d.9e1a.f919a0ef5636</th>\n",
       "      <th>cf0052ae.8440.49df.a20c.f260980c9922</th>\n",
       "      <th>...</th>\n",
       "      <th>CD8_Effector.sampleid.2</th>\n",
       "      <th>CD8_Effector.sampleid.20</th>\n",
       "      <th>CD8_Effector.sampleid.25</th>\n",
       "      <th>CD8_Effector.sampleid.56</th>\n",
       "      <th>CD8_Effector.sampleid.8</th>\n",
       "      <th>CD8_Naive.sampleid.27</th>\n",
       "      <th>CD8_Naive.sampleid.40</th>\n",
       "      <th>CD8_Naive.sampleid.47</th>\n",
       "      <th>CD8_Naive.sampleid.58</th>\n",
       "      <th>CD8_Naive.sampleid.62</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ABCB4</th>\n",
       "      <td>5.604789</td>\n",
       "      <td>4.989663</td>\n",
       "      <td>5.441197</td>\n",
       "      <td>4.695368</td>\n",
       "      <td>3.206950</td>\n",
       "      <td>5.721657</td>\n",
       "      <td>3.990956</td>\n",
       "      <td>5.293097</td>\n",
       "      <td>5.204355</td>\n",
       "      <td>4.803009</td>\n",
       "      <td>...</td>\n",
       "      <td>3.045455</td>\n",
       "      <td>3.828179</td>\n",
       "      <td>3.723255</td>\n",
       "      <td>4.006253</td>\n",
       "      <td>2.721885</td>\n",
       "      <td>3.923108</td>\n",
       "      <td>4.216035</td>\n",
       "      <td>3.829148</td>\n",
       "      <td>3.453659</td>\n",
       "      <td>3.828179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCC4</th>\n",
       "      <td>6.723146</td>\n",
       "      <td>7.416427</td>\n",
       "      <td>5.762887</td>\n",
       "      <td>6.656285</td>\n",
       "      <td>6.501623</td>\n",
       "      <td>6.792601</td>\n",
       "      <td>5.578697</td>\n",
       "      <td>7.299733</td>\n",
       "      <td>7.384469</td>\n",
       "      <td>6.945320</td>\n",
       "      <td>...</td>\n",
       "      <td>6.410316</td>\n",
       "      <td>6.040199</td>\n",
       "      <td>7.119551</td>\n",
       "      <td>6.418939</td>\n",
       "      <td>7.070746</td>\n",
       "      <td>6.902028</td>\n",
       "      <td>6.112029</td>\n",
       "      <td>5.972084</td>\n",
       "      <td>6.610424</td>\n",
       "      <td>7.205973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCG2</th>\n",
       "      <td>6.101578</td>\n",
       "      <td>6.699815</td>\n",
       "      <td>4.599124</td>\n",
       "      <td>5.950794</td>\n",
       "      <td>5.061290</td>\n",
       "      <td>5.557687</td>\n",
       "      <td>6.169653</td>\n",
       "      <td>4.768931</td>\n",
       "      <td>5.999285</td>\n",
       "      <td>3.386272</td>\n",
       "      <td>...</td>\n",
       "      <td>3.231073</td>\n",
       "      <td>6.076638</td>\n",
       "      <td>5.720532</td>\n",
       "      <td>5.249430</td>\n",
       "      <td>4.949900</td>\n",
       "      <td>5.693383</td>\n",
       "      <td>6.800186</td>\n",
       "      <td>6.232055</td>\n",
       "      <td>5.408317</td>\n",
       "      <td>5.060114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD2</th>\n",
       "      <td>8.297409</td>\n",
       "      <td>8.716740</td>\n",
       "      <td>7.644649</td>\n",
       "      <td>8.264061</td>\n",
       "      <td>7.220558</td>\n",
       "      <td>8.116327</td>\n",
       "      <td>7.344384</td>\n",
       "      <td>8.562640</td>\n",
       "      <td>8.593636</td>\n",
       "      <td>8.467564</td>\n",
       "      <td>...</td>\n",
       "      <td>7.483745</td>\n",
       "      <td>8.625189</td>\n",
       "      <td>8.123277</td>\n",
       "      <td>8.303010</td>\n",
       "      <td>7.758614</td>\n",
       "      <td>7.594671</td>\n",
       "      <td>8.786192</td>\n",
       "      <td>7.820276</td>\n",
       "      <td>7.983917</td>\n",
       "      <td>7.456407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD5</th>\n",
       "      <td>5.445609</td>\n",
       "      <td>6.019959</td>\n",
       "      <td>4.747008</td>\n",
       "      <td>5.668057</td>\n",
       "      <td>5.292061</td>\n",
       "      <td>5.867080</td>\n",
       "      <td>4.839893</td>\n",
       "      <td>5.278973</td>\n",
       "      <td>5.407671</td>\n",
       "      <td>5.251516</td>\n",
       "      <td>...</td>\n",
       "      <td>5.544617</td>\n",
       "      <td>5.649848</td>\n",
       "      <td>5.337894</td>\n",
       "      <td>5.481953</td>\n",
       "      <td>5.322352</td>\n",
       "      <td>5.322353</td>\n",
       "      <td>5.052832</td>\n",
       "      <td>4.460813</td>\n",
       "      <td>5.169545</td>\n",
       "      <td>5.153469</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 142 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       X953473f4.9927.4fd4.bfee.8f5908638cc7  \\\n",
       "ABCB4                               5.604789   \n",
       "ABCC4                               6.723146   \n",
       "ABCG2                               6.101578   \n",
       "ABHD2                               8.297409   \n",
       "ABHD5                               5.445609   \n",
       "\n",
       "       df66c814.1407.4899.a6e3.9c2a928e6ea0  \\\n",
       "ABCB4                              4.989663   \n",
       "ABCC4                              7.416427   \n",
       "ABCG2                              6.699815   \n",
       "ABHD2                              8.716740   \n",
       "ABHD5                              6.019959   \n",
       "\n",
       "       X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d  \\\n",
       "ABCB4                               5.441197   \n",
       "ABCC4                               5.762887   \n",
       "ABCG2                               4.599124   \n",
       "ABHD2                               7.644649   \n",
       "ABHD5                               4.747008   \n",
       "\n",
       "       X87c1d789.9482.442a.8b42.97bb479060e3  \\\n",
       "ABCB4                               4.695368   \n",
       "ABCC4                               6.656285   \n",
       "ABCG2                               5.950794   \n",
       "ABHD2                               8.264061   \n",
       "ABHD5                               5.668057   \n",
       "\n",
       "       X4ddfd589.5a62.4149.9ccc.6a539f0b37b7  \\\n",
       "ABCB4                               3.206950   \n",
       "ABCC4                               6.501623   \n",
       "ABCG2                               5.061290   \n",
       "ABHD2                               7.220558   \n",
       "ABHD5                               5.292061   \n",
       "\n",
       "       X383ba5ca.1706.4959.b543.ea87fbd367bd  \\\n",
       "ABCB4                               5.721657   \n",
       "ABCC4                               6.792601   \n",
       "ABCG2                               5.557687   \n",
       "ABHD2                               8.116327   \n",
       "ABHD5                               5.867080   \n",
       "\n",
       "       d9f44b98.58b7.4ebd.a6a6.8de472bf95a8  \\\n",
       "ABCB4                              3.990956   \n",
       "ABCC4                              5.578697   \n",
       "ABCG2                              6.169653   \n",
       "ABHD2                              7.344384   \n",
       "ABHD5                              4.839893   \n",
       "\n",
       "       X550af27b.de50.402f.8a81.c508e3dc6781  \\\n",
       "ABCB4                               5.293097   \n",
       "ABCC4                               7.299733   \n",
       "ABCG2                               4.768931   \n",
       "ABHD2                               8.562640   \n",
       "ABHD5                               5.278973   \n",
       "\n",
       "       c81f9d5b.f58d.411d.9e1a.f919a0ef5636  \\\n",
       "ABCB4                              5.204355   \n",
       "ABCC4                              7.384469   \n",
       "ABCG2                              5.999285   \n",
       "ABHD2                              8.593636   \n",
       "ABHD5                              5.407671   \n",
       "\n",
       "       cf0052ae.8440.49df.a20c.f260980c9922          ...            \\\n",
       "ABCB4                              4.803009          ...             \n",
       "ABCC4                              6.945320          ...             \n",
       "ABCG2                              3.386272          ...             \n",
       "ABHD2                              8.467564          ...             \n",
       "ABHD5                              5.251516          ...             \n",
       "\n",
       "       CD8_Effector.sampleid.2  CD8_Effector.sampleid.20  \\\n",
       "ABCB4                 3.045455                  3.828179   \n",
       "ABCC4                 6.410316                  6.040199   \n",
       "ABCG2                 3.231073                  6.076638   \n",
       "ABHD2                 7.483745                  8.625189   \n",
       "ABHD5                 5.544617                  5.649848   \n",
       "\n",
       "       CD8_Effector.sampleid.25  CD8_Effector.sampleid.56  \\\n",
       "ABCB4                  3.723255                  4.006253   \n",
       "ABCC4                  7.119551                  6.418939   \n",
       "ABCG2                  5.720532                  5.249430   \n",
       "ABHD2                  8.123277                  8.303010   \n",
       "ABHD5                  5.337894                  5.481953   \n",
       "\n",
       "       CD8_Effector.sampleid.8  CD8_Naive.sampleid.27  CD8_Naive.sampleid.40  \\\n",
       "ABCB4                 2.721885               3.923108               4.216035   \n",
       "ABCC4                 7.070746               6.902028               6.112029   \n",
       "ABCG2                 4.949900               5.693383               6.800186   \n",
       "ABHD2                 7.758614               7.594671               8.786192   \n",
       "ABHD5                 5.322352               5.322353               5.052832   \n",
       "\n",
       "       CD8_Naive.sampleid.47  CD8_Naive.sampleid.58  CD8_Naive.sampleid.62  \n",
       "ABCB4               3.829148               3.453659               3.828179  \n",
       "ABCC4               5.972084               6.610424               7.205973  \n",
       "ABCG2               6.232055               5.408317               5.060114  \n",
       "ABHD2               7.820276               7.983917               7.456407  \n",
       "ABHD5               4.460813               5.169545               5.153469  \n",
       "\n",
       "[5 rows x 142 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combat_output_counts = pd.read_csv('infino-rcc/tcgakirc.combat_output.counts.tsv', sep='\\t')\n",
    "combat_output_tpm = pd.read_csv('infino-rcc/tcgakirc.combat_output.tpm.tsv', sep='\\t')\n",
    "combat_output_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:26:33.591595Z",
     "start_time": "2017-11-04T04:26:33.581592Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 142)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combat_output_counts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:26:34.728642Z",
     "start_time": "2017-11-04T04:26:34.359948Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X953473f4.9927.4fd4.bfee.8f5908638cc7</th>\n",
       "      <th>df66c814.1407.4899.a6e3.9c2a928e6ea0</th>\n",
       "      <th>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</th>\n",
       "      <th>X87c1d789.9482.442a.8b42.97bb479060e3</th>\n",
       "      <th>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</th>\n",
       "      <th>X383ba5ca.1706.4959.b543.ea87fbd367bd</th>\n",
       "      <th>d9f44b98.58b7.4ebd.a6a6.8de472bf95a8</th>\n",
       "      <th>X550af27b.de50.402f.8a81.c508e3dc6781</th>\n",
       "      <th>c81f9d5b.f58d.411d.9e1a.f919a0ef5636</th>\n",
       "      <th>cf0052ae.8440.49df.a20c.f260980c9922</th>\n",
       "      <th>...</th>\n",
       "      <th>CD8_Effector.sampleid.2</th>\n",
       "      <th>CD8_Effector.sampleid.20</th>\n",
       "      <th>CD8_Effector.sampleid.25</th>\n",
       "      <th>CD8_Effector.sampleid.56</th>\n",
       "      <th>CD8_Effector.sampleid.8</th>\n",
       "      <th>CD8_Naive.sampleid.27</th>\n",
       "      <th>CD8_Naive.sampleid.40</th>\n",
       "      <th>CD8_Naive.sampleid.47</th>\n",
       "      <th>CD8_Naive.sampleid.58</th>\n",
       "      <th>CD8_Naive.sampleid.62</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>5.225500</td>\n",
       "      <td>5.593956</td>\n",
       "      <td>4.679717</td>\n",
       "      <td>5.517372</td>\n",
       "      <td>4.308833</td>\n",
       "      <td>5.343265</td>\n",
       "      <td>4.648263</td>\n",
       "      <td>5.305688</td>\n",
       "      <td>5.204240</td>\n",
       "      <td>5.356100</td>\n",
       "      <td>...</td>\n",
       "      <td>4.699433</td>\n",
       "      <td>5.464439</td>\n",
       "      <td>5.366999</td>\n",
       "      <td>5.810710</td>\n",
       "      <td>5.294760</td>\n",
       "      <td>4.930439</td>\n",
       "      <td>4.974777</td>\n",
       "      <td>4.877542</td>\n",
       "      <td>4.944777</td>\n",
       "      <td>4.939211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.542453</td>\n",
       "      <td>2.616350</td>\n",
       "      <td>2.466029</td>\n",
       "      <td>2.503334</td>\n",
       "      <td>2.727410</td>\n",
       "      <td>2.493382</td>\n",
       "      <td>2.467800</td>\n",
       "      <td>2.536932</td>\n",
       "      <td>2.551166</td>\n",
       "      <td>2.435279</td>\n",
       "      <td>...</td>\n",
       "      <td>2.348580</td>\n",
       "      <td>2.376212</td>\n",
       "      <td>2.590956</td>\n",
       "      <td>2.118447</td>\n",
       "      <td>2.541979</td>\n",
       "      <td>2.509687</td>\n",
       "      <td>2.578924</td>\n",
       "      <td>2.503913</td>\n",
       "      <td>2.448534</td>\n",
       "      <td>2.607432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-0.601396</td>\n",
       "      <td>-0.344818</td>\n",
       "      <td>-0.562608</td>\n",
       "      <td>-0.609586</td>\n",
       "      <td>-1.604174</td>\n",
       "      <td>-0.578340</td>\n",
       "      <td>-0.649708</td>\n",
       "      <td>-0.422892</td>\n",
       "      <td>-0.423921</td>\n",
       "      <td>-0.107256</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.285562</td>\n",
       "      <td>-0.197449</td>\n",
       "      <td>-0.887762</td>\n",
       "      <td>-0.459678</td>\n",
       "      <td>-0.459678</td>\n",
       "      <td>-0.241122</td>\n",
       "      <td>-0.453357</td>\n",
       "      <td>-0.418825</td>\n",
       "      <td>-0.453357</td>\n",
       "      <td>-0.887762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>3.598187</td>\n",
       "      <td>3.951919</td>\n",
       "      <td>3.081598</td>\n",
       "      <td>4.058520</td>\n",
       "      <td>1.953967</td>\n",
       "      <td>3.922753</td>\n",
       "      <td>2.904010</td>\n",
       "      <td>3.741660</td>\n",
       "      <td>3.583875</td>\n",
       "      <td>3.816684</td>\n",
       "      <td>...</td>\n",
       "      <td>3.194402</td>\n",
       "      <td>4.074219</td>\n",
       "      <td>3.670891</td>\n",
       "      <td>4.770627</td>\n",
       "      <td>3.478867</td>\n",
       "      <td>3.156217</td>\n",
       "      <td>3.085942</td>\n",
       "      <td>3.215449</td>\n",
       "      <td>3.283113</td>\n",
       "      <td>3.087806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>5.694761</td>\n",
       "      <td>6.097771</td>\n",
       "      <td>5.069787</td>\n",
       "      <td>5.953220</td>\n",
       "      <td>4.526278</td>\n",
       "      <td>5.860035</td>\n",
       "      <td>5.187832</td>\n",
       "      <td>5.917613</td>\n",
       "      <td>5.801557</td>\n",
       "      <td>5.825126</td>\n",
       "      <td>...</td>\n",
       "      <td>5.117267</td>\n",
       "      <td>5.964332</td>\n",
       "      <td>5.902906</td>\n",
       "      <td>6.274942</td>\n",
       "      <td>5.823063</td>\n",
       "      <td>5.320128</td>\n",
       "      <td>5.346794</td>\n",
       "      <td>5.245195</td>\n",
       "      <td>5.354794</td>\n",
       "      <td>5.409009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.216736</td>\n",
       "      <td>7.706026</td>\n",
       "      <td>6.652118</td>\n",
       "      <td>7.486979</td>\n",
       "      <td>6.475770</td>\n",
       "      <td>7.312470</td>\n",
       "      <td>6.656430</td>\n",
       "      <td>7.232208</td>\n",
       "      <td>7.218129</td>\n",
       "      <td>7.195161</td>\n",
       "      <td>...</td>\n",
       "      <td>6.569283</td>\n",
       "      <td>7.312682</td>\n",
       "      <td>7.487353</td>\n",
       "      <td>7.311775</td>\n",
       "      <td>7.348365</td>\n",
       "      <td>6.946547</td>\n",
       "      <td>7.126563</td>\n",
       "      <td>6.935408</td>\n",
       "      <td>6.915119</td>\n",
       "      <td>7.090236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>10.797200</td>\n",
       "      <td>10.562137</td>\n",
       "      <td>10.412852</td>\n",
       "      <td>10.455643</td>\n",
       "      <td>10.463155</td>\n",
       "      <td>10.487673</td>\n",
       "      <td>9.884699</td>\n",
       "      <td>10.583364</td>\n",
       "      <td>10.240961</td>\n",
       "      <td>12.093059</td>\n",
       "      <td>...</td>\n",
       "      <td>9.256282</td>\n",
       "      <td>10.474259</td>\n",
       "      <td>10.762238</td>\n",
       "      <td>10.410330</td>\n",
       "      <td>10.636382</td>\n",
       "      <td>10.735853</td>\n",
       "      <td>11.042533</td>\n",
       "      <td>10.452145</td>\n",
       "      <td>10.789398</td>\n",
       "      <td>10.699343</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 142 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       X953473f4.9927.4fd4.bfee.8f5908638cc7  \\\n",
       "count                             900.000000   \n",
       "mean                                5.225500   \n",
       "std                                 2.542453   \n",
       "min                                -0.601396   \n",
       "25%                                 3.598187   \n",
       "50%                                 5.694761   \n",
       "75%                                 7.216736   \n",
       "max                                10.797200   \n",
       "\n",
       "       df66c814.1407.4899.a6e3.9c2a928e6ea0  \\\n",
       "count                            900.000000   \n",
       "mean                               5.593956   \n",
       "std                                2.616350   \n",
       "min                               -0.344818   \n",
       "25%                                3.951919   \n",
       "50%                                6.097771   \n",
       "75%                                7.706026   \n",
       "max                               10.562137   \n",
       "\n",
       "       X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d  \\\n",
       "count                             900.000000   \n",
       "mean                                4.679717   \n",
       "std                                 2.466029   \n",
       "min                                -0.562608   \n",
       "25%                                 3.081598   \n",
       "50%                                 5.069787   \n",
       "75%                                 6.652118   \n",
       "max                                10.412852   \n",
       "\n",
       "       X87c1d789.9482.442a.8b42.97bb479060e3  \\\n",
       "count                             900.000000   \n",
       "mean                                5.517372   \n",
       "std                                 2.503334   \n",
       "min                                -0.609586   \n",
       "25%                                 4.058520   \n",
       "50%                                 5.953220   \n",
       "75%                                 7.486979   \n",
       "max                                10.455643   \n",
       "\n",
       "       X4ddfd589.5a62.4149.9ccc.6a539f0b37b7  \\\n",
       "count                             900.000000   \n",
       "mean                                4.308833   \n",
       "std                                 2.727410   \n",
       "min                                -1.604174   \n",
       "25%                                 1.953967   \n",
       "50%                                 4.526278   \n",
       "75%                                 6.475770   \n",
       "max                                10.463155   \n",
       "\n",
       "       X383ba5ca.1706.4959.b543.ea87fbd367bd  \\\n",
       "count                             900.000000   \n",
       "mean                                5.343265   \n",
       "std                                 2.493382   \n",
       "min                                -0.578340   \n",
       "25%                                 3.922753   \n",
       "50%                                 5.860035   \n",
       "75%                                 7.312470   \n",
       "max                                10.487673   \n",
       "\n",
       "       d9f44b98.58b7.4ebd.a6a6.8de472bf95a8  \\\n",
       "count                            900.000000   \n",
       "mean                               4.648263   \n",
       "std                                2.467800   \n",
       "min                               -0.649708   \n",
       "25%                                2.904010   \n",
       "50%                                5.187832   \n",
       "75%                                6.656430   \n",
       "max                                9.884699   \n",
       "\n",
       "       X550af27b.de50.402f.8a81.c508e3dc6781  \\\n",
       "count                             900.000000   \n",
       "mean                                5.305688   \n",
       "std                                 2.536932   \n",
       "min                                -0.422892   \n",
       "25%                                 3.741660   \n",
       "50%                                 5.917613   \n",
       "75%                                 7.232208   \n",
       "max                                10.583364   \n",
       "\n",
       "       c81f9d5b.f58d.411d.9e1a.f919a0ef5636  \\\n",
       "count                            900.000000   \n",
       "mean                               5.204240   \n",
       "std                                2.551166   \n",
       "min                               -0.423921   \n",
       "25%                                3.583875   \n",
       "50%                                5.801557   \n",
       "75%                                7.218129   \n",
       "max                               10.240961   \n",
       "\n",
       "       cf0052ae.8440.49df.a20c.f260980c9922          ...            \\\n",
       "count                            900.000000          ...             \n",
       "mean                               5.356100          ...             \n",
       "std                                2.435279          ...             \n",
       "min                               -0.107256          ...             \n",
       "25%                                3.816684          ...             \n",
       "50%                                5.825126          ...             \n",
       "75%                                7.195161          ...             \n",
       "max                               12.093059          ...             \n",
       "\n",
       "       CD8_Effector.sampleid.2  CD8_Effector.sampleid.20  \\\n",
       "count               900.000000                900.000000   \n",
       "mean                  4.699433                  5.464439   \n",
       "std                   2.348580                  2.376212   \n",
       "min                  -0.285562                 -0.197449   \n",
       "25%                   3.194402                  4.074219   \n",
       "50%                   5.117267                  5.964332   \n",
       "75%                   6.569283                  7.312682   \n",
       "max                   9.256282                 10.474259   \n",
       "\n",
       "       CD8_Effector.sampleid.25  CD8_Effector.sampleid.56  \\\n",
       "count                900.000000                900.000000   \n",
       "mean                   5.366999                  5.810710   \n",
       "std                    2.590956                  2.118447   \n",
       "min                   -0.887762                 -0.459678   \n",
       "25%                    3.670891                  4.770627   \n",
       "50%                    5.902906                  6.274942   \n",
       "75%                    7.487353                  7.311775   \n",
       "max                   10.762238                 10.410330   \n",
       "\n",
       "       CD8_Effector.sampleid.8  CD8_Naive.sampleid.27  CD8_Naive.sampleid.40  \\\n",
       "count               900.000000             900.000000             900.000000   \n",
       "mean                  5.294760               4.930439               4.974777   \n",
       "std                   2.541979               2.509687               2.578924   \n",
       "min                  -0.459678              -0.241122              -0.453357   \n",
       "25%                   3.478867               3.156217               3.085942   \n",
       "50%                   5.823063               5.320128               5.346794   \n",
       "75%                   7.348365               6.946547               7.126563   \n",
       "max                  10.636382              10.735853              11.042533   \n",
       "\n",
       "       CD8_Naive.sampleid.47  CD8_Naive.sampleid.58  CD8_Naive.sampleid.62  \n",
       "count             900.000000             900.000000             900.000000  \n",
       "mean                4.877542               4.944777               4.939211  \n",
       "std                 2.503913               2.448534               2.607432  \n",
       "min                -0.418825              -0.453357              -0.887762  \n",
       "25%                 3.215449               3.283113               3.087806  \n",
       "50%                 5.245195               5.354794               5.409009  \n",
       "75%                 6.935408               6.915119               7.090236  \n",
       "max                10.452145              10.789398              10.699343  \n",
       "\n",
       "[8 rows x 142 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combat_output_counts.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:26:36.826536Z",
     "start_time": "2017-11-04T04:26:36.438730Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X953473f4.9927.4fd4.bfee.8f5908638cc7</th>\n",
       "      <th>df66c814.1407.4899.a6e3.9c2a928e6ea0</th>\n",
       "      <th>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</th>\n",
       "      <th>X87c1d789.9482.442a.8b42.97bb479060e3</th>\n",
       "      <th>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</th>\n",
       "      <th>X383ba5ca.1706.4959.b543.ea87fbd367bd</th>\n",
       "      <th>d9f44b98.58b7.4ebd.a6a6.8de472bf95a8</th>\n",
       "      <th>X550af27b.de50.402f.8a81.c508e3dc6781</th>\n",
       "      <th>c81f9d5b.f58d.411d.9e1a.f919a0ef5636</th>\n",
       "      <th>cf0052ae.8440.49df.a20c.f260980c9922</th>\n",
       "      <th>...</th>\n",
       "      <th>CD8_Effector.sampleid.2</th>\n",
       "      <th>CD8_Effector.sampleid.20</th>\n",
       "      <th>CD8_Effector.sampleid.25</th>\n",
       "      <th>CD8_Effector.sampleid.56</th>\n",
       "      <th>CD8_Effector.sampleid.8</th>\n",
       "      <th>CD8_Naive.sampleid.27</th>\n",
       "      <th>CD8_Naive.sampleid.40</th>\n",
       "      <th>CD8_Naive.sampleid.47</th>\n",
       "      <th>CD8_Naive.sampleid.58</th>\n",
       "      <th>CD8_Naive.sampleid.62</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "      <td>900.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1254.533980</td>\n",
       "      <td>1814.603845</td>\n",
       "      <td>780.175103</td>\n",
       "      <td>1446.654069</td>\n",
       "      <td>986.507633</td>\n",
       "      <td>1232.324898</td>\n",
       "      <td>664.985031</td>\n",
       "      <td>1247.238477</td>\n",
       "      <td>1177.216487</td>\n",
       "      <td>1738.857006</td>\n",
       "      <td>...</td>\n",
       "      <td>626.760807</td>\n",
       "      <td>1252.327774</td>\n",
       "      <td>1420.743040</td>\n",
       "      <td>1252.577211</td>\n",
       "      <td>1265.957328</td>\n",
       "      <td>1039.660490</td>\n",
       "      <td>1091.234611</td>\n",
       "      <td>916.269872</td>\n",
       "      <td>929.744209</td>\n",
       "      <td>1074.444233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2811.134086</td>\n",
       "      <td>3451.976535</td>\n",
       "      <td>2238.436478</td>\n",
       "      <td>2731.699620</td>\n",
       "      <td>2713.992759</td>\n",
       "      <td>2427.228811</td>\n",
       "      <td>1377.518198</td>\n",
       "      <td>2513.193712</td>\n",
       "      <td>2324.026540</td>\n",
       "      <td>9633.185147</td>\n",
       "      <td>...</td>\n",
       "      <td>1201.947019</td>\n",
       "      <td>2500.317700</td>\n",
       "      <td>2856.587494</td>\n",
       "      <td>2186.208727</td>\n",
       "      <td>2548.184928</td>\n",
       "      <td>2615.203534</td>\n",
       "      <td>2781.128096</td>\n",
       "      <td>1999.159231</td>\n",
       "      <td>2403.555375</td>\n",
       "      <td>2499.270210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.548046</td>\n",
       "      <td>0.708350</td>\n",
       "      <td>0.569721</td>\n",
       "      <td>0.543576</td>\n",
       "      <td>0.201056</td>\n",
       "      <td>0.560829</td>\n",
       "      <td>0.522198</td>\n",
       "      <td>0.655149</td>\n",
       "      <td>0.654476</td>\n",
       "      <td>0.898295</td>\n",
       "      <td>...</td>\n",
       "      <td>0.751592</td>\n",
       "      <td>0.820822</td>\n",
       "      <td>0.411576</td>\n",
       "      <td>0.631487</td>\n",
       "      <td>0.631487</td>\n",
       "      <td>0.785746</td>\n",
       "      <td>0.635491</td>\n",
       "      <td>0.657819</td>\n",
       "      <td>0.635491</td>\n",
       "      <td>0.411576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>36.533780</td>\n",
       "      <td>52.035263</td>\n",
       "      <td>21.794908</td>\n",
       "      <td>57.888842</td>\n",
       "      <td>7.056647</td>\n",
       "      <td>50.539526</td>\n",
       "      <td>18.248215</td>\n",
       "      <td>42.169250</td>\n",
       "      <td>36.012862</td>\n",
       "      <td>45.453747</td>\n",
       "      <td>...</td>\n",
       "      <td>24.396603</td>\n",
       "      <td>58.804748</td>\n",
       "      <td>39.287328</td>\n",
       "      <td>117.993219</td>\n",
       "      <td>32.423489</td>\n",
       "      <td>23.481897</td>\n",
       "      <td>21.888658</td>\n",
       "      <td>24.915260</td>\n",
       "      <td>26.659918</td>\n",
       "      <td>21.931058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>297.305661</td>\n",
       "      <td>444.865143</td>\n",
       "      <td>159.144474</td>\n",
       "      <td>384.991670</td>\n",
       "      <td>92.414841</td>\n",
       "      <td>350.737484</td>\n",
       "      <td>179.079938</td>\n",
       "      <td>371.523731</td>\n",
       "      <td>330.814972</td>\n",
       "      <td>338.704174</td>\n",
       "      <td>...</td>\n",
       "      <td>166.881342</td>\n",
       "      <td>389.292803</td>\n",
       "      <td>366.103267</td>\n",
       "      <td>531.095800</td>\n",
       "      <td>338.007419</td>\n",
       "      <td>204.410563</td>\n",
       "      <td>209.934387</td>\n",
       "      <td>189.653480</td>\n",
       "      <td>211.623345</td>\n",
       "      <td>223.411093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1362.043620</td>\n",
       "      <td>2221.712031</td>\n",
       "      <td>774.422710</td>\n",
       "      <td>1784.653452</td>\n",
       "      <td>649.223422</td>\n",
       "      <td>1499.026674</td>\n",
       "      <td>777.772538</td>\n",
       "      <td>1383.276020</td>\n",
       "      <td>1363.954533</td>\n",
       "      <td>1332.966279</td>\n",
       "      <td>...</td>\n",
       "      <td>712.864121</td>\n",
       "      <td>1499.192509</td>\n",
       "      <td>1785.349533</td>\n",
       "      <td>1497.835210</td>\n",
       "      <td>1553.654914</td>\n",
       "      <td>1039.554445</td>\n",
       "      <td>1244.621127</td>\n",
       "      <td>1028.041509</td>\n",
       "      <td>1007.390978</td>\n",
       "      <td>1200.192452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>48883.729250</td>\n",
       "      <td>38643.615155</td>\n",
       "      <td>33284.668152</td>\n",
       "      <td>34739.876432</td>\n",
       "      <td>35001.815403</td>\n",
       "      <td>35870.574107</td>\n",
       "      <td>19627.739618</td>\n",
       "      <td>39472.679945</td>\n",
       "      <td>28028.061287</td>\n",
       "      <td>178627.772183</td>\n",
       "      <td>...</td>\n",
       "      <td>10470.133078</td>\n",
       "      <td>35392.619824</td>\n",
       "      <td>47204.177683</td>\n",
       "      <td>33200.809120</td>\n",
       "      <td>41621.893408</td>\n",
       "      <td>45975.016213</td>\n",
       "      <td>62475.689220</td>\n",
       "      <td>34618.561922</td>\n",
       "      <td>48503.824317</td>\n",
       "      <td>44326.726783</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 142 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       X953473f4.9927.4fd4.bfee.8f5908638cc7  \\\n",
       "count                             900.000000   \n",
       "mean                             1254.533980   \n",
       "std                              2811.134086   \n",
       "min                                 0.548046   \n",
       "25%                                36.533780   \n",
       "50%                               297.305661   \n",
       "75%                              1362.043620   \n",
       "max                             48883.729250   \n",
       "\n",
       "       df66c814.1407.4899.a6e3.9c2a928e6ea0  \\\n",
       "count                            900.000000   \n",
       "mean                            1814.603845   \n",
       "std                             3451.976535   \n",
       "min                                0.708350   \n",
       "25%                               52.035263   \n",
       "50%                              444.865143   \n",
       "75%                             2221.712031   \n",
       "max                            38643.615155   \n",
       "\n",
       "       X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d  \\\n",
       "count                             900.000000   \n",
       "mean                              780.175103   \n",
       "std                              2238.436478   \n",
       "min                                 0.569721   \n",
       "25%                                21.794908   \n",
       "50%                               159.144474   \n",
       "75%                               774.422710   \n",
       "max                             33284.668152   \n",
       "\n",
       "       X87c1d789.9482.442a.8b42.97bb479060e3  \\\n",
       "count                             900.000000   \n",
       "mean                             1446.654069   \n",
       "std                              2731.699620   \n",
       "min                                 0.543576   \n",
       "25%                                57.888842   \n",
       "50%                               384.991670   \n",
       "75%                              1784.653452   \n",
       "max                             34739.876432   \n",
       "\n",
       "       X4ddfd589.5a62.4149.9ccc.6a539f0b37b7  \\\n",
       "count                             900.000000   \n",
       "mean                              986.507633   \n",
       "std                              2713.992759   \n",
       "min                                 0.201056   \n",
       "25%                                 7.056647   \n",
       "50%                                92.414841   \n",
       "75%                               649.223422   \n",
       "max                             35001.815403   \n",
       "\n",
       "       X383ba5ca.1706.4959.b543.ea87fbd367bd  \\\n",
       "count                             900.000000   \n",
       "mean                             1232.324898   \n",
       "std                              2427.228811   \n",
       "min                                 0.560829   \n",
       "25%                                50.539526   \n",
       "50%                               350.737484   \n",
       "75%                              1499.026674   \n",
       "max                             35870.574107   \n",
       "\n",
       "       d9f44b98.58b7.4ebd.a6a6.8de472bf95a8  \\\n",
       "count                            900.000000   \n",
       "mean                             664.985031   \n",
       "std                             1377.518198   \n",
       "min                                0.522198   \n",
       "25%                               18.248215   \n",
       "50%                              179.079938   \n",
       "75%                              777.772538   \n",
       "max                            19627.739618   \n",
       "\n",
       "       X550af27b.de50.402f.8a81.c508e3dc6781  \\\n",
       "count                             900.000000   \n",
       "mean                             1247.238477   \n",
       "std                              2513.193712   \n",
       "min                                 0.655149   \n",
       "25%                                42.169250   \n",
       "50%                               371.523731   \n",
       "75%                              1383.276020   \n",
       "max                             39472.679945   \n",
       "\n",
       "       c81f9d5b.f58d.411d.9e1a.f919a0ef5636  \\\n",
       "count                            900.000000   \n",
       "mean                            1177.216487   \n",
       "std                             2324.026540   \n",
       "min                                0.654476   \n",
       "25%                               36.012862   \n",
       "50%                              330.814972   \n",
       "75%                             1363.954533   \n",
       "max                            28028.061287   \n",
       "\n",
       "       cf0052ae.8440.49df.a20c.f260980c9922          ...            \\\n",
       "count                            900.000000          ...             \n",
       "mean                            1738.857006          ...             \n",
       "std                             9633.185147          ...             \n",
       "min                                0.898295          ...             \n",
       "25%                               45.453747          ...             \n",
       "50%                              338.704174          ...             \n",
       "75%                             1332.966279          ...             \n",
       "max                           178627.772183          ...             \n",
       "\n",
       "       CD8_Effector.sampleid.2  CD8_Effector.sampleid.20  \\\n",
       "count               900.000000                900.000000   \n",
       "mean                626.760807               1252.327774   \n",
       "std                1201.947019               2500.317700   \n",
       "min                   0.751592                  0.820822   \n",
       "25%                  24.396603                 58.804748   \n",
       "50%                 166.881342                389.292803   \n",
       "75%                 712.864121               1499.192509   \n",
       "max               10470.133078              35392.619824   \n",
       "\n",
       "       CD8_Effector.sampleid.25  CD8_Effector.sampleid.56  \\\n",
       "count                900.000000                900.000000   \n",
       "mean                1420.743040               1252.577211   \n",
       "std                 2856.587494               2186.208727   \n",
       "min                    0.411576                  0.631487   \n",
       "25%                   39.287328                117.993219   \n",
       "50%                  366.103267                531.095800   \n",
       "75%                 1785.349533               1497.835210   \n",
       "max                47204.177683              33200.809120   \n",
       "\n",
       "       CD8_Effector.sampleid.8  CD8_Naive.sampleid.27  CD8_Naive.sampleid.40  \\\n",
       "count               900.000000             900.000000             900.000000   \n",
       "mean               1265.957328            1039.660490            1091.234611   \n",
       "std                2548.184928            2615.203534            2781.128096   \n",
       "min                   0.631487               0.785746               0.635491   \n",
       "25%                  32.423489              23.481897              21.888658   \n",
       "50%                 338.007419             204.410563             209.934387   \n",
       "75%                1553.654914            1039.554445            1244.621127   \n",
       "max               41621.893408           45975.016213           62475.689220   \n",
       "\n",
       "       CD8_Naive.sampleid.47  CD8_Naive.sampleid.58  CD8_Naive.sampleid.62  \n",
       "count             900.000000             900.000000             900.000000  \n",
       "mean              916.269872             929.744209            1074.444233  \n",
       "std              1999.159231            2403.555375            2499.270210  \n",
       "min                 0.657819               0.635491               0.411576  \n",
       "25%                24.915260              26.659918              21.931058  \n",
       "50%               189.653480             211.623345             223.411093  \n",
       "75%              1028.041509            1007.390978            1200.192452  \n",
       "max             34618.561922           48503.824317           44326.726783  \n",
       "\n",
       "[8 rows x 142 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# exponentiate\n",
    "batch_corrected_counts = combat_output_counts.apply(np.exp)\n",
    "batch_corrected_tpm = combat_output_tpm.apply(np.exp)\n",
    "batch_corrected_counts.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:26:53.678348Z",
     "start_time": "2017-11-04T04:26:53.667672Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "assert all(batch_corrected_counts > 0)\n",
    "assert all(batch_corrected_tpm > 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:27:18.770422Z",
     "start_time": "2017-11-04T04:27:18.184850Z"
    },
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "batch_corrected_counts.to_csv('infino-rcc/tcgakirc.plus-singleorigin.counts.batch_corrected.tsv', sep='\\t')\n",
    "batch_corrected_tpm.to_csv('infino-rcc/tcgakirc.plus-singleorigin.tpm.batch_corrected.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Split into training and testing datasets to form Stan runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:27:20.647116Z",
     "start_time": "2017-11-04T04:27:20.578550Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X953473f4.9927.4fd4.bfee.8f5908638cc7</th>\n",
       "      <th>df66c814.1407.4899.a6e3.9c2a928e6ea0</th>\n",
       "      <th>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</th>\n",
       "      <th>X87c1d789.9482.442a.8b42.97bb479060e3</th>\n",
       "      <th>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</th>\n",
       "      <th>X383ba5ca.1706.4959.b543.ea87fbd367bd</th>\n",
       "      <th>d9f44b98.58b7.4ebd.a6a6.8de472bf95a8</th>\n",
       "      <th>X550af27b.de50.402f.8a81.c508e3dc6781</th>\n",
       "      <th>c81f9d5b.f58d.411d.9e1a.f919a0ef5636</th>\n",
       "      <th>cf0052ae.8440.49df.a20c.f260980c9922</th>\n",
       "      <th>...</th>\n",
       "      <th>CD8_Effector.sampleid.2</th>\n",
       "      <th>CD8_Effector.sampleid.20</th>\n",
       "      <th>CD8_Effector.sampleid.25</th>\n",
       "      <th>CD8_Effector.sampleid.56</th>\n",
       "      <th>CD8_Effector.sampleid.8</th>\n",
       "      <th>CD8_Naive.sampleid.27</th>\n",
       "      <th>CD8_Naive.sampleid.40</th>\n",
       "      <th>CD8_Naive.sampleid.47</th>\n",
       "      <th>CD8_Naive.sampleid.58</th>\n",
       "      <th>CD8_Naive.sampleid.62</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ABCB4</th>\n",
       "      <td>6.132064</td>\n",
       "      <td>2.804926</td>\n",
       "      <td>7.783835</td>\n",
       "      <td>3.056185</td>\n",
       "      <td>2.162606</td>\n",
       "      <td>8.955646</td>\n",
       "      <td>3.524200</td>\n",
       "      <td>5.674138</td>\n",
       "      <td>4.678708</td>\n",
       "      <td>3.775569</td>\n",
       "      <td>...</td>\n",
       "      <td>1.919938</td>\n",
       "      <td>2.037311</td>\n",
       "      <td>1.703170</td>\n",
       "      <td>2.636545</td>\n",
       "      <td>1.686988</td>\n",
       "      <td>1.819757</td>\n",
       "      <td>2.037381</td>\n",
       "      <td>1.843299</td>\n",
       "      <td>1.590187</td>\n",
       "      <td>2.490911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCC4</th>\n",
       "      <td>11.497252</td>\n",
       "      <td>15.794061</td>\n",
       "      <td>7.264524</td>\n",
       "      <td>10.438258</td>\n",
       "      <td>13.972745</td>\n",
       "      <td>13.085842</td>\n",
       "      <td>4.049046</td>\n",
       "      <td>16.932871</td>\n",
       "      <td>21.671690</td>\n",
       "      <td>11.883387</td>\n",
       "      <td>...</td>\n",
       "      <td>12.526329</td>\n",
       "      <td>6.477029</td>\n",
       "      <td>12.644819</td>\n",
       "      <td>12.652790</td>\n",
       "      <td>13.683508</td>\n",
       "      <td>10.401006</td>\n",
       "      <td>5.651743</td>\n",
       "      <td>4.174520</td>\n",
       "      <td>8.706744</td>\n",
       "      <td>9.697253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCG2</th>\n",
       "      <td>8.901503</td>\n",
       "      <td>10.482242</td>\n",
       "      <td>3.624627</td>\n",
       "      <td>7.413748</td>\n",
       "      <td>4.918174</td>\n",
       "      <td>5.694519</td>\n",
       "      <td>7.106305</td>\n",
       "      <td>3.082687</td>\n",
       "      <td>8.116328</td>\n",
       "      <td>1.348902</td>\n",
       "      <td>...</td>\n",
       "      <td>1.751195</td>\n",
       "      <td>5.436474</td>\n",
       "      <td>8.815472</td>\n",
       "      <td>3.040920</td>\n",
       "      <td>2.637813</td>\n",
       "      <td>3.750379</td>\n",
       "      <td>10.065195</td>\n",
       "      <td>6.692215</td>\n",
       "      <td>3.173498</td>\n",
       "      <td>2.900208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD2</th>\n",
       "      <td>29.926762</td>\n",
       "      <td>35.320411</td>\n",
       "      <td>23.053732</td>\n",
       "      <td>28.182072</td>\n",
       "      <td>16.851865</td>\n",
       "      <td>27.178261</td>\n",
       "      <td>11.327711</td>\n",
       "      <td>35.390610</td>\n",
       "      <td>43.885329</td>\n",
       "      <td>32.779586</td>\n",
       "      <td>...</td>\n",
       "      <td>54.414951</td>\n",
       "      <td>25.004480</td>\n",
       "      <td>12.530635</td>\n",
       "      <td>25.845545</td>\n",
       "      <td>8.743583</td>\n",
       "      <td>12.620173</td>\n",
       "      <td>27.631923</td>\n",
       "      <td>13.168362</td>\n",
       "      <td>13.204393</td>\n",
       "      <td>9.398938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD5</th>\n",
       "      <td>10.481962</td>\n",
       "      <td>13.037807</td>\n",
       "      <td>7.916075</td>\n",
       "      <td>12.449738</td>\n",
       "      <td>13.777537</td>\n",
       "      <td>16.412296</td>\n",
       "      <td>5.142477</td>\n",
       "      <td>8.114578</td>\n",
       "      <td>11.515093</td>\n",
       "      <td>7.621882</td>\n",
       "      <td>...</td>\n",
       "      <td>15.669661</td>\n",
       "      <td>9.155207</td>\n",
       "      <td>7.498200</td>\n",
       "      <td>9.146203</td>\n",
       "      <td>9.780008</td>\n",
       "      <td>7.055579</td>\n",
       "      <td>4.691434</td>\n",
       "      <td>3.457789</td>\n",
       "      <td>7.206154</td>\n",
       "      <td>8.047937</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 142 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       X953473f4.9927.4fd4.bfee.8f5908638cc7  \\\n",
       "ABCB4                               6.132064   \n",
       "ABCC4                              11.497252   \n",
       "ABCG2                               8.901503   \n",
       "ABHD2                              29.926762   \n",
       "ABHD5                              10.481962   \n",
       "\n",
       "       df66c814.1407.4899.a6e3.9c2a928e6ea0  \\\n",
       "ABCB4                              2.804926   \n",
       "ABCC4                             15.794061   \n",
       "ABCG2                             10.482242   \n",
       "ABHD2                             35.320411   \n",
       "ABHD5                             13.037807   \n",
       "\n",
       "       X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d  \\\n",
       "ABCB4                               7.783835   \n",
       "ABCC4                               7.264524   \n",
       "ABCG2                               3.624627   \n",
       "ABHD2                              23.053732   \n",
       "ABHD5                               7.916075   \n",
       "\n",
       "       X87c1d789.9482.442a.8b42.97bb479060e3  \\\n",
       "ABCB4                               3.056185   \n",
       "ABCC4                              10.438258   \n",
       "ABCG2                               7.413748   \n",
       "ABHD2                              28.182072   \n",
       "ABHD5                              12.449738   \n",
       "\n",
       "       X4ddfd589.5a62.4149.9ccc.6a539f0b37b7  \\\n",
       "ABCB4                               2.162606   \n",
       "ABCC4                              13.972745   \n",
       "ABCG2                               4.918174   \n",
       "ABHD2                              16.851865   \n",
       "ABHD5                              13.777537   \n",
       "\n",
       "       X383ba5ca.1706.4959.b543.ea87fbd367bd  \\\n",
       "ABCB4                               8.955646   \n",
       "ABCC4                              13.085842   \n",
       "ABCG2                               5.694519   \n",
       "ABHD2                              27.178261   \n",
       "ABHD5                              16.412296   \n",
       "\n",
       "       d9f44b98.58b7.4ebd.a6a6.8de472bf95a8  \\\n",
       "ABCB4                              3.524200   \n",
       "ABCC4                              4.049046   \n",
       "ABCG2                              7.106305   \n",
       "ABHD2                             11.327711   \n",
       "ABHD5                              5.142477   \n",
       "\n",
       "       X550af27b.de50.402f.8a81.c508e3dc6781  \\\n",
       "ABCB4                               5.674138   \n",
       "ABCC4                              16.932871   \n",
       "ABCG2                               3.082687   \n",
       "ABHD2                              35.390610   \n",
       "ABHD5                               8.114578   \n",
       "\n",
       "       c81f9d5b.f58d.411d.9e1a.f919a0ef5636  \\\n",
       "ABCB4                              4.678708   \n",
       "ABCC4                             21.671690   \n",
       "ABCG2                              8.116328   \n",
       "ABHD2                             43.885329   \n",
       "ABHD5                             11.515093   \n",
       "\n",
       "       cf0052ae.8440.49df.a20c.f260980c9922          ...            \\\n",
       "ABCB4                              3.775569          ...             \n",
       "ABCC4                             11.883387          ...             \n",
       "ABCG2                              1.348902          ...             \n",
       "ABHD2                             32.779586          ...             \n",
       "ABHD5                              7.621882          ...             \n",
       "\n",
       "       CD8_Effector.sampleid.2  CD8_Effector.sampleid.20  \\\n",
       "ABCB4                 1.919938                  2.037311   \n",
       "ABCC4                12.526329                  6.477029   \n",
       "ABCG2                 1.751195                  5.436474   \n",
       "ABHD2                54.414951                 25.004480   \n",
       "ABHD5                15.669661                  9.155207   \n",
       "\n",
       "       CD8_Effector.sampleid.25  CD8_Effector.sampleid.56  \\\n",
       "ABCB4                  1.703170                  2.636545   \n",
       "ABCC4                 12.644819                 12.652790   \n",
       "ABCG2                  8.815472                  3.040920   \n",
       "ABHD2                 12.530635                 25.845545   \n",
       "ABHD5                  7.498200                  9.146203   \n",
       "\n",
       "       CD8_Effector.sampleid.8  CD8_Naive.sampleid.27  CD8_Naive.sampleid.40  \\\n",
       "ABCB4                 1.686988               1.819757               2.037381   \n",
       "ABCC4                13.683508              10.401006               5.651743   \n",
       "ABCG2                 2.637813               3.750379              10.065195   \n",
       "ABHD2                 8.743583              12.620173              27.631923   \n",
       "ABHD5                 9.780008               7.055579               4.691434   \n",
       "\n",
       "       CD8_Naive.sampleid.47  CD8_Naive.sampleid.58  CD8_Naive.sampleid.62  \n",
       "ABCB4               1.843299               1.590187               2.490911  \n",
       "ABCC4               4.174520               8.706744               9.697253  \n",
       "ABCG2               6.692215               3.173498               2.900208  \n",
       "ABHD2              13.168362              13.204393               9.398938  \n",
       "ABHD5               3.457789               7.206154               8.047937  \n",
       "\n",
       "[5 rows x 142 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_corrected_tpm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:27:26.535930Z",
     "start_time": "2017-11-04T04:27:26.460990Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>B_CD5.sampleid.26</th>\n",
       "      <th>B_CD5.sampleid.3</th>\n",
       "      <th>B_CD5.sampleid.31</th>\n",
       "      <th>B_CD5.sampleid.34</th>\n",
       "      <th>B_Memory.sampleid.29</th>\n",
       "      <th>B_Memory.sampleid.4</th>\n",
       "      <th>B_Memory.sampleid.43</th>\n",
       "      <th>B_Memory.sampleid.48</th>\n",
       "      <th>B_Memory.sampleid.52</th>\n",
       "      <th>B_Naive.sampleid.21</th>\n",
       "      <th>...</th>\n",
       "      <th>CD8_Effector.sampleid.2</th>\n",
       "      <th>CD8_Effector.sampleid.20</th>\n",
       "      <th>CD8_Effector.sampleid.25</th>\n",
       "      <th>CD8_Effector.sampleid.56</th>\n",
       "      <th>CD8_Effector.sampleid.8</th>\n",
       "      <th>CD8_Naive.sampleid.27</th>\n",
       "      <th>CD8_Naive.sampleid.40</th>\n",
       "      <th>CD8_Naive.sampleid.47</th>\n",
       "      <th>CD8_Naive.sampleid.58</th>\n",
       "      <th>CD8_Naive.sampleid.62</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ABCB4</th>\n",
       "      <td>1996.463900</td>\n",
       "      <td>1783.028671</td>\n",
       "      <td>894.751436</td>\n",
       "      <td>1371.221289</td>\n",
       "      <td>424.493435</td>\n",
       "      <td>453.937718</td>\n",
       "      <td>1087.460971</td>\n",
       "      <td>1052.492393</td>\n",
       "      <td>317.249213</td>\n",
       "      <td>2820.729977</td>\n",
       "      <td>...</td>\n",
       "      <td>21.019595</td>\n",
       "      <td>45.978743</td>\n",
       "      <td>41.398947</td>\n",
       "      <td>54.940601</td>\n",
       "      <td>15.208961</td>\n",
       "      <td>50.557309</td>\n",
       "      <td>67.764243</td>\n",
       "      <td>46.023314</td>\n",
       "      <td>31.615869</td>\n",
       "      <td>45.978739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCC4</th>\n",
       "      <td>133.383296</td>\n",
       "      <td>107.210737</td>\n",
       "      <td>265.460036</td>\n",
       "      <td>141.186284</td>\n",
       "      <td>1082.829544</td>\n",
       "      <td>1373.263274</td>\n",
       "      <td>1063.116061</td>\n",
       "      <td>1115.301265</td>\n",
       "      <td>638.842814</td>\n",
       "      <td>213.115285</td>\n",
       "      <td>...</td>\n",
       "      <td>608.085830</td>\n",
       "      <td>419.976756</td>\n",
       "      <td>1235.895537</td>\n",
       "      <td>613.351975</td>\n",
       "      <td>1177.025372</td>\n",
       "      <td>994.288941</td>\n",
       "      <td>451.253410</td>\n",
       "      <td>392.322613</td>\n",
       "      <td>742.797719</td>\n",
       "      <td>1347.455032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCG2</th>\n",
       "      <td>75.165654</td>\n",
       "      <td>50.295542</td>\n",
       "      <td>182.251124</td>\n",
       "      <td>25.306785</td>\n",
       "      <td>33.654222</td>\n",
       "      <td>58.595936</td>\n",
       "      <td>41.982500</td>\n",
       "      <td>50.295542</td>\n",
       "      <td>16.933511</td>\n",
       "      <td>58.595936</td>\n",
       "      <td>...</td>\n",
       "      <td>25.306785</td>\n",
       "      <td>435.562388</td>\n",
       "      <td>305.067278</td>\n",
       "      <td>190.457758</td>\n",
       "      <td>141.160803</td>\n",
       "      <td>296.896265</td>\n",
       "      <td>898.014668</td>\n",
       "      <td>508.799851</td>\n",
       "      <td>223.255532</td>\n",
       "      <td>157.608463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD2</th>\n",
       "      <td>1732.682819</td>\n",
       "      <td>1687.424955</td>\n",
       "      <td>1526.681612</td>\n",
       "      <td>2028.915185</td>\n",
       "      <td>1846.911317</td>\n",
       "      <td>1816.427034</td>\n",
       "      <td>7027.470266</td>\n",
       "      <td>7790.440344</td>\n",
       "      <td>1161.320949</td>\n",
       "      <td>3464.394467</td>\n",
       "      <td>...</td>\n",
       "      <td>1778.891041</td>\n",
       "      <td>5570.216338</td>\n",
       "      <td>3372.054132</td>\n",
       "      <td>4036.000976</td>\n",
       "      <td>2341.657489</td>\n",
       "      <td>1987.576600</td>\n",
       "      <td>6543.265603</td>\n",
       "      <td>2490.592895</td>\n",
       "      <td>2933.398023</td>\n",
       "      <td>1730.917197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD5</th>\n",
       "      <td>249.670025</td>\n",
       "      <td>236.818182</td>\n",
       "      <td>73.254299</td>\n",
       "      <td>149.909264</td>\n",
       "      <td>271.653366</td>\n",
       "      <td>377.687279</td>\n",
       "      <td>960.849087</td>\n",
       "      <td>851.863885</td>\n",
       "      <td>211.279741</td>\n",
       "      <td>268.498534</td>\n",
       "      <td>...</td>\n",
       "      <td>255.856480</td>\n",
       "      <td>284.248342</td>\n",
       "      <td>208.074100</td>\n",
       "      <td>240.315583</td>\n",
       "      <td>204.865252</td>\n",
       "      <td>204.865413</td>\n",
       "      <td>156.464895</td>\n",
       "      <td>86.557860</td>\n",
       "      <td>175.834844</td>\n",
       "      <td>173.030744</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 63 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       B_CD5.sampleid.26  B_CD5.sampleid.3  B_CD5.sampleid.31  \\\n",
       "ABCB4        1996.463900       1783.028671         894.751436   \n",
       "ABCC4         133.383296        107.210737         265.460036   \n",
       "ABCG2          75.165654         50.295542         182.251124   \n",
       "ABHD2        1732.682819       1687.424955        1526.681612   \n",
       "ABHD5         249.670025        236.818182          73.254299   \n",
       "\n",
       "       B_CD5.sampleid.34  B_Memory.sampleid.29  B_Memory.sampleid.4  \\\n",
       "ABCB4        1371.221289            424.493435           453.937718   \n",
       "ABCC4         141.186284           1082.829544          1373.263274   \n",
       "ABCG2          25.306785             33.654222            58.595936   \n",
       "ABHD2        2028.915185           1846.911317          1816.427034   \n",
       "ABHD5         149.909264            271.653366           377.687279   \n",
       "\n",
       "       B_Memory.sampleid.43  B_Memory.sampleid.48  B_Memory.sampleid.52  \\\n",
       "ABCB4           1087.460971           1052.492393            317.249213   \n",
       "ABCC4           1063.116061           1115.301265            638.842814   \n",
       "ABCG2             41.982500             50.295542             16.933511   \n",
       "ABHD2           7027.470266           7790.440344           1161.320949   \n",
       "ABHD5            960.849087            851.863885            211.279741   \n",
       "\n",
       "       B_Naive.sampleid.21          ...            CD8_Effector.sampleid.2  \\\n",
       "ABCB4          2820.729977          ...                          21.019595   \n",
       "ABCC4           213.115285          ...                         608.085830   \n",
       "ABCG2            58.595936          ...                          25.306785   \n",
       "ABHD2          3464.394467          ...                        1778.891041   \n",
       "ABHD5           268.498534          ...                         255.856480   \n",
       "\n",
       "       CD8_Effector.sampleid.20  CD8_Effector.sampleid.25  \\\n",
       "ABCB4                 45.978743                 41.398947   \n",
       "ABCC4                419.976756               1235.895537   \n",
       "ABCG2                435.562388                305.067278   \n",
       "ABHD2               5570.216338               3372.054132   \n",
       "ABHD5                284.248342                208.074100   \n",
       "\n",
       "       CD8_Effector.sampleid.56  CD8_Effector.sampleid.8  \\\n",
       "ABCB4                 54.940601                15.208961   \n",
       "ABCC4                613.351975              1177.025372   \n",
       "ABCG2                190.457758               141.160803   \n",
       "ABHD2               4036.000976              2341.657489   \n",
       "ABHD5                240.315583               204.865252   \n",
       "\n",
       "       CD8_Naive.sampleid.27  CD8_Naive.sampleid.40  CD8_Naive.sampleid.47  \\\n",
       "ABCB4              50.557309              67.764243              46.023314   \n",
       "ABCC4             994.288941             451.253410             392.322613   \n",
       "ABCG2             296.896265             898.014668             508.799851   \n",
       "ABHD2            1987.576600            6543.265603            2490.592895   \n",
       "ABHD5             204.865413             156.464895              86.557860   \n",
       "\n",
       "       CD8_Naive.sampleid.58  CD8_Naive.sampleid.62  \n",
       "ABCB4              31.615869              45.978739  \n",
       "ABCC4             742.797719            1347.455032  \n",
       "ABCG2             223.255532             157.608463  \n",
       "ABHD2            2933.398023            1730.917197  \n",
       "ABHD5             175.834844             173.030744  \n",
       "\n",
       "[5 rows x 63 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract columns in training set\n",
    "single_origin_corrections_counts = batch_corrected_counts[single_origin_cols]\n",
    "single_origin_corrections_tpm = batch_corrected_tpm[single_origin_cols]\n",
    "single_origin_corrections_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:27:28.869355Z",
     "start_time": "2017-11-04T04:27:28.796144Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>sample_id</th>\n",
       "      <th>filename</th>\n",
       "      <th>gene_name</th>\n",
       "      <th>est_counts</th>\n",
       "      <th>tpm</th>\n",
       "      <th>log1p_tpm</th>\n",
       "      <th>log1p_counts</th>\n",
       "      <th>CCR6+</th>\n",
       "      <th>CCR7+</th>\n",
       "      <th>...</th>\n",
       "      <th>log1p_tpm_rescaled</th>\n",
       "      <th>gene_cat</th>\n",
       "      <th>gene_id</th>\n",
       "      <th>B_cell</th>\n",
       "      <th>T_cell</th>\n",
       "      <th>new_gene_cat</th>\n",
       "      <th>new_gene_id</th>\n",
       "      <th>new_sample_cat</th>\n",
       "      <th>new_sample_id</th>\n",
       "      <th>SubSet_and_sample</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4347</th>\n",
       "      <td>4347</td>\n",
       "      <td>1</td>\n",
       "      <td>ERR431566</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.851882</td>\n",
       "      <td>0.616202</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.188314</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>CD4_Th2.sampleid.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4348</th>\n",
       "      <td>4348</td>\n",
       "      <td>2</td>\n",
       "      <td>ERR431567</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.397441</td>\n",
       "      <td>0.334642</td>\n",
       "      <td>1.098612</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.636778</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>CD8_Effector.sampleid.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4349</th>\n",
       "      <td>4349</td>\n",
       "      <td>3</td>\n",
       "      <td>ERR431568</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>782.000170</td>\n",
       "      <td>28.925644</td>\n",
       "      <td>3.398716</td>\n",
       "      <td>6.663133</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.243620</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>B_CD5.sampleid.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4350</th>\n",
       "      <td>4350</td>\n",
       "      <td>4</td>\n",
       "      <td>ERR431569</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>139.999971</td>\n",
       "      <td>5.299960</td>\n",
       "      <td>1.840543</td>\n",
       "      <td>4.948760</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.761793</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>B_Memory.sampleid.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4351</th>\n",
       "      <td>4351</td>\n",
       "      <td>5</td>\n",
       "      <td>ERR431570</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>3.007070</td>\n",
       "      <td>0.197082</td>\n",
       "      <td>0.179887</td>\n",
       "      <td>1.388060</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.883270</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>CD4_Th1.sampleid.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index  sample_id   filename gene_name  est_counts        tpm  log1p_tpm  \\\n",
       "4347   4347          1  ERR431566     ABCB4    1.000000   0.851882   0.616202   \n",
       "4348   4348          2  ERR431567     ABCB4    2.000000   0.397441   0.334642   \n",
       "4349   4349          3  ERR431568     ABCB4  782.000170  28.925644   3.398716   \n",
       "4350   4350          4  ERR431569     ABCB4  139.999971   5.299960   1.840543   \n",
       "4351   4351          5  ERR431570     ABCB4    3.007070   0.197082   0.179887   \n",
       "\n",
       "      log1p_counts  CCR6+  CCR7+           ...             log1p_tpm_rescaled  \\\n",
       "4347      0.693147    0.0    0.0           ...                      -0.188314   \n",
       "4348      1.098612    0.0    0.0           ...                      -0.636778   \n",
       "4349      6.663133    0.0    0.0           ...                       4.243620   \n",
       "4350      4.948760    0.0    0.0           ...                       1.761793   \n",
       "4351      1.388060    0.0    0.0           ...                      -0.883270   \n",
       "\n",
       "      gene_cat  gene_id  B_cell  T_cell  new_gene_cat  new_gene_id  \\\n",
       "4347     ABCB4       70       0       1         ABCB4            1   \n",
       "4348     ABCB4       70       0       1         ABCB4            1   \n",
       "4349     ABCB4       70       1       0         ABCB4            1   \n",
       "4350     ABCB4       70       1       0         ABCB4            1   \n",
       "4351     ABCB4       70       0       1         ABCB4            1   \n",
       "\n",
       "      new_sample_cat  new_sample_id        SubSet_and_sample  \n",
       "4347               1              1       CD4_Th2.sampleid.1  \n",
       "4348               2              2  CD8_Effector.sampleid.2  \n",
       "4349               3              3         B_CD5.sampleid.3  \n",
       "4350               4              4      B_Memory.sampleid.4  \n",
       "4351               5              5       CD4_Th1.sampleid.5  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:27:33.425767Z",
     "start_time": "2017-11-04T04:27:33.369983Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene_name</th>\n",
       "      <th>SubSet_and_sample</th>\n",
       "      <th>tpm_corrected</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABCB4</td>\n",
       "      <td>B_CD5.sampleid.26</td>\n",
       "      <td>26.087430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ABCC4</td>\n",
       "      <td>B_CD5.sampleid.26</td>\n",
       "      <td>3.256069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ABCG2</td>\n",
       "      <td>B_CD5.sampleid.26</td>\n",
       "      <td>1.964039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ABHD2</td>\n",
       "      <td>B_CD5.sampleid.26</td>\n",
       "      <td>17.549267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ABHD5</td>\n",
       "      <td>B_CD5.sampleid.26</td>\n",
       "      <td>8.407926</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  gene_name  SubSet_and_sample  tpm_corrected\n",
       "0     ABCB4  B_CD5.sampleid.26      26.087430\n",
       "1     ABCC4  B_CD5.sampleid.26       3.256069\n",
       "2     ABCG2  B_CD5.sampleid.26       1.964039\n",
       "3     ABHD2  B_CD5.sampleid.26      17.549267\n",
       "4     ABHD5  B_CD5.sampleid.26       8.407926"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# take training_df existing\n",
    "# join by new_sample_id and gene_name\n",
    "melted_singleorigin_corrections_counts = pd.melt(single_origin_corrections_counts.reset_index(),\n",
    "                             id_vars='index',\n",
    "                             var_name='SubSet_and_sample',\n",
    "                             value_name='counts_corrected')\n",
    "melted_singleorigin_corrections_tpm = pd.melt(single_origin_corrections_tpm.reset_index(),\n",
    "                             id_vars='index',\n",
    "                             var_name='SubSet_and_sample',\n",
    "                             value_name='tpm_corrected')\n",
    "melted_singleorigin_corrections_counts = melted_singleorigin_corrections_counts.rename(columns={'index': 'gene_name'})\n",
    "melted_singleorigin_corrections_tpm = melted_singleorigin_corrections_tpm.rename(columns={'index': 'gene_name'})\n",
    "melted_singleorigin_corrections_tpm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:27:43.538146Z",
     "start_time": "2017-11-04T04:27:43.340844Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(56700, 42) (56700, 44)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SubSet_and_sample</th>\n",
       "      <th>gene_name</th>\n",
       "      <th>est_counts</th>\n",
       "      <th>tpm</th>\n",
       "      <th>counts_corrected</th>\n",
       "      <th>tpm_corrected</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CD4_Th2.sampleid.1</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.851882</td>\n",
       "      <td>15.208961</td>\n",
       "      <td>2.400542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CD8_Effector.sampleid.2</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.397441</td>\n",
       "      <td>21.019595</td>\n",
       "      <td>1.919938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B_CD5.sampleid.3</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>782.000170</td>\n",
       "      <td>28.925644</td>\n",
       "      <td>1783.028671</td>\n",
       "      <td>21.833836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B_Memory.sampleid.4</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>139.999971</td>\n",
       "      <td>5.299960</td>\n",
       "      <td>453.937718</td>\n",
       "      <td>6.341653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CD4_Th1.sampleid.5</td>\n",
       "      <td>ABCB4</td>\n",
       "      <td>3.007070</td>\n",
       "      <td>0.197082</td>\n",
       "      <td>26.481363</td>\n",
       "      <td>1.698088</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         SubSet_and_sample gene_name  est_counts        tpm  counts_corrected  \\\n",
       "0       CD4_Th2.sampleid.1     ABCB4    1.000000   0.851882         15.208961   \n",
       "1  CD8_Effector.sampleid.2     ABCB4    2.000000   0.397441         21.019595   \n",
       "2         B_CD5.sampleid.3     ABCB4  782.000170  28.925644       1783.028671   \n",
       "3      B_Memory.sampleid.4     ABCB4  139.999971   5.299960        453.937718   \n",
       "4       CD4_Th1.sampleid.5     ABCB4    3.007070   0.197082         26.481363   \n",
       "\n",
       "   tpm_corrected  \n",
       "0       2.400542  \n",
       "1       1.919938  \n",
       "2      21.833836  \n",
       "3       6.341653  \n",
       "4       1.698088  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add as new columns\n",
    "new_training_df = pd.merge(training_df, melted_singleorigin_corrections_counts, on=['gene_name', 'SubSet_and_sample'], how='left')\n",
    "new_training_df = pd.merge(new_training_df, melted_singleorigin_corrections_tpm, on=['gene_name', 'SubSet_and_sample'], how='left')\n",
    "print(training_df.shape, new_training_df.shape)\n",
    "assert training_df.shape[0] == new_training_df.shape[0]\n",
    "new_training_df.head()[['SubSet_and_sample', 'gene_name', 'est_counts', 'tpm', 'counts_corrected', 'tpm_corrected']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:27:45.657366Z",
     "start_time": "2017-11-04T04:27:45.648629Z"
    },
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "assert new_training_df.new_gene_id.nunique() == new_training_df.new_gene_id.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:27:51.243072Z",
     "start_time": "2017-11-04T04:27:51.117650Z"
    },
    "collapsed": true,
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "outputs": [],
   "source": [
    "pickle.dump(new_training_df, open('infino-rcc/tcgakirc.new_training_df_with_corrections_cols.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:27:59.224248Z",
     "start_time": "2017-11-04T04:27:56.927149Z"
    },
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# call models.prep_stan_data\n",
    "# note that it sets:\n",
    "# 'y': sample_df.est_counts.astype(int).values\n",
    "# if want to replace, feed manually as kwarg (don't forget to cast to int -- since negative binomial distribution)\n",
    "\n",
    "[\n",
    "    stan_data_training_counts_raw, stan_data_training_tpm_raw,\n",
    "    stan_data_training_counts_corrected, stan_data_training_tpm_corrected\n",
    "] = [\n",
    "    models.prep_stan_data(\n",
    "        sample_df=new_training_df,\n",
    "        test_df=None,\n",
    "        by='SubSet',\n",
    "        y=new_training_df[colname].astype(int).values,\n",
    "        colname=colname\n",
    "    )\n",
    "    for colname in ['est_counts', 'tpm', 'counts_corrected', 'tpm_corrected']\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:27:59.315059Z",
     "start_time": "2017-11-04T04:27:59.227052Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "est_counts\n",
      "tpm\n",
      "counts_corrected\n",
      "tpm_corrected\n"
     ]
    }
   ],
   "source": [
    "for standatadict in [\n",
    "        stan_data_training_counts_raw, stan_data_training_tpm_raw,\n",
    "        stan_data_training_counts_corrected, stan_data_training_tpm_corrected\n",
    "]:\n",
    "    assert all(type(t) == np.int64 for t in standatadict['y'])\n",
    "    print(standatadict['colname'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:28:00.819603Z",
     "start_time": "2017-11-04T04:28:00.810306Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['N', 'G', 'S', 'C', 'gene', 'sample', 'x', 'y', 'cell_features', 'M', 'colname'])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stan_data_training_counts_raw.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## extract cohort columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:28:24.948435Z",
     "start_time": "2017-11-04T04:28:24.931815Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 79)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cohort_corrected_counts = batch_corrected_counts[cohort_cols]\n",
    "cohort_corrected_tpm = batch_corrected_tpm[cohort_cols]\n",
    "cohort_corrected_counts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:28:28.652245Z",
     "start_time": "2017-11-04T04:28:28.584033Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X953473f4.9927.4fd4.bfee.8f5908638cc7</th>\n",
       "      <th>df66c814.1407.4899.a6e3.9c2a928e6ea0</th>\n",
       "      <th>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</th>\n",
       "      <th>X87c1d789.9482.442a.8b42.97bb479060e3</th>\n",
       "      <th>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</th>\n",
       "      <th>X383ba5ca.1706.4959.b543.ea87fbd367bd</th>\n",
       "      <th>d9f44b98.58b7.4ebd.a6a6.8de472bf95a8</th>\n",
       "      <th>X550af27b.de50.402f.8a81.c508e3dc6781</th>\n",
       "      <th>c81f9d5b.f58d.411d.9e1a.f919a0ef5636</th>\n",
       "      <th>cf0052ae.8440.49df.a20c.f260980c9922</th>\n",
       "      <th>...</th>\n",
       "      <th>X14b8d86c.2fef.4c50.8633.a84e12d81b51</th>\n",
       "      <th>X0cc91a13.32d9.4262.a33d.264096b4102c</th>\n",
       "      <th>X9bae55b1.5a27.4a9f.b376.ad1ab4fc4ee2</th>\n",
       "      <th>X6b124b2e.dce9.48ef.bedb.7a2da9cd2ff8</th>\n",
       "      <th>X3b4be095.1d76.49de.8357.f932ba5d55bb</th>\n",
       "      <th>e644640b.78d3.460a.83fa.ea475620a3f0</th>\n",
       "      <th>X01f8736a.85bc.498d.91c6.1c053c479512</th>\n",
       "      <th>e40e21de.f116.4eb1.87af.3b16797ec3b2</th>\n",
       "      <th>X19de7074.f195.47f6.81fa.6d3bf01c1685</th>\n",
       "      <th>X0153168a.39ce.47d6.b7f5.ac1e0f312245</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ABCB4</th>\n",
       "      <td>271.724524</td>\n",
       "      <td>146.886978</td>\n",
       "      <td>230.718236</td>\n",
       "      <td>109.439052</td>\n",
       "      <td>24.703625</td>\n",
       "      <td>305.410588</td>\n",
       "      <td>54.106583</td>\n",
       "      <td>198.958657</td>\n",
       "      <td>182.063470</td>\n",
       "      <td>121.876628</td>\n",
       "      <td>...</td>\n",
       "      <td>12.059764</td>\n",
       "      <td>50.600730</td>\n",
       "      <td>41.994015</td>\n",
       "      <td>22.634876</td>\n",
       "      <td>122.664761</td>\n",
       "      <td>114.492055</td>\n",
       "      <td>438.379423</td>\n",
       "      <td>127.216490</td>\n",
       "      <td>11.712959</td>\n",
       "      <td>30.633392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCC4</th>\n",
       "      <td>831.429172</td>\n",
       "      <td>1663.080801</td>\n",
       "      <td>318.265939</td>\n",
       "      <td>777.656770</td>\n",
       "      <td>666.222031</td>\n",
       "      <td>891.229022</td>\n",
       "      <td>264.726493</td>\n",
       "      <td>1479.904552</td>\n",
       "      <td>1610.772320</td>\n",
       "      <td>1038.279478</td>\n",
       "      <td>...</td>\n",
       "      <td>3106.700003</td>\n",
       "      <td>1100.346546</td>\n",
       "      <td>423.348771</td>\n",
       "      <td>759.126933</td>\n",
       "      <td>875.937192</td>\n",
       "      <td>454.449486</td>\n",
       "      <td>361.059298</td>\n",
       "      <td>381.127671</td>\n",
       "      <td>423.106482</td>\n",
       "      <td>247.472922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCG2</th>\n",
       "      <td>446.561761</td>\n",
       "      <td>812.255664</td>\n",
       "      <td>99.397253</td>\n",
       "      <td>384.058331</td>\n",
       "      <td>157.793904</td>\n",
       "      <td>259.222446</td>\n",
       "      <td>478.020154</td>\n",
       "      <td>117.793226</td>\n",
       "      <td>403.140287</td>\n",
       "      <td>29.555554</td>\n",
       "      <td>...</td>\n",
       "      <td>9.305463</td>\n",
       "      <td>114.447453</td>\n",
       "      <td>191.697509</td>\n",
       "      <td>35.464107</td>\n",
       "      <td>73.599907</td>\n",
       "      <td>580.514047</td>\n",
       "      <td>157.049135</td>\n",
       "      <td>133.228123</td>\n",
       "      <td>31.955271</td>\n",
       "      <td>307.780362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD2</th>\n",
       "      <td>4013.460562</td>\n",
       "      <td>6104.244702</td>\n",
       "      <td>2089.435501</td>\n",
       "      <td>3881.827514</td>\n",
       "      <td>1367.252094</td>\n",
       "      <td>3348.699452</td>\n",
       "      <td>1547.480950</td>\n",
       "      <td>5232.475978</td>\n",
       "      <td>5397.201954</td>\n",
       "      <td>4757.911387</td>\n",
       "      <td>...</td>\n",
       "      <td>12081.302795</td>\n",
       "      <td>2836.704826</td>\n",
       "      <td>2561.585963</td>\n",
       "      <td>2567.891426</td>\n",
       "      <td>1483.693915</td>\n",
       "      <td>2622.970734</td>\n",
       "      <td>2304.556472</td>\n",
       "      <td>1207.471578</td>\n",
       "      <td>1185.583044</td>\n",
       "      <td>2773.538872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABHD5</th>\n",
       "      <td>231.738420</td>\n",
       "      <td>411.561635</td>\n",
       "      <td>115.238971</td>\n",
       "      <td>289.471465</td>\n",
       "      <td>198.752713</td>\n",
       "      <td>353.216027</td>\n",
       "      <td>126.455767</td>\n",
       "      <td>196.168396</td>\n",
       "      <td>223.111443</td>\n",
       "      <td>190.855346</td>\n",
       "      <td>...</td>\n",
       "      <td>540.434445</td>\n",
       "      <td>421.282708</td>\n",
       "      <td>161.795912</td>\n",
       "      <td>191.096400</td>\n",
       "      <td>94.902846</td>\n",
       "      <td>130.049607</td>\n",
       "      <td>144.094925</td>\n",
       "      <td>135.310505</td>\n",
       "      <td>73.447862</td>\n",
       "      <td>134.541336</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 79 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       X953473f4.9927.4fd4.bfee.8f5908638cc7  \\\n",
       "ABCB4                             271.724524   \n",
       "ABCC4                             831.429172   \n",
       "ABCG2                             446.561761   \n",
       "ABHD2                            4013.460562   \n",
       "ABHD5                             231.738420   \n",
       "\n",
       "       df66c814.1407.4899.a6e3.9c2a928e6ea0  \\\n",
       "ABCB4                            146.886978   \n",
       "ABCC4                           1663.080801   \n",
       "ABCG2                            812.255664   \n",
       "ABHD2                           6104.244702   \n",
       "ABHD5                            411.561635   \n",
       "\n",
       "       X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d  \\\n",
       "ABCB4                             230.718236   \n",
       "ABCC4                             318.265939   \n",
       "ABCG2                              99.397253   \n",
       "ABHD2                            2089.435501   \n",
       "ABHD5                             115.238971   \n",
       "\n",
       "       X87c1d789.9482.442a.8b42.97bb479060e3  \\\n",
       "ABCB4                             109.439052   \n",
       "ABCC4                             777.656770   \n",
       "ABCG2                             384.058331   \n",
       "ABHD2                            3881.827514   \n",
       "ABHD5                             289.471465   \n",
       "\n",
       "       X4ddfd589.5a62.4149.9ccc.6a539f0b37b7  \\\n",
       "ABCB4                              24.703625   \n",
       "ABCC4                             666.222031   \n",
       "ABCG2                             157.793904   \n",
       "ABHD2                            1367.252094   \n",
       "ABHD5                             198.752713   \n",
       "\n",
       "       X383ba5ca.1706.4959.b543.ea87fbd367bd  \\\n",
       "ABCB4                             305.410588   \n",
       "ABCC4                             891.229022   \n",
       "ABCG2                             259.222446   \n",
       "ABHD2                            3348.699452   \n",
       "ABHD5                             353.216027   \n",
       "\n",
       "       d9f44b98.58b7.4ebd.a6a6.8de472bf95a8  \\\n",
       "ABCB4                             54.106583   \n",
       "ABCC4                            264.726493   \n",
       "ABCG2                            478.020154   \n",
       "ABHD2                           1547.480950   \n",
       "ABHD5                            126.455767   \n",
       "\n",
       "       X550af27b.de50.402f.8a81.c508e3dc6781  \\\n",
       "ABCB4                             198.958657   \n",
       "ABCC4                            1479.904552   \n",
       "ABCG2                             117.793226   \n",
       "ABHD2                            5232.475978   \n",
       "ABHD5                             196.168396   \n",
       "\n",
       "       c81f9d5b.f58d.411d.9e1a.f919a0ef5636  \\\n",
       "ABCB4                            182.063470   \n",
       "ABCC4                           1610.772320   \n",
       "ABCG2                            403.140287   \n",
       "ABHD2                           5397.201954   \n",
       "ABHD5                            223.111443   \n",
       "\n",
       "       cf0052ae.8440.49df.a20c.f260980c9922  \\\n",
       "ABCB4                            121.876628   \n",
       "ABCC4                           1038.279478   \n",
       "ABCG2                             29.555554   \n",
       "ABHD2                           4757.911387   \n",
       "ABHD5                            190.855346   \n",
       "\n",
       "                       ...                    \\\n",
       "ABCB4                  ...                     \n",
       "ABCC4                  ...                     \n",
       "ABCG2                  ...                     \n",
       "ABHD2                  ...                     \n",
       "ABHD5                  ...                     \n",
       "\n",
       "       X14b8d86c.2fef.4c50.8633.a84e12d81b51  \\\n",
       "ABCB4                              12.059764   \n",
       "ABCC4                            3106.700003   \n",
       "ABCG2                               9.305463   \n",
       "ABHD2                           12081.302795   \n",
       "ABHD5                             540.434445   \n",
       "\n",
       "       X0cc91a13.32d9.4262.a33d.264096b4102c  \\\n",
       "ABCB4                              50.600730   \n",
       "ABCC4                            1100.346546   \n",
       "ABCG2                             114.447453   \n",
       "ABHD2                            2836.704826   \n",
       "ABHD5                             421.282708   \n",
       "\n",
       "       X9bae55b1.5a27.4a9f.b376.ad1ab4fc4ee2  \\\n",
       "ABCB4                              41.994015   \n",
       "ABCC4                             423.348771   \n",
       "ABCG2                             191.697509   \n",
       "ABHD2                            2561.585963   \n",
       "ABHD5                             161.795912   \n",
       "\n",
       "       X6b124b2e.dce9.48ef.bedb.7a2da9cd2ff8  \\\n",
       "ABCB4                              22.634876   \n",
       "ABCC4                             759.126933   \n",
       "ABCG2                              35.464107   \n",
       "ABHD2                            2567.891426   \n",
       "ABHD5                             191.096400   \n",
       "\n",
       "       X3b4be095.1d76.49de.8357.f932ba5d55bb  \\\n",
       "ABCB4                             122.664761   \n",
       "ABCC4                             875.937192   \n",
       "ABCG2                              73.599907   \n",
       "ABHD2                            1483.693915   \n",
       "ABHD5                              94.902846   \n",
       "\n",
       "       e644640b.78d3.460a.83fa.ea475620a3f0  \\\n",
       "ABCB4                            114.492055   \n",
       "ABCC4                            454.449486   \n",
       "ABCG2                            580.514047   \n",
       "ABHD2                           2622.970734   \n",
       "ABHD5                            130.049607   \n",
       "\n",
       "       X01f8736a.85bc.498d.91c6.1c053c479512  \\\n",
       "ABCB4                             438.379423   \n",
       "ABCC4                             361.059298   \n",
       "ABCG2                             157.049135   \n",
       "ABHD2                            2304.556472   \n",
       "ABHD5                             144.094925   \n",
       "\n",
       "       e40e21de.f116.4eb1.87af.3b16797ec3b2  \\\n",
       "ABCB4                            127.216490   \n",
       "ABCC4                            381.127671   \n",
       "ABCG2                            133.228123   \n",
       "ABHD2                           1207.471578   \n",
       "ABHD5                            135.310505   \n",
       "\n",
       "       X19de7074.f195.47f6.81fa.6d3bf01c1685  \\\n",
       "ABCB4                              11.712959   \n",
       "ABCC4                             423.106482   \n",
       "ABCG2                              31.955271   \n",
       "ABHD2                            1185.583044   \n",
       "ABHD5                              73.447862   \n",
       "\n",
       "       X0153168a.39ce.47d6.b7f5.ac1e0f312245  \n",
       "ABCB4                              30.633392  \n",
       "ABCC4                             247.472922  \n",
       "ABCG2                             307.780362  \n",
       "ABHD2                            2773.538872  \n",
       "ABHD5                             134.541336  \n",
       "\n",
       "[5 rows x 79 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cohort_corrected_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:28:31.021879Z",
     "start_time": "2017-11-04T04:28:30.949015Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gene_symbols</th>\n",
       "      <th>X953473f4.9927.4fd4.bfee.8f5908638cc7</th>\n",
       "      <th>df66c814.1407.4899.a6e3.9c2a928e6ea0</th>\n",
       "      <th>X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d</th>\n",
       "      <th>X87c1d789.9482.442a.8b42.97bb479060e3</th>\n",
       "      <th>X4ddfd589.5a62.4149.9ccc.6a539f0b37b7</th>\n",
       "      <th>X383ba5ca.1706.4959.b543.ea87fbd367bd</th>\n",
       "      <th>d9f44b98.58b7.4ebd.a6a6.8de472bf95a8</th>\n",
       "      <th>X550af27b.de50.402f.8a81.c508e3dc6781</th>\n",
       "      <th>c81f9d5b.f58d.411d.9e1a.f919a0ef5636</th>\n",
       "      <th>...</th>\n",
       "      <th>X14b8d86c.2fef.4c50.8633.a84e12d81b51</th>\n",
       "      <th>X0cc91a13.32d9.4262.a33d.264096b4102c</th>\n",
       "      <th>X9bae55b1.5a27.4a9f.b376.ad1ab4fc4ee2</th>\n",
       "      <th>X6b124b2e.dce9.48ef.bedb.7a2da9cd2ff8</th>\n",
       "      <th>X3b4be095.1d76.49de.8357.f932ba5d55bb</th>\n",
       "      <th>e644640b.78d3.460a.83fa.ea475620a3f0</th>\n",
       "      <th>X01f8736a.85bc.498d.91c6.1c053c479512</th>\n",
       "      <th>e40e21de.f116.4eb1.87af.3b16797ec3b2</th>\n",
       "      <th>X19de7074.f195.47f6.81fa.6d3bf01c1685</th>\n",
       "      <th>X0153168a.39ce.47d6.b7f5.ac1e0f312245</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>ABCB4</td>\n",
       "      <td>706.33700</td>\n",
       "      <td>446.87304</td>\n",
       "      <td>625.38770</td>\n",
       "      <td>358.914900</td>\n",
       "      <td>118.112890</td>\n",
       "      <td>770.49687</td>\n",
       "      <td>212.26514</td>\n",
       "      <td>560.123426</td>\n",
       "      <td>524.32215</td>\n",
       "      <td>...</td>\n",
       "      <td>68.918632</td>\n",
       "      <td>201.910938</td>\n",
       "      <td>175.665020</td>\n",
       "      <td>110.61954</td>\n",
       "      <td>390.75120</td>\n",
       "      <td>371.18904</td>\n",
       "      <td>1008.13570</td>\n",
       "      <td>401.50031</td>\n",
       "      <td>67.419253</td>\n",
       "      <td>138.757250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>ABCC4</td>\n",
       "      <td>4507.46215</td>\n",
       "      <td>8910.53015</td>\n",
       "      <td>1753.47137</td>\n",
       "      <td>4220.717535</td>\n",
       "      <td>3625.371412</td>\n",
       "      <td>4825.97577</td>\n",
       "      <td>1462.94769</td>\n",
       "      <td>7944.877950</td>\n",
       "      <td>8634.97037</td>\n",
       "      <td>...</td>\n",
       "      <td>16468.615420</td>\n",
       "      <td>5937.068360</td>\n",
       "      <td>2321.357970</td>\n",
       "      <td>4121.82838</td>\n",
       "      <td>4744.56236</td>\n",
       "      <td>2488.93732</td>\n",
       "      <td>1985.07173</td>\n",
       "      <td>2093.51727</td>\n",
       "      <td>2320.051640</td>\n",
       "      <td>1369.117680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>ABCG2</td>\n",
       "      <td>2396.99700</td>\n",
       "      <td>4347.00272</td>\n",
       "      <td>537.00004</td>\n",
       "      <td>2062.999600</td>\n",
       "      <td>851.000260</td>\n",
       "      <td>1394.99960</td>\n",
       "      <td>2565.00448</td>\n",
       "      <td>635.999900</td>\n",
       "      <td>2164.99560</td>\n",
       "      <td>...</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>618.000740</td>\n",
       "      <td>1032.999310</td>\n",
       "      <td>191.99994</td>\n",
       "      <td>398.00040</td>\n",
       "      <td>3111.99950</td>\n",
       "      <td>847.00007</td>\n",
       "      <td>719.00078</td>\n",
       "      <td>173.000001</td>\n",
       "      <td>1655.000178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>ABHD2</td>\n",
       "      <td>10384.18660</td>\n",
       "      <td>16818.22951</td>\n",
       "      <td>4901.97921</td>\n",
       "      <td>9993.524850</td>\n",
       "      <td>3009.864000</td>\n",
       "      <td>8432.18246</td>\n",
       "      <td>3470.54752</td>\n",
       "      <td>14087.238540</td>\n",
       "      <td>14598.38364</td>\n",
       "      <td>...</td>\n",
       "      <td>36871.128370</td>\n",
       "      <td>6967.434770</td>\n",
       "      <td>6196.171419</td>\n",
       "      <td>6213.71428</td>\n",
       "      <td>3306.53000</td>\n",
       "      <td>6367.22714</td>\n",
       "      <td>5486.73707</td>\n",
       "      <td>2608.96686</td>\n",
       "      <td>2554.641760</td>\n",
       "      <td>6789.322640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>ABHD5</td>\n",
       "      <td>612.12094</td>\n",
       "      <td>1056.32190</td>\n",
       "      <td>315.00000</td>\n",
       "      <td>756.191570</td>\n",
       "      <td>529.000980</td>\n",
       "      <td>913.56245</td>\n",
       "      <td>344.11214</td>\n",
       "      <td>522.460330</td>\n",
       "      <td>590.44428</td>\n",
       "      <td>...</td>\n",
       "      <td>1368.165060</td>\n",
       "      <td>1080.002470</td>\n",
       "      <td>435.020880</td>\n",
       "      <td>509.61072</td>\n",
       "      <td>261.83651</td>\n",
       "      <td>353.41108</td>\n",
       "      <td>389.63008</td>\n",
       "      <td>366.99986</td>\n",
       "      <td>205.103910</td>\n",
       "      <td>365.014830</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 80 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Gene_symbols  X953473f4.9927.4fd4.bfee.8f5908638cc7  \\\n",
       "76         ABCB4                              706.33700   \n",
       "89         ABCC4                             4507.46215   \n",
       "110        ABCG2                             2396.99700   \n",
       "136        ABHD2                            10384.18660   \n",
       "139        ABHD5                              612.12094   \n",
       "\n",
       "     df66c814.1407.4899.a6e3.9c2a928e6ea0  \\\n",
       "76                              446.87304   \n",
       "89                             8910.53015   \n",
       "110                            4347.00272   \n",
       "136                           16818.22951   \n",
       "139                            1056.32190   \n",
       "\n",
       "     X2bdf69ff.b6fe.48c8.ac05.74ca195c6e1d  \\\n",
       "76                               625.38770   \n",
       "89                              1753.47137   \n",
       "110                              537.00004   \n",
       "136                             4901.97921   \n",
       "139                              315.00000   \n",
       "\n",
       "     X87c1d789.9482.442a.8b42.97bb479060e3  \\\n",
       "76                              358.914900   \n",
       "89                             4220.717535   \n",
       "110                            2062.999600   \n",
       "136                            9993.524850   \n",
       "139                             756.191570   \n",
       "\n",
       "     X4ddfd589.5a62.4149.9ccc.6a539f0b37b7  \\\n",
       "76                              118.112890   \n",
       "89                             3625.371412   \n",
       "110                             851.000260   \n",
       "136                            3009.864000   \n",
       "139                             529.000980   \n",
       "\n",
       "     X383ba5ca.1706.4959.b543.ea87fbd367bd  \\\n",
       "76                               770.49687   \n",
       "89                              4825.97577   \n",
       "110                             1394.99960   \n",
       "136                             8432.18246   \n",
       "139                              913.56245   \n",
       "\n",
       "     d9f44b98.58b7.4ebd.a6a6.8de472bf95a8  \\\n",
       "76                              212.26514   \n",
       "89                             1462.94769   \n",
       "110                            2565.00448   \n",
       "136                            3470.54752   \n",
       "139                             344.11214   \n",
       "\n",
       "     X550af27b.de50.402f.8a81.c508e3dc6781  \\\n",
       "76                              560.123426   \n",
       "89                             7944.877950   \n",
       "110                             635.999900   \n",
       "136                           14087.238540   \n",
       "139                             522.460330   \n",
       "\n",
       "     c81f9d5b.f58d.411d.9e1a.f919a0ef5636  \\\n",
       "76                              524.32215   \n",
       "89                             8634.97037   \n",
       "110                            2164.99560   \n",
       "136                           14598.38364   \n",
       "139                             590.44428   \n",
       "\n",
       "                     ...                    \\\n",
       "76                   ...                     \n",
       "89                   ...                     \n",
       "110                  ...                     \n",
       "136                  ...                     \n",
       "139                  ...                     \n",
       "\n",
       "     X14b8d86c.2fef.4c50.8633.a84e12d81b51  \\\n",
       "76                               68.918632   \n",
       "89                            16468.615420   \n",
       "110                              50.000000   \n",
       "136                           36871.128370   \n",
       "139                            1368.165060   \n",
       "\n",
       "     X0cc91a13.32d9.4262.a33d.264096b4102c  \\\n",
       "76                              201.910938   \n",
       "89                             5937.068360   \n",
       "110                             618.000740   \n",
       "136                            6967.434770   \n",
       "139                            1080.002470   \n",
       "\n",
       "     X9bae55b1.5a27.4a9f.b376.ad1ab4fc4ee2  \\\n",
       "76                              175.665020   \n",
       "89                             2321.357970   \n",
       "110                            1032.999310   \n",
       "136                            6196.171419   \n",
       "139                             435.020880   \n",
       "\n",
       "     X6b124b2e.dce9.48ef.bedb.7a2da9cd2ff8  \\\n",
       "76                               110.61954   \n",
       "89                              4121.82838   \n",
       "110                              191.99994   \n",
       "136                             6213.71428   \n",
       "139                              509.61072   \n",
       "\n",
       "     X3b4be095.1d76.49de.8357.f932ba5d55bb  \\\n",
       "76                               390.75120   \n",
       "89                              4744.56236   \n",
       "110                              398.00040   \n",
       "136                             3306.53000   \n",
       "139                              261.83651   \n",
       "\n",
       "     e644640b.78d3.460a.83fa.ea475620a3f0  \\\n",
       "76                              371.18904   \n",
       "89                             2488.93732   \n",
       "110                            3111.99950   \n",
       "136                            6367.22714   \n",
       "139                             353.41108   \n",
       "\n",
       "     X01f8736a.85bc.498d.91c6.1c053c479512  \\\n",
       "76                              1008.13570   \n",
       "89                              1985.07173   \n",
       "110                              847.00007   \n",
       "136                             5486.73707   \n",
       "139                              389.63008   \n",
       "\n",
       "     e40e21de.f116.4eb1.87af.3b16797ec3b2  \\\n",
       "76                              401.50031   \n",
       "89                             2093.51727   \n",
       "110                             719.00078   \n",
       "136                            2608.96686   \n",
       "139                             366.99986   \n",
       "\n",
       "     X19de7074.f195.47f6.81fa.6d3bf01c1685  \\\n",
       "76                               67.419253   \n",
       "89                             2320.051640   \n",
       "110                             173.000001   \n",
       "136                            2554.641760   \n",
       "139                             205.103910   \n",
       "\n",
       "     X0153168a.39ce.47d6.b7f5.ac1e0f312245  \n",
       "76                              138.757250  \n",
       "89                             1369.117680  \n",
       "110                            1655.000178  \n",
       "136                            6789.322640  \n",
       "139                             365.014830  \n",
       "\n",
       "[5 rows x 80 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:28:55.603576Z",
     "start_time": "2017-11-04T04:28:55.509418Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene_name</th>\n",
       "      <th>sample</th>\n",
       "      <th>counts_corrected</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABCB4</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>271.724524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ABCC4</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>831.429172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ABCG2</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>446.561761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ABHD2</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>4013.460562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ABHD5</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>231.738420</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  gene_name                                 sample  counts_corrected\n",
       "0     ABCB4  X953473f4.9927.4fd4.bfee.8f5908638cc7        271.724524\n",
       "1     ABCC4  X953473f4.9927.4fd4.bfee.8f5908638cc7        831.429172\n",
       "2     ABCG2  X953473f4.9927.4fd4.bfee.8f5908638cc7        446.561761\n",
       "3     ABHD2  X953473f4.9927.4fd4.bfee.8f5908638cc7       4013.460562\n",
       "4     ABHD5  X953473f4.9927.4fd4.bfee.8f5908638cc7        231.738420"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[\n",
    "    melted_cohort_counts_raw, melted_cohort_tpm_raw,\n",
    "    melted_cohort_counts_corrected, melted_cohort_tpm_corrected\n",
    "] = [\n",
    "    pd.melt(\n",
    "        df.reset_index(),\n",
    "        id_vars='index',\n",
    "        var_name='sample',\n",
    "        value_name=value_name).rename(columns={\n",
    "            'index': 'gene_name'\n",
    "        })\n",
    "    for (df, value_name) in zip([\n",
    "        test_df_counts.rename(columns={'Gene_symbols': 'index'}).set_index('index'),\n",
    "        test_df_tpm.rename(columns={'Gene_symbols': 'index'}).set_index('index'),\n",
    "        cohort_corrected_counts,\n",
    "        cohort_corrected_tpm\n",
    "    ], ['counts_raw', 'tpm_raw', 'counts_corrected', 'tpm_corrected'])\n",
    "]\n",
    "\n",
    "melted_cohort_counts_corrected.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:29:09.653167Z",
     "start_time": "2017-11-04T04:29:09.630540Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene_name</th>\n",
       "      <th>sample</th>\n",
       "      <th>counts_raw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABCB4</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>706.33700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ABCC4</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>4507.46215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ABCG2</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>2396.99700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ABHD2</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>10384.18660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ABHD5</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>612.12094</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  gene_name                                 sample   counts_raw\n",
       "0     ABCB4  X953473f4.9927.4fd4.bfee.8f5908638cc7    706.33700\n",
       "1     ABCC4  X953473f4.9927.4fd4.bfee.8f5908638cc7   4507.46215\n",
       "2     ABCG2  X953473f4.9927.4fd4.bfee.8f5908638cc7   2396.99700\n",
       "3     ABHD2  X953473f4.9927.4fd4.bfee.8f5908638cc7  10384.18660\n",
       "4     ABHD5  X953473f4.9927.4fd4.bfee.8f5908638cc7    612.12094"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melted_cohort_counts_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:29:30.359684Z",
     "start_time": "2017-11-04T04:29:30.171894Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene_name</th>\n",
       "      <th>sample</th>\n",
       "      <th>counts_raw</th>\n",
       "      <th>tpm_raw</th>\n",
       "      <th>counts_corrected</th>\n",
       "      <th>tpm_corrected</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABCB4</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>706.33700</td>\n",
       "      <td>4.968986</td>\n",
       "      <td>271.724524</td>\n",
       "      <td>6.132064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ABCC4</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>4507.46215</td>\n",
       "      <td>20.158407</td>\n",
       "      <td>831.429172</td>\n",
       "      <td>11.497252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ABCG2</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>2396.99700</td>\n",
       "      <td>17.879440</td>\n",
       "      <td>446.561761</td>\n",
       "      <td>8.901503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ABHD2</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>10384.18660</td>\n",
       "      <td>27.948001</td>\n",
       "      <td>4013.460562</td>\n",
       "      <td>29.926762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ABHD5</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>612.12094</td>\n",
       "      <td>10.334948</td>\n",
       "      <td>231.738420</td>\n",
       "      <td>10.481962</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  gene_name                                 sample   counts_raw    tpm_raw  \\\n",
       "0     ABCB4  X953473f4.9927.4fd4.bfee.8f5908638cc7    706.33700   4.968986   \n",
       "1     ABCC4  X953473f4.9927.4fd4.bfee.8f5908638cc7   4507.46215  20.158407   \n",
       "2     ABCG2  X953473f4.9927.4fd4.bfee.8f5908638cc7   2396.99700  17.879440   \n",
       "3     ABHD2  X953473f4.9927.4fd4.bfee.8f5908638cc7  10384.18660  27.948001   \n",
       "4     ABHD5  X953473f4.9927.4fd4.bfee.8f5908638cc7    612.12094  10.334948   \n",
       "\n",
       "   counts_corrected  tpm_corrected  \n",
       "0        271.724524       6.132064  \n",
       "1        831.429172      11.497252  \n",
       "2        446.561761       8.901503  \n",
       "3       4013.460562      29.926762  \n",
       "4        231.738420      10.481962  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# merge them all\n",
    "\n",
    "all_melted_cohorts = [\n",
    "    melted_cohort_counts_raw, melted_cohort_tpm_raw,\n",
    "    melted_cohort_counts_corrected, melted_cohort_tpm_corrected\n",
    "]\n",
    "\n",
    "# merge one\n",
    "#melted_cohort = pd.merge(melted_cohort_counts, melted_cohort_tpm, how='inner', on=['gene_name', 'sample'])\n",
    "#assert melted_cohort.shape[0] == melted_cohort_counts.shape[0] == melted_cohort_counts.shape[0]\n",
    "\n",
    "\n",
    "from functools import reduce\n",
    "# https://stackoverflow.com/a/44338256/130164\n",
    "#e.g. reduce(lambda left,right: left+right, [1,2,3,4,5]) adds (1+2)+3 ... etc\n",
    "melted_cohort = reduce(lambda left, right: pd.merge(left, right, how='inner', on=['gene_name', 'sample']),\n",
    "                        all_melted_cohorts)\n",
    "\n",
    "for i in all_melted_cohorts:\n",
    "    assert melted_cohort.shape[0] == i.shape[0], i.columns\n",
    "\n",
    "melted_cohort.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:29:37.427793Z",
     "start_time": "2017-11-04T04:29:37.390285Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(900, 2) 900\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene_name</th>\n",
       "      <th>new_gene_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABCB4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>ABCC4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>ABCG2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>ABHD2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>252</th>\n",
       "      <td>ABHD5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    gene_name  new_gene_id\n",
       "0       ABCB4            1\n",
       "63      ABCC4            2\n",
       "126     ABCG2            3\n",
       "189     ABHD2            4\n",
       "252     ABHD5            5"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_name_to_id_map = new_training_df[['gene_name', 'new_gene_id']].drop_duplicates()\n",
    "print(gene_name_to_id_map.shape, gene_name_to_id_map.new_gene_id.max())\n",
    "gene_name_to_id_map.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:29:55.198404Z",
     "start_time": "2017-11-04T04:29:55.131366Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gene_name</th>\n",
       "      <th>sample</th>\n",
       "      <th>counts_raw</th>\n",
       "      <th>tpm_raw</th>\n",
       "      <th>counts_corrected</th>\n",
       "      <th>tpm_corrected</th>\n",
       "      <th>new_gene_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABCB4</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>706.33700</td>\n",
       "      <td>4.968986</td>\n",
       "      <td>271.724524</td>\n",
       "      <td>6.132064</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ABCC4</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>4507.46215</td>\n",
       "      <td>20.158407</td>\n",
       "      <td>831.429172</td>\n",
       "      <td>11.497252</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ABCG2</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>2396.99700</td>\n",
       "      <td>17.879440</td>\n",
       "      <td>446.561761</td>\n",
       "      <td>8.901503</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ABHD2</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>10384.18660</td>\n",
       "      <td>27.948001</td>\n",
       "      <td>4013.460562</td>\n",
       "      <td>29.926762</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ABHD5</td>\n",
       "      <td>X953473f4.9927.4fd4.bfee.8f5908638cc7</td>\n",
       "      <td>612.12094</td>\n",
       "      <td>10.334948</td>\n",
       "      <td>231.738420</td>\n",
       "      <td>10.481962</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  gene_name                                 sample   counts_raw    tpm_raw  \\\n",
       "0     ABCB4  X953473f4.9927.4fd4.bfee.8f5908638cc7    706.33700   4.968986   \n",
       "1     ABCC4  X953473f4.9927.4fd4.bfee.8f5908638cc7   4507.46215  20.158407   \n",
       "2     ABCG2  X953473f4.9927.4fd4.bfee.8f5908638cc7   2396.99700  17.879440   \n",
       "3     ABHD2  X953473f4.9927.4fd4.bfee.8f5908638cc7  10384.18660  27.948001   \n",
       "4     ABHD5  X953473f4.9927.4fd4.bfee.8f5908638cc7    612.12094  10.334948   \n",
       "\n",
       "   counts_corrected  tpm_corrected  new_gene_id  \n",
       "0        271.724524       6.132064            1  \n",
       "1        831.429172      11.497252            2  \n",
       "2        446.561761       8.901503            3  \n",
       "3       4013.460562      29.926762            4  \n",
       "4        231.738420      10.481962            5  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# gene2 ids must correspond to gene ids (per model 6.2 inline comments)\n",
    "# so don't do this:\n",
    "# melted_rcc['new_gene_cat'] = melted_rcc['gene_name'].astype('category')\n",
    "# melted_rcc['new_gene_id'] = melted_rcc['new_gene_cat'].cat.codes+1\n",
    "\n",
    "# instead do:\n",
    "melted_cohort2 = pd.merge(melted_cohort, gene_name_to_id_map, how='left', on='gene_name')\n",
    "assert not any(pd.isnull(melted_cohort2['new_gene_id']))\n",
    "melted_cohort2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:30:05.863754Z",
     "start_time": "2017-11-04T04:30:05.817574Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    71100.000000\n",
       "mean        40.000000\n",
       "std         22.803669\n",
       "min          1.000000\n",
       "25%         20.000000\n",
       "50%         40.000000\n",
       "75%         60.000000\n",
       "max         79.000000\n",
       "Name: new_sample_id, dtype: float64"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sample2 ids don't correspond to sample ids\n",
    "melted_cohort2['new_sample_cat'] = melted_cohort2['sample'].astype('category')\n",
    "melted_cohort2['new_sample_id'] = melted_cohort2['new_sample_cat'].cat.codes+1\n",
    "melted_cohort2['new_sample_id'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pickle out stan data dicts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:30:26.868031Z",
     "start_time": "2017-11-04T04:30:26.763555Z"
    },
    "collapsed": false,
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['N', 'G', 'S', 'C', 'gene', 'sample', 'x', 'y', 'cell_features', 'M', 'colname']) est_counts infino-rcc/tcgakirc.singleorigin.stan_data.counts_raw.pkl\n",
      "dict_keys(['N', 'G', 'S', 'C', 'gene', 'sample', 'x', 'y', 'cell_features', 'M', 'colname']) tpm infino-rcc/tcgakirc.singleorigin.stan_data.tpm_raw.pkl\n",
      "dict_keys(['N', 'G', 'S', 'C', 'gene', 'sample', 'x', 'y', 'cell_features', 'M', 'colname']) counts_corrected infino-rcc/tcgakirc.singleorigin.stan_data.counts_corrected.pkl\n",
      "dict_keys(['N', 'G', 'S', 'C', 'gene', 'sample', 'x', 'y', 'cell_features', 'M', 'colname']) tpm_corrected infino-rcc/tcgakirc.singleorigin.stan_data.tpm_corrected.pkl\n"
     ]
    }
   ],
   "source": [
    "for standict, fname in zip([\n",
    "        stan_data_training_counts_raw, stan_data_training_tpm_raw,\n",
    "        stan_data_training_counts_corrected, stan_data_training_tpm_corrected\n",
    "], [\n",
    "        'infino-rcc/tcgakirc.singleorigin.stan_data.counts_raw.pkl',\n",
    "        'infino-rcc/tcgakirc.singleorigin.stan_data.tpm_raw.pkl',\n",
    "        'infino-rcc/tcgakirc.singleorigin.stan_data.counts_corrected.pkl',\n",
    "        'infino-rcc/tcgakirc.singleorigin.stan_data.tpm_corrected.pkl'\n",
    "]):\n",
    "    print(standict.keys(), standict['colname'], fname)\n",
    "    pickle.dump(standict, open(fname, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:30:47.773569Z",
     "start_time": "2017-11-04T04:30:47.737151Z"
    },
    "collapsed": true,
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "outputs": [],
   "source": [
    "pickle.dump(melted_cohort2, open('infino-rcc/tcgakirc.precursor_to_stan_dicts.counts_and_tpm.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:31:05.666462Z",
     "start_time": "2017-11-04T04:31:05.638039Z"
    },
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def make_test_dict_for_sample_ids(melted_dat, sampleids=None, metric='counts', processing='raw'):\n",
    "    \"\"\"\n",
    "    melted_dat is melted_cohort2 from above or identical\n",
    "    sampleids is list of samples to include in this cut. if None, include all samples\n",
    "    metric must be counts or tpm\n",
    "    processing must be raw or corrected\n",
    "    \"\"\"\n",
    "    if not sampleids:\n",
    "        test_df_for_dict = melted_dat.copy()\n",
    "    else:\n",
    "        test_df_for_dict = melted_dat[melted_dat['new_sample_id'].isin(sampleids)].copy()\n",
    "    \n",
    "    # reset sample id (otherwise stan complains/errors out)\n",
    "    test_df_for_dict['new_sample_cat'] = test_df_for_dict['sample'].astype('category')\n",
    "    test_df_for_dict['new_sample_id'] = test_df_for_dict['new_sample_cat'].cat.codes+1\n",
    "    \n",
    "    # return dict\n",
    "    relevant_col_name = '%s_%s' % (metric, processing)\n",
    "    assert relevant_col_name in test_df_for_dict.columns, 'invalid column specified'\n",
    "    \n",
    "    return {'N2': len(test_df_for_dict.index),\n",
    "             'S2': len(test_df_for_dict.new_sample_id.unique()),\n",
    "             'gene2': test_df_for_dict.new_gene_id.values,\n",
    "             'sample2': test_df_for_dict.new_sample_id.values,\n",
    "             'y2': test_df_for_dict[relevant_col_name].astype(int).values, # crucial to set as int (for negative binomial)\n",
    "             'colname_test': relevant_col_name\n",
    "           }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:31:59.522685Z",
     "start_time": "2017-11-04T04:31:59.234178Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "AssertionError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-58-c998651bea9e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# since only 26, put them all together\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32massert\u001b[0m \u001b[0mmake_test_dict_for_sample_ids\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmelted_cohort2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'S2'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m26\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m for (metric, processing) in [('counts', 'raw'), ('tpm', 'raw'),\n\u001b[1;32m      5\u001b[0m                              ('counts', 'corrected'), ('tpm', 'corrected')]:\n",
      "\u001b[0;31mAssertionError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# since only 26, put them all together\n",
    "assert make_test_dict_for_sample_ids(melted_cohort2)['S2'] == 26\n",
    "\n",
    "for (metric, processing) in [('counts', 'raw'), ('tpm', 'raw'),\n",
    "                             ('counts', 'corrected'), ('tpm', 'corrected')]:\n",
    "    output_fname = 'infino-rcc/blahblah.standict.%s.%s.pkl' % (\n",
    "        metric, processing)\n",
    "    print(output_fname,)\n",
    "    dict_to_write = make_test_dict_for_sample_ids(\n",
    "            melted_cohort2,\n",
    "            sampleids=None,\n",
    "            metric=metric,\n",
    "            processing=processing)\n",
    "    print(dict_to_write['colname_test'])\n",
    "    #pickle.dump(dict_to_write, open(output_fname, 'wb'))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Above cell is code from bladder cohort, where all unknown samples fit into one experiment.\n",
    "\n",
    "We're not using that here, but keeping it around for future clones of this notebook.\n",
    "\n",
    "Instead we'll cut into several experiments:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:33:25.014339Z",
     "start_time": "2017-11-04T04:33:24.980362Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "min     1.0\n",
       "max    79.0\n",
       "Name: new_sample_id, dtype: float64"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melted_cohort2.new_sample_id.describe()[['min', 'max']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:34:21.182781Z",
     "start_time": "2017-11-04T04:34:21.152681Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39] [40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79] 39 40 79\n"
     ]
    }
   ],
   "source": [
    "cut_1 = list(range(1, 40))\n",
    "#cut_2 = list(range(10, 20))\n",
    "cut_2 = list(range(40, max(melted_cohort2.new_sample_id)+1))\n",
    "print(cut_1,\n",
    "      cut_2,\n",
    "      #cut_3,\n",
    "      len(cut_1),\n",
    "      len(cut_2),\n",
    "      #len(cut_3),\n",
    "      len(cut_1) + len(cut_2) #+ len(cut_3)\n",
    "     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:36:51.604894Z",
     "start_time": "2017-11-04T04:36:50.391005Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "infino-rcc/tcgakirc.standict.cut1.counts.raw.pkl\n",
      "counts_raw\n",
      "39\n",
      "\n",
      "infino-rcc/tcgakirc.standict.cut1.tpm.raw.pkl\n",
      "tpm_raw\n",
      "39\n",
      "\n",
      "infino-rcc/tcgakirc.standict.cut1.counts.corrected.pkl\n",
      "counts_corrected\n",
      "39\n",
      "\n",
      "infino-rcc/tcgakirc.standict.cut1.tpm.corrected.pkl\n",
      "tpm_corrected\n",
      "39\n",
      "\n",
      "infino-rcc/tcgakirc.standict.cut2.counts.raw.pkl\n",
      "counts_raw\n",
      "40\n",
      "\n",
      "infino-rcc/tcgakirc.standict.cut2.tpm.raw.pkl\n",
      "tpm_raw\n",
      "40\n",
      "\n",
      "infino-rcc/tcgakirc.standict.cut2.counts.corrected.pkl\n",
      "counts_corrected\n",
      "40\n",
      "\n",
      "infino-rcc/tcgakirc.standict.cut2.tpm.corrected.pkl\n",
      "tpm_corrected\n",
      "40\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for (sampleids, cutname) in zip([cut_1, cut_2], ['cut1', 'cut2']):\n",
    "    for (metric, processing) in [('counts', 'raw'), ('tpm', 'raw'),\n",
    "                                 ('counts', 'corrected'), ('tpm', 'corrected')]:\n",
    "        output_fname = 'infino-rcc/tcgakirc.standict.%s.%s.%s.pkl' % (\n",
    "            cutname, metric, processing)\n",
    "        print(output_fname,)\n",
    "        dict_to_write = make_test_dict_for_sample_ids(\n",
    "                melted_cohort2,\n",
    "                sampleids=sampleids,\n",
    "                metric=metric,\n",
    "                processing=processing)\n",
    "        print(dict_to_write['colname_test'])\n",
    "        print(dict_to_write['S2'])\n",
    "        pickle.dump(dict_to_write, open(output_fname, 'wb'))\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-02T21:56:42.674065Z",
     "start_time": "2017-11-02T21:56:42.549975Z"
    },
    "collapsed": true,
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "outputs": [],
   "source": [
    "#pickle.dump(make_test_dict_for_sample_ids(melted_bladder2, cut_1), open('infino-rcc/bladder_dict_1.pkl', 'wb'))\n",
    "#pickle.dump(make_test_dict_for_sample_ids(melted_bladder2, cut_2), open('infino-rcc/bladder_dict_2.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Launch docker containers\n",
    "\n",
    "\n",
    "```\n",
    "cd infino-rcc\n",
    "\n",
    "mkdir -p logs\n",
    "chmod -R 777 logs\n",
    "\n",
    "# run each of the below in a separate tmux:\n",
    "```\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T04:43:18.193731Z",
     "start_time": "2017-11-04T04:43:18.168047Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "        # tcgakirc_cut1_counts_raw: tcgakirc.singleorigin.stan_data.counts_raw.pkl, tcgakirc.standict.cut1.counts.raw.pkl\n",
      "        mkdir -p logs/tcgakirc_cut1_counts_raw && chmod -R 777 logs/tcgakirc_cut1_counts_raw && docker run -it -d --name tcgakirc_cut1_counts_raw \\\n",
      "        -v $HOME/immune-infiltrate-explorations:/home/jovyan/work \\\n",
      "        hammerlab/infino-docker:latest \\\n",
      "        bash -c \"cd work/model-single-origin-samples/infino-rcc/ && python \\\n",
      "        cohort_runstan.py tcgakirc.singleorigin.stan_data.counts_raw.pkl tcgakirc.standict.cut1.counts.raw.pkl tcgakirc_cut1_counts_raw 2>&1 | tee logs/tcgakirc_cut1_counts_raw/run_stan.tcgakirc_cut1_counts_raw.consoleout.txt\"\n",
      "    \n",
      "############################################################\n",
      "\n",
      "        # tcgakirc_cut1_tpm_raw: tcgakirc.singleorigin.stan_data.tpm_raw.pkl, tcgakirc.standict.cut1.tpm.raw.pkl\n",
      "        mkdir -p logs/tcgakirc_cut1_tpm_raw && chmod -R 777 logs/tcgakirc_cut1_tpm_raw && docker run -it -d --name tcgakirc_cut1_tpm_raw \\\n",
      "        -v $HOME/immune-infiltrate-explorations:/home/jovyan/work \\\n",
      "        hammerlab/infino-docker:latest \\\n",
      "        bash -c \"cd work/model-single-origin-samples/infino-rcc/ && python \\\n",
      "        cohort_runstan.py tcgakirc.singleorigin.stan_data.tpm_raw.pkl tcgakirc.standict.cut1.tpm.raw.pkl tcgakirc_cut1_tpm_raw 2>&1 | tee logs/tcgakirc_cut1_tpm_raw/run_stan.tcgakirc_cut1_tpm_raw.consoleout.txt\"\n",
      "    \n",
      "############################################################\n",
      "\n",
      "        # tcgakirc_cut1_counts_corrected: tcgakirc.singleorigin.stan_data.counts_corrected.pkl, tcgakirc.standict.cut1.counts.corrected.pkl\n",
      "        mkdir -p logs/tcgakirc_cut1_counts_corrected && chmod -R 777 logs/tcgakirc_cut1_counts_corrected && docker run -it -d --name tcgakirc_cut1_counts_corrected \\\n",
      "        -v $HOME/immune-infiltrate-explorations:/home/jovyan/work \\\n",
      "        hammerlab/infino-docker:latest \\\n",
      "        bash -c \"cd work/model-single-origin-samples/infino-rcc/ && python \\\n",
      "        cohort_runstan.py tcgakirc.singleorigin.stan_data.counts_corrected.pkl tcgakirc.standict.cut1.counts.corrected.pkl tcgakirc_cut1_counts_corrected 2>&1 | tee logs/tcgakirc_cut1_counts_corrected/run_stan.tcgakirc_cut1_counts_corrected.consoleout.txt\"\n",
      "    \n",
      "############################################################\n",
      "\n",
      "        # tcgakirc_cut1_tpm_corrected: tcgakirc.singleorigin.stan_data.tpm_corrected.pkl, tcgakirc.standict.cut1.tpm.corrected.pkl\n",
      "        mkdir -p logs/tcgakirc_cut1_tpm_corrected && chmod -R 777 logs/tcgakirc_cut1_tpm_corrected && docker run -it -d --name tcgakirc_cut1_tpm_corrected \\\n",
      "        -v $HOME/immune-infiltrate-explorations:/home/jovyan/work \\\n",
      "        hammerlab/infino-docker:latest \\\n",
      "        bash -c \"cd work/model-single-origin-samples/infino-rcc/ && python \\\n",
      "        cohort_runstan.py tcgakirc.singleorigin.stan_data.tpm_corrected.pkl tcgakirc.standict.cut1.tpm.corrected.pkl tcgakirc_cut1_tpm_corrected 2>&1 | tee logs/tcgakirc_cut1_tpm_corrected/run_stan.tcgakirc_cut1_tpm_corrected.consoleout.txt\"\n",
      "    \n",
      "############################################################\n",
      "\n",
      "        # tcgakirc_cut2_counts_raw: tcgakirc.singleorigin.stan_data.counts_raw.pkl, tcgakirc.standict.cut2.counts.raw.pkl\n",
      "        mkdir -p logs/tcgakirc_cut2_counts_raw && chmod -R 777 logs/tcgakirc_cut2_counts_raw && docker run -it -d --name tcgakirc_cut2_counts_raw \\\n",
      "        -v $HOME/immune-infiltrate-explorations:/home/jovyan/work \\\n",
      "        hammerlab/infino-docker:latest \\\n",
      "        bash -c \"cd work/model-single-origin-samples/infino-rcc/ && python \\\n",
      "        cohort_runstan.py tcgakirc.singleorigin.stan_data.counts_raw.pkl tcgakirc.standict.cut2.counts.raw.pkl tcgakirc_cut2_counts_raw 2>&1 | tee logs/tcgakirc_cut2_counts_raw/run_stan.tcgakirc_cut2_counts_raw.consoleout.txt\"\n",
      "    \n",
      "############################################################\n",
      "\n",
      "        # tcgakirc_cut2_tpm_raw: tcgakirc.singleorigin.stan_data.tpm_raw.pkl, tcgakirc.standict.cut2.tpm.raw.pkl\n",
      "        mkdir -p logs/tcgakirc_cut2_tpm_raw && chmod -R 777 logs/tcgakirc_cut2_tpm_raw && docker run -it -d --name tcgakirc_cut2_tpm_raw \\\n",
      "        -v $HOME/immune-infiltrate-explorations:/home/jovyan/work \\\n",
      "        hammerlab/infino-docker:latest \\\n",
      "        bash -c \"cd work/model-single-origin-samples/infino-rcc/ && python \\\n",
      "        cohort_runstan.py tcgakirc.singleorigin.stan_data.tpm_raw.pkl tcgakirc.standict.cut2.tpm.raw.pkl tcgakirc_cut2_tpm_raw 2>&1 | tee logs/tcgakirc_cut2_tpm_raw/run_stan.tcgakirc_cut2_tpm_raw.consoleout.txt\"\n",
      "    \n",
      "############################################################\n",
      "\n",
      "        # tcgakirc_cut2_counts_corrected: tcgakirc.singleorigin.stan_data.counts_corrected.pkl, tcgakirc.standict.cut2.counts.corrected.pkl\n",
      "        mkdir -p logs/tcgakirc_cut2_counts_corrected && chmod -R 777 logs/tcgakirc_cut2_counts_corrected && docker run -it -d --name tcgakirc_cut2_counts_corrected \\\n",
      "        -v $HOME/immune-infiltrate-explorations:/home/jovyan/work \\\n",
      "        hammerlab/infino-docker:latest \\\n",
      "        bash -c \"cd work/model-single-origin-samples/infino-rcc/ && python \\\n",
      "        cohort_runstan.py tcgakirc.singleorigin.stan_data.counts_corrected.pkl tcgakirc.standict.cut2.counts.corrected.pkl tcgakirc_cut2_counts_corrected 2>&1 | tee logs/tcgakirc_cut2_counts_corrected/run_stan.tcgakirc_cut2_counts_corrected.consoleout.txt\"\n",
      "    \n",
      "############################################################\n",
      "\n",
      "        # tcgakirc_cut2_tpm_corrected: tcgakirc.singleorigin.stan_data.tpm_corrected.pkl, tcgakirc.standict.cut2.tpm.corrected.pkl\n",
      "        mkdir -p logs/tcgakirc_cut2_tpm_corrected && chmod -R 777 logs/tcgakirc_cut2_tpm_corrected && docker run -it -d --name tcgakirc_cut2_tpm_corrected \\\n",
      "        -v $HOME/immune-infiltrate-explorations:/home/jovyan/work \\\n",
      "        hammerlab/infino-docker:latest \\\n",
      "        bash -c \"cd work/model-single-origin-samples/infino-rcc/ && python \\\n",
      "        cohort_runstan.py tcgakirc.singleorigin.stan_data.tpm_corrected.pkl tcgakirc.standict.cut2.tpm.corrected.pkl tcgakirc_cut2_tpm_corrected 2>&1 | tee logs/tcgakirc_cut2_tpm_corrected/run_stan.tcgakirc_cut2_tpm_corrected.consoleout.txt\"\n",
      "    \n",
      "############################################################\n"
     ]
    }
   ],
   "source": [
    "for (sampleids, cutname) in zip([cut_1, cut_2], ['cut1', 'cut2']):\n",
    "    for (metric, processing) in [('counts', 'raw'), ('tpm', 'raw'),\n",
    "                                 ('counts', 'corrected'), ('tpm', 'corrected')]:\n",
    "        training_dict_fname = 'tcgakirc.singleorigin.stan_data.%s_%s.pkl' % (metric, processing)\n",
    "        cohort_dict_fname = 'tcgakirc.standict.%s.%s.%s.pkl' % (cutname, metric, processing)\n",
    "        runname = 'tcgakirc_%s_%s_%s' % (cutname, metric, processing)\n",
    "        print(\"\"\"\n",
    "        # {runname}: {training_dict_fname}, {cohort_dict_fname}\n",
    "        mkdir -p logs/{runname} && chmod -R 777 logs/{runname} && docker run -it -d --name {runname} \\\\\n",
    "        -v $HOME/immune-infiltrate-explorations:/home/jovyan/work \\\\\n",
    "        hammerlab/infino-docker:latest \\\\\n",
    "        bash -c \"cd work/model-single-origin-samples/infino-rcc/ && python \\\\\n",
    "        cohort_runstan.py {training_dict_fname} {cohort_dict_fname} {runname} 2>&1 | tee logs/{runname}/run_stan.{runname}.consoleout.txt\"\n",
    "    \"\"\".format(training_dict_fname=training_dict_fname, cohort_dict_fname=cohort_dict_fname, runname=runname))\n",
    "        print(\"#\" * 60)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "# monitor output with:\n",
    "tail -f logs/tcgakirc*/*.consoleout.txt\n",
    "```\n",
    "\n",
    "### some scripts to make gcloud instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-04T05:55:10.769639Z",
     "start_time": "2017-11-04T05:55:10.755148Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "gcloud beta compute --project \"pici-hammerlab\" instances create \"infino-1\" \\\n",
      "--zone \"us-east1-b\" --machine-type \"n1-highmem-4\" --subnet \"default\" --no-restart-on-failure \\\n",
      "--maintenance-policy \"MIGRATE\" \\\n",
      "--service-account \"infino@pici-hammerlab.iam.gserviceaccount.com\" \\\n",
      "--scopes \"https://www.googleapis.com/auth/cloud-platform\" \\\n",
      "--min-cpu-platform \"Automatic\" \\\n",
      "--image \"ubuntu-1604-xenial-v20171026a\" --image-project \"ubuntu-os-cloud\" \\\n",
      "--boot-disk-size \"60\" --boot-disk-type \"pd-standard\" --boot-disk-device-name \"infino-1\";\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "gcloud beta compute --project \"pici-hammerlab\" instances create \"infino-2\" \\\n",
      "--zone \"us-east1-b\" --machine-type \"n1-highmem-4\" --subnet \"default\" --no-restart-on-failure \\\n",
      "--maintenance-policy \"MIGRATE\" \\\n",
      "--service-account \"infino@pici-hammerlab.iam.gserviceaccount.com\" \\\n",
      "--scopes \"https://www.googleapis.com/auth/cloud-platform\" \\\n",
      "--min-cpu-platform \"Automatic\" \\\n",
      "--image \"ubuntu-1604-xenial-v20171026a\" --image-project \"ubuntu-os-cloud\" \\\n",
      "--boot-disk-size \"60\" --boot-disk-type \"pd-standard\" --boot-disk-device-name \"infino-2\";\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "gcloud beta compute --project \"pici-hammerlab\" instances create \"infino-3\" \\\n",
      "--zone \"us-east1-b\" --machine-type \"n1-highmem-4\" --subnet \"default\" --no-restart-on-failure \\\n",
      "--maintenance-policy \"MIGRATE\" \\\n",
      "--service-account \"infino@pici-hammerlab.iam.gserviceaccount.com\" \\\n",
      "--scopes \"https://www.googleapis.com/auth/cloud-platform\" \\\n",
      "--min-cpu-platform \"Automatic\" \\\n",
      "--image \"ubuntu-1604-xenial-v20171026a\" --image-project \"ubuntu-os-cloud\" \\\n",
      "--boot-disk-size \"60\" --boot-disk-type \"pd-standard\" --boot-disk-device-name \"infino-3\";\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "gcloud beta compute --project \"pici-hammerlab\" instances create \"infino-4\" \\\n",
      "--zone \"us-east1-b\" --machine-type \"n1-highmem-4\" --subnet \"default\" --no-restart-on-failure \\\n",
      "--maintenance-policy \"MIGRATE\" \\\n",
      "--service-account \"infino@pici-hammerlab.iam.gserviceaccount.com\" \\\n",
      "--scopes \"https://www.googleapis.com/auth/cloud-platform\" \\\n",
      "--min-cpu-platform \"Automatic\" \\\n",
      "--image \"ubuntu-1604-xenial-v20171026a\" --image-project \"ubuntu-os-cloud\" \\\n",
      "--boot-disk-size \"60\" --boot-disk-type \"pd-standard\" --boot-disk-device-name \"infino-4\";\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "gcloud beta compute --project \"pici-hammerlab\" instances create \"infino-5\" \\\n",
      "--zone \"us-east1-b\" --machine-type \"n1-highmem-4\" --subnet \"default\" --no-restart-on-failure \\\n",
      "--maintenance-policy \"MIGRATE\" \\\n",
      "--service-account \"infino@pici-hammerlab.iam.gserviceaccount.com\" \\\n",
      "--scopes \"https://www.googleapis.com/auth/cloud-platform\" \\\n",
      "--min-cpu-platform \"Automatic\" \\\n",
      "--image \"ubuntu-1604-xenial-v20171026a\" --image-project \"ubuntu-os-cloud\" \\\n",
      "--boot-disk-size \"60\" --boot-disk-type \"pd-standard\" --boot-disk-device-name \"infino-5\";\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "gcloud beta compute --project \"pici-hammerlab\" instances create \"infino-6\" \\\n",
      "--zone \"us-east1-b\" --machine-type \"n1-highmem-4\" --subnet \"default\" --no-restart-on-failure \\\n",
      "--maintenance-policy \"MIGRATE\" \\\n",
      "--service-account \"infino@pici-hammerlab.iam.gserviceaccount.com\" \\\n",
      "--scopes \"https://www.googleapis.com/auth/cloud-platform\" \\\n",
      "--min-cpu-platform \"Automatic\" \\\n",
      "--image \"ubuntu-1604-xenial-v20171026a\" --image-project \"ubuntu-os-cloud\" \\\n",
      "--boot-disk-size \"60\" --boot-disk-type \"pd-standard\" --boot-disk-device-name \"infino-6\";\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "gcloud beta compute --project \"pici-hammerlab\" instances create \"infino-7\" \\\n",
      "--zone \"us-east1-b\" --machine-type \"n1-highmem-4\" --subnet \"default\" --no-restart-on-failure \\\n",
      "--maintenance-policy \"MIGRATE\" \\\n",
      "--service-account \"infino@pici-hammerlab.iam.gserviceaccount.com\" \\\n",
      "--scopes \"https://www.googleapis.com/auth/cloud-platform\" \\\n",
      "--min-cpu-platform \"Automatic\" \\\n",
      "--image \"ubuntu-1604-xenial-v20171026a\" --image-project \"ubuntu-os-cloud\" \\\n",
      "--boot-disk-size \"60\" --boot-disk-type \"pd-standard\" --boot-disk-device-name \"infino-7\";\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "gcloud beta compute --project \"pici-hammerlab\" instances create \"infino-8\" \\\n",
      "--zone \"us-east1-b\" --machine-type \"n1-highmem-4\" --subnet \"default\" --no-restart-on-failure \\\n",
      "--maintenance-policy \"MIGRATE\" \\\n",
      "--service-account \"infino@pici-hammerlab.iam.gserviceaccount.com\" \\\n",
      "--scopes \"https://www.googleapis.com/auth/cloud-platform\" \\\n",
      "--min-cpu-platform \"Automatic\" \\\n",
      "--image \"ubuntu-1604-xenial-v20171026a\" --image-project \"ubuntu-os-cloud\" \\\n",
      "--boot-disk-size \"60\" --boot-disk-type \"pd-standard\" --boot-disk-device-name \"infino-8\";\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for machine_id in range(1, 9):\n",
    "    print(\"\"\"\n",
    "\n",
    "gcloud beta compute --project \"pici-hammerlab\" instances create \"infino-{machine_id}\" \\\\\n",
    "--zone \"us-east1-b\" --machine-type \"n1-highmem-4\" --subnet \"default\" --no-restart-on-failure \\\\\n",
    "--maintenance-policy \"MIGRATE\" \\\\\n",
    "--service-account \"infino@pici-hammerlab.iam.gserviceaccount.com\" \\\\\n",
    "--scopes \"https://www.googleapis.com/auth/cloud-platform\" \\\\\n",
    "--min-cpu-platform \"Automatic\" \\\\\n",
    "--image \"ubuntu-1604-xenial-v20171026a\" --image-project \"ubuntu-os-cloud\" \\\\\n",
    "--boot-disk-size \"60\" --boot-disk-type \"pd-standard\" --boot-disk-device-name \"infino-{machine_id}\";\n",
    "\n",
    "\"\"\".format(machine_id=machine_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Initialization Cell",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}