4.1. Random Forest Classification-1,25-nested CV (Python).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This notebook is to perform random forest classification with nested CV on 1,25-(OH)2D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy import interp\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import cycle\n",
    "\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.model_selection import StratifiedKFold, GridSearchCV, ParameterGrid\n",
    "from nested_cv import NestedCV\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "taxa = pd.read_csv('../data/57316_deblur_taxonomy/taxonomy.tsv', sep='\\t', index_col='Feature ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Taxon</th>\n",
       "      <th>Confidence</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Feature ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGACGCTCAACGTCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG</th>\n",
       "      <td>k__Bacteria; p__Bacteroidetes; c__Bacteroidia;...</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGATGCTCAACATCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG</th>\n",
       "      <td>k__Bacteria; p__Bacteroidetes; c__Bacteroidia;...</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGCAGGCGGCGGCGTAAGTCAGTTGTGAAATCGTGCGGCTTAACCGTGCAATTGCAGTTGATACTGCGTCGCTTGAGTGCACACAGGGATGTTGGAATTCATGG</th>\n",
       "      <td>k__Bacteria; p__Bacteroidetes; c__Bacteroidia;...</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGAGGCTCAACCTCTGCACTGCAGCGCGAACTGGTCTTCTTGAGTACGCACAACGTGGGCGGAATTCGTGG</th>\n",
       "      <td>k__Bacteria; p__Bacteroidetes; c__Bacteroidia;...</td>\n",
       "      <td>0.999998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGTTTGTTAAGCGTGTTGTGAAATGTCGGGGCTCAACCTGGGCATTGCAGCGCGAACTGGCAGACTTGAGTGCGCGGGAAGTAGGCGGAATTCGTCG</th>\n",
       "      <td>k__Bacteria; p__Bacteroidetes; c__Bacteroidia;...</td>\n",
       "      <td>0.999991</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                Taxon  \\\n",
       "Feature ID                                                                                              \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...  k__Bacteria; p__Bacteroidetes; c__Bacteroidia;...   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...  k__Bacteria; p__Bacteroidetes; c__Bacteroidia;...   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...  k__Bacteria; p__Bacteroidetes; c__Bacteroidia;...   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...  k__Bacteria; p__Bacteroidetes; c__Bacteroidia;...   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...  k__Bacteria; p__Bacteroidetes; c__Bacteroidia;...   \n",
       "\n",
       "                                                    Confidence  \n",
       "Feature ID                                                      \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...    1.000000  \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...    1.000000  \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...    1.000000  \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...    0.999998  \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...    0.999991  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "taxa.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "mf = pd.read_csv('../data/mros_mapping_alpha.txt', sep='\\t', index_col='#SampleID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BarcodeSequence</th>\n",
       "      <th>LinkerPrimerSequence</th>\n",
       "      <th>Experiment_Design_Description</th>\n",
       "      <th>Library_Construction_Protocol</th>\n",
       "      <th>Linker</th>\n",
       "      <th>Platform</th>\n",
       "      <th>Center_Name</th>\n",
       "      <th>Center_Project</th>\n",
       "      <th>Instrument_Model</th>\n",
       "      <th>Title</th>\n",
       "      <th>...</th>\n",
       "      <th>ratio_catabolism</th>\n",
       "      <th>QLCOMP</th>\n",
       "      <th>M1STATIN</th>\n",
       "      <th>Antihistamine</th>\n",
       "      <th>Laxative</th>\n",
       "      <th>MIDATA</th>\n",
       "      <th>Rstarch_total</th>\n",
       "      <th>Rstarches_c</th>\n",
       "      <th>M1PPUMP</th>\n",
       "      <th>alpha_pd</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>#SampleID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BI0023</th>\n",
       "      <td>TCTGGTGACATT</td>\n",
       "      <td>GGACTACHVGGGTWTCTAAT</td>\n",
       "      <td>16S stool samples sequenced for MrOS Vitamin D...</td>\n",
       "      <td>16S rRNA v4</td>\n",
       "      <td>GT</td>\n",
       "      <td>Illumina</td>\n",
       "      <td>BI</td>\n",
       "      <td>MrOS</td>\n",
       "      <td>Illumina MiSeq</td>\n",
       "      <td>MrOS_VitaminD</td>\n",
       "      <td>...</td>\n",
       "      <td>0.068605</td>\n",
       "      <td>1:GOOD/EXCELLENT</td>\n",
       "      <td>0: No</td>\n",
       "      <td>0:No</td>\n",
       "      <td>0:No</td>\n",
       "      <td>1: Yes</td>\n",
       "      <td>3.066569</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0: No</td>\n",
       "      <td>30.021398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BI0056</th>\n",
       "      <td>CAAGCATGCCTA</td>\n",
       "      <td>GGACTACHVGGGTWTCTAAT</td>\n",
       "      <td>16S stool samples sequenced for MrOS Vitamin D...</td>\n",
       "      <td>16S rRNA v4</td>\n",
       "      <td>GT</td>\n",
       "      <td>Illumina</td>\n",
       "      <td>BI</td>\n",
       "      <td>MrOS</td>\n",
       "      <td>Illumina MiSeq</td>\n",
       "      <td>MrOS_VitaminD</td>\n",
       "      <td>...</td>\n",
       "      <td>0.099745</td>\n",
       "      <td>1:GOOD/EXCELLENT</td>\n",
       "      <td>0: No</td>\n",
       "      <td>0:No</td>\n",
       "      <td>0:No</td>\n",
       "      <td>1: Yes</td>\n",
       "      <td>3.038136</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1: Yes</td>\n",
       "      <td>18.414977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BI0131</th>\n",
       "      <td>CTATTTGCGACA</td>\n",
       "      <td>GGACTACHVGGGTWTCTAAT</td>\n",
       "      <td>16S stool samples sequenced for MrOS Vitamin D...</td>\n",
       "      <td>16S rRNA v4</td>\n",
       "      <td>GT</td>\n",
       "      <td>Illumina</td>\n",
       "      <td>BI</td>\n",
       "      <td>MrOS</td>\n",
       "      <td>Illumina MiSeq</td>\n",
       "      <td>MrOS_VitaminD</td>\n",
       "      <td>...</td>\n",
       "      <td>0.064502</td>\n",
       "      <td>1:GOOD/EXCELLENT</td>\n",
       "      <td>1: Yes</td>\n",
       "      <td>0:No</td>\n",
       "      <td>0:No</td>\n",
       "      <td>1: Yes</td>\n",
       "      <td>2.840599</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0: No</td>\n",
       "      <td>25.756952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BI0153</th>\n",
       "      <td>ATCGGCGTTACA</td>\n",
       "      <td>GGACTACHVGGGTWTCTAAT</td>\n",
       "      <td>16S stool samples sequenced for MrOS Vitamin D...</td>\n",
       "      <td>16S rRNA v4</td>\n",
       "      <td>GT</td>\n",
       "      <td>Illumina</td>\n",
       "      <td>BI</td>\n",
       "      <td>MrOS</td>\n",
       "      <td>Illumina MiSeq</td>\n",
       "      <td>MrOS_VitaminD</td>\n",
       "      <td>...</td>\n",
       "      <td>0.078388</td>\n",
       "      <td>1:GOOD/EXCELLENT</td>\n",
       "      <td>1: Yes</td>\n",
       "      <td>0:No</td>\n",
       "      <td>0:No</td>\n",
       "      <td>1: Yes</td>\n",
       "      <td>4.106798</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0: No</td>\n",
       "      <td>20.167914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BI0215</th>\n",
       "      <td>CCTCTCGTGATC</td>\n",
       "      <td>GGACTACHVGGGTWTCTAAT</td>\n",
       "      <td>16S stool samples sequenced for MrOS Vitamin D...</td>\n",
       "      <td>16S rRNA v4</td>\n",
       "      <td>GT</td>\n",
       "      <td>Illumina</td>\n",
       "      <td>BI</td>\n",
       "      <td>MrOS</td>\n",
       "      <td>Illumina MiSeq</td>\n",
       "      <td>MrOS_VitaminD</td>\n",
       "      <td>...</td>\n",
       "      <td>0.109697</td>\n",
       "      <td>1:GOOD/EXCELLENT</td>\n",
       "      <td>0: No</td>\n",
       "      <td>0:No</td>\n",
       "      <td>0:No</td>\n",
       "      <td>1: Yes</td>\n",
       "      <td>0.971114</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0: No</td>\n",
       "      <td>20.714331</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 75 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          BarcodeSequence  LinkerPrimerSequence  \\\n",
       "#SampleID                                         \n",
       "BI0023       TCTGGTGACATT  GGACTACHVGGGTWTCTAAT   \n",
       "BI0056       CAAGCATGCCTA  GGACTACHVGGGTWTCTAAT   \n",
       "BI0131       CTATTTGCGACA  GGACTACHVGGGTWTCTAAT   \n",
       "BI0153       ATCGGCGTTACA  GGACTACHVGGGTWTCTAAT   \n",
       "BI0215       CCTCTCGTGATC  GGACTACHVGGGTWTCTAAT   \n",
       "\n",
       "                               Experiment_Design_Description  \\\n",
       "#SampleID                                                      \n",
       "BI0023     16S stool samples sequenced for MrOS Vitamin D...   \n",
       "BI0056     16S stool samples sequenced for MrOS Vitamin D...   \n",
       "BI0131     16S stool samples sequenced for MrOS Vitamin D...   \n",
       "BI0153     16S stool samples sequenced for MrOS Vitamin D...   \n",
       "BI0215     16S stool samples sequenced for MrOS Vitamin D...   \n",
       "\n",
       "          Library_Construction_Protocol Linker  Platform Center_Name  \\\n",
       "#SampleID                                                              \n",
       "BI0023                      16S rRNA v4     GT  Illumina          BI   \n",
       "BI0056                      16S rRNA v4     GT  Illumina          BI   \n",
       "BI0131                      16S rRNA v4     GT  Illumina          BI   \n",
       "BI0153                      16S rRNA v4     GT  Illumina          BI   \n",
       "BI0215                      16S rRNA v4     GT  Illumina          BI   \n",
       "\n",
       "          Center_Project Instrument_Model          Title  ...  \\\n",
       "#SampleID                                                 ...   \n",
       "BI0023              MrOS   Illumina MiSeq  MrOS_VitaminD  ...   \n",
       "BI0056              MrOS   Illumina MiSeq  MrOS_VitaminD  ...   \n",
       "BI0131              MrOS   Illumina MiSeq  MrOS_VitaminD  ...   \n",
       "BI0153              MrOS   Illumina MiSeq  MrOS_VitaminD  ...   \n",
       "BI0215              MrOS   Illumina MiSeq  MrOS_VitaminD  ...   \n",
       "\n",
       "          ratio_catabolism            QLCOMP M1STATIN Antihistamine Laxative  \\\n",
       "#SampleID                                                                      \n",
       "BI0023            0.068605  1:GOOD/EXCELLENT    0: No          0:No     0:No   \n",
       "BI0056            0.099745  1:GOOD/EXCELLENT    0: No          0:No     0:No   \n",
       "BI0131            0.064502  1:GOOD/EXCELLENT   1: Yes          0:No     0:No   \n",
       "BI0153            0.078388  1:GOOD/EXCELLENT   1: Yes          0:No     0:No   \n",
       "BI0215            0.109697  1:GOOD/EXCELLENT    0: No          0:No     0:No   \n",
       "\n",
       "           MIDATA Rstarch_total Rstarches_c M1PPUMP   alpha_pd  \n",
       "#SampleID                                                       \n",
       "BI0023     1: Yes      3.066569         0.0   0: No  30.021398  \n",
       "BI0056     1: Yes      3.038136         0.0  1: Yes  18.414977  \n",
       "BI0131     1: Yes      2.840599         0.0   0: No  25.756952  \n",
       "BI0153     1: Yes      4.106798         0.0   0: No  20.167914  \n",
       "BI0215     1: Yes      0.971114         0.0   0: No  20.714331  \n",
       "\n",
       "[5 rows x 75 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(598, 6)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create extreme categories on Vitamin D measurements\n",
    "mf = mf[['OHV1D3', 'OHV24D3', 'OHVD3', 'ratio_activation', 'ratio_catabolism', 'VDstatus']]\n",
    "mf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OHV1D3</th>\n",
       "      <th>OHV24D3</th>\n",
       "      <th>OHVD3</th>\n",
       "      <th>ratio_activation</th>\n",
       "      <th>ratio_catabolism</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>566.000000</td>\n",
       "      <td>566.000000</td>\n",
       "      <td>555.000000</td>\n",
       "      <td>555.000000</td>\n",
       "      <td>555.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.057802</td>\n",
       "      <td>3.431343</td>\n",
       "      <td>35.224865</td>\n",
       "      <td>0.001774</td>\n",
       "      <td>0.094795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.019780</td>\n",
       "      <td>1.836358</td>\n",
       "      <td>12.461583</td>\n",
       "      <td>0.000735</td>\n",
       "      <td>0.029794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.010700</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>7.800000</td>\n",
       "      <td>0.000398</td>\n",
       "      <td>0.018788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.044125</td>\n",
       "      <td>2.167500</td>\n",
       "      <td>27.400000</td>\n",
       "      <td>0.001319</td>\n",
       "      <td>0.074162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.055500</td>\n",
       "      <td>3.185000</td>\n",
       "      <td>33.600000</td>\n",
       "      <td>0.001661</td>\n",
       "      <td>0.092838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.066350</td>\n",
       "      <td>4.237500</td>\n",
       "      <td>41.850000</td>\n",
       "      <td>0.002081</td>\n",
       "      <td>0.113049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.156000</td>\n",
       "      <td>14.070000</td>\n",
       "      <td>104.000000</td>\n",
       "      <td>0.006727</td>\n",
       "      <td>0.197786</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           OHV1D3     OHV24D3       OHVD3  ratio_activation  ratio_catabolism\n",
       "count  566.000000  566.000000  555.000000        555.000000        555.000000\n",
       "mean     0.057802    3.431343   35.224865          0.001774          0.094795\n",
       "std      0.019780    1.836358   12.461583          0.000735          0.029794\n",
       "min      0.010700    0.300000    7.800000          0.000398          0.018788\n",
       "25%      0.044125    2.167500   27.400000          0.001319          0.074162\n",
       "50%      0.055500    3.185000   33.600000          0.001661          0.092838\n",
       "75%      0.066350    4.237500   41.850000          0.002081          0.113049\n",
       "max      0.156000   14.070000  104.000000          0.006727          0.197786"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mf.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ssun/miniconda3/envs/python3-new/lib/python3.6/site-packages/ipykernel_launcher.py:1: MatplotlibDeprecationWarning: \n",
      "The 'normed' kwarg was deprecated in Matplotlib 2.1 and will be removed in 3.1. Use 'density' instead.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x10b8e5978>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD4CAYAAAATpHZ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAObklEQVR4nO3dfYxldX3H8fdHVqSC1qUMZFXaVYOk2ODSjtjGtsEaLLVJwWhTMcWltVkTIZFErdSaAP2LWpE0obFZA4qNzw9UkhJlJTzUxtLOAvK0RZBuEdiwg7TFh6S68O0f91Cn487MnXvPnbnz2/crubnnnnvuPZ/M7P3sb87TTVUhSWrDs9Y7gCSpP5a6JDXEUpekhljqktQQS12SGrJpLVd2zDHH1NatW9dyldPvvvsG9yeeuL45DnX+HjTFdu/e/XhVzQyz7JqW+tatW5mbm1vLVU6/004b3N9003qmkL8HTbEk/zHssm5+kaSGWOqS1BBLXZIaYqlLUkMsdUlqiKUuSQ2x1CWpIZa6JDXEUpekhqzpGaWaLrkkB51fF/nFKdJG5UhdkhpiqUtSQyx1SWqIpS5JDbHUJakhlrokNcRSl6SGWOqS1BBPPtqAPGlI0lJWHKknOT7JjUn2JLknybu6+RcneSTJHd3tDZOPK0lazjAj9QPAu6vqtiTPA3Yn2dU9d3lVfWhy8SRJq7FiqVfVPmBfN/29JHuAF006mCRp9Va1ozTJVuAU4NZu1vlJ7kxyVZLNS7xmR5K5JHPz8/NjhZUkLW/oUk9yFPBF4IKqehL4CPAyYBuDkfxlB3tdVe2sqtmqmp2ZmekhsiRpKUOVepJnMyj0T1bVlwCq6rGqeqqqngY+Cpw6uZiSpGEMc/RLgCuBPVX14QXztyxY7I3A3f3HkyStxjBHv7wGOAe4K8kd3bz3A2cn2QYUsBd4x0QSSpKGNszRL18HDna2y3X9x5EkjcMzStULz3KVpoPXfpGkhljqktQQS12SGmKpS1JDLHVJaoilLkkNsdQlqSGWuiQ1xFKXpIZY6pLUEEtdkhpiqUtSQyx1SWqIV2k8BCx1BcW+lpc0PRypS1JDLHVJaoilLkkNsdQlqSGWuiQ1xFKXpIZY6pLUEEtdkhriyUdTzJOAJK2WI3VJaoilLkkNsdQlqSGWuiQ1xB2lmqildvbWRbXGSaRDgyN1SWrIiqWe5PgkNybZk+SeJO/q5h+dZFeS+7v7zZOPK0lazjAj9QPAu6vqF4FfBc5LchJwIXBDVZ0A3NA9liStoxVLvar2VdVt3fT3gD3Ai4Azgau7xa4GzppUSEnScFa1TT3JVuAU4FbguKraB4PiB45d4jU7kswlmZufnx8vrSRpWUOXepKjgC8CF1TVk8O+rqp2VtVsVc3OzMyMklGSNKShSj3JsxkU+ier6kvd7MeSbOme3wLsn0xESdKwhjn6JcCVwJ6q+vCCp64FtnfT24Ev9x9PkrQaw5x89BrgHOCuJHd0894PXAp8LsnbgYeA359MRA3LqzpKWrHUq+rrwFJt8bp+40iSxuEZpZLUEEtdkhpiqUtSQyx1SWqIpS5JDbHUJakhlrokNcRSl6SGWOqS1BBLXZIaYqlLUkMsdUlqiKUuSQ2x1CWpIZa6JDXEUpekhljqktQQS12SGmKpS1JDLHVJaoilLkkNsdQlqSGWuiQ1xFKXpIZY6pLUEEtdkhpiqUtSQyx1SWqIpS5JDbHUJakhK5Z6kquS7E9y94J5Fyd5JMkd3e0Nk40pSRrGMCP1jwNnHGT+5VW1rbtd128sSdIoViz1qroFeGINskiSxjTONvXzk9zZbZ7ZvNRCSXYkmUsyNz8/P8bqJEkrGbXUPwK8DNgG7AMuW2rBqtpZVbNVNTszMzPi6iRJwxip1Kvqsap6qqqeBj4KnNpvLEnSKDaN8qIkW6pqX/fwjcDdyy2v5d2092Zee0nWO4akBqxY6kk+DZwGHJPkYeAi4LQk24AC9gLvmGBGSdKQViz1qjr7ILOvnEAWSdKYPKNUkhpiqUtSQyx1SWqIpS5JDbHUJakhlrokNcRSl6SGWOqS1BBLXZIaYqlLUkMsdUlqiKUuSQ2x1CWpIZa6JDXEUpekhljqktSQkb7OThpXlvj6vrqoelleOlQ5UpekhljqktQQS12SGmKpS1JDLHVJaoilLkkNsdQlqSGWuiQ1xFKXpIZY6pLUEEtdkhpiqUtSQ1Ys9SRXJdmf5O4F845OsivJ/d395snGlCQNY5irNH4cuAL4xIJ5FwI3VNWlSS7sHr+v/3htOdiVBm/cu/Y5JLVrxZF6Vd0CPLFo9pnA1d301cBZPeeSJI1g1G3qx1XVPoDu/tilFkyyI8lckrn5+fkRVydJGsbEd5RW1c6qmq2q2ZmZmUmvTpIOaaN+89FjSbZU1b4kW4D9fYbSoWupbziSNJxRR+rXAtu76e3Al/uJI0kaxzCHNH4a+AZwYpKHk7wduBQ4Pcn9wOndY0nSOltx80tVnb3EU6/rOYskaUyeUSpJDbHUJakhlrokNcRSl6SGWOqS1BBLXZIaMuoZpVqGZ0VKWi+O1CWpIZa6JDXEUpekhljqktQQd5RqQ1tqp3RdVKt6n5v23gzAaxe932rfR1pvjtQlqSGWuiQ1xFKXpIZY6pLUEEtdkhpiqUtSQyx1SWqIpS5JDfHkIzWpr5OS1nsd0mo5UpekhljqktQQS12SGmKpS1JD3FE6Br+2TtK0caQuSQ2x1CWpIZa6JDXEUpekhoy1ozTJXuB7wFPAgaqa7SOUJGk0fRz98tqqeryH95EkjcnNL5LUkHFLvYDrk+xOsuNgCyTZkWQuydz8/PyYq5MkLWfczS+vqapHkxwL7Eryb1V1y8IFqmonsBNgdnbWy9dpXS11wtiNa5xDmpSxRupV9Wh3vx+4Bji1j1CSpNGMXOpJjkzyvGemgdcDd/cVTJK0euNsfjkOuCbJM+/zqar6Si+pJEkjGbnUq+pB4JU9ZpEkjcmrNA7BqzFK2ig8Tl2SGmKpS1JDLHVJaoilLkkNsdQlqSGWuiQ1xFKXpIZY6pLUEE8+ktbIUiex1UVevFT9caQuSQ2x1CWpIZa6JDXEUpekhrijVFrGKFfoXO1r3IGqPjlSl6SGWOqS1BBLXZIaYqlLUkOa3VE6ys4nv7ZOcsftRudIXZIaYqlLUkMsdUlqyIbZpt7Xdj63m2ujW+1nwX/zP3Eo7C9wpC5JDbHUJakhlrokNcRSl6SGbJgdpdKhpq+rPU7acutd7c7bvg58WK8dn6P8LPrmSF2SGjJWqSc5I8l9SR5IcmFfoSRJoxm51JMcBvwN8DvAScDZSU7qK5gkafXGGamfCjxQVQ9W1Y+AzwBn9hNLkjSKVI228T7Jm4EzqupPusfnAK+uqvMXLbcD2NE9PBG4b/S4vToGeHy9QwxhI+TcCBnBnH0zZ7+Wy/kLVTUzzJuMc/TLwXbz/tT/EFW1E9g5xnomIslcVc2ud46VbIScGyEjmLNv5uxXXznH2fzyMHD8gscvBh4dL44kaRzjlPq/AickeUmSw4G3ANf2E0uSNIqRN79U1YEk5wNfBQ4Drqqqe3pLNnlTt0loCRsh50bICObsmzn71UvOkXeUSpKmj2eUSlJDLHVJakhzpb7SpQuSPCfJZ7vnb02ytZt/epLdSe7q7n9rGnMueP7nk3w/yXumNWeSk5N8I8k93c/1iGnLmeTZSa7u8u1J8meTyjhkzt9McluSA925IAuf257k/u62fRpzJtm24Hd+Z5I/mMacC55/fpJHklwxjRm7z/n13b/Nexf3wEFVVTM3Bjtsvw28FDgc+CZw0qJl3gn8bTf9FuCz3fQpwAu76V8CHpnGnAue/yLweeA905iTwU74O4FXdo9/DjhsCnO+FfhMN/1cYC+wdR1zbgVOBj4BvHnB/KOBB7v7zd305inM+XLghG76hcA+4AXTlnPB838NfAq4YhozAjcBp3fTRwHPXWmdrY3Uh7l0wZnA1d30F4DXJUlV3V5Vzxxnfw9wRJLnTFtOgCRnMfhQT/poo3Fyvh64s6q+CVBV362qp6YwZwFHJtkE/AzwI+DJ9cpZVXur6k7g6UWv/W1gV1U9UVX/CewCzpi2nFX1raq6v5t+FNgPDHUm5FrmBEjyK8BxwPUTyjdWxgyupbWpqnZ1y32/qn640gpbK/UXAd9Z8Pjhbt5Bl6mqA8B/MxhFLvQm4Paq+p9py5nkSOB9wCUTytZLTgYjtkry1e5Pyz+d0pxfAH7AYET5EPChqnpiHXNO4rWr1cu6kpzKYHT67Z5yLTZyziTPAi4D3juBXAuN87N8OfBfSb6U5PYkf5XBhRSX1dqXZAxz6YJll0nyCuAvGYw0J2WcnJcAl1fV97uB+ySNk3MT8OvAq4AfAjck2V1VN/QbcdkMwyxzKvAUg00Fm4F/TPK1qnqw34jLZpj0a1dr7HUl2QL8HbC9qn5qlNyTcXK+E7iuqr4z4c/ROBk3Ab/BYNPwQ8BngXOBK5d7UWsj9WEuXfB/y3R/cv8s8ET3+MXANcDbqmpSo4txc74a+GCSvcAFwPszOAls2nI+DNxcVY93fzJeB/zyFOZ8K/CVqvpxVe0H/gmY1HVCxrm0xlpelmOsdSV5PvAPwAeq6p97zrbQODl/DTi/+xx9CHhbkkv7jQeM/zu/vdt0cwD4e4b5DE1i58B63Rj8z/Yg8BJ+slPiFYuWOY//v8Psc930C7rl3zTNORctczGT3VE6zs9zM3Abg52Pm4CvAb87hTnfB3yMwYjqSOBe4OT1yrlg2Y/z0ztK/737uW7upo+ewpyHAzcAF0zq32UfORc9dy6T21E6zs/ysG75me7xx4DzVlznpH/wa30D3gB8i8F2vD/v5v0F8Hvd9BEMjhp5APgX4KXd/A8w2LZ6x4LbsdOWc9F7XMwES33cnMAfMtiZezfwwWnMyeCIgs93Oe8F3rvOOV/FYIT2A+C7wD0LXvvHXf4HgD+axpzd7/zHiz5H26Yt56L3OJcJlXoPv/PTGRxFdheD0j98pfV5mQBJakhr29Ql6ZBmqUtSQyx1SWqIpS5JDbHUJakhlrokNcRSl6SG/C8nbYFwW/ZxVgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "n, bins, patches = plt.hist(mf.OHV1D3.dropna(), 50, normed=1, facecolor='green')\n",
    "plt.axvline(x=mf['OHV1D3'].quantile([0.10]).values, color='r')\n",
    "plt.axvline(x=mf['OHV1D3'].quantile([0.90]).values, color='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "q1 = mf['OHV1D3'].quantile([0.10]).values\n",
    "q3 = mf['OHV1D3'].quantile([0.90]).values\n",
    "def V1D3_extreme(x):\n",
    "    if (x <= q1):\n",
    "        return(0)\n",
    "    elif (x >= q3):\n",
    "        return(1)\n",
    "    else:\n",
    "        return('NaN')\n",
    "mf['V1D3_cat'] = mf['OHV1D3'].apply(V1D3_extreme)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NaN    483\n",
       "1       58\n",
       "0       57\n",
       "Name: V1D3_cat, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mf['V1D3_cat'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = pd.read_csv('../data/57316_mros_otus_rare_exp/57316_feature-table-rare.txt', sep='\\t', skiprows = 1, index_col = \"#OTU ID\").transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>#OTU ID</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGACGCTCAACGTCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGATGCTCAACATCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG</th>\n",
       "      <th>TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGCAGGCGGCGGCGTAAGTCAGTTGTGAAATCGTGCGGCTTAACCGTGCAATTGCAGTTGATACTGCGTCGCTTGAGTGCACACAGGGATGTTGGAATTCATGG</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGAGGCTCAACCTCTGCACTGCAGCGCGAACTGGTCTTCTTGAGTACGCACAACGTGGGCGGAATTCGTGG</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGTTTGTTAAGCGTGTTGTGAAATGTCGGGGCTCAACCTGGGCATTGCAGCGCGAACTGGCAGACTTGAGTGCGCGGGAAGTAGGCGGAATTCGTCG</th>\n",
       "      <th>AACGTAGGTCACAAGCGTTGTCCGGAATTACTGGGTGTAAAGGGAGCGCAGGCGGGAAGACAAGTTGGAAGTGAAATCCATGGGCTCAACCCATGAACTGCTTTCAAAACTGTTTTTCTTGAGTAGTGCAGAGGTAGGCGGAATTCCCGG</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGCAGGTTAAGCGTGTTGTGAAATGTAGGGGCTCAACCTCTGCACTGCAGCGCGAACTGGCTTGCTTGAGTACGCACAACGTGGGCGGAATTCGTGG</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGTCTTATAAGCGTGTTGTGAAATGTCGGGGCTCAACCTGGGCATTGCAGCGCGAACTGTGAGACTTGAGTGCGCAGGAAGTAGGCGGAATTCGTCG</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGATGCTCAACATCTGAACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG</th>\n",
       "      <th>TACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATGTTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGG</th>\n",
       "      <th>...</th>\n",
       "      <th>TACGTATGGAGCAAGCGTTGTCCGGATTTACTGGGTGTAAAGGGCGTGTAGCCGGGAGGGCAAGTCAGATGTGAAATCCACGGGCTCAACTCGTGAACTGCATTGGAAACTGTCGTACTAGAGTGTCGGAGGGGTAAGCGGAATTCCTAG</th>\n",
       "      <th>TACGTATGGTGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGATAGGCAAGTCTGGAGTGAAAACCCAGGGCTCAACCCTGGGACTGCTTTGGAAACTGCAGATCTGGAGTGCTGGAGAGGTAAGCGGAATTCCTAG</th>\n",
       "      <th>TACGGAGGGGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGGGTACGTAGGTGGCCTTTTAAGCGTAGGGTATAAGGCAATGGCCCAACCATTGTTCGCCCTATGAACTGGAAGGCTTGAGTGCAGGAGAGGAAAGCGGAATTCCTAGT</th>\n",
       "      <th>TACGTAGGGGGCAAGCGTTGTCCGGAATGACTGGGCGTAAAGGGAGTGTAGGCGGCTTGGCAAGTTAGATGTGAAAGCCCGCAGCTCAACTGCGGAACTGCATCTAAAACTGTCAGGCTTGAGTGCAGGAGAGGTAAGTGGAACTCCTAG</th>\n",
       "      <th>TACGTATGGTGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGTGCGTAGGCGGGATGGCAAGTCAGATGTGAAATACCGGGGCTTAACCCCGGGGCTGCATTTGAAACTGTCGTTCTTGAGTGAAGTAGAGGCAGGCGGAATTCCTAG</th>\n",
       "      <th>TACGTAGGGGGCGAGCGTTGTCCGGAATGACTGGGCGTAAAGGGCGTGTAGGCGGCCGATTAAGTGTGGAGTGAAAGTCCACTTGTCAAGGGTGGAATTGCTTTGCAGACTGGTTGGCTTGAGTGCGGAAGAGGTAAGTGGAATTCCCAG</th>\n",
       "      <th>TACGTAGGGGGCAAGCGTTATCCGGAATTACTGGGTGTAAAGGGTGCGTAGGCGGTACGGCAAGTTTGATGTGAAACCCATGGGCTTAACCCATGGCATGCATCGAAAACTACTGAACTAGAGTGCAGGAGAGGAGAGTGGAATTCCTAG</th>\n",
       "      <th>TACGGAGGGGGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGACTGGCAAGTCTGATGTGAAAGGCGCGGGCTCAACCCGCGGACGGCACATGATACTGCGAGACTAGAGTAATGGAGGGGGAACCGGAATTCTCGG</th>\n",
       "      <th>TACGTAGGGGGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGAAGAGCAAGTCTGATGTGAAAGGCTGGGGCTTAACCCCAGGACTGCATTGGAAACTGTTTTTCTTGAGTGCCGGAGAGGTAAGCGGAATTCTCGG</th>\n",
       "      <th>TACGTAGGGAGCGAGCGTTGTCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGAATGGCAAGTCTGATGTGAGAGGCCGGGGCTCAACCCCGGGACTGCATTGGAAACTGTCAATCTAGAGTACCGGAGGGGTAAGTGGAATTCCTAG</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MN1837</th>\n",
       "      <td>3080.0</td>\n",
       "      <td>1062.0</td>\n",
       "      <td>527.0</td>\n",
       "      <td>505.0</td>\n",
       "      <td>499.0</td>\n",
       "      <td>459.0</td>\n",
       "      <td>361.0</td>\n",
       "      <td>365.0</td>\n",
       "      <td>287.0</td>\n",
       "      <td>262.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PO7410</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>178.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>790.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SD8837</th>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>427.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MN2250</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>533.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SD8603</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2490.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25292 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "#OTU ID  TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGACGCTCAACGTCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG  \\\n",
       "MN1837                                              3080.0                                                                                                        \n",
       "PO7410                                                 2.0                                                                                                        \n",
       "SD8837                                                 2.0                                                                                                        \n",
       "MN2250                                                 2.0                                                                                                        \n",
       "SD8603                                                 2.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGATGCTCAACATCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG  \\\n",
       "MN1837                                              1062.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 3.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGCAGGCGGCGGCGTAAGTCAGTTGTGAAATCGTGCGGCTTAACCGTGCAATTGCAGTTGATACTGCGTCGCTTGAGTGCACACAGGGATGTTGGAATTCATGG  \\\n",
       "MN1837                                               527.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGAGGCTCAACCTCTGCACTGCAGCGCGAACTGGTCTTCTTGAGTACGCACAACGTGGGCGGAATTCGTGG  \\\n",
       "MN1837                                               505.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGTTTGTTAAGCGTGTTGTGAAATGTCGGGGCTCAACCTGGGCATTGCAGCGCGAACTGGCAGACTTGAGTGCGCGGGAAGTAGGCGGAATTCGTCG  \\\n",
       "MN1837                                               499.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  AACGTAGGTCACAAGCGTTGTCCGGAATTACTGGGTGTAAAGGGAGCGCAGGCGGGAAGACAAGTTGGAAGTGAAATCCATGGGCTCAACCCATGAACTGCTTTCAAAACTGTTTTTCTTGAGTAGTGCAGAGGTAGGCGGAATTCCCGG  \\\n",
       "MN1837                                               459.0                                                                                                        \n",
       "PO7410                                               178.0                                                                                                        \n",
       "SD8837                                                46.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                55.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGCAGGTTAAGCGTGTTGTGAAATGTAGGGGCTCAACCTCTGCACTGCAGCGCGAACTGGCTTGCTTGAGTACGCACAACGTGGGCGGAATTCGTGG  \\\n",
       "MN1837                                               361.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGTCTTATAAGCGTGTTGTGAAATGTCGGGGCTCAACCTGGGCATTGCAGCGCGAACTGTGAGACTTGAGTGCGCAGGAAGTAGGCGGAATTCGTCG  \\\n",
       "MN1837                                               365.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGATGCTCAACATCTGAACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG  \\\n",
       "MN1837                                               287.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATGTTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGG  \\\n",
       "MN1837                                               262.0                                                                                                        \n",
       "PO7410                                               790.0                                                                                                        \n",
       "SD8837                                               427.0                                                                                                        \n",
       "MN2250                                               533.0                                                                                                        \n",
       "SD8603                                              2490.0                                                                                                        \n",
       "\n",
       "#OTU ID  ...  \\\n",
       "MN1837   ...   \n",
       "PO7410   ...   \n",
       "SD8837   ...   \n",
       "MN2250   ...   \n",
       "SD8603   ...   \n",
       "\n",
       "#OTU ID  TACGTATGGAGCAAGCGTTGTCCGGATTTACTGGGTGTAAAGGGCGTGTAGCCGGGAGGGCAAGTCAGATGTGAAATCCACGGGCTCAACTCGTGAACTGCATTGGAAACTGTCGTACTAGAGTGTCGGAGGGGTAAGCGGAATTCCTAG  \\\n",
       "MN1837                                                 0.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGTATGGTGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGATAGGCAAGTCTGGAGTGAAAACCCAGGGCTCAACCCTGGGACTGCTTTGGAAACTGCAGATCTGGAGTGCTGGAGAGGTAAGCGGAATTCCTAG  \\\n",
       "MN1837                                                 0.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGGAGGGGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGGGTACGTAGGTGGCCTTTTAAGCGTAGGGTATAAGGCAATGGCCCAACCATTGTTCGCCCTATGAACTGGAAGGCTTGAGTGCAGGAGAGGAAAGCGGAATTCCTAGT  \\\n",
       "MN1837                                                 0.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGTAGGGGGCAAGCGTTGTCCGGAATGACTGGGCGTAAAGGGAGTGTAGGCGGCTTGGCAAGTTAGATGTGAAAGCCCGCAGCTCAACTGCGGAACTGCATCTAAAACTGTCAGGCTTGAGTGCAGGAGAGGTAAGTGGAACTCCTAG  \\\n",
       "MN1837                                                 0.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGTATGGTGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGTGCGTAGGCGGGATGGCAAGTCAGATGTGAAATACCGGGGCTTAACCCCGGGGCTGCATTTGAAACTGTCGTTCTTGAGTGAAGTAGAGGCAGGCGGAATTCCTAG  \\\n",
       "MN1837                                                 0.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGTAGGGGGCGAGCGTTGTCCGGAATGACTGGGCGTAAAGGGCGTGTAGGCGGCCGATTAAGTGTGGAGTGAAAGTCCACTTGTCAAGGGTGGAATTGCTTTGCAGACTGGTTGGCTTGAGTGCGGAAGAGGTAAGTGGAATTCCCAG  \\\n",
       "MN1837                                                 0.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGTAGGGGGCAAGCGTTATCCGGAATTACTGGGTGTAAAGGGTGCGTAGGCGGTACGGCAAGTTTGATGTGAAACCCATGGGCTTAACCCATGGCATGCATCGAAAACTACTGAACTAGAGTGCAGGAGAGGAGAGTGGAATTCCTAG  \\\n",
       "MN1837                                                 0.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGGAGGGGGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGACTGGCAAGTCTGATGTGAAAGGCGCGGGCTCAACCCGCGGACGGCACATGATACTGCGAGACTAGAGTAATGGAGGGGGAACCGGAATTCTCGG  \\\n",
       "MN1837                                                 0.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGTAGGGGGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGAAGAGCAAGTCTGATGTGAAAGGCTGGGGCTTAACCCCAGGACTGCATTGGAAACTGTTTTTCTTGAGTGCCGGAGAGGTAAGCGGAATTCTCGG  \\\n",
       "MN1837                                                 0.0                                                                                                        \n",
       "PO7410                                                 0.0                                                                                                        \n",
       "SD8837                                                 0.0                                                                                                        \n",
       "MN2250                                                 0.0                                                                                                        \n",
       "SD8603                                                 0.0                                                                                                        \n",
       "\n",
       "#OTU ID  TACGTAGGGAGCGAGCGTTGTCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGAATGGCAAGTCTGATGTGAGAGGCCGGGGCTCAACCCCGGGACTGCATTGGAAACTGTCAATCTAGAGTACCGGAGGGGTAAGTGGAATTCCTAG  \n",
       "MN1837                                                 0.0                                                                                                       \n",
       "PO7410                                                 0.0                                                                                                       \n",
       "SD8837                                                 0.0                                                                                                       \n",
       "MN2250                                                 0.0                                                                                                       \n",
       "SD8603                                                 0.0                                                                                                       \n",
       "\n",
       "[5 rows x 25292 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(598, 25292)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['BI0023', 'BI0056', 'BI0131', 'BI0153', 'BI0215', 'BI0353', 'BI0371',\n",
       "       'BI0372', 'BI0380', 'BI0389',\n",
       "       ...\n",
       "       'SD8954', 'SD8964', 'SD8966', 'SD8972', 'SD8973', 'SD8985', 'SD8996',\n",
       "       'SD8999', 'SD9003', 'SD9009'],\n",
       "      dtype='object', name='#SampleID', length=598)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## match metadata and OTU table by sampleID\n",
    "sample_ids = mf.index\n",
    "sample_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = table.loc[sample_ids] # order rows in same order as matrix's rows (sampleIDs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>#OTU ID</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGACGCTCAACGTCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGATGCTCAACATCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG</th>\n",
       "      <th>TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGCAGGCGGCGGCGTAAGTCAGTTGTGAAATCGTGCGGCTTAACCGTGCAATTGCAGTTGATACTGCGTCGCTTGAGTGCACACAGGGATGTTGGAATTCATGG</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGAGGCTCAACCTCTGCACTGCAGCGCGAACTGGTCTTCTTGAGTACGCACAACGTGGGCGGAATTCGTGG</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGTTTGTTAAGCGTGTTGTGAAATGTCGGGGCTCAACCTGGGCATTGCAGCGCGAACTGGCAGACTTGAGTGCGCGGGAAGTAGGCGGAATTCGTCG</th>\n",
       "      <th>AACGTAGGTCACAAGCGTTGTCCGGAATTACTGGGTGTAAAGGGAGCGCAGGCGGGAAGACAAGTTGGAAGTGAAATCCATGGGCTCAACCCATGAACTGCTTTCAAAACTGTTTTTCTTGAGTAGTGCAGAGGTAGGCGGAATTCCCGG</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGCAGGTTAAGCGTGTTGTGAAATGTAGGGGCTCAACCTCTGCACTGCAGCGCGAACTGGCTTGCTTGAGTACGCACAACGTGGGCGGAATTCGTGG</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGTCTTATAAGCGTGTTGTGAAATGTCGGGGCTCAACCTGGGCATTGCAGCGCGAACTGTGAGACTTGAGTGCGCAGGAAGTAGGCGGAATTCGTCG</th>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGATGCTCAACATCTGAACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG</th>\n",
       "      <th>TACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATGTTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGG</th>\n",
       "      <th>...</th>\n",
       "      <th>TACGTATGGAGCAAGCGTTGTCCGGATTTACTGGGTGTAAAGGGCGTGTAGCCGGGAGGGCAAGTCAGATGTGAAATCCACGGGCTCAACTCGTGAACTGCATTGGAAACTGTCGTACTAGAGTGTCGGAGGGGTAAGCGGAATTCCTAG</th>\n",
       "      <th>TACGTATGGTGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGATAGGCAAGTCTGGAGTGAAAACCCAGGGCTCAACCCTGGGACTGCTTTGGAAACTGCAGATCTGGAGTGCTGGAGAGGTAAGCGGAATTCCTAG</th>\n",
       "      <th>TACGGAGGGGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGGGTACGTAGGTGGCCTTTTAAGCGTAGGGTATAAGGCAATGGCCCAACCATTGTTCGCCCTATGAACTGGAAGGCTTGAGTGCAGGAGAGGAAAGCGGAATTCCTAGT</th>\n",
       "      <th>TACGTAGGGGGCAAGCGTTGTCCGGAATGACTGGGCGTAAAGGGAGTGTAGGCGGCTTGGCAAGTTAGATGTGAAAGCCCGCAGCTCAACTGCGGAACTGCATCTAAAACTGTCAGGCTTGAGTGCAGGAGAGGTAAGTGGAACTCCTAG</th>\n",
       "      <th>TACGTATGGTGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGTGCGTAGGCGGGATGGCAAGTCAGATGTGAAATACCGGGGCTTAACCCCGGGGCTGCATTTGAAACTGTCGTTCTTGAGTGAAGTAGAGGCAGGCGGAATTCCTAG</th>\n",
       "      <th>TACGTAGGGGGCGAGCGTTGTCCGGAATGACTGGGCGTAAAGGGCGTGTAGGCGGCCGATTAAGTGTGGAGTGAAAGTCCACTTGTCAAGGGTGGAATTGCTTTGCAGACTGGTTGGCTTGAGTGCGGAAGAGGTAAGTGGAATTCCCAG</th>\n",
       "      <th>TACGTAGGGGGCAAGCGTTATCCGGAATTACTGGGTGTAAAGGGTGCGTAGGCGGTACGGCAAGTTTGATGTGAAACCCATGGGCTTAACCCATGGCATGCATCGAAAACTACTGAACTAGAGTGCAGGAGAGGAGAGTGGAATTCCTAG</th>\n",
       "      <th>TACGGAGGGGGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGACTGGCAAGTCTGATGTGAAAGGCGCGGGCTCAACCCGCGGACGGCACATGATACTGCGAGACTAGAGTAATGGAGGGGGAACCGGAATTCTCGG</th>\n",
       "      <th>TACGTAGGGGGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGAAGAGCAAGTCTGATGTGAAAGGCTGGGGCTTAACCCCAGGACTGCATTGGAAACTGTTTTTCTTGAGTGCCGGAGAGGTAAGCGGAATTCTCGG</th>\n",
       "      <th>TACGTAGGGAGCGAGCGTTGTCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGAATGGCAAGTCTGATGTGAGAGGCCGGGGCTCAACCCCGGGACTGCATTGGAAACTGTCAATCTAGAGTACCGGAGGGGTAAGTGGAATTCCTAG</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>#SampleID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BI0023</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>111.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2181.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BI0056</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3718.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BI0131</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BI0153</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>220.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>217.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BI0215</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>542.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25292 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "#OTU ID    TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGACGCTCAACGTCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   3.0                                                                                                        \n",
       "BI0056                                                   1.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   1.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGATGCTCAACATCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   1.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGCAGGCGGCGGCGTAAGTCAGTTGTGAAATCGTGCGGCTTAACCGTGCAATTGCAGTTGATACTGCGTCGCTTGAGTGCACACAGGGATGTTGGAATTCATGG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGAGGCTCAACCTCTGCACTGCAGCGCGAACTGGTCTTCTTGAGTACGCACAACGTGGGCGGAATTCGTGG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGTTTGTTAAGCGTGTTGTGAAATGTCGGGGCTCAACCTGGGCATTGCAGCGCGAACTGGCAGACTTGAGTGCGCGGGAAGTAGGCGGAATTCGTCG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    AACGTAGGTCACAAGCGTTGTCCGGAATTACTGGGTGTAAAGGGAGCGCAGGCGGGAAGACAAGTTGGAAGTGAAATCCATGGGCTCAACCCATGAACTGCTTTCAAAACTGTTTTTCTTGAGTAGTGCAGAGGTAGGCGGAATTCCCGG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                 111.0                                                                                                        \n",
       "BI0056                                                  64.0                                                                                                        \n",
       "BI0131                                                  43.0                                                                                                        \n",
       "BI0153                                                 220.0                                                                                                        \n",
       "BI0215                                                 176.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGCAGGTTAAGCGTGTTGTGAAATGTAGGGGCTCAACCTCTGCACTGCAGCGCGAACTGGCTTGCTTGAGTACGCACAACGTGGGCGGAATTCGTGG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGTCTTATAAGCGTGTTGTGAAATGTCGGGGCTCAACCTGGGCATTGCAGCGCGAACTGTGAGACTTGAGTGCGCAGGAAGTAGGCGGAATTCGTCG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGATGCTCAACATCTGAACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   1.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGGAGGATCCGAGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGATGGATGTTTAAGTCAGTTGTGAAAGTTTGCGGCTCAACCGTAAAATTGCAGTTGATACTGGATATCTTGAGTGCAGTTGAGGCAGGCGGAATTCGTGG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                2181.0                                                                                                        \n",
       "BI0056                                                3718.0                                                                                                        \n",
       "BI0131                                                 222.0                                                                                                        \n",
       "BI0153                                                 217.0                                                                                                        \n",
       "BI0215                                                 542.0                                                                                                        \n",
       "\n",
       "#OTU ID    ...  \\\n",
       "#SampleID  ...   \n",
       "BI0023     ...   \n",
       "BI0056     ...   \n",
       "BI0131     ...   \n",
       "BI0153     ...   \n",
       "BI0215     ...   \n",
       "\n",
       "#OTU ID    TACGTATGGAGCAAGCGTTGTCCGGATTTACTGGGTGTAAAGGGCGTGTAGCCGGGAGGGCAAGTCAGATGTGAAATCCACGGGCTCAACTCGTGAACTGCATTGGAAACTGTCGTACTAGAGTGTCGGAGGGGTAAGCGGAATTCCTAG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGTATGGTGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGATAGGCAAGTCTGGAGTGAAAACCCAGGGCTCAACCCTGGGACTGCTTTGGAAACTGCAGATCTGGAGTGCTGGAGAGGTAAGCGGAATTCCTAG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGGAGGGGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGGGTACGTAGGTGGCCTTTTAAGCGTAGGGTATAAGGCAATGGCCCAACCATTGTTCGCCCTATGAACTGGAAGGCTTGAGTGCAGGAGAGGAAAGCGGAATTCCTAGT  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGTAGGGGGCAAGCGTTGTCCGGAATGACTGGGCGTAAAGGGAGTGTAGGCGGCTTGGCAAGTTAGATGTGAAAGCCCGCAGCTCAACTGCGGAACTGCATCTAAAACTGTCAGGCTTGAGTGCAGGAGAGGTAAGTGGAACTCCTAG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGTATGGTGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGTGCGTAGGCGGGATGGCAAGTCAGATGTGAAATACCGGGGCTTAACCCCGGGGCTGCATTTGAAACTGTCGTTCTTGAGTGAAGTAGAGGCAGGCGGAATTCCTAG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGTAGGGGGCGAGCGTTGTCCGGAATGACTGGGCGTAAAGGGCGTGTAGGCGGCCGATTAAGTGTGGAGTGAAAGTCCACTTGTCAAGGGTGGAATTGCTTTGCAGACTGGTTGGCTTGAGTGCGGAAGAGGTAAGTGGAATTCCCAG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGTAGGGGGCAAGCGTTATCCGGAATTACTGGGTGTAAAGGGTGCGTAGGCGGTACGGCAAGTTTGATGTGAAACCCATGGGCTTAACCCATGGCATGCATCGAAAACTACTGAACTAGAGTGCAGGAGAGGAGAGTGGAATTCCTAG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGGAGGGGGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGACTGGCAAGTCTGATGTGAAAGGCGCGGGCTCAACCCGCGGACGGCACATGATACTGCGAGACTAGAGTAATGGAGGGGGAACCGGAATTCTCGG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGTAGGGGGCAAGCGTTATCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGAAGAGCAAGTCTGATGTGAAAGGCTGGGGCTTAACCCCAGGACTGCATTGGAAACTGTTTTTCTTGAGTGCCGGAGAGGTAAGCGGAATTCTCGG  \\\n",
       "#SampleID                                                                                                                                                           \n",
       "BI0023                                                   0.0                                                                                                        \n",
       "BI0056                                                   0.0                                                                                                        \n",
       "BI0131                                                   0.0                                                                                                        \n",
       "BI0153                                                   0.0                                                                                                        \n",
       "BI0215                                                   0.0                                                                                                        \n",
       "\n",
       "#OTU ID    TACGTAGGGAGCGAGCGTTGTCCGGATTTACTGGGTGTAAAGGGAGCGTAGACGGAATGGCAAGTCTGATGTGAGAGGCCGGGGCTCAACCCCGGGACTGCATTGGAAACTGTCAATCTAGAGTACCGGAGGGGTAAGTGGAATTCCTAG  \n",
       "#SampleID                                                                                                                                                          \n",
       "BI0023                                                   0.0                                                                                                       \n",
       "BI0056                                                   0.0                                                                                                       \n",
       "BI0131                                                   0.0                                                                                                       \n",
       "BI0153                                                   0.0                                                                                                       \n",
       "BI0215                                                   0.0                                                                                                       \n",
       "\n",
       "[5 rows x 25292 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## random forest classification "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1, 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(115, 7)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filter away samples with NaN values\n",
    "Y = mf[mf.V1D3_cat != 'NaN']\n",
    "Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(115, 25292)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = table.loc[Y.index]\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>290</th>\n",
       "      <th>291</th>\n",
       "      <th>292</th>\n",
       "      <th>293</th>\n",
       "      <th>294</th>\n",
       "      <th>295</th>\n",
       "      <th>296</th>\n",
       "      <th>297</th>\n",
       "      <th>298</th>\n",
       "      <th>299</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.00000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.0</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>114.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.219298</td>\n",
       "      <td>16.745614</td>\n",
       "      <td>0.368421</td>\n",
       "      <td>0.921053</td>\n",
       "      <td>5.00000</td>\n",
       "      <td>0.350877</td>\n",
       "      <td>1.377193</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.587719</td>\n",
       "      <td>0.868421</td>\n",
       "      <td>...</td>\n",
       "      <td>0.184211</td>\n",
       "      <td>1.078947</td>\n",
       "      <td>0.517544</td>\n",
       "      <td>0.096491</td>\n",
       "      <td>1.096491</td>\n",
       "      <td>0.745614</td>\n",
       "      <td>2.447368</td>\n",
       "      <td>0.078947</td>\n",
       "      <td>0.280702</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.761447</td>\n",
       "      <td>58.291275</td>\n",
       "      <td>3.840315</td>\n",
       "      <td>2.331564</td>\n",
       "      <td>10.01945</td>\n",
       "      <td>1.713984</td>\n",
       "      <td>5.497208</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.269621</td>\n",
       "      <td>3.098710</td>\n",
       "      <td>...</td>\n",
       "      <td>0.804476</td>\n",
       "      <td>3.814608</td>\n",
       "      <td>2.881840</td>\n",
       "      <td>0.398433</td>\n",
       "      <td>10.433073</td>\n",
       "      <td>2.367970</td>\n",
       "      <td>4.850129</td>\n",
       "      <td>0.329782</td>\n",
       "      <td>1.043384</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>472.000000</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>61.00000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>46.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>111.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 300 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              0           1           2           3          4           5    \\\n",
       "count  114.000000  114.000000  114.000000  114.000000  114.00000  114.000000   \n",
       "mean     0.219298   16.745614    0.368421    0.921053    5.00000    0.350877   \n",
       "std      0.761447   58.291275    3.840315    2.331564   10.01945    1.713984   \n",
       "min      0.000000    0.000000    0.000000    0.000000    0.00000    0.000000   \n",
       "25%      0.000000    0.000000    0.000000    0.000000    0.00000    0.000000   \n",
       "50%      0.000000    0.000000    0.000000    0.000000    0.00000    0.000000   \n",
       "75%      0.000000    2.000000    0.000000    0.000000    6.00000    0.000000   \n",
       "max      6.000000  472.000000   41.000000   14.000000   61.00000   13.000000   \n",
       "\n",
       "              6      7           8           9    ...         290         291  \\\n",
       "count  114.000000  114.0  114.000000  114.000000  ...  114.000000  114.000000   \n",
       "mean     1.377193    0.0    5.587719    0.868421  ...    0.184211    1.078947   \n",
       "std      5.497208    0.0    9.269621    3.098710  ...    0.804476    3.814608   \n",
       "min      0.000000    0.0    0.000000    0.000000  ...    0.000000    0.000000   \n",
       "25%      0.000000    0.0    0.000000    0.000000  ...    0.000000    0.000000   \n",
       "50%      0.000000    0.0    2.000000    0.000000  ...    0.000000    0.000000   \n",
       "75%      0.000000    0.0    7.000000    0.000000  ...    0.000000    0.000000   \n",
       "max     46.000000    0.0   73.000000   24.000000  ...    6.000000   29.000000   \n",
       "\n",
       "              292         293         294         295         296         297  \\\n",
       "count  114.000000  114.000000  114.000000  114.000000  114.000000  114.000000   \n",
       "mean     0.517544    0.096491    1.096491    0.745614    2.447368    0.078947   \n",
       "std      2.881840    0.398433   10.433073    2.367970    4.850129    0.329782   \n",
       "min      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "25%      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "50%      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "75%      0.000000    0.000000    0.000000    0.000000    2.000000    0.000000   \n",
       "max     26.000000    2.000000  111.000000   19.000000   24.000000    2.000000   \n",
       "\n",
       "              298    299  \n",
       "count  114.000000  114.0  \n",
       "mean     0.280702    0.0  \n",
       "std      1.043384    0.0  \n",
       "min      0.000000    0.0  \n",
       "25%      0.000000    0.0  \n",
       "50%      0.000000    0.0  \n",
       "75%      0.000000    0.0  \n",
       "max      7.000000    0.0  \n",
       "\n",
       "[8 rows x 300 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_describe = pd.DataFrame(X[1:115, 200:500])\n",
    "df_describe.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ssun/miniconda3/envs/python3-new/lib/python3.6/site-packages/ipykernel_launcher.py:4: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "# convert to numpy array\n",
    "# X - rarefied OTU table; Y - vitamin D categories\n",
    "y = pd.factorize(Y['V1D3_cat'])[0] # to get dtype 'int' instead of 'O'\n",
    "X = X.as_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(31)\n",
    "m = np.logspace(2, 8, num=9, base=2, dtype=int)\n",
    "p_grid = {\"max_features\": m,\n",
    "          \"min_samples_split\": [2, 3, 7],\n",
    "          \"min_samples_leaf\": [1, 3, 7],\n",
    "          \"bootstrap\": [True, False]}\n",
    "\n",
    "rf = RandomForestClassifier(n_estimators=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "nested_CV_search = NestedCV(model=rf, params_grid=p_grid , outer_kfolds=5, inner_kfolds=5, n_jobs = 4,\n",
    "                            cv_options={'metric':roc_auc_score, \n",
    "                                        'metric_score_indicator_lower':False, # higher auc score is better\n",
    "                                        'randomized_search_iter':30, \n",
    "                                        'predict_proba':True})\n",
    "nested_CV_search.fit(X=X,y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7264280164280164"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(nested_CV_search.outer_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7301587301587301"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.median(nested_CV_search.outer_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'bootstrap': True,\n",
       "  'max_features': 4,\n",
       "  'min_samples_leaf': 3,\n",
       "  'min_samples_split': 7},\n",
       " {'bootstrap': False,\n",
       "  'max_features': 19,\n",
       "  'min_samples_leaf': 3,\n",
       "  'min_samples_split': 7},\n",
       " {'bootstrap': True,\n",
       "  'max_features': 32,\n",
       "  'min_samples_leaf': 3,\n",
       "  'min_samples_split': 7},\n",
       " {'bootstrap': False,\n",
       "  'max_features': 6,\n",
       "  'min_samples_leaf': 1,\n",
       "  'min_samples_split': 3},\n",
       " {'bootstrap': True,\n",
       "  'max_features': 11,\n",
       "  'min_samples_leaf': 1,\n",
       "  'min_samples_split': 3}]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nested_CV_search.best_inner_params_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}