1.3. Prepare biom table (Python).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import biom\n",
    "from biom.util import biom_open\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ref: https://github.com/sjanssen2/ggmap/blob/master/ggmap/snippets.py\n",
    "def biom2pandas(file_biom, withTaxonomy=False, astype=int):\n",
    "    \"\"\" Converts a biom file into a Pandas.DataFrame\n",
    "    Parameters\n",
    "    ----------\n",
    "    file_biom : str\n",
    "        The path to the biom file.\n",
    "    withTaxonomy : bool\n",
    "        If TRUE, returns a second Pandas.Series with lineage information for\n",
    "        each feature, e.g. OTU or deblur-sequence. Default: FALSE\n",
    "    astype : type\n",
    "        datatype into each value of the biom table is casted. Default: int.\n",
    "        Use e.g. float if biom table contains relative abundances instead of\n",
    "        raw reads.\n",
    "    Returns\n",
    "    -------\n",
    "    A Pandas.DataFrame holding holding numerical values from the biom file.\n",
    "    If withTaxonomy is TRUE then a second Pandas.DataFrame is returned, holding\n",
    "    lineage information about each feature.\n",
    "    Raises\n",
    "    ------\n",
    "    IOError\n",
    "        If file_biom cannot be read.\n",
    "    ValueError\n",
    "        If withTaxonomy=TRUE but biom file does not hold taxonomy information.\n",
    "    \"\"\"\n",
    "    try:\n",
    "        table = biom.load_table(file_biom)\n",
    "        counts = pd.DataFrame(table.matrix_data.T.todense().astype(astype),\n",
    "                              index=table.ids(axis='sample'),\n",
    "                              columns=table.ids(axis='observation')).T\n",
    "        if withTaxonomy:\n",
    "            try:\n",
    "                md = table.metadata_to_dataframe('observation')\n",
    "                levels = [col\n",
    "                          for col in md.columns\n",
    "                          if col.startswith('taxonomy_')]\n",
    "                if levels == []:\n",
    "                    raise ValueError(('No taxonomy information found in '\n",
    "                                      'biom file.'))\n",
    "                else:\n",
    "                    taxonomy = md.apply(lambda row:\n",
    "                                        \";\".join([row[l] for l in levels]),\n",
    "                                        axis=1)\n",
    "                    return counts, taxonomy\n",
    "            except KeyError:\n",
    "                raise ValueError(('Biom file does not have any '\n",
    "                                  'observation metadata!'))\n",
    "        else:\n",
    "            return counts\n",
    "    except IOError:\n",
    "        raise IOError('Cannot read file \"%s\"' % file_biom)\n",
    "\n",
    "\n",
    "def pandas2biom(file_biom, table, taxonomy=None, err=sys.stderr):\n",
    "    \"\"\" Writes a Pandas.DataFrame into a biom file.\n",
    "    Parameters\n",
    "    ----------\n",
    "    file_biom: str\n",
    "        The filename of the BIOM file to be created.\n",
    "    table: a Pandas.DataFrame\n",
    "        The table that should be written as BIOM.\n",
    "    taxonomy : pandas.Series\n",
    "        Index is taxons corresponding to table, values are lineage strings like\n",
    "        'k__Bacteria; p__Actinobacteria'\n",
    "    err : StringIO\n",
    "        Stream onto which errors / warnings should be printed.\n",
    "        Default is sys.stderr\n",
    "    Raises\n",
    "    ------\n",
    "    IOError\n",
    "        If file_biom cannot be written.\n",
    "    TODO\n",
    "    ----\n",
    "        1) also store taxonomy information\n",
    "    \"\"\"\n",
    "    try:\n",
    "        bt = biom.Table(table.values,\n",
    "                        observation_ids=table.index,\n",
    "                        sample_ids=table.columns)\n",
    "\n",
    "        # add taxonomy metadata if provided, i.e. is not None\n",
    "        if taxonomy is not None:\n",
    "            if not isinstance(taxonomy, pd.core.series.Series):\n",
    "                raise AttributeError('taxonomy must be a pandas.Series!')\n",
    "            idx_missing_intable = set(table.index) - set(taxonomy.index)\n",
    "            if len(idx_missing_intable) > 0:\n",
    "                err.write(('Warning: following %i taxa are not in the '\n",
    "                           'provided taxonomy:\\n%s\\n') % (\n",
    "                          len(idx_missing_intable),\n",
    "                          \", \".join(idx_missing_intable)))\n",
    "                missing = pd.Series(\n",
    "                    index=idx_missing_intable,\n",
    "                    name='taxonomy',\n",
    "                    data='k__missing_lineage_information')\n",
    "                taxonomy = taxonomy.append(missing)\n",
    "            idx_missing_intaxonomy = set(taxonomy.index) - set(table.index)\n",
    "            if (len(idx_missing_intaxonomy) > 0) and err:\n",
    "                err.write(('Warning: following %i taxa are not in the '\n",
    "                           'provided count table, but in taxonomy:\\n%s\\n') % (\n",
    "                          len(idx_missing_intaxonomy),\n",
    "                          \", \".join(idx_missing_intaxonomy)))\n",
    "\n",
    "            t = dict()\n",
    "            for taxon, linstr in taxonomy.iteritems():\n",
    "                # fill missing rank annotations with rank__\n",
    "                orig_lineage = {annot[0].lower(): annot\n",
    "                                for annot\n",
    "                                in (map(str.strip, linstr.split(';')))}\n",
    "                lineage = []\n",
    "                for rank in settings.RANKS:\n",
    "                    rank_char = rank[0].lower()\n",
    "                    if rank_char in orig_lineage:\n",
    "                        lineage.append(orig_lineage[rank_char])\n",
    "                    else:\n",
    "                        lineage.append(rank_char+'__')\n",
    "                t[taxon] = {'taxonomy': \";\".join(lineage)}\n",
    "            bt.add_metadata(t, axis='observation')\n",
    "\n",
    "        with biom_open(file_biom, 'w') as f:\n",
    "            bt.to_hdf5(f, \"example\")\n",
    "    except IOError:\n",
    "        raise IOError('Cannot write to file \"%s\"' % file_biom)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "microb_unrare = biom2pandas('../Qiita_Study11274_ID57316/57316_reference-hit.biom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(43660, 599)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "microb_unrare.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>11274.MN1837</th>\n",
       "      <th>11274.PO7410</th>\n",
       "      <th>11274.SD8837</th>\n",
       "      <th>11274.MN2250</th>\n",
       "      <th>11274.SD8603</th>\n",
       "      <th>11274.PI5263</th>\n",
       "      <th>11274.MN2373</th>\n",
       "      <th>11274.PI5340</th>\n",
       "      <th>11274.MN1590</th>\n",
       "      <th>11274.PI4847</th>\n",
       "      <th>...</th>\n",
       "      <th>11274.BI0904</th>\n",
       "      <th>11274.PA3993</th>\n",
       "      <th>11274.PA3846</th>\n",
       "      <th>11274.PO7476</th>\n",
       "      <th>11274.BI0730</th>\n",
       "      <th>11274.PO6998</th>\n",
       "      <th>11274.BI0539</th>\n",
       "      <th>11274.MN2181</th>\n",
       "      <th>11274.PO7226</th>\n",
       "      <th>11274.BI0552</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGACGCTCAACGTCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG</th>\n",
       "      <td>9178</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>493</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>524</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGATGCTCAACATCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG</th>\n",
       "      <td>3054</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>291</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1209</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGCAGGCGGCGGCGTAAGTCAGTTGTGAAATCGTGCGGCTTAACCGTGCAATTGCAGTTGATACTGCGTCGCTTGAGTGCACACAGGGATGTTGGAATTCATGG</th>\n",
       "      <td>1608</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 599 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    11274.MN1837  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...          9178   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...          3054   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...          1608   \n",
       "\n",
       "                                                    11274.PO7410  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             4   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.SD8837  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             2   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             4   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.MN2250  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             2   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.SD8603  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             6   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.PI5263  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             5   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             3   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.MN2373  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             2   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.PI5340  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             2   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.MN1590  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.PI4847  ...  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             3  ...   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0  ...   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0  ...   \n",
       "\n",
       "                                                    11274.BI0904  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             5   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.PA3993  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.PA3846  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.PO7476  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             7   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.BI0730  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...           493   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...           291   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.PO6998  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             3   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             2   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.BI0539  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.MN2181  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0   \n",
       "\n",
       "                                                    11274.PO7226  11274.BI0552  \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...           524             0  \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...          1209             0  \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...             0             0  \n",
       "\n",
       "[3 rows x 599 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "microb_unrare.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# correct subject ID\n",
    "microb_unrare.columns = microb_unrare.columns.str[6:13].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MN1837</th>\n",
       "      <th>PO7410</th>\n",
       "      <th>SD8837</th>\n",
       "      <th>MN2250</th>\n",
       "      <th>SD8603</th>\n",
       "      <th>PI5263</th>\n",
       "      <th>MN2373</th>\n",
       "      <th>PI5340</th>\n",
       "      <th>MN1590</th>\n",
       "      <th>PI4847</th>\n",
       "      <th>...</th>\n",
       "      <th>BI0904</th>\n",
       "      <th>PA3993</th>\n",
       "      <th>PA3846</th>\n",
       "      <th>PO7476</th>\n",
       "      <th>BI0730</th>\n",
       "      <th>PO6998</th>\n",
       "      <th>BI0539</th>\n",
       "      <th>MN2181</th>\n",
       "      <th>PO7226</th>\n",
       "      <th>BI0552</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGACGCTCAACGTCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG</th>\n",
       "      <td>9178</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>493</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>524</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGTAGGCCGGAGATTAAGCGTGTTGTGAAATGTAGATGCTCAACATCTGCACTGCAGCGCGAACTGGTTTCCTTGAGTACGCACAAAGTGGGCGGAATTCGTGG</th>\n",
       "      <td>3054</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>291</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1209</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGCGCAGGCGGCGGCGTAAGTCAGTTGTGAAATCGTGCGGCTTAACCGTGCAATTGCAGTTGATACTGCGTCGCTTGAGTGCACACAGGGATGTTGGAATTCATGG</th>\n",
       "      <td>1608</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 599 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    MN1837  PO7410  SD8837  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...    9178       4       2   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...    3054       0       4   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...    1608       0       0   \n",
       "\n",
       "                                                    MN2250  SD8603  PI5263  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       2       6       5   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       0       0       3   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       0       0       0   \n",
       "\n",
       "                                                    MN2373  PI5340  MN1590  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       2       2       0   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       0       0       0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       0       0       0   \n",
       "\n",
       "                                                    PI4847  ...  BI0904  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       3  ...       5   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       0  ...       0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       0  ...       0   \n",
       "\n",
       "                                                    PA3993  PA3846  PO7476  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       0       0       7   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       0       0       0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       0       0       0   \n",
       "\n",
       "                                                    BI0730  PO6998  BI0539  \\\n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...     493       3       0   \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...     291       2       0   \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       0       0       0   \n",
       "\n",
       "                                                    MN2181  PO7226  BI0552  \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       0     524       0  \n",
       "TACGGAAGGTCCGGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       0    1209       0  \n",
       "TACGGAAGGTCCAGGCGTTATCCGGATTTATTGGGTTTAAAGGGAGC...       0       0       0  \n",
       "\n",
       "[3 rows x 599 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "microb_unrare.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pandas2biom('../data/57316_mros_deblur_otus_unrare.biom', microb_unrare)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### simple examination of the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count      599.000000\n",
       "mean     34003.534224\n",
       "std       6480.764579\n",
       "min      10117.000000\n",
       "25%      30310.000000\n",
       "50%      33768.000000\n",
       "75%      37009.500000\n",
       "max      78762.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "microb_unrare.sum(axis=0).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.min(microb_unrare.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28840"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(microb_unrare.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7788257953208011"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(microb_unrare.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49.61767283321315"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.std(microb_unrare.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7384845868476778"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(np.count_nonzero(microb_unrare.values)/(microb_unrare.shape[0]*microb_unrare.shape[1]))*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}