notebooks/bte_export.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "wxyJcqSap3YR"
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from io import BytesIO\n",
    "from pathlib import Path\n",
    "from zipfile import ZipFile\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import requests\n",
    "import requests_cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "AxuXSnJQEl-f"
   },
   "outputs": [],
   "source": [
    "requests_cache.install_cache(\"pfocr_cache\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "zUI3A6iFDaoL"
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "import rpy2.robjects as ro\n",
    "from rpy2.ipython import html\n",
    "from rpy2.robjects import default_converter, pandas2ri\n",
    "from rpy2.robjects.conversion import localconverter\n",
    "from rpy2.robjects.lib.dplyr import DataFrame\n",
    "from rpy2.robjects.packages import importr\n",
    "\n",
    "html.html_rdataframe = partial(html.html_rdataframe, table_class=\"docutils\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "AR6t3LGZEoXr"
   },
   "outputs": [],
   "source": [
    "pandas2ri.activate()\n",
    "base = importr(\"base\")\n",
    "readRDS = ro.r[\"readRDS\"]\n",
    "saveRDS = ro.r[\"saveRDS\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "zFYzw7reEohC"
   },
   "outputs": [],
   "source": [
    "def rds2pandas(rds_path):\n",
    "    r_df = readRDS(str(rds_path))\n",
    "    with localconverter(ro.default_converter + pandas2ri.converter):\n",
    "        pandas_df = ro.conversion.rpy2py(r_df)\n",
    "    return pandas_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "-XItpmjhEX85"
   },
   "outputs": [],
   "source": [
    "def pandas2rds(pandas_df, rds_path):\n",
    "    with localconverter(default_converter + pandas2ri.converter) as cv:\n",
    "        r_df = DataFrame(pandas_df)\n",
    "\n",
    "    saveRDS(r_df, str(rds_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "AhdBrNEGMkmF"
   },
   "outputs": [],
   "source": [
    "current_dir = Path(\"./\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "W6rxlDKpH8Uz",
    "outputId": "2a645042-0ebc-4654-e56d-6b8b4f7336d9"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'/content/pfocr_genes_draft.rds'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "latest_pfocr_data_url = (\n",
    "    \"https://www.dropbox.com/sh/nmm0pddamgmu32m/AACCNdFXJrpQRD-EZOPe73RCa?dl=1\"\n",
    ")\n",
    "\n",
    "zipfile = ZipFile(BytesIO(requests.get(latest_pfocr_data_url).content))\n",
    "figures_rds_path = current_dir.joinpath(\"pfocr_figures_draft.rds\")\n",
    "zipfile.extract(str(figures_rds_path))\n",
    "genes_rds_path = current_dir.joinpath(\"pfocr_genes_draft.rds\")\n",
    "zipfile.extract(str(genes_rds_path))\n",
    "chemicals_rds_path = current_dir.joinpath(\"pfocr_chemicals_draft.rds\")\n",
    "zipfile.extract(str(chemicals_rds_path))\n",
    "diseases_rds_path = current_dir.joinpath(\"pfocr_diseases_draft.rds\")\n",
    "zipfile.extract(str(diseases_rds_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "q7-RauAUKpdh",
    "outputId": "d476ea01-0807-4e15-f7ed-fed4cb08590a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-35c3eef0-0aa4-4fba-a932-43a26df24a30\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pfocr_id</th>\n",
       "      <th>figure_number</th>\n",
       "      <th>figure_caption</th>\n",
       "      <th>pmc_id</th>\n",
       "      <th>paper_title</th>\n",
       "      <th>reference</th>\n",
       "      <th>pmc_search_index</th>\n",
       "      <th>pathway_score</th>\n",
       "      <th>figure_title</th>\n",
       "      <th>organism</th>\n",
       "      <th>publication_year</th>\n",
       "      <th>paper_url</th>\n",
       "      <th>figure_page_url</th>\n",
       "      <th>figure_thumbnail_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PMC5732092__cshperspect-CYT-028522_F2.jpg</td>\n",
       "      <td>Figure 2</td>\n",
       "      <td>Interleukin (IL)-17RA/RC signaling pathways. I...</td>\n",
       "      <td>PMC5732092</td>\n",
       "      <td>Interleukin 17 Family Cytokines: Signaling Mec...</td>\n",
       "      <td>Leticia Monin, et al. Cold Spring Harb Perspec...</td>\n",
       "      <td>33056</td>\n",
       "      <td>0.900423</td>\n",
       "      <td>Interleukin (IL)-17RA/RC signaling pathways</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>2018</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.govpmc/articles/PMC57...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PMC5793760__cshperspect-TGF-022210_F4.jpg</td>\n",
       "      <td>Figure 4</td>\n",
       "      <td>TGF-β signaling pathways. TGF-βs signal by bin...</td>\n",
       "      <td>PMC5793760</td>\n",
       "      <td>TGF-β Signaling in Control of Cardiovascular F...</td>\n",
       "      <td>Marie-José Goumans, et al. Cold Spring Harb Pe...</td>\n",
       "      <td>54906</td>\n",
       "      <td>0.932830</td>\n",
       "      <td>TGFB signaling pathways</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>2018</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.govpmc/articles/PMC57...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PMC5793761__cshperspect-TGF-031989_F1.jpg</td>\n",
       "      <td>Figure 1</td>\n",
       "      <td>Bone morphogenetic protein (BMP) signaling pat...</td>\n",
       "      <td>PMC5793761</td>\n",
       "      <td>Bone Morphogenetic Proteins in Vascular Homeos...</td>\n",
       "      <td>Marie-José Goumans, et al. Cold Spring Harb Pe...</td>\n",
       "      <td>13758</td>\n",
       "      <td>0.966398</td>\n",
       "      <td>Bone morphogenetic protein (BMP) signaling pat...</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>2018</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.govpmc/articles/PMC57...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PMC5830892__cshperspect-CEL-027961_F2.jpg</td>\n",
       "      <td>Figure 2</td>\n",
       "      <td>FERM-binding partners of Crumbs3. Through its ...</td>\n",
       "      <td>PMC5830892</td>\n",
       "      <td>The Crumbs3 Polarity Protein.</td>\n",
       "      <td>Ben Margolis. Cold Spring Harb Perspect Biol. ...</td>\n",
       "      <td>38670</td>\n",
       "      <td>0.693111</td>\n",
       "      <td>FERM-binding partners of Crumbs3</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>2018</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.govpmc/articles/PMC58...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>PMC5830900__cshperspect-TGF-031997_F1.jpg</td>\n",
       "      <td>Figure 1</td>\n",
       "      <td>Role of the TGF-β family in mammary gland deve...</td>\n",
       "      <td>PMC5830900</td>\n",
       "      <td>TGF-β Family Signaling in Ductal Differentiati...</td>\n",
       "      <td>Kaoru Kahata, et al. Cold Spring Harb Perspect...</td>\n",
       "      <td>19123</td>\n",
       "      <td>0.600142</td>\n",
       "      <td>Role of the TGFB family in mammary gland devel...</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>2018</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.govpmc/articles/PMC58...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79945</th>\n",
       "      <td>PMC4216988__zh20221474360006.jpg</td>\n",
       "      <td>Fig. 6</td>\n",
       "      <td>Hypothetical mechanism of activation of ENaC b...</td>\n",
       "      <td>PMC4216988</td>\n",
       "      <td>Prostasin interacts with the epithelial Na+ ch...</td>\n",
       "      <td>Marcelo D. Carattino, et al. Am J Physiol Rena...</td>\n",
       "      <td>108774</td>\n",
       "      <td>0.143076</td>\n",
       "      <td>Hypothetical mechanism of activation of ENaC b...</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>2014</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79946</th>\n",
       "      <td>PMC2873070__nihms128887f5.jpg</td>\n",
       "      <td>Scheme 1</td>\n",
       "      <td>A schematic diagram of a proposed working mode...</td>\n",
       "      <td>PMC2873070</td>\n",
       "      <td>Apolipoprotein E mediates sulfatide depletion ...</td>\n",
       "      <td>Hua Cheng, et al. Neurobiol Aging. ;31(7):1188...</td>\n",
       "      <td>143547</td>\n",
       "      <td>0.127176</td>\n",
       "      <td>A schematic diagram of a proposed working mode...</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>None</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79947</th>\n",
       "      <td>PMC3651446__pnas.1220523110fig06.jpg</td>\n",
       "      <td>Fig. 6</td>\n",
       "      <td>Models for nucleation of centrosomal and kinet...</td>\n",
       "      <td>PMC3651446</td>\n",
       "      <td>Aurora kinase inhibitors reveal mechanisms of ...</td>\n",
       "      <td>Jiun-Ming Wu, et al. Proc Natl Acad Sci U S A....</td>\n",
       "      <td>159643</td>\n",
       "      <td>0.055546</td>\n",
       "      <td>Models for nucleation of centrosomal and kinet...</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>2013</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79948</th>\n",
       "      <td>PMC6770832__cancers-11-01236-g005.jpg</td>\n",
       "      <td>Figure 5</td>\n",
       "      <td>A simplified TGFβ pathway leading to p21 expre...</td>\n",
       "      <td>PMC6770832</td>\n",
       "      <td>Relevance of Non-Targeted Effects for Radiothe...</td>\n",
       "      <td>Carmel Mothersill, et al. Cancers (Basel). 201...</td>\n",
       "      <td>618</td>\n",
       "      <td>0.140041</td>\n",
       "      <td>Simplified TGFB pathway leading to p21 expression</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>2019</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79949</th>\n",
       "      <td>PMC4609065__12885_2015_1721_Fig5_HTML.jpg</td>\n",
       "      <td>Fig. 5</td>\n",
       "      <td>Hypothetical model of EGFR and ANXA3 to fluoro...</td>\n",
       "      <td>PMC4609065</td>\n",
       "      <td>Construction of possible integrated predictive...</td>\n",
       "      <td>Hiro Takahashi, et al. BMC Cancer. 2015;15:718.</td>\n",
       "      <td>20865</td>\n",
       "      <td>0.090798</td>\n",
       "      <td>EGFR and ANXA3 to fluoropyrimidine resistance ...</td>\n",
       "      <td>Homo sapiens</td>\n",
       "      <td>2015</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>79949 rows × 14 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-35c3eef0-0aa4-4fba-a932-43a26df24a30')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-35c3eef0-0aa4-4fba-a932-43a26df24a30 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-35c3eef0-0aa4-4fba-a932-43a26df24a30');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                                        pfocr_id figure_number  \\\n",
       "1      PMC5732092__cshperspect-CYT-028522_F2.jpg      Figure 2   \n",
       "2      PMC5793760__cshperspect-TGF-022210_F4.jpg      Figure 4   \n",
       "3      PMC5793761__cshperspect-TGF-031989_F1.jpg      Figure 1   \n",
       "4      PMC5830892__cshperspect-CEL-027961_F2.jpg      Figure 2   \n",
       "5      PMC5830900__cshperspect-TGF-031997_F1.jpg      Figure 1   \n",
       "...                                          ...           ...   \n",
       "79945           PMC4216988__zh20221474360006.jpg        Fig. 6   \n",
       "79946              PMC2873070__nihms128887f5.jpg      Scheme 1   \n",
       "79947       PMC3651446__pnas.1220523110fig06.jpg        Fig. 6   \n",
       "79948      PMC6770832__cancers-11-01236-g005.jpg      Figure 5   \n",
       "79949  PMC4609065__12885_2015_1721_Fig5_HTML.jpg        Fig. 5   \n",
       "\n",
       "                                          figure_caption      pmc_id  \\\n",
       "1      Interleukin (IL)-17RA/RC signaling pathways. I...  PMC5732092   \n",
       "2      TGF-β signaling pathways. TGF-βs signal by bin...  PMC5793760   \n",
       "3      Bone morphogenetic protein (BMP) signaling pat...  PMC5793761   \n",
       "4      FERM-binding partners of Crumbs3. Through its ...  PMC5830892   \n",
       "5      Role of the TGF-β family in mammary gland deve...  PMC5830900   \n",
       "...                                                  ...         ...   \n",
       "79945  Hypothetical mechanism of activation of ENaC b...  PMC4216988   \n",
       "79946  A schematic diagram of a proposed working mode...  PMC2873070   \n",
       "79947  Models for nucleation of centrosomal and kinet...  PMC3651446   \n",
       "79948  A simplified TGFβ pathway leading to p21 expre...  PMC6770832   \n",
       "79949  Hypothetical model of EGFR and ANXA3 to fluoro...  PMC4609065   \n",
       "\n",
       "                                             paper_title  \\\n",
       "1      Interleukin 17 Family Cytokines: Signaling Mec...   \n",
       "2      TGF-β Signaling in Control of Cardiovascular F...   \n",
       "3      Bone Morphogenetic Proteins in Vascular Homeos...   \n",
       "4                          The Crumbs3 Polarity Protein.   \n",
       "5      TGF-β Family Signaling in Ductal Differentiati...   \n",
       "...                                                  ...   \n",
       "79945  Prostasin interacts with the epithelial Na+ ch...   \n",
       "79946  Apolipoprotein E mediates sulfatide depletion ...   \n",
       "79947  Aurora kinase inhibitors reveal mechanisms of ...   \n",
       "79948  Relevance of Non-Targeted Effects for Radiothe...   \n",
       "79949  Construction of possible integrated predictive...   \n",
       "\n",
       "                                               reference pmc_search_index  \\\n",
       "1      Leticia Monin, et al. Cold Spring Harb Perspec...            33056   \n",
       "2      Marie-José Goumans, et al. Cold Spring Harb Pe...            54906   \n",
       "3      Marie-José Goumans, et al. Cold Spring Harb Pe...            13758   \n",
       "4      Ben Margolis. Cold Spring Harb Perspect Biol. ...            38670   \n",
       "5      Kaoru Kahata, et al. Cold Spring Harb Perspect...            19123   \n",
       "...                                                  ...              ...   \n",
       "79945  Marcelo D. Carattino, et al. Am J Physiol Rena...           108774   \n",
       "79946  Hua Cheng, et al. Neurobiol Aging. ;31(7):1188...           143547   \n",
       "79947  Jiun-Ming Wu, et al. Proc Natl Acad Sci U S A....           159643   \n",
       "79948  Carmel Mothersill, et al. Cancers (Basel). 201...              618   \n",
       "79949    Hiro Takahashi, et al. BMC Cancer. 2015;15:718.            20865   \n",
       "\n",
       "       pathway_score                                       figure_title  \\\n",
       "1           0.900423        Interleukin (IL)-17RA/RC signaling pathways   \n",
       "2           0.932830                            TGFB signaling pathways   \n",
       "3           0.966398  Bone morphogenetic protein (BMP) signaling pat...   \n",
       "4           0.693111                   FERM-binding partners of Crumbs3   \n",
       "5           0.600142  Role of the TGFB family in mammary gland devel...   \n",
       "...              ...                                                ...   \n",
       "79945       0.143076  Hypothetical mechanism of activation of ENaC b...   \n",
       "79946       0.127176  A schematic diagram of a proposed working mode...   \n",
       "79947       0.055546  Models for nucleation of centrosomal and kinet...   \n",
       "79948       0.140041  Simplified TGFB pathway leading to p21 expression   \n",
       "79949       0.090798  EGFR and ANXA3 to fluoropyrimidine resistance ...   \n",
       "\n",
       "           organism publication_year  \\\n",
       "1      Homo sapiens             2018   \n",
       "2      Homo sapiens             2018   \n",
       "3      Homo sapiens             2018   \n",
       "4      Homo sapiens             2018   \n",
       "5      Homo sapiens             2018   \n",
       "...             ...              ...   \n",
       "79945  Homo sapiens             2014   \n",
       "79946  Homo sapiens             None   \n",
       "79947  Homo sapiens             2013   \n",
       "79948  Homo sapiens             2019   \n",
       "79949  Homo sapiens             2015   \n",
       "\n",
       "                                               paper_url  \\\n",
       "1      https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...   \n",
       "2      https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...   \n",
       "3      https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...   \n",
       "4      https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...   \n",
       "5      https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...   \n",
       "...                                                  ...   \n",
       "79945  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4...   \n",
       "79946  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2...   \n",
       "79947  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3...   \n",
       "79948  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6...   \n",
       "79949  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4...   \n",
       "\n",
       "                                         figure_page_url  \\\n",
       "1      https://www.ncbi.nlm.nih.govpmc/articles/PMC57...   \n",
       "2      https://www.ncbi.nlm.nih.govpmc/articles/PMC57...   \n",
       "3      https://www.ncbi.nlm.nih.govpmc/articles/PMC57...   \n",
       "4      https://www.ncbi.nlm.nih.govpmc/articles/PMC58...   \n",
       "5      https://www.ncbi.nlm.nih.govpmc/articles/PMC58...   \n",
       "...                                                  ...   \n",
       "79945  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4...   \n",
       "79946  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2...   \n",
       "79947  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3...   \n",
       "79948  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6...   \n",
       "79949  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4...   \n",
       "\n",
       "                                    figure_thumbnail_url  \n",
       "1      https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...  \n",
       "2      https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...  \n",
       "3      https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...  \n",
       "4      https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...  \n",
       "5      https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5...  \n",
       "...                                                  ...  \n",
       "79945  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4...  \n",
       "79946  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2...  \n",
       "79947  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3...  \n",
       "79948  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6...  \n",
       "79949  https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4...  \n",
       "\n",
       "[79949 rows x 14 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "figures_df = rds2pandas(figures_rds_path).rename(\n",
    "    columns={\n",
    "        \"figid\": \"pfocr_id\",\n",
    "        \"pmcid\": \"pmc_id\",\n",
    "        \"filename\": \"figure_filename\",\n",
    "        \"number\": \"figure_number\",\n",
    "        \"pmc_ranked_result_index\": \"pmc_search_index\",\n",
    "        \"figtitle\": \"figure_title\",\n",
    "        \"papertitle\": \"paper_title\",\n",
    "        \"caption\": \"figure_caption\",\n",
    "        \"figlink\": \"relative_figure_page_url\",\n",
    "        \"reftext\": \"reference\",\n",
    "        \"year\": \"publication_year\",\n",
    "    }\n",
    ")\n",
    "figures_df[\"paper_url\"] = (\n",
    "    \"https://www.ncbi.nlm.nih.gov/pmc/articles/\" + figures_df[\"pmc_id\"]\n",
    ")\n",
    "\n",
    "figures_df[\"figure_page_url\"] = (\n",
    "    \"https://www.ncbi.nlm.nih.gov\" + figures_df[\"relative_figure_page_url\"]\n",
    ")\n",
    "\n",
    "figures_df[\"figure_thumbnail_url\"] = (\n",
    "    \"https://www.ncbi.nlm.nih.gov/pmc/articles/\"\n",
    "    + figures_df[\"pmc_id\"]\n",
    "    + \"/bin/\"\n",
    "    + figures_df[\"figure_filename\"]\n",
    ")\n",
    "\n",
    "figures_df.drop(\n",
    "    columns=[\n",
    "        \"figure_filename\",\n",
    "        \"relative_figure_page_url\",\n",
    "    ],\n",
    "    inplace=True,\n",
    ")\n",
    "\n",
    "figures_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chemicals_df = rds2pandas(chemicals_rds_path)\n",
    "chemicals_by_pfocr_id = dict()\n",
    "for pfocr_id, df in chemicals_df[[\"pfocr_id\", \"identifier\"]].groupby(\n",
    "    \"pfocr_id\"\n",
    "):\n",
    "    # TODO: could any of these be something other than MESH IDs?\n",
    "    chemicals_by_pfocr_id[pfocr_id] = set(df[\"identifier\"].dropna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "diseases_df = rds2pandas(diseases_rds_path)\n",
    "diseases_by_pfocr_id = dict()\n",
    "for pfocr_id, df in diseases_df[[\"pfocr_id\", \"identifier\"]].groupby(\"pfocr_id\"):\n",
    "    # TODO: could any of these be something other than MESH IDs?\n",
    "    diseases_by_pfocr_id[pfocr_id] = set(df[\"identifier\"].dropna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "s5sTnpXsHtZE"
   },
   "outputs": [],
   "source": [
    "genes_df = rds2pandas(genes_rds_path).rename(\n",
    "    columns={\n",
    "        \"figid\": \"pfocr_id\",\n",
    "        \"pmcid\": \"pmc_id\",\n",
    "        \"entrez\": \"ncbigene_id\",\n",
    "        \"word\": \"matched_ocr_text\",\n",
    "        \"symbol\": \"lexicon_term\",\n",
    "        \"source\": \"lexicon_term_source\",\n",
    "    }\n",
    ")\n",
    "genes_by_pfocr_id = dict()\n",
    "for pfocr_id, df in genes_df[[\"pfocr_id\", \"ncbigene_id\"]].groupby(\"pfocr_id\"):\n",
    "    genes_by_pfocr_id[pfocr_id] = set(df[\"ncbigene_id\"].dropna())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0_KNorjuHm6f"
   },
   "source": [
    "## Export\n",
    "\n",
    "https://github.com/wikipathways/pathway-figure-ocr/issues/16#issuecomment-1075574192\n",
    "```json\n",
    "{\n",
    "  \"_id\": \"PMC0000000__nihm00000000\",\n",
    "  \"associatedWith\": {\n",
    "    \"title\": \"signaling in immune response\",\n",
    "    \"figureUrl\": \"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC0000000/bin/nihm00000000.jpg\",\n",
    "    \"pmc\": \"PMC0000000\",\n",
    "    \"pubmed\": \"00000000\",\n",
    "    \"mentions\": {\n",
    "        \"diseases\": {\n",
    "              \"mesh\": [\"D000860\", \"D000333\"]\n",
    "        },\n",
    "        \"chemicals\": {\n",
    "               \"mesh\": [\"D0001\", \"D00002\"]\n",
    "        },\n",
    "        \"genes\": {\n",
    "               \"ncbigene\": [\"1234\", \"2345\"]\n",
    "        }\n",
    "     }\n",
    "}\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "D3ymA-mAHtUc"
   },
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/a/8230505/5354298\n",
    "class SetEncoder(json.JSONEncoder):\n",
    "    def default(self, obj):\n",
    "        if isinstance(obj, set):\n",
    "            return list(obj)\n",
    "        return json.JSONEncoder.default(self, obj)\n",
    "\n",
    "\n",
    "export_dir = current_dir.joinpath(\"export\")\n",
    "if not export_dir.exists():\n",
    "    export_dir.mkdir()\n",
    "\n",
    "with open(export_dir.joinpath(\"bte_chemicals_diseases_genes.ndjson\"), \"w\") as f:\n",
    "    for i, row in figures_df.iterrows():\n",
    "        pfocr_id = row[\"pfocr_id\"]\n",
    "\n",
    "        chemicals = chemicals_by_pfocr_id.get(pfocr_id, set())\n",
    "        diseases = diseases_by_pfocr_id.get(pfocr_id, set())\n",
    "        genes = genes_by_pfocr_id.get(pfocr_id, set())\n",
    "\n",
    "        if chemicals or diseases or genes:\n",
    "            f.write(\n",
    "                json.dumps(\n",
    "                    {\n",
    "                        \"_id\": pfocr_id,\n",
    "                        \"associatedWith\": {\n",
    "                            \"title\": row[\"figure_title\"],\n",
    "                            \"figureUrl\": row[\"figure_thumbnail_url\"],\n",
    "                            \"pmc\": row[\"pmc_id\"],\n",
    "                            # \"pubmed\": \"\",\n",
    "                            \"mentions\": {\n",
    "                                \"chemicals\": {\"mesh\": chemicals},\n",
    "                                \"diseases\": {\"mesh\": diseases},\n",
    "                                \"genes\": {\"ncbigene\": genes},\n",
    "                            },\n",
    "                        },\n",
    "                    },\n",
    "                    cls=SetEncoder,\n",
    "                )\n",
    "            )\n",
    "\n",
    "            f.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZVGH0ZE_VGCr",
    "outputId": "8d3fc4bd-c658-4497-8993-63c405447bc9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"_id\": \"PMC5732092__cshperspect-CYT-028522_F2.jpg\", \"associatedWith\": {\"title\": \"Interleukin (IL)-17RA/RC signaling pathways\", \"figureUrl\": \"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5732092/bin/cshperspect-CYT-028522_F2.jpg\", \"pmc\": \"PMC5732092\", \"mentions\": {\"chemicals\": {\"mesh\": []}, \"diseases\": {\"mesh\": []}, \"genes\": {\"ncbigene\": [\"84818\", \"5196\", \"10131\", \"7188\", \"6387\", \"4791\", \"1994\", \"4283\", \"2920\", \"6426\", \"5966\", \"5473\", \"4117\", \"1432\", \"7189\", \"6374\", \"7186\", \"6300\", \"5594\", \"3627\", \"2919\", \"3326\", \"1051\", \"5601\", \"4790\", \"10758\", \"5603\", \"6373\", \"10563\", \"5600\", \"3576\", \"23765\", \"2921\", \"5602\", \"3605\", \"5971\", \"6372\", \"5599\", \"9547\", \"5595\", \"112744\", \"7184\", \"5970\"]}}}}\n",
      "{\"_id\": \"PMC5793760__cshperspect-TGF-022210_F4.jpg\", \"associatedWith\": {\"title\": \"TGFB signaling pathways\", \"figureUrl\": \"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5793760/bin/cshperspect-TGF-022210_F4.jpg\", \"pmc\": \"PMC5793760\", \"mentions\": {\"chemicals\": {\"mesh\": []}, \"diseases\": {\"mesh\": []}, \"genes\": {\"ncbigene\": [\"658\", \"7040\", \"650\", \"4088\", \"657\", \"7042\", \"4090\", \"652\", \"392255\", \"27302\", \"659\", \"656\", \"2658\", \"8200\", \"4089\", \"649\", \"7043\", \"654\", \"651\", \"653\", \"9210\", \"4087\", \"4086\", \"353500\", \"7048\", \"90\", \"9573\", \"6498\", \"7046\", \"655\", \"2657\", \"151449\"]}}}}\n",
      "{\"_id\": \"PMC5793761__cshperspect-TGF-031989_F1.jpg\", \"associatedWith\": {\"title\": \"Bone morphogenetic protein (BMP) signaling pathway\", \"figureUrl\": \"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5793761/bin/cshperspect-TGF-031989_F1.jpg\", \"pmc\": \"PMC5793761\", \"mentions\": {\"chemicals\": {\"mesh\": []}, \"diseases\": {\"mesh\": []}, \"genes\": {\"ncbigene\": [\"658\", \"650\", \"657\", \"4090\", \"652\", \"392255\", \"25805\", \"4117\", \"2077\", \"27302\", \"1432\", \"659\", \"656\", \"5594\", \"4093\", \"64750\", \"2658\", \"8200\", \"168667\", \"4091\", \"4089\", \"649\", \"57154\", \"654\", \"5601\", \"651\", \"653\", \"56963\", \"9210\", \"5603\", \"4086\", \"5600\", \"4035\", \"353500\", \"5602\", \"6445\", \"90\", \"9573\", \"4256\", \"5599\", \"5595\", \"4092\", \"655\", \"6300\", \"2657\", \"151449\"]}}}}\n",
      "{\"_id\": \"PMC5830892__cshperspect-CEL-027961_F2.jpg\", \"associatedWith\": {\"title\": \"FERM-binding partners of Crumbs3\", \"figureUrl\": \"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5830892/bin/cshperspect-CEL-027961_F2.jpg\", \"pmc\": \"PMC5830892\", \"mentions\": {\"chemicals\": {\"mesh\": []}, \"diseases\": {\"mesh\": []}, \"genes\": {\"ncbigene\": [\"71\", \"59\", \"58\", \"2119\", \"72\", \"70\", \"55249\", \"60\"]}}}}\n",
      "{\"_id\": \"PMC5830900__cshperspect-TGF-031997_F1.jpg\", \"associatedWith\": {\"title\": \"Role of the TGFB family in mammary gland development and maturation\", \"figureUrl\": \"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5830900/bin/cshperspect-TGF-031997_F1.jpg\", \"pmc\": \"PMC5830900\", \"mentions\": {\"chemicals\": {\"mesh\": [\"D011374\"]}, \"diseases\": {\"mesh\": []}, \"genes\": {\"ncbigene\": [\"6776\", \"7040\", \"650\", \"6777\", \"4088\", \"7042\", \"4090\", \"3082\", \"652\", \"392255\", \"3479\", \"27302\", \"4488\", \"656\", \"3625\", \"4093\", \"2658\", \"8200\", \"4091\", \"4089\", \"649\", \"7043\", \"3624\", \"654\", \"651\", \"653\", \"9210\", \"4086\", \"4087\", \"353500\", \"9573\", \"6498\", \"4092\", \"655\", \"2657\", \"151449\"]}}}}\n",
      "{\"_id\": \"PMC5843509__nihms906417f3.jpg\", \"associatedWith\": {\"title\": \"Design Principles of Electrical Synaptic Plasticity\", \"figureUrl\": \"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5843509/bin/nihms906417f3.jpg\", \"pmc\": \"PMC5843509\", \"mentions\": {\"chemicals\": {\"mesh\": [\"D018350\", \"D004298\", \"D002118\", \"C094686\"]}, \"diseases\": {\"mesh\": []}, \"genes\": {\"ncbigene\": [\"109\", \"111\", \"114\", \"889\", \"113\", \"112\", \"5493\", \"613\", \"115\", \"196883\", \"108\", \"9735\", \"55811\", \"107\", \"816\", \"8740\", \"57369\"]}}}}\n",
      "{\"_id\": \"PMC5880173__cshperspect-TGF-022327_F1.jpg\", \"associatedWith\": {\"title\": \"Bone morphogenetic protein (BMP) pathway and potential strategies for therapeutic modulation\", \"figureUrl\": \"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5880173/bin/cshperspect-TGF-022327_F1.jpg\", \"pmc\": \"PMC5880173\", \"mentions\": {\"chemicals\": {\"mesh\": []}, \"diseases\": {\"mesh\": []}, \"genes\": {\"ncbigene\": [\"650\", \"657\", \"5293\", \"10000\", \"652\", \"4090\", \"392255\", \"5291\", \"1432\", \"27302\", \"659\", \"656\", \"6300\", \"5594\", \"8503\", \"4093\", \"2658\", \"8200\", \"4089\", \"649\", \"654\", \"30849\", \"651\", \"5294\", \"653\", \"5603\", \"9210\", \"23533\", \"5600\", \"4086\", \"353500\", \"207\", \"208\", \"146850\", \"5602\", \"90\", \"9573\", \"5599\", \"5290\", \"5595\", \"92\", \"655\", \"5601\", \"2657\", \"151449\"]}}}}\n",
      "{\"_id\": \"PMC5920792__nihms917800f1.jpg\", \"associatedWith\": {\"title\": \"Feedback Regulation of RTK Signaling in Development\", \"figureUrl\": \"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5920792/bin/nihms917800f1.jpg\", \"pmc\": \"PMC5920792\", \"mentions\": {\"chemicals\": {\"mesh\": []}, \"diseases\": {\"mesh\": []}, \"genes\": {\"ncbigene\": [\"2261\", \"54756\", \"2260\", \"53834\", \"2264\", \"23767\", \"2263\"]}}}}\n",
      "{\"_id\": \"PMC5920792__nihms917800f2.jpg\", \"associatedWith\": {\"title\": \"Feedback Regulation of RTK Signaling in Development\", \"figureUrl\": \"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5920792/bin/nihms917800f2.jpg\", \"pmc\": \"PMC5920792\", \"mentions\": {\"chemicals\": {\"mesh\": []}, \"diseases\": {\"mesh\": []}, \"genes\": {\"ncbigene\": [\"2885\", \"1846\", \"3845\", \"3265\", \"4893\", \"5293\", \"10000\", \"6903\", \"5291\", \"5605\", \"5604\", \"5728\", \"10252\", \"1431\", \"161742\", \"8503\", \"5594\", \"5781\", \"10251\", \"81848\", \"5037\", \"1844\", \"30849\", \"5294\", \"10253\", \"369\", \"1843\", \"23533\", \"6654\", \"6655\", \"5894\", \"207\", \"146850\", \"208\", \"1845\", \"1848\", \"399473\", \"5290\", \"5595\", \"2475\", \"673\", \"200734\", \"1847\"]}}}}\n",
      "{\"_id\": \"PMC5920792__nihms917800f3.jpg\", \"associatedWith\": {\"title\": \"Feedback Regulation of RTK Signaling in Development\", \"figureUrl\": \"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5920792/bin/nihms917800f3.jpg\", \"pmc\": \"PMC5920792\", \"mentions\": {\"chemicals\": {\"mesh\": []}, \"diseases\": {\"mesh\": [\"C000656865\"]}, \"genes\": {\"ncbigene\": [\"867\", \"60495\", \"23327\", \"1154\", \"4734\"]}}}}\n"
     ]
    }
   ],
   "source": [
    "!head export/bte_chemicals_diseases_genes.ndjson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "8QBP-60GaUHM"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyPMwUPl/glVw3tOGzrE2vhG",
   "collapsed_sections": [],
   "include_colab_link": true,
   "name": "bte_export.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python3 - mypkgs",
   "language": "python",
   "name": "ipython_mypkgs"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}