script_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "53df6f72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setup complete\n"
     ]
    }
   ],
   "source": [
    "import palimpzest as pz\n",
    "import pandas as pd\n",
    "import time\n",
    "import os\n",
    "import IPython\n",
    "\n",
    "formatter = IPython.get_ipython().display_formatter.formatters['text/plain']\n",
    "formatter.max_seq_length = 0\n",
    "\n",
    "# set OPENAI_API_KEY environment variable based on OPENAI_API_KEY\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"{{ OPENAI_API_KEY }}\" \n",
    "\n",
    "class ScientificPaper(pz.PDFFile):\n",
    "   \"\"\"Represents a scientific research paper, which in practice is usually from a PDF file\"\"\"\n",
    "   paper_title = pz.Field(desc=\"The title of the paper. This is a natural language title, not a number or letter.\", required=True)\n",
    "   author = pz.Field(desc=\"The name of the first author of the paper\", required=True)\n",
    "   abstract = pz.Field(desc=\"A short description of the paper contributions and findings\", required=False)\n",
    "\n",
    "class Reference(pz.Schema):\n",
    "    \"\"\" Represents a reference to another paper, which is cited in a scientific paper\"\"\"\n",
    "    index = pz.Field(desc=\"The index of the reference in the paper\", required=True)\n",
    "    title = pz.Field(desc=\"The title of the paper being cited\", required=True)\n",
    "    first_author = pz.Field(desc=\"The author of the paper being cited\", required=True)\n",
    "    year = pz.Field(desc=\"The year in which the cited paper was published\", required=True)\n",
    "\n",
    "from palimpzest.corelib.schemas import File, Number, TextFile, RawJSONObject, PDFFile, ImageFile, EquationImage, PlotImage, URL, Download, WebPage, XLSFile, Table\n",
    "\n",
    "existing_schemas = {\n",
    "    \"File\": File,\n",
    "    \"Number\": Number,\n",
    "    \"TextFile\": TextFile,\n",
    "    \"RawJSONObject\": RawJSONObject,\n",
    "    \"PDFFile\": PDFFile,\n",
    "    \"ImageFile\": ImageFile,\n",
    "    \"EquationImage\": EquationImage,\n",
    "    \"PlotImage\": PlotImage,\n",
    "    \"URL\": URL,\n",
    "    \"Download\": Download,\n",
    "    \"WebPage\": WebPage,\n",
    "    \"XLSFile\": XLSFile,\n",
    "    \"Table\": Table,\n",
    "    \"ScientificPaper\": ScientificPaper,\n",
    "    \"Reference\": Reference\n",
    "}\n",
    "\n",
    "print(\"Setup complete\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39f830aa",
   "metadata": {},
   "source": [
    "What are the datasets available in the system?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "40ed3fff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "    <thead>\n",
       "        <tr>\n",
       "            <th>Name</th>\n",
       "            <th>Type</th>\n",
       "            <th>Path</th>\n",
       "        </tr>\n",
       "    </thead>\n",
       "    <tbody>\n",
       "        <tr>\n",
       "            <td>real-estate-eval-20</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/real-estate-eval-20</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>biofabric-tiny-filtered</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/biofabric-tiny-filtered</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>real-estate-eval-tiny</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/real-estate-eval-tiny</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>groundtruth</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/groundtruth</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>askem</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/askem</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>enron-eval</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/enron-eval</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>pdfs-tiny</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/pdfs-tiny</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>enron-tiny</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/enron-tiny</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>enron-small</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/enron-small</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>bdf-usecase3-references-pdffull</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/bdf-usecase3-references-pdffull</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>bdf-usecase3-tiny</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/bdf-pz/testdata/bdf-usecase3-tiny</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>real-estate-eval-15</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/real-estate-eval-15</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>skilling-raptor-emails.txt</td>\n",
       "            <td>file</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/skilling-raptor-emails.txt</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>vldbdownload</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/vldbdownload</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>images-tiny</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/images-tiny</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>bdf-usecase3-references</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/bdf-usecase3-references</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>equation-tiny</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/equation-tiny</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>biofabric-urls</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/biofabric-urls</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>enron-eval-tiny</td>\n",
       "            <td>dir</td>\n",
       "            <td>testdata/enron-eval-tiny</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>biofabric-pdf</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/biofabric-pdf</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>biofabric-tiny</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/biofabric-tiny</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>bdf-usecase3-references-pdf</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/bdf-usecase3-references-pdf</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>target_matching.csv</td>\n",
       "            <td>file</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/target_matching.csv</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>real-estate-eval</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/real-estate-eval</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>biofabric-html</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/biofabric-html</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>bdf-usecase3-pdf</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/bdf-usecase3-pdf</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>real-estate-eval-5</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/real-estate-eval-5</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>real-estate-eval-25</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/real-estate-eval-25</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>real-estate-eval-30</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/real-estate-eval-30</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>biofabric-matching</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/biofabric-matching</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>biofabric-medium</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/biofabric-medium</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>askem-tiny</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/askem-tiny</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>real-estate-eval-10</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/real-estate-eval-10</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>biofabric-mimic</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/biofabric-mimic</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>movies-title</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/movies-title</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>movies-rating</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/palimpzest/testdata/movies-rating</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>bdf-demo</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/bdf-pz/testdata/bdf-demo</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>schwannoma-tiny</td>\n",
       "            <td>dir</td>\n",
       "            <td>/home/gerardo/synapse-pz/testdata/schwannoma-tiny</td>\n",
       "        </tr>\n",
       "    </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "+---------------------------------+------+-------------------------------------------------------------------+\n",
       "|               Name              | Type |                                Path                               |\n",
       "+---------------------------------+------+-------------------------------------------------------------------+\n",
       "|       real-estate-eval-20       | dir  |       /home/gerardo/palimpzest/testdata/real-estate-eval-20       |\n",
       "|     biofabric-tiny-filtered     | dir  |     /home/gerardo/palimpzest/testdata/biofabric-tiny-filtered     |\n",
       "|      real-estate-eval-tiny      | dir  |      /home/gerardo/palimpzest/testdata/real-estate-eval-tiny      |\n",
       "|           groundtruth           | dir  |           /home/gerardo/palimpzest/testdata/groundtruth           |\n",
       "|              askem              | dir  |              /home/gerardo/palimpzest/testdata/askem              |\n",
       "|            enron-eval           | dir  |            /home/gerardo/palimpzest/testdata/enron-eval           |\n",
       "|            pdfs-tiny            | dir  |            /home/gerardo/palimpzest/testdata/pdfs-tiny            |\n",
       "|            enron-tiny           | dir  |            /home/gerardo/palimpzest/testdata/enron-tiny           |\n",
       "|           enron-small           | dir  |           /home/gerardo/palimpzest/testdata/enron-small           |\n",
       "| bdf-usecase3-references-pdffull | dir  | /home/gerardo/palimpzest/testdata/bdf-usecase3-references-pdffull |\n",
       "|        bdf-usecase3-tiny        | dir  |          /home/gerardo/bdf-pz/testdata/bdf-usecase3-tiny          |\n",
       "|       real-estate-eval-15       | dir  |       /home/gerardo/palimpzest/testdata/real-estate-eval-15       |\n",
       "|    skilling-raptor-emails.txt   | file |    /home/gerardo/palimpzest/testdata/skilling-raptor-emails.txt   |\n",
       "|           vldbdownload          | dir  |           /home/gerardo/palimpzest/testdata/vldbdownload          |\n",
       "|           images-tiny           | dir  |           /home/gerardo/palimpzest/testdata/images-tiny           |\n",
       "|     bdf-usecase3-references     | dir  |     /home/gerardo/palimpzest/testdata/bdf-usecase3-references     |\n",
       "|          equation-tiny          | dir  |          /home/gerardo/palimpzest/testdata/equation-tiny          |\n",
       "|          biofabric-urls         | dir  |          /home/gerardo/palimpzest/testdata/biofabric-urls         |\n",
       "|         enron-eval-tiny         | dir  |                      testdata/enron-eval-tiny                     |\n",
       "|          biofabric-pdf          | dir  |          /home/gerardo/palimpzest/testdata/biofabric-pdf          |\n",
       "|          biofabric-tiny         | dir  |          /home/gerardo/palimpzest/testdata/biofabric-tiny         |\n",
       "|   bdf-usecase3-references-pdf   | dir  |   /home/gerardo/palimpzest/testdata/bdf-usecase3-references-pdf   |\n",
       "|       target_matching.csv       | file |       /home/gerardo/palimpzest/testdata/target_matching.csv       |\n",
       "|         real-estate-eval        | dir  |         /home/gerardo/palimpzest/testdata/real-estate-eval        |\n",
       "|          biofabric-html         | dir  |          /home/gerardo/palimpzest/testdata/biofabric-html         |\n",
       "|         bdf-usecase3-pdf        | dir  |         /home/gerardo/palimpzest/testdata/bdf-usecase3-pdf        |\n",
       "|        real-estate-eval-5       | dir  |        /home/gerardo/palimpzest/testdata/real-estate-eval-5       |\n",
       "|       real-estate-eval-25       | dir  |       /home/gerardo/palimpzest/testdata/real-estate-eval-25       |\n",
       "|       real-estate-eval-30       | dir  |       /home/gerardo/palimpzest/testdata/real-estate-eval-30       |\n",
       "|        biofabric-matching       | dir  |        /home/gerardo/palimpzest/testdata/biofabric-matching       |\n",
       "|         biofabric-medium        | dir  |         /home/gerardo/palimpzest/testdata/biofabric-medium        |\n",
       "|            askem-tiny           | dir  |            /home/gerardo/palimpzest/testdata/askem-tiny           |\n",
       "|       real-estate-eval-10       | dir  |       /home/gerardo/palimpzest/testdata/real-estate-eval-10       |\n",
       "|         biofabric-mimic         | dir  |         /home/gerardo/palimpzest/testdata/biofabric-mimic         |\n",
       "|           movies-title          | dir  |           /home/gerardo/palimpzest/testdata/movies-title          |\n",
       "|          movies-rating          | dir  |          /home/gerardo/palimpzest/testdata/movies-rating          |\n",
       "|             bdf-demo            | dir  |               /home/gerardo/bdf-pz/testdata/bdf-demo              |\n",
       "|         schwannoma-tiny         | dir  |         /home/gerardo/synapse-pz/testdata/schwannoma-tiny         |\n",
       "+---------------------------------+------+-------------------------------------------------------------------+"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from prettytable import PrettyTable\n",
    "\n",
    "ds = pz.DataDirectory().list_registered_datasets()\n",
    "\n",
    "# construct table for printing\n",
    "table = [[\"Name\", \"Type\", \"Path\"]]\n",
    "for path, descriptor in ds:\n",
    "    table.append([path, descriptor[0], descriptor[1]])\n",
    "\n",
    "# print table of registered datasets\n",
    "t = PrettyTable(table[0])\n",
    "t.add_rows(table[1:])\n",
    "t"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "558fbbcb",
   "metadata": {},
   "source": [
    "Show me the files available in bdf-demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a2e8a7a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bolderson2010.pdf',\n",
       " 'bonfiglio2023.pdf',\n",
       " 'marques2021.pdf',\n",
       " 'brunyanszki2014.pdf',\n",
       " 'liu2024.pdf',\n",
       " 'li2021.pdf',\n",
       " 'zethoven2022.pdf',\n",
       " 'vasaikar2019.pdf',\n",
       " 'ceccaldi2015.pdf',\n",
       " 'munkhbaatar2020.pdf',\n",
       " 'gupte2017.pdf']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "files = os.listdir(\"/home/gerardo/bdf-pz/testdata/bdf-demo\")\n",
    "\n",
    "files"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1006af8",
   "metadata": {},
   "source": [
    "Create a schema to extract authors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fe45d526",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "__main__.Author"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define a class name\n",
    "class_name = \"Author\"\n",
    "\n",
    "# Create the class dynamically\n",
    "attributes = {\"__doc__\": \" Schema for extracting author information from documents.\"}\n",
    "\n",
    "# Custom __repr__ to output detailed information about the class and its fields\n",
    "def custom_repr(self):\n",
    "    class_info = [f\"{class_name}: {self.__doc__}\"]\n",
    "    for name, field in self.__class__.__dict__.items():\n",
    "        if isinstance(field, pz.Field):\n",
    "            class_info.append(f\"{name}: description='{field.desc}', required={field.required}\")\n",
    "    return \"\\n\".join(class_info)\n",
    "\n",
    "# Add the custom __repr__ method to the class attributes\n",
    "attributes[\"__repr__\"] = custom_repr\n",
    "\n",
    "for name, desc, required in zip(['name', 'affiliation', 'email'], ['The full name of the author.', 'The affiliation of the author.', 'The email address of the author.'], [True, False, False]):\n",
    "    attributes[name] = pz.Field(desc=desc, required=required)\n",
    "\n",
    "# Create the class dynamically using type()\n",
    "new_class = type(class_name, (pz.Schema,), attributes)\n",
    "\n",
    "# Assign the dynamically created class to a variable using globals()\n",
    "globals()[class_name] = new_class\n",
    "existing_schemas[class_name] = new_class\n",
    "new_class"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fb28f34",
   "metadata": {},
   "source": [
    "Consider the bdf-demo dataset. This is a dataset of scientific papers. Extract all authors of papers about brain cancer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ec2cf727",
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    schema = existing_schemas[\"Author\"]\n",
    "except KeyError:\n",
    "    raise ValueError(f\"Schema 'Author' not found in existing schemas!\")\n",
    "dataset = pz.Dataset(source=\"bdf-demo\", schema=schema)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c3e57e12",
   "metadata": {},
   "outputs": [],
   "source": [
    "condition = \"The paper is about brain cancer.\"\n",
    "dataset = dataset.filter(condition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "19c1fd70",
   "metadata": {},
   "outputs": [],
   "source": [
    "convert_schema = Author\n",
    "cardinality_str = \"one_to_many\"\n",
    "cardinality = pz.Cardinality.ONE_TO_MANY if cardinality_str == \"one_to_many\" else pz.Cardinality.ONE_TO_ONE\n",
    "dataset = dataset.convert(convert_schema, desc=Author.__doc__, cardinality=cardinality)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17a863cf",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'PDFFile.56364ce7be.contents'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[19], line 24\u001b[0m\n\u001b[1;32m     21\u001b[0m results \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m     22\u001b[0m statistics \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 24\u001b[0m \u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43midx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mextraction\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mplan\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstats\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43menumerate\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43miterable\u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[1;32m     26\u001b[0m \u001b[43m    \u001b[49m\u001b[43mrecord_time\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mtime\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtime\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     27\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstatistics\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mappend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstats\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/palimpzest/src/palimpzest/execution/streaming_execution.py:93\u001b[0m, in \u001b[0;36mStreamingSequentialExecution.execute\u001b[0;34m(self, dataset, policy)\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[38;5;66;03m# Always delete cache\u001b[39;00m\n\u001b[1;32m     92\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mplan_generated:\n\u001b[0;32m---> 93\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate_plan\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpolicy\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     95\u001b[0m input_records \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_input_records()\n\u001b[1;32m     96\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m idx, record \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(input_records):\n\u001b[1;32m     97\u001b[0m     \u001b[38;5;66;03m# print(\"Iteration number: \", idx+1, \"out of\", len(input_records))\u001b[39;00m\n",
      "File \u001b[0;32m~/palimpzest/src/palimpzest/execution/streaming_execution.py:73\u001b[0m, in \u001b[0;36mStreamingSequentialExecution.generate_plan\u001b[0;34m(self, dataset, policy)\u001b[0m\n\u001b[1;32m     59\u001b[0m optimizer \u001b[38;5;241m=\u001b[39m Optimizer(\n\u001b[1;32m     60\u001b[0m     policy\u001b[38;5;241m=\u001b[39mpolicy,\n\u001b[1;32m     61\u001b[0m     cost_model\u001b[38;5;241m=\u001b[39mcost_model,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     69\u001b[0m     optimization_strategy\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moptimization_strategy,\n\u001b[1;32m     70\u001b[0m )\n\u001b[1;32m     72\u001b[0m \u001b[38;5;66;03m# Effectively always use the optimal strategy\u001b[39;00m\n\u001b[0;32m---> 73\u001b[0m plans \u001b[38;5;241m=\u001b[39m \u001b[43moptimizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptimize\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     74\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mplan \u001b[38;5;241m=\u001b[39m plans[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m     75\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mplan_stats \u001b[38;5;241m=\u001b[39m PlanStats(plan_id\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mplan\u001b[38;5;241m.\u001b[39mplan_id)\n",
      "File \u001b[0;32m~/palimpzest/src/palimpzest/optimizer/optimizer.py:450\u001b[0m, in \u001b[0;36mOptimizer.optimize\u001b[0;34m(self, query_plan)\u001b[0m\n\u001b[1;32m    445\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    446\u001b[0m \u001b[38;5;124;03mThe optimize function takes in an initial query plan and searches the space of\u001b[39;00m\n\u001b[1;32m    447\u001b[0m \u001b[38;5;124;03mlogical and physical plans in order to cost and produce a (near) optimal physical plan.\u001b[39;00m\n\u001b[1;32m    448\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    449\u001b[0m \u001b[38;5;66;03m# compute the initial group tree for the user plan\u001b[39;00m\n\u001b[0;32m--> 450\u001b[0m final_group_id \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconvert_query_plan_to_group_tree\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery_plan\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    452\u001b[0m \u001b[38;5;66;03m# use transformation rules if we're not doing sentinel optimization\u001b[39;00m\n\u001b[1;32m    453\u001b[0m include_transformations \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moptimization_strategy \u001b[38;5;241m!=\u001b[39m OptimizationStrategy\u001b[38;5;241m.\u001b[39mSENTINEL\n",
      "File \u001b[0;32m~/palimpzest/src/palimpzest/optimizer/optimizer.py:303\u001b[0m, in \u001b[0;36mOptimizer.convert_query_plan_to_group_tree\u001b[0;34m(self, query_plan)\u001b[0m\n\u001b[1;32m    301\u001b[0m \u001b[38;5;66;03m# If the node already has depends_on specified, then resolve each field name to a full (unique) field name\u001b[39;00m\n\u001b[1;32m    302\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(node\u001b[38;5;241m.\u001b[39m_depends_on) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 303\u001b[0m     node\u001b[38;5;241m.\u001b[39m_depends_on \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(\u001b[38;5;28mmap\u001b[39m(\u001b[38;5;28;01mlambda\u001b[39;00m field: short_to_full_field_name[field], node\u001b[38;5;241m.\u001b[39m_depends_on))\n\u001b[1;32m    304\u001b[0m     \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[1;32m    306\u001b[0m \u001b[38;5;66;03m# otherwise, make the node depend on all upstream nodes\u001b[39;00m\n",
      "File \u001b[0;32m~/palimpzest/src/palimpzest/optimizer/optimizer.py:303\u001b[0m, in \u001b[0;36mOptimizer.convert_query_plan_to_group_tree.<locals>.<lambda>\u001b[0;34m(field)\u001b[0m\n\u001b[1;32m    301\u001b[0m \u001b[38;5;66;03m# If the node already has depends_on specified, then resolve each field name to a full (unique) field name\u001b[39;00m\n\u001b[1;32m    302\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(node\u001b[38;5;241m.\u001b[39m_depends_on) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 303\u001b[0m     node\u001b[38;5;241m.\u001b[39m_depends_on \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(\u001b[38;5;28mmap\u001b[39m(\u001b[38;5;28;01mlambda\u001b[39;00m field: \u001b[43mshort_to_full_field_name\u001b[49m\u001b[43m[\u001b[49m\u001b[43mfield\u001b[49m\u001b[43m]\u001b[49m, node\u001b[38;5;241m.\u001b[39m_depends_on))\n\u001b[1;32m    304\u001b[0m     \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[1;32m    306\u001b[0m \u001b[38;5;66;03m# otherwise, make the node depend on all upstream nodes\u001b[39;00m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'PDFFile.56364ce7be.contents'"
     ]
    }
   ],
   "source": [
    "if \"dataset\" not in locals():\n",
    "    output = \"bdf-demo\"\n",
    "else:\n",
    "    output = dataset\n",
    "\n",
    "policy_method = \"min_cost\"\n",
    "\n",
    "# optimization block\n",
    "engine = pz.StreamingSequentialExecution\n",
    "if policy_method == \"min_cost\":\n",
    "    policy = pz.MinCost()\n",
    "elif policy_method == \"max_quality\":\n",
    "    policy = pz.MaxQuality()\n",
    "iterable  =  pz.Execute(output,\n",
    "                        policy = policy,\n",
    "                        nocache=True,\n",
    "                        allow_code_synth=False,\n",
    "                        allow_token_reduction=False,\n",
    "                        execution_engine=engine)\n",
    "\n",
    "results = []\n",
    "statistics = []\n",
    "\n",
    "for idx, (extraction, plan, stats) in enumerate(iterable):\n",
    "    \n",
    "    record_time = time.time()\n",
    "    statistics.append(stats)\n",
    "\n",
    "    for ex in extraction:\n",
    "        ex_obj = {}\n",
    "        for name in output.schema.field_names():\n",
    "            ex_obj[name] = ex.__getattr__(name)\n",
    "        print(ex_obj)\n",
    "        results.append(ex_obj)\n",
    "\n",
    "results_df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2aa99409",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'results_df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[20], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mresults_df\u001b[49m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'results_df' is not defined"
     ]
    }
   ],
   "source": [
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1be5ca24",
   "metadata": {},
   "outputs": [],
   "source": [
    "if \"dataset\" not in locals():\n",
    "    print(\"Setting dataset\")\n",
    "    dataset = pz.Dataset(\"bdf-demo\", schema=ScientificPaper)\n",
    "condition = \"The paper is published from 2019 onwards\"\n",
    "\n",
    "dataset = dataset.filter(condition)\n",
    "\n",
    "dataset\n",
    "if \"dataset\" not in locals():\n",
    "    dataset = pz.Dataset(\"bdf-demo\", schema=ScientificPaper)\n",
    "convert_schema = AuthorAffiliationSchema\n",
    "\n",
    "caridinality_str = \"one_to_many\"\n",
    "cardinality = pz.Cardinality.ONE_TO_MANY if caridinality_str == \"one_to_many\" else pz.Cardinality.ONE_TO_ONE\n",
    "dataset = dataset.convert(convert_schema, desc=AuthorAffiliationSchema.__doc__, cardinality=cardinality)\n",
    "dataset\n",
    "# Error: KeyError 'PDFFile.b434331434.contents'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81d5695f",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "if \"dataset\" not in locals():\n",
    "    output = \"bdf-demo\"\n",
    "else:\n",
    "    output = dataset\n",
    "\n",
    "policy_method = \"min_cost\"\n",
    "\n",
    "# optimization block\n",
    "engine = pz.StreamingSequentialExecution\n",
    "if policy_method == \"min_cost\":\n",
    "    policy = pz.MinCost()\n",
    "elif policy_method == \"max_quality\":\n",
    "    policy = pz.MaxQuality()\n",
    "iterable  =  pz.Execute(output,\n",
    "                        policy = policy,\n",
    "                        nocache=True,\n",
    "                        allow_code_synth=False,\n",
    "                        allow_token_reduction=False,\n",
    "                        execution_engine=engine)\n",
    "\n",
    "results = []\n",
    "statistics = []\n",
    "\n",
    "for idx, (extraction, plan, stats) in enumerate(iterable):\n",
    "\n",
    "    record_time = time.time()\n",
    "    statistics.append(stats)\n",
    "\n",
    "    for ex in extraction:\n",
    "        ex_obj = {}\n",
    "        for name in output.schema.field_names():\n",
    "            ex_obj[name] = ex.__getattr__(name)\n",
    "        print(ex_obj)\n",
    "        results.append(ex_obj)\n",
    "\n",
    "results_df = pd.DataFrame(results)\n",
    "print(results_df)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "palimpzest",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}