adding code for training

src/patent_train.ipynb

@@ -11,78 +11,51 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Importing Packages"
+    "## Importing Packages\n",
+    "\n",
+    "We first need to import the actual USPTO dataset."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Collecting datasets\n",
-      "  Downloading datasets-2.11.0-py3-none-any.whl (468 kB)\n",
-      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m468.7/468.7 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
-      "\u001b[?25hRequirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (4.64.1)\n",
-      "Requirement already satisfied: pyarrow>=8.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (9.0.0)\n",
+      "Requirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (2.11.0)\n",
       "Requirement already satisfied: fsspec[http]>=2021.11.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (2022.8.2)\n",
-      "Collecting aiohttp\n",
-      "  Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl (1.0 MB)\n",
-      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m19.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
-      "\u001b[?25hCollecting huggingface-hub<1.0.0,>=0.11.0\n",
-      "  Downloading huggingface_hub-0.13.3-py3-none-any.whl (199 kB)\n",
-      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.8/199.8 kB\u001b[0m \u001b[31m18.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-      "\u001b[?25hRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets) (1.5.0)\n",
-      "Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from datasets) (21.3)\n",
-      "Collecting responses<0.19\n",
-      "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
-      "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.3.5.1)\n",
-      "Collecting xxhash\n",
-      "  Downloading xxhash-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl (242 kB)\n",
-      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m242.7/242.7 kB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-      "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (2.28.1)\n",
-      "Collecting multiprocess\n",
-      "  Downloading multiprocess-0.70.14-py310-none-any.whl (134 kB)\n",
-      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.3/134.3 kB\u001b[0m \u001b[31m11.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-      "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (6.0)\n",
       "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from datasets) (1.23.3)\n",
-      "Collecting multidict<7.0,>=4.5\n",
-      "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl (116 kB)\n",
-      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.6/116.6 kB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-      "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
-      "  Downloading yarl-1.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl (257 kB)\n",
-      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m257.3/257.3 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-      "\u001b[?25hCollecting frozenlist>=1.1.1\n",
-      "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl (148 kB)\n",
-      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m148.1/148.1 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-      "\u001b[?25hRequirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (2.1.1)\n",
+      "Requirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (4.64.1)\n",
+      "Requirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (2.28.1)\n",
+      "Requirement already satisfied: pyarrow>=8.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (9.0.0)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (6.0)\n",
+      "Requirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets) (1.5.0)\n",
+      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.13.4)\n",
+      "Requirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.18.0)\n",
+      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets) (3.2.0)\n",
+      "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.3.6)\n",
+      "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets) (3.8.4)\n",
+      "Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from datasets) (21.3)\n",
+      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets) (0.70.14)\n",
       "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (22.1.0)\n",
-      "Collecting aiosignal>=1.1.2\n",
-      "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
-      "Collecting async-timeout<5.0,>=4.0.0a3\n",
-      "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
-      "Collecting filelock\n",
-      "  Downloading filelock-3.10.7-py3-none-any.whl (10 kB)\n",
+      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\n",
+      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.2)\n",
+      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.4)\n",
+      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.8.2)\n",
+      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.3)\n",
+      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (2.1.1)\n",
       "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (4.4.0)\n",
+      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (3.12.0)\n",
       "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->datasets) (3.0.9)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (2022.9.24)\n",
       "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.4)\n",
       "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (1.26.11)\n",
-      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (2022.9.24)\n",
-      "Collecting dill<0.3.7,>=0.3.0\n",
-      "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
-      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-      "\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2.8.2)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2.8.2)\n",
       "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2022.4)\n",
-      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
-      "Installing collected packages: xxhash, multidict, frozenlist, filelock, dill, async-timeout, yarl, responses, multiprocess, huggingface-hub, aiosignal, aiohttp, datasets\n",
-      "  Attempting uninstall: dill\n",
-      "    Found existing installation: dill 0.3.5.1\n",
-      "    Uninstalling dill-0.3.5.1:\n",
-      "      Successfully uninstalled dill-0.3.5.1\n",
-      "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 datasets-2.11.0 dill-0.3.6 filelock-3.10.7 frozenlist-1.3.3 huggingface-hub-0.13.3 multidict-6.0.4 multiprocess-0.70.14 responses-0.18.0 xxhash-3.2.0 yarl-1.8.2\n"
+      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n"
      ]
     }
    ],
@@ -92,14 +65,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
     "from datasets import load_dataset\n",
     "import pandas as pd\n",
-    "import numpy as np\n",
-    "import matplotlib.pyplot as plt"
+    "import numpy as np"
    ]
   },
   {
@@ -113,25 +85,25 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "We first need to extract the dataset. We filter only for those in January 2016."
+    "We need to extract the dataset. We filter only for those in January 2016."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Found cached dataset hupd (/home/jovyan/.cache/huggingface/datasets/HUPD___hupd/sample-ba3b43e1cc5c9c76/0.0.0/6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142)\n"
+      "Found cached dataset hupd (/home/jovyan/.cache/huggingface/datasets/HUPD___hupd/sample-a4eeba92b4229e93/0.0.0/6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142)\n"
      ]
     },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e9c97a02e1834189bcdd4c188ef555d7",
+       "model_id": "e39fd26828774c8e9d159a8b5d91c4f5",
        "version_major": 2,
        "version_minor": 0
       },
@@ -164,7 +136,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -197,7 +169,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -214,7 +186,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -555,7 +527,7 @@
        "[16153 rows x 14 columns]"
       ]
      },
-     "execution_count": 20,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -587,7 +559,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -597,7 +569,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -609,7 +581,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -740,7 +712,7 @@
        "[8719 rows x 3 columns]"
       ]
      },
-     "execution_count": 30,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -751,7 +723,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -763,7 +735,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
@@ -894,7 +866,7 @@
        "[4888 rows x 3 columns]"
       ]
      },
-     "execution_count": 32,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -912,7 +884,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -921,7 +893,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -931,7 +903,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -1062,7 +1034,7 @@
        "[8719 rows x 3 columns]"
       ]
      },
-     "execution_count": 35,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1073,7 +1045,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -1204,7 +1176,7 @@
        "[4888 rows x 3 columns]"
       ]
      },
-     "execution_count": 36,
+     "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1213,6 +1185,432 @@
     "valDF2"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We combine the `abstract` and `claims` columns into a single `text` column. We also re-label the `decision` column to `label`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>label</th>\n",
+       "      <th>text</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1</td>\n",
+       "      <td>The present invention relates to passive optic...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1</td>\n",
+       "      <td>Embodiments of the invention provide a method ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>1</td>\n",
+       "      <td>A crystal growth furnace comprising a crucible...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0</td>\n",
+       "      <td>A shoe midsole is composed of a base plate (1)...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>1</td>\n",
+       "      <td>A ratchet tool includes a shaft member, a hand...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16144</th>\n",
+       "      <td>1</td>\n",
+       "      <td>A wavelength tunable laser device, including: ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16145</th>\n",
+       "      <td>1</td>\n",
+       "      <td>In one aspect, a method for use in preparing a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16148</th>\n",
+       "      <td>1</td>\n",
+       "      <td>A robot hand controlling method executes calcu...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16149</th>\n",
+       "      <td>0</td>\n",
+       "      <td>A fusion protein is disclosed. The fusion prot...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16150</th>\n",
+       "      <td>0</td>\n",
+       "      <td>A pipe extraction tool that grips the inside o...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>8719 rows × 2 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "       label                                               text\n",
+       "0          1  The present invention relates to passive optic...\n",
+       "1          1  Embodiments of the invention provide a method ...\n",
+       "3          1  A crystal growth furnace comprising a crucible...\n",
+       "4          0  A shoe midsole is composed of a base plate (1)...\n",
+       "5          1  A ratchet tool includes a shaft member, a hand...\n",
+       "...      ...                                                ...\n",
+       "16144      1  A wavelength tunable laser device, including: ...\n",
+       "16145      1  In one aspect, a method for use in preparing a...\n",
+       "16148      1  A robot hand controlling method executes calcu...\n",
+       "16149      0  A fusion protein is disclosed. The fusion prot...\n",
+       "16150      0  A pipe extraction tool that grips the inside o...\n",
+       "\n",
+       "[8719 rows x 2 columns]"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trainDF3 = trainDF2.rename(columns={'decision': 'label'})\n",
+    "trainDF3['text'] = trainDF3['abstract'] + ' ' + trainDF3['claims']\n",
+    "trainDF3.drop(columns=[\"abstract\",\"claims\"],inplace=True)\n",
+    "trainDF3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>label</th>\n",
+       "      <th>text</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>Regimen for the treatment of rosacea include t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1</td>\n",
+       "      <td>A clamp arrangement includes a pair of bracket...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0</td>\n",
+       "      <td>A system and method for device action and conf...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0</td>\n",
+       "      <td>Systems and methods for managing datasets prod...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>1</td>\n",
+       "      <td>A scan driving circuit is provided. The scan d...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9085</th>\n",
+       "      <td>0</td>\n",
+       "      <td>The non-rigid gate device as described may be ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9090</th>\n",
+       "      <td>0</td>\n",
+       "      <td>The present invention provides an improved unc...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9091</th>\n",
+       "      <td>1</td>\n",
+       "      <td>A method for detecting a software-race conditi...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9092</th>\n",
+       "      <td>1</td>\n",
+       "      <td>The present application relates to multi-stage...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9093</th>\n",
+       "      <td>1</td>\n",
+       "      <td>A paper feeder includes a housing, a driving u...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>4888 rows × 2 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "      label                                               text\n",
+       "0         0  Regimen for the treatment of rosacea include t...\n",
+       "1         1  A clamp arrangement includes a pair of bracket...\n",
+       "2         0  A system and method for device action and conf...\n",
+       "4         0  Systems and methods for managing datasets prod...\n",
+       "9         1  A scan driving circuit is provided. The scan d...\n",
+       "...     ...                                                ...\n",
+       "9085      0  The non-rigid gate device as described may be ...\n",
+       "9090      0  The present invention provides an improved unc...\n",
+       "9091      1  A method for detecting a software-race conditi...\n",
+       "9092      1  The present application relates to multi-stage...\n",
+       "9093      1  A paper feeder includes a housing, a driving u...\n",
+       "\n",
+       "[4888 rows x 2 columns]"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "valDF3 = valDF2.rename(columns={'decision': 'label'})\n",
+    "valDF3['text'] = valDF3['abstract'] + ' ' + valDF3['claims']\n",
+    "valDF3.drop(columns=[\"abstract\",\"claims\"],inplace=True)\n",
+    "valDF3"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can grab the data for each column so that we have a list of values for training labels, training texts, validation labels, and validation texts."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trainLabels = trainDF3[\"label\"].tolist()\n",
+    "trainText = trainDF3[\"text\"].tolist()\n",
+    "\n",
+    "valLabels = valDF3[\"label\"].tolist()\n",
+    "valText = valDF3[\"text\"].tolist()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Loading the Trainer\n",
+    "\n",
+    "Now we can start training! This time, we will just go with `distilbert-base-uncased` for simplicity."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: torch in /opt/conda/lib/python3.10/site-packages (2.0.0)\n",
+      "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.4.91)\n",
+      "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.91)\n",
+      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch) (3.1.2)\n",
+      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch) (4.4.0)\n",
+      "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /opt/conda/lib/python3.10/site-packages (from torch) (10.2.10.91)\n",
+      "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /opt/conda/lib/python3.10/site-packages (from torch) (11.4.0.1)\n",
+      "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /opt/conda/lib/python3.10/site-packages (from torch) (11.10.3.66)\n",
+      "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /opt/conda/lib/python3.10/site-packages (from torch) (10.9.0.58)\n",
+      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch) (1.11.1)\n",
+      "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.99)\n",
+      "Requirement already satisfied: triton==2.0.0 in /opt/conda/lib/python3.10/site-packages (from torch) (2.0.0)\n",
+      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch) (3.12.0)\n",
+      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch) (2.8.7)\n",
+      "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /opt/conda/lib/python3.10/site-packages (from torch) (2.14.3)\n",
+      "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.101)\n",
+      "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.99)\n",
+      "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /opt/conda/lib/python3.10/site-packages (from torch) (8.5.0.96)\n",
+      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (65.4.1)\n",
+      "Requirement already satisfied: wheel in /opt/conda/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (0.37.1)\n",
+      "Requirement already satisfied: cmake in /opt/conda/lib/python3.10/site-packages (from triton==2.0.0->torch) (3.26.3)\n",
+      "Requirement already satisfied: lit in /opt/conda/lib/python3.10/site-packages (from triton==2.0.0->torch) (16.0.1)\n",
+      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch) (2.1.1)\n",
+      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch) (1.2.1)\n",
+      "Requirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (4.28.1)\n",
+      "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.13.4)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (6.0)\n",
+      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (1.23.3)\n",
+      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers) (3.12.0)\n",
+      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (2023.3.23)\n",
+      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers) (4.64.1)\n",
+      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers) (2.28.1)\n",
+      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.13.3)\n",
+      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers) (21.3)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.4.0)\n",
+      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->transformers) (3.0.9)\n",
+      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (1.26.11)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (2022.9.24)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (3.4)\n",
+      "Requirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (2.1.1)\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install torch\n",
+    "!pip install transformers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "from torch.utils.data import Dataset\n",
+    "from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification\n",
+    "from transformers import Trainer, TrainingArguments"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_name = \"distilbert-base-uncased\"\n",
+    "class USPTODataset(Dataset):\n",
+    "    def __init__(self, encodings, labels):\n",
+    "        self.encodings = encodings\n",
+    "        self.labels = labels\n",
+    "    def __getitem__(self, idx):\n",
+    "        item = {key: torch.tensor(val[idx]) for key, val in self.encoding.items()}\n",
+    "        item['labels'] = torch.tensor(self.labels[idx])\n",
+    "        return item\n",
+    "    def __len__(self):\n",
+    "        return len(self.labels)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_encodings = tokenizer(trainText, truncation=True, padding=True)\n",
+    "val_encodings = tokenizer(valText, truncation=True, padding=True)\n",
+    "\n",
+    "train_dataset = USPTODataset(train_encodings, trainLabels)\n",
+    "val_dataset = USPTODataset(val_encodings, valLabels)\n",
+    "\n",
+    "train_args = TrainingArguments(\n",
+    "    output_dir=\"./results\",\n",
+    "    num_train_epochs=2,\n",
+    "    per_device_train_batch_size=16,\n",
+    "    per_device_eval_batch_size=64,\n",
+    "    warmup_steps=500,\n",
+    "    learning_rate=5e-5,\n",
+    "    weight_decay=0.01,\n",
+    "    logging_dir=\"./logs\",\n",
+    "    logging_steps=10\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": null,

src/train.py

@@ -0,0 +1,152 @@
+from datasets import load_dataset
+import pandas as pd
+import numpy as np
+import os
+import json
+import torch
+from torch.utils.data import Dataset, DataLoader
+from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification
+from transformers import Trainer, TrainingArguments, AdamW
+
+model_name = "distilbert-base-uncased"
+
+class USPTODataset(Dataset):
+    def __init__(self, encodings, labels):
+        self.encodings = encodings
+        self.labels = labels
+    def __getitem__(self, idx):
+        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
+        item['labels'] = torch.tensor(self.labels[idx])
+        return item
+    def __len__(self):
+        return len(self.labels)
+
+def LoadDataset():
+    print("=== LOADING THE DATASET ===")
+    # Extracting the dataset, filtering only for Jan. 2016
+    dataset_dict = load_dataset('HUPD/hupd',
+        name='sample',
+        data_files="https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather", 
+        icpr_label=None,
+        train_filing_start_date='2016-01-01',
+        train_filing_end_date='2016-01-21',
+        val_filing_start_date='2016-01-22',
+        val_filing_end_date='2016-01-31',
+    )
+
+    print("Separating between training and validation data")
+    df_train = pd.DataFrame(dataset_dict['train'] )
+    df_val = pd.DataFrame(dataset_dict['validation'] )
+
+    
+    print("=== PRE-PROCESSING THE DATASET ===")
+    #We are interested in the following columns:
+    # - Abstract
+    # - Claims
+    # - Decision <- our `y`
+    # Let's preprocess them both out of our training and validation data
+    # Also, consider that the "Decision" column has three types of values: "Accepted", "Rejected", and "Pending". To remove unecessary baggage, we will be only looking for "Accepted" and "Rejected".
+
+    necessary_columns = ["abstract","claims","decision"]
+    output_values = ['ACCEPTED','REJECTED']
+
+    print("Dropping unused columns")
+    trainFeaturesToDrop = [col for col in list(df_train.columns) if col not in necessary_columns]
+    trainDF = df_train.dropna()
+    trainDF.drop(columns=trainFeaturesToDrop, inplace=True)
+    trainDF = trainDF[trainDF['decision'].isin(output_values)]
+    valFeaturesToDrop = [col for col in list(df_val.columns) if col not in necessary_columns]
+    valDF = df_val.dropna()
+    valDF.drop(columns=valFeaturesToDrop, inplace=True)
+    valDF = valDF[valDF['decision'].isin(output_values)]
+
+    # We need to replace the values in the `decision` column to numerical representations. ]
+    # We will set "ACCEPTED" as `1` and "REJECTED" as `0`.
+    print("Replacing values in `decision` column")
+    yKey = {"ACCEPTED":1,"REJECTED":0}
+    trainDF2 = trainDF.replace({"decision": yKey})
+    valDF2 = valDF.replace({"decision": yKey})
+
+    # We combine the `abstract` and `claims` columns into a single `text` column. 
+    # We also re-label the `decision` column to `label`.
+    print("Combining columns and renaming `decision` to `label`")
+    trainDF3 = trainDF2.rename(columns={'decision': 'label'})
+    trainDF3['text'] = trainDF3['abstract'] + ' ' + trainDF3['claims']
+    trainDF3.drop(columns=["abstract","claims"],inplace=True)
+
+    valDF3 = valDF2.rename(columns={'decision': 'label'})
+    valDF3['text'] = valDF3['abstract'] + ' ' + valDF3['claims']
+    valDF3.drop(columns=["abstract","claims"],inplace=True)
+
+    # We can grab the data for each column so that we have a list of values for training labels, 
+    #   training texts, validation labels, and validation texts.
+    print("Extracting label and text data from dataframes")
+    trainData = {
+        "labels":trainDF3["label"].tolist(),
+        "text":trainDF3["text"].tolist()
+    }
+    valData = {
+        "labels":valDF3["label"].tolist(),
+        "text":valDF3["text"].tolist()
+    }
+    print(f'TRAINING:\t# labels: {len(trainData["labels"])}\t# texts: {len(trainData["text"])}')
+    print(f'VALID:\t# labels: {len(valData["labels"])}\t# texts: {len(valData["text"])}')
+
+    if not os.path.exists("./data"):
+        os.makedirs('./data')
+
+    with open("./data/train.json", "w") as outfile:
+        json.dump(trainData, outfile, indent=2)
+    with open("./data/val.json", "w") as outfile:
+        json.dump(valData, outfile, indent=2)
+
+    return trainData, valData
+
+def main():
+    trainDataPath = "./data/train.json"
+    valDataPath = "./data/val.json"
+    trainData = None
+    valData = None
+
+    if os.path.exists(trainDataPath) and os.path.exists(valDataPath):
+        ftrain = open(trainDataPath)
+        trainData = json.load(ftrain)
+        ftrain.close()
+        fval = open(valDataPath)
+        valData = json.load(fval)
+        fval.close()
+    else:
+        trainData, valData = LoadDataset()
+    
+    print(len(trainData["labels"]), len(trainData["text"]), len(valData["labels"]), len(valData["text"]))
+
+    tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)
+    train_encodings = tokenizer(trainData["text"], truncation=True, padding=True)
+    val_encodings = tokenizer(valData["text"], truncation=True, padding=True)
+
+    train_dataset = USPTODataset(train_encodings, trainData["labels"])
+    val_dataset = USPTODataset(val_encodings, valData["labels"])
+
+    train_args = TrainingArguments(
+        output_dir="./results",
+        num_train_epochs=2,
+        per_device_train_batch_size=16,
+        per_device_eval_batch_size=64,
+        warmup_steps=500,
+        learning_rate=5e-5,
+        weight_decay=0.01,
+        logging_dir="./logs",
+        logging_steps=10
+    )
+
+    model = DistilBertForSequenceClassification.from_pretrained(model_name)
+    trainer = Trainer(
+        model=model,
+        args=train_args,
+        train_dataset=train_dataset,
+        eval_dataset=val_dataset
+    )
+    trainer.train()
+
+if __name__ == "__main__":
+    main()