Spaces:
Sleeping
Sleeping
File size: 12,106 Bytes
afd7f5e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 |
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/anaconda3/envs/rag_llm/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"import os\n",
"import hopsworks\n",
"from sentence_transformers import SentenceTransformer\n",
"import numpy as np\n",
"import pandas as pd\n",
"from langchain_docling import DoclingLoader\n",
"from langchain_docling.loader import ExportType\n",
"from docling.chunking import HybridChunker\n",
"\n",
"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"PDF_PATH = \"content/Building+Machine+Learning+Systems+with+a+Feature+Store.pdf\"\n",
"EMBED_MODEL_ID = \"sentence-transformers/all-MiniLM-L6-v2\"\n",
"EXPORT_TYPE = ExportType.DOC_CHUNKS"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2025-12-02 19:43:33,611 INFO: detected formats: [<InputFormat.PDF: 'pdf'>]\n",
"2025-12-02 19:43:33,861 INFO: Going to convert document batch...\n",
"2025-12-02 19:43:33,863 INFO: Initializing pipeline for StandardPdfPipeline with options hash e15bc6f248154cc62f8db15ef18a8ab7\n",
"2025-12-02 19:43:33,913 WARNING: The plugin langchain_docling will not be loaded because Docling is being executed with allow_external_plugins=false.\n",
"2025-12-02 19:43:33,914 INFO: Loading plugin 'docling_defaults'\n",
"2025-12-02 19:43:33,926 INFO: Registered picture descriptions: ['vlm', 'api']\n",
"2025-12-02 19:43:33,981 WARNING: The plugin langchain_docling will not be loaded because Docling is being executed with allow_external_plugins=false.\n",
"2025-12-02 19:43:33,982 INFO: Loading plugin 'docling_defaults'\n",
"2025-12-02 19:43:34,010 INFO: Registered ocr engines: ['auto', 'easyocr', 'ocrmac', 'rapidocr', 'tesserocr', 'tesseract']\n",
"2025-12-02 19:43:42,281 INFO: Auto OCR model selected ocrmac.\n",
"2025-12-02 19:43:42,299 WARNING: The plugin langchain_docling will not be loaded because Docling is being executed with allow_external_plugins=false.\n",
"2025-12-02 19:43:42,299 INFO: Loading plugin 'docling_defaults'\n",
"2025-12-02 19:43:42,323 INFO: Registered layout engines: ['docling_layout_default', 'docling_experimental_table_crops_layout']\n",
"2025-12-02 19:43:42,347 INFO: Accelerator device: 'mps'\n",
"2025-12-02 19:43:57,889 WARNING: The plugin langchain_docling will not be loaded because Docling is being executed with allow_external_plugins=false.\n",
"2025-12-02 19:43:57,907 INFO: Loading plugin 'docling_defaults'\n",
"2025-12-02 19:43:57,919 INFO: Registered table structure engines: ['docling_tableformer']\n",
"2025-12-02 19:44:40,325 INFO: Accelerator device: 'mps'\n",
"2025-12-02 19:44:41,261 INFO: Processing document Building+Machine+Learning+Systems+with+a+Feature+Store.pdf\n",
"2025-12-02 19:51:45,276 INFO: Finished converting document Building+Machine+Learning+Systems+with+a+Feature+Store.pdf in 491.52 sec.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Token indices sequence length is longer than the specified maximum sequence length for this model (1143 > 512). Running this sequence through the model will result in indexing errors\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loaded 1333 document chunks\n"
]
}
],
"source": [
"loader = DoclingLoader(\n",
" file_path=PDF_PATH,\n",
" export_type=EXPORT_TYPE,\n",
" chunker=HybridChunker(tokenizer=EMBED_MODEL_ID),\n",
")\n",
"\n",
"docs = loader.load()\n",
"print(f\"Loaded {len(docs)} document chunks\")"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"page_content='Praise for Building Machine Learning Systems with a Feature Store\n",
"It' s easy to be lost in quality metrics land and forget about the crucial systems aspect to ML. Jim does a great job explaining those aspects and gives a lot of practical tips on how to survive a long deployment.\n",
"-Hannes Mühleisen, cocreator of DuckDB\n",
"Building machine learning systems in production has historically involved a lot of black magic and undocumented learnings. Jim Dowling is doing a great service to ML practitioners by sharing the best practices and putting together clear step-by-step guide.' metadata={'source': 'content/Building+Machine+Learning+Systems+with+a+Feature+Store.pdf', 'dl_meta': {'schema_name': 'docling_core.transforms.chunker.DocMeta', 'version': '1.0.0', 'doc_items': [{'self_ref': '#/texts/7', 'parent': {'$ref': '#/body'}, 'children': [], 'content_layer': 'body', 'label': 'text', 'prov': [{'page_no': 1, 'bbox': {'l': 97.75, 't': 162.01999999999998, 'r': 432.0, 'b': 126.02999999999997, 'coord_origin': 'BOTTOMLEFT'}, 'charspan': [0, 213]}]}, {'self_ref': '#/texts/8', 'parent': {'$ref': '#/body'}, 'children': [], 'content_layer': 'body', 'label': 'text', 'prov': [{'page_no': 1, 'bbox': {'l': 264.75, 't': 122.13, 'r': 432.0, 'b': 110.03200000000004, 'coord_origin': 'BOTTOMLEFT'}, 'charspan': [0, 38]}]}, {'self_ref': '#/texts/9', 'parent': {'$ref': '#/body'}, 'children': [], 'content_layer': 'body', 'label': 'text', 'prov': [{'page_no': 2, 'bbox': {'l': 81.2, 't': 608.02, 'r': 432.0, 'b': 572.03, 'coord_origin': 'BOTTOMLEFT'}, 'charspan': [0, 256]}]}], 'headings': ['Praise for Building Machine Learning Systems with a Feature Store'], 'origin': {'mimetype': 'application/pdf', 'binary_hash': 2591788756701469466, 'filename': 'Building+Machine+Learning+Systems+with+a+Feature+Store.pdf'}}}\n"
]
}
],
"source": [
"print(docs[1])"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Created 1333 splits\n",
"Sample: Praise for Building Machine Learning Systems with a Feature Store\n",
"I witnessed the rise of feature st...\n"
]
}
],
"source": [
"if EXPORT_TYPE == ExportType.DOC_CHUNKS:\n",
" splits = docs\n",
"else:\n",
" from langchain_text_splitters import MarkdownHeaderTextSplitter\n",
" splitter = MarkdownHeaderTextSplitter(\n",
" headers_to_split_on=[\n",
" (\"#\", \"Header_1\"),\n",
" (\"##\", \"Header_2\"),\n",
" (\"###\", \"Header_3\"),\n",
" ],\n",
" )\n",
" splits = [split for doc in docs for split in splitter.split_text(doc.page_content)]\n",
"\n",
"print(f\"Created {len(splits)} splits\")\n",
"print(f\"Sample: {splits[0].page_content[:100]}...\")"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2025-12-02 19:52:07,229 INFO: Use pytorch device_name: mps\n",
"2025-12-02 19:52:07,232 INFO: Load pretrained SentenceTransformer: sentence-transformers/all-MiniLM-L6-v2\n"
]
}
],
"source": [
"embeddings = SentenceTransformer(EMBED_MODEL_ID)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Batches: 100%|██████████| 42/42 [00:18<00:00, 2.31it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Created 1333 embeddings\n"
]
}
],
"source": [
"texts = [split.page_content for split in splits]\n",
"metadatas = [split.metadata for split in splits]\n",
"\n",
"vectors = embeddings.encode(texts, show_progress_bar=True, batch_size=32)\n",
"print(f\"Created {len(vectors)} embeddings\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2025-12-02 19:52:44,050 INFO: Initializing external client\n",
"2025-12-02 19:52:44,064 INFO: Base URL: https://c.app.hopsworks.ai:443\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"\n",
"UserWarning: The installed hopsworks client version 4.4.2 may not be compatible with the connected Hopsworks backend version 4.2.2. \n",
"To ensure compatibility please install the latest bug fix release matching the minor version of your backend (4.2) by running 'pip install hopsworks==4.2.*'\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"2025-12-02 19:52:47,302 INFO: Python Engine initialized.\n",
"\n",
"Logged in to project, explore it here https://c.app.hopsworks.ai:443/p/1271977\n"
]
}
],
"source": [
"project = hopsworks.login()\n",
"fs = project.get_feature_store()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Created dataframe with 1333 rows\n"
]
}
],
"source": [
"data = []\n",
"for i, (text, vector, metadata) in enumerate(zip(texts, vectors, metadatas)):\n",
" data.append({\n",
" 'id': i,\n",
" 'text': text,\n",
" 'page': metadata.get('page', metadata.get('page_number', 0)),\n",
" 'embedding': vector\n",
" })\n",
"\n",
"df = pd.DataFrame(data)\n",
"print(f\"Created dataframe with {len(df)} rows\")"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Feature Group created successfully, explore it at \n",
"https://c.app.hopsworks.ai:443/p/1271977/fs/1258579/fg/1790385\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Uploading Dataframe: 100.00% |██████████| Rows 1333/1333 | Elapsed Time: 00:01 | Remaining Time: 00:00\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Launching job: book_embeddings_2_offline_fg_materialization\n",
"Job started successfully, you can follow the progress at \n",
"https://c.app.hopsworks.ai:443/p/1271977/jobs/named/book_embeddings_2_offline_fg_materialization/executions\n"
]
},
{
"data": {
"text/plain": [
"(Job('book_embeddings_2_offline_fg_materialization', 'SPARK'), None)"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"book_fg = fs.get_or_create_feature_group(\n",
" name=\"book_embeddings\",\n",
" version=2,\n",
" primary_key=[\"id\"],\n",
" description=\"Book text chunks with embeddings\"\n",
")\n",
"\n",
"book_fg.insert(df)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "rag_llm",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.14"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
|