cleaned code
Browse files- lightrag/lightrag.py +7 -7
lightrag/lightrag.py
CHANGED
@@ -5,7 +5,6 @@ from dataclasses import asdict, dataclass, field
|
|
5 |
from datetime import datetime
|
6 |
from functools import partial
|
7 |
from typing import Any, Callable, Coroutine, Optional, Type, Union, cast
|
8 |
-
import traceback
|
9 |
from .operate import (
|
10 |
chunking_by_token_size,
|
11 |
extract_entities,
|
@@ -560,16 +559,17 @@ class LightRAG:
|
|
560 |
batch_docs_list = [
|
561 |
pending_doc_ids[i : i + batch_size] for i in range(0, len(pending_doc_ids), batch_size)
|
562 |
]
|
563 |
-
|
564 |
-
|
565 |
# 3. iterate over batches
|
566 |
tasks: dict[str, list[Coroutine[Any, Any, None]]] = {}
|
567 |
-
for batch_idx, doc_ids in
|
568 |
-
|
|
|
|
|
569 |
# 4. iterate over batch
|
570 |
for doc_id in tqdm_async(
|
571 |
doc_ids,
|
572 |
-
desc=f"
|
573 |
):
|
574 |
# Update status in processing
|
575 |
status_doc = await self.doc_status.get_by_id(doc_id)
|
@@ -631,7 +631,7 @@ class LightRAG:
|
|
631 |
|
632 |
except Exception as e:
|
633 |
logger.error(
|
634 |
-
f"Failed to process document {doc_id}: {str(e)}
|
635 |
)
|
636 |
await self.doc_status.upsert(
|
637 |
{
|
|
|
5 |
from datetime import datetime
|
6 |
from functools import partial
|
7 |
from typing import Any, Callable, Coroutine, Optional, Type, Union, cast
|
|
|
8 |
from .operate import (
|
9 |
chunking_by_token_size,
|
10 |
extract_entities,
|
|
|
559 |
batch_docs_list = [
|
560 |
pending_doc_ids[i : i + batch_size] for i in range(0, len(pending_doc_ids), batch_size)
|
561 |
]
|
562 |
+
|
|
|
563 |
# 3. iterate over batches
|
564 |
tasks: dict[str, list[Coroutine[Any, Any, None]]] = {}
|
565 |
+
for batch_idx, doc_ids in tqdm_async(
|
566 |
+
enumerate(batch_docs_list),
|
567 |
+
desc=f"Process Batches",
|
568 |
+
):
|
569 |
# 4. iterate over batch
|
570 |
for doc_id in tqdm_async(
|
571 |
doc_ids,
|
572 |
+
desc=f"Process Batch {batch_idx}",
|
573 |
):
|
574 |
# Update status in processing
|
575 |
status_doc = await self.doc_status.get_by_id(doc_id)
|
|
|
631 |
|
632 |
except Exception as e:
|
633 |
logger.error(
|
634 |
+
f"Failed to process document {doc_id}: {str(e)}"
|
635 |
)
|
636 |
await self.doc_status.upsert(
|
637 |
{
|