YanSte commited on
Commit
e4307a4
·
1 Parent(s): a6a0d04

cleaned code

Browse files
Files changed (1) hide show
  1. lightrag/lightrag.py +7 -7
lightrag/lightrag.py CHANGED
@@ -5,7 +5,6 @@ from dataclasses import asdict, dataclass, field
5
  from datetime import datetime
6
  from functools import partial
7
  from typing import Any, Callable, Coroutine, Optional, Type, Union, cast
8
- import traceback
9
  from .operate import (
10
  chunking_by_token_size,
11
  extract_entities,
@@ -560,16 +559,17 @@ class LightRAG:
560
  batch_docs_list = [
561
  pending_doc_ids[i : i + batch_size] for i in range(0, len(pending_doc_ids), batch_size)
562
  ]
563
- batch_len = len(batch_docs_list) + 1
564
-
565
  # 3. iterate over batches
566
  tasks: dict[str, list[Coroutine[Any, Any, None]]] = {}
567
- for batch_idx, doc_ids in enumerate(batch_docs_list):
568
-
 
 
569
  # 4. iterate over batch
570
  for doc_id in tqdm_async(
571
  doc_ids,
572
- desc=f"Level 1 - Batch {batch_idx} / {batch_len}",
573
  ):
574
  # Update status in processing
575
  status_doc = await self.doc_status.get_by_id(doc_id)
@@ -631,7 +631,7 @@ class LightRAG:
631
 
632
  except Exception as e:
633
  logger.error(
634
- f"Failed to process document {doc_id}: {str(e)}\n{traceback.format_exc()}"
635
  )
636
  await self.doc_status.upsert(
637
  {
 
5
  from datetime import datetime
6
  from functools import partial
7
  from typing import Any, Callable, Coroutine, Optional, Type, Union, cast
 
8
  from .operate import (
9
  chunking_by_token_size,
10
  extract_entities,
 
559
  batch_docs_list = [
560
  pending_doc_ids[i : i + batch_size] for i in range(0, len(pending_doc_ids), batch_size)
561
  ]
562
+
 
563
  # 3. iterate over batches
564
  tasks: dict[str, list[Coroutine[Any, Any, None]]] = {}
565
+ for batch_idx, doc_ids in tqdm_async(
566
+ enumerate(batch_docs_list),
567
+ desc=f"Process Batches",
568
+ ):
569
  # 4. iterate over batch
570
  for doc_id in tqdm_async(
571
  doc_ids,
572
+ desc=f"Process Batch {batch_idx}",
573
  ):
574
  # Update status in processing
575
  status_doc = await self.doc_status.get_by_id(doc_id)
 
631
 
632
  except Exception as e:
633
  logger.error(
634
+ f"Failed to process document {doc_id}: {str(e)}"
635
  )
636
  await self.doc_status.upsert(
637
  {