yangdx
commited on
Commit
·
fd1695b
1
Parent(s):
0edbfbb
Remove redundancy set_logger function and related calls
Browse files- lightrag/kg/shared_storage.py +17 -15
- lightrag/lightrag.py +1 -3
- lightrag/operate.py +0 -1
- lightrag/utils.py +0 -46
- run_with_gunicorn.py +0 -1
lightrag/kg/shared_storage.py
CHANGED
@@ -80,24 +80,26 @@ def initialize_share_data(workers: int = 1):
|
|
80 |
|
81 |
# Mark as initialized
|
82 |
_initialized = True
|
83 |
-
|
84 |
# Initialize pipeline status for document indexing control
|
85 |
pipeline_namespace = get_namespace_data("pipeline_status")
|
86 |
-
|
87 |
-
#
|
88 |
history_messages = _manager.list() if is_multiprocess else []
|
89 |
-
|
90 |
-
pipeline_namespace.update(
|
91 |
-
|
92 |
-
|
93 |
-
|
94 |
-
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
|
|
|
|
|
101 |
|
102 |
|
103 |
def try_initialize_namespace(namespace: str) -> bool:
|
|
|
80 |
|
81 |
# Mark as initialized
|
82 |
_initialized = True
|
83 |
+
|
84 |
# Initialize pipeline status for document indexing control
|
85 |
pipeline_namespace = get_namespace_data("pipeline_status")
|
86 |
+
|
87 |
+
# Create a shared list object for history_messages
|
88 |
history_messages = _manager.list() if is_multiprocess else []
|
89 |
+
|
90 |
+
pipeline_namespace.update(
|
91 |
+
{
|
92 |
+
"busy": False, # Control concurrent processes
|
93 |
+
"job_name": "Default Job", # Current job name (indexing files/indexing texts)
|
94 |
+
"job_start": None, # Job start time
|
95 |
+
"docs": 0, # Total number of documents to be indexed
|
96 |
+
"batchs": 0, # Number of batches for processing documents
|
97 |
+
"cur_batch": 0, # Current processing batch
|
98 |
+
"request_pending": False, # Flag for pending request for processing
|
99 |
+
"latest_message": "", # Latest message from pipeline processing
|
100 |
+
"history_messages": history_messages, # 使用共享列表对象
|
101 |
+
}
|
102 |
+
)
|
103 |
|
104 |
|
105 |
def try_initialize_namespace(namespace: str) -> bool:
|
lightrag/lightrag.py
CHANGED
@@ -45,7 +45,6 @@ from .utils import (
|
|
45 |
lazy_external_import,
|
46 |
limit_async_func_call,
|
47 |
logger,
|
48 |
-
set_logger,
|
49 |
)
|
50 |
from .types import KnowledgeGraph
|
51 |
from dotenv import load_dotenv
|
@@ -268,7 +267,6 @@ class LightRAG:
|
|
268 |
|
269 |
def __post_init__(self):
|
270 |
os.makedirs(os.path.dirname(self.log_file_path), exist_ok=True)
|
271 |
-
set_logger(self.log_file_path, self.log_level)
|
272 |
logger.info(f"Logger initialized for working directory: {self.working_dir}")
|
273 |
|
274 |
from lightrag.kg.shared_storage import (
|
@@ -682,7 +680,7 @@ class LightRAG:
|
|
682 |
with storage_lock:
|
683 |
# Ensure only one worker is processing documents
|
684 |
if not pipeline_status.get("busy", False):
|
685 |
-
# Cleaning history_messages without breaking it as a shared list object
|
686 |
current_history = pipeline_status.get("history_messages", [])
|
687 |
if hasattr(current_history, "clear"):
|
688 |
current_history.clear()
|
|
|
45 |
lazy_external_import,
|
46 |
limit_async_func_call,
|
47 |
logger,
|
|
|
48 |
)
|
49 |
from .types import KnowledgeGraph
|
50 |
from dotenv import load_dotenv
|
|
|
267 |
|
268 |
def __post_init__(self):
|
269 |
os.makedirs(os.path.dirname(self.log_file_path), exist_ok=True)
|
|
|
270 |
logger.info(f"Logger initialized for working directory: {self.working_dir}")
|
271 |
|
272 |
from lightrag.kg.shared_storage import (
|
|
|
680 |
with storage_lock:
|
681 |
# Ensure only one worker is processing documents
|
682 |
if not pipeline_status.get("busy", False):
|
683 |
+
# Cleaning history_messages without breaking it as a shared list object
|
684 |
current_history = pipeline_status.get("history_messages", [])
|
685 |
if hasattr(current_history, "clear"):
|
686 |
current_history.clear()
|
lightrag/operate.py
CHANGED
@@ -336,7 +336,6 @@ async def extract_entities(
|
|
336 |
global_config: dict[str, str],
|
337 |
llm_response_cache: BaseKVStorage | None = None,
|
338 |
) -> None:
|
339 |
-
|
340 |
from lightrag.kg.shared_storage import get_namespace_data
|
341 |
|
342 |
pipeline_status = get_namespace_data("pipeline_status")
|
|
|
336 |
global_config: dict[str, str],
|
337 |
llm_response_cache: BaseKVStorage | None = None,
|
338 |
) -> None:
|
|
|
339 |
from lightrag.kg.shared_storage import get_namespace_data
|
340 |
|
341 |
pipeline_status = get_namespace_data("pipeline_status")
|
lightrag/utils.py
CHANGED
@@ -68,52 +68,6 @@ logger.setLevel(logging.INFO)
|
|
68 |
logging.getLogger("httpx").setLevel(logging.WARNING)
|
69 |
|
70 |
|
71 |
-
def set_logger(log_file: str, level: int = logging.DEBUG):
|
72 |
-
"""Set up file logging with the specified level.
|
73 |
-
|
74 |
-
Args:
|
75 |
-
log_file: Path to the log file
|
76 |
-
level: Logging level (e.g. logging.DEBUG, logging.INFO)
|
77 |
-
"""
|
78 |
-
|
79 |
-
logger.setLevel(level)
|
80 |
-
log_file = os.path.abspath(log_file)
|
81 |
-
formatter = logging.Formatter(
|
82 |
-
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
83 |
-
)
|
84 |
-
has_file_handler = False
|
85 |
-
has_console_handler = False
|
86 |
-
|
87 |
-
for handler in logger.handlers:
|
88 |
-
if isinstance(handler, logging.FileHandler):
|
89 |
-
has_file_handler = True
|
90 |
-
elif isinstance(handler, logging.StreamHandler) and not isinstance(
|
91 |
-
handler, logging.FileHandler
|
92 |
-
):
|
93 |
-
has_console_handler = True
|
94 |
-
|
95 |
-
if not has_file_handler:
|
96 |
-
from logging.handlers import RotatingFileHandler
|
97 |
-
|
98 |
-
file_handler = RotatingFileHandler(
|
99 |
-
log_file,
|
100 |
-
maxBytes=10 * 1024 * 1024, # 10MB
|
101 |
-
backupCount=5,
|
102 |
-
encoding="utf-8",
|
103 |
-
)
|
104 |
-
file_handler.setLevel(level)
|
105 |
-
file_handler.setFormatter(formatter)
|
106 |
-
logger.addHandler(file_handler)
|
107 |
-
|
108 |
-
if not has_console_handler:
|
109 |
-
console_handler = logging.StreamHandler()
|
110 |
-
console_handler.setLevel(level)
|
111 |
-
console_handler.setFormatter(formatter)
|
112 |
-
logger.addHandler(console_handler)
|
113 |
-
|
114 |
-
logger.propagate = False
|
115 |
-
|
116 |
-
|
117 |
class UnlimitedSemaphore:
|
118 |
"""A context manager that allows unlimited access."""
|
119 |
|
|
|
68 |
logging.getLogger("httpx").setLevel(logging.WARNING)
|
69 |
|
70 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
71 |
class UnlimitedSemaphore:
|
72 |
"""A context manager that allows unlimited access."""
|
73 |
|
run_with_gunicorn.py
CHANGED
@@ -125,7 +125,6 @@ def main():
|
|
125 |
if callable(value):
|
126 |
self.cfg.set(key, value)
|
127 |
|
128 |
-
|
129 |
if hasattr(gunicorn_config, "logconfig_dict"):
|
130 |
self.cfg.set(
|
131 |
"logconfig_dict", getattr(gunicorn_config, "logconfig_dict")
|
|
|
125 |
if callable(value):
|
126 |
self.cfg.set(key, value)
|
127 |
|
|
|
128 |
if hasattr(gunicorn_config, "logconfig_dict"):
|
129 |
self.cfg.set(
|
130 |
"logconfig_dict", getattr(gunicorn_config, "logconfig_dict")
|