Merge branch 'main' into optimize-config-management

# Conflicts:
# env.example
# lightrag/api/utils_api.py

README.md

@@ -441,11 +441,16 @@ if __name__ == "__main__":
 - [Direct OpenAI Example](examples/lightrag_llamaindex_direct_demo.py)
 - [LiteLLM Proxy Example](examples/lightrag_llamaindex_litellm_demo.py)
 
+</details>
+
 ### Conversation History Support
 
 
 LightRAG now supports multi-turn dialogue through the conversation history feature. Here's how to use it:
 
+<details>
+  <summary> <b> Usage Example </b></summary>
+
 ```python
 # Create conversation history
 conversation_history = [
@@ -468,10 +473,15 @@ response = rag.query(
 )
 ```
 
+</details>
+
 ### Custom Prompt Support
 
 LightRAG now supports custom prompts for fine-tuned control over the system's behavior. Here's how to use it:
 
+<details>
+  <summary> <b> Usage Example </b></summary>
+
 ```python
 # Create query parameters
 query_param = QueryParam(
@@ -506,6 +516,8 @@ response_custom = rag.query(
 print(response_custom)
 ```
 
+</details>
+
 ### Separate Keyword Extraction
 
 We've introduced a new function `query_with_separate_keyword_extraction` to enhance the keyword extraction capabilities. This function separates the keyword extraction process from the user's prompt, focusing solely on the query to improve the relevance of extracted keywords.
@@ -519,7 +531,8 @@ The function operates by dividing the input into two parts:
 
 It then performs keyword extraction exclusively on the `user query`. This separation ensures that the extraction process is focused and relevant, unaffected by any additional language in the `prompt`. It also allows the `prompt` to serve purely for response formatting, maintaining the intent and clarity of the user's original question.
 
-**Usage Example**
+<details>
+  <summary> <b> Usage Example </b></summary>
 
 This `example` shows how to tailor the function for educational content, focusing on detailed explanations for older students.
 
@@ -531,67 +544,6 @@ rag.query_with_separate_keyword_extraction(
 )
 ```
 
-### Insert Custom KG
-
-```python
-custom_kg = {
-    "chunks": [
-        {
-            "content": "Alice and Bob are collaborating on quantum computing research.",
-            "source_id": "doc-1"
-        }
-    ],
-    "entities": [
-        {
-            "entity_name": "Alice",
-            "entity_type": "person",
-            "description": "Alice is a researcher specializing in quantum physics.",
-            "source_id": "doc-1"
-        },
-        {
-            "entity_name": "Bob",
-            "entity_type": "person",
-            "description": "Bob is a mathematician.",
-            "source_id": "doc-1"
-        },
-        {
-            "entity_name": "Quantum Computing",
-            "entity_type": "technology",
-            "description": "Quantum computing utilizes quantum mechanical phenomena for computation.",
-            "source_id": "doc-1"
-        }
-    ],
-    "relationships": [
-        {
-            "src_id": "Alice",
-            "tgt_id": "Bob",
-            "description": "Alice and Bob are research partners.",
-            "keywords": "collaboration research",
-            "weight": 1.0,
-            "source_id": "doc-1"
-        },
-        {
-            "src_id": "Alice",
-            "tgt_id": "Quantum Computing",
-            "description": "Alice conducts research on quantum computing.",
-            "keywords": "research expertise",
-            "weight": 1.0,
-            "source_id": "doc-1"
-        },
-        {
-            "src_id": "Bob",
-            "tgt_id": "Quantum Computing",
-            "description": "Bob researches quantum computing.",
-            "keywords": "research application",
-            "weight": 1.0,
-            "source_id": "doc-1"
-        }
-    ]
-}
-
-rag.insert_custom_kg(custom_kg)
-```
-
 </details>
 
 ## Insert
@@ -683,6 +635,70 @@ rag.insert(text_content.decode('utf-8'))
 
 </details>
 
+<details>
+  <summary> <b> Insert Custom KG </b></summary>
+
+```python
+custom_kg = {
+    "chunks": [
+        {
+            "content": "Alice and Bob are collaborating on quantum computing research.",
+            "source_id": "doc-1"
+        }
+    ],
+    "entities": [
+        {
+            "entity_name": "Alice",
+            "entity_type": "person",
+            "description": "Alice is a researcher specializing in quantum physics.",
+            "source_id": "doc-1"
+        },
+        {
+            "entity_name": "Bob",
+            "entity_type": "person",
+            "description": "Bob is a mathematician.",
+            "source_id": "doc-1"
+        },
+        {
+            "entity_name": "Quantum Computing",
+            "entity_type": "technology",
+            "description": "Quantum computing utilizes quantum mechanical phenomena for computation.",
+            "source_id": "doc-1"
+        }
+    ],
+    "relationships": [
+        {
+            "src_id": "Alice",
+            "tgt_id": "Bob",
+            "description": "Alice and Bob are research partners.",
+            "keywords": "collaboration research",
+            "weight": 1.0,
+            "source_id": "doc-1"
+        },
+        {
+            "src_id": "Alice",
+            "tgt_id": "Quantum Computing",
+            "description": "Alice conducts research on quantum computing.",
+            "keywords": "research expertise",
+            "weight": 1.0,
+            "source_id": "doc-1"
+        },
+        {
+            "src_id": "Bob",
+            "tgt_id": "Quantum Computing",
+            "description": "Bob researches quantum computing.",
+            "keywords": "research application",
+            "weight": 1.0,
+            "source_id": "doc-1"
+        }
+    ]
+}
+
+rag.insert_custom_kg(custom_kg)
+```
+
+</details>
+
 <details>
   <summary><b>Citation Functionality</b></summary>
 
@@ -842,7 +858,8 @@ rag.delete_by_doc_id("doc_id")
 
 LightRAG now supports comprehensive knowledge graph management capabilities, allowing you to create, edit, and delete entities and relationships within your knowledge graph.
 
-### Create Entities and Relations
+<details>
+  <summary> <b> Create Entities and Relations </b></summary>
 
 ```python
 # Create new entity
@@ -865,7 +882,10 @@ relation = rag.create_relation("Google", "Gmail", {
 })
 ```
 
-### Edit Entities and Relations
+</details>
+
+<details>
+  <summary> <b> Edit Entities and Relations </b></summary>
 
 ```python
 # Edit an existing entity
@@ -902,6 +922,8 @@ All operations are available in both synchronous and asynchronous versions. The
 
 These operations maintain data consistency across both the graph database and vector database components, ensuring your knowledge graph remains coherent.
 
+</details>
+
 ## Data Export Functions
 
 ### Overview
@@ -910,7 +932,8 @@ LightRAG allows you to export your knowledge graph data in various formats for a
 
 ### Export Functions
 
-#### Basic Usage
+<details>
+  <summary> <b> Basic Usage </b></summary>
 
 ```python
 # Basic CSV export (default format)
@@ -920,7 +943,10 @@ rag.export_data("knowledge_graph.csv")
 rag.export_data("output.xlsx", file_format="excel")
 ```
 
-#### Different File Formats supported
+</details>
+
+<details>
+  <summary> <b> Different File Formats supported </b></summary>
 
 ```python
 #Export data in CSV format
@@ -935,13 +961,18 @@ rag.export_data("graph_data.md", file_format="md")
 # Export data in Text
 rag.export_data("graph_data.txt", file_format="txt")
 ```
-#### Additional Options
+</details>
+
+<details>
+  <summary> <b> Additional Options </b></summary>
 
 Include vector embeddings in the export (optional):
 
 ```python
 rag.export_data("complete_data.csv", include_vector_data=True)
 ```
+</details>
+
 ### Data Included in Export
 
 All exports include:

env.example

@@ -3,9 +3,11 @@
 ### Server Configuration
 # HOST=0.0.0.0
 # PORT=9621
-# WORKERS=1
-# NAMESPACE_PREFIX=lightrag  # separating data from difference Lightrag instances
-# MAX_GRAPH_NODES=1000       # Max nodes return from grap retrieval
+# WORKERS=2
+### separating data from difference Lightrag instances
+# NAMESPACE_PREFIX=lightrag
+### Max nodes return from grap retrieval
+# MAX_GRAPH_NODES=1000
 # CORS_ORIGINS=http://localhost:3000,http://localhost:8080
 
 ### Optional SSL Configuration
@@ -13,7 +15,7 @@
 # SSL_CERTFILE=/path/to/cert.pem
 # SSL_KEYFILE=/path/to/key.pem
 
-### Directory Configuration
+### Directory Configuration (defaults to current working directory)
 # WORKING_DIR=<absolute_path_for_working_dir>
 # INPUT_DIR=<absolute_path_for_doc_input_dir>
 
@@ -23,9 +25,10 @@
 ### Logging level
 # LOG_LEVEL=INFO
 # VERBOSE=False
-# LOG_DIR=/path/to/log/directory  # Log file directory path, defaults to current working directory
-# LOG_MAX_BYTES=10485760          # Log file max size in bytes, defaults to 10MB
-# LOG_BACKUP_COUNT=5              # Number of backup files to keep, defaults to 5
+# LOG_MAX_BYTES=10485760
+# LOG_BACKUP_COUNT=5
+### Logfile location (defaults to current working directory)
+# LOG_DIR=/path/to/log/directory
 
 ### Settings for RAG query
 # HISTORY_TURNS=3
@@ -36,28 +39,37 @@
 # MAX_TOKEN_ENTITY_DESC=4000
 
 ### Settings for document indexing
-ENABLE_LLM_CACHE_FOR_EXTRACT=true    # Enable LLM cache for entity extraction
+ENABLE_LLM_CACHE_FOR_EXTRACT=true
 SUMMARY_LANGUAGE=English
 # CHUNK_SIZE=1200
 # CHUNK_OVERLAP_SIZE=100
-# MAX_TOKEN_SUMMARY=500                # Max tokens for entity or relations summary
-# MAX_PARALLEL_INSERT=2                # Number of parallel processing documents in one patch
-
-# EMBEDDING_BATCH_NUM=32               # num of chunks send to Embedding in one request
-# EMBEDDING_FUNC_MAX_ASYNC=16          # Max concurrency requests for Embedding
+### Max tokens for entity or relations summary
+# MAX_TOKEN_SUMMARY=500
+### Number of parallel processing documents in one patch
+# MAX_PARALLEL_INSERT=2
+
+### Num of chunks send to Embedding in single request
+# EMBEDDING_BATCH_NUM=32
+### Max concurrency requests for Embedding
+# EMBEDDING_FUNC_MAX_ASYNC=16
 # MAX_EMBED_TOKENS=8192
 
-### LLM Configuration (Use valid host. For local services installed with docker, you can use host.docker.internal)
-TIMEOUT=150                            # Time out in seconds for LLM, None for infinite timeout
+### LLM Configuration
+### Time out in seconds for LLM, None for infinite timeout
+TIMEOUT=150
+### Some models like o1-mini require temperature to be set to 1
 TEMPERATURE=0.5
-MAX_ASYNC=4                            # Max concurrency requests of LLM
-MAX_TOKENS=32768                       # Max tokens send to LLM (less than context size of the model)
+### Max concurrency requests of LLM
+MAX_ASYNC=4
+### Max tokens send to LLM (less than context size of the model)
+MAX_TOKENS=32768
 
+### Ollama example (For local services installed with docker, you can use host.docker.internal as host)
 LLM_BINDING=ollama
 LLM_MODEL=mistral-nemo:latest
 LLM_BINDING_API_KEY=your_api_key
-### Ollama example
 LLM_BINDING_HOST=http://localhost:11434
+
 ### OpenAI alike example
 # LLM_BINDING=openai
 # LLM_MODEL=gpt-4o
@@ -103,9 +115,10 @@ ORACLE_DSN=localhost:1521/XEPDB1
 ORACLE_USER=your_username
 ORACLE_PASSWORD='your_password'
 ORACLE_CONFIG_DIR=/path/to/oracle/config
-#ORACLE_WALLET_LOCATION=/path/to/wallet  # optional
-#ORACLE_WALLET_PASSWORD='your_password'  # optional
-#ORACLE_WORKSPACE=default  # separating all data from difference Lightrag instances(deprecated, use NAMESPACE_PREFIX in future)
+#ORACLE_WALLET_LOCATION=/path/to/wallet
+#ORACLE_WALLET_PASSWORD='your_password'
+### separating all data from difference Lightrag instances(deprecating, use NAMESPACE_PREFIX in future)
+#ORACLE_WORKSPACE=default
 
 ### TiDB Configuration
 TIDB_HOST=localhost
@@ -113,7 +126,8 @@ TIDB_PORT=4000
 TIDB_USER=your_username
 TIDB_PASSWORD='your_password'
 TIDB_DATABASE=your_database
-#TIDB_WORKSPACE=default  # separating all data from difference Lightrag instances(deprecated, use NAMESPACE_PREFIX in future)
+### separating all data from difference Lightrag instances(deprecating, use NAMESPACE_PREFIX in future)
+#TIDB_WORKSPACE=default
 
 ### PostgreSQL Configuration
 POSTGRES_HOST=localhost
@@ -121,7 +135,8 @@ POSTGRES_PORT=5432
 POSTGRES_USER=your_username
 POSTGRES_PASSWORD='your_password'
 POSTGRES_DATABASE=your_database
-#POSTGRES_WORKSPACE=default  # separating all data from difference Lightrag instances(deprecated, use NAMESPACE_PREFIX in future)
+### separating all data from difference Lightrag instances(deprecating, use NAMESPACE_PREFIX in future)
+#POSTGRES_WORKSPACE=default
 
 ### Independent AGM Configuration(not for AMG embedded in PostreSQL)
 AGE_POSTGRES_DB=
@@ -130,8 +145,9 @@ AGE_POSTGRES_PASSWORD=
 AGE_POSTGRES_HOST=
 # AGE_POSTGRES_PORT=8529
 
+### separating all data from difference Lightrag instances(deprecating, use NAMESPACE_PREFIX in future)
 # AGE Graph Name(apply to PostgreSQL and independent AGM)
-# AGE_GRAPH_NAME=lightrag  # deprecated, use NAME_SPACE_PREFIX instead
+# AGE_GRAPH_NAME=lightrag
 
 ### Neo4j Configuration
 NEO4J_URI=neo4j+s://xxxxxxxx.databases.neo4j.io
@@ -141,7 +157,8 @@ NEO4J_PASSWORD='your_password'
 ### MongoDB Configuration
 MONGO_URI=mongodb://root:root@localhost:27017/
 MONGO_DATABASE=LightRAG
-MONGODB_GRAPH=false # deprecated (keep for backward compatibility)
+### separating all data from difference Lightrag instances(deprecating, use NAMESPACE_PREFIX in future)
+# MONGODB_GRAPH=false
 
 ### Milvus Configuration
 MILVUS_URI=http://localhost:19530
@@ -158,11 +175,11 @@ QDRANT_URL=http://localhost:16333
 REDIS_URI=redis://localhost:6379
 
 ### For JWT Auth
-AUTH_ACCOUNTS='admin:admin123,user1:pass456'     # username:password,username:password
-TOKEN_SECRET=Your-Key-For-LightRAG-API-Server    # JWT key
-#TOKEN_EXPIRE_HOURS=4                             # Expire duration, default 4
-#GUEST_TOKEN_EXPIRE_HOURS=2                       # Guest expire duration, default 2
-#JWT_ALGORITHM=HS256                              # JWT encode algorithm, default HS256
+#AUTH_ACCOUNTS='admin:admin123,user1:pass456'
+#TOKEN_SECRET=Your-Key-For-LightRAG-API-Server
+#TOKEN_EXPIRE_HOURS=4
+#GUEST_TOKEN_EXPIRE_HOURS=2
+#JWT_ALGORITHM=HS256
 
 ### API-Key to access LightRAG Server API
 # LIGHTRAG_API_KEY=your-secure-api-key-here

examples/lightrag_gemini_track_token_demo.py

@@ -0,0 +1,151 @@
+# pip install -q -U google-genai to use gemini as a client
+
+import os
+import asyncio
+import numpy as np
+import nest_asyncio
+from google import genai
+from google.genai import types
+from dotenv import load_dotenv
+from lightrag.utils import EmbeddingFunc
+from lightrag import LightRAG, QueryParam
+from lightrag.kg.shared_storage import initialize_pipeline_status
+from lightrag.llm.siliconcloud import siliconcloud_embedding
+from lightrag.utils import setup_logger
+from lightrag.utils import TokenTracker
+
+setup_logger("lightrag", level="DEBUG")
+
+# Apply nest_asyncio to solve event loop issues
+nest_asyncio.apply()
+
+load_dotenv()
+gemini_api_key = os.getenv("GEMINI_API_KEY")
+siliconflow_api_key = os.getenv("SILICONFLOW_API_KEY")
+
+WORKING_DIR = "./dickens"
+
+if not os.path.exists(WORKING_DIR):
+    os.mkdir(WORKING_DIR)
+
+token_tracker = TokenTracker()
+
+
+async def llm_model_func(
+    prompt, system_prompt=None, history_messages=[], keyword_extraction=False, **kwargs
+) -> str:
+    # 1. Initialize the GenAI Client with your Gemini API Key
+    client = genai.Client(api_key=gemini_api_key)
+
+    # 2. Combine prompts: system prompt, history, and user prompt
+    if history_messages is None:
+        history_messages = []
+
+    combined_prompt = ""
+    if system_prompt:
+        combined_prompt += f"{system_prompt}\n"
+
+    for msg in history_messages:
+        # Each msg is expected to be a dict: {"role": "...", "content": "..."}
+        combined_prompt += f"{msg['role']}: {msg['content']}\n"
+
+    # Finally, add the new user prompt
+    combined_prompt += f"user: {prompt}"
+
+    # 3. Call the Gemini model
+    response = client.models.generate_content(
+        model="gemini-2.0-flash",
+        contents=[combined_prompt],
+        config=types.GenerateContentConfig(
+            max_output_tokens=5000, temperature=0, top_k=10
+        ),
+    )
+
+    # 4. Get token counts with null safety
+    usage = getattr(response, "usage_metadata", None)
+    prompt_tokens = getattr(usage, "prompt_token_count", 0) or 0
+    completion_tokens = getattr(usage, "candidates_token_count", 0) or 0
+    total_tokens = getattr(usage, "total_token_count", 0) or (
+        prompt_tokens + completion_tokens
+    )
+
+    token_counts = {
+        "prompt_tokens": prompt_tokens,
+        "completion_tokens": completion_tokens,
+        "total_tokens": total_tokens,
+    }
+
+    token_tracker.add_usage(token_counts)
+
+    # 5. Return the response text
+    return response.text
+
+
+async def embedding_func(texts: list[str]) -> np.ndarray:
+    return await siliconcloud_embedding(
+        texts,
+        model="BAAI/bge-m3",
+        api_key=siliconflow_api_key,
+        max_token_size=512,
+    )
+
+
+async def initialize_rag():
+    rag = LightRAG(
+        working_dir=WORKING_DIR,
+        entity_extract_max_gleaning=1,
+        enable_llm_cache=True,
+        enable_llm_cache_for_entity_extract=True,
+        embedding_cache_config={"enabled": True, "similarity_threshold": 0.90},
+        llm_model_func=llm_model_func,
+        embedding_func=EmbeddingFunc(
+            embedding_dim=1024,
+            max_token_size=8192,
+            func=embedding_func,
+        ),
+    )
+
+    await rag.initialize_storages()
+    await initialize_pipeline_status()
+
+    return rag
+
+
+def main():
+    # Initialize RAG instance
+    rag = asyncio.run(initialize_rag())
+
+    with open("./book.txt", "r", encoding="utf-8") as f:
+        rag.insert(f.read())
+
+    # Context Manager Method
+    with token_tracker:
+        print(
+            rag.query(
+                "What are the top themes in this story?", param=QueryParam(mode="naive")
+            )
+        )
+
+        print(
+            rag.query(
+                "What are the top themes in this story?", param=QueryParam(mode="local")
+            )
+        )
+
+        print(
+            rag.query(
+                "What are the top themes in this story?",
+                param=QueryParam(mode="global"),
+            )
+        )
+
+        print(
+            rag.query(
+                "What are the top themes in this story?",
+                param=QueryParam(mode="hybrid"),
+            )
+        )
+
+
+if __name__ == "__main__":
+    main()

examples/lightrag_siliconcloud_track_token_demo.py

@@ -0,0 +1,110 @@
+import os
+import asyncio
+from lightrag import LightRAG, QueryParam
+from lightrag.llm.openai import openai_complete_if_cache
+from lightrag.llm.siliconcloud import siliconcloud_embedding
+from lightrag.utils import EmbeddingFunc
+from lightrag.utils import TokenTracker
+import numpy as np
+from lightrag.kg.shared_storage import initialize_pipeline_status
+from dotenv import load_dotenv
+
+load_dotenv()
+
+token_tracker = TokenTracker()
+WORKING_DIR = "./dickens"
+
+if not os.path.exists(WORKING_DIR):
+    os.mkdir(WORKING_DIR)
+
+
+async def llm_model_func(
+    prompt, system_prompt=None, history_messages=[], keyword_extraction=False, **kwargs
+) -> str:
+    return await openai_complete_if_cache(
+        "Qwen/Qwen2.5-7B-Instruct",
+        prompt,
+        system_prompt=system_prompt,
+        history_messages=history_messages,
+        api_key=os.getenv("SILICONFLOW_API_KEY"),
+        base_url="https://api.siliconflow.cn/v1/",
+        token_tracker=token_tracker,
+        **kwargs,
+    )
+
+
+async def embedding_func(texts: list[str]) -> np.ndarray:
+    return await siliconcloud_embedding(
+        texts,
+        model="BAAI/bge-m3",
+        api_key=os.getenv("SILICONFLOW_API_KEY"),
+        max_token_size=512,
+    )
+
+
+# function test
+async def test_funcs():
+    # Context Manager Method
+    with token_tracker:
+        result = await llm_model_func("How are you?")
+        print("llm_model_func: ", result)
+
+
+asyncio.run(test_funcs())
+
+
+async def initialize_rag():
+    rag = LightRAG(
+        working_dir=WORKING_DIR,
+        llm_model_func=llm_model_func,
+        embedding_func=EmbeddingFunc(
+            embedding_dim=1024, max_token_size=512, func=embedding_func
+        ),
+    )
+
+    await rag.initialize_storages()
+    await initialize_pipeline_status()
+
+    return rag
+
+
+def main():
+    # Initialize RAG instance
+    rag = asyncio.run(initialize_rag())
+
+    # Reset tracker before processing queries
+    token_tracker.reset()
+
+    with open("./book.txt", "r", encoding="utf-8") as f:
+        rag.insert(f.read())
+
+    print(
+        rag.query(
+            "What are the top themes in this story?", param=QueryParam(mode="naive")
+        )
+    )
+
+    print(
+        rag.query(
+            "What are the top themes in this story?", param=QueryParam(mode="local")
+        )
+    )
+
+    print(
+        rag.query(
+            "What are the top themes in this story?", param=QueryParam(mode="global")
+        )
+    )
+
+    print(
+        rag.query(
+            "What are the top themes in this story?", param=QueryParam(mode="hybrid")
+        )
+    )
+
+    # Display final token usage after main query
+    print("Token usage:", token_tracker.get_usage())
+
+
+if __name__ == "__main__":
+    main()

lightrag/__init__.py

@@ -1,5 +1,5 @@
 from .lightrag import LightRAG as LightRAG, QueryParam as QueryParam
 
-__version__ = "1.3.0"
+__version__ = "1.3.1"
 __author__ = "Zirui Guo"
 __url__ = "https://github.com/HKUDS/LightRAG"

lightrag/api/README-zh.md

@@ -52,7 +52,8 @@ LLM_BINDING=openai
 LLM_MODEL=gpt-4o
 LLM_BINDING_HOST=https://api.openai.com/v1
 LLM_BINDING_API_KEY=your_api_key
-MAX_TOKENS=32768                # 发送给 LLM 的最大 token 数（小于模型上下文大小）
+### 发送给 LLM 的最大 token 数（小于模型上下文大小）
+MAX_TOKENS=32768
 
 EMBEDDING_BINDING=ollama
 EMBEDDING_BINDING_HOST=http://localhost:11434
@@ -68,7 +69,8 @@ LLM_BINDING=ollama
 LLM_MODEL=mistral-nemo:latest
 LLM_BINDING_HOST=http://localhost:11434
 # LLM_BINDING_API_KEY=your_api_key
-MAX_TOKENS=8192                  # 发送给 LLM 的最大 token 数（基于您的 Ollama 服务器容量）
+### 发送给 LLM 的最大 token 数（基于您的 Ollama 服务器容量）
+MAX_TOKENS=8192
 
 EMBEDDING_BINDING=ollama
 EMBEDDING_BINDING_HOST=http://localhost:11434
@@ -117,9 +119,12 @@ LightRAG 服务器可以在 `Gunicorn + Uvicorn` 预加载模式下运行。Guni
 虽然 LightRAG 服务器使用一个工作进程来处理文档索引流程，但通过 Uvicorn 的异步任务支持，可以并行处理多个文件。文档索引速度的瓶颈主要在于 LLM。如果您的 LLM 支持高并发，您可以通过增加 LLM 的并发级别来加速文档索引。以下是几个与并发处理相关的环境变量及其默认值：
 
 ```
-WORKERS=2                    # 工作进程数，不大于 (2 x 核心数) + 1
-MAX_PARALLEL_INSERT=2        # 一批中并行处理的文件数
-MAX_ASYNC=4                  # LLM 的最大并发请求数
+### 工作进程数，数字不大于 (2 x 核心数) + 1
+WORKERS=2
+### 一批中并行处理的文件数
+MAX_PARALLEL_INSERT=2
+# LLM 的最大并发请求数
+MAX_ASYNC=4
 ```
 
 ### 将 Lightrag 安装为 Linux 服务
@@ -201,10 +206,9 @@ LightRAG API 服务器使用基于 HS256 算法的 JWT 认证。要启用安全
 
 ```bash
 # JWT 认证
-AUTH_USERNAME=admin      # 登录名
-AUTH_PASSWORD=admin123   # 密码
-TOKEN_SECRET=your-key    # JWT 密钥
-TOKEN_EXPIRE_HOURS=4     # 过期时间
+AUTH_ACCOUNTS='admin:admin123,user1:pass456'
+TOKEN_SECRET='your-key'
+TOKEN_EXPIRE_HOURS=4
 ```
 
 > 目前仅支持配置一个管理员账户和密码。尚未开发和实现完整的账户系统。
@@ -238,8 +242,11 @@ LLM_BINDING=azure_openai
 LLM_BINDING_HOST=your-azure-endpoint
 LLM_MODEL=your-model-deployment-name
 LLM_BINDING_API_KEY=your-azure-api-key
-AZURE_OPENAI_API_VERSION=2024-08-01-preview  # 可选，默认为最新版本
-EMBEDDING_BINDING=azure_openai  # 如果使用 Azure OpenAI 进行嵌入
+### API Version可选，默认为最新版本
+AZURE_OPENAI_API_VERSION=2024-08-01-preview
+
+### 如果使用 Azure OpenAI 进行嵌入
+EMBEDDING_BINDING=azure_openai
 EMBEDDING_MODEL=your-embedding-deployment-name
 ```
 
@@ -362,7 +369,47 @@ LIGHTRAG_DOC_STATUS_STORAGE=PGDocStatusStorage
 | --embedding-binding | ollama | 嵌入绑定类型（lollms、ollama、openai、azure_openai） |
 | auto-scan-at-startup | - | 扫描输入目录中的新文件并开始索引 |
 
-### 使用示例
+### .env 文件示例
+
+```bash
+### Server Configuration
+# HOST=0.0.0.0
+PORT=9621
+WORKERS=2
+
+### Settings for document indexing
+ENABLE_LLM_CACHE_FOR_EXTRACT=true
+SUMMARY_LANGUAGE=Chinese
+MAX_PARALLEL_INSERT=2
+
+### LLM Configuration (Use valid host. For local services installed with docker, you can use host.docker.internal)
+TIMEOUT=200
+TEMPERATURE=0.0
+MAX_ASYNC=4
+MAX_TOKENS=32768
+
+LLM_BINDING=openai
+LLM_MODEL=gpt-4o-mini
+LLM_BINDING_HOST=https://api.openai.com/v1
+LLM_BINDING_API_KEY=your-api-key
+
+### Embedding Configuration (Use valid host. For local services installed with docker, you can use host.docker.internal)
+EMBEDDING_MODEL=bge-m3:latest
+EMBEDDING_DIM=1024
+EMBEDDING_BINDING=ollama
+EMBEDDING_BINDING_HOST=http://localhost:11434
+
+### For JWT Auth
+# AUTH_ACCOUNTS='admin:admin123,user1:pass456'
+# TOKEN_SECRET=your-key-for-LightRAG-API-Server-xxx
+# TOKEN_EXPIRE_HOURS=48
+
+# LIGHTRAG_API_KEY=your-secure-api-key-here-123
+# WHITELIST_PATHS=/api/*
+# WHITELIST_PATHS=/health,/api/*
+```
+
+
 
 #### 使用 ollama 默认本地服务器作为 llm 和嵌入后端运行 Lightrag 服务器
 

lightrag/api/README.md

@@ -52,7 +52,8 @@ LLM_BINDING=openai
 LLM_MODEL=gpt-4o
 LLM_BINDING_HOST=https://api.openai.com/v1
 LLM_BINDING_API_KEY=your_api_key
-MAX_TOKENS=32768                # Max tokens send to LLM (less than model context size)
+### Max tokens send to LLM (less than model context size)
+MAX_TOKENS=32768
 
 EMBEDDING_BINDING=ollama
 EMBEDDING_BINDING_HOST=http://localhost:11434
@@ -68,7 +69,8 @@ LLM_BINDING=ollama
 LLM_MODEL=mistral-nemo:latest
 LLM_BINDING_HOST=http://localhost:11434
 # LLM_BINDING_API_KEY=your_api_key
-MAX_TOKENS=8192                  # Max tokens send to LLM (base on your Ollama Server capacity)
+### Max tokens send to LLM (base on your Ollama Server capacity)
+MAX_TOKENS=8192
 
 EMBEDDING_BINDING=ollama
 EMBEDDING_BINDING_HOST=http://localhost:11434
@@ -90,7 +92,9 @@ lightrag-server
 ```
 lightrag-gunicorn --workers 4
 ```
-The `.env` file must be placed in the startup directory. Upon launching, the LightRAG Server will create a documents directory (default is `./inputs`) and a data directory (default is `./rag_storage`). This allows you to initiate multiple instances of LightRAG Server from different directories, with each instance configured to listen on a distinct network port.
+The `.env` file **must be placed in the startup directory**.
+
+Upon launching, the LightRAG Server will create a documents directory (default is `./inputs`) and a data directory (default is `./rag_storage`). This allows you to initiate multiple instances of LightRAG Server from different directories, with each instance configured to listen on a distinct network port.
 
 Here are some common used startup parameters:
 
@@ -100,6 +104,8 @@ Here are some common used startup parameters:
 - `--log-level`: Logging level (default: INFO)
 - --input-dir: specifying the directory to scan for documents (default: ./input)
 
+> The requirement for the .env file to be in the startup directory is intentionally designed this way. The purpose is to support users in launching multiple LightRAG instances simultaneously. Allow different .env files for different instances.
+
 ### Auto scan on startup
 
 When starting any of the servers with the `--auto-scan-at-startup` parameter, the system will automatically:
@@ -117,9 +123,12 @@ The LightRAG Server can operate in the `Gunicorn + Uvicorn` preload mode. Gunico
 Though LightRAG Server uses one workers to process the document indexing pipeline, with aysnc task supporting of Uvicorn, multiple files can be processed in parallell. The bottleneck of document indexing speed mainly lies with the LLM. If your LLM supports high concurrency, you can accelerate document indexing by increasing the concurrency level of the LLM. Below are several environment variables related to concurrent processing, along with their default values:
 
 ```
-WORKERS=2                    # Num of worker processes, not greater then (2 x number_of_cores) + 1
-MAX_PARALLEL_INSERT=2        # Num of parallel files to process in one batch
-MAX_ASYNC=4                  # Max concurrency requests of LLM
+### Num of worker processes, not greater then (2 x number_of_cores) + 1
+WORKERS=2
+### Num of parallel files to process in one batch
+MAX_PARALLEL_INSERT=2
+### Max concurrency requests of LLM
+MAX_ASYNC=4
 ```
 
 ### Install Lightrag as a Linux Service
@@ -203,10 +212,9 @@ LightRAG API Server implements JWT-based authentication using HS256 algorithm. T
 
 ```bash
 # For jwt auth
-AUTH_USERNAME=admin      # login name
-AUTH_PASSWORD=admin123   # password
-TOKEN_SECRET=your-key    # JWT key
-TOKEN_EXPIRE_HOURS=4     # expire duration
+AUTH_ACCOUNTS='admin:admin123,user1:pass456'
+TOKEN_SECRET='your-key'
+TOKEN_EXPIRE_HOURS=4
 ```
 
 > Currently, only the configuration of an administrator account and password is supported. A comprehensive account system is yet to be developed and implemented.
@@ -243,10 +251,12 @@ LLM_BINDING=azure_openai
 LLM_BINDING_HOST=your-azure-endpoint
 LLM_MODEL=your-model-deployment-name
 LLM_BINDING_API_KEY=your-azure-api-key
-AZURE_OPENAI_API_VERSION=2024-08-01-preview  # optional, defaults to latest version
-EMBEDDING_BINDING=azure_openai  # if using Azure OpenAI for embeddings
-EMBEDDING_MODEL=your-embedding-deployment-name
+### API version is optional, defaults to latest version
+AZURE_OPENAI_API_VERSION=2024-08-01-preview
 
+### if using Azure OpenAI for embeddings
+EMBEDDING_BINDING=azure_openai
+EMBEDDING_MODEL=your-embedding-deployment-name
 ```
 
 
@@ -370,76 +380,47 @@ You can not change storage implementation selection after you add documents to L
 | --embedding-binding | ollama | Embedding binding type (lollms, ollama, openai, azure_openai) |
 | auto-scan-at-startup | - | Scan input directory for new files and start indexing |
 
-### Example Usage
-
-#### Running a Lightrag server with ollama default local server as llm and embedding backends
-
-Ollama is the default backend for both llm and embedding, so by default you can run lightrag-server with no parameters and the default ones will be used. Make sure ollama is installed and is running and default models are already installed on ollama.
+### .env Examples
 
 ```bash
-# Run lightrag with ollama, mistral-nemo:latest for llm, and bge-m3:latest for embedding
-lightrag-server
-
-# Using an authentication key
-lightrag-server --key my-key
-```
+### Server Configuration
+# HOST=0.0.0.0
+PORT=9621
+WORKERS=2
+
+### Settings for document indexing
+ENABLE_LLM_CACHE_FOR_EXTRACT=true
+SUMMARY_LANGUAGE=Chinese
+MAX_PARALLEL_INSERT=2
+
+### LLM Configuration (Use valid host. For local services installed with docker, you can use host.docker.internal)
+TIMEOUT=200
+TEMPERATURE=0.0
+MAX_ASYNC=4
+MAX_TOKENS=32768
 
-#### Running a Lightrag server with lollms default local server as llm and embedding backends
-
-```bash
-# Run lightrag with lollms, mistral-nemo:latest for llm, and bge-m3:latest for embedding
-# Configure LLM_BINDING=lollms and EMBEDDING_BINDING=lollms in .env or config.ini
-lightrag-server
-
-# Using an authentication key
-lightrag-server --key my-key
-```
-
-#### Running a Lightrag server with openai server as llm and embedding backends
-
-```bash
-# Run lightrag with openai, GPT-4o-mini for llm, and text-embedding-3-small for embedding
-# Configure in .env or config.ini:
-# LLM_BINDING=openai
-# LLM_MODEL=GPT-4o-mini
-# EMBEDDING_BINDING=openai
-# EMBEDDING_MODEL=text-embedding-3-small
-lightrag-server
-
-# Using an authentication key
-lightrag-server --key my-key
-```
-
-#### Running a Lightrag server with azure openai server as llm and embedding backends
+LLM_BINDING=openai
+LLM_MODEL=gpt-4o-mini
+LLM_BINDING_HOST=https://api.openai.com/v1
+LLM_BINDING_API_KEY=your-api-key
 
-```bash
-# Run lightrag with azure_openai
-# Configure in .env or config.ini:
-# LLM_BINDING=azure_openai
-# LLM_MODEL=your-model
-# EMBEDDING_BINDING=azure_openai
-# EMBEDDING_MODEL=your-embedding-model
-lightrag-server
+### Embedding Configuration (Use valid host. For local services installed with docker, you can use host.docker.internal)
+EMBEDDING_MODEL=bge-m3:latest
+EMBEDDING_DIM=1024
+EMBEDDING_BINDING=ollama
+EMBEDDING_BINDING_HOST=http://localhost:11434
 
-# Using an authentication key
-lightrag-server --key my-key
-```
+### For JWT Auth
+# AUTH_ACCOUNTS='admin:admin123,user1:pass456'
+# TOKEN_SECRET=your-key-for-LightRAG-API-Server-xxx
+# TOKEN_EXPIRE_HOURS=48
 
-**Important Notes:**
-- For LoLLMs: Make sure the specified models are installed in your LoLLMs instance
-- For Ollama: Make sure the specified models are installed in your Ollama instance
-- For OpenAI: Ensure you have set up your OPENAI_API_KEY environment variable
-- For Azure OpenAI: Build and configure your server as stated in the Prequisites section
+# LIGHTRAG_API_KEY=your-secure-api-key-here-123
+# WHITELIST_PATHS=/api/*
+# WHITELIST_PATHS=/health,/api/*
 
-For help on any server, use the --help flag:
-```bash
-lightrag-server --help
 ```
 
-Note: If you don't need the API functionality, you can install the base package without API support using:
-```bash
-pip install lightrag-hku
-```
 
 ## API Endpoints
 

lightrag/api/__init__.py

@@ -1 +1 @@
-__api_version__ = "1.2.7"
+__api_version__ = "1.2.8"

lightrag/api/auth.py

@@ -7,7 +7,10 @@ from pydantic import BaseModel
 
 from .config import global_args
 
-load_dotenv()
+# use the .env that is inside the current folder
+# allows to use different .env file for each lightrag instance
+# the OS environment variables take precedence over the .env file
+load_dotenv(dotenv_path=".env", override=False)
 
 
 class TokenPayload(BaseModel):

lightrag/api/lightrag_server.py

@@ -50,15 +50,18 @@ from lightrag.kg.shared_storage import (
 from fastapi.security import OAuth2PasswordRequestForm
 from lightrag.api.auth import auth_handler
 
-# Load environment variables
-# Updated to use the .env that is inside the current folder
-# This update allows the user to put a different.env file for each lightrag folder
-load_dotenv(".env")
+# use the .env that is inside the current folder
+# allows to use different .env file for each lightrag instance
+# the OS environment variables take precedence over the .env file
+load_dotenv(dotenv_path=".env", override=False)
 
 # Initialize config parser
 config = configparser.ConfigParser()
 config.read("config.ini")
 
+# Global authentication configuration
+auth_configured = bool(auth_handler.accounts)
+
 
 def create_app(args):
     # Setup logging
@@ -429,9 +432,7 @@ def create_app(args):
         try:
             pipeline_status = await get_namespace_data("pipeline_status")
 
-            username = os.getenv("AUTH_USERNAME")
-            password = os.getenv("AUTH_PASSWORD")
-            if not (username and password):
+            if not auth_configured:
                 auth_mode = "disabled"
             else:
                 auth_mode = "enabled"

lightrag/api/routers/document_routes.py

@@ -540,6 +540,7 @@ def create_document_routes(
 
         Returns:
             InsertResponse: A response object containing the upload status and a message.
+                status can be "success", "duplicated", or error is thrown.
 
         Raises:
             HTTPException: If the file type is not supported (400) or other errors occur (500).
@@ -552,6 +553,13 @@ def create_document_routes(
                 )
 
             file_path = doc_manager.input_dir / file.filename
+            # Check if file already exists
+            if file_path.exists():
+                return InsertResponse(
+                    status="duplicated",
+                    message=f"File '{file.filename}' already exists in the input directory.",
+                )
+
             with open(file_path, "wb") as buffer:
                 shutil.copyfileobj(file.file, buffer)
 

lightrag/api/run_with_gunicorn.py

@@ -11,9 +11,10 @@ from lightrag.api.utils_api import parse_args, display_splash_screen, check_env_
 from lightrag.kg.shared_storage import initialize_share_data, finalize_share_data
 from dotenv import load_dotenv
 
-# Updated to use the .env that is inside the current folder
-# This update allows the user to put a different.env file for each lightrag folder
-load_dotenv(".env")
+# use the .env that is inside the current folder
+# allows to use different .env file for each lightrag instance
+# the OS environment variables take precedence over the .env file
+load_dotenv(dotenv_path=".env", override=False)
 
 
 def check_and_install_dependencies():

lightrag/api/utils_api.py

@@ -2,18 +2,17 @@
 Utility functions for the LightRAG API.
 """
 
-import argparse
 import os
-import sys
+import argparse
 from typing import Optional, List, Tuple
-
+import sys
 from ascii_colors import ASCIIColors
-from dotenv import load_dotenv
+from lightrag.api import __api_version__ as api_version
+from lightrag import __version__ as core_version
 from fastapi import HTTPException, Security, Request, status
+from dotenv import load_dotenv
 from fastapi.security import APIKeyHeader, OAuth2PasswordBearer
 from starlette.status import HTTP_403_FORBIDDEN
-
-from lightrag.api import __api_version__
 from .auth import auth_handler
 from .config import ollama_server_infos
 from ..prompt import PROMPTS
@@ -25,9 +24,7 @@ def check_env_file():
     Returns True if should continue, False if should exit.
     """
     if not os.path.exists(".env"):
-        warning_msg = (
-            "Warning: .env file not found. Some features may not work properly."
-        )
+        warning_msg = "Warning: Startup directory must contain .env file for multi-instance support."
         ASCIIColors.yellow(warning_msg)
 
         # Check if running in interactive terminal
@@ -39,8 +36,10 @@ def check_env_file():
     return True
 
 
-# Load environment variables
-load_dotenv()
+# use the .env that is inside the current folder
+# allows to use different .env file for each lightrag instance
+# the OS environment variables take precedence over the .env file
+load_dotenv(dotenv_path=".env", override=False)
 
 # Get whitelist paths from environment variable, only once during initialization
 default_whitelist = "/health,/api/*"
@@ -182,7 +181,7 @@ def display_splash_screen(args: argparse.Namespace) -> None:
     # Banner
     ASCIIColors.cyan(f"""
     ╔══════════════════════════════════════════════════════════════╗
-    ║                   🚀 LightRAG Server v{__api_version__}                  ║
+    ║                   🚀 LightRAG Server v{core_version}/{api_version}            ║
     ║          Fast, Lightweight RAG Server Implementation         ║
     ╚══════════════════════════════════════════════════════════════╝
     """)

lightrag/base.py

@@ -16,7 +16,10 @@ import numpy as np
 from .utils import EmbeddingFunc
 from .types import KnowledgeGraph
 
-load_dotenv()
+# use the .env that is inside the current folder
+# allows to use different .env file for each lightrag instance
+# the OS environment variables take precedence over the .env file
+load_dotenv(dotenv_path=".env", override=False)
 
 
 class TextChunkSchema(TypedDict):

lightrag/lightrag.py

@@ -55,8 +55,10 @@ from .utils import (
 from .types import KnowledgeGraph
 from dotenv import load_dotenv
 
-# Load environment variables
-load_dotenv(override=True)
+# use the .env that is inside the current folder
+# allows to use different .env file for each lightrag instance
+# the OS environment variables take precedence over the .env file
+load_dotenv(dotenv_path=".env", override=False)
 
 # TODO: TO REMOVE @Yannick
 config = configparser.ConfigParser()

lightrag/llm/openai.py

@@ -58,6 +58,7 @@ async def openai_complete_if_cache(
     history_messages: list[dict[str, Any]] | None = None,
     base_url: str | None = None,
     api_key: str | None = None,
+    token_tracker: Any | None = None,
     **kwargs: Any,
 ) -> str:
     if history_messages is None:
@@ -89,11 +90,13 @@ async def openai_complete_if_cache(
     messages.extend(history_messages)
     messages.append({"role": "user", "content": prompt})
 
-    logger.debug("===== Sending Query to LLM =====")
+    logger.debug("===== Entering func of LLM =====")
     logger.debug(f"Model: {model}   Base URL: {base_url}")
     logger.debug(f"Additional kwargs: {kwargs}")
-    verbose_debug(f"Query: {prompt}")
+    logger.debug(f"Num of history messages: {len(history_messages)}")
     verbose_debug(f"System prompt: {system_prompt}")
+    verbose_debug(f"Query: {prompt}")
+    logger.debug("===== Sending Query to LLM =====")
 
     try:
         if "response_format" in kwargs:
@@ -154,6 +157,18 @@ async def openai_complete_if_cache(
 
         if r"\u" in content:
             content = safe_unicode_decode(content.encode("utf-8"))
+
+        if token_tracker and hasattr(response, "usage"):
+            token_counts = {
+                "prompt_tokens": getattr(response.usage, "prompt_tokens", 0),
+                "completion_tokens": getattr(response.usage, "completion_tokens", 0),
+                "total_tokens": getattr(response.usage, "total_tokens", 0),
+            }
+            token_tracker.add_usage(token_counts)
+
+        logger.debug(f"Response content len: {len(content)}")
+        verbose_debug(f"Response: {response}")
+
         return content
 
 

lightrag/operate.py

@@ -38,8 +38,10 @@ from .prompt import GRAPH_FIELD_SEP, PROMPTS
 import time
 from dotenv import load_dotenv
 
-# Load environment variables
-load_dotenv(override=True)
+# use the .env that is inside the current folder
+# allows to use different .env file for each lightrag instance
+# the OS environment variables take precedence over the .env file
+load_dotenv(dotenv_path=".env", override=False)
 
 
 def chunking_by_token_size(
@@ -589,7 +591,7 @@ async def extract_entities(
         processed_chunks += 1
         entities_count = len(maybe_nodes)
         relations_count = len(maybe_edges)
-        log_message = f"  Chunk {processed_chunks}/{total_chunks}: extracted {entities_count} entities and {relations_count} relationships (deduplicated)"
+        log_message = f"  Chk {processed_chunks}/{total_chunks}: extracted {entities_count} Ent + {relations_count} Rel (deduplicated)"
         logger.info(log_message)
         if pipeline_status is not None:
             async with pipeline_status_lock:
@@ -654,7 +656,7 @@ async def extract_entities(
                 pipeline_status["latest_message"] = log_message
                 pipeline_status["history_messages"].append(log_message)
 
-    log_message = f"Extracted {len(all_entities_data)} entities and {len(all_relationships_data)} relationships (deduplicated)"
+    log_message = f"Extracted {len(all_entities_data)} entities + {len(all_relationships_data)} relationships (deduplicated)"
     logger.info(log_message)
     if pipeline_status is not None:
         async with pipeline_status_lock:
@@ -1038,7 +1040,7 @@ async def mix_kg_vector_query(
             # Include time information in content
             formatted_chunks = []
             for c in maybe_trun_chunks:
-                chunk_text = c["content"]
+                chunk_text = "File path: " + c["file_path"] + "\n" + c["content"]
                 if c["created_at"]:
                     chunk_text = f"[Created at: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(c['created_at']))}]\n{chunk_text}"
                 formatted_chunks.append(chunk_text)
@@ -1334,9 +1336,9 @@ async def _get_node_data(
         )
     relations_context = list_of_list_to_csv(relations_section_list)
 
-    text_units_section_list = [["id", "content"]]
+    text_units_section_list = [["id", "content", "file_path"]]
     for i, t in enumerate(use_text_units):
-        text_units_section_list.append([i, t["content"]])
+        text_units_section_list.append([i, t["content"], t["file_path"]])
     text_units_context = list_of_list_to_csv(text_units_section_list)
     return entities_context, relations_context, text_units_context
 
@@ -1597,9 +1599,9 @@ async def _get_edge_data(
         )
     entities_context = list_of_list_to_csv(entites_section_list)
 
-    text_units_section_list = [["id", "content"]]
+    text_units_section_list = [["id", "content", "file_path"]]
     for i, t in enumerate(use_text_units):
-        text_units_section_list.append([i, t["content"]])
+        text_units_section_list.append([i, t["content"], t["file_path"]])
     text_units_context = list_of_list_to_csv(text_units_section_list)
     return entities_context, relations_context, text_units_context
 
@@ -1785,7 +1787,12 @@ async def naive_query(
         f"Truncate chunks from {len(chunks)} to {len(maybe_trun_chunks)} (max tokens:{query_param.max_token_for_text_unit})"
     )
 
-    section = "\n--New Chunk--\n".join([c["content"] for c in maybe_trun_chunks])
+    section = "\n--New Chunk--\n".join(
+        [
+            "File path: " + c["file_path"] + "\n" + c["content"]
+            for c in maybe_trun_chunks
+        ]
+    )
 
     if query_param.only_need_context:
         return section

lightrag/prompt.py

@@ -222,7 +222,7 @@ When handling relationships with timestamps:
 - Use markdown formatting with appropriate section headings
 - Please respond in the same language as the user's question.
 - Ensure the response maintains continuity with the conversation history.
-- List up to 5 most important reference sources at the end under "References" section. Clearly indicating whether each source is from Knowledge Graph (KG) or Vector Data (DC), and include the file path if available, in the following format: [KG/DC] Source content (File: file_path)
+- List up to 5 most important reference sources at the end under "References" section. Clearly indicating whether each source is from Knowledge Graph (KG) or Vector Data (DC), and include the file path if available, in the following format: [KG/DC] file_path
 - If you don't know the answer, just say so.
 - Do not make anything up. Do not include information not provided by the Knowledge Base."""
 
@@ -320,7 +320,7 @@ When handling content with timestamps:
 - Use markdown formatting with appropriate section headings
 - Please respond in the same language as the user's question.
 - Ensure the response maintains continuity with the conversation history.
-- List up to 5 most important reference sources at the end under "References" section. Clearly indicating whether each source is from Knowledge Graph (KG) or Vector Data (DC), and include the file path if available, in the following format: [KG/DC] Source content (File: file_path)
+- List up to 5 most important reference sources at the end under "References" section. Clearly indicating whether each source is from Knowledge Graph (KG) or Vector Data (DC), and include the file path if available, in the following format: [KG/DC] file_path
 - If you don't know the answer, just say so.
 - Do not include information not provided by the Document Chunks."""
 
@@ -382,6 +382,6 @@ When handling information with timestamps:
 - Ensure the response maintains continuity with the conversation history.
 - Organize answer in sections focusing on one main point or aspect of the answer
 - Use clear and descriptive section titles that reflect the content
-- List up to 5 most important reference sources at the end under "References" section. Clearly indicating whether each source is from Knowledge Graph (KG) or Vector Data (DC), and include the file path if available, in the following format: [KG/DC] Source content (File: file_path)
+- List up to 5 most important reference sources at the end under "References" section. Clearly indicating whether each source is from Knowledge Graph (KG) or Vector Data (DC), and include the file path if available, in the following format: [KG/DC] file_path
 - If you don't know the answer, just say so. Do not make anything up.
 - Do not include information not provided by the Data Sources."""

lightrag/utils.py

@@ -19,9 +19,10 @@ import tiktoken
 from lightrag.prompt import PROMPTS
 from dotenv import load_dotenv
 
-# Load environment variables
-load_dotenv(override=True)
-
+# use the .env that is inside the current folder
+# allows to use different .env file for each lightrag instance
+# the OS environment variables take precedence over the .env file
+load_dotenv(dotenv_path=".env", override=False)
 
 VERBOSE_DEBUG = os.getenv("VERBOSE", "false").lower() == "true"
 
@@ -46,7 +47,7 @@ def verbose_debug(msg: str, *args, **kwargs):
             formatted_msg = msg
         # Then truncate the formatted message
         truncated_msg = (
-            formatted_msg[:50] + "..." if len(formatted_msg) > 50 else formatted_msg
+            formatted_msg[:100] + "..." if len(formatted_msg) > 100 else formatted_msg
         )
         logger.debug(truncated_msg, **kwargs)
 
@@ -953,3 +954,60 @@ def check_storage_env_vars(storage_name: str) -> None:
             f"Storage implementation '{storage_name}' requires the following "
             f"environment variables: {', '.join(missing_vars)}"
         )
+
+
+class TokenTracker:
+    """Track token usage for LLM calls."""
+
+    def __init__(self):
+        self.reset()
+
+    def __enter__(self):
+        self.reset()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        print(self)
+
+    def reset(self):
+        self.prompt_tokens = 0
+        self.completion_tokens = 0
+        self.total_tokens = 0
+        self.call_count = 0
+
+    def add_usage(self, token_counts):
+        """Add token usage from one LLM call.
+
+        Args:
+            token_counts: A dictionary containing prompt_tokens, completion_tokens, total_tokens
+        """
+        self.prompt_tokens += token_counts.get("prompt_tokens", 0)
+        self.completion_tokens += token_counts.get("completion_tokens", 0)
+
+        # If total_tokens is provided, use it directly; otherwise calculate the sum
+        if "total_tokens" in token_counts:
+            self.total_tokens += token_counts["total_tokens"]
+        else:
+            self.total_tokens += token_counts.get(
+                "prompt_tokens", 0
+            ) + token_counts.get("completion_tokens", 0)
+
+        self.call_count += 1
+
+    def get_usage(self):
+        """Get current usage statistics."""
+        return {
+            "prompt_tokens": self.prompt_tokens,
+            "completion_tokens": self.completion_tokens,
+            "total_tokens": self.total_tokens,
+            "call_count": self.call_count,
+        }
+
+    def __str__(self):
+        usage = self.get_usage()
+        return (
+            f"LLM call count: {usage['call_count']}, "
+            f"Prompt tokens: {usage['prompt_tokens']}, "
+            f"Completion tokens: {usage['completion_tokens']}, "
+            f"Total tokens: {usage['total_tokens']}"
+        )

lightrag_webui/src/api/lightrag.ts

@@ -109,7 +109,7 @@ export type QueryResponse = {
 }
 
 export type DocActionResponse = {
-  status: 'success' | 'partial_success' | 'failure'
+  status: 'success' | 'partial_success' | 'failure' | 'duplicated'
   message: string
 }
 

lightrag_webui/src/components/documents/UploadDocumentsDialog.tsx

@@ -1,4 +1,5 @@
 import { useState, useCallback } from 'react'
+import { FileRejection } from 'react-dropzone'
 import Button from '@/components/ui/Button'
 import {
   Dialog,
@@ -23,57 +24,132 @@ export default function UploadDocumentsDialog() {
   const [progresses, setProgresses] = useState<Record<string, number>>({})
   const [fileErrors, setFileErrors] = useState<Record<string, string>>({})
 
+  const handleRejectedFiles = useCallback(
+    (rejectedFiles: FileRejection[]) => {
+      // Process rejected files and add them to fileErrors
+      rejectedFiles.forEach(({ file, errors }) => {
+        // Get the first error message
+        let errorMsg = errors[0]?.message || t('documentPanel.uploadDocuments.fileUploader.fileRejected', { name: file.name })
+
+        // Simplify error message for unsupported file types
+        if (errorMsg.includes('file-invalid-type')) {
+          errorMsg = t('documentPanel.uploadDocuments.fileUploader.unsupportedType')
+        }
+
+        // Set progress to 100% to display error message
+        setProgresses((pre) => ({
+          ...pre,
+          [file.name]: 100
+        }))
+
+        // Add error message to fileErrors
+        setFileErrors(prev => ({
+          ...prev,
+          [file.name]: errorMsg
+        }))
+      })
+    },
+    [setProgresses, setFileErrors, t]
+  )
+
   const handleDocumentsUpload = useCallback(
     async (filesToUpload: File[]) => {
       setIsUploading(true)
-      setFileErrors({})
+
+      // Only clear errors for files that are being uploaded, keep errors for rejected files
+      setFileErrors(prev => {
+        const newErrors = { ...prev };
+        filesToUpload.forEach(file => {
+          delete newErrors[file.name];
+        });
+        return newErrors;
+      });
+
+      // Show uploading toast
+      const toastId = toast.loading(t('documentPanel.uploadDocuments.batch.uploading'))
 
       try {
-        toast.promise(
-          (async () => {
+        // Track errors locally to ensure we have the final state
+        const uploadErrors: Record<string, string> = {}
+
+        await Promise.all(
+          filesToUpload.map(async (file) => {
             try {
-              await Promise.all(
-                filesToUpload.map(async (file) => {
-                  try {
-                    const result = await uploadDocument(file, (percentCompleted: number) => {
-                      console.debug(t('documentPanel.uploadDocuments.single.uploading', { name: file.name, percent: percentCompleted }))
-                      setProgresses((pre) => ({
-                        ...pre,
-                        [file.name]: percentCompleted
-                      }))
-                    })
-
-                    if (result.status !== 'success') {
-                      setFileErrors(prev => ({
-                        ...prev,
-                        [file.name]: result.message
-                      }))
-                    }
-                  } catch (err) {
-                    setFileErrors(prev => ({
-                      ...prev,
-                      [file.name]: errorMessage(err)
-                    }))
-                  }
-                })
-              )
-            } catch (error) {
-              console.error('Upload failed:', error)
+              // Initialize upload progress
+              setProgresses((pre) => ({
+                ...pre,
+                [file.name]: 0
+              }))
+
+              const result = await uploadDocument(file, (percentCompleted: number) => {
+                console.debug(t('documentPanel.uploadDocuments.single.uploading', { name: file.name, percent: percentCompleted }))
+                setProgresses((pre) => ({
+                  ...pre,
+                  [file.name]: percentCompleted
+                }))
+              })
+
+              if (result.status === 'duplicated') {
+                uploadErrors[file.name] = t('documentPanel.uploadDocuments.fileUploader.duplicateFile')
+                setFileErrors(prev => ({
+                  ...prev,
+                  [file.name]: t('documentPanel.uploadDocuments.fileUploader.duplicateFile')
+                }))
+              } else if (result.status !== 'success') {
+                uploadErrors[file.name] = result.message
+                setFileErrors(prev => ({
+                  ...prev,
+                  [file.name]: result.message
+                }))
+              }
+            } catch (err) {
+              console.error(`Upload failed for ${file.name}:`, err)
+
+              // Handle HTTP errors, including 400 errors
+              let errorMsg = errorMessage(err)
+
+              // If it's an axios error with response data, try to extract more detailed error info
+              if (err && typeof err === 'object' && 'response' in err) {
+                const axiosError = err as { response?: { status: number, data?: { detail?: string } } }
+                if (axiosError.response?.status === 400) {
+                  // Extract specific error message from backend response
+                  errorMsg = axiosError.response.data?.detail || errorMsg
+                }
+
+                // Set progress to 100% to display error message
+                setProgresses((pre) => ({
+                  ...pre,
+                  [file.name]: 100
+                }))
+              }
+
+              // Record error message in both local tracking and state
+              uploadErrors[file.name] = errorMsg
+              setFileErrors(prev => ({
+                ...prev,
+                [file.name]: errorMsg
+              }))
             }
-          })(),
-          {
-            loading: t('documentPanel.uploadDocuments.batch.uploading'),
-            success: t('documentPanel.uploadDocuments.batch.success'),
-            error: t('documentPanel.uploadDocuments.batch.error')
-          }
+          })
         )
+
+        // Check if any files failed to upload using our local tracking
+        const hasErrors = Object.keys(uploadErrors).length > 0
+
+        // Update toast status
+        if (hasErrors) {
+          toast.error(t('documentPanel.uploadDocuments.batch.error'), { id: toastId })
+        } else {
+          toast.success(t('documentPanel.uploadDocuments.batch.success'), { id: toastId })
+        }
       } catch (err) {
-        toast.error(t('documentPanel.uploadDocuments.generalError', { error: errorMessage(err) }))
+        console.error('Unexpected error during upload:', err)
+        toast.error(t('documentPanel.uploadDocuments.generalError', { error: errorMessage(err) }), { id: toastId })
       } finally {
         setIsUploading(false)
       }
     },
-    [setIsUploading, setProgresses, t]
+    [setIsUploading, setProgresses, setFileErrors, t]
   )
 
   return (
@@ -107,6 +183,7 @@ export default function UploadDocumentsDialog() {
           maxSize={200 * 1024 * 1024}
           description={t('documentPanel.uploadDocuments.fileTypes')}
           onUpload={handleDocumentsUpload}
+          onReject={handleRejectedFiles}
           progresses={progresses}
           fileErrors={fileErrors}
           disabled={isUploading}

lightrag_webui/src/components/ui/FileUploader.tsx

@@ -39,6 +39,14 @@ interface FileUploaderProps extends React.HTMLAttributes<HTMLDivElement> {
    */
   onUpload?: (files: File[]) => Promise<void>
 
+  /**
+   * Function to be called when files are rejected.
+   * @type (rejections: FileRejection[]) => void
+   * @default undefined
+   * @example onReject={(rejections) => handleRejectedFiles(rejections)}
+   */
+  onReject?: (rejections: FileRejection[]) => void
+
   /**
    * Progress of the uploaded files.
    * @type Record<string, number> | undefined
@@ -125,6 +133,7 @@ function FileUploader(props: FileUploaderProps) {
     value: valueProp,
     onValueChange,
     onUpload,
+    onReject,
     progresses,
     fileErrors,
     accept = supportedFileTypes,
@@ -144,38 +153,77 @@ function FileUploader(props: FileUploaderProps) {
 
   const onDrop = React.useCallback(
     (acceptedFiles: File[], rejectedFiles: FileRejection[]) => {
-      if (!multiple && maxFileCount === 1 && acceptedFiles.length > 1) {
+      // Calculate total file count including both accepted and rejected files
+      const totalFileCount = (files?.length ?? 0) + acceptedFiles.length + rejectedFiles.length
+
+      // Check file count limits
+      if (!multiple && maxFileCount === 1 && (acceptedFiles.length + rejectedFiles.length) > 1) {
         toast.error(t('documentPanel.uploadDocuments.fileUploader.singleFileLimit'))
         return
       }
 
-      if ((files?.length ?? 0) + acceptedFiles.length > maxFileCount) {
+      if (totalFileCount > maxFileCount) {
         toast.error(t('documentPanel.uploadDocuments.fileUploader.maxFilesLimit', { count: maxFileCount }))
         return
       }
 
-      const newFiles = acceptedFiles.map((file) =>
+      // Handle rejected files first - this will set error states
+      if (rejectedFiles.length > 0) {
+        if (onReject) {
+          // Use the onReject callback if provided
+          onReject(rejectedFiles)
+        } else {
+          // Fall back to toast notifications if no callback is provided
+          rejectedFiles.forEach(({ file }) => {
+            toast.error(t('documentPanel.uploadDocuments.fileUploader.fileRejected', { name: file.name }))
+          })
+        }
+      }
+
+      // Process accepted files
+      const newAcceptedFiles = acceptedFiles.map((file) =>
         Object.assign(file, {
           preview: URL.createObjectURL(file)
         })
       )
 
-      const updatedFiles = files ? [...files, ...newFiles] : newFiles
+      // Process rejected files for UI display
+      const newRejectedFiles = rejectedFiles.map(({ file }) =>
+        Object.assign(file, {
+          preview: URL.createObjectURL(file),
+          rejected: true
+        })
+      )
+
+      // Combine all files for display
+      const allNewFiles = [...newAcceptedFiles, ...newRejectedFiles]
+      const updatedFiles = files ? [...files, ...allNewFiles] : allNewFiles
 
+      // Update the files state with all files
       setFiles(updatedFiles)
 
-      if (rejectedFiles.length > 0) {
-        rejectedFiles.forEach(({ file }) => {
-          toast.error(t('documentPanel.uploadDocuments.fileUploader.fileRejected', { name: file.name }))
-        })
-      }
+      // Only upload accepted files - make sure we're not uploading rejected files
+      if (onUpload && acceptedFiles.length > 0) {
+        // Filter out any files that might have been rejected by our custom validator
+        const validFiles = acceptedFiles.filter(file => {
+          // Check if file type is accepted
+          const fileExt = `.${file.name.split('.').pop()?.toLowerCase() || ''}`;
+          const isAccepted = Object.entries(accept || {}).some(([mimeType, extensions]) => {
+            return file.type === mimeType || extensions.includes(fileExt);
+          });
 
-      if (onUpload && updatedFiles.length > 0 && updatedFiles.length <= maxFileCount) {
-        onUpload(updatedFiles)
+          // Check file size
+          const isSizeValid = file.size <= maxSize;
+
+          return isAccepted && isSizeValid;
+        });
+
+        if (validFiles.length > 0) {
+          onUpload(validFiles);
+        }
       }
     },
-
-    [files, maxFileCount, multiple, onUpload, setFiles, t]
+    [files, maxFileCount, multiple, onUpload, onReject, setFiles, t, accept, maxSize]
   )
 
   function onRemove(index: number) {
@@ -204,11 +252,39 @@ function FileUploader(props: FileUploaderProps) {
     <div className="relative flex flex-col gap-6 overflow-hidden">
       <Dropzone
         onDrop={onDrop}
-        accept={accept}
+        // remove accept，use customizd validator
+        noClick={false}
+        noKeyboard={false}
         maxSize={maxSize}
         maxFiles={maxFileCount}
         multiple={maxFileCount > 1 || multiple}
         disabled={isDisabled}
+        validator={(file) => {
+          // Check if file type is accepted
+          const fileExt = `.${file.name.split('.').pop()?.toLowerCase() || ''}`;
+          const isAccepted = Object.entries(accept || {}).some(([mimeType, extensions]) => {
+            return file.type === mimeType || extensions.includes(fileExt);
+          });
+
+          if (!isAccepted) {
+            return {
+              code: 'file-invalid-type',
+              message: t('documentPanel.uploadDocuments.fileUploader.unsupportedType')
+            };
+          }
+
+          // Check file size
+          if (file.size > maxSize) {
+            return {
+              code: 'file-too-large',
+              message: t('documentPanel.uploadDocuments.fileUploader.fileTooLarge', {
+                maxSize: formatBytes(maxSize)
+              })
+            };
+          }
+
+          return null;
+        }}
       >
         {({ getRootProps, getInputProps, isDragActive }) => (
           <div
@@ -279,18 +355,21 @@ function FileUploader(props: FileUploaderProps) {
 interface ProgressProps {
   value: number
   error?: boolean
+  showIcon?: boolean  // New property to control icon display
 }
 
 function Progress({ value, error }: ProgressProps) {
   return (
-    <div className="h-2 w-full overflow-hidden rounded-full bg-secondary">
-      <div
-        className={cn(
-          'h-full transition-all',
-          error ? 'bg-destructive' : 'bg-primary'
-        )}
-        style={{ width: `${value}%` }}
-      />
+    <div className="relative h-2 w-full">
+      <div className="h-full w-full overflow-hidden rounded-full bg-secondary">
+        <div
+          className={cn(
+            'h-full transition-all',
+            error ? 'bg-red-400' : 'bg-primary'
+          )}
+          style={{ width: `${value}%` }}
+        />
+      </div>
     </div>
   )
 }
@@ -307,16 +386,22 @@ function FileCard({ file, progress, error, onRemove }: FileCardProps) {
   return (
     <div className="relative flex items-center gap-2.5">
       <div className="flex flex-1 gap-2.5">
-        {isFileWithPreview(file) ? <FilePreview file={file} /> : null}
+        {error ? (
+          <FileText className="text-red-400 size-10" aria-hidden="true" />
+        ) : (
+          isFileWithPreview(file) ? <FilePreview file={file} /> : null
+        )}
         <div className="flex w-full flex-col gap-2">
           <div className="flex flex-col gap-px">
             <p className="text-foreground/80 line-clamp-1 text-sm font-medium">{file.name}</p>
             <p className="text-muted-foreground text-xs">{formatBytes(file.size)}</p>
           </div>
           {error ? (
-            <div className="text-destructive text-sm">
-              <Progress value={100} error={true} />
-              <p className="mt-1">{error}</p>
+            <div className="text-red-400 text-sm">
+              <div className="relative mb-2">
+                <Progress value={100} error={true} />
+              </div>
+              <p>{error}</p>
             </div>
           ) : (
             progress ? <Progress value={progress} /> : null

lightrag_webui/src/features/DocumentManager.tsx

@@ -21,7 +21,7 @@ import { errorMessage } from '@/lib/utils'
 import { toast } from 'sonner'
 import { useBackendState } from '@/stores/state'
 
-import { RefreshCwIcon, ActivityIcon } from 'lucide-react'
+import { RefreshCwIcon, ActivityIcon, ArrowUpIcon, ArrowDownIcon } from 'lucide-react'
 import { DocStatusResponse } from '@/api/lightrag'
 import PipelineStatusDialog from '@/components/documents/PipelineStatusDialog'
 
@@ -47,6 +47,49 @@ const getDisplayFileName = (doc: DocStatusResponse, maxLength: number = 20): str
 };
 
 const pulseStyle = `
+/* Tooltip styles */
+.tooltip-container {
+  position: relative;
+  overflow: visible !important;
+}
+
+.tooltip {
+  position: fixed; /* Use fixed positioning to escape overflow constraints */
+  z-index: 9999; /* Ensure tooltip appears above all other elements */
+  max-width: 600px;
+  white-space: normal;
+  border-radius: 0.375rem;
+  padding: 0.5rem 0.75rem;
+  background-color: rgba(0, 0, 0, 0.95);
+  color: white;
+  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+  pointer-events: none; /* Prevent tooltip from interfering with mouse events */
+  opacity: 0;
+  visibility: hidden;
+  transition: opacity 0.15s, visibility 0.15s;
+}
+
+.tooltip.visible {
+  opacity: 1;
+  visibility: visible;
+}
+
+.dark .tooltip {
+  background-color: rgba(255, 255, 255, 0.95);
+  color: black;
+}
+
+/* Position tooltip helper class */
+.tooltip-helper {
+  position: absolute;
+  visibility: hidden;
+  pointer-events: none;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 0;
+}
+
 @keyframes pulse {
   0% {
     background-color: rgb(255 0 0 / 0.1);
@@ -87,6 +130,10 @@ const pulseStyle = `
 }
 `;
 
+// Type definitions for sort field and direction
+type SortField = 'created_at' | 'updated_at' | 'id';
+type SortDirection = 'asc' | 'desc';
+
 export default function DocumentManager() {
   const [showPipelineStatus, setShowPipelineStatus] = useState(false)
   const { t } = useTranslation()
@@ -97,6 +144,52 @@ export default function DocumentManager() {
   const showFileName = useSettingsStore.use.showFileName()
   const setShowFileName = useSettingsStore.use.setShowFileName()
 
+  // Sort state
+  const [sortField, setSortField] = useState<SortField>('updated_at')
+  const [sortDirection, setSortDirection] = useState<SortDirection>('desc')
+
+  // Handle sort column click
+  const handleSort = (field: SortField) => {
+    if (sortField === field) {
+      // Toggle sort direction if clicking the same field
+      setSortDirection(prev => prev === 'asc' ? 'desc' : 'asc')
+    } else {
+      // Set new sort field with default desc direction
+      setSortField(field)
+      setSortDirection('desc')
+    }
+  }
+
+  // Sort documents based on current sort field and direction
+  const sortDocuments = (documents: DocStatusResponse[]) => {
+    return [...documents].sort((a, b) => {
+      let valueA, valueB;
+
+      // Special handling for ID field based on showFileName setting
+      if (sortField === 'id' && showFileName) {
+        valueA = getDisplayFileName(a);
+        valueB = getDisplayFileName(b);
+      } else if (sortField === 'id') {
+        valueA = a.id;
+        valueB = b.id;
+      } else {
+        // Date fields
+        valueA = new Date(a[sortField]).getTime();
+        valueB = new Date(b[sortField]).getTime();
+      }
+
+      // Apply sort direction
+      const sortMultiplier = sortDirection === 'asc' ? 1 : -1;
+
+      // Compare values
+      if (typeof valueA === 'string' && typeof valueB === 'string') {
+        return sortMultiplier * valueA.localeCompare(valueB);
+      } else {
+        return sortMultiplier * (valueA > valueB ? 1 : valueA < valueB ? -1 : 0);
+      }
+    });
+  }
+
   // Store previous status counts
   const prevStatusCounts = useRef({
     processed: 0,
@@ -115,6 +208,71 @@ export default function DocumentManager() {
     }
   }, [])
 
+  // Reference to the card content element
+  const cardContentRef = useRef<HTMLDivElement>(null);
+
+  // Add tooltip position adjustment for fixed positioning
+  useEffect(() => {
+    if (!docs) return;
+
+    // Function to position tooltips
+    const positionTooltips = () => {
+      // Get all tooltip containers
+      const containers = document.querySelectorAll<HTMLElement>('.tooltip-container');
+
+      containers.forEach(container => {
+        const tooltip = container.querySelector<HTMLElement>('.tooltip');
+        if (!tooltip) return;
+
+        // Skip tooltips that aren't visible
+        if (!tooltip.classList.contains('visible')) return;
+
+        // Get container position
+        const rect = container.getBoundingClientRect();
+
+        // Position tooltip above the container
+        tooltip.style.left = `${rect.left}px`;
+        tooltip.style.top = `${rect.top - 5}px`;
+        tooltip.style.transform = 'translateY(-100%)';
+      });
+    };
+
+    // Set up event listeners
+    const handleMouseOver = (e: MouseEvent) => {
+      // Check if target or its parent is a tooltip container
+      const target = e.target as HTMLElement;
+      const container = target.closest('.tooltip-container');
+      if (!container) return;
+
+      // Find tooltip and make it visible
+      const tooltip = container.querySelector<HTMLElement>('.tooltip');
+      if (tooltip) {
+        tooltip.classList.add('visible');
+        // Position immediately without delay
+        positionTooltips();
+      }
+    };
+
+    const handleMouseOut = (e: MouseEvent) => {
+      const target = e.target as HTMLElement;
+      const container = target.closest('.tooltip-container');
+      if (!container) return;
+
+      const tooltip = container.querySelector<HTMLElement>('.tooltip');
+      if (tooltip) {
+        tooltip.classList.remove('visible');
+      }
+    };
+
+    document.addEventListener('mouseover', handleMouseOver);
+    document.addEventListener('mouseout', handleMouseOut);
+
+    return () => {
+      document.removeEventListener('mouseover', handleMouseOver);
+      document.removeEventListener('mouseout', handleMouseOut);
+    };
+  }, [docs]);
+
   const fetchDocuments = useCallback(async () => {
     try {
       const docs = await getDocuments()
@@ -192,13 +350,18 @@ export default function DocumentManager() {
     return () => clearInterval(interval)
   }, [health, fetchDocuments, t, currentTab])
 
+  // Add dependency on sort state to re-render when sort changes
+  useEffect(() => {
+    // This effect ensures the component re-renders when sort state changes
+  }, [sortField, sortDirection]);
+
   return (
-    <Card className="!size-full !rounded-none !border-none">
-      <CardHeader>
+    <Card className="!rounded-none !overflow-hidden flex flex-col h-full min-h-0">
+      <CardHeader className="py-2 px-6">
         <CardTitle className="text-lg">{t('documentPanel.documentManager.title')}</CardTitle>
       </CardHeader>
-      <CardContent className="space-y-4">
-        <div className="flex gap-2">
+      <CardContent className="flex-1 flex flex-col min-h-0 overflow-auto">
+        <div className="flex gap-2 mb-2">
           <div className="flex gap-2">
             <Button
               variant="outline"
@@ -231,8 +394,8 @@ export default function DocumentManager() {
           />
         </div>
 
-        <Card>
-          <CardHeader>
+        <Card className="flex-1 flex flex-col border rounded-md min-h-0 mb-2">
+          <CardHeader className="flex-none py-2 px-4">
             <div className="flex justify-between items-center">
               <CardTitle>{t('documentPanel.documentManager.uploadedTitle')}</CardTitle>
               <div className="flex items-center gap-2">
@@ -250,95 +413,140 @@ export default function DocumentManager() {
                 </Button>
               </div>
             </div>
-            <CardDescription>{t('documentPanel.documentManager.uploadedDescription')}</CardDescription>
+            <CardDescription aria-hidden="true" className="hidden">{t('documentPanel.documentManager.uploadedDescription')}</CardDescription>
           </CardHeader>
 
-          <CardContent>
+          <CardContent className="flex-1 relative p-0" ref={cardContentRef}>
             {!docs && (
-              <EmptyCard
-                title={t('documentPanel.documentManager.emptyTitle')}
-                description={t('documentPanel.documentManager.emptyDescription')}
-              />
+              <div className="absolute inset-0 p-0">
+                <EmptyCard
+                  title={t('documentPanel.documentManager.emptyTitle')}
+                  description={t('documentPanel.documentManager.emptyDescription')}
+                />
+              </div>
             )}
             {docs && (
-              <Table>
-                <TableHeader>
-                  <TableRow>
-                    <TableHead>{t('documentPanel.documentManager.columns.id')}</TableHead>
-                    <TableHead>{t('documentPanel.documentManager.columns.summary')}</TableHead>
-                    <TableHead>{t('documentPanel.documentManager.columns.status')}</TableHead>
-                    <TableHead>{t('documentPanel.documentManager.columns.length')}</TableHead>
-                    <TableHead>{t('documentPanel.documentManager.columns.chunks')}</TableHead>
-                    <TableHead>{t('documentPanel.documentManager.columns.created')}</TableHead>
-                    <TableHead>{t('documentPanel.documentManager.columns.updated')}</TableHead>
-                  </TableRow>
-                </TableHeader>
-                <TableBody className="text-sm">
-                  {Object.entries(docs.statuses).map(([status, documents]) =>
-                    documents.map((doc) => (
-                      <TableRow key={doc.id}>
-                        <TableCell className="truncate font-mono overflow-visible">
-                          {showFileName ? (
-                            <>
-                              <div className="group relative overflow-visible">
+              <div className="absolute inset-0 flex flex-col p-0">
+                <div className="absolute inset-[-1px] flex flex-col p-0 border rounded-md border-gray-200 dark:border-gray-700 overflow-hidden">
+                  <Table className="w-full">
+                    <TableHeader className="sticky top-0 bg-background z-10 shadow-sm">
+                      <TableRow className="border-b bg-card/95 backdrop-blur supports-[backdrop-filter]:bg-card/75 shadow-[inset_0_-1px_0_rgba(0,0,0,0.1)]">
+                        <TableHead
+                          onClick={() => handleSort('id')}
+                          className="cursor-pointer hover:bg-gray-200 dark:hover:bg-gray-800 select-none"
+                        >
+                          <div className="flex items-center">
+                            {t('documentPanel.documentManager.columns.id')}
+                            {sortField === 'id' && (
+                              <span className="ml-1">
+                                {sortDirection === 'asc' ? <ArrowUpIcon size={14} /> : <ArrowDownIcon size={14} />}
+                              </span>
+                            )}
+                          </div>
+                        </TableHead>
+                        <TableHead>{t('documentPanel.documentManager.columns.summary')}</TableHead>
+                        <TableHead>{t('documentPanel.documentManager.columns.status')}</TableHead>
+                        <TableHead>{t('documentPanel.documentManager.columns.length')}</TableHead>
+                        <TableHead>{t('documentPanel.documentManager.columns.chunks')}</TableHead>
+                        <TableHead
+                          onClick={() => handleSort('created_at')}
+                          className="cursor-pointer hover:bg-gray-200 dark:hover:bg-gray-800 select-none"
+                        >
+                          <div className="flex items-center">
+                            {t('documentPanel.documentManager.columns.created')}
+                            {sortField === 'created_at' && (
+                              <span className="ml-1">
+                                {sortDirection === 'asc' ? <ArrowUpIcon size={14} /> : <ArrowDownIcon size={14} />}
+                              </span>
+                            )}
+                          </div>
+                        </TableHead>
+                        <TableHead
+                          onClick={() => handleSort('updated_at')}
+                          className="cursor-pointer hover:bg-gray-200 dark:hover:bg-gray-800 select-none"
+                        >
+                          <div className="flex items-center">
+                            {t('documentPanel.documentManager.columns.updated')}
+                            {sortField === 'updated_at' && (
+                              <span className="ml-1">
+                                {sortDirection === 'asc' ? <ArrowUpIcon size={14} /> : <ArrowDownIcon size={14} />}
+                              </span>
+                            )}
+                          </div>
+                        </TableHead>
+                      </TableRow>
+                    </TableHeader>
+                    <TableBody className="text-sm overflow-auto">
+                      {Object.entries(docs.statuses).flatMap(([status, documents]) => {
+                        // Apply sorting to documents
+                        const sortedDocuments = sortDocuments(documents);
+
+                        return sortedDocuments.map(doc => (
+                          <TableRow key={doc.id}>
+                            <TableCell className="truncate font-mono overflow-visible max-w-[250px]">
+                              {showFileName ? (
+                                <>
+                                  <div className="group relative overflow-visible tooltip-container">
+                                    <div className="truncate">
+                                      {getDisplayFileName(doc, 30)}
+                                    </div>
+                                    <div className="invisible group-hover:visible tooltip">
+                                      {doc.file_path}
+                                    </div>
+                                  </div>
+                                  <div className="text-xs text-gray-500">{doc.id}</div>
+                                </>
+                              ) : (
+                                <div className="group relative overflow-visible tooltip-container">
+                                  <div className="truncate">
+                                    {doc.id}
+                                  </div>
+                                  <div className="invisible group-hover:visible tooltip">
+                                    {doc.file_path}
+                                  </div>
+                                </div>
+                              )}
+                            </TableCell>
+                            <TableCell className="max-w-xs min-w-45 truncate overflow-visible">
+                              <div className="group relative overflow-visible tooltip-container">
                                 <div className="truncate">
-                                  {getDisplayFileName(doc, 35)}
+                                  {doc.content_summary}
                                 </div>
-                                <div className="invisible group-hover:visible absolute z-[9999] mt-1 max-w-[800px] whitespace-normal break-all rounded-md bg-black/95 px-3 py-2 text-sm text-white shadow-lg dark:bg-white/95 dark:text-black">
-                                  {doc.file_path}
+                                <div className="invisible group-hover:visible tooltip">
+                                  {doc.content_summary}
                                 </div>
                               </div>
-                              <div className="text-xs text-gray-500">{doc.id}</div>
-                            </>
-                          ) : (
-                            <div className="group relative overflow-visible">
-                              <div className="truncate">
-                                {doc.id}
-                              </div>
-                              <div className="invisible group-hover:visible absolute z-[9999] mt-1 max-w-[800px] whitespace-normal break-all rounded-md bg-black/95 px-3 py-2 text-sm text-white shadow-lg dark:bg-white/95 dark:text-black">
-                                {doc.file_path}
-                              </div>
-                            </div>
-                          )}
-                        </TableCell>
-                        <TableCell className="max-w-xs min-w-24 truncate overflow-visible">
-                          <div className="group relative overflow-visible">
-                            <div className="truncate">
-                              {doc.content_summary}
-                            </div>
-                            <div className="invisible group-hover:visible absolute z-[9999] mt-1 max-w-[800px] whitespace-normal break-all rounded-md bg-black/95 px-3 py-2 text-sm text-white shadow-lg dark:bg-white/95 dark:text-black">
-                              {doc.content_summary}
-                            </div>
-                          </div>
-                        </TableCell>
-                        <TableCell>
-                          {status === 'processed' && (
-                            <span className="text-green-600">{t('documentPanel.documentManager.status.completed')}</span>
-                          )}
-                          {status === 'processing' && (
-                            <span className="text-blue-600">{t('documentPanel.documentManager.status.processing')}</span>
-                          )}
-                          {status === 'pending' && <span className="text-yellow-600">{t('documentPanel.documentManager.status.pending')}</span>}
-                          {status === 'failed' && <span className="text-red-600">{t('documentPanel.documentManager.status.failed')}</span>}
-                          {doc.error && (
-                            <span className="ml-2 text-red-500" title={doc.error}>
-                              ⚠️
-                            </span>
-                          )}
-                        </TableCell>
-                        <TableCell>{doc.content_length ?? '-'}</TableCell>
-                        <TableCell>{doc.chunks_count ?? '-'}</TableCell>
-                        <TableCell className="truncate">
-                          {new Date(doc.created_at).toLocaleString()}
-                        </TableCell>
-                        <TableCell className="truncate">
-                          {new Date(doc.updated_at).toLocaleString()}
-                        </TableCell>
-                      </TableRow>
-                    ))
-                  )}
-                </TableBody>
-              </Table>
+                            </TableCell>
+                            <TableCell>
+                              {status === 'processed' && (
+                                <span className="text-green-600">{t('documentPanel.documentManager.status.completed')}</span>
+                              )}
+                              {status === 'processing' && (
+                                <span className="text-blue-600">{t('documentPanel.documentManager.status.processing')}</span>
+                              )}
+                              {status === 'pending' && <span className="text-yellow-600">{t('documentPanel.documentManager.status.pending')}</span>}
+                              {status === 'failed' && <span className="text-red-600">{t('documentPanel.documentManager.status.failed')}</span>}
+                              {doc.error && (
+                                <span className="ml-2 text-red-500" title={doc.error}>
+                                  ⚠️
+                                </span>
+                              )}
+                            </TableCell>
+                            <TableCell>{doc.content_length ?? '-'}</TableCell>
+                            <TableCell>{doc.chunks_count ?? '-'}</TableCell>
+                            <TableCell className="truncate">
+                              {new Date(doc.created_at).toLocaleString()}
+                            </TableCell>
+                            <TableCell className="truncate">
+                              {new Date(doc.updated_at).toLocaleString()}
+                            </TableCell>
+                          </TableRow>
+                        ));
+                      })}
+                    </TableBody>
+                  </Table>
+                </div>
+              </div>
             )}
           </CardContent>
         </Card>

lightrag_webui/src/hooks/useLightragGraph.tsx

@@ -205,7 +205,7 @@ const createSigmaGraph = (rawGraph: RawGraph | null) => {
   // Add edges from raw graph data
   for (const rawEdge of rawGraph?.edges ?? []) {
     rawEdge.dynamicId = graph.addDirectedEdge(rawEdge.source, rawEdge.target, {
-      label: rawEdge.type || undefined
+      label: rawEdge.properties?.keywords || undefined
     })
   }
 
@@ -660,7 +660,7 @@ const useLightrangeGraph = () => {
 
           // Add the edge to the sigma graph
           newEdge.dynamicId = sigmaGraph.addDirectedEdge(newEdge.source, newEdge.target, {
-            label: newEdge.type || undefined
+            label: newEdge.properties?.keywords || undefined
           });
 
           // Add the edge to the raw graph

lightrag_webui/src/locales/ar.json

@@ -65,16 +65,19 @@
         "singleFileLimit": "لا يمكن رفع أكثر من ملف واحد في المرة الواحدة",
         "maxFilesLimit": "لا يمكن رفع أكثر من {{count}} ملفات",
         "fileRejected": "تم رفض الملف {{name}}",
+        "unsupportedType": "نوع الملف غير مدعوم",
+        "fileTooLarge": "حجم الملف كبير جدًا، الحد الأقصى {{maxSize}}",
         "dropHere": "أفلت الملفات هنا",
         "dragAndDrop": "اسحب وأفلت الملفات هنا، أو انقر للاختيار",
         "removeFile": "إزالة الملف",
-        "uploadDescription": "يمكنك رفع {{isMultiple ? 'عدة' : count}} ملفات (حتى {{maxSize}} لكل منها)"
+        "uploadDescription": "يمكنك رفع {{isMultiple ? 'عدة' : count}} ملفات (حتى {{maxSize}} لكل منها)",
+        "duplicateFile": "اسم الملف موجود بالفعل في ذاكرة التخزين المؤقت للخادم"
       }
     },
     "documentManager": {
       "title": "إدارة المستندات",
       "scanButton": "مسح ضوئي",
-      "scanTooltip": "مسح المستندات ضوئيًا",
+      "scanTooltip": "مسح المستندات ضوئيًا في مجلد الإدخال",
       "pipelineStatusButton": "حالة خط المعالجة",
       "pipelineStatusTooltip": "عرض حالة خط المعالجة",
       "uploadedTitle": "المستندات المرفوعة",
@@ -212,7 +215,8 @@
           "entity_id": "الاسم",
           "entity_type": "النوع",
           "source_id": "معرف المصدر",
-          "Neighbour": "الجار"
+          "Neighbour": "الجار",
+          "file_path": "المصدر"
         }
       },
       "edge": {

lightrag_webui/src/locales/en.json

@@ -65,16 +65,19 @@
         "singleFileLimit": "Cannot upload more than 1 file at a time",
         "maxFilesLimit": "Cannot upload more than {{count}} files",
         "fileRejected": "File {{name}} was rejected",
+        "unsupportedType": "Unsupported file type",
+        "fileTooLarge": "File too large, maximum size is {{maxSize}}",
         "dropHere": "Drop the files here",
         "dragAndDrop": "Drag and drop files here, or click to select files",
         "removeFile": "Remove file",
-        "uploadDescription": "You can upload {{isMultiple ? 'multiple' : count}} files (up to {{maxSize}} each)"
+        "uploadDescription": "You can upload {{isMultiple ? 'multiple' : count}} files (up to {{maxSize}} each)",
+        "duplicateFile": "File name already exists in server cache"
       }
     },
     "documentManager": {
       "title": "Document Management",
       "scanButton": "Scan",
-      "scanTooltip": "Scan documents",
+      "scanTooltip": "Scan documents in input folder",
       "pipelineStatusButton": "Pipeline Status",
       "pipelineStatusTooltip": "View pipeline status",
       "uploadedTitle": "Uploaded Documents",
@@ -212,7 +215,8 @@
           "entity_id": "Name",
           "entity_type": "Type",
           "source_id": "SrcID",
-          "Neighbour": "Neigh"
+          "Neighbour": "Neigh",
+          "file_path": "Source"
         }
       },
       "edge": {

lightrag_webui/src/locales/fr.json

@@ -65,16 +65,19 @@
         "singleFileLimit": "Impossible de télécharger plus d'un fichier à la fois",
         "maxFilesLimit": "Impossible de télécharger plus de {{count}} fichiers",
         "fileRejected": "Le fichier {{name}} a été rejeté",
+        "unsupportedType": "Type de fichier non pris en charge",
+        "fileTooLarge": "Fichier trop volumineux, taille maximale {{maxSize}}",
         "dropHere": "Déposez les fichiers ici",
         "dragAndDrop": "Glissez et déposez les fichiers ici, ou cliquez pour sélectionner",
         "removeFile": "Supprimer le fichier",
-        "uploadDescription": "Vous pouvez télécharger {{isMultiple ? 'plusieurs' : count}} fichiers (jusqu'à {{maxSize}} chacun)"
+        "uploadDescription": "Vous pouvez télécharger {{isMultiple ? 'plusieurs' : count}} fichiers (jusqu'à {{maxSize}} chacun)",
+        "duplicateFile": "Le nom du fichier existe déjà dans le cache du serveur"
       }
     },
     "documentManager": {
       "title": "Gestion des documents",
       "scanButton": "Scanner",
-      "scanTooltip": "Scanner les documents",
+      "scanTooltip": "Scanner les documents dans le dossier d'entrée",
       "pipelineStatusButton": "État du Pipeline",
       "pipelineStatusTooltip": "Voir l'état du pipeline",
       "uploadedTitle": "Documents téléchargés",
@@ -212,7 +215,8 @@
           "entity_id": "Nom",
           "entity_type": "Type",
           "source_id": "ID source",
-          "Neighbour": "Voisin"
+          "Neighbour": "Voisin",
+          "file_path": "Source"
         }
       },
       "edge": {

lightrag_webui/src/locales/zh.json

@@ -65,16 +65,19 @@
         "singleFileLimit": "一次只能上传一个文件",
         "maxFilesLimit": "最多只能上传 {{count}} 个文件",
         "fileRejected": "文件 {{name}} 被拒绝",
+        "unsupportedType": "不支持的文件类型",
+        "fileTooLarge": "文件过大，最大允许 {{maxSize}}",
         "dropHere": "将文件拖放到此处",
         "dragAndDrop": "拖放文件到此处，或点击选择文件",
         "removeFile": "移除文件",
-        "uploadDescription": "您可以上传{{isMultiple ? '多个' : count}}个文件（每个文件最大{{maxSize}}）"
+        "uploadDescription": "您可以上传{{isMultiple ? '多个' : count}}个文件（每个文件最大{{maxSize}}）",
+        "duplicateFile": "文件名与服务器上的缓存重复"
       }
     },
     "documentManager": {
       "title": "文档管理",
       "scanButton": "扫描",
-      "scanTooltip": "扫描文档",
+      "scanTooltip": "扫描输入目录中的文档",
       "pipelineStatusButton": "流水线状态",
       "pipelineStatusTooltip": "查看流水线状态",
       "uploadedTitle": "已上传文档",
@@ -212,7 +215,8 @@
           "entity_id": "名称",
           "entity_type": "类型",
           "source_id": "信源ID",
-          "Neighbour": "邻接"
+          "Neighbour": "邻接",
+          "file_path": "信源"
         }
       },
       "edge": {