diff --git a/.gitattributes b/.gitattributes index 02a362823572caf7a521a5ebea271cb56fa489fe..b967487de257bc1d982c9adf5c6bbfc8c3363483 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ +lightrag/api/webui/** -diff *.png filter=lfs diff=lfs merge=lfs -text *.ttf filter=lfs diff=lfs merge=lfs -text *.ico filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index a4afe4ea146a24cda0299fd2c166949769ab8fff..dd1c386b3e5a82dcce8b234f0ac243dbb2fa1eeb 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,6 @@ gui/ # unit-test files test_* + +# Cline files +memory-bank/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 169a7cc75b263efb5cd9938826210a2a4b25495e..0629b7432846aafba89a927dcadc7c157d503baf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,16 +3,21 @@ repos: rev: v5.0.0 hooks: - id: trailing-whitespace + exclude: ^lightrag/api/webui/ - id: end-of-file-fixer + exclude: ^lightrag/api/webui/ - id: requirements-txt-fixer + exclude: ^lightrag/api/webui/ - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.6.4 hooks: - id: ruff-format + exclude: ^lightrag/api/webui/ - id: ruff args: [--fix, --ignore=E402] + exclude: ^lightrag/api/webui/ - repo: https://github.com/mgedmin/check-manifest @@ -20,3 +25,4 @@ repos: hooks: - id: check-manifest stages: [manual] + exclude: ^lightrag/api/webui/ diff --git a/README.md b/README.md index 018a94e659fd4e1cdc20f4627b6253bab50da4cb..61e7b20f313c4428da3da49145d0fb14e4dd9ebb 100644 --- a/README.md +++ b/README.md @@ -37,28 +37,30 @@ This repository hosts the code of LightRAG. The structure of this code is based
+ + +
🎉 News - -- [x] [2025.02.05]🎯📢Our team has released [VideoRAG](https://github.com/HKUDS/VideoRAG) understanding extremely long-context videos. -- [x] [2025.01.13]🎯📢Our team has released [MiniRAG](https://github.com/HKUDS/MiniRAG) making RAG simpler with small models. -- [x] [2025.01.06]🎯📢You can now [use PostgreSQL for Storage](#using-postgresql-for-storage). -- [x] [2024.12.31]🎯📢LightRAG now supports [deletion by document ID](https://github.com/HKUDS/LightRAG?tab=readme-ov-file#delete). -- [x] [2024.11.25]🎯📢LightRAG now supports seamless integration of [custom knowledge graphs](https://github.com/HKUDS/LightRAG?tab=readme-ov-file#insert-custom-kg), empowering users to enhance the system with their own domain expertise. -- [x] [2024.11.19]🎯📢A comprehensive guide to LightRAG is now available on [LearnOpenCV](https://learnopencv.com/lightrag). Many thanks to the blog author. -- [x] [2024.11.12]🎯📢LightRAG now supports [Oracle Database 23ai for all storage types (KV, vector, and graph)](https://github.com/HKUDS/LightRAG/blob/main/examples/lightrag_oracle_demo.py). -- [x] [2024.11.11]🎯📢LightRAG now supports [deleting entities by their names](https://github.com/HKUDS/LightRAG?tab=readme-ov-file#delete). -- [x] [2024.11.09]🎯📢Introducing the [LightRAG Gui](https://lightrag-gui.streamlit.app), which allows you to insert, query, visualize, and download LightRAG knowledge. -- [x] [2024.11.04]🎯📢You can now [use Neo4J for Storage](https://github.com/HKUDS/LightRAG?tab=readme-ov-file#using-neo4j-for-storage). -- [x] [2024.10.29]🎯📢LightRAG now supports multiple file types, including PDF, DOC, PPT, and CSV via `textract`. -- [x] [2024.10.20]🎯📢We've added a new feature to LightRAG: Graph Visualization. -- [x] [2024.10.18]🎯📢We've added a link to a [LightRAG Introduction Video](https://youtu.be/oageL-1I0GE). Thanks to the author! -- [x] [2024.10.17]🎯📢We have created a [Discord channel](https://discord.gg/yF2MmDJyGJ)! Welcome to join for sharing and discussions! 🎉🎉 -- [x] [2024.10.16]🎯📢LightRAG now supports [Ollama models](https://github.com/HKUDS/LightRAG?tab=readme-ov-file#quick-start)! -- [x] [2024.10.15]🎯📢LightRAG now supports [Hugging Face models](https://github.com/HKUDS/LightRAG?tab=readme-ov-file#quick-start)! +- [X] [2025.02.05]🎯📢Our team has released [VideoRAG](https://github.com/HKUDS/VideoRAG) understanding extremely long-context videos. +- [X] [2025.01.13]🎯📢Our team has released [MiniRAG](https://github.com/HKUDS/MiniRAG) making RAG simpler with small models. +- [X] [2025.01.06]🎯📢You can now [use PostgreSQL for Storage](#using-postgresql-for-storage). +- [X] [2024.12.31]🎯📢LightRAG now supports [deletion by document ID](https://github.com/HKUDS/LightRAG?tab=readme-ov-file#delete). +- [X] [2024.11.25]🎯📢LightRAG now supports seamless integration of [custom knowledge graphs](https://github.com/HKUDS/LightRAG?tab=readme-ov-file#insert-custom-kg), empowering users to enhance the system with their own domain expertise. +- [X] [2024.11.19]🎯📢A comprehensive guide to LightRAG is now available on [LearnOpenCV](https://learnopencv.com/lightrag). Many thanks to the blog author. +- [X] [2024.11.12]🎯📢LightRAG now supports [Oracle Database 23ai for all storage types (KV, vector, and graph)](https://github.com/HKUDS/LightRAG/blob/main/examples/lightrag_oracle_demo.py). +- [X] [2024.11.11]🎯📢LightRAG now supports [deleting entities by their names](https://github.com/HKUDS/LightRAG?tab=readme-ov-file#delete). +- [X] [2024.11.09]🎯📢Introducing the [LightRAG Gui](https://lightrag-gui.streamlit.app), which allows you to insert, query, visualize, and download LightRAG knowledge. +- [X] [2024.11.04]🎯📢You can now [use Neo4J for Storage](https://github.com/HKUDS/LightRAG?tab=readme-ov-file#using-neo4j-for-storage). +- [X] [2024.10.29]🎯📢LightRAG now supports multiple file types, including PDF, DOC, PPT, and CSV via `textract`. +- [X] [2024.10.20]🎯📢We've added a new feature to LightRAG: Graph Visualization. +- [X] [2024.10.18]🎯📢We've added a link to a [LightRAG Introduction Video](https://youtu.be/oageL-1I0GE). Thanks to the author! +- [X] [2024.10.17]🎯📢We have created a [Discord channel](https://discord.gg/yF2MmDJyGJ)! Welcome to join for sharing and discussions! 🎉🎉 +- [X] [2024.10.16]🎯📢LightRAG now supports [Ollama models](https://github.com/HKUDS/LightRAG?tab=readme-ov-file#quick-start)! +- [X] [2024.10.15]🎯📢LightRAG now supports [Hugging Face models](https://github.com/HKUDS/LightRAG?tab=readme-ov-file#quick-start)!
@@ -82,16 +84,20 @@ This repository hosts the code of LightRAG. The structure of this code is based cd LightRAG pip install -e . ``` + * Install from PyPI + ```bash pip install lightrag-hku ``` ## Quick Start + * [Video demo](https://www.youtube.com/watch?v=g21royNJ4fw) of running LightRAG locally. * All the code can be found in the `examples`. * Set OpenAI API key in environment if using OpenAI models: `export OPENAI_API_KEY="sk-...".` * Download the demo text "A Christmas Carol by Charles Dickens": + ```bash curl https://raw.githubusercontent.com/gusye1234/nano-graphrag/main/tests/mock_data.txt > ./book.txt ``` @@ -187,6 +193,7 @@ class QueryParam: Using Open AI-like APIs * LightRAG also supports Open AI-like chat/embeddings APIs: + ```python async def llm_model_func( prompt, system_prompt=None, history_messages=[], keyword_extraction=False, **kwargs @@ -225,6 +232,7 @@ async def initialize_rag(): return rag ``` +
@@ -252,12 +260,14 @@ rag = LightRAG( ), ) ``` +
Using Ollama Models ### Overview + If you want to use Ollama models, you need to pull model you plan to use and embedding model, for example `nomic-embed-text`. Then you only need to set LightRAG as follows: @@ -281,31 +291,37 @@ rag = LightRAG( ``` ### Increasing context size + In order for LightRAG to work context should be at least 32k tokens. By default Ollama models have context size of 8k. You can achieve this using one of two ways: #### Increasing the `num_ctx` parameter in Modelfile. 1. Pull the model: + ```bash ollama pull qwen2 ``` 2. Display the model file: + ```bash ollama show --modelfile qwen2 > Modelfile ``` 3. Edit the Modelfile by adding the following line: + ```bash PARAMETER num_ctx 32768 ``` 4. Create the modified model: + ```bash ollama create -f Modelfile qwen2m ``` #### Setup `num_ctx` via Ollama API. + Tiy can use `llm_model_kwargs` param to configure ollama: ```python @@ -325,6 +341,7 @@ rag = LightRAG( ), ) ``` + #### Low RAM GPUs In order to run this experiment on low RAM GPU you should select small model and tune context window (increasing context increase memory consumption). For example, running this ollama example on repurposed mining GPU with 6Gb of RAM required to set context size to 26k while using `gemma2:2b`. It was able to find 197 entities and 19 relations on `book.txt`. @@ -402,6 +419,7 @@ if __name__ == "__main__": ``` #### For detailed documentation and examples, see: + - [LlamaIndex Documentation](lightrag/llm/Readme.md) - [Direct OpenAI Example](examples/lightrag_llamaindex_direct_demo.py) - [LiteLLM Proxy Example](examples/lightrag_llamaindex_litellm_demo.py) @@ -483,13 +501,16 @@ print(response_custom) We've introduced a new function `query_with_separate_keyword_extraction` to enhance the keyword extraction capabilities. This function separates the keyword extraction process from the user's prompt, focusing solely on the query to improve the relevance of extracted keywords. ##### How It Works? + The function operates by dividing the input into two parts: + - `User Query` - `Prompt` It then performs keyword extraction exclusively on the `user query`. This separation ensures that the extraction process is focused and relevant, unaffected by any additional language in the `prompt`. It also allows the `prompt` to serve purely for response formatting, maintaining the intent and clarity of the user's original question. ##### Usage Example + This `example` shows how to tailor the function for educational content, focusing on detailed explanations for older students. ```python @@ -563,6 +584,7 @@ custom_kg = { rag.insert_custom_kg(custom_kg) ``` +
## Insert @@ -593,6 +615,7 @@ rag.insert(["TEXT1", "TEXT2", "TEXT3", ...]) # Documents will be processed in b ``` The `insert_batch_size` parameter in `addon_params` controls how many documents are processed in each batch during insertion. This is useful for: + - Managing memory usage with large document collections - Optimizing processing speed - Providing better progress tracking @@ -647,6 +670,7 @@ text_content = textract.process(file_path) rag.insert(text_content.decode('utf-8')) ``` + ## Storage @@ -685,6 +709,7 @@ async def initialize_rag(): return rag ``` + see test_neo4j.py for a working example. @@ -693,6 +718,7 @@ see test_neo4j.py for a working example. Using PostgreSQL for Storage For production level scenarios you will most likely want to leverage an enterprise solution. PostgreSQL can provide a one-stop solution for you as KV store, VectorDB (pgvector) and GraphDB (apache AGE). + * PostgreSQL is lightweight,the whole binary distribution including all necessary plugins can be zipped to 40MB: Ref to [Windows Release](https://github.com/ShanGor/apache-age-windows/releases/tag/PG17%2Fv1.5.0-rc0) as it is easy to install for Linux/Mac. * If you prefer docker, please start with this image if you are a beginner to avoid hiccups (DO read the overview): https://hub.docker.com/r/shangor/postgres-for-rag * How to start? Ref to: [examples/lightrag_zhipu_postgres_demo.py](https://github.com/HKUDS/LightRAG/blob/main/examples/lightrag_zhipu_postgres_demo.py) @@ -735,6 +761,7 @@ For production level scenarios you will most likely want to leverage an enterpri > It is a known issue of the release version: https://github.com/apache/age/pull/1721 > > You can Compile the AGE from source code and fix it. + > @@ -742,9 +769,11 @@ For production level scenarios you will most likely want to leverage an enterpri Using Faiss for Storage - Install the required dependencies: + ``` pip install faiss-cpu ``` + You can also install `faiss-gpu` if you have GPU support. - Here we are using `sentence-transformers` but you can also use `OpenAIEmbedding` model with `3072` dimensions. @@ -810,6 +839,7 @@ relation = rag.create_relation("Google", "Gmail", { "weight": 2.0 }) ``` +
@@ -835,6 +865,7 @@ updated_relation = rag.edit_relation("Google", "Google Mail", { "weight": 3.0 }) ``` +
All operations are available in both synchronous and asynchronous versions. The asynchronous versions have the prefix "a" (e.g., `acreate_entity`, `aedit_relation`). @@ -851,6 +882,55 @@ All operations are available in both synchronous and asynchronous versions. The These operations maintain data consistency across both the graph database and vector database components, ensuring your knowledge graph remains coherent. +## Data Export Functions + +## Overview + +LightRAG allows you to export your knowledge graph data in various formats for analysis, sharing, and backup purposes. The system supports exporting entities, relations, and relationship data. + +## Export Functions + +### Basic Usage + +```python +# Basic CSV export (default format) +rag.export_data("knowledge_graph.csv") + +# Specify any format +rag.export_data("output.xlsx", file_format="excel") +``` + +### Different File Formats supported + +```python +#Export data in CSV format +rag.export_data("graph_data.csv", file_format="csv") + +# Export data in Excel sheet +rag.export_data("graph_data.xlsx", file_format="excel") + +# Export data in markdown format +rag.export_data("graph_data.md", file_format="md") + +# Export data in Text +rag.export_data("graph_data.txt", file_format="txt") +``` +## Additional Options + +Include vector embeddings in the export (optional): + +```python +rag.export_data("complete_data.csv", include_vector_data=True) +``` +## Data Included in Export + +All exports include: + +* Entity information (names, IDs, metadata) +* Relation data (connections between entities) +* Relationship information from vector database + + ## Entity Merging
@@ -913,6 +993,7 @@ rag.merge_entities( ``` When merging entities: + * All relationships from source entities are redirected to the target entity * Duplicate relationships are intelligently merged * Self-relationships (loops) are prevented @@ -946,6 +1027,7 @@ rag.clear_cache(modes=["local"]) ``` Valid modes are: + - `"default"`: Extraction cache - `"naive"`: Naive search cache - `"local"`: Local search cache @@ -960,33 +1042,33 @@ Valid modes are:
Parameters -| **Parameter** | **Type** | **Explanation** | **Default** | -|----------------------------------------------| --- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------| -| **working\_dir** | `str` | Directory where the cache will be stored | `lightrag_cache+timestamp` | -| **kv\_storage** | `str` | Storage type for documents and text chunks. Supported types: `JsonKVStorage`, `OracleKVStorage` | `JsonKVStorage` | -| **vector\_storage** | `str` | Storage type for embedding vectors. Supported types: `NanoVectorDBStorage`, `OracleVectorDBStorage` | `NanoVectorDBStorage` | -| **graph\_storage** | `str` | Storage type for graph edges and nodes. Supported types: `NetworkXStorage`, `Neo4JStorage`, `OracleGraphStorage` | `NetworkXStorage` | -| **chunk\_token\_size** | `int` | Maximum token size per chunk when splitting documents | `1200` | -| **chunk\_overlap\_token\_size** | `int` | Overlap token size between two chunks when splitting documents | `100` | -| **tiktoken\_model\_name** | `str` | Model name for the Tiktoken encoder used to calculate token numbers | `gpt-4o-mini` | -| **entity\_extract\_max\_gleaning** | `int` | Number of loops in the entity extraction process, appending history messages | `1` | -| **entity\_summary\_to\_max\_tokens** | `int` | Maximum token size for each entity summary | `500` | -| **node\_embedding\_algorithm** | `str` | Algorithm for node embedding (currently not used) | `node2vec` | -| **node2vec\_params** | `dict` | Parameters for node embedding | `{"dimensions": 1536,"num_walks": 10,"walk_length": 40,"window_size": 2,"iterations": 3,"random_seed": 3,}` | -| **embedding\_func** | `EmbeddingFunc` | Function to generate embedding vectors from text | `openai_embed` | -| **embedding\_batch\_num** | `int` | Maximum batch size for embedding processes (multiple texts sent per batch) | `32` | -| **embedding\_func\_max\_async** | `int` | Maximum number of concurrent asynchronous embedding processes | `16` | -| **llm\_model\_func** | `callable` | Function for LLM generation | `gpt_4o_mini_complete` | -| **llm\_model\_name** | `str` | LLM model name for generation | `meta-llama/Llama-3.2-1B-Instruct` | -| **llm\_model\_max\_token\_size** | `int` | Maximum token size for LLM generation (affects entity relation summaries) | `32768`（default value changed by env var MAX_TOKENS) | -| **llm\_model\_max\_async** | `int` | Maximum number of concurrent asynchronous LLM processes | `16`（default value changed by env var MAX_ASYNC) | -| **llm\_model\_kwargs** | `dict` | Additional parameters for LLM generation | | -| **vector\_db\_storage\_cls\_kwargs** | `dict` | Additional parameters for vector database, like setting the threshold for nodes and relations retrieval. | cosine_better_than_threshold: 0.2（default value changed by env var COSINE_THRESHOLD) | -| **enable\_llm\_cache** | `bool` | If `TRUE`, stores LLM results in cache; repeated prompts return cached responses | `TRUE` | -| **enable\_llm\_cache\_for\_entity\_extract** | `bool` | If `TRUE`, stores LLM results in cache for entity extraction; Good for beginners to debug your application | `TRUE` | -| **addon\_params** | `dict` | Additional parameters, e.g., `{"example_number": 1, "language": "Simplified Chinese", "entity_types": ["organization", "person", "geo", "event"], "insert_batch_size": 10}`: sets example limit, output language, and batch size for document processing | `example_number: all examples, language: English, insert_batch_size: 10` | -| **convert\_response\_to\_json\_func** | `callable` | Not used | `convert_response_to_json` | -| **embedding\_cache\_config** | `dict` | Configuration for question-answer caching. Contains three parameters:
- `enabled`: Boolean value to enable/disable cache lookup functionality. When enabled, the system will check cached responses before generating new answers.
- `similarity_threshold`: Float value (0-1), similarity threshold. When a new question's similarity with a cached question exceeds this threshold, the cached answer will be returned directly without calling the LLM.
- `use_llm_check`: Boolean value to enable/disable LLM similarity verification. When enabled, LLM will be used as a secondary check to verify the similarity between questions before returning cached answers. | Default: `{"enabled": False, "similarity_threshold": 0.95, "use_llm_check": False}` | +| **Parameter** | **Type** | **Explanation** | **Default** | +| -------------------------------------------------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| **working\_dir** | `str` | Directory where the cache will be stored | `lightrag_cache+timestamp` | +| **kv\_storage** | `str` | Storage type for documents and text chunks. Supported types:`JsonKVStorage`, `OracleKVStorage` | `JsonKVStorage` | +| **vector\_storage** | `str` | Storage type for embedding vectors. Supported types:`NanoVectorDBStorage`, `OracleVectorDBStorage` | `NanoVectorDBStorage` | +| **graph\_storage** | `str` | Storage type for graph edges and nodes. Supported types:`NetworkXStorage`, `Neo4JStorage`, `OracleGraphStorage` | `NetworkXStorage` | +| **chunk\_token\_size** | `int` | Maximum token size per chunk when splitting documents | `1200` | +| **chunk\_overlap\_token\_size** | `int` | Overlap token size between two chunks when splitting documents | `100` | +| **tiktoken\_model\_name** | `str` | Model name for the Tiktoken encoder used to calculate token numbers | `gpt-4o-mini` | +| **entity\_extract\_max\_gleaning** | `int` | Number of loops in the entity extraction process, appending history messages | `1` | +| **entity\_summary\_to\_max\_tokens** | `int` | Maximum token size for each entity summary | `500` | +| **node\_embedding\_algorithm** | `str` | Algorithm for node embedding (currently not used) | `node2vec` | +| **node2vec\_params** | `dict` | Parameters for node embedding | `{"dimensions": 1536,"num_walks": 10,"walk_length": 40,"window_size": 2,"iterations": 3,"random_seed": 3,}` | +| **embedding\_func** | `EmbeddingFunc` | Function to generate embedding vectors from text | `openai_embed` | +| **embedding\_batch\_num** | `int` | Maximum batch size for embedding processes (multiple texts sent per batch) | `32` | +| **embedding\_func\_max\_async** | `int` | Maximum number of concurrent asynchronous embedding processes | `16` | +| **llm\_model\_func** | `callable` | Function for LLM generation | `gpt_4o_mini_complete` | +| **llm\_model\_name** | `str` | LLM model name for generation | `meta-llama/Llama-3.2-1B-Instruct` | +| **llm\_model\_max\_token\_size** | `int` | Maximum token size for LLM generation (affects entity relation summaries) | `32768`（default value changed by env var MAX_TOKENS) | +| **llm\_model\_max\_async** | `int` | Maximum number of concurrent asynchronous LLM processes | `16`（default value changed by env var MAX_ASYNC) | +| **llm\_model\_kwargs** | `dict` | Additional parameters for LLM generation | | +| **vector\_db\_storage\_cls\_kwargs** | `dict` | Additional parameters for vector database, like setting the threshold for nodes and relations retrieval. | cosine_better_than_threshold: 0.2（default value changed by env var COSINE_THRESHOLD) | +| **enable\_llm\_cache** | `bool` | If `TRUE`, stores LLM results in cache; repeated prompts return cached responses | `TRUE` | +| **enable\_llm\_cache\_for\_entity\_extract** | `bool` | If `TRUE`, stores LLM results in cache for entity extraction; Good for beginners to debug your application | `TRUE` | +| **addon\_params** | `dict` | Additional parameters, e.g.,`{"example_number": 1, "language": "Simplified Chinese", "entity_types": ["organization", "person", "geo", "event"], "insert_batch_size": 10}`: sets example limit, output language, and batch size for document processing | `example_number: all examples, language: English, insert_batch_size: 10` | +| **convert\_response\_to\_json\_func** | `callable` | Not used | `convert_response_to_json` | +| **embedding\_cache\_config** | `dict` | Configuration for question-answer caching. Contains three parameters:`
`- `enabled`: Boolean value to enable/disable cache lookup functionality. When enabled, the system will check cached responses before generating new answers.`
`- `similarity_threshold`: Float value (0-1), similarity threshold. When a new question's similarity with a cached question exceeds this threshold, the cached answer will be returned directly without calling the LLM.`
`- `use_llm_check`: Boolean value to enable/disable LLM similarity verification. When enabled, LLM will be used as a secondary check to verify the similarity between questions before returning cached answers. | Default:`{"enabled": False, "similarity_threshold": 0.95, "use_llm_check": False}` |
@@ -996,12 +1078,15 @@ Valid modes are: Click to view error handling details The API includes comprehensive error handling: + - File not found errors (404) - Processing errors (500) - Supports multiple file encodings (UTF-8 and GBK) +
## API + LightRag can be installed with API support to serve a Fast api interface to perform data upload and indexing/Rag operations/Rescan of the input folder etc.. [LightRag API](lightrag/api/README.md) @@ -1035,7 +1120,6 @@ net.show('knowledge_graph.html')
Graph visualization with Neo4 - * The following code can be found in `examples/graph_visual_with_neo4j.py` ```python @@ -1171,10 +1255,13 @@ LightRag can be installed with Tools support to add extra tools like the graphml
## Evaluation + ### Dataset + The dataset used in LightRAG can be downloaded from [TommyChien/UltraDomain](https://huggingface.co/datasets/TommyChien/UltraDomain). ### Generate Query + LightRAG uses the following prompt to generate high-level queries, with the corresponding code in `example/generate_query.py`.
@@ -1203,9 +1290,11 @@ Output the results in the following structure: - User 5: [user description] ... ``` +
### Batch Eval + To evaluate the performance of two RAG systems on high-level queries, LightRAG uses the following prompt, with the specific code available in `example/batch_eval.py`.
@@ -1253,37 +1342,40 @@ Output your evaluation in the following JSON format: }} }} ``` +
### Overall Performance Table -| | **Agriculture** | | **CS** | | **Legal** | | **Mix** | | -|----------------------|-------------------------|-----------------------|-----------------------|-----------------------|-----------------------|-----------------------|-----------------------|-----------------------| -| | NaiveRAG | **LightRAG** | NaiveRAG | **LightRAG** | NaiveRAG | **LightRAG** | NaiveRAG | **LightRAG** | -| **Comprehensiveness** | 32.4% | **67.6%** | 38.4% | **61.6%** | 16.4% | **83.6%** | 38.8% | **61.2%** | -| **Diversity** | 23.6% | **76.4%** | 38.0% | **62.0%** | 13.6% | **86.4%** | 32.4% | **67.6%** | -| **Empowerment** | 32.4% | **67.6%** | 38.8% | **61.2%** | 16.4% | **83.6%** | 42.8% | **57.2%** | -| **Overall** | 32.4% | **67.6%** | 38.8% | **61.2%** | 15.2% | **84.8%** | 40.0% | **60.0%** | -| | RQ-RAG | **LightRAG** | RQ-RAG | **LightRAG** | RQ-RAG | **LightRAG** | RQ-RAG | **LightRAG** | -| **Comprehensiveness** | 31.6% | **68.4%** | 38.8% | **61.2%** | 15.2% | **84.8%** | 39.2% | **60.8%** | -| **Diversity** | 29.2% | **70.8%** | 39.2% | **60.8%** | 11.6% | **88.4%** | 30.8% | **69.2%** | -| **Empowerment** | 31.6% | **68.4%** | 36.4% | **63.6%** | 15.2% | **84.8%** | 42.4% | **57.6%** | -| **Overall** | 32.4% | **67.6%** | 38.0% | **62.0%** | 14.4% | **85.6%** | 40.0% | **60.0%** | -| | HyDE | **LightRAG** | HyDE | **LightRAG** | HyDE | **LightRAG** | HyDE | **LightRAG** | -| **Comprehensiveness** | 26.0% | **74.0%** | 41.6% | **58.4%** | 26.8% | **73.2%** | 40.4% | **59.6%** | -| **Diversity** | 24.0% | **76.0%** | 38.8% | **61.2%** | 20.0% | **80.0%** | 32.4% | **67.6%** | -| **Empowerment** | 25.2% | **74.8%** | 40.8% | **59.2%** | 26.0% | **74.0%** | 46.0% | **54.0%** | -| **Overall** | 24.8% | **75.2%** | 41.6% | **58.4%** | 26.4% | **73.6%** | 42.4% | **57.6%** | -| | GraphRAG | **LightRAG** | GraphRAG | **LightRAG** | GraphRAG | **LightRAG** | GraphRAG | **LightRAG** | -| **Comprehensiveness** | 45.6% | **54.4%** | 48.4% | **51.6%** | 48.4% | **51.6%** | **50.4%** | 49.6% | -| **Diversity** | 22.8% | **77.2%** | 40.8% | **59.2%** | 26.4% | **73.6%** | 36.0% | **64.0%** | -| **Empowerment** | 41.2% | **58.8%** | 45.2% | **54.8%** | 43.6% | **56.4%** | **50.8%** | 49.2% | -| **Overall** | 45.2% | **54.8%** | 48.0% | **52.0%** | 47.2% | **52.8%** | **50.4%** | 49.6% | +| | **Agriculture** | | **CS** | | **Legal** | | **Mix** | | +| --------------------------- | --------------------- | ------------------ | ------------ | ------------------ | --------------- | ------------------ | --------------- | ------------------ | +| | NaiveRAG | **LightRAG** | NaiveRAG | **LightRAG** | NaiveRAG | **LightRAG** | NaiveRAG | **LightRAG** | +| **Comprehensiveness** | 32.4% | **67.6%** | 38.4% | **61.6%** | 16.4% | **83.6%** | 38.8% | **61.2%** | +| **Diversity** | 23.6% | **76.4%** | 38.0% | **62.0%** | 13.6% | **86.4%** | 32.4% | **67.6%** | +| **Empowerment** | 32.4% | **67.6%** | 38.8% | **61.2%** | 16.4% | **83.6%** | 42.8% | **57.2%** | +| **Overall** | 32.4% | **67.6%** | 38.8% | **61.2%** | 15.2% | **84.8%** | 40.0% | **60.0%** | +| | RQ-RAG | **LightRAG** | RQ-RAG | **LightRAG** | RQ-RAG | **LightRAG** | RQ-RAG | **LightRAG** | +| **Comprehensiveness** | 31.6% | **68.4%** | 38.8% | **61.2%** | 15.2% | **84.8%** | 39.2% | **60.8%** | +| **Diversity** | 29.2% | **70.8%** | 39.2% | **60.8%** | 11.6% | **88.4%** | 30.8% | **69.2%** | +| **Empowerment** | 31.6% | **68.4%** | 36.4% | **63.6%** | 15.2% | **84.8%** | 42.4% | **57.6%** | +| **Overall** | 32.4% | **67.6%** | 38.0% | **62.0%** | 14.4% | **85.6%** | 40.0% | **60.0%** | +| | HyDE | **LightRAG** | HyDE | **LightRAG** | HyDE | **LightRAG** | HyDE | **LightRAG** | +| **Comprehensiveness** | 26.0% | **74.0%** | 41.6% | **58.4%** | 26.8% | **73.2%** | 40.4% | **59.6%** | +| **Diversity** | 24.0% | **76.0%** | 38.8% | **61.2%** | 20.0% | **80.0%** | 32.4% | **67.6%** | +| **Empowerment** | 25.2% | **74.8%** | 40.8% | **59.2%** | 26.0% | **74.0%** | 46.0% | **54.0%** | +| **Overall** | 24.8% | **75.2%** | 41.6% | **58.4%** | 26.4% | **73.6%** | 42.4% | **57.6%** | +| | GraphRAG | **LightRAG** | GraphRAG | **LightRAG** | GraphRAG | **LightRAG** | GraphRAG | **LightRAG** | +| **Comprehensiveness** | 45.6% | **54.4%** | 48.4% | **51.6%** | 48.4% | **51.6%** | **50.4%** | 49.6% | +| **Diversity** | 22.8% | **77.2%** | 40.8% | **59.2%** | 26.4% | **73.6%** | 36.0% | **64.0%** | +| **Empowerment** | 41.2% | **58.8%** | 45.2% | **54.8%** | 43.6% | **56.4%** | **50.8%** | 49.2% | +| **Overall** | 45.2% | **54.8%** | 48.0% | **52.0%** | 47.2% | **52.8%** | **50.4%** | 49.6% | ## Reproduce + All the code can be found in the `./reproduce` directory. ### Step-0 Extract Unique Contexts + First, we need to extract unique contexts in the datasets.
@@ -1340,9 +1432,11 @@ def extract_unique_contexts(input_directory, output_directory): print("All files have been processed.") ``` +
### Step-1 Insert Contexts + For the extracted contexts, we insert them into the LightRAG system.
@@ -1366,6 +1460,7 @@ def insert_text(rag, file_path): if retries == max_retries: print("Insertion failed after exceeding the maximum number of retries") ``` +
### Step-2 Generate Queries @@ -1390,9 +1485,11 @@ def get_summary(context, tot_tokens=2000): return summary ``` + ### Step-3 Query + For the queries generated in Step-2, we will extract them and query LightRAG.
@@ -1409,6 +1506,7 @@ def extract_queries(file_path): return queries ``` +
## Star History @@ -1441,4 +1539,5 @@ archivePrefix={arXiv}, primaryClass={cs.IR} } ``` + **Thank you for your interest in our work!** diff --git a/lightrag/__init__.py b/lightrag/__init__.py index 382060f72b482c5d8df57c725c151de9f75999a4..89475dca3e361886c5f5d2f0ea1b520512768d7e 100644 --- a/lightrag/__init__.py +++ b/lightrag/__init__.py @@ -1,5 +1,5 @@ from .lightrag import LightRAG as LightRAG, QueryParam as QueryParam -__version__ = "1.2.5" +__version__ = "1.2.6" __author__ = "Zirui Guo" __url__ = "https://github.com/HKUDS/LightRAG" diff --git a/lightrag/api/gunicorn_config.py b/lightrag/api/gunicorn_config.py index 0594ceae5b37500242c0fd4cf9fb7e3eecca2fb1..23e468078478be73f9690d45eda5fd4dcff4f844 100644 --- a/lightrag/api/gunicorn_config.py +++ b/lightrag/api/gunicorn_config.py @@ -59,7 +59,7 @@ logconfig_dict = { }, "filters": { "path_filter": { - "()": "lightrag.api.lightrag_server.LightragPathFilter", + "()": "lightrag.utils.LightragPathFilter", }, }, "loggers": { diff --git a/lightrag/api/lightrag_server.py b/lightrag/api/lightrag_server.py index 5d223759ba7758735f85be791e06c30d3d801675..16f4a439818a86ce500c0eb9f6b9f05d1e7395df 100644 --- a/lightrag/api/lightrag_server.py +++ b/lightrag/api/lightrag_server.py @@ -55,41 +55,6 @@ config = configparser.ConfigParser() config.read("config.ini") -class LightragPathFilter(logging.Filter): - """Filter for lightrag logger to filter out frequent path access logs""" - - def __init__(self): - super().__init__() - # Define paths to be filtered - self.filtered_paths = ["/documents", "/health", "/webui/"] - - def filter(self, record): - try: - # Check if record has the required attributes for an access log - if not hasattr(record, "args") or not isinstance(record.args, tuple): - return True - if len(record.args) < 5: - return True - - # Extract method, path and status from the record args - method = record.args[1] - path = record.args[2] - status = record.args[4] - - # Filter out successful GET requests to filtered paths - if ( - method == "GET" - and (status == 200 or status == 304) - and path in self.filtered_paths - ): - return False - - return True - except Exception: - # In case of any error, let the message through - return True - - def create_app(args): # Setup logging logger.setLevel(args.log_level) @@ -177,6 +142,9 @@ def create_app(args): if api_key else "", version=__api_version__, + openapi_url="/openapi.json", # Explicitly set OpenAPI schema URL + docs_url="/docs", # Explicitly set docs URL + redoc_url="/redoc", # Explicitly set redoc URL openapi_tags=[{"name": "api"}], lifespan=lifespan, ) @@ -423,12 +391,24 @@ def create_app(args): "update_status": update_status, } + # Custom StaticFiles class to prevent caching of HTML files + class NoCacheStaticFiles(StaticFiles): + async def get_response(self, path: str, scope): + response = await super().get_response(path, scope) + if path.endswith(".html"): + response.headers["Cache-Control"] = ( + "no-cache, no-store, must-revalidate" + ) + response.headers["Pragma"] = "no-cache" + response.headers["Expires"] = "0" + return response + # Webui mount webui/index.html static_dir = Path(__file__).parent / "webui" static_dir.mkdir(exist_ok=True) app.mount( "/webui", - StaticFiles(directory=static_dir, html=True, check_dir=True), + NoCacheStaticFiles(directory=static_dir, html=True, check_dir=True), name="webui", ) @@ -516,7 +496,7 @@ def configure_logging(): }, "filters": { "path_filter": { - "()": "lightrag.api.lightrag_server.LightragPathFilter", + "()": "lightrag.utils.LightragPathFilter", }, }, } diff --git a/lightrag/api/routers/document_routes.py b/lightrag/api/routers/document_routes.py index c166619252d992c745f168655e42b8d4403994d8..7b6f11c1e73d6a4870e9604d6159a717f757fbf3 100644 --- a/lightrag/api/routers/document_routes.py +++ b/lightrag/api/routers/document_routes.py @@ -99,6 +99,37 @@ class DocsStatusesResponse(BaseModel): statuses: Dict[DocStatus, List[DocStatusResponse]] = {} +class PipelineStatusResponse(BaseModel): + """Response model for pipeline status + + Attributes: + autoscanned: Whether auto-scan has started + busy: Whether the pipeline is currently busy + job_name: Current job name (e.g., indexing files/indexing texts) + job_start: Job start time as ISO format string (optional) + docs: Total number of documents to be indexed + batchs: Number of batches for processing documents + cur_batch: Current processing batch + request_pending: Flag for pending request for processing + latest_message: Latest message from pipeline processing + history_messages: List of history messages + """ + + autoscanned: bool = False + busy: bool = False + job_name: str = "Default Job" + job_start: Optional[str] = None + docs: int = 0 + batchs: int = 0 + cur_batch: int = 0 + request_pending: bool = False + latest_message: str = "" + history_messages: Optional[List[str]] = None + + class Config: + extra = "allow" # Allow additional fields from the pipeline status + + class DocumentManager: def __init__( self, @@ -247,7 +278,7 @@ async def pipeline_enqueue_file(rag: LightRAG, file_path: Path) -> bool: if global_args["main_args"].document_loading_engine == "DOCLING": if not pm.is_installed("docling"): # type: ignore pm.install("docling") - from docling.document_converter import DocumentConverter + from docling.document_converter import DocumentConverter # type: ignore converter = DocumentConverter() result = converter.convert(file_path) @@ -266,7 +297,7 @@ async def pipeline_enqueue_file(rag: LightRAG, file_path: Path) -> bool: if global_args["main_args"].document_loading_engine == "DOCLING": if not pm.is_installed("docling"): # type: ignore pm.install("docling") - from docling.document_converter import DocumentConverter + from docling.document_converter import DocumentConverter # type: ignore converter = DocumentConverter() result = converter.convert(file_path) @@ -286,7 +317,7 @@ async def pipeline_enqueue_file(rag: LightRAG, file_path: Path) -> bool: if global_args["main_args"].document_loading_engine == "DOCLING": if not pm.is_installed("docling"): # type: ignore pm.install("docling") - from docling.document_converter import DocumentConverter + from docling.document_converter import DocumentConverter # type: ignore converter = DocumentConverter() result = converter.convert(file_path) @@ -307,7 +338,7 @@ async def pipeline_enqueue_file(rag: LightRAG, file_path: Path) -> bool: if global_args["main_args"].document_loading_engine == "DOCLING": if not pm.is_installed("docling"): # type: ignore pm.install("docling") - from docling.document_converter import DocumentConverter + from docling.document_converter import DocumentConverter # type: ignore converter = DocumentConverter() result = converter.convert(file_path) @@ -718,17 +749,33 @@ def create_document_routes( logger.error(traceback.format_exc()) raise HTTPException(status_code=500, detail=str(e)) - @router.get("/pipeline_status", dependencies=[Depends(optional_api_key)]) - async def get_pipeline_status(): + @router.get( + "/pipeline_status", + dependencies=[Depends(optional_api_key)], + response_model=PipelineStatusResponse, + ) + async def get_pipeline_status() -> PipelineStatusResponse: """ Get the current status of the document indexing pipeline. This endpoint returns information about the current state of the document processing pipeline, - including whether it's busy, the current job name, when it started, how many documents - are being processed, how many batches there are, and which batch is currently being processed. + including the processing status, progress information, and history messages. Returns: - dict: A dictionary containing the pipeline status information + PipelineStatusResponse: A response object containing: + - autoscanned (bool): Whether auto-scan has started + - busy (bool): Whether the pipeline is currently busy + - job_name (str): Current job name (e.g., indexing files/indexing texts) + - job_start (str, optional): Job start time as ISO format string + - docs (int): Total number of documents to be indexed + - batchs (int): Number of batches for processing documents + - cur_batch (int): Current processing batch + - request_pending (bool): Flag for pending request for processing + - latest_message (str): Latest message from pipeline processing + - history_messages (List[str], optional): List of history messages + + Raises: + HTTPException: If an error occurs while retrieving pipeline status (500) """ try: from lightrag.kg.shared_storage import get_namespace_data @@ -746,7 +793,7 @@ def create_document_routes( if status_dict.get("job_start"): status_dict["job_start"] = str(status_dict["job_start"]) - return status_dict + return PipelineStatusResponse(**status_dict) except Exception as e: logger.error(f"Error getting pipeline status: {str(e)}") logger.error(traceback.format_exc()) diff --git a/lightrag/api/webui/index.html b/lightrag/api/webui/index.html new file mode 100644 index 0000000000000000000000000000000000000000..49fc0ea6ebcac282b3edc530da229efe63be7ad1 Binary files /dev/null and b/lightrag/api/webui/index.html differ diff --git a/lightrag/kg/chroma_impl.py b/lightrag/kg/chroma_impl.py index f668c87ab4dd368ea301b0ff1b56ad2567219643..84d43326d86356e957d347ac5b6889251275061b 100644 --- a/lightrag/kg/chroma_impl.py +++ b/lightrag/kg/chroma_impl.py @@ -156,7 +156,9 @@ class ChromaVectorDBStorage(BaseVectorStorage): logger.error(f"Error during ChromaDB upsert: {str(e)}") raise - async def query(self, query: str, top_k: int) -> list[dict[str, Any]]: + async def query( + self, query: str, top_k: int, ids: list[str] | None = None + ) -> list[dict[str, Any]]: try: embedding = await self.embedding_func([query]) diff --git a/lightrag/kg/faiss_impl.py b/lightrag/kg/faiss_impl.py index a5716e9c16ca2297739d81968dca112b11ee7ef3..57b0cae057d17bc0acab0bfd53648d6e8b26e445 100644 --- a/lightrag/kg/faiss_impl.py +++ b/lightrag/kg/faiss_impl.py @@ -171,7 +171,9 @@ class FaissVectorDBStorage(BaseVectorStorage): logger.info(f"Upserted {len(list_data)} vectors into Faiss index.") return [m["__id__"] for m in list_data] - async def query(self, query: str, top_k: int) -> list[dict[str, Any]]: + async def query( + self, query: str, top_k: int, ids: list[str] | None = None + ) -> list[dict[str, Any]]: """ Search by a textual query; returns top_k results with their metadata + similarity distance. """ diff --git a/lightrag/kg/milvus_impl.py b/lightrag/kg/milvus_impl.py index 4fb5f0123c6a9ce017b5a13a952661195ae71377..4b4577caf2135d0b96dd15b06119299cf017ccd6 100644 --- a/lightrag/kg/milvus_impl.py +++ b/lightrag/kg/milvus_impl.py @@ -101,7 +101,9 @@ class MilvusVectorDBStorage(BaseVectorStorage): results = self._client.upsert(collection_name=self.namespace, data=list_data) return results - async def query(self, query: str, top_k: int) -> list[dict[str, Any]]: + async def query( + self, query: str, top_k: int, ids: list[str] | None = None + ) -> list[dict[str, Any]]: embedding = await self.embedding_func([query]) results = self._client.search( collection_name=self.namespace, diff --git a/lightrag/kg/mongo_impl.py b/lightrag/kg/mongo_impl.py index da4dc32c3763a0d38a301558758e2b7350895504..7d43e4f4f4586ae8585357fdc7cc7cfd5f8ae510 100644 --- a/lightrag/kg/mongo_impl.py +++ b/lightrag/kg/mongo_impl.py @@ -938,7 +938,9 @@ class MongoVectorDBStorage(BaseVectorStorage): return list_data - async def query(self, query: str, top_k: int) -> list[dict[str, Any]]: + async def query( + self, query: str, top_k: int, ids: list[str] | None = None + ) -> list[dict[str, Any]]: """Queries the vector database using Atlas Vector Search.""" # Generate the embedding embedding = await self.embedding_func([query]) diff --git a/lightrag/kg/nano_vector_db_impl.py b/lightrag/kg/nano_vector_db_impl.py index ac010f164c0642db4500e36ab207dbd0d30c455b..4f739091b48da9817035bf6ad73feeef71e63a79 100644 --- a/lightrag/kg/nano_vector_db_impl.py +++ b/lightrag/kg/nano_vector_db_impl.py @@ -120,7 +120,9 @@ class NanoVectorDBStorage(BaseVectorStorage): f"embedding is not 1-1 with data, {len(embeddings)} != {len(list_data)}" ) - async def query(self, query: str, top_k: int) -> list[dict[str, Any]]: + async def query( + self, query: str, top_k: int, ids: list[str] | None = None + ) -> list[dict[str, Any]]: # Execute embedding outside of lock to avoid long lock times embedding = await self.embedding_func([query]) embedding = embedding[0] diff --git a/lightrag/kg/neo4j_impl.py b/lightrag/kg/neo4j_impl.py index d0c6c77976b20ae86ad5e34684423d4c81b7b91b..2df420dffcb6e344b03c6b820effeeb89662412c 100644 --- a/lightrag/kg/neo4j_impl.py +++ b/lightrag/kg/neo4j_impl.py @@ -553,18 +553,6 @@ class Neo4JStorage(BaseGraphStorage): logger.error(f"Error during upsert: {str(e)}") raise - @retry( - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=4, max=10), - retry=retry_if_exception_type( - ( - neo4jExceptions.ServiceUnavailable, - neo4jExceptions.TransientError, - neo4jExceptions.WriteServiceUnavailable, - neo4jExceptions.ClientError, - ) - ), - ) @retry( stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10), @@ -666,14 +654,14 @@ class Neo4JStorage(BaseGraphStorage): main_query = """ MATCH (n) OPTIONAL MATCH (n)-[r]-() - WITH n, count(r) AS degree + WITH n, COALESCE(count(r), 0) AS degree WHERE degree >= $min_degree ORDER BY degree DESC LIMIT $max_nodes WITH collect({node: n}) AS filtered_nodes UNWIND filtered_nodes AS node_info WITH collect(node_info.node) AS kept_nodes, filtered_nodes - MATCH (a)-[r]-(b) + OPTIONAL MATCH (a)-[r]-(b) WHERE a IN kept_nodes AND b IN kept_nodes RETURN filtered_nodes AS node_info, collect(DISTINCT r) AS relationships @@ -703,7 +691,7 @@ class Neo4JStorage(BaseGraphStorage): WITH start, nodes, relationships UNWIND nodes AS node OPTIONAL MATCH (node)-[r]-() - WITH node, count(r) AS degree, start, nodes, relationships + WITH node, COALESCE(count(r), 0) AS degree, start, nodes, relationships WHERE node = start OR EXISTS((start)--(node)) OR degree >= $min_degree ORDER BY CASE @@ -716,7 +704,7 @@ class Neo4JStorage(BaseGraphStorage): WITH collect({node: node}) AS filtered_nodes UNWIND filtered_nodes AS node_info WITH collect(node_info.node) AS kept_nodes, filtered_nodes - MATCH (a)-[r]-(b) + OPTIONAL MATCH (a)-[r]-(b) WHERE a IN kept_nodes AND b IN kept_nodes RETURN filtered_nodes AS node_info, collect(DISTINCT r) AS relationships @@ -744,11 +732,7 @@ class Neo4JStorage(BaseGraphStorage): result.nodes.append( KnowledgeGraphNode( id=f"{node_id}", - labels=[ - label - for label in node.labels - if label != "base" - ], + labels=[node.get("entity_id")], properties=dict(node), ) ) @@ -865,9 +849,7 @@ class Neo4JStorage(BaseGraphStorage): # Create KnowledgeGraphNode for target target_node = KnowledgeGraphNode( id=f"{target_id}", - labels=[ - label for label in b_node.labels if label != "base" - ], + labels=list(f"{target_id}"), properties=dict(b_node.properties), ) @@ -907,9 +889,7 @@ class Neo4JStorage(BaseGraphStorage): # Create initial KnowledgeGraphNode start_node = KnowledgeGraphNode( id=f"{node_record['n'].get('entity_id')}", - labels=[ - label for label in node_record["n"].labels if label != "base" - ], + labels=list(f"{node_record['n'].get('entity_id')}"), properties=dict(node_record["n"].properties), ) finally: diff --git a/lightrag/kg/oracle_impl.py b/lightrag/kg/oracle_impl.py index 32790f4f757abe2b0247307b60218f0abd5d9310..c42f0f767bf58797455914cf7884cab6cc6c3407 100644 --- a/lightrag/kg/oracle_impl.py +++ b/lightrag/kg/oracle_impl.py @@ -417,7 +417,9 @@ class OracleVectorDBStorage(BaseVectorStorage): self.db = None #################### query method ############### - async def query(self, query: str, top_k: int) -> list[dict[str, Any]]: + async def query( + self, query: str, top_k: int, ids: list[str] | None = None + ) -> list[dict[str, Any]]: embeddings = await self.embedding_func([query]) embedding = embeddings[0] # 转换精度 diff --git a/lightrag/kg/qdrant_impl.py b/lightrag/kg/qdrant_impl.py index 53a59c2f69d86fa40cba47261f7ded2d213c4003..e32c43351d433d47dd9704d80b70d898a79ddee7 100644 --- a/lightrag/kg/qdrant_impl.py +++ b/lightrag/kg/qdrant_impl.py @@ -123,7 +123,9 @@ class QdrantVectorDBStorage(BaseVectorStorage): ) return results - async def query(self, query: str, top_k: int) -> list[dict[str, Any]]: + async def query( + self, query: str, top_k: int, ids: list[str] | None = None + ) -> list[dict[str, Any]]: embedding = await self.embedding_func([query]) results = self._client.search( collection_name=self.namespace, diff --git a/lightrag/kg/tidb_impl.py b/lightrag/kg/tidb_impl.py index c4485df6a201028115e9fec54dae8133534cbfcb..0982c9140e5a968e3038da3c257313b2e5a1e51b 100644 --- a/lightrag/kg/tidb_impl.py +++ b/lightrag/kg/tidb_impl.py @@ -306,7 +306,9 @@ class TiDBVectorDBStorage(BaseVectorStorage): await ClientManager.release_client(self.db) self.db = None - async def query(self, query: str, top_k: int) -> list[dict[str, Any]]: + async def query( + self, query: str, top_k: int, ids: list[str] | None = None + ) -> list[dict[str, Any]]: """Search from tidb vector""" embeddings = await self.embedding_func([query]) embedding = embeddings[0] diff --git a/lightrag/lightrag.py b/lightrag/lightrag.py index 3a5e4e84f89f9b5ca773137daab36a3564dd1b37..a466e572e408d4941a59d2ae53b0960a9f470aa5 100644 --- a/lightrag/lightrag.py +++ b/lightrag/lightrag.py @@ -3,11 +3,14 @@ from __future__ import annotations import asyncio import configparser import os +import csv import warnings from dataclasses import asdict, dataclass, field from datetime import datetime from functools import partial -from typing import Any, AsyncIterator, Callable, Iterator, cast, final +from typing import Any, AsyncIterator, Callable, Iterator, cast, final, Literal +import pandas as pd + from lightrag.kg import ( STORAGE_ENV_REQUIREMENTS, @@ -1111,6 +1114,7 @@ class LightRAG: # Prepare node data node_data: dict[str, str] = { + "entity_id": entity_name, "entity_type": entity_type, "description": description, "source_id": source_id, @@ -1148,6 +1152,7 @@ class LightRAG: await self.chunk_entity_relation_graph.upsert_node( need_insert_id, node_data={ + "entity_id": need_insert_id, "source_id": source_id, "description": "UNKNOWN", "entity_type": "UNKNOWN", @@ -2157,6 +2162,7 @@ class LightRAG: # Prepare node data with defaults if missing node_data = { + "entity_id": entity_name, "entity_type": entity_data.get("entity_type", "UNKNOWN"), "description": entity_data.get("description", ""), "source_id": entity_data.get("source_id", "manual"), @@ -2592,6 +2598,322 @@ class LightRAG: logger.error(f"Error merging entities: {e}") raise + async def aexport_data( + self, + output_path: str, + file_format: Literal["csv", "excel", "md", "txt"] = "csv", + include_vector_data: bool = False, + ) -> None: + """ + Asynchronously exports all entities, relations, and relationships to various formats. + Args: + output_path: The path to the output file (including extension). + file_format: Output format - "csv", "excel", "md", "txt". + - csv: Comma-separated values file + - excel: Microsoft Excel file with multiple sheets + - md: Markdown tables + - txt: Plain text formatted output + - table: Print formatted tables to console + include_vector_data: Whether to include data from the vector database. + """ + # Collect data + entities_data = [] + relations_data = [] + relationships_data = [] + + # --- Entities --- + all_entities = await self.chunk_entity_relation_graph.get_all_labels() + for entity_name in all_entities: + entity_info = await self.get_entity_info( + entity_name, include_vector_data=include_vector_data + ) + entity_row = { + "entity_name": entity_name, + "source_id": entity_info["source_id"], + "graph_data": str( + entity_info["graph_data"] + ), # Convert to string to ensure compatibility + } + if include_vector_data and "vector_data" in entity_info: + entity_row["vector_data"] = str(entity_info["vector_data"]) + entities_data.append(entity_row) + + # --- Relations --- + for src_entity in all_entities: + for tgt_entity in all_entities: + if src_entity == tgt_entity: + continue + + edge_exists = await self.chunk_entity_relation_graph.has_edge( + src_entity, tgt_entity + ) + if edge_exists: + relation_info = await self.get_relation_info( + src_entity, tgt_entity, include_vector_data=include_vector_data + ) + relation_row = { + "src_entity": src_entity, + "tgt_entity": tgt_entity, + "source_id": relation_info["source_id"], + "graph_data": str( + relation_info["graph_data"] + ), # Convert to string + } + if include_vector_data and "vector_data" in relation_info: + relation_row["vector_data"] = str(relation_info["vector_data"]) + relations_data.append(relation_row) + + # --- Relationships (from VectorDB) --- + all_relationships = await self.relationships_vdb.client_storage + for rel in all_relationships["data"]: + relationships_data.append( + { + "relationship_id": rel["__id__"], + "data": str(rel), # Convert to string for compatibility + } + ) + + # Export based on format + if file_format == "csv": + # CSV export + with open(output_path, "w", newline="", encoding="utf-8") as csvfile: + # Entities + if entities_data: + csvfile.write("# ENTITIES\n") + writer = csv.DictWriter(csvfile, fieldnames=entities_data[0].keys()) + writer.writeheader() + writer.writerows(entities_data) + csvfile.write("\n\n") + + # Relations + if relations_data: + csvfile.write("# RELATIONS\n") + writer = csv.DictWriter( + csvfile, fieldnames=relations_data[0].keys() + ) + writer.writeheader() + writer.writerows(relations_data) + csvfile.write("\n\n") + + # Relationships + if relationships_data: + csvfile.write("# RELATIONSHIPS\n") + writer = csv.DictWriter( + csvfile, fieldnames=relationships_data[0].keys() + ) + writer.writeheader() + writer.writerows(relationships_data) + + elif file_format == "excel": + # Excel export + entities_df = ( + pd.DataFrame(entities_data) if entities_data else pd.DataFrame() + ) + relations_df = ( + pd.DataFrame(relations_data) if relations_data else pd.DataFrame() + ) + relationships_df = ( + pd.DataFrame(relationships_data) + if relationships_data + else pd.DataFrame() + ) + + with pd.ExcelWriter(output_path, engine="xlsxwriter") as writer: + if not entities_df.empty: + entities_df.to_excel(writer, sheet_name="Entities", index=False) + if not relations_df.empty: + relations_df.to_excel(writer, sheet_name="Relations", index=False) + if not relationships_df.empty: + relationships_df.to_excel( + writer, sheet_name="Relationships", index=False + ) + + elif file_format == "md": + # Markdown export + with open(output_path, "w", encoding="utf-8") as mdfile: + mdfile.write("# LightRAG Data Export\n\n") + + # Entities + mdfile.write("## Entities\n\n") + if entities_data: + # Write header + mdfile.write("| " + " | ".join(entities_data[0].keys()) + " |\n") + mdfile.write( + "| " + + " | ".join(["---"] * len(entities_data[0].keys())) + + " |\n" + ) + + # Write rows + for entity in entities_data: + mdfile.write( + "| " + " | ".join(str(v) for v in entity.values()) + " |\n" + ) + mdfile.write("\n\n") + else: + mdfile.write("*No entity data available*\n\n") + + # Relations + mdfile.write("## Relations\n\n") + if relations_data: + # Write header + mdfile.write("| " + " | ".join(relations_data[0].keys()) + " |\n") + mdfile.write( + "| " + + " | ".join(["---"] * len(relations_data[0].keys())) + + " |\n" + ) + + # Write rows + for relation in relations_data: + mdfile.write( + "| " + + " | ".join(str(v) for v in relation.values()) + + " |\n" + ) + mdfile.write("\n\n") + else: + mdfile.write("*No relation data available*\n\n") + + # Relationships + mdfile.write("## Relationships\n\n") + if relationships_data: + # Write header + mdfile.write( + "| " + " | ".join(relationships_data[0].keys()) + " |\n" + ) + mdfile.write( + "| " + + " | ".join(["---"] * len(relationships_data[0].keys())) + + " |\n" + ) + + # Write rows + for relationship in relationships_data: + mdfile.write( + "| " + + " | ".join(str(v) for v in relationship.values()) + + " |\n" + ) + else: + mdfile.write("*No relationship data available*\n\n") + + elif file_format == "txt": + # Plain text export + with open(output_path, "w", encoding="utf-8") as txtfile: + txtfile.write("LIGHTRAG DATA EXPORT\n") + txtfile.write("=" * 80 + "\n\n") + + # Entities + txtfile.write("ENTITIES\n") + txtfile.write("-" * 80 + "\n") + if entities_data: + # Create fixed width columns + col_widths = { + k: max(len(k), max(len(str(e[k])) for e in entities_data)) + for k in entities_data[0] + } + header = " ".join(k.ljust(col_widths[k]) for k in entities_data[0]) + txtfile.write(header + "\n") + txtfile.write("-" * len(header) + "\n") + + # Write rows + for entity in entities_data: + row = " ".join( + str(v).ljust(col_widths[k]) for k, v in entity.items() + ) + txtfile.write(row + "\n") + txtfile.write("\n\n") + else: + txtfile.write("No entity data available\n\n") + + # Relations + txtfile.write("RELATIONS\n") + txtfile.write("-" * 80 + "\n") + if relations_data: + # Create fixed width columns + col_widths = { + k: max(len(k), max(len(str(r[k])) for r in relations_data)) + for k in relations_data[0] + } + header = " ".join( + k.ljust(col_widths[k]) for k in relations_data[0] + ) + txtfile.write(header + "\n") + txtfile.write("-" * len(header) + "\n") + + # Write rows + for relation in relations_data: + row = " ".join( + str(v).ljust(col_widths[k]) for k, v in relation.items() + ) + txtfile.write(row + "\n") + txtfile.write("\n\n") + else: + txtfile.write("No relation data available\n\n") + + # Relationships + txtfile.write("RELATIONSHIPS\n") + txtfile.write("-" * 80 + "\n") + if relationships_data: + # Create fixed width columns + col_widths = { + k: max(len(k), max(len(str(r[k])) for r in relationships_data)) + for k in relationships_data[0] + } + header = " ".join( + k.ljust(col_widths[k]) for k in relationships_data[0] + ) + txtfile.write(header + "\n") + txtfile.write("-" * len(header) + "\n") + + # Write rows + for relationship in relationships_data: + row = " ".join( + str(v).ljust(col_widths[k]) for k, v in relationship.items() + ) + txtfile.write(row + "\n") + else: + txtfile.write("No relationship data available\n\n") + + else: + raise ValueError( + f"Unsupported file format: {file_format}. " + f"Choose from: csv, excel, md, txt" + ) + if file_format is not None: + print(f"Data exported to: {output_path} with format: {file_format}") + else: + print("Data displayed as table format") + + def export_data( + self, + output_path: str, + file_format: Literal["csv", "excel", "md", "txt"] = "csv", + include_vector_data: bool = False, + ) -> None: + """ + Synchronously exports all entities, relations, and relationships to various formats. + Args: + output_path: The path to the output file (including extension). + file_format: Output format - "csv", "excel", "md", "txt". + - csv: Comma-separated values file + - excel: Microsoft Excel file with multiple sheets + - md: Markdown tables + - txt: Plain text formatted output + - table: Print formatted tables to console + include_vector_data: Whether to include data from the vector database. + """ + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + loop.run_until_complete( + self.aexport_data(output_path, file_format, include_vector_data) + ) + def merge_entities( self, source_entities: list[str], diff --git a/lightrag/utils.py b/lightrag/utils.py index b8f00c5d75517a97157e6a1dcdfdab5beb025571..362e553116f6f1005125856c68fa873918e39df3 100644 --- a/lightrag/utils.py +++ b/lightrag/utils.py @@ -76,6 +76,7 @@ class LightragPathFilter(logging.Filter): super().__init__() # Define paths to be filtered self.filtered_paths = ["/documents", "/health", "/webui/"] + # self.filtered_paths = ["/health", "/webui/"] def filter(self, record): try: diff --git a/lightrag_webui/bun.lock b/lightrag_webui/bun.lock index a0fe0b89524ba5263c1839900eef28e762fad188..0e85a228911c7761d94c99e65dcd301e4ed15f7e 100644 --- a/lightrag_webui/bun.lock +++ b/lightrag_webui/bun.lock @@ -63,6 +63,7 @@ "@types/node": "^22.13.5", "@types/react": "^19.0.10", "@types/react-dom": "^19.0.4", + "@types/react-i18next": "^8.1.0", "@types/react-syntax-highlighter": "^15.5.13", "@types/seedrandom": "^3.0.8", "@vitejs/plugin-react-swc": "^3.8.0", @@ -446,6 +447,8 @@ "@types/react-dom": ["@types/react-dom@19.0.4", "", { "peerDependencies": { "@types/react": "^19.0.0" } }, "sha512-4fSQ8vWFkg+TGhePfUzVmat3eC14TXYSsiiDSLI0dVLsrm9gZFABjPy/Qu6TKgl1tq1Bu1yDsuQgY3A3DOjCcg=="], + "@types/react-i18next": ["@types/react-i18next@8.1.0", "", { "dependencies": { "react-i18next": "*" } }, "sha512-d4xhcjX5b3roNMObRNMfb1HinHQlQLPo8xlDj60dnHeeAw2bBymR2cy/l1giJpHzo/ZFgSvgVUvIWr4kCrenCg=="], + "@types/react-syntax-highlighter": ["@types/react-syntax-highlighter@15.5.13", "", { "dependencies": { "@types/react": "*" } }, "sha512-uLGJ87j6Sz8UaBAooU0T6lWJ0dBmjZgN1PZTrj05TNql2/XpC6+4HhMT5syIdFUUt+FASfCeLLv4kBygNU+8qA=="], "@types/react-transition-group": ["@types/react-transition-group@4.4.12", "", { "peerDependencies": { "@types/react": "*" } }, "sha512-8TV6R3h2j7a91c+1DXdJi3Syo69zzIZbz7Lg5tORM5LEJG7X/E6a1V3drRyBRZq7/utz7A+c4OgYLiLcYGHG6w=="], diff --git a/lightrag_webui/index.html b/lightrag_webui/index.html index 32d18acd7c4d81c54c396956cd788081b5f4c1fb..3dd1ebbc2cc79e324b645bcbee8787d22c0383ec 100644 --- a/lightrag_webui/index.html +++ b/lightrag_webui/index.html @@ -2,6 +2,9 @@ + + + Lightrag diff --git a/lightrag_webui/package.json b/lightrag_webui/package.json index fff2d9d85c1c0ac96b3c6d7870f6e58432ee34bb..1c87f77cd78f6b04e06f9d100dd332c7c40b7d22 100644 --- a/lightrag_webui/package.json +++ b/lightrag_webui/package.json @@ -72,6 +72,7 @@ "@types/node": "^22.13.5", "@types/react": "^19.0.10", "@types/react-dom": "^19.0.4", + "@types/react-i18next": "^8.1.0", "@types/react-syntax-highlighter": "^15.5.13", "@types/seedrandom": "^3.0.8", "@vitejs/plugin-react-swc": "^3.8.0", diff --git a/lightrag_webui/src/App.tsx b/lightrag_webui/src/App.tsx index 80dd57a57cabbeea9bfa0c1283cb15e29f4e723e..67e3bb9c3238a32004ee24e355d7cf06a4253b7f 100644 --- a/lightrag_webui/src/App.tsx +++ b/lightrag_webui/src/App.tsx @@ -1,4 +1,6 @@ import { useState, useCallback } from 'react' +import ThemeProvider from '@/components/ThemeProvider' +import TabVisibilityProvider from '@/contexts/TabVisibilityProvider' import MessageAlert from '@/components/MessageAlert' import ApiKeyAlert from '@/components/ApiKeyAlert' import StatusIndicator from '@/components/graph/StatusIndicator' @@ -19,7 +21,7 @@ import { Tabs, TabsContent } from '@/components/ui/Tabs' function App() { const message = useBackendState.use.message() const enableHealthCheck = useSettingsStore.use.enableHealthCheck() - const [currentTab] = useState(() => useSettingsStore.getState().currentTab) + const currentTab = useSettingsStore.use.currentTab() const [apiKeyInvalid, setApiKeyInvalid] = useState(false) // Health check @@ -51,32 +53,36 @@ function App() { }, [message, setApiKeyInvalid]) return ( -
- - -
- - - - - - - - - - - - -
- - {enableHealthCheck && } - {message !== null && !apiKeyInvalid && } - {apiKeyInvalid && } -
+ + +
+ + +
+ + + + + + + + + + + + +
+ + {enableHealthCheck && } + {message !== null && !apiKeyInvalid && } + {apiKeyInvalid && } +
+ + ) } diff --git a/lightrag_webui/src/components/AppSettings.tsx b/lightrag_webui/src/components/AppSettings.tsx new file mode 100644 index 0000000000000000000000000000000000000000..284ad67f33b1cb8eee308533653e966c484194cd --- /dev/null +++ b/lightrag_webui/src/components/AppSettings.tsx @@ -0,0 +1,66 @@ +import { useState, useCallback } from 'react' +import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/Popover' +import Button from '@/components/ui/Button' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/Select' +import { useSettingsStore } from '@/stores/settings' +import { PaletteIcon } from 'lucide-react' +import { useTranslation } from 'react-i18next' + +export default function AppSettings() { + const [opened, setOpened] = useState(false) + const { t } = useTranslation() + + const language = useSettingsStore.use.language() + const setLanguage = useSettingsStore.use.setLanguage() + + const theme = useSettingsStore.use.theme() + const setTheme = useSettingsStore.use.setTheme() + + const handleLanguageChange = useCallback((value: string) => { + setLanguage(value as 'en' | 'zh') + }, [setLanguage]) + + const handleThemeChange = useCallback((value: string) => { + setTheme(value as 'light' | 'dark' | 'system') + }, [setTheme]) + + return ( + + + + + +
+
+ + +
+ +
+ + +
+
+ + + ) +} diff --git a/lightrag_webui/src/components/Root.tsx b/lightrag_webui/src/components/Root.tsx new file mode 100644 index 0000000000000000000000000000000000000000..6191ace8e9c4c70e20fe63f1b13fca3cba29b9fb --- /dev/null +++ b/lightrag_webui/src/components/Root.tsx @@ -0,0 +1,24 @@ +import { StrictMode, useEffect, useState } from 'react' +import { initializeI18n } from '@/i18n' +import App from '@/App' + +export const Root = () => { + const [isI18nInitialized, setIsI18nInitialized] = useState(false) + + useEffect(() => { + // Initialize i18n immediately with persisted language + initializeI18n().then(() => { + setIsI18nInitialized(true) + }) + }, []) + + if (!isI18nInitialized) { + return null // or a loading spinner + } + + return ( + + + + ) +} diff --git a/lightrag_webui/src/components/ThemeProvider.tsx b/lightrag_webui/src/components/ThemeProvider.tsx index 873b92a4b36d93c614552c116684007bec48e3cb..df5816c6eb79ba8223fc3a8f55c8c7efbf18c845 100644 --- a/lightrag_webui/src/components/ThemeProvider.tsx +++ b/lightrag_webui/src/components/ThemeProvider.tsx @@ -1,4 +1,4 @@ -import { createContext, useEffect, useState } from 'react' +import { createContext, useEffect } from 'react' import { Theme, useSettingsStore } from '@/stores/settings' type ThemeProviderProps = { @@ -21,30 +21,32 @@ const ThemeProviderContext = createContext(initialState) * Component that provides the theme state and setter function to its children. */ export default function ThemeProvider({ children, ...props }: ThemeProviderProps) { - const [theme, setTheme] = useState(useSettingsStore.getState().theme) + const theme = useSettingsStore.use.theme() + const setTheme = useSettingsStore.use.setTheme() useEffect(() => { const root = window.document.documentElement root.classList.remove('light', 'dark') if (theme === 'system') { - const systemTheme = window.matchMedia('(prefers-color-scheme: dark)').matches - ? 'dark' - : 'light' - root.classList.add(systemTheme) - setTheme(systemTheme) - return + const mediaQuery = window.matchMedia('(prefers-color-scheme: dark)') + const handleChange = (e: MediaQueryListEvent) => { + root.classList.remove('light', 'dark') + root.classList.add(e.matches ? 'dark' : 'light') + } + + root.classList.add(mediaQuery.matches ? 'dark' : 'light') + mediaQuery.addEventListener('change', handleChange) + + return () => mediaQuery.removeEventListener('change', handleChange) + } else { + root.classList.add(theme) } - - root.classList.add(theme) }, [theme]) const value = { theme, - setTheme: (theme: Theme) => { - useSettingsStore.getState().setTheme(theme) - setTheme(theme) - } + setTheme } return ( diff --git a/lightrag_webui/src/components/graph/FocusOnNode.tsx b/lightrag_webui/src/components/graph/FocusOnNode.tsx index cfefb7bb4616eef2205a8f8e6028ce686125f544..70af75251b3cdf8b2698140110d0b9d083199c2d 100644 --- a/lightrag_webui/src/components/graph/FocusOnNode.tsx +++ b/lightrag_webui/src/components/graph/FocusOnNode.tsx @@ -13,15 +13,24 @@ const FocusOnNode = ({ node, move }: { node: string | null; move?: boolean }) => * When the selected item changes, highlighted the node and center the camera on it. */ useEffect(() => { - if (!node) return - sigma.getGraph().setNodeAttribute(node, 'highlighted', true) if (move) { - gotoNode(node) + if (node) { + sigma.getGraph().setNodeAttribute(node, 'highlighted', true) + gotoNode(node) + } else { + // If no node is selected but move is true, reset to default view + sigma.setCustomBBox(null) + sigma.getCamera().animate({ x: 0.5, y: 0.5, ratio: 1 }, { duration: 0 }) + } useGraphStore.getState().setMoveToSelectedNode(false) + } else if (node) { + sigma.getGraph().setNodeAttribute(node, 'highlighted', true) } return () => { - sigma.getGraph().setNodeAttribute(node, 'highlighted', false) + if (node) { + sigma.getGraph().setNodeAttribute(node, 'highlighted', false) + } } }, [node, move, sigma, gotoNode]) diff --git a/lightrag_webui/src/components/graph/GraphControl.tsx b/lightrag_webui/src/components/graph/GraphControl.tsx index 3200fe5e4552be5e1532f2b2a5346dfe5fcd89a2..7d0143162b0de9ee5dd27084aa11bb7d04caf4d2 100644 --- a/lightrag_webui/src/components/graph/GraphControl.tsx +++ b/lightrag_webui/src/components/graph/GraphControl.tsx @@ -1,10 +1,11 @@ import { useLoadGraph, useRegisterEvents, useSetSettings, useSigma } from '@react-sigma/core' +import Graph from 'graphology' // import { useLayoutCircular } from '@react-sigma/layout-circular' import { useLayoutForceAtlas2 } from '@react-sigma/layout-forceatlas2' import { useEffect } from 'react' // import useRandomGraph, { EdgeType, NodeType } from '@/hooks/useRandomGraph' -import useLightragGraph, { EdgeType, NodeType } from '@/hooks/useLightragGraph' +import { EdgeType, NodeType } from '@/hooks/useLightragGraph' import useTheme from '@/hooks/useTheme' import * as Constants from '@/lib/constants' @@ -21,7 +22,6 @@ const isButtonPressed = (ev: MouseEvent | TouchEvent) => { } const GraphControl = ({ disableHoverEffect }: { disableHoverEffect?: boolean }) => { - const { lightrageGraph } = useLightragGraph() const sigma = useSigma() const registerEvents = useRegisterEvents() const setSettings = useSetSettings() @@ -34,21 +34,25 @@ const GraphControl = ({ disableHoverEffect }: { disableHoverEffect?: boolean }) const { theme } = useTheme() const hideUnselectedEdges = useSettingsStore.use.enableHideUnselectedEdges() + const enableEdgeEvents = useSettingsStore.use.enableEdgeEvents() + const renderEdgeLabels = useSettingsStore.use.showEdgeLabel() + const renderLabels = useSettingsStore.use.showNodeLabel() const selectedNode = useGraphStore.use.selectedNode() const focusedNode = useGraphStore.use.focusedNode() const selectedEdge = useGraphStore.use.selectedEdge() const focusedEdge = useGraphStore.use.focusedEdge() + const sigmaGraph = useGraphStore.use.sigmaGraph() /** * When component mount or maxIterations changes * => load the graph and apply layout */ useEffect(() => { - // Create & load the graph - const graph = lightrageGraph() - loadGraph(graph) - assignLayout() - }, [assignLayout, loadGraph, lightrageGraph, maxIterations]) + if (sigmaGraph) { + loadGraph(sigmaGraph as unknown as Graph) + assignLayout() + } + }, [assignLayout, loadGraph, sigmaGraph, maxIterations]) /** * When component mount @@ -58,39 +62,52 @@ const GraphControl = ({ disableHoverEffect }: { disableHoverEffect?: boolean }) const { setFocusedNode, setSelectedNode, setFocusedEdge, setSelectedEdge, clearSelection } = useGraphStore.getState() - // Register the events - registerEvents({ - enterNode: (event) => { + // Define event types + type NodeEvent = { node: string; event: { original: MouseEvent | TouchEvent } } + type EdgeEvent = { edge: string; event: { original: MouseEvent | TouchEvent } } + + // Register all events, but edge events will only be processed if enableEdgeEvents is true + const events: Record = { + enterNode: (event: NodeEvent) => { if (!isButtonPressed(event.event.original)) { setFocusedNode(event.node) } }, - leaveNode: (event) => { + leaveNode: (event: NodeEvent) => { if (!isButtonPressed(event.event.original)) { setFocusedNode(null) } }, - clickNode: (event) => { + clickNode: (event: NodeEvent) => { setSelectedNode(event.node) setSelectedEdge(null) }, - clickEdge: (event) => { + clickStage: () => clearSelection() + } + + // Only add edge event handlers if enableEdgeEvents is true + if (enableEdgeEvents) { + events.clickEdge = (event: EdgeEvent) => { setSelectedEdge(event.edge) setSelectedNode(null) - }, - enterEdge: (event) => { + } + + events.enterEdge = (event: EdgeEvent) => { if (!isButtonPressed(event.event.original)) { setFocusedEdge(event.edge) } - }, - leaveEdge: (event) => { + } + + events.leaveEdge = (event: EdgeEvent) => { if (!isButtonPressed(event.event.original)) { setFocusedEdge(null) } - }, - clickStage: () => clearSelection() - }) - }, [registerEvents]) + } + } + + // Register the events + registerEvents(events) + }, [registerEvents, enableEdgeEvents]) /** * When component mount or hovered node change @@ -101,7 +118,14 @@ const GraphControl = ({ disableHoverEffect }: { disableHoverEffect?: boolean }) const labelColor = isDarkTheme ? Constants.labelColorDarkTheme : undefined const edgeColor = isDarkTheme ? Constants.edgeColorDarkTheme : undefined + // Update all dynamic settings directly without recreating the sigma container setSettings({ + // Update display settings + enableEdgeEvents, + renderEdgeLabels, + renderLabels, + + // Node reducer for node appearance nodeReducer: (node, data) => { const graph = sigma.getGraph() const newData: NodeType & { @@ -140,6 +164,8 @@ const GraphControl = ({ disableHoverEffect }: { disableHoverEffect?: boolean }) } return newData }, + + // Edge reducer for edge appearance edgeReducer: (edge, data) => { const graph = sigma.getGraph() const newData = { ...data, hidden: false, labelColor, color: edgeColor } @@ -181,7 +207,10 @@ const GraphControl = ({ disableHoverEffect }: { disableHoverEffect?: boolean }) sigma, disableHoverEffect, theme, - hideUnselectedEdges + hideUnselectedEdges, + enableEdgeEvents, + renderEdgeLabels, + renderLabels ]) return null diff --git a/lightrag_webui/src/components/graph/GraphLabels.tsx b/lightrag_webui/src/components/graph/GraphLabels.tsx index 7bc26c8826cc17f027966cdebd7ded9c2c41e6f1..bd2c8ea0762353105499aead49762a0a36566ef5 100644 --- a/lightrag_webui/src/components/graph/GraphLabels.tsx +++ b/lightrag_webui/src/components/graph/GraphLabels.tsx @@ -1,37 +1,48 @@ -import { useCallback } from 'react' +import { useCallback, useEffect, useRef } from 'react' import { AsyncSelect } from '@/components/ui/AsyncSelect' -import { getGraphLabels } from '@/api/lightrag' import { useSettingsStore } from '@/stores/settings' import { useGraphStore } from '@/stores/graph' import { labelListLimit } from '@/lib/constants' import MiniSearch from 'minisearch' import { useTranslation } from 'react-i18next' -const lastGraph: any = { - graph: null, - searchEngine: null, - labels: [] -} - const GraphLabels = () => { const { t } = useTranslation() const label = useSettingsStore.use.queryLabel() - const graph = useGraphStore.use.sigmaGraph() + const allDatabaseLabels = useGraphStore.use.allDatabaseLabels() + const labelsLoadedRef = useRef(false) - const getSearchEngine = useCallback(async () => { - if (lastGraph.graph == graph) { - return { - labels: lastGraph.labels, - searchEngine: lastGraph.searchEngine - } - } - const labels = ['*'].concat(await getGraphLabels()) + // Track if a fetch is in progress to prevent multiple simultaneous fetches + const fetchInProgressRef = useRef(false) + + // Fetch labels once on component mount, using global flag to prevent duplicates + useEffect(() => { + // Check if we've already attempted to fetch labels in this session + const labelsFetchAttempted = useGraphStore.getState().labelsFetchAttempted - // Ensure query label exists - if (!labels.includes(useSettingsStore.getState().queryLabel)) { - useSettingsStore.getState().setQueryLabel(labels[0]) + // Only fetch if we haven't attempted in this session and no fetch is in progress + if (!labelsFetchAttempted && !fetchInProgressRef.current) { + fetchInProgressRef.current = true + // Set global flag to indicate we've attempted to fetch in this session + useGraphStore.getState().setLabelsFetchAttempted(true) + + console.log('Fetching graph labels (once per session)...') + + useGraphStore.getState().fetchAllDatabaseLabels() + .then(() => { + labelsLoadedRef.current = true + fetchInProgressRef.current = false + }) + .catch((error) => { + console.error('Failed to fetch labels:', error) + fetchInProgressRef.current = false + // Reset global flag to allow retry + useGraphStore.getState().setLabelsFetchAttempted(false) + }) } + }, []) // Empty dependency array ensures this only runs once on mount + const getSearchEngine = useCallback(() => { // Create search engine const searchEngine = new MiniSearch({ idField: 'id', @@ -46,41 +57,32 @@ const GraphLabels = () => { }) // Add documents - const documents = labels.map((str, index) => ({ id: index, value: str })) + const documents = allDatabaseLabels.map((str, index) => ({ id: index, value: str })) searchEngine.addAll(documents) - lastGraph.graph = graph - lastGraph.searchEngine = searchEngine - lastGraph.labels = labels - return { - labels, + labels: allDatabaseLabels, searchEngine } - }, [graph]) + }, [allDatabaseLabels]) const fetchData = useCallback( async (query?: string): Promise => { - const { labels, searchEngine } = await getSearchEngine() + const { labels, searchEngine } = getSearchEngine() let result: string[] = labels if (query) { // Search labels - result = searchEngine.search(query).map((r) => labels[r.id]) + result = searchEngine.search(query).map((r: { id: number }) => labels[r.id]) } return result.length <= labelListLimit ? result - : [...result.slice(0, labelListLimit), t('graphLabels.andOthers', { count: result.length - labelListLimit })] + : [...result.slice(0, labelListLimit), '...'] }, [getSearchEngine] ) - const setQueryLabel = useCallback((label: string) => { - if (label.startsWith('And ') && label.endsWith(' others')) return - useSettingsStore.getState().setQueryLabel(label) - }, []) - return ( className="ml-2" @@ -94,8 +96,38 @@ const GraphLabels = () => { notFound={
No labels found
} label={t('graphPanel.graphLabels.label')} placeholder={t('graphPanel.graphLabels.placeholder')} - value={label !== null ? label : ''} - onChange={setQueryLabel} + value={label !== null ? label : '*'} + onChange={(newLabel) => { + const currentLabel = useSettingsStore.getState().queryLabel + + // select the last item means query all + if (newLabel === '...') { + newLabel = '*' + } + + // Reset the fetch attempted flag to force a new data fetch + useGraphStore.getState().setGraphDataFetchAttempted(false) + + // Clear current graph data to ensure complete reload when label changes + if (newLabel !== currentLabel) { + const graphStore = useGraphStore.getState(); + graphStore.clearSelection(); + + // Reset the graph state but preserve the instance + if (graphStore.sigmaGraph) { + const nodes = Array.from(graphStore.sigmaGraph.nodes()); + nodes.forEach(node => graphStore.sigmaGraph?.dropNode(node)); + } + } + + if (newLabel === currentLabel && newLabel !== '*') { + // reselect the same itme means qery all + useSettingsStore.getState().setQueryLabel('*') + } else { + useSettingsStore.getState().setQueryLabel(newLabel) + } + }} + clearable={false} // Prevent clearing value on reselect /> ) } diff --git a/lightrag_webui/src/components/graph/GraphSearch.tsx b/lightrag_webui/src/components/graph/GraphSearch.tsx index bbb8cb5bb07d735c91ed2e61d1a67d038d80d93c..2ba36bdac73f33f5b3d4292e1c1b3334b9078952 100644 --- a/lightrag_webui/src/components/graph/GraphSearch.tsx +++ b/lightrag_webui/src/components/graph/GraphSearch.tsx @@ -1,4 +1,4 @@ -import { FC, useCallback, useMemo } from 'react' +import { FC, useCallback, useEffect, useMemo } from 'react' import { EdgeById, NodeById, @@ -28,6 +28,7 @@ function OptionComponent(item: OptionItem) { } const messageId = '__message_item' +// Reset this cache when graph changes to ensure fresh search results const lastGraph: any = { graph: null, searchEngine: null @@ -48,6 +49,15 @@ export const GraphSearchInput = ({ const { t } = useTranslation() const graph = useGraphStore.use.sigmaGraph() + // Force reset the cache when graph changes + useEffect(() => { + if (graph) { + // Reset cache to ensure fresh search results with new graph data + lastGraph.graph = null; + lastGraph.searchEngine = null; + } + }, [graph]); + const searchEngine = useMemo(() => { if (lastGraph.graph == graph) { return lastGraph.searchEngine @@ -85,8 +95,19 @@ export const GraphSearchInput = ({ const loadOptions = useCallback( async (query?: string): Promise => { if (onFocus) onFocus(null) - if (!query || !searchEngine) return [] - const result: OptionItem[] = searchEngine.search(query).map((r) => ({ + if (!graph || !searchEngine) return [] + + // If no query, return first searchResultLimit nodes + if (!query) { + const nodeIds = graph.nodes().slice(0, searchResultLimit) + return nodeIds.map(id => ({ + id, + type: 'nodes' + })) + } + + // If has query, search nodes + const result: OptionItem[] = searchEngine.search(query).map((r: { id: string }) => ({ id: r.id, type: 'nodes' })) @@ -103,7 +124,7 @@ export const GraphSearchInput = ({ } ] }, - [searchEngine, onFocus] + [graph, searchEngine, onFocus, t] ) return ( diff --git a/lightrag_webui/src/components/graph/PropertiesView.tsx b/lightrag_webui/src/components/graph/PropertiesView.tsx index 4571b02bd1dd04598f7bc99d56a07e2dbc2a72be..0931e46d587b9fa649b179f9cf5c2dfce7ccc555 100644 --- a/lightrag_webui/src/components/graph/PropertiesView.tsx +++ b/lightrag_webui/src/components/graph/PropertiesView.tsx @@ -96,9 +96,9 @@ const refineNodeProperties = (node: RawNodeType): NodeType => { const neighbour = state.rawGraph.getNode(neighbourId) if (neighbour) { relationships.push({ - type: isTarget ? 'Target' : 'Source', + type: 'Neighbour', id: neighbourId, - label: neighbour.labels.join(', ') + label: neighbour.properties['entity_id'] ? neighbour.properties['entity_id'] : neighbour.labels.join(', ') }) } } @@ -132,14 +132,22 @@ const PropertyRow = ({ onClick?: () => void tooltip?: string }) => { + const { t } = useTranslation() + + const getPropertyNameTranslation = (name: string) => { + const translationKey = `graphPanel.propertiesView.node.propertyNames.${name}` + const translation = t(translationKey) + return translation === translationKey ? name : translation + } + return (
- : + : @@ -174,7 +182,7 @@ const NodePropertiesView = ({ node }: { node: NodeType }) => { {node.relationships.length > 0 && ( <>
{node.relationships.map(({ type, id, label }) => { diff --git a/lightrag_webui/src/components/graph/Settings.tsx b/lightrag_webui/src/components/graph/Settings.tsx index 4a4b15a52f981a72dee59469526ee264da4b12cb..a24c86e92bfe193d7f85ecbc8a65cfc1dd79d694 100644 --- a/lightrag_webui/src/components/graph/Settings.tsx +++ b/lightrag_webui/src/components/graph/Settings.tsx @@ -8,9 +8,10 @@ import Input from '@/components/ui/Input' import { controlButtonVariant } from '@/lib/constants' import { useSettingsStore } from '@/stores/settings' import { useBackendState } from '@/stores/state' +import { useGraphStore } from '@/stores/graph' -import { SettingsIcon } from 'lucide-react' -import { useTranslation } from "react-i18next"; +import { SettingsIcon, RefreshCwIcon } from 'lucide-react' +import { useTranslation } from 'react-i18next'; /** * Component that displays a checkbox with a label. @@ -114,6 +115,7 @@ const LabeledNumberInput = ({ export default function Settings() { const [opened, setOpened] = useState(false) const [tempApiKey, setTempApiKey] = useState('') + const refreshLayout = useGraphStore.use.refreshLayout() const showPropertyPanel = useSettingsStore.use.showPropertyPanel() const showNodeSearchBar = useSettingsStore.use.showNodeSearchBar() @@ -208,116 +210,126 @@ export default function Settings() { const { t } = useTranslation(); return ( - - - - - e.preventDefault()} + <> + + + + + + e.preventDefault()} + > +
+ - + - - + + - + - - + + - + - - - + + + - - - - - + + + + + -
- -
e.preventDefault()}> -
- -
- -
+
+ +
e.preventDefault()}> +
+ +
+ +
+
-
- - + + + ) } diff --git a/lightrag_webui/src/components/graph/SettingsDisplay.tsx b/lightrag_webui/src/components/graph/SettingsDisplay.tsx new file mode 100644 index 0000000000000000000000000000000000000000..7e63dd98afe1321b974bfe116096ad4977b167f1 --- /dev/null +++ b/lightrag_webui/src/components/graph/SettingsDisplay.tsx @@ -0,0 +1,21 @@ +import { useSettingsStore } from '@/stores/settings' +import { useTranslation } from 'react-i18next' + +/** + * Component that displays current values of important graph settings + * Positioned to the right of the toolbar at the bottom-left corner + */ +const SettingsDisplay = () => { + const { t } = useTranslation() + const graphQueryMaxDepth = useSettingsStore.use.graphQueryMaxDepth() + const graphMinDegree = useSettingsStore.use.graphMinDegree() + + return ( +
+
{t('graphPanel.sideBar.settings.depth')}: {graphQueryMaxDepth}
+
{t('graphPanel.sideBar.settings.degree')}: {graphMinDegree}
+
+ ) +} + +export default SettingsDisplay diff --git a/lightrag_webui/src/components/retrieval/QuerySettings.tsx b/lightrag_webui/src/components/retrieval/QuerySettings.tsx index ee785fc8b281d9b8f619d542ed49a6e1cf402a40..7be587df880b7c6bedaf0cbaf7c204e8fee3fa63 100644 --- a/lightrag_webui/src/components/retrieval/QuerySettings.tsx +++ b/lightrag_webui/src/components/retrieval/QuerySettings.tsx @@ -25,7 +25,7 @@ export default function QuerySettings() { }, []) return ( - + {t('retrievePanel.querySettings.parametersTitle')} {t('retrievePanel.querySettings.parametersDescription')} diff --git a/lightrag_webui/src/components/ui/AsyncSearch.tsx b/lightrag_webui/src/components/ui/AsyncSearch.tsx index a9b3a6cea65c4a785ca6531b844a677b34805850..b1c25fe93929432fad0f748784cf16f60ab79822 100644 --- a/lightrag_webui/src/components/ui/AsyncSearch.tsx +++ b/lightrag_webui/src/components/ui/AsyncSearch.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useCallback } from 'react' +import React, { useState, useEffect, useCallback } from 'react' import { Loader2 } from 'lucide-react' import { useDebounce } from '@/hooks/useDebounce' @@ -193,7 +193,7 @@ export function AsyncSearch({
)}
-