ArnoChen
commited on
Commit
·
c832152
1
Parent(s):
5f2a32b
fix examples
Browse files
examples/lightrag_api_oracle_demo.py
CHANGED
@@ -48,6 +48,14 @@ print(f"EMBEDDING_MAX_TOKEN_SIZE: {EMBEDDING_MAX_TOKEN_SIZE}")
|
|
48 |
if not os.path.exists(WORKING_DIR):
|
49 |
os.mkdir(WORKING_DIR)
|
50 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
51 |
|
52 |
async def llm_model_func(
|
53 |
prompt, system_prompt=None, history_messages=[], keyword_extraction=False, **kwargs
|
@@ -89,20 +97,6 @@ async def init():
|
|
89 |
# We storage data in unified tables, so we need to set a `workspace` parameter to specify which docs we want to store and query
|
90 |
# Below is an example of how to connect to Oracle Autonomous Database on Oracle Cloud
|
91 |
|
92 |
-
oracle_db = OracleDB(
|
93 |
-
config={
|
94 |
-
"user": "",
|
95 |
-
"password": "",
|
96 |
-
"dsn": "",
|
97 |
-
"config_dir": "path_to_config_dir",
|
98 |
-
"wallet_location": "path_to_wallet_location",
|
99 |
-
"wallet_password": "wallet_password",
|
100 |
-
"workspace": "company",
|
101 |
-
} # specify which docs you want to store and query
|
102 |
-
)
|
103 |
-
|
104 |
-
# Check if Oracle DB tables exist, if not, tables will be created
|
105 |
-
await oracle_db.check_tables()
|
106 |
# Initialize LightRAG
|
107 |
# We use Oracle DB as the KV/vector/graph storage
|
108 |
# You can add `addon_params={"example_number": 1, "language": "Simplfied Chinese"}` to control the prompt
|
@@ -121,11 +115,6 @@ async def init():
|
|
121 |
vector_storage="OracleVectorDBStorage",
|
122 |
)
|
123 |
|
124 |
-
# Setthe KV/vector/graph storage's `db` property, so all operation will use same connection pool
|
125 |
-
rag.graph_storage_cls.db = oracle_db
|
126 |
-
rag.key_string_value_json_storage_cls.db = oracle_db
|
127 |
-
rag.vector_db_storage_cls.db = oracle_db
|
128 |
-
|
129 |
return rag
|
130 |
|
131 |
|
|
|
48 |
if not os.path.exists(WORKING_DIR):
|
49 |
os.mkdir(WORKING_DIR)
|
50 |
|
51 |
+
os.environ["ORACLE_USER"] = ""
|
52 |
+
os.environ["ORACLE_PASSWORD"] = ""
|
53 |
+
os.environ["ORACLE_DSN"] = ""
|
54 |
+
os.environ["ORACLE_CONFIG_DIR"] = "path_to_config_dir"
|
55 |
+
os.environ["ORACLE_WALLET_LOCATION"] = "path_to_wallet_location"
|
56 |
+
os.environ["ORACLE_WALLET_PASSWORD"] = "wallet_password"
|
57 |
+
os.environ["ORACLE_WORKSPACE"] = "company"
|
58 |
+
|
59 |
|
60 |
async def llm_model_func(
|
61 |
prompt, system_prompt=None, history_messages=[], keyword_extraction=False, **kwargs
|
|
|
97 |
# We storage data in unified tables, so we need to set a `workspace` parameter to specify which docs we want to store and query
|
98 |
# Below is an example of how to connect to Oracle Autonomous Database on Oracle Cloud
|
99 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
100 |
# Initialize LightRAG
|
101 |
# We use Oracle DB as the KV/vector/graph storage
|
102 |
# You can add `addon_params={"example_number": 1, "language": "Simplfied Chinese"}` to control the prompt
|
|
|
115 |
vector_storage="OracleVectorDBStorage",
|
116 |
)
|
117 |
|
|
|
|
|
|
|
|
|
|
|
118 |
return rag
|
119 |
|
120 |
|
examples/lightrag_oracle_demo.py
CHANGED
@@ -26,6 +26,14 @@ MAX_TOKENS = 4000
|
|
26 |
if not os.path.exists(WORKING_DIR):
|
27 |
os.mkdir(WORKING_DIR)
|
28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
29 |
|
30 |
async def llm_model_func(
|
31 |
prompt, system_prompt=None, history_messages=[], keyword_extraction=False, **kwargs
|
@@ -63,26 +71,6 @@ async def main():
|
|
63 |
embedding_dimension = await get_embedding_dim()
|
64 |
print(f"Detected embedding dimension: {embedding_dimension}")
|
65 |
|
66 |
-
# Create Oracle DB connection
|
67 |
-
# The `config` parameter is the connection configuration of Oracle DB
|
68 |
-
# More docs here https://python-oracledb.readthedocs.io/en/latest/user_guide/connection_handling.html
|
69 |
-
# We storage data in unified tables, so we need to set a `workspace` parameter to specify which docs we want to store and query
|
70 |
-
# Below is an example of how to connect to Oracle Autonomous Database on Oracle Cloud
|
71 |
-
oracle_db = OracleDB(
|
72 |
-
config={
|
73 |
-
"user": "username",
|
74 |
-
"password": "xxxxxxxxx",
|
75 |
-
"dsn": "xxxxxxx_medium",
|
76 |
-
"config_dir": "dir/path/to/oracle/config",
|
77 |
-
"wallet_location": "dir/path/to/oracle/wallet",
|
78 |
-
"wallet_password": "xxxxxxxxx",
|
79 |
-
"workspace": "company", # specify which docs you want to store and query
|
80 |
-
}
|
81 |
-
)
|
82 |
-
|
83 |
-
# Check if Oracle DB tables exist, if not, tables will be created
|
84 |
-
await oracle_db.check_tables()
|
85 |
-
|
86 |
# Initialize LightRAG
|
87 |
# We use Oracle DB as the KV/vector/graph storage
|
88 |
# You can add `addon_params={"example_number": 1, "language": "Simplfied Chinese"}` to control the prompt
|
@@ -112,26 +100,6 @@ async def main():
|
|
112 |
},
|
113 |
)
|
114 |
|
115 |
-
# Setthe KV/vector/graph storage's `db` property, so all operation will use same connection pool
|
116 |
-
|
117 |
-
for storage in [
|
118 |
-
rag.vector_db_storage_cls,
|
119 |
-
rag.graph_storage_cls,
|
120 |
-
rag.doc_status,
|
121 |
-
rag.full_docs,
|
122 |
-
rag.text_chunks,
|
123 |
-
rag.llm_response_cache,
|
124 |
-
rag.key_string_value_json_storage_cls,
|
125 |
-
rag.chunks_vdb,
|
126 |
-
rag.relationships_vdb,
|
127 |
-
rag.entities_vdb,
|
128 |
-
rag.graph_storage_cls,
|
129 |
-
rag.chunk_entity_relation_graph,
|
130 |
-
rag.llm_response_cache,
|
131 |
-
]:
|
132 |
-
# set client
|
133 |
-
storage.db = oracle_db
|
134 |
-
|
135 |
# Extract and Insert into LightRAG storage
|
136 |
with open(WORKING_DIR + "/docs.txt", "r", encoding="utf-8") as f:
|
137 |
all_text = f.read()
|
|
|
26 |
if not os.path.exists(WORKING_DIR):
|
27 |
os.mkdir(WORKING_DIR)
|
28 |
|
29 |
+
os.environ["ORACLE_USER"] = "username"
|
30 |
+
os.environ["ORACLE_PASSWORD"] = "xxxxxxxxx"
|
31 |
+
os.environ["ORACLE_DSN"] = "xxxxxxx_medium"
|
32 |
+
os.environ["ORACLE_CONFIG_DIR"] = "path_to_config_dir"
|
33 |
+
os.environ["ORACLE_WALLET_LOCATION"] = "path_to_wallet_location"
|
34 |
+
os.environ["ORACLE_WALLET_PASSWORD"] = "wallet_password"
|
35 |
+
os.environ["ORACLE_WORKSPACE"] = "company"
|
36 |
+
|
37 |
|
38 |
async def llm_model_func(
|
39 |
prompt, system_prompt=None, history_messages=[], keyword_extraction=False, **kwargs
|
|
|
71 |
embedding_dimension = await get_embedding_dim()
|
72 |
print(f"Detected embedding dimension: {embedding_dimension}")
|
73 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
74 |
# Initialize LightRAG
|
75 |
# We use Oracle DB as the KV/vector/graph storage
|
76 |
# You can add `addon_params={"example_number": 1, "language": "Simplfied Chinese"}` to control the prompt
|
|
|
100 |
},
|
101 |
)
|
102 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
103 |
# Extract and Insert into LightRAG storage
|
104 |
with open(WORKING_DIR + "/docs.txt", "r", encoding="utf-8") as f:
|
105 |
all_text = f.read()
|
examples/lightrag_tidb_demo.py
CHANGED
@@ -17,11 +17,11 @@ APIKEY = ""
|
|
17 |
CHATMODEL = ""
|
18 |
EMBEDMODEL = ""
|
19 |
|
20 |
-
TIDB_HOST = ""
|
21 |
-
TIDB_PORT = ""
|
22 |
-
TIDB_USER = ""
|
23 |
-
TIDB_PASSWORD = ""
|
24 |
-
TIDB_DATABASE = "lightrag"
|
25 |
|
26 |
if not os.path.exists(WORKING_DIR):
|
27 |
os.mkdir(WORKING_DIR)
|
@@ -62,21 +62,6 @@ async def main():
|
|
62 |
embedding_dimension = await get_embedding_dim()
|
63 |
print(f"Detected embedding dimension: {embedding_dimension}")
|
64 |
|
65 |
-
# Create TiDB DB connection
|
66 |
-
tidb = TiDB(
|
67 |
-
config={
|
68 |
-
"host": TIDB_HOST,
|
69 |
-
"port": TIDB_PORT,
|
70 |
-
"user": TIDB_USER,
|
71 |
-
"password": TIDB_PASSWORD,
|
72 |
-
"database": TIDB_DATABASE,
|
73 |
-
"workspace": "company", # specify which docs you want to store and query
|
74 |
-
}
|
75 |
-
)
|
76 |
-
|
77 |
-
# Check if TiDB DB tables exist, if not, tables will be created
|
78 |
-
await tidb.check_tables()
|
79 |
-
|
80 |
# Initialize LightRAG
|
81 |
# We use TiDB DB as the KV/vector
|
82 |
# You can add `addon_params={"example_number": 1, "language": "Simplfied Chinese"}` to control the prompt
|
@@ -95,15 +80,6 @@ async def main():
|
|
95 |
graph_storage="TiDBGraphStorage",
|
96 |
)
|
97 |
|
98 |
-
if rag.llm_response_cache:
|
99 |
-
rag.llm_response_cache.db = tidb
|
100 |
-
rag.full_docs.db = tidb
|
101 |
-
rag.text_chunks.db = tidb
|
102 |
-
rag.entities_vdb.db = tidb
|
103 |
-
rag.relationships_vdb.db = tidb
|
104 |
-
rag.chunks_vdb.db = tidb
|
105 |
-
rag.chunk_entity_relation_graph.db = tidb
|
106 |
-
|
107 |
# Extract and Insert into LightRAG storage
|
108 |
with open("./dickens/demo.txt", "r", encoding="utf-8") as f:
|
109 |
await rag.ainsert(f.read())
|
|
|
17 |
CHATMODEL = ""
|
18 |
EMBEDMODEL = ""
|
19 |
|
20 |
+
os.environ["TIDB_HOST"] = ""
|
21 |
+
os.environ["TIDB_PORT"] = ""
|
22 |
+
os.environ["TIDB_USER"] = ""
|
23 |
+
os.environ["TIDB_PASSWORD"] = ""
|
24 |
+
os.environ["TIDB_DATABASE"] = "lightrag"
|
25 |
|
26 |
if not os.path.exists(WORKING_DIR):
|
27 |
os.mkdir(WORKING_DIR)
|
|
|
62 |
embedding_dimension = await get_embedding_dim()
|
63 |
print(f"Detected embedding dimension: {embedding_dimension}")
|
64 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
65 |
# Initialize LightRAG
|
66 |
# We use TiDB DB as the KV/vector
|
67 |
# You can add `addon_params={"example_number": 1, "language": "Simplfied Chinese"}` to control the prompt
|
|
|
80 |
graph_storage="TiDBGraphStorage",
|
81 |
)
|
82 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
83 |
# Extract and Insert into LightRAG storage
|
84 |
with open("./dickens/demo.txt", "r", encoding="utf-8") as f:
|
85 |
await rag.ainsert(f.read())
|
examples/lightrag_zhipu_postgres_demo.py
CHANGED
@@ -22,22 +22,14 @@ if not os.path.exists(WORKING_DIR):
|
|
22 |
# AGE
|
23 |
os.environ["AGE_GRAPH_NAME"] = "dickens"
|
24 |
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
"password": "rag",
|
31 |
-
"database": "rag",
|
32 |
-
}
|
33 |
-
)
|
34 |
|
35 |
|
36 |
async def main():
|
37 |
-
await postgres_db.initdb()
|
38 |
-
# Check if PostgreSQL DB tables exist, if not, tables will be created
|
39 |
-
await postgres_db.check_tables()
|
40 |
-
|
41 |
rag = LightRAG(
|
42 |
working_dir=WORKING_DIR,
|
43 |
llm_model_func=zhipu_complete,
|
@@ -57,17 +49,7 @@ async def main():
|
|
57 |
graph_storage="PGGraphStorage",
|
58 |
vector_storage="PGVectorStorage",
|
59 |
)
|
60 |
-
|
61 |
-
rag.doc_status.db = postgres_db
|
62 |
-
rag.full_docs.db = postgres_db
|
63 |
-
rag.text_chunks.db = postgres_db
|
64 |
-
rag.llm_response_cache.db = postgres_db
|
65 |
-
rag.key_string_value_json_storage_cls.db = postgres_db
|
66 |
-
rag.chunks_vdb.db = postgres_db
|
67 |
-
rag.relationships_vdb.db = postgres_db
|
68 |
-
rag.entities_vdb.db = postgres_db
|
69 |
-
rag.graph_storage_cls.db = postgres_db
|
70 |
-
rag.chunk_entity_relation_graph.db = postgres_db
|
71 |
# add embedding_func for graph database, it's deleted in commit 5661d76860436f7bf5aef2e50d9ee4a59660146c
|
72 |
rag.chunk_entity_relation_graph.embedding_func = rag.embedding_func
|
73 |
|
|
|
22 |
# AGE
|
23 |
os.environ["AGE_GRAPH_NAME"] = "dickens"
|
24 |
|
25 |
+
os.environ["POSTGRES_HOST"] = "localhost"
|
26 |
+
os.environ["POSTGRES_PORT"] = "15432"
|
27 |
+
os.environ["POSTGRES_USER"] = "rag"
|
28 |
+
os.environ["POSTGRES_PASSWORD"] = "rag"
|
29 |
+
os.environ["POSTGRES_DATABASE"] = "rag"
|
|
|
|
|
|
|
|
|
30 |
|
31 |
|
32 |
async def main():
|
|
|
|
|
|
|
|
|
33 |
rag = LightRAG(
|
34 |
working_dir=WORKING_DIR,
|
35 |
llm_model_func=zhipu_complete,
|
|
|
49 |
graph_storage="PGGraphStorage",
|
50 |
vector_storage="PGVectorStorage",
|
51 |
)
|
52 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
53 |
# add embedding_func for graph database, it's deleted in commit 5661d76860436f7bf5aef2e50d9ee4a59660146c
|
54 |
rag.chunk_entity_relation_graph.embedding_func = rag.embedding_func
|
55 |
|