增加仅字符分割参数，如果开启，仅采用字符分割，不开启，在分割完以后如果chunk过大，会继续根据token size分割，更新测试文件

test.ipynb → examples/test_split_by_character.ipynb

@@ -6,8 +6,8 @@
    "id": "4b5690db12e34685",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2025-01-07T05:38:34.174205Z",
-     "start_time": "2025-01-07T05:38:29.978194Z"
+     "end_time": "2025-01-09T03:40:58.307102Z",
+     "start_time": "2025-01-09T03:40:51.935233Z"
     }
    },
    "outputs": [],
@@ -21,34 +21,40 @@
     "import nest_asyncio"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "dd17956ec322b361",
+   "metadata": {},
+   "source": "#### split by character"
+  },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "id": "8c8ee7c061bf9159",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2025-01-07T05:38:37.440083Z",
-     "start_time": "2025-01-07T05:38:37.437666Z"
+     "end_time": "2025-01-09T03:41:13.961167Z",
+     "start_time": "2025-01-09T03:41:13.958357Z"
     }
    },
    "outputs": [],
    "source": [
     "nest_asyncio.apply()\n",
-    "WORKING_DIR = \"../llm_rag/paper_db/R000088_test2\"\n",
+    "WORKING_DIR = \"../../llm_rag/paper_db/R000088_test1\"\n",
     "logging.basicConfig(format=\"%(levelname)s:%(message)s\", level=logging.INFO)\n",
     "if not os.path.exists(WORKING_DIR):\n",
     "    os.mkdir(WORKING_DIR)\n",
-    "os.environ[\"doubao_api\"] = \"6b890250-0cf6-4eb1-aa82-9c9d711398a7\""
+    "API = os.environ.get(\"DOUBAO_API_KEY\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "id": "a5009d16e0851dca",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2025-01-07T05:38:42.594315Z",
-     "start_time": "2025-01-07T05:38:42.590800Z"
+     "end_time": "2025-01-09T03:41:16.862036Z",
+     "start_time": "2025-01-09T03:41:16.859306Z"
     }
    },
    "outputs": [],
@@ -61,7 +67,7 @@
     "        prompt,\n",
     "        system_prompt=system_prompt,\n",
     "        history_messages=history_messages,\n",
-    "        api_key=os.getenv(\"doubao_api\"),\n",
+    "        api_key=API,\n",
     "        base_url=\"https://ark.cn-beijing.volces.com/api/v3\",\n",
     "        **kwargs,\n",
     "    )\n",
@@ -71,19 +77,19 @@
     "    return await openai_embedding(\n",
     "        texts,\n",
     "        model=\"ep-20241231173413-pgjmk\",\n",
-    "        api_key=os.getenv(\"doubao_api\"),\n",
+    "        api_key=API,\n",
     "        base_url=\"https://ark.cn-beijing.volces.com/api/v3\",\n",
     "    )"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "id": "397fcad24ce4d0ed",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2025-01-07T05:38:44.016901Z",
-     "start_time": "2025-01-07T05:38:44.006291Z"
+     "end_time": "2025-01-09T03:41:24.950307Z",
+     "start_time": "2025-01-09T03:41:24.940353Z"
     }
    },
    "outputs": [
@@ -91,13 +97,13 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "INFO:lightrag:Logger initialized for working directory: ../llm_rag/paper_db/R000088_test2\n",
+      "INFO:lightrag:Logger initialized for working directory: ../../llm_rag/paper_db/R000088_test1\n",
       "INFO:lightrag:Load KV llm_response_cache with 0 data\n",
       "INFO:lightrag:Load KV full_docs with 0 data\n",
       "INFO:lightrag:Load KV text_chunks with 0 data\n",
-      "INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../llm_rag/paper_db/R000088_test2/vdb_entities.json'} 0 data\n",
-      "INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../llm_rag/paper_db/R000088_test2/vdb_relationships.json'} 0 data\n",
-      "INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../llm_rag/paper_db/R000088_test2/vdb_chunks.json'} 0 data\n",
+      "INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../../llm_rag/paper_db/R000088_test1/vdb_entities.json'} 0 data\n",
+      "INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../../llm_rag/paper_db/R000088_test1/vdb_relationships.json'} 0 data\n",
+      "INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../../llm_rag/paper_db/R000088_test1/vdb_chunks.json'} 0 data\n",
       "INFO:lightrag:Loaded document status storage with 0 records\n"
      ]
     }
@@ -109,23 +115,24 @@
     "    embedding_func=EmbeddingFunc(\n",
     "        embedding_dim=4096, max_token_size=8192, func=embedding_func\n",
     "    ),\n",
+    "    chunk_token_size=512,\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "id": "1dc3603677f7484d",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2025-01-07T05:38:47.509111Z",
-     "start_time": "2025-01-07T05:38:47.501997Z"
+     "end_time": "2025-01-09T03:41:37.947456Z",
+     "start_time": "2025-01-09T03:41:37.941901Z"
     }
    },
    "outputs": [],
    "source": [
     "with open(\n",
-    "    \"../llm_rag/example/R000088/auto/R000088_full_txt.md\", \"r\", encoding=\"utf-8\"\n",
+    "    \"../../llm_rag/example/R000088/auto/R000088_full_txt.md\", \"r\", encoding=\"utf-8\"\n",
     ") as f:\n",
     "    content = f.read()\n",
     "\n",
@@ -134,7 +141,7 @@
     "    return await openai_embedding(\n",
     "        texts,\n",
     "        model=\"ep-20241231173413-pgjmk\",\n",
-    "        api_key=os.getenv(\"doubao_api\"),\n",
+    "        api_key=API,\n",
     "        base_url=\"https://ark.cn-beijing.volces.com/api/v3\",\n",
     "    )\n",
     "\n",
@@ -148,12 +155,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 7,
    "id": "6844202606acfbe5",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2025-01-07T05:38:50.666764Z",
-     "start_time": "2025-01-07T05:38:50.247712Z"
+     "end_time": "2025-01-09T03:41:39.608541Z",
+     "start_time": "2025-01-09T03:41:39.165057Z"
     }
    },
    "outputs": [
@@ -171,12 +178,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 8,
    "id": "d6273839d9681403",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2025-01-07T05:42:33.085507Z",
-     "start_time": "2025-01-07T05:38:56.789348Z"
+     "end_time": "2025-01-09T03:44:34.295345Z",
+     "start_time": "2025-01-09T03:41:48.324171Z"
     }
    },
    "outputs": [
@@ -185,13 +192,48 @@
      "output_type": "stream",
      "text": [
       "INFO:lightrag:Processing 1 new unique documents\n",
-      "Processing batch 1:   0%|          | 0/1 [00:00<?, ?it/s]INFO:lightrag:Inserting 22 vectors to chunks\n",
+      "Processing batch 1:   0%|          | 0/1 [00:00<?, ?it/s]INFO:lightrag:Inserting 35 vectors to chunks\n",
       "\n",
-      "Generating embeddings:   0%|          | 0/1 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "Generating embeddings:   0%|          | 0/2 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "\n",
-      "Generating embeddings: 100%|██████████| 1/1 [00:03<00:00,  3.85s/batch]\u001b[A\n",
+      "Generating embeddings:  50%|█████     | 1/2 [00:00<00:00,  1.36batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "\n",
-      "Extracting entities from chunks:   0%|          | 0/22 [00:00<?, ?chunk/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "Generating embeddings: 100%|██████████| 2/2 [00:04<00:00,  2.25s/batch]\u001b[A\n",
+      "\n",
+      "Extracting entities from chunks:   0%|          | 0/35 [00:00<?, ?chunk/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠙ Processed 1 chunks, 1 entities(duplicated), 0 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:   3%|▎         | 1/35 [00:04<02:47,  4.93s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠹ Processed 2 chunks, 2 entities(duplicated), 0 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:   6%|▌         | 2/35 [00:05<01:18,  2.37s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
@@ -206,7 +248,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠙ Processed 1 chunks, 7 entities(duplicated), 6 relations(duplicated)\r"
+      "⠸ Processed 3 chunks, 9 entities(duplicated), 5 relations(duplicated)\r"
      ]
     },
     {
@@ -214,7 +256,22 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:   5%|▍         | 1/22 [00:23<08:21, 23.90s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "Extracting entities from chunks:   9%|▊         | 3/35 [00:26<05:43, 10.73s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠼ Processed 4 chunks, 16 entities(duplicated), 11 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  11%|█▏        | 4/35 [00:26<03:24,  6.60s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
@@ -222,7 +279,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠹ Processed 2 chunks, 12 entities(duplicated), 15 relations(duplicated)\r"
+      "⠴ Processed 5 chunks, 24 entities(duplicated), 18 relations(duplicated)\r"
      ]
     },
     {
@@ -230,7 +287,7 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:   9%|▉         | 2/22 [00:26<03:50, 11.51s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "Extracting entities from chunks:  14%|█▍        | 5/35 [00:33<03:24,  6.82s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
@@ -238,7 +295,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠸ Processed 3 chunks, 20 entities(duplicated), 22 relations(duplicated)\r"
+      "⠦ Processed 6 chunks, 35 entities(duplicated), 28 relations(duplicated)\r"
      ]
     },
     {
@@ -246,7 +303,7 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  14%|█▎        | 3/22 [00:34<03:08,  9.93s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "Extracting entities from chunks:  17%|█▋        | 6/35 [00:42<03:38,  7.53s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
@@ -254,7 +311,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠼ Processed 4 chunks, 30 entities(duplicated), 30 relations(duplicated)\r"
+      "⠧ Processed 7 chunks, 47 entities(duplicated), 36 relations(duplicated)\r"
      ]
     },
     {
@@ -262,14 +319,14 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  18%|█▊        | 4/22 [00:37<02:09,  7.21s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+      "Extracting entities from chunks:  20%|██        | 7/35 [00:43<02:28,  5.31s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠴ Processed 5 chunks, 39 entities(duplicated), 39 relations(duplicated)\r"
+      "⠇ Processed 8 chunks, 61 entities(duplicated), 49 relations(duplicated)\r"
      ]
     },
     {
@@ -277,14 +334,29 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  23%|██▎       | 5/22 [00:38<01:19,  4.70s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+      "Extracting entities from chunks:  23%|██▎       | 8/35 [00:45<01:52,  4.16s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠦ Processed 6 chunks, 39 entities(duplicated), 39 relations(duplicated)\r"
+      "⠏ Processed 9 chunks, 81 entities(duplicated), 49 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠋ Processed 10 chunks, 90 entities(duplicated), 62 relations(duplicated)\r"
      ]
     },
     {
@@ -292,14 +364,16 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  27%|██▋       | 6/22 [00:38<00:53,  3.32s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+      "Extracting entities from chunks:  29%|██▊       | 10/35 [00:46<01:06,  2.64s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠧ Processed 7 chunks, 47 entities(duplicated), 50 relations(duplicated)\r"
+      "⠙ Processed 11 chunks, 101 entities(duplicated), 80 relations(duplicated)\r"
      ]
     },
     {
@@ -307,7 +381,7 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  32%|███▏      | 7/22 [00:39<00:39,  2.65s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "Extracting entities from chunks:  31%|███▏      | 11/35 [00:52<01:19,  3.31s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
@@ -315,7 +389,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠇ Processed 8 chunks, 56 entities(duplicated), 58 relations(duplicated)\r"
+      "⠹ Processed 12 chunks, 108 entities(duplicated), 85 relations(duplicated)\r"
      ]
     },
     {
@@ -323,7 +397,7 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  36%|███▋      | 8/22 [00:40<00:29,  2.13s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "Extracting entities from chunks:  34%|███▍      | 12/35 [00:54<01:11,  3.12s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
@@ -332,7 +406,38 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠏ Processed 9 chunks, 63 entities(duplicated), 69 relations(duplicated)\r"
+      "⠸ Processed 13 chunks, 120 entities(duplicated), 100 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  37%|███▋      | 13/35 [00:59<01:18,  3.55s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠼ Processed 14 chunks, 131 entities(duplicated), 110 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  40%|████      | 14/35 [01:00<00:59,  2.82s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠴ Processed 15 chunks, 143 entities(duplicated), 110 relations(duplicated)\r"
      ]
     },
     {
@@ -340,14 +445,14 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  41%|████      | 9/22 [00:47<00:43,  3.38s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+      "Extracting entities from chunks:  43%|████▎     | 15/35 [01:02<00:52,  2.64s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠋ Processed 10 chunks, 81 entities(duplicated), 81 relations(duplicated)\r"
+      "⠦ Processed 16 chunks, 162 entities(duplicated), 124 relations(duplicated)\r"
      ]
     },
     {
@@ -355,7 +460,39 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  45%|████▌     | 10/22 [00:48<00:32,  2.73s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "Extracting entities from chunks:  46%|████▌     | 16/35 [01:05<00:53,  2.80s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠧ Processed 17 chunks, 174 entities(duplicated), 132 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  49%|████▊     | 17/35 [01:06<00:39,  2.19s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠇ Processed 18 chunks, 185 entities(duplicated), 137 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  51%|█████▏    | 18/35 [01:12<00:53,  3.15s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
@@ -366,7 +503,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠙ Processed 11 chunks, 92 entities(duplicated), 89 relations(duplicated)\r"
+      "⠏ Processed 19 chunks, 193 entities(duplicated), 149 relations(duplicated)\r"
      ]
     },
     {
@@ -374,14 +511,15 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  50%|█████     | 11/22 [01:01<01:05,  5.99s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+      "Extracting entities from chunks:  54%|█████▍    | 19/35 [01:18<01:06,  4.14s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠹ Processed 12 chunks, 107 entities(duplicated), 107 relations(duplicated)\r"
+      "⠋ Processed 20 chunks, 205 entities(duplicated), 158 relations(duplicated)\r"
      ]
     },
     {
@@ -389,7 +527,7 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  55%|█████▍    | 12/22 [01:10<01:09,  6.94s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "Extracting entities from chunks:  57%|█████▋    | 20/35 [01:19<00:50,  3.35s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
@@ -397,7 +535,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠸ Processed 13 chunks, 127 entities(duplicated), 126 relations(duplicated)\r"
+      "⠙ Processed 21 chunks, 220 entities(duplicated), 187 relations(duplicated)\r"
      ]
     },
     {
@@ -405,14 +543,18 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  59%|█████▉    | 13/22 [01:16<00:59,  6.59s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+      "Extracting entities from chunks:  60%|██████    | 21/35 [01:27<01:02,  4.47s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠼ Processed 14 chunks, 151 entities(duplicated), 137 relations(duplicated)\r"
+      "⠹ Processed 22 chunks, 247 entities(duplicated), 216 relations(duplicated)\r"
      ]
     },
     {
@@ -420,14 +562,14 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  64%|██████▎   | 14/22 [01:16<00:37,  4.68s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+      "Extracting entities from chunks:  63%|██████▎   | 22/35 [01:30<00:54,  4.16s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠴ Processed 15 chunks, 161 entities(duplicated), 144 relations(duplicated)\r"
+      "⠸ Processed 23 chunks, 260 entities(duplicated), 230 relations(duplicated)\r"
      ]
     },
     {
@@ -435,14 +577,14 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  68%|██████▊   | 15/22 [01:17<00:23,  3.31s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+      "Extracting entities from chunks:  66%|██████▌   | 23/35 [01:34<00:48,  4.05s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠦ Processed 16 chunks, 176 entities(duplicated), 154 relations(duplicated)\r"
+      "⠼ Processed 24 chunks, 291 entities(duplicated), 253 relations(duplicated)\r"
      ]
     },
     {
@@ -450,14 +592,14 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  73%|███████▎  | 16/22 [01:19<00:18,  3.04s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+      "Extracting entities from chunks:  69%|██████▊   | 24/35 [01:38<00:44,  4.03s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠧ Processed 17 chunks, 189 entities(duplicated), 162 relations(duplicated)\r"
+      "⠴ Processed 25 chunks, 304 entities(duplicated), 262 relations(duplicated)\r"
      ]
     },
     {
@@ -465,7 +607,7 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  77%|███████▋  | 17/22 [01:21<00:13,  2.80s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "Extracting entities from chunks:  71%|███████▏  | 25/35 [01:41<00:36,  3.67s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
@@ -473,7 +615,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠇ Processed 18 chunks, 207 entities(duplicated), 186 relations(duplicated)\r"
+      "⠦ Processed 26 chunks, 313 entities(duplicated), 271 relations(duplicated)\r"
      ]
     },
     {
@@ -481,14 +623,14 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  82%|████████▏ | 18/22 [01:38<00:28,  7.06s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+      "Extracting entities from chunks:  74%|███████▍  | 26/35 [01:41<00:24,  2.76s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠏ Processed 19 chunks, 222 entities(duplicated), 200 relations(duplicated)\r"
+      "⠧ Processed 27 chunks, 321 entities(duplicated), 283 relations(duplicated)\r"
      ]
     },
     {
@@ -496,15 +638,29 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  86%|████████▋ | 19/22 [01:44<00:19,  6.61s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
-      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+      "Extracting entities from chunks:  77%|███████▋  | 27/35 [01:47<00:28,  3.52s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠇ Processed 28 chunks, 333 entities(duplicated), 290 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  80%|████████  | 28/35 [01:52<00:28,  4.08s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠋ Processed 20 chunks, 310 entities(duplicated), 219 relations(duplicated)\r"
+      "⠏ Processed 29 chunks, 348 entities(duplicated), 307 relations(duplicated)\r"
      ]
     },
     {
@@ -512,7 +668,7 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  91%|█████████ | 20/22 [02:12<00:26, 13.19s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "Extracting entities from chunks:  83%|████████▎ | 29/35 [01:59<00:29,  4.88s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
@@ -520,7 +676,52 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠙ Processed 21 chunks, 345 entities(duplicated), 263 relations(duplicated)\r"
+      "⠋ Processed 30 chunks, 362 entities(duplicated), 329 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  86%|████████▌ | 30/35 [02:02<00:21,  4.29s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠙ Processed 31 chunks, 373 entities(duplicated), 337 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  89%|████████▊ | 31/35 [02:03<00:13,  3.28s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠹ Processed 32 chunks, 390 entities(duplicated), 369 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  91%|█████████▏| 32/35 [02:03<00:07,  2.55s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠸ Processed 33 chunks, 405 entities(duplicated), 378 relations(duplicated)\r"
      ]
     },
     {
@@ -528,14 +729,14 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks:  95%|█████████▌| 21/22 [02:32<00:15, 15.15s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+      "Extracting entities from chunks:  94%|█████████▍| 33/35 [02:07<00:05,  2.84s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "⠹ Processed 22 chunks, 417 entities(duplicated), 285 relations(duplicated)\r"
+      "⠼ Processed 34 chunks, 435 entities(duplicated), 395 relations(duplicated)\r"
      ]
     },
     {
@@ -543,20 +744,33 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "Extracting entities from chunks: 100%|██████████| 22/22 [03:21<00:00,  9.18s/chunk]\u001b[A\n",
+      "Extracting entities from chunks:  97%|█████████▋| 34/35 [02:10<00:02,  2.94s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠴ Processed 35 chunks, 456 entities(duplicated), 440 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks: 100%|██████████| 35/35 [02:23<00:00,  4.10s/chunk]\u001b[A\n",
       "INFO:lightrag:Inserting entities into storage...\n",
       "\n",
-      "Inserting entities: 100%|██████████| 327/327 [00:00<00:00, 13446.31entity/s]\n",
+      "Inserting entities: 100%|██████████| 324/324 [00:00<00:00, 17456.96entity/s]\n",
       "INFO:lightrag:Inserting relationships into storage...\n",
       "\n",
-      "Inserting relationships: 100%|██████████| 272/272 [00:00<00:00, 16740.29relationship/s]\n",
-      "INFO:lightrag:Inserting 327 vectors to entities\n",
+      "Inserting relationships: 100%|██████████| 427/427 [00:00<00:00, 29956.31relationship/s]\n",
+      "INFO:lightrag:Inserting 324 vectors to entities\n",
       "\n",
       "Generating embeddings:   0%|          | 0/11 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "\n",
-      "Generating embeddings:   9%|▉         | 1/11 [00:00<00:09,  1.02batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
-      "\n",
-      "Generating embeddings:  18%|█▊        | 2/11 [00:02<00:09,  1.07s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "Generating embeddings:   9%|▉         | 1/11 [00:00<00:06,  1.48batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
@@ -564,40 +778,52 @@
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "\n",
-      "Generating embeddings:  27%|██▋       | 3/11 [00:02<00:06,  1.33batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "Generating embeddings:  18%|█▊        | 2/11 [00:02<00:11,  1.25s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "\n",
-      "Generating embeddings:  36%|███▋      | 4/11 [00:02<00:04,  1.67batch/s]\u001b[A\n",
-      "Generating embeddings:  45%|████▌     | 5/11 [00:03<00:03,  1.93batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "Generating embeddings:  27%|██▋       | 3/11 [00:02<00:06,  1.17batch/s]\u001b[A\n",
+      "Generating embeddings:  36%|███▋      | 4/11 [00:03<00:04,  1.50batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "\n",
-      "Generating embeddings:  55%|█████▍    | 6/11 [00:03<00:02,  2.15batch/s]\u001b[A\n",
-      "Generating embeddings:  64%|██████▎   | 7/11 [00:03<00:01,  2.33batch/s]\u001b[A\n",
-      "Generating embeddings:  73%|███████▎  | 8/11 [00:04<00:01,  2.46batch/s]\u001b[A\n",
-      "Generating embeddings:  82%|████████▏ | 9/11 [00:04<00:00,  2.55batch/s]\u001b[A\n",
-      "Generating embeddings:  91%|█████████ | 10/11 [00:05<00:00,  2.64batch/s]\u001b[A\n",
-      "Generating embeddings: 100%|██████████| 11/11 [00:05<00:00,  2.04batch/s]\u001b[A\n",
-      "INFO:lightrag:Inserting 272 vectors to relationships\n",
+      "Generating embeddings:  45%|████▌     | 5/11 [00:03<00:03,  1.78batch/s]\u001b[A\n",
+      "Generating embeddings:  55%|█████▍    | 6/11 [00:03<00:02,  2.01batch/s]\u001b[A\n",
+      "Generating embeddings:  64%|██████▎   | 7/11 [00:04<00:01,  2.19batch/s]\u001b[A\n",
+      "Generating embeddings:  73%|███████▎  | 8/11 [00:04<00:01,  2.31batch/s]\u001b[A\n",
+      "Generating embeddings:  82%|████████▏ | 9/11 [00:04<00:00,  2.41batch/s]\u001b[A\n",
+      "Generating embeddings:  91%|█████████ | 10/11 [00:05<00:00,  2.48batch/s]\u001b[A\n",
+      "Generating embeddings: 100%|██████████| 11/11 [00:05<00:00,  1.91batch/s]\u001b[A\n",
+      "INFO:lightrag:Inserting 427 vectors to relationships\n",
       "\n",
-      "Generating embeddings:   0%|          | 0/9 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "Generating embeddings:   0%|          | 0/14 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "\n",
-      "Generating embeddings:  11%|█         | 1/9 [00:01<00:11,  1.39s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "Generating embeddings:   7%|▋         | 1/14 [00:01<00:14,  1.11s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "\n",
-      "Generating embeddings:  22%|██▏       | 2/9 [00:02<00:07,  1.01s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "Generating embeddings:  14%|█▍        | 2/14 [00:02<00:14,  1.18s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "\n",
-      "Generating embeddings:  33%|███▎      | 3/9 [00:02<00:04,  1.40batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "Generating embeddings:  21%|██▏       | 3/14 [00:02<00:08,  1.23batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
       "\n",
-      "Generating embeddings:  44%|████▍     | 4/9 [00:02<00:02,  1.74batch/s]\u001b[A\n",
-      "Generating embeddings:  56%|█████▌    | 5/9 [00:03<00:01,  2.01batch/s]\u001b[A\n",
-      "Generating embeddings:  67%|██████▋   | 6/9 [00:03<00:01,  2.23batch/s]\u001b[A\n",
-      "Generating embeddings:  78%|███████▊  | 7/9 [00:03<00:00,  2.39batch/s]\u001b[A\n",
-      "Generating embeddings:  89%|████████▉ | 8/9 [00:04<00:00,  2.52batch/s]\u001b[A\n",
-      "Generating embeddings: 100%|██████████| 9/9 [00:04<00:00,  1.93batch/s]\u001b[A\n",
-      "INFO:lightrag:Writing graph with 331 nodes, 272 edges\n",
-      "Processing batch 1: 100%|██████████| 1/1 [03:36<00:00, 216.27s/it]\n"
+      "Generating embeddings:  29%|██▊       | 4/14 [00:03<00:06,  1.56batch/s]\u001b[A\n",
+      "Generating embeddings:  36%|███▌      | 5/14 [00:03<00:04,  1.85batch/s]\u001b[A\n",
+      "Generating embeddings:  43%|████▎     | 6/14 [00:03<00:03,  2.05batch/s]\u001b[A\n",
+      "Generating embeddings:  50%|█████     | 7/14 [00:04<00:03,  2.23batch/s]\u001b[A\n",
+      "Generating embeddings:  57%|█████▋    | 8/14 [00:04<00:02,  2.37batch/s]\u001b[A\n",
+      "Generating embeddings:  64%|██████▍   | 9/14 [00:04<00:02,  2.46batch/s]\u001b[A\n",
+      "Generating embeddings:  71%|███████▏  | 10/14 [00:05<00:01,  2.54batch/s]\u001b[A\n",
+      "Generating embeddings:  79%|███████▊  | 11/14 [00:05<00:01,  2.59batch/s]\u001b[A\n",
+      "Generating embeddings:  86%|████████▌ | 12/14 [00:06<00:00,  2.64batch/s]\u001b[A\n",
+      "Generating embeddings:  93%|█████████▎| 13/14 [00:06<00:00,  2.65batch/s]\u001b[A\n",
+      "Generating embeddings: 100%|██████████| 14/14 [00:06<00:00,  2.05batch/s]\u001b[A\n",
+      "INFO:lightrag:Writing graph with 333 nodes, 427 edges\n",
+      "Processing batch 1: 100%|██████████| 1/1 [02:45<00:00, 165.90s/it]\n"
      ]
     }
    ],
@@ -608,40 +834,107 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 9,
    "id": "c4f9ae517151a01d",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2025-01-07T05:42:50.044809Z",
-     "start_time": "2025-01-07T05:42:50.041256Z"
+     "end_time": "2025-01-09T03:45:11.668987Z",
+     "start_time": "2025-01-09T03:45:11.664744Z"
     }
    },
    "outputs": [],
    "source": [
-    "prompt1 = \"\"\"\n",
-    "你是一名经验丰富的论文分析科学家，你的任务是对一篇英文学术研究论文进行关键信息提取并深入分析。\n",
-    "\n",
+    "prompt1 = \"\"\"你是一名经验丰富的论文分析科学家，你的任务是对一篇英文学术研究论文进行关键信息提取并深入分析。\n",
     "请按照以下步骤进行分析：\n",
-    "1. 对于论文的分析对象相关问题：\n",
-    "    - 仔细查找论文中的研究队列相关信息，确定分析对象来自哪些研究队列。\n",
-    "    - 查看如果来自多个队列，文中是单独分析还是联合分析。\n",
-    "    - 找出这些队列的名称。\n",
-    "    - 确定这些队列开展的国家有哪些（注意：“澳门”记为“中国澳门”，“香港”记为“中国香港”，“台湾”记为“中国台湾”，其余采用国家回答）。\n",
-    "    - 明确队列研究对象的性别分布（“男性”、“女性”或“全体”）。\n",
-    "    - 查找队列收集结束时，研究对象年龄分布（平均值/中位值、标准差或范围），若信息缺失则根据年龄推理规则进行推理：当论文只提供了队列开展时对象的年龄，应根据队列结束时间推算最终年龄范围。例如：1989建立队列时年龄为25 - 42岁，随访至2011年结束，则推算年龄范围为47 - 64岁。\n",
-    "    - 确定队列研究时间线，即哪一年开始收集信息/建立队列，哪一年结束，若信息缺失则根据队列时间线推理规则进行推理：如论文只提供了建立队列时间为1995，进行了10年的随访，则推算队列结束时间为2005年。\n",
-    "    - 找出队列结束时实际参与研究人数是多少。\n",
+    "1. 该文献主要研究的问题是什么？\n",
+    "2. 该文献采用什么方法进行分析？\n",
+    "3. 该文献的主要结论是什么？\n",
     "首先在<分析>标签中，针对每个问题详细分析你的思考过程。然后在<回答>标签中给出所有问题的最终答案。\"\"\""
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 10,
    "id": "7a6491385b050095",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2025-01-07T05:43:24.751628Z",
-     "start_time": "2025-01-07T05:42:50.865679Z"
+     "end_time": "2025-01-09T03:45:40.829111Z",
+     "start_time": "2025-01-09T03:45:13.530298Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:lightrag:Local query uses 5 entites, 12 relations, 3 text units\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:lightrag:Global query uses 8 entites, 5 relations, 4 text units\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<分析>\n",
+      "1. **该文献主要研究的问题是什么？**\n",
+      "    - 思考过程：通过浏览论文内容，查找作者明确阐述研究目的的部分。文中多处提及“Our study was performed to explore whether folic acid treatment was associated with cancer outcomes and all-cause mortality after extended follow-up”，表明作者旨在探究叶酸治疗与癌症结局及全因死亡率之间的关系，尤其是在经过长期随访后。\n",
+      "2. **该文献采用什么方法进行分析？**\n",
+      "    - 思考过程：寻找描述研究方法和数据分析过程的段落。文中提到“Survival curves were constructed using the Kaplan-Meier method and differences in survival between groups were analyzed using the log-rank test. Estimates of hazard ratios (HRs) with 95% CIs were obtained by using Cox proportional hazards regression models stratified by trial”，可以看出作者使用了Kaplan-Meier法构建生存曲线、log-rank检验分析组间生存差异以及Cox比例风险回归模型估计风险比等方法。\n",
+      "3. **该文献的主要结论是什么？**\n",
+      "    - 思考过程：定位到论文中总结结论的部分，如“Conclusion Treatment with folic acid plus vitamin $\\mathsf{B}_{12}$ was associated with increased cancer outcomes and all-cause mortality in patients with ischemic heart disease in Norway, where there is no folic acid fortification of foods”，可知作者得出叶酸加维生素$\\mathsf{B}_{12}$治疗与癌症结局和全因死亡率增加有关的结论。\n",
+      "<回答>\n",
+      "1. 该文献主要研究的问题是：叶酸治疗与癌症结局及全因死亡率之间的关系，尤其是在经过长期随访后，叶酸治疗是否与癌症结局和全因死亡率相关。\n",
+      "2. 该文献采用的分析方法包括：使用Kaplan-Meier法构建生存曲线、log-rank检验分析组间生存差异、Cox比例风险回归模型估计风险比等。\n",
+      "3. 该文献的主要结论是：在挪威没有叶酸强化食品的情况下，叶酸加维生素$\\mathsf{B}_{12}$治疗与缺血性心脏病患者的癌症结局和全因死亡率增加有关。\n",
+      "\n",
+      "**参考文献**\n",
+      "- [VD] In2Norwegianhomocysteine-lowering trialsamongpatientswithischemicheart disease, there was a statistically nonsignificantincreaseincancerincidenceinthe groupsassignedtofolicacidtreatment.15,16 Our study was performed to explore whetherfolicacidtreatmentwasassociatedwithcanceroutcomesandall-cause mortality after extended follow-up.\n",
+      "- [VD] Survivalcurveswereconstructedusing theKaplan-Meiermethodanddifferences insurvivalbetweengroupswereanalyzed usingthelog-ranktest.Estimatesofhazard ratios (HRs) with $95\\%$ CIs were obtainedbyusingCoxproportionalhazards regressionmodelsstratifiedbytrial.\n",
+      "- [VD] Conclusion Treatment with folic acid plus vitamin $\\mathsf{B}_{12}$ was associated with increased cancer outcomes and all-cause mortality in patients with ischemic heart disease in Norway, where there is no folic acid fortification of foods.\n"
+     ]
+    }
+   ],
+   "source": [
+    "resp = rag.query(prompt1, param=QueryParam(mode=\"mix\", top_k=5))\n",
+    "print(resp)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4e5bfad24cb721a8",
+   "metadata": {},
+   "source": "#### split by character only"
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "44e2992dc95f8ce0",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-01-09T03:47:40.988796Z",
+     "start_time": "2025-01-09T03:47:40.982648Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "WORKING_DIR = \"../../llm_rag/paper_db/R000088_test2\"\n",
+    "if not os.path.exists(WORKING_DIR):\n",
+    "    os.mkdir(WORKING_DIR)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "62c63385d2d973d5",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-01-09T03:51:39.951329Z",
+     "start_time": "2025-01-09T03:49:15.218976Z"
     }
    },
    "outputs": [
@@ -649,28 +942,292 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
+      "INFO:lightrag:Logger initialized for working directory: ../../llm_rag/paper_db/R000088_test2\n",
+      "INFO:lightrag:Load KV llm_response_cache with 0 data\n",
+      "INFO:lightrag:Load KV full_docs with 0 data\n",
+      "INFO:lightrag:Load KV text_chunks with 0 data\n",
+      "INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../../llm_rag/paper_db/R000088_test2/vdb_entities.json'} 0 data\n",
+      "INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../../llm_rag/paper_db/R000088_test2/vdb_relationships.json'} 0 data\n",
+      "INFO:nano-vectordb:Init {'embedding_dim': 4096, 'metric': 'cosine', 'storage_file': '../../llm_rag/paper_db/R000088_test2/vdb_chunks.json'} 0 data\n",
+      "INFO:lightrag:Loaded document status storage with 0 records\n",
+      "INFO:lightrag:Processing 1 new unique documents\n",
+      "Processing batch 1:   0%|          | 0/1 [00:00<?, ?it/s]INFO:lightrag:Inserting 12 vectors to chunks\n",
+      "\n",
+      "Generating embeddings:   0%|          | 0/1 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "\n",
+      "Generating embeddings: 100%|██████████| 1/1 [00:02<00:00,  2.95s/batch]\u001b[A\n",
+      "\n",
+      "Extracting entities from chunks:   0%|          | 0/12 [00:00<?, ?chunk/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠙ Processed 1 chunks, 0 entities(duplicated), 0 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:   8%|▊         | 1/12 [00:03<00:43,  3.93s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
-      "INFO:lightrag:kw_prompt result:\n"
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{\n",
-      "  \"high_level_keywords\": [\"英文学术研究论文分析\", \"关键信息提取\", \"深入分析\"],\n",
-      "  \"low_level_keywords\": [\"研究队列\", \"队列名称\", \"队列开展国家\", \"性别分布\", \"年龄分布\", \"队列研究时间线\", \"实际参与研究人数\"]\n",
-      "}\n"
+      "⠹ Processed 2 chunks, 8 entities(duplicated), 8 relations(duplicated)\r"
      ]
     },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  17%|█▋        | 2/12 [00:29<02:44, 16.46s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠸ Processed 3 chunks, 17 entities(duplicated), 15 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  25%|██▌       | 3/12 [00:30<01:25,  9.45s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠼ Processed 4 chunks, 27 entities(duplicated), 22 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  33%|███▎      | 4/12 [00:39<01:16,  9.52s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠴ Processed 5 chunks, 36 entities(duplicated), 33 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  42%|████▏     | 5/12 [00:40<00:43,  6.24s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠦ Processed 6 chunks, 49 entities(duplicated), 42 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  50%|█████     | 6/12 [00:49<00:43,  7.33s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠧ Processed 7 chunks, 62 entities(duplicated), 65 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  58%|█████▊    | 7/12 [01:05<00:50, 10.05s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠇ Processed 8 chunks, 81 entities(duplicated), 90 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  67%|██████▋   | 8/12 [01:23<00:50, 12.69s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠏ Processed 9 chunks, 99 entities(duplicated), 117 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  75%|███████▌  | 9/12 [01:32<00:34, 11.54s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠋ Processed 10 chunks, 123 entities(duplicated), 140 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  83%|████████▎ | 10/12 [01:48<00:25, 12.79s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠙ Processed 11 chunks, 158 entities(duplicated), 174 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks:  92%|█████████▏| 11/12 [02:03<00:13, 13.50s/chunk]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⠹ Processed 12 chunks, 194 entities(duplicated), 221 relations(duplicated)\r"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Extracting entities from chunks: 100%|██████████| 12/12 [02:13<00:00, 11.15s/chunk]\u001b[A\n",
+      "INFO:lightrag:Inserting entities into storage...\n",
+      "\n",
+      "Inserting entities: 100%|██████████| 170/170 [00:00<00:00, 11610.25entity/s]\n",
+      "INFO:lightrag:Inserting relationships into storage...\n",
+      "\n",
+      "Inserting relationships: 100%|██████████| 218/218 [00:00<00:00, 15913.51relationship/s]\n",
+      "INFO:lightrag:Inserting 170 vectors to entities\n",
+      "\n",
+      "Generating embeddings:   0%|          | 0/6 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "\n",
+      "Generating embeddings:  17%|█▋        | 1/6 [00:01<00:05,  1.10s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "\n",
+      "Generating embeddings:  33%|███▎      | 2/6 [00:02<00:04,  1.07s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "\n",
+      "Generating embeddings:  50%|█████     | 3/6 [00:02<00:02,  1.33batch/s]\u001b[A\n",
+      "Generating embeddings:  67%|██████▋   | 4/6 [00:02<00:01,  1.67batch/s]\u001b[A\n",
+      "Generating embeddings:  83%|████████▎ | 5/6 [00:03<00:00,  1.95batch/s]\u001b[A\n",
+      "Generating embeddings: 100%|██████████| 6/6 [00:03<00:00,  1.66batch/s]\u001b[A\n",
+      "INFO:lightrag:Inserting 218 vectors to relationships\n",
+      "\n",
+      "Generating embeddings:   0%|          | 0/7 [00:00<?, ?batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "\n",
+      "Generating embeddings:  14%|█▍        | 1/7 [00:01<00:10,  1.74s/batch]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "\n",
+      "Generating embeddings:  29%|██▊       | 2/7 [00:02<00:05,  1.04s/batch]\u001b[A\n",
+      "Generating embeddings:  43%|████▎     | 3/7 [00:02<00:02,  1.35batch/s]\u001b[A\n",
+      "Generating embeddings:  57%|█████▋    | 4/7 [00:03<00:01,  1.69batch/s]\u001b[AINFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "\n",
+      "Generating embeddings:  71%|███████▏  | 5/7 [00:03<00:01,  1.96batch/s]\u001b[A\n",
+      "Generating embeddings:  86%|████████▌ | 6/7 [00:03<00:00,  2.17batch/s]\u001b[A\n",
+      "Generating embeddings: 100%|███���██████| 7/7 [00:04<00:00,  1.68batch/s]\u001b[A\n",
+      "INFO:lightrag:Writing graph with 174 nodes, 218 edges\n",
+      "Processing batch 1: 100%|██████████| 1/1 [02:24<00:00, 144.69s/it]\n"
+     ]
+    }
+   ],
+   "source": [
+    "rag = LightRAG(\n",
+    "    working_dir=WORKING_DIR,\n",
+    "    llm_model_func=llm_model_func,\n",
+    "    embedding_func=EmbeddingFunc(\n",
+    "        embedding_dim=4096, max_token_size=8192, func=embedding_func\n",
+    "    ),\n",
+    "    chunk_token_size=512,\n",
+    ")\n",
+    "\n",
+    "# rag.insert(content)\n",
+    "rag.insert(content, split_by_character=\"\\n#\", split_by_character_only=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "3c7aa9836d8d43c7",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-01-09T03:52:37.000418Z",
+     "start_time": "2025-01-09T03:52:09.933584Z"
+    }
+   },
+   "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
-      "INFO:lightrag:Local query uses 60 entites, 38 relations, 6 text units\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
+      "INFO:lightrag:Local query uses 5 entites, 3 relations, 2 text units\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/embeddings \"HTTP/1.1 200 OK\"\n",
-      "INFO:lightrag:Global query uses 72 entites, 60 relations, 4 text units\n",
+      "INFO:lightrag:Global query uses 9 entites, 5 relations, 4 text units\n",
       "INFO:httpx:HTTP Request: POST https://ark.cn-beijing.volces.com/api/v3/chat/completions \"HTTP/1.1 200 OK\"\n"
      ]
     },
@@ -679,38 +1236,37 @@
      "output_type": "stream",
      "text": [
       "<分析>\n",
-      "- **分析对象来自哪些研究队列及是单独分析还是联合分析**：\n",
-      "    通过查找论文内容，发现文中提到“This is a combined analysis of data from 2 randomized, double-blind, placebo-controlled clinical trials (Norwegian Vitamin [NORVIT] trial15 and Western Norway B Vitamin Intervention Trial [WENBIT]16)”，明确是对两个队列的数据进行联合分析，队列名称分别为“Norwegian Vitamin (NORVIT) trial”和“Western Norway B Vitamin Intervention Trial (WENBIT)”。\n",
-      "- **队列开展的国家**：\n",
-      "    文中多次提及研究在挪威进行，如“combined analyses and extended follow-up of 2 vitamin B intervention trials among patients with ischemic heart disease in Norway”，所以确定研究开展的国家是挪威。\n",
-      "- **队列研究对象的性别分布**：\n",
-      "    从“Mean (SD) age was 62.3 (11.0) years and 23.5% of participants were women”可知，研究对象包含男性和女性，即全体。\n",
-      "- **队列收集结束时研究对象年龄分布**：\n",
-      "    已知“Mean (SD) age was 62.3 (11.0) years”是基线时年龄信息，“Median (interquartile range) duration of extended follow-up through December 31, 2007, was 78 (61 - 90) months”，由于随访的中位时间是78个月（约6.5年），所以可推算队列收集结束时研究对象年龄均值约为62.3 + 6.5 = 68.8岁（标准差仍为11.0年）。\n",
-      "- **队列研究时间线**：\n",
-      "    根据“2 randomized, double-blind, placebo-controlled clinical trials (Norwegian Vitamin [NORVIT] trial15 and Western Norway B Vitamin Intervention Trial [WENBIT]16) conducted between 1998 and 2005, and an observational posttrial follow-up through December 31, 2007”可知，队列开始收集信息时间为1998年，结束时间为2007年12月31日。\n",
-      "- **队列结束时实际参与研究人数**：\n",
-      "    由“A total of 6837 individuals were included in the combined analyses, of whom 6261 (91.6%) participated in posttrial follow-up”可知，队列结束时实际参与研究人数为6261人。\n",
+      "- **该文献主要研究的问题是什么？**\n",
+      "    - **思考过程**：通过浏览论文的标题、摘要、引言等部分，寻找关于研究目的和问题的描述。论文标题为“Cancer Incidence and Mortality After Treatment With Folic Acid and Vitamin B12”，摘要中的“Objective”部分明确指出研究目的是“To evaluate effects of treatment with B vitamins on cancer outcomes and all-cause mortality in 2 randomized controlled trials”。因此，可以确定该文献主要研究的问题是评估B族维生素治疗对两项随机对照试验中癌症结局和全因死亡率的影响。\n",
+      "- **该文献采用什么方法进行分析？**\n",
+      "    - **思考过程**：在论文的“METHODS”部分详细描述了研究方法。文中提到这是一个对两项随机、双盲、安慰剂对照临床试验（Norwegian Vitamin [NORVIT] trial和Western Norway B Vitamin Intervention Trial [WENBIT]）数据的联合分析，并进行了观察性的试验后随访。具体包括对参与者进行分组干预（不同剂量的叶酸、维生素B12、维生素B6或安慰剂），收集临床信息和血样，分析循环B族维生素、同型半胱氨酸和可替宁等指标，并进行基因分型等，还涉及到多种统计分析方法，如计算预期癌症发生率、构建生存曲线、进行Cox比例风险回归模型分析等。\n",
+      "- **该文献的主要结论是什么？**\n",
+      "    - **思考过程**：在论文的“Results”和“Conclusion”部分寻找主要结论。研究结果表明，在治疗期间，接受叶酸加维生素B12治疗的参与者血清叶酸浓度显著增加，且在后续随访中，该组癌症发病率、癌症死亡率和全因死亡率均有所上升，主要是肺癌发病率增加，而维生素B6治疗未显示出显著影响。结论部分明确指出“Treatment with folic acid plus vitamin $\\mathsf{B}_{12}$ was associated with increased cancer outcomes and all-cause mortality in patients with ischemic heart disease in Norway, where there is no folic acid fortification of foods”。\n",
       "</分析>\n",
       "\n",
       "<回答>\n",
-      "- 分析对象来自“Norwegian Vitamin (NORVIT) trial”和“Western Norway B Vitamin Intervention Trial (WENBIT)”两个研究队列，文中是对这两个队列的数据进行联合分析。\n",
-      "- 队列开展的国家是挪威。\n",
-      "- 队列研究对象的性别分布为全体。\n",
-      "- 队列收集结束时，研究对象年龄分布均值约为68.8岁，标准差为11.0年。\n",
-      "- 队列研究时间线为1998年开始收集信息/建立队列，2007年12月31日结束。\n",
-      "- 队列结束时实际参与研究人数是6261人。\n"
+      "- **主要研究问题**：评估B族维生素治疗对两项随机对照试验中癌症结局和全因死亡率的影响。\n",
+      "- **研究方法**：采用对两项随机、双盲、安慰剂对照临床试验（Norwegian Vitamin [NORVIT] trial和Western Norway B Vitamin Intervention Trial [WENBIT]）数据的联合分析，并进行观察性的试验后随访，涉及分组干预、多种指标检测以及多种统计分析方法。\n",
+      "- **主要结论**：在挪威（食品中未添加叶酸），对于缺血性心脏病患者，叶酸加维生素B12治疗与癌症结局和全因死亡率的增加有关，而维生素B6治疗未显示出显著影响。\n",
+      "\n",
+      "**参考文献**\n",
+      "- [VD] Cancer Incidence and Mortality After Treatment With Folic Acid and Vitamin B12\n",
+      "- [VD] METHODS Study Design, Participants, and Study Intervention\n",
+      "- [VD] RESULTS\n",
+      "- [VD] Conclusion\n",
+      "- [VD] Objective To evaluate effects of treatment with B vitamins on cancer outcomes and all-cause mortality in 2 randomized controlled trials.\n"
      ]
     }
    ],
    "source": [
-    "print(rag.query(prompt1, param=QueryParam(mode=\"hybrid\")))"
+    "resp = rag.query(prompt1, param=QueryParam(mode=\"mix\", top_k=5))\n",
+    "print(resp)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "fef9d06983da47af",
+   "id": "7ba6fa79a2550d10",
    "metadata": {},
    "outputs": [],
    "source": []

lightrag/lightrag.py

@@ -314,18 +314,25 @@ class LightRAG:
             "JsonDocStatusStorage": JsonDocStatusStorage,
         }
 
-    def insert(self, string_or_strings, split_by_character=None):
+    def insert(
+        self, string_or_strings, split_by_character=None, split_by_character_only=False
+    ):
         loop = always_get_an_event_loop()
         return loop.run_until_complete(
-            self.ainsert(string_or_strings, split_by_character)
+            self.ainsert(string_or_strings, split_by_character, split_by_character_only)
         )
 
-    async def ainsert(self, string_or_strings, split_by_character):
+    async def ainsert(
+        self, string_or_strings, split_by_character, split_by_character_only
+    ):
         """Insert documents with checkpoint support
 
         Args:
             string_or_strings: Single document string or list of document strings
-            split_by_character: if split_by_character is not None, split the string by character
+            split_by_character: if split_by_character is not None, split the string by character, if chunk longer than
+            chunk_size, split the sub chunk by token size.
+            split_by_character_only: if split_by_character_only is True, split the string by character only, when
+            split_by_character is None, this parameter is ignored.
         """
         if isinstance(string_or_strings, str):
             string_or_strings = [string_or_strings]
@@ -384,6 +391,7 @@ class LightRAG:
                         for dp in chunking_by_token_size(
                             doc["content"],
                             split_by_character=split_by_character,
+                            split_by_character_only=split_by_character_only,
                             overlap_token_size=self.chunk_overlap_token_size,
                             max_token_size=self.chunk_token_size,
                             tiktoken_model=self.tiktoken_model_name,

lightrag/operate.py

@@ -36,6 +36,7 @@ import time
 def chunking_by_token_size(
     content: str,
     split_by_character=None,
+    split_by_character_only=False,
     overlap_token_size=128,
     max_token_size=1024,
     tiktoken_model="gpt-4o",
@@ -45,21 +46,26 @@ def chunking_by_token_size(
     if split_by_character:
         raw_chunks = content.split(split_by_character)
         new_chunks = []
-        for chunk in raw_chunks:
-            _tokens = encode_string_by_tiktoken(chunk, model_name=tiktoken_model)
-            if len(_tokens) > max_token_size:
-                for start in range(
-                    0, len(_tokens), max_token_size - overlap_token_size
-                ):
-                    chunk_content = decode_tokens_by_tiktoken(
-                        _tokens[start : start + max_token_size],
-                        model_name=tiktoken_model,
-                    )
-                    new_chunks.append(
-                        (min(max_token_size, len(_tokens) - start), chunk_content)
-                    )
-            else:
+        if split_by_character_only:
+            for chunk in raw_chunks:
+                _tokens = encode_string_by_tiktoken(chunk, model_name=tiktoken_model)
                 new_chunks.append((len(_tokens), chunk))
+        else:
+            for chunk in raw_chunks:
+                _tokens = encode_string_by_tiktoken(chunk, model_name=tiktoken_model)
+                if len(_tokens) > max_token_size:
+                    for start in range(
+                        0, len(_tokens), max_token_size - overlap_token_size
+                    ):
+                        chunk_content = decode_tokens_by_tiktoken(
+                            _tokens[start : start + max_token_size],
+                            model_name=tiktoken_model,
+                        )
+                        new_chunks.append(
+                            (min(max_token_size, len(_tokens) - start), chunk_content)
+                        )
+                else:
+                    new_chunks.append((len(_tokens), chunk))
         for index, (_len, chunk) in enumerate(new_chunks):
             results.append(
                 {