Hugging Face's logo

cocoshe
/
diskann_files

Model card Files
xet
Community
cocoshe commited on
Commit
103f07d
·
1 Parent(s): 37a0aea

init

Browse files
Files changed (20) hide show
  1. .gitattributes +2 -0
  2. bigann_learn.bin +3 -0
  3. bigann_query.bin +3 -0
  4. bigann_query_learn_gt100 +3 -0
  5. disk_sift_bigann_learn_R64_L200_disk.index +3 -0
  6. disk_sift_bigann_learn_R64_L200_disk.index_centroids.bin +3 -0
  7. disk_sift_bigann_learn_R64_L200_disk.index_medoids.bin +3 -0
  8. disk_sift_bigann_learn_R64_L200_pq_compressed.bin +3 -0
  9. disk_sift_bigann_learn_R64_L200_pq_pivots.bin +3 -0
  10. disk_sift_bigann_learn_R64_L200_sample_data.bin +3 -0
  11. disk_sift_bigann_learn_R64_L200_sample_ids.bin +3 -0
  12. exp.txt +27 -0
  13. res_20_dists_float.bin +3 -0
  14. res_20_idx_uint32.bin +3 -0
  15. res_40_dists_float.bin +3 -0
  16. res_40_idx_uint32.bin +3 -0
  17. res_60_dists_float.bin +3 -0
  18. res_60_idx_uint32.bin +3 -0
  19. res_80_dists_float.bin +3 -0
  20. res_80_idx_uint32.bin +3 -0
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ disk_sift_bigann_learn_R64_L200_disk.index filter=lfs diff=lfs merge=lfs -text
37
+ bigann_query_learn_gt100 filter=lfs diff=lfs merge=lfs -text
bigann_learn.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1427191d2d9210a1ea44e734ee4b311fa3e08ddff112dfbdb136f0758ab96997
3
+ size 12800000008
bigann_query.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eca755831fc9a8004e14886df48b81109a8ed3bfc6b632509c93c0460a30d552
3
+ size 1280008
bigann_query_learn_gt100 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da11c1be87d8f1cf6b7ca03a0ae5fe2a43b30429e1c4c1aaea82e506e9c0396b
3
+ size 8000008
disk_sift_bigann_learn_R64_L200_disk.index ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:829ca5d7fbd68f23372e43b15555b338ef3b623d103bc1cf5865c044cb9f4aa7
3
+ size 40960004096
disk_sift_bigann_learn_R64_L200_disk.index_centroids.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:56d0031e2dcb3e8ed3db0b6ea49731e13308270f4d63aa0c09aa7bd47b9abdf8
3
+ size 1544
disk_sift_bigann_learn_R64_L200_disk.index_medoids.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3485d750a337fc5a95bf684eed2b46a990ae17a7e3fa9b7b46976ef6fe4caf9e
3
+ size 20
disk_sift_bigann_learn_R64_L200_pq_compressed.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f510ab6ad933ffbcccee908daa53bbf4b17f88afbba5eb4e1cf2879205b17851
3
+ size 12800000008
disk_sift_bigann_learn_R64_L200_pq_pivots.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:305549de6594e7962255ad634580e36278591485bf748f34e273b76e15f23edd
3
+ size 136220
disk_sift_bigann_learn_R64_L200_sample_data.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b8d473585d9c1455678ad1c76c3d8901fd78d65a6897a54ec5e48e44b6832227
3
+ size 12792840
disk_sift_bigann_learn_R64_L200_sample_ids.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:974616c5733ee8436725b2d3e5bcec83b03336c79b8b5ecf8819119aad7bbf47
3
+ size 399784
exp.txt ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ ./apps/search_disk_index --data_type uint8 --dist_fn l2 --index_path_prefix data/disk_sift_bigann_learn_R64_L200 --query_file data/bigann_query.bin --gt_file data/bigann_query_learn_gt100 -K 10 -L 20 40 60 80 --num_nodes_to_cache 500000 --result_path data/res
3
+ ./apps/search_disk_index --data_type uint8 --dist_fn l2 --index_path_prefix data_backup/disk_sift_bigann_learn_R64_L200 --query_file data_backup/bigann_query.bin --gt_file data_backup/bigann_query_learn_gt100 -K 10 -L 20 40 60 80 --num_nodes_to_cache 500000 --result_path data_backup/res
4
+ ./apps/search_disk_index --data_type uint8 --dist_fn l2 --index_path_prefix data_backup_clean_test/disk_sift_bigann_learn_R64_L200 --query_file data_backup_clean_test/bigann_query.bin --gt_file data_backup_clean_test/bigann_query_learn_gt100 -K 10 -L 20 40 60 80 --num_nodes_to_cache 500000 --result_path data_backup_clean_test/res
5
+
6
+ 13G+内存
7
+ 压缩数据12G
8
+ 图索引39G
9
+ L Beamwidth QPS Mean Latency 99.9 Latency Mean IOs Mean IO (us) CPU (s) Recall@10
10
+ ===================================================================================================================================
11
+ 20 2 62.17 1027376.45 1966731.00 26.05 1025264.30 1982.37 82.89
12
+ 40 2 34.98 1826255.43 2891463.00 44.71 1823065.03 2972.43 91.40
13
+ 60 2 24.01 2661614.25 3849402.00 63.87 2658473.16 2867.13 94.91
14
+ 80 2 18.37 3478590.97 4610630.00 83.25 3474978.81 3277.12 96.59
15
+ 100 2 15.68 4075180.62 5313749.00 102.84 4068131.79 6632.12 97.55
16
+
17
+
18
+
19
+
20
+ ./apps/search_disk_index --data_type uint8 --dist_fn l2 --index_path_prefix data/disk_sift_bigann_learn_R64_L200 --query_file data/bigann_query.bin --gt_file data/bigann_query_learn_gt100 -K 10 -L 20 40 60 80 --num_nodes_to_cache 100000000 --result_path data/res -T 16
21
+ 22G内存
22
+ L Beamwidth QPS Mean Latency 99.9 Latency Mean IOs Mean IO (us) CPU (s) Recall@10
23
+ ===================================================================================================================================
24
+ 20 2 50.12 318922.83 646162.00 22.46 316961.21 1740.45 82.89
25
+ 40 2 28.68 557495.46 894321.00 39.55 555376.91 1859.22 91.40
26
+ 60 2 19.65 813717.81 1189216.00 57.09 810930.56 2359.29 94.91
27
+ 80 2 14.47 1105305.05 1585758.00 74.80 1101771.55 3034.36 96.59
res_20_dists_float.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f7fedb19ea43112b7bc303acc3f586aa67b080270cb2e217e2ec5130e6f43dd7
3
+ size 400008
res_20_idx_uint32.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ded0c108e72b60b08d6902a4078877c269f7168b9f04caf0ab7970d75784bd80
3
+ size 400008
res_40_dists_float.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f09e62495f38a96e9ed6631ebeed4bafa2dce15978cab8552b2361bc8873fdf8
3
+ size 400008
res_40_idx_uint32.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d5631dd48adf5e8d45d8f21492b1ec440d4edd3cd1c9771b728684daf1bc8b5
3
+ size 400008
res_60_dists_float.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab26ad8e94dfdaf951002a606c2d5067ca761c3bcb19fdbac06d31b441e798a9
3
+ size 400008
res_60_idx_uint32.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e6849c344d00d49a51a0b843104fb791b66b71d97d1960cd7b6c81a8ceb83141
3
+ size 400008
res_80_dists_float.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:566a642763dd698a7162a0289709cd7842178a154963bb32b4c03ed6b8987c8c
3
+ size 400008
res_80_idx_uint32.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca20949d162b0a856a84b2d46e2ac98e8cbd467996d557a78930f4dda616a928
3
+ size 400008