File size: 2,514 Bytes
8352b84 120e951 8352b84 120e951 8352b84 120e951 8352b84 120e951 8352b84 120e951 8352b84 120e951 8352b84 120e951 8352b84 120e951 8352b84 120e951 8352b84 120e951 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 |
#!/usr/bin/env python
"""
Example script demonstrating the basic usage of MinerU parser
This example shows how to:
1. Parse different types of documents (PDF, images, office documents)
2. Use different parsing methods
3. Display document statistics
"""
import os
import argparse
from lightrag.mineru_parser import MineruParser
def parse_document(
file_path: str, output_dir: str = None, method: str = "auto", stats: bool = False
):
"""
Parse a document using MinerU parser
Args:
file_path: Path to the document
output_dir: Output directory for parsed results
method: Parsing method (auto, ocr, txt)
stats: Whether to display content statistics
"""
try:
# Parse the document
content_list, md_content = MineruParser.parse_document(
file_path=file_path, parse_method=method, output_dir=output_dir
)
# Display statistics if requested
if stats:
print("\nDocument Statistics:")
print(f"Total content blocks: {len(content_list)}")
# Count different types of content
content_types = {}
for item in content_list:
content_type = item.get("type", "unknown")
content_types[content_type] = content_types.get(content_type, 0) + 1
print("\nContent Type Distribution:")
for content_type, count in content_types.items():
print(f"- {content_type}: {count}")
return content_list, md_content
except Exception as e:
print(f"Error parsing document: {str(e)}")
return None, None
def main():
"""Main function to run the example"""
parser = argparse.ArgumentParser(description="MinerU Parser Example")
parser.add_argument("file_path", help="Path to the document to parse")
parser.add_argument("--output", "-o", help="Output directory path")
parser.add_argument(
"--method",
"-m",
choices=["auto", "ocr", "txt"],
default="auto",
help="Parsing method (auto, ocr, txt)",
)
parser.add_argument(
"--stats", action="store_true", help="Display content statistics"
)
args = parser.parse_args()
# Create output directory if specified
if args.output:
os.makedirs(args.output, exist_ok=True)
# Parse document
content_list, md_content = parse_document(
args.file_path, args.output, args.method, args.stats
)
if __name__ == "__main__":
main()
|