Extract text from pdfs using tesseract (per-page OCR).
504597
def extract_tesseract(self, filename, **kwargs):
        """"""
        temp_dir = mkdtemp()
        base = os.path.join(temp_dir, 'conv')
        contents = []
        try:
            stdout, _ = self.run(['pdftoppm', filename, base])

            for page in sorted(os.listdir(temp_dir)):
                page_path = os.path.join(temp_dir, page)
                page_content = TesseractParser().extract(page_path, **kwargs)
                contents.append(page_content)
            return six.b('').join(contents)
        finally:
            shutil.rmtree(temp_dir)
782625
def extract_images(text):
    
    extensions = [ExtractImagesExtension()]
    mkd = markdown.Markdown(extensions=extensions)
    html = mkd.convert(text)
    return mkd.images
237370
def clean(self, text):
        """"""
        return ''.join([c for c in text if c in self.alphabet])
423965
def to_text(self, digest, blob, mime_type):
        
        # Special case, for now (XXX).
        if mime_type.startswith("image/"):
            return ""

        cache_key = "txt:" + digest

        text = self.cache.get(cache_key)
        if text:
            return text

        # Direct conversion possible
        for handler in self.handlers:
            if handler.accept(mime_type, "text/plain"):
                text = handler.convert(blob)
                self.cache[cache_key] = text
                return text

        # Use PDF as a pivot format
        pdf = self.to_pdf(digest, blob, mime_type)
        for handler in self.handlers:
            if handler.accept("application/pdf", "text/plain"):
                text = handler.convert(pdf)
                self.cache[cache_key] = text
                return text

        raise HandlerNotFound(f"No handler found to convert from {mime_type} to text")
224057
def parse_text(document, container, element):
    ""

    txt = None

    alternate = element.find(_name('{{{mc}}}AlternateContent'))

    if alternate is not None:
        parse_alternate(document, container, alternate)

    br = element.find(_name('{{{w}}}br'))

    if br is not None:
        if _name('{{{w}}}type') in br.attrib:
            _type = br.attrib[_name('{{{w}}}type')]
            brk = doc.Break(_type)
        else:
            brk = doc.Break()

        container.elements.append(brk)

    t = element.find(_name('{{{w}}}t'))

    if t is not None:
        txt = doc.Text(t.text)
        txt.parent = container

        container.elements.append(txt)

    rpr = element.find(_name('{{{w}}}rPr'))

    if rpr is not None:
        # Notice it is using txt as container
        parse_previous_properties(document, txt, rpr)

    for r in element.findall(_name('{{{w}}}r')):
        parse_text(document, container, r)

    foot = element.find(_name('{{{w}}}footnoteReference'))

    if foot is not None:
        parse_footnote(document, container, foot)

    end = element.find(_name('{{{w}}}endnoteReference'))

    if end is not None:
        parse_endnote(document, container, end)

    sym = element.find(_name('{{{w}}}sym'))

    if sym is not None:
        _font = sym.attrib[_name('{{{w}}}font')]
        _char = sym.attrib[_name('{{{w}}}char')]

        container.elements.append(doc.Symbol(font=_font, character=_char))

    image = element.find(_name('{{{w}}}drawing'))

    if image is not None:
        parse_drawing(document, container, image)

    refe = element.find(_name('{{{w}}}commentReference'))

    if refe is not None:
        _m = doc.Comment(refe.attrib[_name('{{{w}}}id')], 'reference')
        container.elements.append(_m)

    return
415911
function extractDescription(texts) {
  let document = '';
  texts.forEach(text => {
    document += text.description || '';
  });
  return document.toLowerCase();
}
433019
def extract_features(self, text):
        
        # Feature extractor may take one or two arguments
        try:
            return self.feature_extractor(text, self.train_set)
        except (TypeError, AttributeError):
            return self.feature_extractor(text)
628241
def select_ocr_image(infiles, output_file, log, context):
    """"""

    image = infiles[0]
    options = context.get_options()
    pageinfo = get_pageinfo(image, context)

    with Image.open(image) as im:
        from PIL import ImageColor
        from PIL import ImageDraw

        white = ImageColor.getcolor('#ffffff', im.mode)
        # pink = ImageColor.getcolor('#ff0080', im.mode)
        draw = ImageDraw.ImageDraw(im)

        xres, yres = im.info['dpi']
        log.debug('resolution %r %r', xres, yres)

        if not options.force_ocr:
            # Do not mask text areas when forcing OCR, because we need to OCR
            # all text areas
            mask = None  # Exclude both visible and invisible text from OCR
            if options.redo_ocr:
                mask = True  # Mask visible text, but not invisible text

            for textarea in pageinfo.get_textareas(visible=mask, corrupt=None):
                # Calculate resolution based on the image size and page dimensions
                # without regard whatever resolution is in pageinfo (may differ or
                # be None)
                bbox = [float(v) for v in textarea]
                xscale, yscale = float(xres) / 72.0, float(yres) / 72.0
                pixcoords = [
                    bbox[0] * xscale,
                    im.height - bbox[3] * yscale,
                    bbox[2] * xscale,
                    im.height - bbox[1] * yscale,
                ]
                pixcoords = [int(round(c)) for c in pixcoords]
                log.debug('blanking %r', pixcoords)
                draw.rectangle(pixcoords, fill=white)
                # draw.rectangle(pixcoords, outline=pink)

        if options.mask_barcodes or options.threshold:
            pix = leptonica.Pix.frompil(im)
            if options.threshold:
                pix = pix.masked_threshold_on_background_norm()
            if options.mask_barcodes:
                barcodes = pix.locate_barcodes()
                for barcode in barcodes:
                    decoded, rect = barcode
                    log.info('masking barcode %s %r', decoded, rect)
                    draw.rectangle(rect, fill=white)
            im = pix.topil()

        del draw
        # Pillow requires integer DPI
        dpi = round(xres), round(yres)
        im.save(output_file, dpi=dpi)
751742
def text_to_data(self, text, elt, ps):
        '''
        '''
        if text is None:
            return None
        
        m = self.lex_pattern.match(text)
        if not m:
            raise EvaluateException('Bad Gregorian: %s' %text, ps.Backtrace(elt))
        try:
            retval = _dict_to_tuple(m.groupdict())
        except ValueError, e:
            #raise EvaluateException(str(e))
            raise
        
        if self.pyclass is not None:
            return self.pyclass(retval)
        return retval
1082467
def _extract_docs_description(self):
        """"""
        # FIXME: the indentation of descriptions is lost
        data = '\n'.join([d.rstrip().replace(self.docs['out']['spaces'], '', 1) for d in self.docs['in']['raw'].splitlines()])
        if self.dst.style['in'] == 'groups':
            idx = self.dst.get_group_index(data)
        elif self.dst.style['in'] == 'google':
            lines = data.splitlines()
            line_num = self.dst.googledoc.get_next_section_start_line(lines)
            if line_num == -1:
                idx = -1
            else:
                idx = len('\n'.join(lines[:line_num]))
        elif self.dst.style['in'] == 'numpydoc':
            lines = data.splitlines()
            line_num = self.dst.numpydoc.get_next_section_start_line(lines)
            if line_num == -1:
                idx = -1
            else:
                idx = len('\n'.join(lines[:line_num]))
        elif self.dst.style['in'] == 'unknown':
            idx = -1
        else:
            idx = self.dst.get_elem_index(data)
        if idx == 0:
            self.docs['in']['desc'] = ''
        elif idx == -1:
            self.docs['in']['desc'] = data
        else:
            self.docs['in']['desc'] = data[:idx]
532194
def convert(document):
    """"""
    raw_tokens = []
    curpos = 0
    text_spans = []
    all_labels = []
    sent_spans = []
    word_texts = []
    for sentence in document:
        startpos = curpos
        for idx, (text, label) in enumerate(sentence):
            raw_tokens.append(text)
            word_texts.append(text)
            all_labels.append(label)
            text_spans.append((curpos, curpos+len(text)))
            curpos += len(text)
            if idx < len(sentence) - 1:
                raw_tokens.append(' ')
                curpos += 1
        sent_spans.append((startpos, curpos))
        raw_tokens.append('\n')
        curpos += 1
    return {
        TEXT: ''.join(raw_tokens),
        WORDS: [{TEXT: text, START: start, END: end, LABEL: label} for text, (start, end), label in zip(word_texts, text_spans, all_labels)],
        SENTENCES: [{START: start, END:end} for start, end in sent_spans]
    }
449740
def analyze_pages(file_name, char_margin=1.0):
    
    log = logging.getLogger(__name__)
    # Open a PDF file.
    with open(os.path.realpath(file_name), "rb") as fp:
        # Create a PDF parser object associated with the file object.
        parser = PDFParser(fp)
        # Create a PDF document object that stores the document structure.
        # Supply the password for initialization.
        document = PDFDocument(parser, password="")
        # Create a PDF resource manager object that stores shared resources.
        rsrcmgr = PDFResourceManager()
        # Set parameters for analysis.
        laparams = LAParams(
            char_margin=char_margin, word_margin=0.1, detect_vertical=True
        )
        # Create a PDF page aggregator object.
        device = CustomPDFPageAggregator(rsrcmgr, laparams=laparams)
        # Create a PDF interpreter object.
        interpreter = PDFPageInterpreter(rsrcmgr, device)
        # Process each page contained in the document.
        for page_num, page in enumerate(PDFPage.create_pages(document)):
            try:
                interpreter.process_page(page)
            except OverflowError as oe:
                log.exception(
                    "{}, skipping page {} of {}".format(oe, page_num, file_name)
                )
                continue
            layout = device.get_result()
            yield layout
105931
def process_Text( self, text, **kwargs ):
        ''' 
        ''' 
        mrf_lines = convert_Text_to_mrf( text )
        return self.process_mrf_lines( mrf_lines, **kwargs )
488939
def extractFont(self, xref=0, info_only=0):
        """"""
        if self.isClosed or self.isEncrypted:
            raise ValueError("operation illegal for closed / encrypted doc")

        return _fitz.Document_extractFont(self, xref, info_only)
914241
def path_to_text(self, path):
        '''
        

        '''
        rsrcmgr = PDFResourceManager()
        retstr = StringIO()
        codec = 'utf-8'
        laparams = LAParams()
        device = TextConverter(rsrcmgr, retstr, codec=codec, laparams=laparams)
        fp = open(path, 'rb')
        interpreter = PDFPageInterpreter(rsrcmgr, device)
        password = ""
        maxpages = 0
        caching = True
        pagenos = set()

        pages_data = PDFPage.get_pages(
            fp,
            pagenos,
            maxpages=maxpages,
            password=password,
            caching=caching,
            check_extractable=True
        )

        for page in pages_data:
            interpreter.process_page(page)

        text = retstr.getvalue()
        text = text.replace("\n", "")

        fp.close()
        device.close()
        retstr.close()
        return text
804347
private void extractTablesFromAPage(ArrayList<TextPiece> linesOfAPage,
	    ArrayList<Table> tables, ArrayList<TextPiece> wordsOfAPage, int i,
	    File pdfFile, ArrayList<ArrayList<TextPiece>> linesByPage) {
	if (linesOfAPage.size() > 30) {
	    /*
	     * Checks for table keywords
	     */
	    boolean tableKwdExist = false;
	    int prevTableEndIndex = 0;
	    for (TextPiece t : linesOfAPage) {
		String textWithoutSpace = t.getText().replace(" ", "");
		if (getMatchedTabledKeyword(textWithoutSpace) != null) {
		    tableKwdExist = true;
		    break;
		}
	    }

	    /*
	     * TODO [improve]: keywords are searched twice, no need to do the
	     * first search in the future
	     */
	    if (tableKwdExist == true) {
		for (int k = 0; k < linesOfAPage.size();) {
		    TextPiece line = linesOfAPage.get(k);
		    TextPiece prevLine = (k == 0) ? null : linesOfAPage
			    .get(k - 1);
		    String textWithoutSpace = line.getText().replace(" ", "");
		    String keyword = getMatchedTabledKeyword(textWithoutSpace);

		    if (keyword != null
			    && (textWithoutSpace.startsWith("tableofcontent") == false)
			    && (textWithoutSpace.startsWith("TABLEOFCONTENT") == false)
			    && (textWithoutSpace.startsWith("TableofContent") == false)) {
			char lastCharInKwd = keyword
				.charAt(keyword.length() - 1);
			String oldKwd = line.getText().substring(0,
				line.getText().indexOf(lastCharInKwd) + 1);
			String newLine = "Table"
				+ line.getText()
					.substring(
						line.getText().indexOf(
							lastCharInKwd) + 1,
						line.getText().length());
			// linesOfAPage.get(k).setText(newLine);

			if (prevLine != null && line.getY() == prevLine.getY())
			    k++;
			else {
			    /*
			     * try to extract this table
			     */
			    TableCandidate tc = new TableCandidate();
			    tc.setKeyword(keyword);
			    // tc.setXMLDoc(xml_doc); // to add all table
			    // metadata elements in TableCandidate

			    k = extractOneTable(k, linesOfAPage, keyword, tc,
				    wordsOfAPage, i, prevTableEndIndex,
				    linesByPage);
			    if (tc.isValid()) {
				if (tc.isTopCaption())
				    prevTableEndIndex = tc.getBodyEndLine();
				else
				    prevTableEndIndex = tc.getCaptionEndLine();
				if (Config.DEBUG_MODE)
				    tc.print();
				String metadata_StructureLevel = tc
					.getMetadata_StructureLevel();
				String tableMeta_thisDoc = m_docInfo
					.getTableMetaInThisDoc();
				String tableMetaHighLevel = tc
					.getMetadata_HighLevel();
				String latestMetaData = tableMeta_thisDoc
					+ tableMetaHighLevel
					+ metadata_StructureLevel;
				m_docInfo.setTableMetadata(latestMetaData);

				/*
				 * Generates the extracted table list, then XML
				 * output from each table
				 */
				// System.out.println(Integer.toString(i)+" Adding");
				Table oneTable = new Table();
				oneTable.setValidTable(tc);
				tables.add(oneTable);

				oneTable.setPageNumber(i + 1);
			    }
			}
		    } else
			k++;
		}
	    } else {// TODO: log
	    }
	} else {
	    /*
	     * Impossible to contain tables because of few texts in the page
	     */
	}
    }
834699
def process_document(self, doc: Document):
        

        text_segments = doc.select_segments("lexisnexis.doc_description")
        for text_segment in text_segments:
            split_sentences = doc.extract(self.sentence_extractor, text_segment)
            doc.store(split_sentences, 'split_sentences')
        # for t, u in zip(text_to_be_split, units_of_text):
        #     split_sentences = doc.extract(self.sentence_extractor, t)
        #     u.store(split_sentences, "split_sentences")
        return list()