Converts Microsoft Word DOC files to text.
517899
def convert_doc_to_text(filename: str = None,
                        blob: bytes = None,
                        config: TextProcessingConfig = _DEFAULT_CONFIG) -> str:
    
    antiword = tools['antiword']
    if antiword:
        if filename:
            return get_cmd_output(antiword, '-w', str(config.width), filename)
        else:
            return get_cmd_output_from_stdin(blob, antiword, '-w',
                                             str(config.width), '-')
    else:
        raise AssertionError("No DOC-reading tool available")
1003105
def convert(self, text):
        """"""
        # Main function. The order in which other subs are called here is
        # essential. Link and image substitutions need to happen before
        # _EscapeSpecialChars(), so that any *'s or _'s in the <a>
        # and <img> tags get encoded.

        # Clear the global hashes. If we don't clear these, you get conflicts
        # from other articles when generating a page which contains more than
        # one article (e.g. an index page that shows the N most recent
        # articles):
        self.reset()

        if not isinstance(text, unicode):
            #TODO: perhaps shouldn't presume UTF-8 for string input?
            text = unicode(text, 'utf-8')

        if self.use_file_vars:
            # Look for emacs-style file variable hints.
            emacs_vars = self._get_emacs_vars(text)
            if "markdown-extras" in emacs_vars:
                splitter = re.compile("[ ,]+")
                for e in splitter.split(emacs_vars["markdown-extras"]):
                    if '=' in e:
                        ename, earg = e.split('=', 1)
                        try:
                            earg = int(earg)
                        except ValueError:
                            pass
                    else:
                        ename, earg = e, None
                    self.extras[ename] = earg

        # Standardize line endings:
        text = re.sub("\r\n|\r", "\n", text)

        # Make sure $text ends with a couple of newlines:
        text += "\n\n"

        # Convert all tabs to spaces.
        text = self._detab(text)

        # Strip any lines consisting only of spaces and tabs.
        # This makes subsequent regexen easier to write, because we can
        # match consecutive blank lines with /\n+/ instead of something
        # contorted like /[ \t]*\n+/ .
        text = self._ws_only_line_re.sub("", text)

        # strip metadata from head and extract
        if "metadata" in self.extras:
            text = self._extract_metadata(text)

        text = self.preprocess(text)

        if self.safe_mode:
            text = self._hash_html_spans(text)

        # Turn block-level HTML blocks into hash entries
        text = self._hash_html_blocks(text, raw=True)

        # Strip link definitions, store in hashes.
        if "footnotes" in self.extras:
            # Must do footnotes first because an unlucky footnote defn
            # looks like a link defn:
            #   [^4]: this "looks like a link defn"
            text = self._strip_footnote_definitions(text)
        text = self._strip_link_definitions(text)

        text = self._run_block_gamut(text)

        if "footnotes" in self.extras:
            text = self._add_footnotes(text)

        text = self.postprocess(text)

        text = self._unescape_special_chars(text)

        if self.safe_mode:
            text = self._unhash_html_spans(text)

        if "nofollow" in self.extras:
            text = self._a_nofollow.sub(r'<\1 rel="nofollow"\2', text)

        text += "\n"

        rv = UnicodeWithAttrs(text)
        if "toc" in self.extras:
            rv._toc = self._toc
        if "metadata" in self.extras:
            rv.metadata = self.metadata
        return rv
21635
def set_plain_text(self, text, is_code):
        """"""

        # text is coming from utils.dochelpers.getdoc
        if type(text) is dict:
            name = text['name']
            if name:
                rst_title = ''.join(['='*len(name), '\n', name, '\n',
                                    '='*len(name), '\n\n'])
            else:
                rst_title = ''

            if text['argspec']:
                definition = ''.join(['Definition: ', name, text['argspec'],
                                      '\n'])
            else:
                definition = ''

            if text['note']:
                note = ''.join(['Type: ', text['note'], '\n\n----\n\n'])
            else:
                note = ''

            full_text = ''.join([rst_title, definition, note,
                                 text['docstring']])
        else:
            full_text = text

        self.plain_text.set_text(full_text, is_code)
        self.save_text([self.plain_text.set_text, full_text, is_code])
1035583
public static void toFile(Document doc, String path, String charset) {
		if (StrUtil.isBlank(charset)) {
			charset = doc.getXmlEncoding();
		}
		if (StrUtil.isBlank(charset)) {
			charset = CharsetUtil.UTF_8;
		}

		BufferedWriter writer = null;
		try {
			writer = FileUtil.getWriter(path, charset, false);
			write(doc, writer, charset, INDENT_DEFAULT);
		} finally {
			IoUtil.close(writer);
		}
	}
586527
def _prepare_docstrings():
    """"""
    if config.USEAUTODOC:
        source = inspect.getsource(Options)
        docstrings = source.split('"""')[3::2]
        attributes = [line.strip().split()[0] for line in source.split('\n')
                      if '_Option(' in line]
        for attribute, docstring in zip(attributes, docstrings):
            Options.__dict__[attribute].__doc__ = docstring
1051286
def markdown(text):
    """"""
    text = gfm(text)
    text = markdown_lib.markdown(text)
    return text
1248221
function(str, lex) {
    if (lex) {
      w2.plugin({
        words: lex
      });
    }
    let doc = buildText(str, w2);
    doc.tagger();
    return doc;
  }
316755
def _format_docstring(self, text):
        
        #The only complication we have here is that we don't want to break words up.
        words = text.split()
        result = []
        line = []
        cumline = 0
        for word in words:
            if len(word) + 1 + cumline < 90:
                line.append(word)
                cumline += len(word) + 1
            else:
                result.append(' '.join(line))
                cumline = len(word) + 1
                line = [word]

        return result
594709
def _cleandoc(doc):
    
    indent_length = lambda s: len(s) - len(s.lstrip(" "))
    not_empty = lambda s: s != ""

    lines = doc.split("\n")
    indent = min(map(indent_length, filter(not_empty, lines)))

    return "\n".join(s[indent:] for s in lines)
527553
def encode_document(document):
    """"""
    tree = etree.Element('document')
    etree.SubElement(tree, 'id').text = document.id
    encode_infons(tree, document.infons)
    for passage in document.passages:
        tree.append(encode_passage(passage))
    for ann in document.annotations:
        tree.append(encode_annotation(ann))
    for rel in document.relations:
        tree.append(encode_relation(rel))
    return tree
766508
def docstring_to_markdown(docstring):
    
    new_docstring_lst = []

    for idx, line in enumerate(docstring.split('\n')):
        line = line.strip()
        if set(line) in ({'-'}, {'='}):
            new_docstring_lst[idx-1] = '**%s**' % new_docstring_lst[idx-1]
        elif line.startswith('>>>'):
            line = '    %s' % line
        new_docstring_lst.append(line)

    for idx, line in enumerate(new_docstring_lst[1:]):
        if line:
            if line.startswith('Description : '):
                new_docstring_lst[idx+1] = (new_docstring_lst[idx+1]
                                            .replace('Description : ', ''))
            elif ' : ' in line:
                line = line.replace(' : ', '` : ')
                new_docstring_lst[idx+1] = '\n- `%s\n' % line
            elif '**' in new_docstring_lst[idx-1] and '**' not in line:
                new_docstring_lst[idx+1] = '\n%s' % line.lstrip()
            elif '**' not in line:
                new_docstring_lst[idx+1] = '    %s' % line.lstrip()

    clean_lst = []
    for line in new_docstring_lst:
        if set(line.strip()) not in ({'-'}, {'='}):
            clean_lst.append(line)
    return clean_lst
135232
def as_text(self, max_rows=0, sep=" | "):
        """"""
        if not max_rows or max_rows > self.num_rows:
            max_rows = self.num_rows
        omitted = max(0, self.num_rows - max_rows)
        labels = self._columns.keys()
        fmts = self._get_column_formatters(max_rows, False)
        rows = [[fmt(label, label=True) for fmt, label in zip(fmts, labels)]]
        for row in itertools.islice(self.rows, max_rows):
            rows.append([f(v, label=False) for v, f in zip(row, fmts)])
        lines = [sep.join(row) for row in rows]
        if omitted:
            lines.append('... ({} rows omitted)'.format(omitted))
        return '\n'.join([line.rstrip() for line in lines])
242314
def write_text(_command, txt_file):
    """"""
    command = _command.strip()
    with open(txt_file, 'w') as txt:
        txt.writelines(command)
913502
function(doc) {
  if (doc.params) {
    _.each(doc.params, function(param) {
      replaceNewLines(param, 'description');
    });
  }

  // Replace new lines in the return and params descriptions.
  var returns = doc.returns;
  if (returns) {
    replaceNewLines(returns, 'description');
  }
}
995132
def _get_text(self):
        
        boxes = self.boxes
        txt = []
        for line in boxes:
            txt_line = u""
            for box in line.word_boxes:
                txt_line += u" " + box.content
            txt.append(txt_line)
        return txt
955589
def text_to_data(self, text, elt, ps):
        '''
        '''
        if self.strip: text = text.strip()
        if self.pyclass is not None:
            return self.pyclass(text.encode(UNICODE_ENCODING))
        return text.encode(UNICODE_ENCODING)
576643
def texts(self: object, fileids: str, plaintext: bool = True):
        

        for doc in self.docs(fileids):
            if plaintext==True:
                doc = re.sub(r'<.+?>\s', '', doc) # Remove citation info
            doc = doc.rstrip() # Clean up final line breaks
            yield doc