Convert a Python object's docstring to markdown

    Parameters
    ----------
    docstring : str
        The docstring body.

    Returns
    ----------
    clean_lst : list
        The markdown formatted docstring as lines (str) in a Python list.
766508
def docstring_to_markdown(docstring):
    
    new_docstring_lst = []

    for idx, line in enumerate(docstring.split('\n')):
        line = line.strip()
        if set(line) in ({'-'}, {'='}):
            new_docstring_lst[idx-1] = '**%s**' % new_docstring_lst[idx-1]
        elif line.startswith('>>>'):
            line = '    %s' % line
        new_docstring_lst.append(line)

    for idx, line in enumerate(new_docstring_lst[1:]):
        if line:
            if line.startswith('Description : '):
                new_docstring_lst[idx+1] = (new_docstring_lst[idx+1]
                                            .replace('Description : ', ''))
            elif ' : ' in line:
                line = line.replace(' : ', '` : ')
                new_docstring_lst[idx+1] = '\n- `%s\n' % line
            elif '**' in new_docstring_lst[idx-1] and '**' not in line:
                new_docstring_lst[idx+1] = '\n%s' % line.lstrip()
            elif '**' not in line:
                new_docstring_lst[idx+1] = '    %s' % line.lstrip()

    clean_lst = []
    for line in new_docstring_lst:
        if set(line.strip()) not in ({'-'}, {'='}):
            clean_lst.append(line)
    return clean_lst
657524
def clean_srt(srt):
    
    with open(srt, 'r') as f:
        text = f.read()
    text = re.sub(r'^\d+[\n\r]', '', text, flags=re.MULTILINE)
    lines = text.splitlines()
    output = OrderedDict()
    key = ''

    for line in lines:
        line = line.strip()
        if line.find('-->') > -1:
            key = line
            output[key] = ''
        else:
            if key != '':
                output[key] += line + ' '

    return output
91720
def clean_text(text):
    
    if text:
        text = html2text.html2text(clean_markdown(text))
        return re.sub(r'\s+', ' ', text).strip()
1346036
def get_output_docs(self):
        
        if not self.parsed:
            self._parse()
        lst = []
        for e in self.docs_list:
            lst.append(e['docs'].get_raw_docs())
        return lst
1157553
def _replace_docstring_header(paragraph):
    """"""

    # Replace Markdown headers in docstrings with light headers in bold.
    paragraph = re.sub(_docstring_header_pattern,
                       r'*\1*',
                       paragraph,
                       )

    paragraph = re.sub(_docstring_parameters_pattern,
                       r'\n* `\1` (\2)\n',
                       paragraph,
                       )

    return paragraph
1315453
def docstring_section_lines(docstring, section_name):
    
    lines = []
    inside_section = False
    underline = '-' * len(section_name)
    expect_underline = False
    for line in docstring.splitlines():
        _line = line.strip().lower()

        if expect_underline:
            expect_underline = False
            if _line == underline:
                inside_section = True
                continue

        if _line == section_name:
            expect_underline = True
        elif _line in DOCSTRING_SECTIONS:
            # next section
            break
        elif inside_section:
            lines.append(line)
    return '\n'.join(lines)
351622
def markdown_to_text(self, source):
        """"""
        if self.comment and self.comment != "#'":
            source = copy(source)
            comment_magic(source, self.language, self.comment_magics)

        return comment_lines(source, self.comment)
523345
def extract_docstring(self):
        
        lines = open(self.filename).readlines()
        start_row = 0
        if lines[0].startswith('#!'):
            lines.pop(0)
            start_row = 1

        docstring = ''
        first_par = ''
        line_iter = lines.__iter__()
        tokens = tokenize.generate_tokens(lambda: next(line_iter))
        for tok_type, tok_content, _, (erow, _), _ in tokens:
            tok_type = token.tok_name[tok_type]
            if tok_type in ('NEWLINE', 'COMMENT', 'NL', 'INDENT', 'DEDENT'):
                continue
            elif tok_type == 'STRING':
                docstring = eval(tok_content)
                # If the docstring is formatted with several paragraphs,
                # extract the first one:
                paragraphs = '\n'.join(line.rstrip()
                                       for line in docstring.split('\n')
                                       ).split('\n\n')
                if len(paragraphs) > 0:
                    first_par = paragraphs[0]
            break

        thumbloc = None
        for i, line in enumerate(docstring.split("\n")):
            m = re.match(r"^_thumb: (\.\d+),\s*(\.\d+)", line)
            if m:
                thumbloc = float(m.group(1)), float(m.group(2))
                break
        if thumbloc is not None:
            self.thumbloc = thumbloc
            docstring = "\n".join([l for l in docstring.split("\n")
                                   if not l.startswith("_thumb")])

        self.docstring = docstring
        self.short_desc = first_par
        self.end_line = erow + 1 + start_row
1053551
def _get_docstring_indent(definition, docstring):
        """"""
        before_docstring, _, _ = definition.source.partition(docstring)
        _, _, indent = before_docstring.rpartition('\n')
        return indent
1237468
def _split_docstring(self, docstring):
        """"""

        lines = docstring.strip().splitlines()

        synopsis = lines[0].strip()
        body = textwrap.dedent('\n'.join(lines[2:]))

        # Remove RST preformatted text markers.
        body = body.replace('\n::\n', '')
        body = body.replace('::\n', ':')

        return (synopsis, body)
1037592
def generate_docstr(element, indent='', wrap=None):
    
    result = []

    txt = element.text and element.text.rstrip()
    if txt:
        result.append(_reindent(txt, indent))
        result.append(indent)

    for d in element.findall('doc') + element.findall('rule'):
        docval = ''.join(d.textlist()).rstrip()
        if not docval:
            continue
        reformat = True
        if 'name' in d.attrib:
            result.append(indent + d.attrib['name'].upper() + ':')
            result.append(indent)
            extra_indent = '    '
            if d.attrib['name'] == 'grammar':
                reformat = False # Don't want re-indenting to mess this up
        elif d.tag == 'rule':
            result.append(indent + 'RULE:')
            result.append(indent)
            extra_indent = '    '
        else:
            extra_indent = ''
        result.append(_reindent(docval, indent + extra_indent, reformat))
        result.append(indent)

    fields = element.findall('field')
    if fields:
        result.append(indent + 'PARAMETERS:')
        for f in fields:
            result.append(indent + '    ' + _fixup_field_name(f) + ': ' + _field_type(f))
            field_docs = generate_docstr(f, indent + '        ')
            if field_docs:
                result.append(indent)
                result.append(field_docs)
            result.append(indent)

    if not result:
        return None

    if wrap is not None:
        result = [wrap] + result + [wrap]

    return '\n'.join(x.rstrip() for x in result) + '\n'
25184
def core_module_docstrings(
    include_core=True, include_user=False, config=None, format="md"
):
    
    paths = {}
    docstrings = {}
    if include_core:
        for file in os.listdir(modules_directory()):
            if file.endswith(".py"):
                name = file[:-3]
                if name != "__init__":
                    paths[name] = (os.path.join(modules_directory(), file), "core")

    if include_user:
        # include user modules
        for include_path in sorted(config["include_paths"]):
            for file in sorted(os.listdir(include_path)):
                if not file.endswith(".py"):
                    continue
                name = file[:-3]
                paths[name] = (os.path.join(include_path, file), "user")
    for name in paths:
        path, module_type = paths[name]
        with open(path) as f:
            try:
                module = ast.parse(f.read())
            except SyntaxError:
                # there is a syntax error so ignore module
                continue
            raw_docstring = ast.get_docstring(module)

            # prevent issue when no docstring exists
            if raw_docstring is None:
                continue

            # remove any sample outputs
            parts = re.split("^SAMPLE OUTPUT$", raw_docstring, flags=re.M)
            docstring = parts[0]

            if format == "md":
                docstring = [
                    d for d in _from_docstring_md(str(docstring).strip().split("\n"))
                ]
            elif format == "rst":
                docstring = [
                    d for d in _from_docstring_rst(str(docstring).strip().split("\n"))
                ]
            else:
                raise Exception("`md` and `rst` format supported only")

            docstrings[name] = docstring + ["\n"]
    return docstrings
14621
def _processDocstring(self, node, tail='', **kwargs):
        
        typeName = type(node).__name__
        # Modules don't have lineno defined, but it's always 0 for them.
        curLineNum = startLineNum = 0
        if typeName != 'Module':
            startLineNum = curLineNum = node.lineno - 1
        # Figure out where both our enclosing object and our docstring start.
        line = ''
        while curLineNum < len(self.lines):
            line = self.lines[curLineNum]
            match = AstWalker.__docstrMarkerRE.match(line)
            if match:
                break
            curLineNum += 1
        docstringStart = curLineNum
        # Figure out where our docstring ends.
        if not AstWalker.__docstrOneLineRE.match(line):
            # Skip for the special case of a single-line docstring.
            curLineNum += 1
            while curLineNum < len(self.lines):
                line = self.lines[curLineNum]
                if line.find(match.group(2)) >= 0:
                    break
                curLineNum += 1
        endLineNum = curLineNum + 1

        # Isolate our enclosing object's declaration.
        defLines = self.lines[startLineNum: docstringStart]
        # Isolate our docstring.
        self.docLines = self.lines[docstringStart: endLineNum]

        # If we have a docstring, extract information from it.
        if self.docLines:
            # Get rid of the docstring delineators.
            self.docLines[0] = AstWalker.__docstrMarkerRE.sub('',
                                                              self.docLines[0])
            self.docLines[-1] = AstWalker.__docstrMarkerRE.sub('',
                                                               self.docLines[-1])
            # Handle special strings within the docstring.
            docstringConverter = self.__alterDocstring(
                tail, self.__writeDocstring())
            for lineInfo in enumerate(self.docLines):
                docstringConverter.send(lineInfo)
            docstringConverter.send((len(self.docLines) - 1, None))

        # Add a Doxygen @brief tag to any single-line description.
        if self.options.autobrief:
            safetyCounter = 0
            while len(self.docLines) > 0 and self.docLines[0].lstrip('#').strip() == '':
                del self.docLines[0]
                self.docLines.append('')
                safetyCounter += 1
                if safetyCounter >= len(self.docLines):
                    # Escape the effectively empty docstring.
                    break
            if len(self.docLines) == 1 or (len(self.docLines) >= 2 and (
                self.docLines[1].strip(whitespace + '#') == '' or
                    self.docLines[1].strip(whitespace + '#').startswith('@'))):
                self.docLines[0] = "## @brief {0}".format(self.docLines[0].lstrip('#'))
                if len(self.docLines) > 1 and self.docLines[1] == '# @par':
                    self.docLines[1] = '#'

        if defLines:
            match = AstWalker.__indentRE.match(defLines[0])
            indentStr = match and match.group(1) or ''
            self.docLines = [AstWalker.__newlineRE.sub(indentStr + '#', docLine)
                             for docLine in self.docLines]

        # Taking away a docstring from an interface method definition sometimes
        # leaves broken code as the docstring may be the only code in it.
        # Here we manually insert a pass statement to rectify this problem.
        if typeName != 'Module':
            if docstringStart < len(self.lines):
                match = AstWalker.__indentRE.match(self.lines[docstringStart])
                indentStr = match and match.group(1) or ''
            else:
                indentStr = ''
            containingNodes = kwargs.get('containingNodes', []) or []
            fullPathNamespace = self._getFullPathName(containingNodes)
            parentType = fullPathNamespace[-2][1]
            if parentType == 'interface' and typeName == 'FunctionDef' \
               or fullPathNamespace[-1][1] == 'interface':
                defLines[-1] = '{0}{1}{2}pass'.format(defLines[-1],
                                                      linesep, indentStr)
            elif self.options.autobrief and typeName == 'ClassDef':
                # If we're parsing docstrings separate out class attribute
                # definitions to get better Doxygen output.
                for firstVarLineNum, firstVarLine in enumerate(self.docLines):
                    if '@property\t' in firstVarLine:
                        break
                lastVarLineNum = len(self.docLines)
                if lastVarLineNum > 0 and '@property\t' in firstVarLine:
                    while lastVarLineNum > firstVarLineNum:
                        lastVarLineNum -= 1
                        if '@property\t' in self.docLines[lastVarLineNum]:
                            break
                    lastVarLineNum += 1
                    if firstVarLineNum < len(self.docLines):
                        indentLineNum = endLineNum
                        indentStr = ''
                        while not indentStr and indentLineNum < len(self.lines):
                            match = AstWalker.__indentRE.match(self.lines[indentLineNum])
                            indentStr = match and match.group(1) or ''
                            indentLineNum += 1
                        varLines = ['{0}{1}'.format(linesep, docLine).replace(
                                    linesep, linesep + indentStr)
                                    for docLine in self.docLines[
                                        firstVarLineNum: lastVarLineNum]]
                        defLines.extend(varLines)
                        self.docLines[firstVarLineNum: lastVarLineNum] = []
                        # After the property shuffling we will need to relocate
                        # any existing namespace information.
                        namespaceLoc = defLines[-1].find('\n# @namespace')
                        if namespaceLoc >= 0:
                            self.docLines[-1] += defLines[-1][namespaceLoc:]
                            defLines[-1] = defLines[-1][:namespaceLoc]

        # For classes and functions, apply our changes and reverse the
        # order of the declaration and docstring, and for modules just
        # apply our changes.
        if typeName != 'Module':
            self.lines[startLineNum: endLineNum] = self.docLines + defLines
        else:
            self.lines[startLineNum: endLineNum] = defLines + self.docLines
181092
def clean(document):
    

    newdocument = document

    # Clean empty text and r tags
    for t in ('t', 'r'):
        rmlist = []
        for element in newdocument.iter():
            if element.tag == '{%s}%s' % (nsprefixes['w'], t):
                if not element.text and not len(element):
                    rmlist.append(element)
        for element in rmlist:
            element.getparent().remove(element)

    return newdocument
649697
def trim_docstring(docstring):
    
    if not docstring or not docstring.strip():
        return ""
    # Convert tabs to spaces and split into lines
    lines = docstring.expandtabs().splitlines()
    indent = min(len(line) - len(line.lstrip()) for line in lines if line.lstrip())
    trimmed = [lines[0].lstrip()] + [line[indent:].rstrip() for line in lines[1:]]
    return "\n".join(trimmed).strip()
986150
def strip_docstrings(tokens):
    
    stack = []
    state = 'wait_string'
    for t in tokens:
        typ = t[0]
        if state == 'wait_string':
            if typ in (tokenize.NL, tokenize.COMMENT):
                yield t
            elif typ in (tokenize.DEDENT, tokenize.INDENT, tokenize.STRING):
                stack.append(t)
            elif typ == tokenize.NEWLINE:
                stack.append(t)
                start_line, end_line = stack[0][2][0], stack[-1][3][0]+1
                for i in range(start_line, end_line):
                    yield tokenize.NL, '\n', (i, 0), (i,1), '\n'
                for t in stack:
                    if t[0] in (tokenize.DEDENT, tokenize.INDENT):
                        yield t[0], t[1], (i+1, t[2][1]), (i+1, t[3][1]), t[4]
                del stack[:]
            else:
                stack.append(t)
                for t in stack: yield t
                del stack[:]
                state = 'wait_newline'
        elif state == 'wait_newline':
            if typ == tokenize.NEWLINE:
                state = 'wait_string'
            yield t
1350288
def markdown_to_safe_html(markdown_string):
  
  warning = ''
  # Convert to utf-8 whenever we have a binary input.
  if isinstance(markdown_string, six.binary_type):
    markdown_string_decoded = markdown_string.decode('utf-8')
    # Remove null bytes and warn if there were any, since it probably means
    # we were given a bad encoding.
    markdown_string = markdown_string_decoded.replace(u'\x00', u'')
    num_null_bytes = len(markdown_string_decoded) - len(markdown_string)
    if num_null_bytes:
      warning = ('<!-- WARNING: discarded %d null bytes in markdown string '
                 'after UTF-8 decoding -->\n') % num_null_bytes

  string_html = markdown.markdown(
      markdown_string, extensions=['markdown.extensions.tables'])
  string_sanitized = bleach.clean(
      string_html, tags=_ALLOWED_TAGS, attributes=_ALLOWED_ATTRIBUTES)
  return warning + string_sanitized