Returns HTML for code blocks formatted with Pygment
5252
def format_code(text)
      simple_format( truncate( Sanitize.clean(markdown(text)).html_safe, escape: false, length: 300, separator: ' ', omission: ' …' ))
    end
891244
def from_pygments_lexer_cls(cls, lexer_cls):
        
        patterns = {
            # For Python, start highlighting at any class/def block.
            'Python':   r'^\s*(class|def)\s+',
            'Python 3': r'^\s*(class|def)\s+',

            # For HTML, start at any open/close tag definition.
            'HTML': r'<[/a-zA-Z]',

            # For javascript, start at a function.
            'JavaScript': r'\bfunction\b'

            # TODO: Add definitions for other languages.
            #       By default, we start at every possible line.
        }
        p = patterns.get(lexer_cls.name, '^')
        return cls(p)
1357837
def code_to_text(self):
        """"""
        source = copy(self.source)
        comment_magic(source, self.language, self.comment_magics)

        options = []
        if self.cell_type == 'code' and self.language:
            options.append(self.language)

        filtered_metadata = {key: self.metadata[key] for key in self.metadata
                             if key not in ['active', 'language']}

        if filtered_metadata:
            options.append(metadata_to_md_options(filtered_metadata))

        return ['```{}'.format(' '.join(options))] + source + ['```']
539351
def get_codeblock_node(self, code, language):
        

        # type: () -> List[nodes.Node]
        document = self.state.document
        location = self.state_machine.get_source_and_line(self.lineno)

        linespec = self.options.get('emphasize-lines')
        if linespec:
            try:
                nlines = len(code.split('\n'))
                hl_lines = parselinenos(linespec, nlines)
                if any(i >= nlines for i in hl_lines):
                    log.warning(__('line number spec is out of range(1-%d): %r') %
                                   (nlines, self.options['emphasize-lines']),
                                   location=location)

                hl_lines = [x + 1 for x in hl_lines if x < nlines]
            except ValueError as err:
                return [document.reporter.warning(str(err), line=self.lineno)]
        else:
            hl_lines = None

        if 'dedent' in self.options:
            location = self.state_machine.get_source_and_line(self.lineno)
            lines = code.split('\n')
            lines = dedent_lines(lines, self.options['dedent'], location=location)
            code = '\n'.join(lines)

        literal = nodes.literal_block(code, code)
        literal['language'] = language
        literal['linenos'] = 'linenos' in self.options or \
                             'lineno-start' in self.options
        literal['classes'] += self.options.get('class', [])
        extra_args = literal['highlight_args'] = {}
        if hl_lines is not None:
            extra_args['hl_lines'] = hl_lines
        if 'lineno-start' in self.options:
            extra_args['linenostart'] = self.options['lineno-start']
        set_source_info(self, literal)

        caption = self.options.get('caption')
        if caption:
            try:
                literal = container_wrapper(self, literal, caption)
            except ValueError as exc:
                return [document.reporter.warning(text_type(exc), line=self.lineno)]

        # literal will be note_implicit_target that is linked from caption and numref.
        # when options['name'] is provided, it should be primary ID.
        self.add_name(literal)

        return [literal]
800928
def get_formatted_content(self, pyobj):
        '''
        '''
        u = urlencode(pyobj, self.reserved)
        return String.get_formatted_content(self, 
            u)
288162
def to_html(self, codebase):
        
        body = ''
        for section in ('params', 'options', 'exceptions'):
            val = getattr(self, section)
            if val:
                body += '<h5>%s</h5>\n<dl class = "%s">%s</dl>' % (
                        printable(section), section, 
                        '\n'.join(param.to_html() for param in val))

        body += codebase.build_see_html(self.see, 'h5', self)
        return ('<a name = "%s" />\n<div class = "function">\n' + 
                '<h4>%s</h4>\n%s\n%s\n</div>\n') % (self.name, self.name, 
                    htmlize_paragraphs(codebase.translate_links(self.doc, self)), body)
664658
def get_highlighted_code(name, code, type='terminal'):
    
    import logging
    try:
        import pygments
        pygments
    except ImportError:
        return code
    from pygments import highlight
    from pygments.lexers import guess_lexer_for_filename, ClassNotFound
    from pygments.formatters import TerminalFormatter

    try:
        lexer = guess_lexer_for_filename(name, code)
        formatter = TerminalFormatter()
        content = highlight(code, lexer, formatter)
    except ClassNotFound:
        logging.debug("Couldn't guess Lexer, will not use pygments.")
        content = code
    return content
1200993
def format(self, soup):
        
        # Remove all the newline characters before a closing tag.
        for node in soup.findAll(text=True):
            if node.rstrip(" ").endswith("\n"):
                node.replaceWith(node.rstrip(" ").rstrip("\n"))
        # Join the block elements lines into a single long line
        for tag in ['p', 'li']:
            for node in soup.findAll(tag):
                text = unicode(node)
                lines = [x.strip() for x in text.splitlines()]
                text = ' '.join(lines)
                node.replaceWith(BeautifulSoup.BeautifulSoup(text))
        soup = BeautifulSoup.BeautifulSoup(unicode(soup))
        # replace all <br/> tag by newline character
        for node in soup.findAll('br'):
            node.replaceWith("\n")
        soup = BeautifulSoup.BeautifulSoup(unicode(soup))
        return soup
268223
def highlightBlock(self, string):
        
        prev_data = self.currentBlock().previous().userData()
        if prev_data is not None:
            self._lexer._saved_state_stack = prev_data.syntax_stack
        elif hasattr(self._lexer, '_saved_state_stack'):
            del self._lexer._saved_state_stack

        # Lex the text using Pygments
        index = 0
        for token, text in self._lexer.get_tokens(string):
            length = len(text)
            self.setFormat(index, length, self._get_format(token))
            index += length

        if hasattr(self._lexer, '_saved_state_stack'):
            data = PygmentsBlockUserData(
                syntax_stack=self._lexer._saved_state_stack)
            self.currentBlock().setUserData(data)
            # Clean up for the next go-round.
            del self._lexer._saved_state_stack
903335
def stylize_comment_block(lines):
  
  normal, sep, in_code = range(3)
  state = normal
  for line in lines:
    indented = line.startswith('    ')
    empty_line = line.strip() == ''

    if state == normal and empty_line:
      state = sep
    elif state in [sep, normal] and indented:
      yield ''
      if indented:
        yield '.. code-block:: javascript'
        yield ''
        yield line
        state = in_code
      else:
        state = normal
    elif state == sep and not empty_line:
      yield ''
      yield line
      state = normal
    else:
      yield line
      if state == in_code and not (indented or empty_line):
        sep = normal
31298
def pre_process_code_block(block):
        
        if 'indent' in block and block['indent']:
            indent = r'^' + block['indent']
            block['content'] = re.sub(indent, '', block['icontent'],
                                      flags=re.MULTILINE)
1016369
function highlight(code) {
  return new Promise((resolve, reject) => {
    pygmentize({ lang: 'html', format: 'html' }, code, function (err, result) {
      if (err) {
        console.log(err);
        reject(err);
      } else {
        resolve(result.toString());
      };
    });
  });
}
1109189
def _markdownify(tag, _listType=None, _blockQuote=False, _listIndex=1):
	""""""
	children = tag.find_all(recursive=False)

	if tag.name == '[document]':
		for child in children:
			_markdownify(child)
		return

	if tag.name not in _supportedTags or not _supportedAttrs(tag):
		if tag.name not in _inlineTags:
			tag.insert_before('\n\n')
			tag.insert_after('\n\n')
		else:
			_escapeCharacters(tag)
			for child in children:
				_markdownify(child)
		return
	if tag.name not in ('pre', 'code'):
		_escapeCharacters(tag)
		_breakRemNewlines(tag)
	if tag.name == 'p':
		if tag.string != None:
			if tag.string.strip() == u'':
				tag.string = u'\xa0'
				tag.unwrap()
				return
		if not _blockQuote:
			tag.insert_before('\n\n')
			tag.insert_after('\n\n')
		else:
			tag.insert_before('\n')
			tag.insert_after('\n')
		tag.unwrap()

		for child in children:
			_markdownify(child)
	elif tag.name == 'br':
		tag.string = '  \n'
		tag.unwrap()
	elif tag.name == 'img':
		alt = ''
		title = ''
		if tag.has_attr('alt'):
			alt = tag['alt']
		if tag.has_attr('title') and tag['title']:
			title = ' "%s"' % tag['title']
		tag.string = '![%s](%s%s)' % (alt, tag['src'], title)
		tag.unwrap()
	elif tag.name == 'hr':
		tag.string = '\n---\n'
		tag.unwrap()
	elif tag.name == 'pre':
		tag.insert_before('\n\n')
		tag.insert_after('\n\n')
		if tag.code:
			if not _supportedAttrs(tag.code):
				return
			for child in tag.code.find_all(recursive=False):
				if child.name != 'br':
					return
			# code block
			for br in tag.code.find_all('br'):
				br.string = '\n'
				br.unwrap()
			tag.code.unwrap()
			lines = unicode(tag).strip().split('\n')
			lines[0] = lines[0][5:]
			lines[-1] = lines[-1][:-6]
			if not lines[-1]:
				lines.pop()
			for i,line in enumerate(lines):
				line = line.replace(u'\xa0', ' ')
				lines[i] = '    %s' % line
			tag.replace_with(BeautifulSoup('\n'.join(lines), 'html.parser'))
		return
	elif tag.name == 'code':
		# inline code
		if children:
			return
		tag.insert_before('`` ')
		tag.insert_after(' ``')
		tag.unwrap()
	elif _recursivelyValid(tag):
		if tag.name == 'blockquote':
			# ! FIXME: hack
			tag.insert_before('<<<BLOCKQUOTE: ')
			tag.insert_after('>>>')
			tag.unwrap()
			for child in children:
				_markdownify(child, _blockQuote=True)
			return
		elif tag.name == 'a':
			# process children first
			for child in children:
				_markdownify(child)
			if not tag.has_attr('href'):
				return
			if tag.string != tag.get('href') or tag.has_attr('title'):
				title = ''
				if tag.has_attr('title'):
					title = ' "%s"' % tag['title']
				tag.string = '[%s](%s%s)' % (BeautifulSoup(unicode(tag), 'html.parser').string,
					tag.get('href', ''),
					title)
			else:
				# ! FIXME: hack
				tag.string = '<<<FLOATING LINK: %s>>>' % tag.string
			tag.unwrap()
			return
		elif tag.name == 'h1':
			tag.insert_before('\n\n# ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h2':
			tag.insert_before('\n\n## ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h3':
			tag.insert_before('\n\n### ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h4':
			tag.insert_before('\n\n#### ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h5':
			tag.insert_before('\n\n##### ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h6':
			tag.insert_before('\n\n###### ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name in ('ul', 'ol'):
			tag.insert_before('\n\n')
			tag.insert_after('\n\n')
			tag.unwrap()
			for i, child in enumerate(children):
				_markdownify(child, _listType=tag.name, _listIndex=i+1)
			return
		elif tag.name == 'li':
			if not _listType:
				# <li> outside of list; ignore
				return
			if _listType == 'ul':
				tag.insert_before('*   ')
			else:
				tag.insert_before('%d.   ' % _listIndex)
			for child in children:
				_markdownify(child)
			for c in tag.contents:
				if type(c) != bs4.element.NavigableString:
					continue
				c.replace_with('\n    '.join(c.split('\n')))
			tag.insert_after('\n')
			tag.unwrap()
			return
		elif tag.name in ('strong','b'):
			tag.insert_before('__')
			tag.insert_after('__')
			tag.unwrap()
		elif tag.name in ('em','i'):
			tag.insert_before('_')
			tag.insert_after('_')
			tag.unwrap()
		for child in children:
			_markdownify(child)
573142
def pformat(self):
        '''
        
        '''
        result = "{\n"
        indent1 = " " * 4
        indent2 = " " * 8
        for line_no, code_objects in sorted(self.items()):
            result += '%s%i: [\n' % (indent1, line_no)
            for code_object in code_objects:
                result += '%s"""<%s:%s>""",\n' % (
                    indent2, code_object.PART_TYPE, code_object.content
                )
            result += '%s],\n' % indent1
        result += "}"

        return result
1196200
def highlight_block(self, text):
        """"""
        text = to_text_string(text)
        prev_state = tbh.get_state(self.currentBlock().previous())
        if prev_state == self.INSIDE_DQ3STRING:
            offset = -4
            text = r'""" '+text
        elif prev_state == self.INSIDE_SQ3STRING:
            offset = -4
            text = r"''' "+text
        elif prev_state == self.INSIDE_DQSTRING:
            offset = -2
            text = r'" '+text
        elif prev_state == self.INSIDE_SQSTRING:
            offset = -2
            text = r"' "+text
        else:
            offset = 0
            prev_state = self.NORMAL
        
        oedata = None
        import_stmt = None

        self.setFormat(0, len(text), self.formats["normal"])
        
        state = self.NORMAL
        match = self.PROG.search(text)
        while match:
            for key, value in list(match.groupdict().items()):
                if value:
                    start, end = match.span(key)
                    start = max([0, start+offset])
                    end = max([0, end+offset])
                    if key == "uf_sq3string":
                        self.setFormat(start, end-start,
                                       self.formats["string"])
                        state = self.INSIDE_SQ3STRING
                    elif key == "uf_dq3string":
                        self.setFormat(start, end-start,
                                       self.formats["string"])
                        state = self.INSIDE_DQ3STRING
                    elif key == "uf_sqstring":
                        self.setFormat(start, end-start,
                                       self.formats["string"])
                        state = self.INSIDE_SQSTRING
                    elif key == "uf_dqstring":
                        self.setFormat(start, end-start,
                                       self.formats["string"])
                        state = self.INSIDE_DQSTRING
                    else:
                        self.setFormat(start, end-start, self.formats[key])
                        if key == "comment":
                            if text.lstrip().startswith(self.cell_separators):
                                self.found_cell_separators = True
                                oedata = OutlineExplorerData()
                                oedata.text = to_text_string(text).strip()
                                # cell_head: string contaning the first group
                                # of '%'s in the cell header
                                cell_head = re.search(r"%+|$",
                                                      text.lstrip()).group()
                                if cell_head == '':
                                    oedata.cell_level = 0
                                else:
                                    oedata.cell_level = len(cell_head) - 2
                                oedata.fold_level = start
                                oedata.def_type = OutlineExplorerData.CELL
                                oedata.def_name = get_code_cell_name(text)
                            elif self.OECOMMENT.match(text.lstrip()):
                                oedata = OutlineExplorerData()
                                oedata.text = to_text_string(text).strip()
                                oedata.fold_level = start
                                oedata.def_type = OutlineExplorerData.COMMENT
                                oedata.def_name = text.strip()
                        elif key == "keyword":
                            if value in ("def", "class"):
                                match1 = self.IDPROG.match(text, end)
                                if match1:
                                    start1, end1 = match1.span(1)
                                    self.setFormat(start1, end1-start1,
                                                   self.formats["definition"])
                                    oedata = OutlineExplorerData()
                                    oedata.text = to_text_string(text)
                                    oedata.fold_level = (len(text)
                                                         - len(text.lstrip()))
                                    oedata.def_type = self.DEF_TYPES[
                                                        to_text_string(value)]
                                    oedata.def_name = text[start1:end1]
                                    oedata.color = self.formats["definition"]
                            elif value in ("elif", "else", "except", "finally",
                                           "for", "if", "try", "while",
                                           "with"):
                                if text.lstrip().startswith(value):
                                    oedata = OutlineExplorerData()
                                    oedata.text = to_text_string(text).strip()
                                    oedata.fold_level = start
                                    oedata.def_type = \
                                        OutlineExplorerData.STATEMENT
                                    oedata.def_name = text.strip()
                            elif value == "import":
                                import_stmt = text.strip()
                                # color all the "as" words on same line, except
                                # if in a comment; cheap approximation to the
                                # truth
                                if '#' in text:
                                    endpos = text.index('#')
                                else:
                                    endpos = len(text)
                                while True:
                                    match1 = self.ASPROG.match(text, end,
                                                               endpos)
                                    if not match1:
                                        break
                                    start, end = match1.span(1)
                                    self.setFormat(start, end-start,
                                                   self.formats["keyword"])
                    
            match = self.PROG.search(text, match.end())
        
        tbh.set_state(self.currentBlock(), state)
        
        # Use normal format for indentation and trailing spaces.
        self.formats['leading'] = self.formats['normal']
        self.formats['trailing'] = self.formats['normal']
        self.highlight_spaces(text, offset)
        
        if oedata is not None:
            block_nb = self.currentBlock().blockNumber()
            self.outlineexplorer_data[block_nb] = oedata
            self.outlineexplorer_data['found_cell_separators'] = self.found_cell_separators
        if import_stmt is not None:
            block_nb = self.currentBlock().blockNumber()
            self.import_statements[block_nb] = import_stmt
235666
def htmlFormat(output, pathParts = (), statDict = None, query = None):
  """"""
  statDict = statDict or scales.getStats()
  if query:
    statDict = runQuery(statDict, query)
  _htmlRenderDict(pathParts, statDict, output)
214016
def highlight_block(self, text):
        """"""
        text = to_text_string(text)
        self.setFormat(0, len(text), self.formats["normal"])
        
        match = self.PROG.search(text)
        index = 0
        while match:
            for key, value in list(match.groupdict().items()):
                if value:
                    start, end = match.span(key)
                    index += end-start
                    self.setFormat(start, end-start, self.formats[key])
                    if value.lower() in ("subroutine", "module", "function"):
                        match1 = self.IDPROG.match(text, end)
                        if match1:
                            start1, end1 = match1.span(1)
                            self.setFormat(start1, end1-start1,
                                           self.formats["definition"])
                    
            match = self.PROG.search(text, match.end())
        
        self.highlight_spaces(text)