Process a paragraph, which includes all content under it
176547
def paragraph(node):
    
    text = ''
    if node.string_content is not None:
        text = node.string_content
    o = nodes.paragraph('', ' '.join(text))
    o.line = node.sourcepos[0][0]
    for n in MarkDown(node):
        o.append(n)

    return o
468829
def _parse_content(fh):
    """"""
    content = iter(fh)
    deps = {}

    for line in content:
        # quickly discard most of the lines
        if 'fades' not in line:
            continue

        # discard other string with 'fades' that isn't a comment
        if '#' not in line:
            continue

        # assure that it's a well commented line and no other stuff
        line = line.strip()
        index_of_last_fades = line.rfind('fades')
        index_of_first_hash = line.index('#')

        # discard when fades does not appear after #
        if index_of_first_hash > index_of_last_fades:
            continue

        import_part, fades_part = line.rsplit("#", 1)

        # discard other comments in the same line that aren't for fades
        if "fades" not in fades_part:
            import_part, fades_part = import_part.rsplit("#", 1)

        fades_part = fades_part.strip()
        if not fades_part.startswith("fades"):
            continue

        if not import_part:
            # the fades comment was done at the beginning of the line,
            # which means that the import info is in the next one
            import_part = next(content).strip()

        if import_part.startswith('#'):
            continue

        # get module
        import_tokens = import_part.split()
        if import_tokens[0] == 'import':
            module_path = import_tokens[1]
        elif import_tokens[0] == 'from' and import_tokens[2] == 'import':
            module_path = import_tokens[1]
        else:
            logger.debug("Not understood import info: %s", import_tokens)
            continue
        module = module_path.split(".")[0]
        # If fades know the real name of the pkg. Replace it!
        if module in PKG_NAMES_DB:
            module = PKG_NAMES_DB[module]
        # To match the "safe" name that pkg_resources creates:
        module = module.replace('_', '-')

        # get the fades info after 'fades' mark, if any
        if len(fades_part) == 5 or fades_part[5:].strip()[0] in "<>=!":
            # just the 'fades' mark, and maybe a version specification, the requirement is what
            # was imported (maybe with that version comparison)
            requirement = module + fades_part[5:]
        elif fades_part[5] != " ":
            # starts with fades but it's part of a longer weird word
            logger.warning("Not understood fades info: %r", fades_part)
            continue
        else:
            # more complex stuff, to be parsed as a normal requirement
            requirement = fades_part[5:]

        # parse and convert the requirement
        parsed_req = parse_fade_requirement(requirement)
        if parsed_req is None:
            continue
        repo, dependency = parsed_req
        deps.setdefault(repo, []).append(dependency)

    return deps
285138
@Override
    public void processOtherContent(Wrapper<Content> content) {
        if (parentElement != null) {
            parentElement.addContent(content.getContent());
        }
    }
34411
def process_summary(article):
    """"""

    summary = article.summary
    summary_parsed = BeautifulSoup(summary, 'html.parser')
    math = summary_parsed.find_all(class_='math')

    if len(math) > 0:
        last_math_text = math[-1].get_text()
        if len(last_math_text) > 3 and last_math_text[-3:] == '...':
            content_parsed = BeautifulSoup(article._content, 'html.parser')
            full_text = content_parsed.find_all(class_='math')[len(math)-1].get_text()
            math[-1].string = "%s ..." % full_text
            summary = summary_parsed.decode()

        # clear memoization cache
        import functools
        if isinstance(article.get_summary, functools.partial):
            memoize_instance = article.get_summary.func.__self__
            memoize_instance.cache.clear()

        article._summary = "%s<script type='text/javascript'>%s</script>" % (summary, process_summary.mathjax_script)
252935
def preprocess(content, options):
    # type: (Text, Optional[optparse.Values]) -> ReqFileLines
    
    lines_enum = enumerate(content.splitlines(), start=1)  # type: ReqFileLines
    lines_enum = join_lines(lines_enum)
    lines_enum = ignore_comments(lines_enum)
    lines_enum = skip_regex(lines_enum, options)
    lines_enum = expand_env_variables(lines_enum)
    return lines_enum
79069
function processKeywords() {
    var nodes = []
    var lastIndex
    var keyword
    var node
    var submatch

    if (!top.keywords) {
      return addText(modeBuffer, nodes)
    }

    lastIndex = 0

    top.lexemesRe.lastIndex = 0

    keyword = top.lexemesRe.exec(modeBuffer)

    while (keyword) {
      addText(modeBuffer.substring(lastIndex, keyword.index), nodes)

      submatch = keywordMatch(top, keyword)

      if (submatch) {
        relevance += submatch[1]

        node = build(submatch[0], [])

        nodes.push(node)

        addText(keyword[0], node.children)
      } else {
        addText(keyword[0], nodes)
      }

      lastIndex = top.lexemesRe.lastIndex
      keyword = top.lexemesRe.exec(modeBuffer)
    }

    addText(modeBuffer.substr(lastIndex), nodes)

    return nodes
  }
987583
def tokenize_paragraphs(self):
        """"""
        tok = self.__paragraph_tokenizer
        spans = tok.span_tokenize(self.text)
        dicts = []
        for start, end in spans:
            dicts.append({'start': start, 'end': end})
        self[PARAGRAPHS] = dicts
        return self
1033111
def paragraphs(self, with_id = False):
        """"""
        prevp = 0
        partext = []
        for word, id, pos, lemma in iter(self):
            doc_id, ptype, p, s, w = re.findall('([\w\d-]+)\.(p|head)\.(\d+)\.s\.(\d+)\.w\.(\d+)',id)[0]
            if prevp != p and partext:
                    yield ( doc_id + "." + ptype + "." + prevp , " ".join(partext) )
                    partext = []
            partext.append(word)
            prevp = p
        if partext:
            yield (doc_id + "." + ptype + "." + prevp, " ".join(partext) )
273364
def parse_content(self, text):
        """"""
        raw_content = self.raw_content
        raw_content.clear()
        formal_collect = {}

        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            try:
                split = self.visible_empty_line_re.split(text)
            except ValueError:  # python >= 3.5
                split = [text]

        option_split_re = self.option_split_re
        name = re.compile(re.escape(self.option_name), re.IGNORECASE)
        for text in filter(lambda x: x and x.strip(), split):

            # logger.warning('get options group:\n%r', text)
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                try:
                    split_options = option_split_re.split(text)
                except ValueError:  # python >= 3.5
                    continue

            split_options.pop(0)

            for title, section in zip(split_options[::2], split_options[1::2]):
                prefix, end = name.split(title)

                prefix = prefix.strip()

                section = section.rstrip()
                if end.endswith('\n'):
                    formal = section
                else:
                    formal = ' ' * len(title) + section

                formal_collect.setdefault(prefix, []).append(formal)

                # logger.error((title, section))
                if prefix in raw_content:
                    # TODO: better handling way?
                    if self.namedoptions:
                        log = logger.warning
                    else:
                        log = logger.debug
                    log('duplicated options section %s', prefix)

                    raw_content[prefix] += '\n%s%s' % (title, section)
                else:
                    raw_content[prefix] = title + section

        if formal_collect:
            for each_title, values in formal_collect.items():
                value = '\n'.join(map(textwrap.dedent, values))
                formal_collect[each_title] = value

        self.formal_content = formal_collect
776312
function processNode(node) {
    if (node.tagName === 'P' || node.tagName === 'DIV' || node.tagName === 'UL' || node.tagName === 'OL' || node.tagName === 'PRE')
      setContent(node, getContent(node), '\n\n', '\n\n');
    else if (node.tagName === 'BR')
      setContent(node, '\n\n');
    else if (node.tagName === 'HR')
      setContent(node, '\n***\n');
    else if (node.tagName === 'H1')
      setContent(node, nltrim(getContent(node)), '\n# ', '\n');
    else if (node.tagName === 'H2') 
      setContent(node, nltrim(getContent(node)), '\n## ', '\n');
    else if (node.tagName === 'H3')
      setContent(node, nltrim(getContent(node)), '\n### ', '\n');
    else if (node.tagName === 'H4') 
      setContent(node, nltrim(getContent(node)), '\n#### ', '\n');
    else if (node.tagName === 'H5') 
      setContent(node, nltrim(getContent(node)), '\n##### ', '\n');
    else if (node.tagName === 'H6')
      setContent(node, nltrim(getContent(node)), '\n###### ', '\n');
    else if (node.tagName === 'B' || node.tagName === 'STRONG')
      setContent(node, nltrim(getContent(node)), '**', '**');
    else if (node.tagName === 'I' || node.tagName === 'EM')
      setContent(node, nltrim(getContent(node)), '_', '_');
    else if (node.tagName === 'A') {
      var href = node.href ? nltrim(node.href) : '', text = nltrim(getContent(node)) || href, title = node.title ? nltrim(node.title) : '';
      if (href.length > 0)
        setContent(node, '[' + text + '](' + href + (title ? ' "' + title + '"' : '') + ')');
      else
        setContent(node, '');
    } else if (node.tagName === 'IMG') {
      var src = node.getAttribute('src') ? nltrim(node.getAttribute('src')) : '', alt = node.alt ? nltrim(node.alt) : '', caption = node.title ? nltrim(node.title) : '';
      if (src.length > 0)
        setContent(node, '![' + alt + '](' + src  + (caption ? ' "' + caption + '"' : '') + ')');
      else
        setContent(node, '');
    } else if (node.tagName === 'BLOCKQUOTE') {
      var block_content = getContent(node);
      if (block_content.length > 0)
        setContent(node, prefixBlock('> ', block_content), '\n\n', '\n\n');
      else
        setContent(node, '');
    } else if (node.tagName === 'CODE') {
      if (node._bfs_parent.tagName === 'PRE' && node._bfs_parent._bfs_parent !== null)
        setContent(node, prefixBlock('    ', getContent(node)));
      else
        setContent(node, nltrim(getContent(node)), '`', '`');
    } else if (node.tagName === 'LI') {
      var list_content = getContent(node);
      if (list_content.length > 0)
        if (node._bfs_parent.tagName === 'OL') 
          setContent(node, trim(prefixBlock('    ', list_content, true)), '1. ', '\n\n');
        else
          setContent(node, trim(prefixBlock('    ', list_content, true)), '- ', '\n\n');
      else
        setContent(node, '');
    } else 
      setContent(node, getContent(node));
  }
842713
def build(self, text):
        
        super(Paragraph, self).build()
        self.content = text
753320
def parse(text, showToc=True):
	""""""
	p = Parser(show_toc=showToc)
	return p.parse(text)
395010
function tokenize(content, rules = parseOrder) {
    let data = []
    if(content) {
        let lines = content.split(/(\r\n|\n)/)
        for (let idx = 0; idx < lines.length; idx++) {
            for (let i = 0; i < lines[idx].length;) {
                let chunk = lines[idx].slice(i)
                let match = rules.some(item => {
                    if (item.pattern.test(chunk)) {
                        let matchObject = chunk.match(item.pattern)
                        i += matchObject[0].length
                        data.push([ item.label, matchObject[0] ])
                        return true
                    }
                })
                if (!match) {
                    data.push([ UNKNOWN, chunk[i] ])
                    i++
                }
            }
        }
    }
    return data
}
422671
def do_content(self, element, decl, pseudo):
        """"""
        # FIXME Need?: link(id)
        step = self.state[self.state['current_step']]
        actions = step['actions']

        wastebin = []
        elem = self.current_target().tree
        if elem == element.etree_element or pseudo == 'inside':
            actions.append(('clear', elem))

        if pseudo:
            current_actions = len(actions)
        # decl.value is parsed representation: loop over it
        # if a string, to pending elem - either text, or tail of last child
        # if a string(x) retrieve value from state and attach as tail
        # if a pending(x), do the target/extend dance
        # content() attr(x), link(x,y) etc.
        for term in decl.value:
            if type(term) is ast.WhitespaceToken:
                continue

            elif type(term) is ast.StringToken:
                actions.append(('string', term.value))

            elif type(term) is ast.LiteralToken:
                actions.append(('string', term.value))

            elif type(term) is ast.FunctionBlock:
                if term.name == 'string':
                    str_args = split(term.arguments, ',')
                    str_name = self.eval_string_value(element,
                                                      str_args[0])[0]
                    val = self.lookup('strings', str_name)
                    if val == '':
                        if len(str_args) > 1:
                            val = self.eval_string_value(element,
                                                         str_args[1])[0]
                        else:
                            log(WARN, u"{} blank string"
                                .format(str_name).encode('utf-8'))
                    if val != '':
                        actions.append(('string', val))

                elif term.name == 'counter':
                    counterargs = [serialize(t).strip(" \'")
                                   for t in split(term.arguments, ',')]
                    count = self.lookup('counters', counterargs)
                    actions.append(('string', (count,)))

                elif term.name.startswith('target-'):
                    target_args = split(term.arguments, ',')
                    vref = self.eval_string_value(element,
                                                  target_args[0])[0]
                    vname = self.eval_string_value(element,
                                                   target_args[1])[0]

                    vtype = term.name[7:]+'s'
                    actions.append(('string', [TargetVal(self, vref[1:],
                                                         vname, vtype)]))

                elif term.name == u'attr':
                    att_args = split(term.arguments, ',')
                    att_name = self.eval_string_value(element,
                                                      att_args[0])[0]
                    att_def = ''
                    if len(att_args) > 1:
                        att_def = self.eval_string_value(element,
                                                         att_args[1])[0]
                    if '|' in att_name:
                        ns, att = att_name.split('|')
                        try:
                            ns = self.css_namespaces[ns]
                        except KeyError:
                            log(WARN, u"Undefined namespace prefix {}"
                                .format(ns).encode('utf-8'))
                            continue
                        att_name = etree.QName(ns, att)
                    att_val = element.etree_element.get(att_name, att_def)
                    actions.append(('string', att_val))

                elif term.name == u'uuid':
                    actions.append(('string', self.generate_id()))

                elif term.name == u'first-letter':
                    tmpstr = self.eval_string_value(element, term.arguments)
                    if tmpstr:
                        actions.append(('string', tmpstr[0]))

                elif term.name == u'content':
                    if pseudo in ('before', 'after'):
                        mycopy = copy_w_id_suffix(element.etree_element)
                        actions.append(('content', mycopy))
                    elif pseudo == 'outside':
                        actions.append(('move', element.etree_element))
                    else:
                        actions.append(('content', None))

                elif term.name == 'pending':
                    target = serialize(term.arguments)
                    val, val_step = self.lookup('pending', target)
                    if val is None:
                        log(INFO, u"{} empty bucket".format(
                            target).encode('utf-8'))
                        continue
                    actions.extend(val)
                    del self.state[val_step]['pending'][target]

                elif term.name == 'nodes':
                    target = serialize(term.arguments)
                    val, val_step = self.lookup('pending', target)
                    if val is None:
                        log(INFO, u"{} empty bucket".format(
                            target).encode('utf-8'))
                        continue
                    for action in val:
                            if action[0] == 'move':
                                actions.append(('nodeset', action[1]))
                            else:
                                actions.append(action)

                elif term.name == u'clear':
                    target = serialize(term.arguments)
                    val, val_step = self.lookup('pending', target)
                    if val is None:
                        log(INFO, u"{} empty bucket".format(
                            target).encode('utf-8'))
                        continue
                    wastebin.extend(val)
                    del self.state[val_step]['pending'][target]

                else:
                    log(WARN, u"Unknown function {}".format(
                        term.name).encode('utf-8'))
            else:
                log(WARN, u"Unknown term {}".format(
                    term).encode('utf-8'))

        if pseudo:
            if len(actions) == current_actions:
                wastebin.append(('move', elem))

        if len(wastebin) > 0:
            trashbucket = etree.Element('div',
                                        attrib={'class': 'delete-me'})
            if actions[-1][0] == 'target':
                actions.pop()
            actions.append(('target', Target(trashbucket)))
            actions.extend(wastebin)
            wastebin = []
857421
def cleanParagraph(self):
        

        runs = self.block.content

        if not runs:
            self.block = None
            return

        if not self.clean_paragraphs:
            return

        joinedRuns = []
        hasContent = False

        for run in runs:

            if run.content[0]:
                hasContent = True
            else:
                continue

            # For whitespace-only groups, remove any property stuff,
            # to avoid extra markup in output
            if not run.content[0].strip():
                run.properties = {}

            # Join runs only if their properties match
            if joinedRuns and (run.properties == joinedRuns[-1].properties):
                joinedRuns[-1].content[0] += run.content[0]
            else:
                joinedRuns.append(run)

        if hasContent:
            # Strip beginning of paragraph
            joinedRuns[0].content[0] = joinedRuns[0].content[0].lstrip()
            # And then strip the end
            joinedRuns[-1].content[0] = joinedRuns[-1].content[0].rstrip()
            self.block.content = joinedRuns
        else:
            self.block = None
446797
def getdocumenttext(document):
    ''''''
    paratextlist = []
    # Compile a list of all paragraph (p) elements
    paralist = []
    for element in document.iter():
        # Find p (paragraph) elements
        if element.tag == '{'+nsprefixes['w']+'}p':
            paralist.append(element)
    # Since a single sentence might be spread over multiple text elements,
    # iterate through each paragraph, appending all text (t) children to that
    # paragraphs text.
    for para in paralist:
        paratext = u''
        # Loop through each paragraph
        for element in para.iter():
            # Find t (text) elements
            if element.tag == '{'+nsprefixes['w']+'}t':
                if element.text:
                    paratext = paratext+element.text
            elif element.tag == '{'+nsprefixes['w']+'}tab':
                paratext = paratext + '\t'
        # Add our completed paragraph text to the list of paragraph text
        if not len(paratext) == 0:
            paratextlist.append(paratext)
    return paratextlist
364358
def format_paragraph(paragraph, wrap_at, ansi_stack)
      cleared_para = Sanitizer.replace(paragraph)
      lines = []
      line = ''
      word = ''
      word_length = 0
      line_length = 0
      char_length = 0 # visible char length
      text_length = display_width(cleared_para)
      total_length = 0
      ansi = ''
      matched = nil
      to_chars(cleared_para) do |char|
        if char == ANSI # found ansi
          ansi << char && next
        end

        if ansi.length > 0
          ansi << char
          if Sanitizer.ansi?(ansi) # we found ansi let's consume
            matched = ansi
          elsif matched
            ansi_stack << [matched[0...-1], line_length + word_length]
            matched = nil
            ansi    = ''
          end
          next if ansi.length > 0
        end

        char_length = display_width(char)
        total_length += char_length
        if line_length + word_length + char_length <= wrap_at
          if char == SPACE || total_length == text_length
            line << word + char
            line_length += word_length + char_length
            word = ''
            word_length = 0
          else
            word << char
            word_length += char_length
          end
          next
        end

        if char == SPACE # ends with space
          lines << insert_ansi(ansi_stack, line)
          line = ''
          line_length = 0
          word += char
          word_length += char_length
        elsif word_length + char_length <= wrap_at
          lines << insert_ansi(ansi_stack, line)
          line = word + char
          line_length = word_length + char_length
          word = ''
          word_length = 0
        else # hyphenate word - too long to fit a line
          lines << insert_ansi(ansi_stack, word)
          line_length = 0
          word = char
          word_length = char_length
        end
      end
      lines << insert_ansi(ansi_stack, line) unless line.empty?
      lines << insert_ansi(ansi_stack, word) unless word.empty?
      lines
    end