:param text: takes the whole text of an article, searches for references, cleans the text,
    marks the reference indeces from zero inside the text.
    :return: the tagged text and a tag:reference dictionary to be used in sectionParser
778079
def referencesFinder(text):
    
    references = referencesRegEx.finditer(text)
    count = 0
    refs = []
    spans = []
    for i in references:
        refs.append(i.group())
        spans.append(i.span())
        count += 1
    done = set()
    nameRegEx = re.compile(r"""(name=["']*.*?["']*)(\s|/|>)""")

    for index, obj in enumerate(refs):

        if obj.startswith('<ref name='):
                nameTag = re.escape(nameRegEx.search(obj).group(1))

                if nameTag not in done:
                    nameTag = re.escape(nameRegEx.search(obj).group(1))
                    indeces = [i for i, x in enumerate(refs) if re.search(nameTag, x)]
                    matches = [refs[i] for i in indeces]
                    full = max(matches, key=len)

                    for i in indeces:
                        refs[i] = full

                    done.add(nameTag)

    #eliminate <ref tag or other rudiments from the ref string

    for i in range(len(refs)):
        #print('SIIT', refs[i])
        lastindex = refs[i].rindex('<')
        firstindex = refs[i].index('>')+1
        refs[i]=refs[i][firstindex:lastindex]

    #Handle cite-references

    for i in range(len(refs)):
        if 'cite' in refs[i].lower():
            newText = ''
            values = refs[i].split('|')
            for j in values:
                if '=' in j:
                    first = j.index('=')
                    newText += j[first+1:].strip() + ';'
            refs[i] = newText
    #a ref string:position int dictionary
    refspos = {}
    c = 0
    for i in refs:
        if i not in refspos.keys():
            refspos[i] = c
            c +=1
        else:
            continue

    #print(refspos)

    #eliminate old, bad <ref> tags and insert clean  ones <ref 1..2..3/> to the same spot.
    newText = ''
    assert len(spans) == len(refs)
    #Could happen... havent yet.
    next = 0
    for i in range(len(spans)):
        start = spans[i][0]
        newText+=text[next:start]+'<ref '+str(refspos[refs[i]])+'/>'
        next = spans[i][1]

    newText+=text[next:]

    #switch keys:values in the dictionary for use in sectionsParser
    #positiontag:ref
    newDict  = {y:x for x,y in refspos.items()}

    return newText, newDict
435855
public static String clear(String text) {
        text = StringCleaner.tokenizeURLs(text);
        text = StringCleaner.tokenizeSmileys(text);
        text = StringCleaner.removeAccents(text);
        text = StringCleaner.removeSymbols(text);
        text = StringCleaner.removeExtraSpaces(text);
        
        return text.toLowerCase(Locale.ENGLISH);
    }
952345
def text(self, cls = 'current', retaintokenisation=False, previousdelimiter="",strict=False, correctionhandling=CorrectionHandling.CURRENT, normalize_spaces=False):
        """"""
        if cls == 'original': correctionhandling = CorrectionHandling.ORIGINAL #backward compatibility
        if correctionhandling in (CorrectionHandling.CURRENT, CorrectionHandling.EITHER):
            for e in self:
                if isinstance(e, New) or isinstance(e, Current):
                    s = previousdelimiter + e.text(cls, retaintokenisation,"", strict, correctionhandling)
                    if normalize_spaces:
                        return norm_spaces(s)
                    else:
                        return s
        if correctionhandling in (CorrectionHandling.ORIGINAL, CorrectionHandling.EITHER):
            for e in self:
                if isinstance(e, Original):
                    s =  previousdelimiter + e.text(cls, retaintokenisation,"", strict, correctionhandling)
                    if normalize_spaces:
                        return norm_spaces(s)
                    else:
                        return s
        raise NoSuchText
244300
def clean_text(self, text):
        ''''''
        if text is None:
            return ''
        text = re.sub(ILLEGAL_CHARACTERS_RE, '', text)
        if '<' in text or '&lt' in text:
            text = clean(text, tags=self.tags, strip=self.strip)

        return unescape(text)
686740
def clause_texts(self):
        
        if not self.is_tagged(CLAUSES):
            self.tag_clauses()
        return self.texts(CLAUSES)
1302431
def parse(self, text):
        
        #: Do not process empty strings (Issue #3)
        if text.strip() == "":
            return ""
        #: Do not process strings consisting of a single punctuation mark (Issue #4)
        elif text.strip() in PUNCTUATION:
            _sym = text.strip()
            if _sym in tuple('.?!'):
                _tag = "."
            else:
                _tag = _sym
            if self.lemmata:
                return "{0}/{1}/O/O/{0}".format(_sym, _tag)
            else:
                return "{0}/{1}/O/O".format(_sym, _tag)
        if self.tokenize:
            _tokenized = " ".join(self.tokenizer.tokenize(text))
        else:
            _tokenized = text

        _parsed = pattern_parse(_tokenized,
                                # text is tokenized before it is passed on to
                                # pattern.de.parse
                                tokenize=False,
                                tags=self.tags, chunks=self.chunks,
                                relations=self.relations, lemmata=self.lemmata,
                                encoding=self.encoding, tagset=self.tagset)
        if self.pprint:
            _parsed = pattern_pprint(_parsed)

        return _parsed
709145
def macronize_text(self, text):
        
        macronized_words = [entry[2] for entry in self.macronize_tags(text)]
        return " ".join(macronized_words)
1180628
def parse(self, text, element, context='eqn'):
        
        
        # Remove the inline comments from `text` before parsing the grammar
        # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039973
        text = re.sub(r"\{[^}]*\}", "", text)
        
        self.ast = self.grammar.parse(text)
        self.context = context
        self.element = element
        self.new_structure = []
        
        py_expr = self.visit(self.ast)
        
        return ({
            'py_expr': py_expr
        }, self.new_structure)
741783
def from_text(cls, text, mapping='mapping'):
        
        graphemes = Counter(grapheme_pattern.findall(text))
        specs = [
            OrderedDict([
                (cls.GRAPHEME_COL, grapheme),
                ('frequency', frequency),
                (mapping, grapheme)])
            for grapheme, frequency in graphemes.most_common()]
        return cls(*specs)
634973
def extract_from_text(text):
    
    return tools.remove_duplicates([i[0]
                                    for i in REGEX.findall(text) if i != ''])
1220381
def get_reference_section_beginning(fulltext):
    """"""
    sect_start = {
        'start_line': None,
        'end_line': None,
        'title_string': None,
        'marker_pattern': None,
        'marker': None,
        'how_found_start': None,
    }

    # Find start of refs section:
    sect_start = find_reference_section(fulltext)
    if sect_start is not None:
        sect_start['how_found_start'] = 1
    else:
        # No references found - try with no title option
        sect_start = find_reference_section_no_title_via_brackets(fulltext)
        if sect_start is not None:
            sect_start['how_found_start'] = 2
        # Try weaker set of patterns if needed
        if sect_start is None:
            # No references found - try with no title option (with weaker
            # patterns..)
            sect_start = find_reference_section_no_title_via_dots(fulltext)
            if sect_start is not None:
                sect_start['how_found_start'] = 3
            if sect_start is None:
                # No references found - try with no title option (with even
                # weaker patterns..)
                sect_start = find_reference_section_no_title_via_numbers(
                    fulltext)
                if sect_start is not None:
                    sect_start['how_found_start'] = 4

    if sect_start:
        current_app.logger.debug('* title %r' % sect_start['title_string'])
        current_app.logger.debug('* marker %r' % sect_start['marker'])
        current_app.logger.debug('* title_marker_same_line %s'
                                 % sect_start['title_marker_same_line'])
    else:
        current_app.logger.debug('* could not find references section')
    return sect_start
458403
def blocks_by_text(cls, text):
        
        text = kurt.BlockType._strip_text(text)
        matches = []
        for block in cls.blocks:
            for pbt in block.conversions:
                if pbt.stripped_text == text:
                    matches.append(block)
                    break
        return matches
735997
def parse_line(text):
    
    indent,text = calculate_indent(text)
    results = line_parser.parseString(text, parseAll=True).asList()
    return indent,results[0]
615107
def _retrieve_tag(self, text):
        
        if self.tagger == 'tag_ngram_123_backoff':  # Data format: Perseus Style (see https://github.com/cltk/latin_treebank_perseus)
            tags = POSTag('latin').tag_ngram_123_backoff(text.lower())
            return [(tag[0], tag[1]) for tag in tags]
        elif self.tagger == 'tag_tnt':
            tags = POSTag('latin').tag_tnt(text.lower())
            return [(tag[0], tag[1]) for tag in tags]
        elif self.tagger == 'tag_crf':
            tags = POSTag('latin').tag_crf(text.lower())
            return [(tag[0], tag[1]) for tag in tags]
448333
public static String getTextInsideTag(String text, String tag)
	{
		return getTextInsideTag(text, tag, EMPTY_MAP);
	}
555992
def predict(self, text):
        
        pred = self.predict_proba(text)
        tags = self._get_tags(pred)

        return tags
91720
def clean_text(text):
    
    if text:
        text = html2text.html2text(clean_markdown(text))
        return re.sub(r'\s+', ' ', text).strip()