Parse documents from a .txt file assuming 1 document per line, yielding lists of filtered tokens
9985
def get_texts(self):
        
        with self.getstream() as text_stream:
            for i, line in enumerate(text_stream):
                line = SMSCorpus.case_normalizer(line)
                if self.mask is not None and not self.mask[i]:
                    continue
                ngrams = []
                for ng in tokens2ngrams(self.tokenizer(line)):
                    if SMSCorpus.ignore_matcher(ng):
                        continue
                    ngrams += [ng]
                yield ngrams
1353964
def lines_from_stream(f, as_interned=False):
    
    if as_interned:
        return [sys.intern(line) for line in f.read().splitlines()]
    return f.read().splitlines()
998302
def get_tokens_unprocessed(self, text, stack=('root',)):
        """"""
        self.content_type = None
        return RegexLexer.get_tokens_unprocessed(self, text, stack)
395010
function tokenize(content, rules = parseOrder) {
    let data = []
    if(content) {
        let lines = content.split(/(\r\n|\n)/)
        for (let idx = 0; idx < lines.length; idx++) {
            for (let i = 0; i < lines[idx].length;) {
                let chunk = lines[idx].slice(i)
                let match = rules.some(item => {
                    if (item.pattern.test(chunk)) {
                        let matchObject = chunk.match(item.pattern)
                        i += matchObject[0].length
                        data.push([ item.label, matchObject[0] ])
                        return true
                    }
                })
                if (!match) {
                    data.push([ UNKNOWN, chunk[i] ])
                    i++
                }
            }
        }
    }
    return data
}
658436
def _parse_document(self):
        """"""
        self._reset()

        doc = self._profile_doc

        self._split_tokens_calculated = True
        self._split_tokens = None

        self._duration_calculated = True
        self._duration = doc[u'millis']

        self._datetime_calculated = True
        self._datetime = doc[u'ts']
        if self._datetime.tzinfo is None:
            self._datetime = self._datetime.replace(tzinfo=tzutc())
        self._datetime_format = None
        self._reformat_timestamp('ctime', force=True)

        self._thread_calculated = True
        self._thread = doc['thread']

        self._operation_calculated = True
        self._operation = doc[u'op']
        self._namespace = doc[u'ns']

        self._command_calculated = True
        if self.operation == 'command':
            self._command = doc[u'command'].keys()[0]

        # query pattern for system.profile events, all three cases.
        # See SERVER-13245
        if 'query' in doc:
            if 'query' in doc['query'] and isinstance(doc['query']['query'],
                                                      dict):
                self._pattern = str(doc['query']['query']).replace("'", '"')
            elif '$query' in doc['query']:
                self._pattern = str(doc['query']['$query']).replace("'", '"')
            else:
                self._pattern = str(doc['query']).replace("'", '"')

            # sort pattern
            if ('orderby' in doc['query'] and
                    isinstance(doc['query']['orderby'], dict)):
                self._sort_pattern = str(doc['query']
                                         ['orderby']).replace("'", '"')
            elif '$orderby' in doc['query']:
                self._sort_pattern = str(doc['query']
                                         ['$orderby']).replace("'", '"')
            else:
                self._sort_pattern = None

        self._counters_calculated = True
        self._nscanned = doc[u'nscanned'] if 'nscanned' in doc else None
        self._ntoreturn = doc[u'ntoreturn'] if 'ntoreturn' in doc else None
        self._nupdated = doc[u'nupdated'] if 'nupdated' in doc else None
        self._nreturned = doc[u'nreturned'] if 'nreturned' in doc else None
        self._ninserted = doc[u'ninserted'] if 'ninserted' in doc else None
        self._ndeleted = doc[u'ndeleted'] if 'ndeleted' in doc else None
        self._numYields = doc[u'numYield'] if 'numYield' in doc else None

        if u'lockStats' in doc:
            self._r = doc[u'lockStats'][u'timeLockedMicros'][u'r']
            self._w = doc[u'lockStats'][u'timeLockedMicros'][u'w']
            self._r_acquiring = doc[u'lockStats']['timeAcquiringMicros'][u'r']
            self._w_acquiring = doc[u'lockStats']['timeAcquiringMicros'][u'w']
            locks = 'w:%i' % self.w if self.w is not None else 'r:%i' % self.r
        elif u'locks' in doc:
            locks = json.dumps(doc[u'locks'])
        else:
            locks = ''

        # build a fake line_str
        payload = ''
        if 'query' in doc:
            payload += ('query: %s' % str(doc[u'query'])
                        .replace("u'", "'").replace("'", '"'))
        if 'command' in doc:
            payload += ('command: %s' % str(doc[u'command'])
                        .replace("u'", "'").replace("'", '"'))
        if 'updateobj' in doc:
            payload += (' update: %s' % str(doc[u'updateobj'])
                        .replace("u'", "'").replace("'", '"'))

        scanned = 'nscanned:%i' % self._nscanned if 'nscanned' in doc else ''
        yields = 'numYields:%i' % self._numYields if 'numYield' in doc else ''
        duration = '%ims' % self.duration if self.duration is not None else ''

        self._line_str = ("[{thread}] {operation} {namespace} {payload} "
                          "{scanned} {yields} locks(micros) {locks} "
                          "{duration}".format(datetime=self.datetime,
                                              thread=self.thread,
                                              operation=self.operation,
                                              namespace=self.namespace,
                                              payload=payload, scanned=scanned,
                                              yields=yields, locks=locks,
                                              duration=duration))
1328020
def __clean_and_tokenize(self, doc_list):
        
        # Some repositories fill entire documentation in description. We ignore
        # such repositories for cleaner tokens.
        doc_list = filter(
            lambda x: x is not None and len(x) <= GitSuggest.MAX_DESC_LEN,
            doc_list,
        )

        cleaned_doc_list = list()

        # Regular expression to remove out all punctuations, numbers and other
        # un-necessary text substrings like emojis etc.
        tokenizer = RegexpTokenizer(r"[a-zA-Z]+")

        # Get stop words.
        stopwords = self.__get_words_to_ignore()

        # Get english words.
        dict_words = self.__get_words_to_consider()

        for doc in doc_list:
            # Lowercase doc.
            lower = doc.lower()

            # Tokenize removing numbers and punctuation.
            tokens = tokenizer.tokenize(lower)

            # Include meaningful words.
            tokens = [tok for tok in tokens if tok in dict_words]

            # Remove stopwords.
            tokens = [tok for tok in tokens if tok not in stopwords]

            # Filter Nones if any are introduced.
            tokens = [tok for tok in tokens if tok is not None]

            cleaned_doc_list.append(tokens)

        return cleaned_doc_list
22788
def parse_doc(doc):
    
    if not doc:
        return {}
    out = {}
    for s in doc.split('\n'):
        s = s.strip().split(':', maxsplit=1)
        if len(s) == 2:
            out[s[0]] = s[1]
    return out
834699
def process_document(self, doc: Document):
        

        text_segments = doc.select_segments("lexisnexis.doc_description")
        for text_segment in text_segments:
            split_sentences = doc.extract(self.sentence_extractor, text_segment)
            doc.store(split_sentences, 'split_sentences')
        # for t, u in zip(text_to_be_split, units_of_text):
        #     split_sentences = doc.extract(self.sentence_extractor, t)
        #     u.store(split_sentences, "split_sentences")
        return list()
314271
def filter_lines(input_file, output_file, translate=lambda line: line):
    
    filepath, lines = get_lines([input_file])[0]
    return filepath, [(tag, translate(line=line, tag=tag)) for (tag, line) in lines]
945011
def filter_lines(lines, filter_regex, groups=None):
    
    pattern = re.compile(filter_regex)
    for line in lines:
        match = pattern.search(line)
        if match:
            if groups is None:
                yield line
            elif len(groups) == 1:
                yield match.group(groups[0])
            else:
                matched_groups = match.groupdict()
                yield tuple(matched_groups.get(group) for group in groups)
195117
def _doc2vec_doc_stream(paths, n, tokenizer=word_tokenize, sentences=True):
    
    i = 0
    p = Progress()
    for path in paths:
        with open(path, 'r') as f:
            for line in f:
                i += 1
                p.print_progress(i/n)

                # We do minimal pre-processing here so the model can learn
                # punctuation
                line = line.lower()

                if sentences:
                    for sent in sent_tokenize(line):
                        tokens = tokenizer(sent)
                        yield LabeledSentence(tokens, ['SENT_{}'.format(i)])
                else:
                    tokens = tokenizer(line)
                    yield LabeledSentence(tokens, ['SENT_{}'.format(i)])
568523
def parse(self, line):
        """"""
        tokens = []

        self.idx = -1   # Bogus value to ensure idx = 0 after first iteration
        self.characters = iter(line)
        self.update_chars()

        while self.char != '\n':
            # Update namelist group status
            if self.char in ('&', '$'):
                self.group_token = self.char

            if self.group_token and (
                    (self.group_token, self.char) in (('&', '/'), ('$', '$'))):
                self.group_token = False

            word = ''
            if self.char in self.whitespace:
                while self.char in self.whitespace:
                    word += self.char
                    self.update_chars()

            elif self.char in ('!', '#') or self.group_token is None:
                # Abort the iteration and build the comment token
                word = line[self.idx:-1]
                self.char = '\n'

            elif self.char in '"\'' or self.prior_delim:
                word = self.parse_string()

            elif self.char.isalpha():
                word = self.parse_name(line)

            elif self.char in ('+', '-'):
                # Lookahead to check for IEEE value
                self.characters, lookahead = itertools.tee(self.characters)
                ieee_val = ''.join(itertools.takewhile(str.isalpha, lookahead))
                if ieee_val.lower() in ('inf', 'infinity', 'nan'):
                    word = self.char + ieee_val
                    self.characters = lookahead
                    self.prior_char = ieee_val[-1]
                    self.char = next(lookahead, '\n')
                else:
                    word = self.parse_numeric()

            elif self.char.isdigit():
                word = self.parse_numeric()

            elif self.char == '.':
                self.update_chars()
                if self.char.isdigit():
                    frac = self.parse_numeric()
                    word = '.' + frac
                else:
                    word = '.'
                    while self.char.isalpha():
                        word += self.char
                        self.update_chars()
                    if self.char == '.':
                        word += self.char
                        self.update_chars()

            elif self.char in Tokenizer.punctuation:
                word = self.char
                self.update_chars()

            else:
                # This should never happen
                raise ValueError

            tokens.append(word)

        return tokens
733002
def tokenize(contents):
    """"""
    tokens = _scan_for_tokens(contents)
    tokens = _compress_tokens(tokens)
    tokens = [token for token in tokens if token.type != TokenType.Whitespace]
    return tokens
114753
def getblock(lines):
    """"""
    try:
        tokenize.tokenize(ListReader(lines).readline, BlockFinder().tokeneater)
    except EndOfBlock, eob:
        return lines[:eob.args[0]]
    # Fooling the indent/dedent logic implies a one-line definition
    return lines[:1]
73126
def parse(tokens):
    """"""
    d = collections.OrderedDict()
    prev_line = 0
    blockname = None
    blockline = None
    for token in tokens:
        if token.type == 'COMMENT':
            continue
        elif token.type == 'BLOCK':
            block = token
            blockline = token.line
            blocktype = token.value.upper()
            blockname = None
            if blocktype not in d:
                d[blocktype] = collections.OrderedDict()
        elif token.line == blockline:
            if blockname is None:
                blockname = token.value
                d[blocktype][blockname] = collections.defaultdict(list)
            else:
                d[blocktype][blockname]['info'].append(numval(token))
        elif token.line != prev_line:
            if blockname is None:
                raise ParseError("Found value outside block!")
            d[blocktype][blockname]['values'].append([numval(token)])
        else:
            if blockname is None:
                raise ParseError("Found value outside block!")
            d[blocktype][blockname]['values'][-1].append(numval(token))
        prev_line = token.line
    return d
247551
def listified_tokenizer(source):
    """"""
    io_obj = io.StringIO(source)
    return [list(a) for a in tokenize.generate_tokens(io_obj.readline)]
787689
def parse_strings(content="", filename=None):
    
    if filename is not None:
        content = _get_content(filename=filename)

    stringset = []
    f = content
    if f.startswith(u'\ufeff'):
        f = f.lstrip(u'\ufeff')
    #regex for finding all comments in a file
    cp = r'(?:/\*(?P<comment>(?:[^*]|(?:\*+[^*/]))*\**)\*/)'
    p = re.compile(r'(?:%s[ \t]*[\n]|[\r\n]|[\r]){0,1}(?P<line>(("(?P<key>[^"\\]*(?:\\.[^"\\]*)*)")|(?P<property>\w+))\s*=\s*"(?P<value>[^"\\]*(?:\\.[^"\\]*)*)"\s*;)'%cp, re.DOTALL|re.U)
    #c = re.compile(r'\s*/\*(.|\s)*?\*/\s*', re.U)
    c = re.compile(r'//[^\n]*\n|/\*(?:.|[\r\n])*?\*/', re.U)
    ws = re.compile(r'\s+', re.U)
    end=0
    start = 0
    for i in p.finditer(f):
        start = i.start('line')
        end_ = i.end()
        key = i.group('key')
        comment = i.group('comment') or ''
        if not key:
            key = i.group('property')
        value = i.group('value')
        while end < start:
            m = c.match(f, end, start) or ws.match(f, end, start)
            if not m or m.start() != end:
                print("Invalid syntax: %s" %\
                        f[end:start])
            end = m.end()
        end = end_
        key = _unescape_key(key)
        stringset.append({'key': key, 'value': _unescape(value), 'comment': comment})
    return stringset