Build compound regex from list of tokens
167694
def build_regex(self, tokens):
        """"""
        patterns = []
        for token in tokens:
            patterns.append(token.pattern_start)
            if token.pattern_end:
                patterns.append(token.pattern_end)
        return re.compile('|'.join(patterns), re.DOTALL)
586992
def tokeniter(self, source, name, filename=None, state=None):
        
        source = '\n'.join(unicode(source).splitlines())
        pos = 0
        lineno = 1
        stack = ['root']
        if state is not None and state != 'root':
            assert state in ('variable', 'block'), 'invalid state'
            stack.append(state + '_begin')
        else:
            state = 'root'
        statetokens = self.rules[stack[-1]]
        source_length = len(source)

        balancing_stack = []

        while 1:
            # tokenizer loop
            for regex, tokens, new_state in statetokens:
                m = regex.match(source, pos)
                # if no match we try again with the next rule
                if m is None:
                    continue

                # we only match blocks and variables if brances / parentheses
                # are balanced. continue parsing with the lower rule which
                # is the operator rule. do this only if the end tags look
                # like operators
                if balancing_stack and \
                   tokens in ('variable_end', 'block_end',
                              'linestatement_end'):
                    continue

                # tuples support more options
                if isinstance(tokens, tuple):
                    for idx, token in enumerate(tokens):
                        # failure group
                        if token.__class__ is Failure:
                            raise token(lineno, filename)
                        # bygroup is a bit more complex, in that case we
                        # yield for the current token the first named
                        # group that matched
                        elif token == '#bygroup':
                            for key, value in m.groupdict().iteritems():
                                if value is not None:
                                    yield lineno, key, value
                                    lineno += value.count('\n')
                                    break
                            else:
                                raise RuntimeError('%r wanted to resolve '
                                                   'the token dynamically'
                                                   ' but no group matched'
                                                   % regex)
                        # normal group
                        else:
                            data = m.group(idx + 1)
                            if data or token not in ignore_if_empty:
                                yield lineno, token, data
                            lineno += data.count('\n')

                # strings as token just are yielded as it.
                else:
                    data = m.group()
                    # update brace/parentheses balance
                    if tokens == 'operator':
                        if data == '{':
                            balancing_stack.append('}')
                        elif data == '(':
                            balancing_stack.append(')')
                        elif data == '[':
                            balancing_stack.append(']')
                        elif data in ('}', ')', ']'):
                            if not balancing_stack:
                                raise TemplateSyntaxError('unexpected \'%s\'' %
                                                          data, lineno, name,
                                                          filename)
                            expected_op = balancing_stack.pop()
                            if expected_op != data:
                                raise TemplateSyntaxError('unexpected \'%s\', '
                                                          'expected \'%s\'' %
                                                          (data, expected_op),
                                                          lineno, name,
                                                          filename)
                    # yield items
                    if data or tokens not in ignore_if_empty:
                        yield lineno, tokens, data
                    lineno += data.count('\n')

                # fetch new position into new variable so that we can check
                # if there is a internal parsing error which would result
                # in an infinite loop
                pos2 = m.end()

                # handle state changes
                if new_state is not None:
                    # remove the uppermost state
                    if new_state == '#pop':
                        stack.pop()
                    # resolve the new state by group checking
                    elif new_state == '#bygroup':
                        for key, value in m.groupdict().iteritems():
                            if value is not None:
                                stack.append(key)
                                break
                        else:
                            raise RuntimeError('%r wanted to resolve the '
                                               'new state dynamically but'
                                               ' no group matched' %
                                               regex)
                    # direct state name given
                    else:
                        stack.append(new_state)
                    statetokens = self.rules[stack[-1]]
                # we are still at the same position and no stack change.
                # this means a loop without break condition, avoid that and
                # raise error
                elif pos2 == pos:
                    raise RuntimeError('%r yielded empty string without '
                                       'stack change' % regex)
                # publish new function and start again
                pos = pos2
                break
            # if loop terminated without break we havn't found a single match
            # either we are at the end of the file or we have a problem
            else:
                # end of text
                if pos >= source_length:
                    return
                # something went wrong
                raise TemplateSyntaxError('unexpected char %r at %d' %
                                          (source[pos], pos), lineno,
                                          name, filename)
1185828
def join(tokens, start, result):
    """"""
    texts = []
    if len(result) > 0:
        for e in result:
            for child in e.iter():
                if child.text is not None:
                    texts.append(child.text)
        return [E(result[0].tag, ' '.join(texts))]
1267516
def _group(tlist, cls, match,
           valid_prev=lambda t: True,
           valid_next=lambda t: True,
           post=None,
           extend=True,
           recurse=True
           ):
    """"""

    tidx_offset = 0
    pidx, prev_ = None, None
    for idx, token in enumerate(list(tlist)):
        tidx = idx - tidx_offset

        if token.is_whitespace:
            continue

        if recurse and token.is_group and not isinstance(token, cls):
            _group(token, cls, match, valid_prev, valid_next, post, extend)

        if match(token):
            nidx, next_ = tlist.token_next(tidx)
            if prev_ and valid_prev(prev_) and valid_next(next_):
                from_idx, to_idx = post(tlist, pidx, tidx, nidx)
                grp = tlist.group_tokens(cls, from_idx, to_idx, extend=extend)

                tidx_offset += to_idx - from_idx
                pidx, prev_ = from_idx, grp
                continue

        pidx, prev_ = tidx, token
1090033
def _get_tokens_and_tags(parse_str):
  """"""
  tokens = []
  parse_split = parse_str.split(' ')
  for p in parse_split:
    assert p.startswith('(') or p.endswith(')')
    if p.endswith(')'):
      token = p.replace(')', '')
      tokens.append(token)

  return tokens
926321
def _process_regex(cls, regex, rflags, state):
        """"""
        if isinstance(regex, Future):
            regex = regex.get()
        return re.compile(regex, rflags).match
993419
def build_tokens_line(self):
        """"""
        logical = []
        comments = []
        length = 0
        prev_row = prev_col = mapping = None
        for token_type, text, start, end, line in self.tokens:
            if token_type in SKIP_TOKENS:
                continue
            if not mapping:
                mapping = [(0, start)]
            if token_type == tokenize.COMMENT:
                comments.append(text)
                continue
            if token_type == tokenize.STRING:
                text = mute_string(text)
            if prev_row:
                (start_row, start_col) = start
                if prev_row != start_row:    # different row
                    prev_text = self.lines[prev_row - 1][prev_col - 1]
                    if prev_text == ',' or (prev_text not in '{[(' and
                                            text not in '}])'):
                        text = ' ' + text
                elif prev_col != start_col:  # different column
                    text = line[prev_col:start_col] + text
            logical.append(text)
            length += len(text)
            mapping.append((length, end))
            (prev_row, prev_col) = end
        self.logical_line = ''.join(logical)
        self.noqa = comments and noqa(''.join(comments))
        return mapping
40545
def _init_from_list(self, subwords):
    """"""
    subwords = [tf.compat.as_text(s) for s in subwords if s]
    self._subwords = subwords
    # Note that internally everything is 0-indexed. Padding is dealt with at the
    # end of encode and the beginning of decode.
    self._subword_to_id = {s: i for i, s in enumerate(subwords)}

    # We remember the maximum length of any subword to avoid having to
    # check arbitrarily long strings.
    self._max_subword_len = max(
        len(_UNDERSCORE_REPLACEMENT), max([len(s) for s in subwords] or [1]))

    # Initialize the cache
    self._cache_size = 2**20
    self._token_to_ids_cache = [(None, None)] * self._cache_size

    # Setup tokenizer
    # Reserved tokens are all tokens that are mixed alphanum and non-alphanum.
    reserved_tokens = set([_UNDERSCORE_REPLACEMENT])
    for t in self._subwords:
      if text_encoder.is_mixed_alphanum(t):
        reserved_tokens.add(t)
    self._tokenizer = text_encoder.Tokenizer(
        alphanum_only=False, reserved_tokens=reserved_tokens)
905914
def convert_tokens_into_matrix(self, token_list):
        '''
        
        '''
        return np.array(self.vectorize(token_list)).astype(np.float32)
1205305
def split_token_in_parts(token):
    
    result = []
    current = []
    for part in token + (':', ):
        if part == ':':
            if current:
                result.append(tuple(current))
                current = []
        else:
            current.append(part)

    return result
395010
function tokenize(content, rules = parseOrder) {
    let data = []
    if(content) {
        let lines = content.split(/(\r\n|\n)/)
        for (let idx = 0; idx < lines.length; idx++) {
            for (let i = 0; i < lines[idx].length;) {
                let chunk = lines[idx].slice(i)
                let match = rules.some(item => {
                    if (item.pattern.test(chunk)) {
                        let matchObject = chunk.match(item.pattern)
                        i += matchObject[0].length
                        data.push([ item.label, matchObject[0] ])
                        return true
                    }
                })
                if (!match) {
                    data.push([ UNKNOWN, chunk[i] ])
                    i++
                }
            }
        }
    }
    return data
}
876465
def completer_tokenize(cls, value, min_length=3):
        ''''''
        tokens = list(itertools.chain(*[
            [m for m in n.split("'") if len(m) > min_length]
            for n in value.split(' ')
        ]))
        return list(set([value] + tokens + [' '.join(tokens)]))
852520
def get_compound_regex(schema='mona'):
    

    # NOTE: will just ignore cases in the regex, to avoid repetition here
    meta_parse = collections.OrderedDict()

    if schema == 'mona':
        meta_parse['name'] = ['^Name(?:=|:)(.*)$']
        meta_parse['inchikey_id'] = ['^inchikey(?:=|:)(.*)$']
        meta_parse['molecular_formula'] = ['^molecular formula(?:=|:)(.*)$', '^formula:(.*)$']
        meta_parse['molecular_weight'] = ['^MW(?:=|:)(\d*[.,]?\d*)$']
        meta_parse['pubchem_id'] = ['^pubchem.*cid(?:=|:)(\d*)".*$']
        meta_parse['chemspider_id'] = ['^chemspider(?:=|:)(\d*)".*$']
        meta_parse['compound_class'] = ['^compound.*class(?:=|:)(.*)$']
        meta_parse['exact_mass'] = ['^exact.*mass(?:=|:)(\d*[.,]?\d*)$']
        meta_parse['smiles'] = ['^SMILES(?:=|:)(.*)$']
        meta_parse['other_names'] = ['^Synonym(?:=|:)(.*)$']
    elif schema == 'massbank':
        meta_parse['name'] = ['^CH\$NAME:\s+(.*)$']
        meta_parse['other_names'] = ['^CH\$NAME:\s+(.*)$']

        meta_parse['inchikey_id'] = ['^CH\$LINK:\s+INCHIKEY\s+(.*)$']
        meta_parse['molecular_formula'] = ['^CH\$FORMULA:\s+(.*)$']
        meta_parse['molecular_weight'] = ['^CH\$MOLECULAR_WEIGHT:\s+(.*)$']
        meta_parse['pubchem_id'] = ['^CH\$LINK:\s+PUBCHEM\s+CID:(.*)$']
        meta_parse['chemspider_id'] = ['^CH\$LINK:\s+CHEMSPIDER\s+(.*)$']
        meta_parse['compound_class'] = ['^CH\$COMPOUND_CLASS:\s+(.*)$']
        meta_parse['exact_mass'] = ['^CH\$EXACT_MASS:\s+(.*)$']
        meta_parse['smiles'] = ['^CH\$SMILES:\s+(.*)$']

    return meta_parse
1123136
def all_matches(grammar, text):
    """"""
    for tokens, start, stop in grammar.parseWithTabs().scanString(text):
        yield unpack(tokens), start, stop
1082567
def _scan_for_tokens(contents):
    """"""
    # Regexes are in priority order. Changing the order may alter the
    # behavior of the lexer
    scanner = re.Scanner([
        # Things inside quotes
        (r"(?<![^\s\(])([\"\'])(?:(?=(\\?))\2.)*?\1(?![^\s\)])",
         lambda s, t: (TokenType.QuotedLiteral, t)),
        # Numbers on their own
        (r"(?<![^\s\(])-?[0-9]+(?![^\s\)\(])", lambda s, t: (TokenType.Number,
                                                             t)),
        # Left Paren
        (r"\(", lambda s, t: (TokenType.LeftParen, t)),
        # Right Paren
        (r"\)", lambda s, t: (TokenType.RightParen, t)),
        # Either a valid function name or variable name.
        (r"(?<![^\s\(])[a-zA-z_][a-zA-Z0-9_]*(?![^\s\)\(])",
         lambda s, t: (TokenType.Word, t)),
        # Variable dereference.
        (r"(?<![^\s\(])\${[a-zA-z_][a-zA-Z0-9_]*}(?![^\s\)])",
         lambda s, t: (TokenType.Deref, t)),
        # Newline
        (r"\n", lambda s, t: (TokenType.Newline, t)),
        # Whitespace
        (r"\s+", lambda s, t: (TokenType.Whitespace, t)),
        # The beginning of a double-quoted string, terminating at end of line
        (r"(?<![^\s\(\\])[\"]([^\"]|\\[\"])*$",
         lambda s, t: (TokenType.BeginDoubleQuotedLiteral, t)),
        # The end of a double-quoted string
        (r"[^\s]*(?<!\\)[\"](?![^\s\)])",
         lambda s, t: (TokenType.EndDoubleQuotedLiteral, t)),
        # The beginning of a single-quoted string, terminating at end of line
        (r"(?<![^\s\(\\])[\']([^\']|\\[\'])*$",
         lambda s, t: (TokenType.BeginSingleQuotedLiteral, t)),
        # The end of a single-quoted string
        (r"[^\s]*(?<!\\)[\'](?![^\s\)])",
         lambda s, t: (TokenType.EndSingleQuotedLiteral, t)),
        # Begin-RST Comment Block
        (r"#.rst:$", lambda s, t: (TokenType.BeginRSTComment, t)),
        # Begin Inline RST
        (r"#\[=*\[.rst:$", lambda s, t: (TokenType.BeginInlineRST, t)),
        # End Inline RST
        (r"#\]=*\]$", lambda s, t: (TokenType.EndInlineRST, t)),
        # Comment
        (r"#", lambda s, t: (TokenType.Comment, t)),
        # Catch-all for literals which are compound statements.
        (r"([^\s\(\)]+|[^\s\(]*[^\)]|[^\(][^\s\)]*)",
         lambda s, t: (TokenType.UnquotedLiteral, t))
    ])

    tokens_return = []

    lines = contents.splitlines(True)
    lineno = 0
    for line in lines:
        lineno += 1
        col = 1

        tokens, remaining = scanner.scan(line)
        if remaining != "":
            msg = "Unknown tokens found on line {0}: {1}".format(lineno,
                                                                 remaining)
            raise RuntimeError(msg)

        for token_type, token_contents in tokens:
            tokens_return.append(Token(type=token_type,
                                       content=token_contents,
                                       line=lineno,
                                       col=col))

            col += len(token_contents)

    return tokens_return
1117433
def parse_compound(self):
        """"""
        compound_tag = Compound()

        for token in self.collect_tokens_until('CLOSE_COMPOUND'):
            item_key = token.value
            if token.type not in ('NUMBER', 'STRING', 'QUOTED_STRING'):
                raise self.error(f'Expected compound key but got {item_key!r}')

            if token.type == 'QUOTED_STRING':
                item_key = self.unquote_string(item_key)

            if self.next().current_token.type != 'COLON':
                raise self.error(f'Expected colon but got '
                                 f'{self.current_token.value!r}')
            self.next()
            compound_tag[item_key] = self.parse()
        return compound_tag
573127
def generate(regex, Ns):
    ""
    return sorted(regex_parse(regex)[0](Ns),
                  key=lambda s: (len(s), s))