Emits a token using the current pattern match and pattern label.
763268
def emit(self, string, match, pattern, **_):
        """"""
        return grammar.Token(name=pattern.name, value=string,
                             start=match.start(), end=match.end())
220085
def tokenize(code):
    """"""
    tok_regex = '|'.join('(?P<{}>{})'.format(*pair) for pair in _tokens)
    tok_regex = re.compile(tok_regex, re.IGNORECASE|re.M)
    line_num = 1
    line_start = 0
    for mo in re.finditer(tok_regex, code):
        kind = mo.lastgroup
        value = mo.group(kind)
        if kind == 'NEWLINE':
            line_start = mo.end()
            line_num += 1
        elif kind == 'SKIP' or value=='':
            pass
        else:
            column = mo.start() - line_start
            yield Token(kind, value, line_num, column)
599024
def match(self, tokens, item):
        """"""
        for flag, get_handler in self.matchers.items():
            if flag in tokens:
                return get_handler(self)(tokens, item)
        raise CoconutInternalException("invalid pattern-matching tokens", tokens)
43820
def _emitHMM(self, token_type, past_states, past_emissions):
        
        assert token_type in self.emissions
        return utils.weighted_choice(self.emissions[token_type].items())
686182
public CoreLabel makeToken(String tokenText, int begin, int length) {
    return makeToken(tokenText, tokenText, begin, length);
  }
372723
def update_text(self, token, match):
        """"""
        if isinstance(self.text, MatchGroup):
            self.text = self.text.get_group_value(token, match)
1254100
def get_matched_token(self, match):
        """"""
        match_groupdict = match.groupdict()
        for group in self.groups:
            if match_groupdict[group] is not None:
                token, match_type = self.groups[group]
                return (token, match_type, group)
759022
function PatternEmitter() {
  EventEmitter.call(this);

  this.event = '';
  this._regexesCount = 0;

  this._events = this._events || {};
  this._patternEvents = this._patternEvents || {};
  this._regexes = this._regexes || {};
}
622995
void _emitJcc(INST_CODE code, Label label, final HINT hint) {
        if (hint == HINT.HINT_NONE) {
            emitX86(code, label);
        } else {
            emitX86(code, label, Immediate.imm(hint.value()));
        }
    }
130250
function (token) {
	if (!token) {
		return '';
	}
	if (typeof token === 'string') {
		return token;
	}
	if (typeof token.content === 'string') {
		return token.content;
	}
	return token.content.map(stringifyToken).join('');
}
645520
function handleToken(tokens, name) {
		var value = tokens[name];

		var type = Prism.util.type(value);
		switch (type) {
			case 'RegExp':
				var inside = {};
				tokens[name] = {
					pattern: value,
					inside: inside
				};
				addInvisibles(inside);
				break;

			case 'Array':
				for (var i = 0, l = value.length; i < l; i++) {
					handleToken(value, i);
				}
				break;

			default: // 'Object'
				var inside = value.inside || (value.inside = {});
				addInvisibles(inside);
				break;
		}
	}
743511
def _process_token(cls, token):
        """"""
        assert type(token) is _TokenType or callable(token), \
            'token type must be simple type or callable, not %r' % (token,)
        return token
10555
public static Token parse(final String pattern) {
		List<Token> tokens = new ArrayList<Token>();

		int start = 0;
		int count = 0;

		for (int i = 0; i < pattern.length(); ++i) {
			char character = pattern.charAt(i);

			if (character == '{') {
				if (count == 0) {
					if (start < i) {
						tokens.add(new PlainTextToken(pattern.substring(start, i)));
					}
					start = i;
				}
				count += 1;
			} else if (character == '}') {
				if (count == 0) {
					InternalLogger.log(Level.ERROR, "Opening curly bracket is missing: '" + pattern + "'");
				} else {
					count -= 1;
					if (count == 0) {
						tokens.add(parse(pattern.substring(start + 1, i)));
						start = i + 1;
					}
				}
			}
		}

		if (count > 0) {
			InternalLogger.log(Level.ERROR, "Closing curly bracket is missing: '" + pattern + "'");
		}

		int splitIndex = pattern.indexOf('|', start);
		if (splitIndex == -1) {
			tokens.add(createPlainToken(pattern.substring(start)));
			return tokens.size() == 1 ? tokens.get(0) : new BundleToken(tokens);
		} else {
			String token = pattern.substring(start, splitIndex).trim();
			tokens.add(createPlainToken(token));
			String[] styleOptions = SPLIT_PATTERN.split(pattern.substring(splitIndex + 1));
			return styleToken(tokens.size() == 1 ? tokens.get(0) : new BundleToken(tokens), styleOptions);
		}
	}
282741
def keyword(tokens, expected):
    """"""
    try:
        token = next(iter(tokens))
    except StopIteration:
        return

    if token and token.name == "symbol" and token.value.lower() == expected:
        return TokenMatch(None, token.value, (token,))
167694
def build_regex(self, tokens):
        """"""
        patterns = []
        for token in tokens:
            patterns.append(token.pattern_start)
            if token.pattern_end:
                patterns.append(token.pattern_end)
        return re.compile('|'.join(patterns), re.DOTALL)
857939
public static TokenRegex compile(String pattern) throws ParseException {
      ExpressionIterator p = QueryToPredicate.PARSER.parse(pattern);
      Expression exp;
      TransitionFunction top = null;
      while ((exp = p.next()) != null) {
         if (top == null) {
            top = consumerize(exp);
         } else {
            top = new TransitionFunction.Sequence(top, consumerize(exp));
         }
      }
      return new TokenRegex(top);
   }
639999
def match_paren(self, tokens, item):
        """"""
        match, = tokens
        return self.match(match, item)