Span from the start of the first token to the end of the last.
18971
public OffsetRange<CharOffset> offsets() {
    final CharOffset start = tokens.get(0).offsets().startInclusive();
    final CharOffset end = tokens.reverse().get(0).offsets().endInclusive();
    return OffsetRange.charOffsetRange(start.asInt(), end.asInt());
  }
913401
function splitOnFirst(str, splitter, callback) {
        var parts = str.split(splitter);
        var first = parts.shift();
        return callback(first, parts.join(splitter));
    }
22396
private void getLastToken(StreamTokenizer tokenizer, boolean endOfFileOk) throws IOException, ParseException {
        if ((tokenizer.nextToken() != StreamTokenizer.TT_EOL) && ((tokenizer.ttype != StreamTokenizer.TT_EOF) || !endOfFileOk)) {
            throw new ParseException("end of line expected", tokenizer.lineno());
        }
    }
709567
def frange(start, end, step):
    """"""
    if start <= end:
        step = abs(step)
    else:
        step = -abs(step)

    while start < end:
        yield start
        start += step
821881
def start_span(self, name='span'):
        
        span = trace_span.BlankSpan(name, context_tracer=self)
        return span
1185828
def join(tokens, start, result):
    """"""
    texts = []
    if len(result) > 0:
        for e in result:
            for child in e.iter():
                if child.text is not None:
                    texts.append(child.text)
        return [E(result[0].tag, ' '.join(texts))]
687351
def _mid(string, start, end=None):
    
    if end is None:
        end = len(string)
    return string[start:start + end]
695764
def paragraph_ends(self):
        """"""
        if not self.is_tagged(PARAGRAPHS):
            self.tokenize_paragraphs()
        return self.ends(PARAGRAPHS)
1255203
private static String nextToken(String[] tokens, int x, String context)
            throws ParseException {
        if (x < tokens.length) {
            return tokens[x];
        }
        throw new ParseException("missing token at end of '" + context + "'", -1);
    }
137212
def span_case(self, i, case):
        """"""

        # A new \L, \C or \E should pop the last in the stack.
        if self.span_stack:
            self.span_stack.pop()
        if self.single_stack:
            self.single_stack.pop()
        self.span_stack.append(case)
        count = len(self.span_stack)
        self.end_found = False
        try:
            while not self.end_found:
                t = next(i)
                if self.use_format and t in _CURLY_BRACKETS:
                    self.handle_format(t, i)
                elif t == '\\':
                    try:
                        t = next(i)
                        self.reference(t, i)
                    except StopIteration:
                        self.result.append(t)
                        raise
                else:
                    self.result.append(self.convert_case(t, case))
                if self.end_found or count > len(self.span_stack):
                    self.end_found = False
                    break
        except StopIteration:
            pass
        if count == len(self.span_stack):
            self.span_stack.pop()
459944
def get_token(self, text, start=0):
        
        best_class = best_match = None

        for token_class, match in self.matching_tokens(text):
            if best_match and best_match.end() >= match.end():
                continue
            best_match = match
            best_class = token_class

        return best_class, best_match
245983
def _wrap_value_with_context(self, tokens: List[Token], start: int, end: int) -> Extraction:
        """"""
        return Extraction(' '.join([x.orth_ if isinstance(x, Token) else x for x in tokens[start:end]]),
                          self.name,
                          start_token=start,
                          end_token=end,
                          start_char=tokens[start].idx if isinstance(tokens[start], Token) else -1,
                          end_char=tokens[end - 1].idx + len(tokens[end - 1].orth_) if isinstance(tokens[end - 1],
                                                                                                  Token) else -1
                          )
1346470
private void checkForTokens( String content, int startOffset, int endOffset ) {
        while( startOffset <= endOffset ) {
            // skip the delimiters to find the start of a new token
            while( isDelimiter(content.substring(startOffset, startOffset + 1)) ) {
                if (startOffset < endOffset)
                    startOffset++;
                else
                    return;
            }
            // Extract and process the entire token
            if (isQuoteDelimiter(content.substring(startOffset, startOffset + 1)))
                startOffset = getQuoteToken(content, startOffset, endOffset);
            else
                startOffset = getOtherToken(content, startOffset, endOffset);
        }
    }
846841
private void advance() {
	String s = null;
	while (tokenizer.hasNext()) {
	    String nextToken = tokenizer.next();
	    // stop if the word should be returned
	    if (filter.accept(nextToken)) {
		s = nextToken;
		break;
	    }
	}
	next = s;
    }
1131072
def tokenise(template):
    ''''''
    upto = 0
    lineno = 0

    for m in tag_re.finditer(template):

        start, end = m.span()
        lineno = template.count('\n', 0, start) + 1  # Humans count from 1
        # If there's a gap between our start and the end of the last match,
        # there's a Text node between.
        if upto < start:
            yield Token(TokenType.text, template[upto:start], lineno)
        upto = end

        mode = m.lastgroup
        content = m.group(mode)
        yield Token(TokenType[mode], content, lineno)

    # if the last match ended before the end of the source, we have a tail Text
    # node.
    if upto < len(template):
        yield Token(TokenType.text, template[upto:], lineno)
440921
def tokenize_wrapper(input):
    """"""
    skip = set((token.NEWLINE, token.INDENT, token.DEDENT))
    tokens = tokenize.generate_tokens(StringIO.StringIO(input).readline)
    for quintuple in tokens:
        type, value, start, end, line_text = quintuple
        if type not in skip:
            yield quintuple
1141492
def detect(self, tokens):
        """"""
        results = []
        for abbr_span, long_span in self.detect_spans(tokens):
            results.append((tokens[abbr_span[0]:abbr_span[1]], tokens[long_span[0]:long_span[1]]))
        return results