// Returns tokens[current-position + shift] or NIL
// (if (current-position + i) >= len(tokens))
500898
func (p *Parser) GetR(shift int) *Token {
	i := p.idx + shift
	return p.Get(i)
}
1348932
func (p PrefixCoded) Shift() (uint, error) {
	if len(p) > 0 {
		shift := p[0] - ShiftStartInt64
		if shift < 0 || shift < 63 {
			return uint(shift), nil
		}
	}
	return 0, fmt.Errorf("invalid prefix coded value")
}
417899
func (p *parser) shift() *lexer.Token {
	var result *lexer.Token

	p.ensure(0)

	result, p.tokens = p.tokens[0], p.tokens[1:]

	// check error token
	if result.Kind == lexer.TokenError {
		errToken(result, "Lexer error")
	}

	return result
}
1112132
func nextToken(s string) (token, rest string) {
	i := 0
	for ; i < len(s); i++ {
		if !isTokenOctet[s[i]] {
			break
		}
	}
	return s[:i], s[i:]
}
192037
func (l *Lexer) Next() (TokenType, []byte) {
	switch l.r.Peek(0) {
	case ' ', '\t', '\n', '\r', '\f':
		l.r.Move(1)
		for l.consumeWhitespace() {
		}
		return WhitespaceToken, l.r.Shift()
	case ':':
		l.r.Move(1)
		return ColonToken, l.r.Shift()
	case ';':
		l.r.Move(1)
		return SemicolonToken, l.r.Shift()
	case ',':
		l.r.Move(1)
		return CommaToken, l.r.Shift()
	case '(', ')', '[', ']', '{', '}':
		if t := l.consumeBracket(); t != ErrorToken {
			return t, l.r.Shift()
		}
	case '#':
		if l.consumeHashToken() {
			return HashToken, l.r.Shift()
		}
	case '"', '\'':
		if t := l.consumeString(); t != ErrorToken {
			return t, l.r.Shift()
		}
	case '.', '+':
		if t := l.consumeNumeric(); t != ErrorToken {
			return t, l.r.Shift()
		}
	case '-':
		if t := l.consumeNumeric(); t != ErrorToken {
			return t, l.r.Shift()
		} else if t := l.consumeIdentlike(); t != ErrorToken {
			return t, l.r.Shift()
		} else if l.consumeCDCToken() {
			return CDCToken, l.r.Shift()
		} else if l.consumeCustomVariableToken() {
			return CustomPropertyNameToken, l.r.Shift()
		}
	case '@':
		if l.consumeAtKeywordToken() {
			return AtKeywordToken, l.r.Shift()
		}
	case '$', '*', '^', '~':
		if t := l.consumeMatch(); t != ErrorToken {
			return t, l.r.Shift()
		}
	case '/':
		if l.consumeComment() {
			return CommentToken, l.r.Shift()
		}
	case '<':
		if l.consumeCDOToken() {
			return CDOToken, l.r.Shift()
		}
	case '\\':
		if t := l.consumeIdentlike(); t != ErrorToken {
			return t, l.r.Shift()
		}
	case 'u', 'U':
		if l.consumeUnicodeRangeToken() {
			return UnicodeRangeToken, l.r.Shift()
		} else if t := l.consumeIdentlike(); t != ErrorToken {
			return t, l.r.Shift()
		}
	case '|':
		if t := l.consumeMatch(); t != ErrorToken {
			return t, l.r.Shift()
		} else if l.consumeColumnToken() {
			return ColumnToken, l.r.Shift()
		}
	case 0:
		if l.r.Err() != nil {
			return ErrorToken, nil
		}
	default:
		if t := l.consumeNumeric(); t != ErrorToken {
			return t, l.r.Shift()
		} else if t := l.consumeIdentlike(); t != ErrorToken {
			return t, l.r.Shift()
		}
	}
	// can't be rune because consumeIdentlike consumes that as an identifier
	l.r.Move(1)
	return DelimToken, l.r.Shift()
}
373003
func (r *Lexer) scanToken() {
	if r.token.kind != tokenUndef || r.fatalError != nil {
		return
	}

	r.FetchToken()
}
855198
func (z *Lexer) Shift() []byte {
	b := z.buf[z.start:z.pos]
	z.start = z.pos
	return b
}
526469
func decodeReferenceToken(token string) string {
	step1 := strings.Replace(token, `~1`, `/`, -1)
	step2 := strings.Replace(step1, `~0`, `~`, -1)
	return step2
}
1043293
func (s *bufScanner) ScanRegex() (tok Token, pos Pos, lit string) {
	return s.scanFunc(s.s.ScanRegex)
}
80014
func (l *lexerContext) nextToken() token {
	buf := new(bytes.Buffer)

	index := 0
	escaped := false
	quoted := false

MainLoop:
	for _, c := range l.value[l.index:] {
		index += utf8.RuneLen(c)

		if escaped {
			escaped = false
			buf.WriteRune(c)
			continue
		}

		switch c {
		case '\\':
			escaped = true

		case '"':
			quoted = !quoted

		case l.delim:
			if quoted {
				buf.WriteRune(c)
			} else {
				break MainLoop
			}

		default:
			buf.WriteRune(c)
		}
	}

	l.index += index
	return token(buf.Bytes())
}
1121575
func (d *Decoder) expect(t tokenType) {
	if tok := d.next(); tok.t != t {
		d.error(d.pos(tok), "expected %v found %v", t, tok)
	}
}
728737
func (p *Parser) Get(i int) *Token {
	if i < len(p.tokens) && i >= 0 {
		return p.tokens[i]
	}
	return nil
}
973694
func (c *cachedGetter) Token() (string, error) {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	var err error
	// no token or exceeds the TTL
	if c.token == "" || time.Since(c.born) > c.ttl {
		c.token, err = c.tokenGetter.Token()
		if err != nil {
			return "", fmt.Errorf("failed to get token: %s", err)
		}
		c.born = time.Now()
	}
	return c.token, nil
}
417234
func (s *ScanOp) Offset(token string) *ScanOp {
	s.token = token
	return s
}
474868
func (z *StreamLexer) Shift() []byte {
	if z.pos > len(z.buf) { // make sure we peeked at least as much as we shift
		z.read(z.pos - 1)
	}
	b := z.buf[z.start:z.pos]
	z.start = z.pos
	return b
}
433594
func (p *Lexer) Token8() (string, bool) {

	var buf []rune

	for {
		c, _ := p.Rune()
		if !isTokenChar(c) {
			break
		}
		buf = append(buf, c)
	}

	p.UnreadRune()
	return string(buf), len(buf) > 0
}
169221
func (p *Parser) PeekN(shift int, typ TokenType, val string) *Token {
	t := p.Get(p.idx + shift)
	if t != nil {
		if t.Typ == typ && t.Val == val {
			return t
		}
	}
	return nil
}