// LexWithScanner creates a Lexer from a user-provided scanner.Scanner.
//
// Useful if you need to customise the Scanner.
760085
func LexWithScanner(r io.Reader, scan *scanner.Scanner) Lexer {
	return lexWithScanner(r, scan)
}
522457
func (p *parser) scanBlank() error {
	if p.useOptionX() {
		for {
			for p.charsRight() > 0 && isSpace(p.rightChar(0)) {
				p.moveRight(1)
			}

			if p.charsRight() == 0 {
				break
			}

			if p.rightChar(0) == '#' {
				for p.charsRight() > 0 && p.rightChar(0) != '\n' {
					p.moveRight(1)
				}
			} else if p.charsRight() >= 3 && p.rightChar(2) == '#' &&
				p.rightChar(1) == '?' && p.rightChar(0) == '(' {
				for p.charsRight() > 0 && p.rightChar(0) != ')' {
					p.moveRight(1)
				}
				if p.charsRight() == 0 {
					return p.getErr(ErrUnterminatedComment)
				}
				p.moveRight(1)
			} else {
				break
			}
		}
	} else {
		for {
			if p.charsRight() < 3 || p.rightChar(2) != '#' ||
				p.rightChar(1) != '?' || p.rightChar(0) != '(' {
				return nil
			}

			for p.charsRight() > 0 && p.rightChar(0) != ')' {
				p.moveRight(1)
			}
			if p.charsRight() == 0 {
				return p.getErr(ErrUnterminatedComment)
			}
			p.moveRight(1)
		}
	}
	return nil
}
771935
func (s *Scanner) Scan() token.Token {
	ch := s.next()

	// skip white space
	for isWhitespace(ch) {
		ch = s.next()
	}

	var tok token.Type

	// token text markings
	s.tokStart = s.srcPos.Offset - s.lastCharLen

	// token position, initial next() is moving the offset by one(size of rune
	// actually), though we are interested with the starting point
	s.tokPos.Offset = s.srcPos.Offset - s.lastCharLen
	if s.srcPos.Column > 0 {
		// common case: last character was not a '\n'
		s.tokPos.Line = s.srcPos.Line
		s.tokPos.Column = s.srcPos.Column
	} else {
		// last character was a '\n'
		// (we cannot be at the beginning of the source
		// since we have called next() at least once)
		s.tokPos.Line = s.srcPos.Line - 1
		s.tokPos.Column = s.lastLineLen
	}

	switch {
	case isLetter(ch):
		lit := s.scanIdentifier()
		if lit == "true" || lit == "false" {
			tok = token.BOOL
		} else if lit == "null" {
			tok = token.NULL
		} else {
			s.err("illegal char")
		}
	case isDecimal(ch):
		tok = s.scanNumber(ch)
	default:
		switch ch {
		case eof:
			tok = token.EOF
		case '"':
			tok = token.STRING
			s.scanString()
		case '.':
			tok = token.PERIOD
			ch = s.peek()
			if isDecimal(ch) {
				tok = token.FLOAT
				ch = s.scanMantissa(ch)
				ch = s.scanExponent(ch)
			}
		case '[':
			tok = token.LBRACK
		case ']':
			tok = token.RBRACK
		case '{':
			tok = token.LBRACE
		case '}':
			tok = token.RBRACE
		case ',':
			tok = token.COMMA
		case ':':
			tok = token.COLON
		case '-':
			if isDecimal(s.peek()) {
				ch := s.next()
				tok = s.scanNumber(ch)
			} else {
				s.err("illegal char")
			}
		default:
			s.err("illegal char: " + string(ch))
		}
	}

	// finish token ending
	s.tokEnd = s.srcPos.Offset

	// create token literal
	var tokenText string
	if s.tokStart >= 0 {
		tokenText = string(s.src[s.tokStart:s.tokEnd])
	}
	s.tokStart = s.tokEnd // ensure idempotency of tokenText() call

	return token.Token{
		Type: tok,
		Pos:  s.tokPos,
		Text: tokenText,
	}
}
190031
func LexDuration(l *Lexer) StateFn {
	l.SkipWhiteSpaces()
	typ, ok := scanNumericOrDuration(l, true)
	if !ok {
		return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
	}
	l.Emit(typ)
	return nil
}
940323
func (l *Parser) Lex(lval *yySymType) int {
	t := l.Lexer.Lex(lval)
	l.currentToken = lval.token
	return t
}
677141
func lexSoyDoc(l *lexer) stateFn {
	l.emit(itemSoyDocStart)
	var star = false
	var startOfLine = true // ignoring whitespace and asterisks.
	for {
		var ch = l.next()
		if ch == eof {
			return l.errorf("unexpected eof when scanning soydoc")
		}
		if star && ch == '/' {
			maybeEmitText(l, 2)
			l.emit(itemSoyDocEnd)
			return lexText
		}
		if startOfLine {
			// ignore any space or asterisks at the beginning of lines
			if isSpaceEOL(ch) {
				continue
			}
			if ch == '*' {
				star = true
				continue
			}
			l.pos--
			l.ignore()

			// start with @param?
			if strings.HasPrefix(l.input[l.pos:], "@param") {
				lexSoyDocParam(l)
			}
			startOfLine = false
		}

		if isEndOfLine(ch) {
			maybeEmitText(l, 1)
			startOfLine = true
		}
		star = ch == '*'
	}
}
654534
func (s *Scanner) Scan() (r rune) {
	s.tokenIdx = 0
	r, _, err := s.r.ReadRune()
	prevRune := r

	if s.Whitespace.Get(int(r)) {
		s.token[s.tokenIdx] = r
		s.tokenIdx++
		return r
	}

	for ; !s.Whitespace.Get(int(r)); r, _, err = s.r.ReadRune() {
		switch err {
		case nil:
			break
		case io.EOF:
			return EOF
		default:
			s.Error(err)
			return EOF
		}
		s.token[s.tokenIdx] = r
		s.tokenIdx++
		prevRune = r
	}

	// We have to UnreadRune because we went too far of one rune
	err = s.r.UnreadRune()
	if err != nil {
		s.Error(err)
	}

	return prevRune
}
204338
func lexSel(input string) *selLexer {
	return &selLexer{
		input: input,
		items: make(chan selItem),
	}
}
76694
func LexRun(l Lexer) {
	for fn := l.GetEntryPoint(); fn != nil; {
		fn = fn(l)
	}
	close(l.Items())
}
249231
func (ls *LineScanner) Scan() bool {
	if ls.text, _, ls.err = ls.Reader.ReadLine(); ls.err != nil {
		if ls.err == io.EOF {
			ls.err = nil
		}
		return false
	}
	ls.lnum++
	return true
}
321988
func lexKeyword(l *lexer) stateFn {
	input := l.input[l.pos:]
	f := func(r rune) bool {
		return !unicode.IsLetter(r)
	}
	if idx := strings.IndexFunc(input, f); idx >= 0 {
		input = input[:idx]
	}
	if strings.EqualFold(input, query) {
		consumeKeyword(l, ItemQuery)
		return lexSpace
	}
	if strings.EqualFold(input, insert) {
		consumeKeyword(l, ItemInsert)
		return lexSpace
	}
	if strings.EqualFold(input, delete) {
		consumeKeyword(l, ItemDelete)
		return lexSpace
	}
	if strings.EqualFold(input, create) {
		consumeKeyword(l, ItemCreate)
		return lexSpace
	}
	if strings.EqualFold(input, construct) {
		consumeKeyword(l, ItemConstruct)
		return lexSpace
	}
	if strings.EqualFold(input, deconstruct) {
		consumeKeyword(l, ItemDeconstruct)
		return lexSpace
	}
	if strings.EqualFold(input, drop) {
		consumeKeyword(l, ItemDrop)
		return lexSpace
	}
	if strings.EqualFold(input, graph) {
		consumeKeyword(l, ItemGraph)
		return lexSpace
	}
	if strings.EqualFold(input, data) {
		consumeKeyword(l, ItemData)
		return lexSpace
	}
	if strings.EqualFold(input, into) {
		consumeKeyword(l, ItemInto)
		return lexSpace
	}
	if strings.EqualFold(input, from) {
		consumeKeyword(l, ItemFrom)
		return lexSpace
	}
	if strings.EqualFold(input, where) {
		consumeKeyword(l, ItemWhere)
		return lexSpace
	}
	if strings.EqualFold(input, as) {
		consumeKeyword(l, ItemAs)
		return lexSpace
	}
	if strings.EqualFold(input, before) {
		consumeKeyword(l, ItemBefore)
		return lexSpace
	}
	if strings.EqualFold(input, after) {
		consumeKeyword(l, ItemAfter)
		return lexSpace
	}
	if strings.EqualFold(input, between) {
		consumeKeyword(l, ItemBetween)
		return lexSpace
	}
	if strings.EqualFold(input, count) {
		consumeKeyword(l, ItemCount)
		return lexSpace
	}
	if strings.EqualFold(input, distinct) {
		consumeKeyword(l, ItemDistinct)
		return lexSpace
	}
	if strings.EqualFold(input, sum) {
		consumeKeyword(l, ItemSum)
		return lexSpace
	}
	if strings.EqualFold(input, group) {
		consumeKeyword(l, ItemGroup)
		return lexSpace
	}
	if strings.EqualFold(input, by) {
		consumeKeyword(l, ItemBy)
		return lexSpace
	}
	if strings.EqualFold(input, order) {
		consumeKeyword(l, ItemOrder)
		return lexSpace
	}
	if strings.EqualFold(input, asc) {
		consumeKeyword(l, ItemAsc)
		return lexSpace
	}
	if strings.EqualFold(input, desc) {
		consumeKeyword(l, ItemDesc)
		return lexSpace
	}
	if strings.EqualFold(input, having) {
		consumeKeyword(l, ItemHaving)
		return lexSpace
	}
	if strings.EqualFold(input, limit) {
		consumeKeyword(l, ItemLimit)
		return lexSpace
	}
	if strings.EqualFold(input, not) {
		consumeKeyword(l, ItemNot)
		return lexSpace
	}
	if strings.EqualFold(input, and) {
		consumeKeyword(l, ItemAnd)
		return lexSpace
	}
	if strings.EqualFold(input, or) {
		consumeKeyword(l, ItemOr)
		return lexSpace
	}
	if strings.EqualFold(input, id) {
		consumeKeyword(l, ItemID)
		return lexSpace
	}
	if strings.EqualFold(input, typeKeyword) {
		consumeKeyword(l, ItemType)
		return lexSpace
	}
	if strings.EqualFold(input, atKeyword) {
		consumeKeyword(l, ItemAt)
		return lexSpace
	}
	if strings.EqualFold(input, inKeyword) {
		consumeKeyword(l, ItemIn)
		return lexSpace
	}
	if strings.EqualFold(input, showKeyword) {
		consumeKeyword(l, ItemShow)
		return lexSpace
	}
	if strings.EqualFold(input, graphsKeyword) {
		consumeKeyword(l, ItemGraphs)
		return lexSpace
	}
	for {
		r := l.next()
		if unicode.IsSpace(r) || r == eof {
			l.backup()
			break
		}
	}
	l.emitError("found unknown keyword")
	return nil
}
333505
func (s *DiffIndexScanner) Scan() bool {
	if !s.prepareScan() {
		return false
	}

	s.next, s.err = s.scan(s.from.Text())
	if s.err != nil {
		s.err = errors.Wrap(s.err, "scan")
	}

	return s.err == nil
}
223881
func (s *Scanner) Scan() (tok Token, pos Pos, lit string) {
	// Read next code point.
	ch0, pos := s.r.read()

	// If we see whitespace then consume all contiguous whitespace.
	// If we see a letter, or certain acceptable special characters, then consume
	// as an ident or reserved word.
	if isWhitespace(ch0) {
		return s.scanWhitespace()
	} else if isLetter(ch0) || ch0 == '_' {
		s.r.unread()
		return s.scanIdent(true)
	} else if isDigit(ch0) {
		return s.scanNumber()
	}

	// Otherwise parse individual characters.
	switch ch0 {
	case eof:
		return EOF, pos, ""
	case '"':
		s.r.unread()
		return s.scanIdent(true)
	case '\'':
		return s.scanString()
	case '.':
		ch1, _ := s.r.read()
		s.r.unread()
		if isDigit(ch1) {
			return s.scanNumber()
		}
		return DOT, pos, ""
	case '$':
		tok, _, lit = s.scanIdent(false)
		if tok != IDENT {
			return tok, pos, "$" + lit
		}
		return BOUNDPARAM, pos, "$" + lit
	case '+':
		return ADD, pos, ""
	case '-':
		ch1, _ := s.r.read()
		if ch1 == '-' {
			s.skipUntilNewline()
			return COMMENT, pos, ""
		}
		s.r.unread()
		return SUB, pos, ""
	case '*':
		return MUL, pos, ""
	case '/':
		ch1, _ := s.r.read()
		if ch1 == '*' {
			if err := s.skipUntilEndComment(); err != nil {
				return ILLEGAL, pos, ""
			}
			return COMMENT, pos, ""
		} else {
			s.r.unread()
		}
		return DIV, pos, ""
	case '%':
		return MOD, pos, ""
	case '&':
		return BITWISE_AND, pos, ""
	case '|':
		return BITWISE_OR, pos, ""
	case '^':
		return BITWISE_XOR, pos, ""
	case '=':
		if ch1, _ := s.r.read(); ch1 == '~' {
			return EQREGEX, pos, ""
		}
		s.r.unread()
		return EQ, pos, ""
	case '!':
		if ch1, _ := s.r.read(); ch1 == '=' {
			return NEQ, pos, ""
		} else if ch1 == '~' {
			return NEQREGEX, pos, ""
		}
		s.r.unread()
	case '>':
		if ch1, _ := s.r.read(); ch1 == '=' {
			return GTE, pos, ""
		}
		s.r.unread()
		return GT, pos, ""
	case '<':
		if ch1, _ := s.r.read(); ch1 == '=' {
			return LTE, pos, ""
		} else if ch1 == '>' {
			return NEQ, pos, ""
		}
		s.r.unread()
		return LT, pos, ""
	case '(':
		return LPAREN, pos, ""
	case ')':
		return RPAREN, pos, ""
	case ',':
		return COMMA, pos, ""
	case ';':
		return SEMICOLON, pos, ""
	case ':':
		if ch1, _ := s.r.read(); ch1 == ':' {
			return DOUBLECOLON, pos, ""
		}
		s.r.unread()
		return COLON, pos, ""
	}

	return ILLEGAL, pos, string(ch0)
}
542461
func NewLexer(r io.Reader) *Lexer {
	var b []byte
	if r != nil {
		if buffer, ok := r.(interface {
			Bytes() []byte
		}); ok {
			b = buffer.Bytes()
		} else {
			var err error
			b, err = ioutil.ReadAll(r)
			if err != nil {
				return &Lexer{
					buf: []byte{0},
					err: err,
				}
			}
		}
	}
	return NewLexerBytes(b)
}
1362992
func NewLexer(r io.Reader) (*Lexer, error) {

	tokenizer, err := NewTokenizer(r)
	if err != nil {
		return nil, err
	}
	lexer := &Lexer{tokenizer: tokenizer}
	return lexer, nil
}
391764
func scanNumericOrDuration(l *Lexer, doDuration bool) (typ TokenType, ok bool) {
	typ = TokenInteger
	// Optional leading sign.
	hasSign := l.accept("+-")
	peek2 := l.PeekX(2)
	peek2nd := byte(0)
	if len(peek2) == 2 {
		peek2nd = peek2[1]
	}
	//u.Debugf("scanNumericOrDuration?  '%v' peek:%v ", string(peek2), len(peek2))
	if peek2 == "0x" {
		// Hexadecimal.
		if hasSign {
			// No signs for hexadecimals.
			return
		}
		l.acceptRun("0x")
		if !l.acceptRun(hexDigits) {
			// Requires at least one digit.
			return
		}
		if l.accept(".") {
			// No dots for hexadecimals.
			return
		}
	} else {
		// Decimal
		if !l.acceptRun(decDigits) {
			// Requires at least one digit
			return
		}
		if l.accept(".") {
			// Float
			if !l.acceptRun(decDigits) {
				// Requires a digit after the dot.
				return
			}
			typ = TokenFloat
		} else {
			if (!hasSign && l.input[l.start] == '0') ||
				(hasSign && l.input[l.start+1] == '0') {
				switch peek2nd {
				case ' ', '\t', '\n', ',', ')', ';', byte(0):
					return typ, true
				}
				// Integers can't start with 0??
				return
			}
		}
		if l.accept("e") {
			l.accept("+-")
			if !l.acceptRun(decDigits) {
				// A digit is required after the scientific notation.
				return
			}
			typ = TokenFloat
		}
	}

	if doDuration {
		if l.acceptRun("yYmMdDuUsSwW") {
			// duration was found
			typ = TokenDuration
		}
	} else {
		// Next thing must not be alphanumeric.
		if isAlNum(l.Peek()) {
			l.Next()
			return
		}
	}

	ok = true
	return
}
473934
func (l *lexer) scan() *token {
	for {
		r, eof := l.next()
		if eof {
			return &token{tEOF, tEOF, ""}
		} else if isWhitespace(r) || l.eof() { // we just scanned some meaningful data
			token := l.scanIdent()
			l.scanWhitespaces()
			l.moveon()
			return token
		}
	}
}