// lex reads SPF record and returns list of Tokens along with
// their modifiers and values. Parser should parse the Tokens and execute
// relevant actions
351657
func lex(input string) []*token {
	var tokens []*token
	l := &lexer{0, 0, 0, len(input), input}
	for {
		token := l.scan()
		if token.mechanism == tEOF {
			break
		}
		tokens = append(tokens, token)
	}
	return tokens
}
707049
func lex(name string, input []byte) *lexer {
	l := &lexer{
		name:  name,
		input: string(input),
		items: make(chan item),
		line:  1,
	}
	go l.run()
	return l
}
580624
func lex(name, input string, run bool) *lexer {
	l := &lexer{
		name:       name,
		input:      input,
		items:      make(chan item),
		leftDelim:  defaultLeftDelim,
		rightDelim: defaultRightDelim,
	}
	if run {
		l.run()
	}
	return l
}
760085
func LexWithScanner(r io.Reader, scan *scanner.Scanner) Lexer {
	return lexWithScanner(r, scan)
}
433594
func (p *Lexer) Token8() (string, bool) {

	var buf []rune

	for {
		c, _ := p.Rune()
		if !isTokenChar(c) {
			break
		}
		buf = append(buf, c)
	}

	p.UnreadRune()
	return string(buf), len(buf) > 0
}
45222
func (l *lexer) token(type_ TokenType, word []rune, e Error) (*Token, Error) {
	// if we have an error, then we return a BAD token
	if e != nil {
		type_ = BAD
	}

	// extract the lexer position
	//      - the column is taken from the current column position
	//        minus the length of the consumed "word"
	_word := len(word)
	_column := l._column - _word
	_offset := l._offset - _word
	position := Position{"", l._line, _column, _offset}

	// if this is a newline token, we adjust the line & column counts
	if type_ == EOL {
		l.newline()
	}

	// return the Token
	return NewToken(type_, word, position), e
}
780402
func lexBlock(lx *lexer) stateFn {
	r := lx.next()

	//u.Debugf("lexBlock() pos=%d len=%d  %q", lx.pos, len(lx.input), lx.input[lx.pos:])

	switch {
	case r == blockEnd:

		lx.backup() // unconsume )  we are going to verify below
		lx.backup() // unconsume previous rune to ensure it is newline

		// Looking for a ')' character on a line by itself, if the previous
		// character isn't a new line, then break so we keep processing the block.
		if lx.next() != '\n' {
			lx.next() // if inline ( this will consume it
			break
		}
		lx.next()

		// Make sure the next character is a new line or an eof. We want a ')' on a
		// bare line by itself.
		switch r = lx.next(); r {
		case '\n', eof:
			if r == eof {
				lx.width = 1
			}
			lx.backup() // unconsume the \n, or eof
			r = lx.peek()
			if r != ')' { // For some reason on EOF we aren't where we think
				lx.backup() // unconsume the )
			}
			lx.backup() // unconsume the \n
			lx.emit(itemString)
			lx.next() // consume the previous line \n
			lx.next() // consume the )
			lx.ignore()
			return lx.pop()
		}
		lx.backup()
	}
	return lexBlock
}
515241
func (l *lexerContext) split() []token {
	result := make([]token, 0, 16)
	for !l.finished() {
		result = append(result, l.nextToken())
	}
	return result
}
1318176
func (p *Parser) scan() {
	for {
		token, literal := p.l.Lex()
		p.scannedItems = append(p.scannedItems, ScannedItem{token, literal})
		if token == EndOfStringToken {
			break
		}
	}
}
1311100
func (r *Lexer) Skip() {
	if r.token.kind == tokenUndef && r.Ok() {
		r.FetchToken()
	}
	r.consume()
}
367016
func (l *Lexer) Lex(lval *yySymType) int {
	tok, lit, pos, err := l.s.Scan()
	if err != nil {
		l.e = &Error{Message: err.Error(), Pos: pos, Fatal: true}
	}
	lval.tok = ast.Token{Tok: tok, Lit: lit}
	lval.tok.SetPosition(pos)
	l.lit = lit
	l.pos = pos
	return tok
}
802183
func (l *tomlLexer) lexKey() tomlLexStateFn {
	growingString := ""

	for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() {
		if r == '"' {
			l.next()
			str, err := l.lexStringAsString(`"`, false, true)
			if err != nil {
				return l.errorf(err.Error())
			}
			growingString += "\"" + str + "\""
			l.next()
			continue
		} else if r == '\'' {
			l.next()
			str, err := l.lexLiteralStringAsString(`'`, false)
			if err != nil {
				return l.errorf(err.Error())
			}
			growingString += "'" + str + "'"
			l.next()
			continue
		} else if r == '\n' {
			return l.errorf("keys cannot contain new lines")
		} else if isSpace(r) {
			break
		} else if r == '.' {
			// skip
		} else if !isValidBareChar(r) {
			return l.errorf("keys cannot contain %c character", r)
		}
		growingString += string(r)
		l.next()
	}
	l.emitWithValue(tokenKey, growingString)
	return l.lexVoid
}
944953
func lexIdentifier(l *Lexer) lexFunc {
	str := l.findRegexp(rID)
	if len(str) == 0 {
		// this is rotten
		panic("Identifier expected")
	}

	l.pos += len(str)
	l.emit(TokenID)

	return lexExpression
}
563360
func (z *StreamLexer) Lexeme() []byte {
	return z.buf[z.start:z.pos]
}
373003
func (r *Lexer) scanToken() {
	if r.token.kind != tokenUndef || r.fatalError != nil {
		return
	}

	r.FetchToken()
}
700273
func lexBinding(l *lexer) stateFn {
	for {
		if r := l.next(); !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != rune('_') || r == eof {
			l.backup()
			l.emit(ItemBinding)
			break
		}
	}
	return lexSpace
}
473934
func (l *lexer) scan() *token {
	for {
		r, eof := l.next()
		if eof {
			return &token{tEOF, tEOF, ""}
		} else if isWhitespace(r) || l.eof() { // we just scanned some meaningful data
			token := l.scanIdent()
			l.scanWhitespaces()
			l.moveon()
			return token
		}
	}
}