// Eat whitespaces and comments until the next token is found.
766085
func yaml_parser_scan_to_next_token(parser *yaml_parser_t) bool {

	// Until the next token is not found.
	for {
		// Allow the BOM mark to start a line.
		if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
			return false
		}
		if parser.mark.column == 0 && is_bom(parser.buffer, parser.buffer_pos) {
			skip(parser)
		}

		// Eat whitespaces.
		// Tabs are allowed:
		//  - in the flow context
		//  - in the block context, but not at the beginning of the line or
		//  after '-', '?', or ':' (complex value).
		if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
			return false
		}

		for parser.buffer[parser.buffer_pos] == ' ' || ((parser.flow_level > 0 || !parser.simple_key_allowed) && parser.buffer[parser.buffer_pos] == '\t') {
			skip(parser)
			if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
				return false
			}
		}

		// Eat a comment until a line break.
		if parser.buffer[parser.buffer_pos] == '#' {
			for !is_breakz(parser.buffer, parser.buffer_pos) {
				skip(parser)
				if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
					return false
				}
			}
		}

		// If it is a line break, eat it.
		if is_break(parser.buffer, parser.buffer_pos) {
			if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
				return false
			}
			skip_line(parser)

			// In the block context, a new line may start a simple key.
			if parser.flow_level == 0 {
				parser.simple_key_allowed = true
			}
		} else {
			break // We have found a token.
		}
	}

	return true
}
1330040
func processLine(d *Directive, token []byte, stripLeftWhitespace bool) ([]byte, error) {
	if stripLeftWhitespace {
		token = trimWhitespace(token)
	}
	return trimComments(token), d.possibleParserDirective(string(token))
}
356086
func (l *Lexer) consumeString() TokenType {
	// assume to be on " or '
	delim := l.r.Peek(0)
	l.r.Move(1)
	for {
		c := l.r.Peek(0)
		if c == 0 && l.r.Err() != nil {
			break
		} else if c == '\n' || c == '\r' || c == '\f' {
			l.r.Move(1)
			return BadStringToken
		} else if c == delim {
			l.r.Move(1)
			break
		} else if c == '\\' {
			if !l.consumeEscape() {
				l.r.Move(1)
				l.consumeNewline()
			}
		} else {
			l.r.Move(1)
		}
	}
	return StringToken
}
38118
func (g *globTokeniser) Peek() bool {
	if !g.hasPeek {
		g.peekToken, g.peekTokenType, g.peekErr = g.parse(g.tokenType)
		g.hasPeek = true
	}

	return g.peekErr == nil
}
219711
func (p *Parser) Peek(typ TokenType, val string) *Token {
	return p.PeekN(0, typ, val)
}
486521
func (z *Tokenizer) skipWhiteSpace() {
	if z.err != nil {
		return
	}
	for {
		c := z.readByte()
		if z.err != nil {
			return
		}
		switch c {
		case ' ', '\n', '\r', '\t', '\f':
			// No-op.
		default:
			z.raw.end--
			return
		}
	}
}
1130160
func (l *Lexer) consumeWhitespace() bool {
	c := l.r.Peek(0)
	if c == ' ' || c == '\t' || c == '\v' || c == '\f' {
		l.r.Move(1)
		return true
	} else if c >= 0xC0 {
		if r, n := l.r.PeekRune(0); r == '\u00A0' || r == '\uFEFF' || unicode.Is(unicode.Zs, r) {
			l.r.Move(n)
			return true
		}
	}
	return false
}
777044
func (l *Lexer) SkipWhiteSpaces() {
	for rune := l.Next(); unicode.IsSpace(rune); rune = l.Next() {
		if rune == '\n' {
			// New line, lets keep track of line position
			l.line++
			l.linepos = l.pos
		}
	}
	l.backup()
	l.ignore()
}
504823
func (l *LLk) Consume(tt lexer.TokenType) bool {
	if l.tkns[0].Type != tt {
		return false
	}
	l.tkns = l.tkns[1:]
	appendNextToken(l)
	return true
}
720942
func (p *parser) consumeParenthesis() bool {
	if p.i < len(p.s) && p.s[p.i] == '(' {
		p.i++
		p.skipWhitespace()
		return true
	}
	return false
}
716771
func (p *parser) snippetTokens() ([]Token, error) {
	// TODO: disallow imports in snippets for simplicity at import time
	// snippet must have curlies.
	err := p.openCurlyBrace()
	if err != nil {
		return nil, err
	}
	count := 1
	tokens := []Token{}
	for p.Next() {
		if p.Val() == "}" {
			count--
			if count == 0 {
				break
			}
		}
		if p.Val() == "{" {
			count++
		}
		tokens = append(tokens, p.tokens[p.cursor])
	}
	// make sure we're matched up
	if count != 0 {
		return nil, p.SyntaxErr("}")
	}
	return tokens, nil
}
304861
func tokenize(s string) []string {
	/*
		TODO: Properly detect if the string starts with a ( and ends with a )
		and that those two form a matching pair.

		Eg: (foo) -> true; (foo)(bar) -> false;
	*/
	if s[0] == '(' && s[len(s)-1] == ')' {
		s = s[1 : len(s)-1]
	}
	ret := []string{}
	for chunk := range split(s) {
		if len(chunk) != 0 {
			if chunk[0] == '(' && chunk[len(chunk)-1] == ')' {
				ret = append(ret, chunk)
			} else {
				for _, token := range pat.FindAllStringSubmatch(chunk, -1) {
					ret = append(ret, token[0])
				}
			}
		} else {
			fmt.Printf("Empty chunk in string '%s'\n", s)
		}
	}
	return ret
}
569447
func (t *SimpleTokenizer) Tokenize(sentence string) []string {
	// is the tokenizer really the best place to be making
	// everything lowercase? is this really a sanitizaion func?
	return strings.Split(strings.ToLower(sentence), t.SplitOn)
}
1298586
func (l *Lexer) Next() (TokenType, []byte) {
	l.text = nil
	var c byte
	if l.inTag {
		l.attrVal = nil
		for { // before attribute name state
			if c = l.r.Peek(0); c == ' ' || c == '\t' || c == '\n' || c == '\r' {
				l.r.Move(1)
				continue
			}
			break
		}
		if c == 0 {
			if l.r.Err() == nil {
				l.err = parse.NewErrorLexer("unexpected null character", l.r)
			}
			return ErrorToken, nil
		} else if c != '>' && (c != '/' && c != '?' || l.r.Peek(1) != '>') {
			return AttributeToken, l.shiftAttribute()
		}
		start := l.r.Pos()
		l.inTag = false
		if c == '/' {
			l.r.Move(2)
			l.text = l.r.Lexeme()[start:]
			return StartTagCloseVoidToken, l.r.Shift()
		} else if c == '?' {
			l.r.Move(2)
			l.text = l.r.Lexeme()[start:]
			return StartTagClosePIToken, l.r.Shift()
		} else {
			l.r.Move(1)
			l.text = l.r.Lexeme()[start:]
			return StartTagCloseToken, l.r.Shift()
		}
	}

	for {
		c = l.r.Peek(0)
		if c == '<' {
			if l.r.Pos() > 0 {
				return TextToken, l.r.Shift()
			}
			c = l.r.Peek(1)
			if c == '/' {
				l.r.Move(2)
				return EndTagToken, l.shiftEndTag()
			} else if c == '!' {
				l.r.Move(2)
				if l.at('-', '-') {
					l.r.Move(2)
					return CommentToken, l.shiftCommentText()
				} else if l.at('[', 'C', 'D', 'A', 'T', 'A', '[') {
					l.r.Move(7)
					return CDATAToken, l.shiftCDATAText()
				} else if l.at('D', 'O', 'C', 'T', 'Y', 'P', 'E') {
					l.r.Move(7)
					return DOCTYPEToken, l.shiftDOCTYPEText()
				}
				l.r.Move(-2)
			} else if c == '?' {
				l.r.Move(2)
				l.inTag = true
				return StartTagPIToken, l.shiftStartTag()
			}
			l.r.Move(1)
			l.inTag = true
			return StartTagToken, l.shiftStartTag()
		} else if c == 0 {
			if l.r.Pos() > 0 {
				return TextToken, l.r.Shift()
			}
			if l.r.Err() == nil {
				l.err = parse.NewErrorLexer("unexpected null character", l.r)
			}
			return ErrorToken, nil
		}
		l.r.Move(1)
	}
}
209394
func (r *Lexer) consume() {
	r.token.kind = tokenUndef
	r.token.delimValue = 0
}
785742
func (p *parser) next() (*Token, Error) {
	// do we have any previously read tokens?
	_length := len(p._undo)
	if _length > 0 {
		_token := p._undo[_length-1]
		p._undo = p._undo[:_length-1]
		return _token, nil
	}

	// otherwise, attempt to retrieve the next token from the lexer
	return p._lexer.Next()
}
1072798
func (b *LookAheadBuffer) ConsumeAll() {
	err := b.Consume(b.bytesBuffered)
	if err != nil { // This should never happen
		panic(err)
	}
}