// separatorsScanner returns a split function for a scanner that returns tokens delimited any of the specified runes.
// Note that the delimiters themselves are counted as tokens, so callers who want to discard the separators must do this
// themselves.
7736
func separatorsScanner(separators []rune) func(data []byte, atEOF bool) (int, []byte, error) {
	return func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		if atEOF && len(data) == 0 {
			return 0, nil, nil
		}

		// Transform the separators into a map (for efficient lookup)
		seps := make(map[rune]bool)
		for _, r := range separators {
			seps[r] = true
		}

		// Scan until separator, marking the end of a token
		for width, i := 0, 0; i < len(data); i += width {
			var r rune
			r, width = utf8.DecodeRune(data[i:])
			if seps[r] {
				if i == 0 {
					// Separator token
					return i + width, data[0 : i+width], nil
				}

				// Normal token
				return i, data[0:i], nil
			}
		}

		// If we're at EOF, we have a final, non-empty, non-terminated token: return it
		if atEOF && len(data) > 0 {
			return len(data), data[0:], nil
		}

		// Request more data
		return 0, nil, nil
	}
}
624326
func (l *Lexer) scanSpecialSymbol() (Token, string) {
	lastScannedItem := ScannedItem{}
	var buffer []byte
SpecialSymbolLoop:
	for {
		switch ch := l.read(); {
		case ch == 0:
			break SpecialSymbolLoop
		case isSpecialSymbol(ch):
			buffer = append(buffer, ch)
			if token, ok := string2token[string(buffer)]; ok {
				lastScannedItem = ScannedItem{tok: token, literal: string(buffer)}
			} else if lastScannedItem.tok != 0 {
				l.unread()
				break SpecialSymbolLoop
			}
		default:
			l.unread()
			break SpecialSymbolLoop
		}
	}
	if lastScannedItem.tok == 0 {
		return ErrorToken, fmt.Sprintf("error expected: keyword found '%s'", buffer)
	}
	return lastScannedItem.tok, lastScannedItem.literal
}
881951
func (s *scanner) SkipSpaces() (rune, bool) {
	r, ok := s.Next()
	for unicode.IsSpace(r) && ok {
		r, ok = s.Next()
	}
	return r, ok
}
24594
func (ctx *Context) ScanReader(r io.Reader) BasicScanner {
	return ctx.SetScanner(&ScanReader{sr: bufio.NewScanner(r)})
}
240857
func SplitTokensIntoLines(tokens []Token) (out [][]Token) {
	var line []Token // nolint: prealloc
	for _, token := range tokens {
		for strings.Contains(token.Value, "\n") {
			parts := strings.SplitAfterN(token.Value, "\n", 2)
			// Token becomes the tail.
			token.Value = parts[1]

			// Append the head to the line and flush the line.
			clone := token.Clone()
			clone.Value = parts[0]
			line = append(line, clone)
			out = append(out, line)
			line = nil
		}
		line = append(line, token)
	}
	if len(line) > 0 {
		out = append(out, line)
	}
	// Strip empty trailing token line.
	if len(out) > 0 {
		last := out[len(out)-1]
		if len(last) == 1 && last[0].Value == "" {
			out = out[:len(out)-1]
		}
	}
	return
}
1293655
func splitSpecial(s string, delimiter string, cutdelimiter bool) (string, string) {
	i := strings.Index(s, delimiter)
	if i < 0 {
		// if delimiter not found return as is.
		return s, ""
	}
	// if delimiter should be removed, remove it.
	if cutdelimiter {
		return s[0:i], s[i+len(delimiter):]
	}
	// return split strings with delimiter
	return s[0:i], s[i:]
}
14689
func (m *FilePager) NextScanner() (schema.ConnScanner, error) {

	fr, err := m.NextFile()
	if err == iterator.Done {
		return nil, err
	} else if err != nil {
		u.Warnf("NextFile Error %v", err)
		return nil, err
	}

	scanner, err := m.fs.fh.Scanner(m.fs.store, fr)
	if err != nil {
		u.Errorf("Could not open file scanner %v err=%v", m.fs.fileType, err)
		return nil, err
	}
	m.ConnScanner = scanner
	return scanner, err
}
1009504
func (c *Client) scanErr() error {
	if err := c.scanner.Err(); err != nil {
		return err
	}
	return io.ErrUnexpectedEOF
}
1276584
func (s *scanner) scan() (token, string, error) {
	ch, _, err := s.r.ReadRune()

	if err != nil && err != io.EOF {
		return tokenError, "", err
	}

	switch ch {
	case zeroRune:
		return eof, "", nil
	case ':':
		return colon, string(ch), nil
	case '~':
		return tilde, string(ch), nil
	case '^':
		return caret, string(ch), nil
	case '.':
		return dot, string(ch), nil
	case '/':
		return slash, string(ch), nil
	case '{':
		return obrace, string(ch), nil
	case '}':
		return cbrace, string(ch), nil
	case '-':
		return minus, string(ch), nil
	case '@':
		return at, string(ch), nil
	case '\\':
		return aslash, string(ch), nil
	case '?':
		return qmark, string(ch), nil
	case '*':
		return asterisk, string(ch), nil
	case '[':
		return obracket, string(ch), nil
	case '!':
		return emark, string(ch), nil
	}

	if unicode.IsSpace(ch) {
		return space, string(ch), nil
	}

	if unicode.IsControl(ch) {
		return control, string(ch), nil
	}

	if unicode.IsLetter(ch) {
		return tokenizeExpression(ch, word, unicode.IsLetter, s.r)
	}

	if unicode.IsNumber(ch) {
		return tokenizeExpression(ch, number, unicode.IsNumber, s.r)
	}

	return tokenError, string(ch), nil
}
1036742
func (p *PointParser) scan() (Token, string) {
	// If we have a token on the buffer, then return it.
	if p.buf.n != 0 {
		idx := p.buf.n % MAX_BUFFER_SIZE
		tok, lit := p.buf.tok[idx], p.buf.lit[idx]
		p.buf.n -= 1
		return tok, lit
	}

	// Otherwise read the next token from the scanner.
	tok, lit := p.s.Scan()

	// Save it to the buffer in case we unscan later.
	p.buffer(tok, lit)

	return tok, lit
}
232198
func (s *Scanner) scanIdentifier() (string, error) {
	var ret []rune
	for {
		if !isLetter(s.peek()) && !isDigit(s.peek()) {
			break
		}
		ret = append(ret, s.peek())
		s.next()
	}
	return string(ret), nil
}
654534
func (s *Scanner) Scan() (r rune) {
	s.tokenIdx = 0
	r, _, err := s.r.ReadRune()
	prevRune := r

	if s.Whitespace.Get(int(r)) {
		s.token[s.tokenIdx] = r
		s.tokenIdx++
		return r
	}

	for ; !s.Whitespace.Get(int(r)); r, _, err = s.r.ReadRune() {
		switch err {
		case nil:
			break
		case io.EOF:
			return EOF
		default:
			s.Error(err)
			return EOF
		}
		s.token[s.tokenIdx] = r
		s.tokenIdx++
		prevRune = r
	}

	// We have to UnreadRune because we went too far of one rune
	err = s.r.UnreadRune()
	if err != nil {
		s.Error(err)
	}

	return prevRune
}
694304
func splitIgnoreGroups(origStr, sep string) []string {
	if len(origStr) == 0 {
		return []string{}
	} else if len(sep) == 0 {
		return []string{origStr}
	}
	retSplit := make([]string, 0)
	cmIdxs := indexStringAll(origStr, ",") // Main indexes of separators
	if len(cmIdxs) == 0 {
		return []string{origStr}
	}
	oCrlyIdxs := indexStringAll(origStr, "{") // Index  { for exceptions
	cCrlyIdxs := indexStringAll(origStr, "}") // Index  } for exceptions closing
	oBrktIdxs := indexStringAll(origStr, "[") // Index [ for exceptions
	cBrktIdxs := indexStringAll(origStr, "]") // Index ] for exceptions closing
	lastNonexcludedIdx := 0
	for i, cmdIdx := range cmIdxs {
		if len(oCrlyIdxs) == len(cCrlyIdxs) && len(oBrktIdxs) == len(cBrktIdxs) { // We assume exceptions and closing them are symetrical, otherwise don't handle exceptions
			exceptFound := false
			for iCrlyIdx := range oCrlyIdxs {
				if oCrlyIdxs[iCrlyIdx] < cmdIdx && cCrlyIdxs[iCrlyIdx] > cmdIdx { // Parentheses canceling indexing found
					exceptFound = true
					break
				}
			}
			for oBrktIdx := range oBrktIdxs {
				if oBrktIdxs[oBrktIdx] < cmdIdx && cBrktIdxs[oBrktIdx] > cmdIdx { // Parentheses canceling indexing found
					exceptFound = true
					break
				}
			}
			if exceptFound {
				continue
			}
		}
		switch i {
		case 0: // First one
			retSplit = append(retSplit, origStr[:cmIdxs[i]])
		case len(cmIdxs) - 1: // Last one
			postpendStr := ""
			if len(origStr) > cmIdxs[i]+1 { // Our separator is not the last character in the string
				postpendStr = origStr[cmIdxs[i]+1:]
			}
			retSplit = append(retSplit, origStr[cmIdxs[lastNonexcludedIdx]+1:cmIdxs[i]], postpendStr)
		default:
			retSplit = append(retSplit, origStr[cmIdxs[lastNonexcludedIdx]+1:cmIdxs[i]]) // Discard the separator from end string
		}
		lastNonexcludedIdx = i
	}
	groupedSplt := make([]string, 0)
	// Merge more consecutive groups (this is how FS displays app data from dial strings)
	for idx, spltData := range retSplit {
		if idx == 0 {
			groupedSplt = append(groupedSplt, spltData)
			continue // Nothing to do for first data
		}
		isGroup, _ := regexp.MatchString("{.*}|[.*]", spltData)
		if !isGroup {
			groupedSplt = append(groupedSplt, spltData)
			continue
		}
		isPrevGroup, _ := regexp.MatchString("{.*}|[.*]", retSplit[idx-1])
		if !isPrevGroup {
			groupedSplt = append(groupedSplt, spltData)
			continue
		}
		groupedSplt[len(groupedSplt)-1] = groupedSplt[len(groupedSplt)-1] + sep + spltData // Merge it with the previous data
	}
	return groupedSplt
}
1107387
func splitTerms(fieldSelector string) []string {
	if len(fieldSelector) == 0 {
		return nil
	}

	terms := make([]string, 0, 1)
	startIndex := 0
	inSlash := false
	for i, c := range fieldSelector {
		switch {
		case inSlash:
			inSlash = false
		case c == '\\':
			inSlash = true
		case c == ',':
			terms = append(terms, fieldSelector[startIndex:i])
			startIndex = i + 1
		}
	}

	terms = append(terms, fieldSelector[startIndex:])

	return terms
}
825855
func ScanString(r io.RuneScanner) (string, error) {
	ending, _, err := r.ReadRune()
	if err != nil {
		return "", errBadString
	}

	var buf bytes.Buffer
	for {
		ch0, _, err := r.ReadRune()
		if ch0 == ending {
			return buf.String(), nil
		} else if err != nil || ch0 == '\n' {
			return buf.String(), errBadString
		} else if ch0 == '\\' {
			// If the next character is an escape then write the escaped char.
			// If it's not a valid escape then return an error.
			ch1, _, _ := r.ReadRune()
			if ch1 == 'n' {
				_, _ = buf.WriteRune('\n')
			} else if ch1 == '\\' {
				_, _ = buf.WriteRune('\\')
			} else if ch1 == '"' {
				_, _ = buf.WriteRune('"')
			} else if ch1 == '\'' {
				_, _ = buf.WriteRune('\'')
			} else {
				return string(ch0) + string(ch1), errBadEscape
			}
		} else {
			_, _ = buf.WriteRune(ch0)
		}
	}
}
1106020
func (t *Tokenizer) scanStream() (*Token, error) {
	state := STATE_START
	var tokenType TokenType
	value := make([]int32, 0, INITIAL_TOKEN_CAPACITY)
	var (
		nextRune     int32
		nextRuneType RuneTokenType
		err          error
	)
SCAN:
	for {
		nextRune, _, err = t.input.ReadRune()
		nextRuneType = t.classifier.ClassifyRune(nextRune)
		if err != nil {
			if err == io.EOF {
				nextRuneType = RUNETOKEN_EOF
				err = nil
			} else {
				return nil, err
			}
		}
		switch state {
		case STATE_START: // no runes read yet
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						return nil, io.EOF
					}
				case RUNETOKEN_CHAR:
					{
						tokenType = TOKEN_WORD
						value = append(value, nextRune)
						state = STATE_INWORD
					}
				case RUNETOKEN_SPACE:
					{
					}
				case RUNETOKEN_ESCAPING_QUOTE:
					{
						tokenType = TOKEN_WORD
						state = STATE_QUOTED_ESCAPING
					}
				case RUNETOKEN_NONESCAPING_QUOTE:
					{
						tokenType = TOKEN_WORD
						state = STATE_QUOTED
					}
				case RUNETOKEN_ESCAPE:
					{
						tokenType = TOKEN_WORD
						state = STATE_ESCAPING
					}
				case RUNETOKEN_COMMENT:
					{
						tokenType = TOKEN_COMMENT
						state = STATE_COMMENT
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Unknown rune: %v", nextRune))
					}
				}
			}
		case STATE_INWORD: // in a regular word
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						break SCAN
					}
				case RUNETOKEN_CHAR, RUNETOKEN_COMMENT:
					{
						value = append(value, nextRune)
					}
				case RUNETOKEN_SPACE:
					{
						t.input.UnreadRune()
						break SCAN
					}
				case RUNETOKEN_ESCAPING_QUOTE:
					{
						state = STATE_QUOTED_ESCAPING
					}
				case RUNETOKEN_NONESCAPING_QUOTE:
					{
						state = STATE_QUOTED
					}
				case RUNETOKEN_ESCAPE:
					{
						state = STATE_ESCAPING
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
					}
				}
			}
		case STATE_ESCAPING: // the next rune after an escape character
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						err = errors.New("EOF found after escape character")
						break SCAN
					}
				case RUNETOKEN_CHAR, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
					{
						state = STATE_INWORD
						value = append(value, nextRune)
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
					}
				}
			}
		case STATE_ESCAPING_QUOTED: // the next rune after an escape character, in double quotes
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						err = errors.New("EOF found after escape character")
						break SCAN
					}
				case RUNETOKEN_CHAR, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
					{
						state = STATE_QUOTED_ESCAPING
						value = append(value, nextRune)
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
					}
				}
			}
		case STATE_QUOTED_ESCAPING: // in escaping double quotes
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						err = errors.New("EOF found when expecting closing quote.")
						break SCAN
					}
				case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_SPACE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_COMMENT:
					{
						value = append(value, nextRune)
					}
				case RUNETOKEN_ESCAPING_QUOTE:
					{
						state = STATE_INWORD
					}
				case RUNETOKEN_ESCAPE:
					{
						state = STATE_ESCAPING_QUOTED
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
					}
				}
			}
		case STATE_QUOTED: // in non-escaping single quotes
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						err = errors.New("EOF found when expecting closing quote.")
						break SCAN
					}
				case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
					{
						value = append(value, nextRune)
					}
				case RUNETOKEN_NONESCAPING_QUOTE:
					{
						state = STATE_INWORD
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
					}
				}
			}
		case STATE_COMMENT:
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						break SCAN
					}
				case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT, RUNETOKEN_NONESCAPING_QUOTE:
					{
						value = append(value, nextRune)
					}
				case RUNETOKEN_SPACE:
					{
						if nextRune == '\n' {
							state = STATE_START
							break SCAN
						} else {
							value = append(value, nextRune)
						}
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
					}
				}
			}
		default:
			{
				panic(fmt.Sprintf("Unexpected state: %v", state))
			}
		}
	}
	token := &Token{
		tokenType: tokenType,
		value:     string(value)}
	return token, err
}
76623
func scanSecrets(r io.Reader) ([]secretPair, error) {
	var pairs []secretPair

	lexer := shlex.NewLexer(r)

	for {
		word, err := lexer.Next()
		if err != nil {
			if err == io.EOF {
				return pairs, nil
			}
			return nil, fmt.Errorf("Error reading input file. %s", err)
		}

		tokens := strings.SplitN(word, "=", 2)
		if len(tokens) < 2 {
			return nil, fmt.Errorf("Error parsing secret %q", word)
		}

		key := tokens[0]
		value := tokens[1]

		if key == "" || value == "" {
			return nil, fmt.Errorf("Error parsing secret %q", word)
		}

		pairs = append(pairs, secretPair{key: key, value: value})
	}
}