// tokens returns a channel that emits seps from tokenizing src.
19631
func tokens(src string) <-chan *sep {
	tc := make(chan *sep)
	go tokenize(src, tc)
	return tc
}
1243967
func (s *RemoteApplication) Token() (string, error) {
	r := s.st.RemoteEntities()
	return r.GetToken(s.Tag())
}
25997
func Tokenise(lexer Lexer, options *TokeniseOptions, text string) ([]Token, error) {
	var out []Token
	it, err := lexer.Tokenise(options, text)
	if err != nil {
		return nil, err
	}
	for t := it(); t != EOF; t = it() {
		out = append(out, t)
	}
	return out, nil
}
1037063
func allTokens(input io.Reader) ([]Token, error) {
	l := new(lexer)
	err := l.load(input)
	if err != nil {
		return nil, err
	}
	var tokens []Token
	for l.next() {
		tokens = append(tokens, l.token)
	}
	return tokens, nil
}
852203
func (i Iterator) Tokens() []Token {
	var out []Token
	for t := i(); t != EOF; t = i() {
		out = append(out, t)
	}
	return out
}
944643
func TokenName(c int) string {
	if c >= TAnd && c-TAnd < len(yyToknames) {
		if yyToknames[c-TAnd] != "" {
			return yyToknames[c-TAnd]
		}
	}
	return string([]byte{byte(c)})
}
50110
func (parser *Parser) ParseDeclarations() ([]*css.Declaration, error) {
	result := []*css.Declaration{}

	if parser.tokenChar("{") {
		parser.shiftToken()
	}

	for parser.tokenParsable() {
		if parser.tokenIgnorable() {
			parser.shiftToken()
		} else if parser.tokenChar("}") {
			// end of block
			parser.shiftToken()
			break
		} else {
			declaration, err := parser.ParseDeclaration()
			if err != nil {
				return result, err
			}

			result = append(result, declaration)
		}
	}

	return result, parser.err()
}
725095
func (e *WordTokenizer) HasSentEndChars(t *sentences.Token) bool {
	enders := []string{
		`."`, `.'`, `.)`, `.’`, `.”`,
		`?`, `?"`, `?'`, `?)`, `?’`, `?”`,
		`!`, `!"`, `!'`, `!)`, `!’`, `!”`,
	}

	for _, ender := range enders {
		if strings.HasSuffix(t.Tok, ender) {
			return true
		}
	}

	parens := []string{
		`.[`, `.(`, `."`, `.'`,
		`?[`, `?(`,
		`![`, `!(`,
	}

	for _, paren := range parens {
		if strings.Index(t.Tok, paren) != -1 {
			return true
		}
	}

	return false
}
938184
func (r RegexpTokenizer) Tokenize(text string) []string {
	var tokens []string
	if r.gaps {
		temp := r.regex.Split(text, -1)
		if r.discard {
			for _, s := range temp {
				if s != "" {
					tokens = append(tokens, s)
				}
			}
		} else {
			tokens = temp
		}
	} else {
		tokens = r.regex.FindAllString(text, -1)
	}
	return tokens
}
420866
func Tokenize(input []rune) ([]Token, error) {
	tokens := []Token{}
	for i, t, err := nextToken(0, input); err != errEndOfInput; i, t, err = nextToken(i, input) {
		if err != nil {
			return nil, err
		}

		tokens = append(tokens, t)
	}

	return tokens, nil
}
186363
func (t *Transactor) ListTokens() (*token.UnspentTokens, error) {
	iterator, err := t.Ledger.GetStateRangeScanIterator(tokenNameSpace, "", "")
	if err != nil {
		return nil, err
	}

	tokens := make([]*token.TokenOutput, 0)
	prefix, err := createPrefix(tokenOutput)
	if err != nil {
		return nil, err
	}
	for {
		next, err := iterator.Next()

		switch {
		case err != nil:
			return nil, err

		case next == nil:
			// nil response from iterator indicates end of query results
			return &token.UnspentTokens{Tokens: tokens}, nil

		default:
			result, ok := next.(*queryresult.KV)
			if !ok {
				return nil, errors.New("failed to retrieve unspent tokens: casting error")
			}
			if strings.HasPrefix(result.Key, prefix) {
				output := &token.PlainOutput{}
				err = proto.Unmarshal(result.Value, output)
				if err != nil {
					return nil, errors.New("failed to retrieve unspent tokens: casting error")
				}
				if string(output.Owner) == string(t.PublicCredential) {
					spent, err := t.isSpent(result.Key)
					if err != nil {
						return nil, err
					}
					if !spent {
						tokens = append(tokens,
							&token.TokenOutput{
								Type:     output.Type,
								Quantity: output.Quantity,
								Id:       getCompositeKeyBytes(result.Key),
							})
					}
				}
			}
		}
	}

}
271399
func (d *Decoder) selectors(tok *token) {
	for {
		if tok.t == tokenHash || tok.t == tokenAttachment || tok.t == tokenClass || tok.t == tokenLBracket {
			d.selector(tok)
			tok = d.next()
			if tok.t == tokenComma {
				tok = d.next() // TODO non-selector after comma?
				if tok.t == tokenLBrace {
					// dangling comma
					d.backup()
					break
				}

				continue
			}
			d.backup()
		} else {
			d.error(d.pos(tok), "expected layer, attachment, class or filter, got %v", tok)
		}
		break
	}
}
608871
func (l *lexer) token() token {
	for {
		select {
		case token := <-l.tokens:
			return token
		default:
			l.state = l.state(l)
		}
	}
}
275874
func (bc Bytecode) Tokens() ([]string, error) {
	// Overestimate of capacity in the presence of pushes
	tokens := make([]string, 0, len(bc))

	for i := 0; i < len(bc); i++ {
		op, ok := asm.GetOpCode(bc[i])
		if !ok {
			return tokens, fmt.Errorf("did not recognise byte %#x at position %v as an OpCode:\n %s",
				bc[i], i, lexingPositionString(bc, i, tokens))
		}
		pushes := op.Pushes()
		tokens = append(tokens, op.Name())
		if pushes > 0 {
			// This is a PUSH<N> OpCode so consume N bytes from the input, render them as hex, and skip to next OpCode
			if i+pushes >= len(bc) {
				return tokens, fmt.Errorf("token %v of input is %s but not enough input remains to push %v: %s",
					i, op.Name(), pushes, lexingPositionString(bc, i, tokens))
			}
			pushedBytes := bc[i+1 : i+pushes+1]
			tokens = append(tokens, fmt.Sprintf("0x%s", pushedBytes))
			i += pushes
		}
	}
	return tokens, nil
}
718662
func (r *Registry) Headers(ctx context.Context, uri string) (map[string]string, error) {
	// Get the token.
	token, err := r.Token(ctx, uri)
	if err != nil {
		if err == ErrBasicAuth {
			// If we couldn't get a token because the server requires basic auth, just return basic auth headers.
			return map[string]string{
				"Authorization": fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(r.Username+":"+r.Password))),
			}, nil
		}
	}

	if len(token) < 1 {
		r.Logf("got empty token for %s", uri)
		return map[string]string{}, nil
	}

	return map[string]string{
		"Authorization": fmt.Sprintf("Bearer %s", token),
	}, nil
}
363790
func (t *TokenSource) Token() (*oauth2.Token, error) {
	token := &oauth2.Token{
		AccessToken: t.AccessToken,
	}
	return token, nil
}
223952
func (t AccessToken) Token() *adal.Token {
	return &adal.Token{
		AccessToken: t.AccessToken,
		Type:        t.TokenType,
	}
}