// Tokenize analyze a sentence in standard tokenize mode.
770618
func (t Tokenizer) Tokenize(input string) []Token {
	return t.Analyze(input, Normal)
}
190159
func (l Lexer) TokenizeString(s string) ([]Token, error) {
	r := bufio.NewReader(strings.NewReader(s))
	tokens := []Token{}
	err := l.Tokenize(r, func(t Token) error {
		tokens = append(tokens, t)
		return nil
	})
	return tokens, err
}
755077
func (s *DefaultSentenceTokenizer) SentencePositions(text string) []int {
	annotatedTokens := s.AnnotatedTokens(text)

	positions := make([]int, 0, len(annotatedTokens))
	for _, token := range annotatedTokens {
		if !token.SentBreak {
			continue
		}

		positions = append(positions, token.Position)
	}

	lastChar := len(text)
	positions = append(positions, lastChar)

	return positions
}
675287
func tokenize(msgText string) tokenMap {
	if msgText == "" {
		return nil
	}

	// split the message text up on basic punctuation/spaces
	tokens := splitExpr.Split(msgText, -1)
	tokenMap := tokenMap{}
	for _, token := range tokens {
		if len(token) < MinTokenLength {
			continue
		}

		token = strings.ToLower(token)
		if _, ok := tokenMap[token]; !ok {
			tokenMap[token] = map[string]chat1.EmptyStruct{}
		}

		// strip separators to raw tokens which we count as an alias to the
		// original token
		stripped := stripExpr.Split(token, -1)
		for _, s := range stripped {
			if s == "" {
				continue
			}
			tokenMap[token][s] = chat1.EmptyStruct{}

			// add the stem as an alias
			stemmed := porterstemmer.StemWithoutLowerCasing([]rune(s))
			tokenMap[token][string(stemmed)] = chat1.EmptyStruct{}

			// calculate prefixes to alias to the token
			for _, prefix := range prefixes(s) {
				tokenMap[token][prefix] = chat1.EmptyStruct{}
			}
		}
		// drop the original token from the set of aliases
		delete(tokenMap[token], token)
	}
	return tokenMap
}
8075
func Camelize(word string) (camelized string) {
	for pos, ru := range []rune(word) {
		if pos > 0 {
			camelized += string(unicode.ToLower(ru))
		} else {
			camelized += string(unicode.ToUpper(ru))
		}
	}
	return
}
694434
func (c *Config) Normalize() (err error) {
	if err = c.Swarming.Normalize(); err != nil {
		return
	}
	if err = c.Isolate.Normalize(); err != nil {
		return
	}
	return
}
461730
func spacify(s string) string {
	v := spaceRE.Split(s, -1)
	if len(v) > 0 && v[0] == "" {
		v = v[1:]
	}
	l := len(v)
	if l > 0 && v[l-1] == "" {
		v = v[0:(l - 1)]
	}
	return strings.Join(v, " ")
}
868774
func tokenize(src string, tc chan *sep) {
	var s scanner.Scanner
	fset := token.NewFileSet() // boilerplate stuff for scanner...
	file := fset.AddFile("", fset.Base(), len(src))
	s.Init(file, []byte(src), nil, 1)
	for {
		filePos, tok, _ := s.Scan()
		tc <- &sep{int(filePos) - file.Base(), tok}
		if tok == token.EOF {
			close(tc)
			break
		}
	}
}
191156
func (l *lexer) emit(t TokenType) {
	l.tokens <- Token{
		Type: t,
		Text: l.input[l.start:l.pos],
	}
	l.start = l.pos
}
964324
func (l *lexer) emit(t tokenType) {
	l.tokens <- token{
		t,
		l.input[l.start:l.pos],
		l.lineNum(),
		l.columnNum(),
	}
	l.start = l.pos
}
597448
func (m *tokenManager) Parse(ctx context.Context, tokenString string) (*jwt.Token, error) {
	keyFunc := m.KeyFunction(ctx)
	jwtToken, err := jwt.Parse(tokenString, keyFunc)
	if err != nil {
		log.Error(ctx, map[string]interface{}{
			"err": err,
		}, "unable to parse token")
		return nil, autherrors.NewUnauthorizedError(err.Error())
	}
	return jwtToken, nil
}
1028144
func (c *ChatRPC) ChatTLFFinalize(
	_ context.Context, _ chat1.ChatTLFFinalizeArg) error {
	return nil
}
815582
func Literator(tokens ...Token) Iterator {
	return func() Token {
		if len(tokens) == 0 {
			return EOF
		}
		token := tokens[0]
		tokens = tokens[1:]
		return token
	}
}
834500
func (s *Etcd) normalize(key string) string {
	key = store.Normalize(key)
	return strings.TrimPrefix(key, "/")
}
356254
func (s *IndicesAnalyzeService) Tokenizer(tokenizer string) *IndicesAnalyzeService {
	s.request.Tokenizer = tokenizer
	return s
}
938184
func (r RegexpTokenizer) Tokenize(text string) []string {
	var tokens []string
	if r.gaps {
		temp := r.regex.Split(text, -1)
		if r.discard {
			for _, s := range temp {
				if s != "" {
					tokens = append(tokens, s)
				}
			}
		} else {
			tokens = temp
		}
	} else {
		tokens = r.regex.FindAllString(text, -1)
	}
	return tokens
}
619326
func (l *Localizer) Localize(lc *LocalizeConfig) (string, error) {
	msg, _, err := l.LocalizeWithTag(lc)
	return msg, err
}