// Tokenizer is the name of the tokenizer to use for the analysis.
356254
func (s *IndicesAnalyzeService) Tokenizer(tokenizer string) *IndicesAnalyzeService {
	s.request.Tokenizer = tokenizer
	return s
}
65140
func NewTokenizer(r io.Reader) *Tokenizer {
	return &Tokenizer{
		InStream: r,
		buf:      make([]byte, defaultBufSize),
	}
}
91216
func (z *Tokenizer) Text() []byte {
	switch z.tt {
	case TextToken, CommentToken, DoctypeToken:
		s := z.buf[z.data.start:z.data.end]
		z.data.start = z.raw.end
		z.data.end = z.raw.end
		s = convertNewlines(s)
		if (z.convertNUL || z.tt == CommentToken) && bytes.Contains(s, nul) {
			s = bytes.Replace(s, nul, replacement, -1)
		}
		if !z.textIsRaw {
			s = unescape(s, false)
		}
		return s
	}
	return nil
}
1259242
func (s *ValidateService) Analyzer(analyzer string) *ValidateService {
	s.analyzer = analyzer
	return s
}
703955
func (l *lexer) tokenize() {
	for l.state = lexData; l.state != nil; {
		l.state = l.state(l)
	}
}
815582
func Literator(tokens ...Token) Iterator {
	return func() Token {
		if len(tokens) == 0 {
			return EOF
		}
		token := tokens[0]
		tokens = tokens[1:]
		return token
	}
}
330320
func (z *Tokenizer) readStartTag() TokenType {
	z.readTag(true)
	if z.err != nil {
		return ErrorToken
	}
	// Several tags flag the tokenizer's next token as raw.
	c, raw := z.buf[z.data.start], false
	if 'A' <= c && c <= 'Z' {
		c += 'a' - 'A'
	}
	switch c {
	case 'i':
		raw = z.startTagIn("iframe")
	case 'n':
		raw = z.startTagIn("noembed", "noframes", "noscript")
	case 'p':
		raw = z.startTagIn("plaintext")
	case 's':
		raw = z.startTagIn("script", "style")
	case 't':
		raw = z.startTagIn("textarea", "title")
	case 'x':
		raw = z.startTagIn("xmp")
	}
	if raw {
		z.rawTag = strings.ToLower(string(z.buf[z.data.start:z.data.end]))
	}
	// Look for a self-closing token like "<br/>".
	if z.err == nil && z.buf[z.raw.end-2] == '/' {
		return SelfClosingTagToken
	}
	return StartTagToken
}
569447
func (t *SimpleTokenizer) Tokenize(sentence string) []string {
	// is the tokenizer really the best place to be making
	// everything lowercase? is this really a sanitizaion func?
	return strings.Split(strings.ToLower(sentence), t.SplitOn)
}
326938
func (s *IndicesAnalyzeService) Analyzer(analyzer string) *IndicesAnalyzeService {
	s.request.Analyzer = analyzer
	return s
}
868774
func tokenize(src string, tc chan *sep) {
	var s scanner.Scanner
	fset := token.NewFileSet() // boilerplate stuff for scanner...
	file := fset.AddFile("", fset.Base(), len(src))
	s.Init(file, []byte(src), nil, 1)
	for {
		filePos, tok, _ := s.Scan()
		tc <- &sep{int(filePos) - file.Base(), tok}
		if tok == token.EOF {
			close(tc)
			break
		}
	}
}
698868
func NewTokenGenerator(s string, err error) TokenGenerator {
	return TokenGenerator{
		TokenFn: func() (string, error) {
			return s, err
		},
	}
}
1265232
func (c *Config) TokenSource(ctx context.Context, t *Token) TokenSource {
	tkr := &tokenRefresher{
		ctx:  ctx,
		conf: c,
	}
	if t != nil {
		tkr.refreshToken = t.RefreshToken
	}
	return &reuseTokenSource{
		t:   t,
		new: tkr,
	}
}
1120620
func (t Tokenizer) Dot(input string, w io.Writer) (tokens []Token) {
	return t.AnalyzeGraph(input, Normal, w)
}
428937
func (t Tokenizer) Analyze(input string, mode TokenizeMode) (tokens []Token) {
	la := lattice.New(t.dic, t.udic)
	defer la.Free()
	la.Build(input)
	m := lattice.Normal
	switch mode {
	case Normal:
		m = lattice.Normal
	case Search:
		m = lattice.Search
	case Extended:
		m = lattice.Extended
	}
	la.Forward(m)
	la.Backward(m)
	size := len(la.Output)
	tokens = make([]Token, 0, size)
	for i := range la.Output {
		n := la.Output[size-1-i]
		tok := Token{
			ID:      n.ID,
			Class:   TokenClass(n.Class),
			Start:   n.Start,
			End:     n.Start + utf8.RuneCountInString(n.Surface),
			Surface: n.Surface,
			dic:     t.dic,
			udic:    t.udic,
		}
		if tok.ID == lattice.BosEosID {
			if i == 0 {
				tok.Surface = "BOS"
			} else {
				tok.Surface = "EOS"
			}
		}
		tokens = append(tokens, tok)
	}
	return
}
973694
func (c *cachedGetter) Token() (string, error) {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	var err error
	// no token or exceeds the TTL
	if c.token == "" || time.Since(c.born) > c.ttl {
		c.token, err = c.tokenGetter.Token()
		if err != nil {
			return "", fmt.Errorf("failed to get token: %s", err)
		}
		c.born = time.Now()
	}
	return c.token, nil
}
45222
func (l *lexer) token(type_ TokenType, word []rune, e Error) (*Token, Error) {
	// if we have an error, then we return a BAD token
	if e != nil {
		type_ = BAD
	}

	// extract the lexer position
	//      - the column is taken from the current column position
	//        minus the length of the consumed "word"
	_word := len(word)
	_column := l._column - _word
	_offset := l._offset - _word
	position := Position{"", l._line, _column, _offset}

	// if this is a newline token, we adjust the line & column counts
	if type_ == EOL {
		l.newline()
	}

	// return the Token
	return NewToken(type_, word, position), e
}
424613
func tokenFromInt(iv int32) lex.Token {
	t, ok := lex.TokenNameMap[lex.TokenType(iv)]
	if ok {
		return lex.Token{T: t.T, V: t.Description}
	}
	return lex.Token{}
}