/*
Expert: Set a new reader on the Tokenizer. Typically, an analyzer (in
its tokenStream method) will use this to re-use a previously created
tokenizer.
*/
357136
func (t *Tokenizer) SetReader(input io.RuneReader) error {
	assert2(input != nil, "input must not be nil")
	assert2(t.Input == ILLEGAL_STATE_READER, "TokenStream contract violation: close() call missing")
	t.inputPending = input
	return nil
}
214045
func NewReader(getReaderFunc func(offset, length int64) (io.ReadCloser, error), args *ReaderArgs) (*Reader, error) {
	reader, err := parquetgo.NewReader(getReaderFunc, nil)
	if err != nil {
		if err != io.EOF {
			return nil, errParquetParsingError(err)
		}

		return nil, err
	}

	return &Reader{
		args:   args,
		reader: reader,
	}, nil
}
1094154
func (s *UpdateByQueryService) Analyzer(analyzer string) *UpdateByQueryService {
	s.analyzer = analyzer
	return s
}
412770
func NewReader(r io.Reader, limiter Limiter) io.Reader {
	return &reader{r, limiter}
}
881735
func NewReadCloser(r io.Reader, c io.Closer) io.ReadCloser {
	return &readCloser{Reader: r, closer: c}
}
1015646
func NewReader(ctx context.Context, r io.Reader) *Reader {
	return &Reader{
		ctx:    ctx,
		reader: r,
	}
}
378040
func NewSentenceTokenizer(s *Storage) *DefaultSentenceTokenizer {
	lang := NewPunctStrings()
	word := NewWordTokenizer(lang)

	annotations := NewAnnotations(s, lang, word)

	tokenizer := &DefaultSentenceTokenizer{
		Storage:       s,
		PunctStrings:  lang,
		WordTokenizer: word,
		Annotations:   annotations,
	}

	return tokenizer

}
776913
func NewPunktSentenceTokenizer() *PunktSentenceTokenizer {
	var pt PunktSentenceTokenizer
	var err error

	pt.tokenizer, err = newSentenceTokenizer(nil)
	util.CheckError(err)

	return &pt
}
59509
func newStandardTokenizer(matchVersion util.Version, input io.RuneReader) *StandardTokenizer {
	ans := &StandardTokenizer{
		Tokenizer:      NewTokenizer(input),
		maxTokenLength: DEFAULT_MAX_TOKEN_LENGTH,
	}
	ans.termAtt = ans.Attributes().Add("CharTermAttribute").(CharTermAttribute)
	ans.offsetAtt = ans.Attributes().Add("OffsetAttribute").(OffsetAttribute)
	ans.posIncrAtt = ans.Attributes().Add("PositionIncrementAttribute").(PositionIncrementAttribute)
	ans.typeAtt = ans.Attributes().Add("TypeAttribute").(TypeAttribute)
	ans.init(matchVersion)
	return ans
}
339014
func NewLexer(src io.Reader, fName string) *Lexer {
	file := t.NewFileSet().AddFile(fName, -1, 1<<31-3)
	lx, err := lex.New(file, bufio.NewReader(src), lex.RuneClass(Rune2Class))
	if err != nil {
		panic(err)
	}

	return &Lexer{
		Lexer:         lx,
		StateStack:    []int{0},
		PhpDocComment: "",
		FreeFloating:  nil,
		heredocLabel:  "",
		tokenBytesBuf: &bytes.Buffer{},
		TokenPool:     &TokenPool{},
	}
}
536657
func NewReader(r io.Reader, limit int64) *Reader {
	return &Reader{r, New(0, 0), limit, true}
}
353639
func NewReaderLexer(in io.Reader, fn LexFn) *ReaderLexer {
	return &ReaderLexer{
		bufio.NewReader(in),
		0,
		-1,
		-1,
		1,
		[]rune{},
		make(chan LexItem, 1),
		fn,
	}
}
943764
func NewAnalyzerWithLogger(resizer options.Resizer, logger Logger) Analyzer {
	if logger.Log == nil {
		logger.Log = log.New(ioutil.Discard, "", 0)
	}
	return &smartcropAnalyzer{Resizer: resizer, logger: logger}
}
69589
func NewAnalyzer(resizer options.Resizer) Analyzer {
	logger := Logger{
		DebugMode: false,
	}

	return NewAnalyzerWithLogger(resizer, logger)
}
593976
func NewReader(charset string, r io.Reader) (io.Reader, error) {
	tr, err := TranslatorFrom(charset)
	if err != nil {
		return nil, err
	}
	return NewTranslatingReader(r, tr), nil
}
517962
func (s *S3) NewReader(id string) (io.ReadCloser, error) {
	r, _, err := s.bucket.GetReader(s.Prefixed.Name(id), nil)
	return r, err
}
930940
func NewReader(r io.Reader) *Reader {
	return &Reader{
		r:       r,
		decoded: make([]byte, maxUncompressedChunkLen),
		buf:     make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize),
	}
}