Tokenizer and Stemmer
501607
def _tokenize(sentence):
    ''''''

    _tokens = nltk.word_tokenize(sentence)
    tokens = [stemmer.stem(tk) for tk in _tokens]
    return tokens
10600
private void nextToken(StreamTokenizer tokenizer) throws IOException {
    tokenizer.nextToken();
    if((tokenizer.ttype == '\'') || (tokenizer.ttype == '"')) {
      tokenizer.ttype = StreamTokenizer.TT_WORD;
    }
    else if((tokenizer.ttype == StreamTokenizer.TT_WORD) && (tokenizer.sval.equals("?"))) {
      tokenizer.ttype = '?';
    }
    if(LOG.isDebugging()) {
      if(tokenizer.ttype == StreamTokenizer.TT_NUMBER) {
        LOG.debug("token: " + tokenizer.nval);
      }
      else if(tokenizer.ttype == StreamTokenizer.TT_WORD) {
        LOG.debug("token: " + tokenizer.sval);
      }
      else if(tokenizer.ttype == StreamTokenizer.TT_EOF) {
        LOG.debug("token: EOF");
      }
      else if(tokenizer.ttype == StreamTokenizer.TT_EOL) {
        LOG.debug("token: EOL");
      }
      else {
        LOG.debug("token type: " + tokenizer.ttype);
      }
    }
  }
778837
function tokenizeText(text, callback) {
    "use strict";
    processTokens(tokenizer.tokenize(text) || [], function (error, tokens) {
        process.nextTick(function () {
            callback(error, tokens);
        });
    });
}
1008192
def stem(self, s):
        
        return javabridge.get_env().get_string(self.__stem(javabridge.get_env().new_string_utf(s)))
603419
function Lexer (options) {
  this.tokens = []
  this.options = options || defaults
  this.rules = block
  if (this.options.gfm) {
    Object.assign(this.rules, block.opts.gfm)
  }
  if (this.options.include) {
    Object.assign(this.rules, block.opts.include)
  }
  if (this.options.numberedheadings) {
    Object.assign(this.rules, block.opts.numberedheadings)
  }
  if (this.options.toc) {
    Object.assign(this.rules, block.opts.toc)
  }
  if (this.options.ref) {
    Object.assign(this.rules, block.opts.ref)
  }
}
599056
def token(name):
    

    def wrap(f):
        tokenizers.append((name, f))
        return f

    return wrap
1338103
public function setTokenizer(Tokenizer $tokenizer)
    {
        if (null === $this->converter) {
            $this->converter = new ToString();
        }

        $this->tokenizer = $tokenizer;
    }
1006206
def make_tokenizer(specs):
    """"""

    def compile_spec(spec):
        name, args = spec
        return name, re.compile(*args)

    compiled = [compile_spec(s) for s in specs]

    def match_specs(specs, str, i, position):
        line, pos = position
        for type, regexp in specs:
            m = regexp.match(str, i)
            if m is not None:
                value = m.group()
                nls = value.count(u'\n')
                n_line = line + nls
                if nls == 0:
                    n_pos = pos + len(value)
                else:
                    n_pos = len(value) - value.rfind(u'\n') - 1
                return Token(type, value, (line, pos + 1), (n_line, n_pos))
        else:
            errline = str.splitlines()[line - 1]
            raise LexerError((line, pos + 1), errline)

    def f(str):
        length = len(str)
        line, pos = 1, 0
        i = 0
        while i < length:
            t = match_specs(compiled, str, i, (line, pos))
            yield t
            line, pos = t.end
            i += len(t.value)

    return f
681857
def tokenize(stream, separator):
        
        for value in stream:
            for token in value.split(separator):
                if token:
                    yield token.strip()
1305674
function matcherFromTokens( tokens, context, xml ) {
	var token, matcher,
		i = 0;

	for ( ; (token = tokens[i]); i++ ) {
		if ( Expr.relative[ token.part ] ) {
			matcher = addCombinator( matcher, Expr.relative[ token.part ], context );
		} else {
			token.captures.push( context, xml );
			matcher = addMatcher( matcher, Expr.filter[ token.part ].apply( null, token.captures ) );
		}
	}

	return matcher;
}
247551
def listified_tokenizer(source):
    """"""
    io_obj = io.StringIO(source)
    return [list(a) for a in tokenize.generate_tokens(io_obj.readline)]
46386
function tokenizer(input) {
  const topLevelTokens = getTopLevelTokens();

  const length = input.length;

  const output = [];

  let cursor = 0;
  let lastBreak = 0;

  while (cursor < length) {
    const slice = input.slice(cursor);
    const found = matchPattern(topLevelTokens, slice, false);

    if (found && input[cursor - 1] === '\\') {
      const text = input.slice(lastBreak, cursor - 1);
      if (text) {
        output.push(new Text(text));
      }

      const escapedText = found[1][0];
      output.push(new Text(escapedText));

      cursor += escapedText.length;
      lastBreak = cursor;
    } else if (found) {
      const [Tok, matches] = found;

      const text = input.slice(lastBreak, cursor);
      if (text) {
        output.push(new Text(text));
      }
      output.push(new Tok(...matches));

      cursor += matches[0].length;
      lastBreak = cursor;
    } else {
      cursor += 1;
    }
  }
  const text = input.slice(lastBreak, cursor);
  if (text) {
    output.push(new Text(text));
  }

  // if there are more closes than opens
  // intelligently remove extra ones
  return removeExtraCloses(output);
}
1321558
def span_tokenize(self, s):
        
        if self._tokenizer is None:
            self._tokenizer = load_model(self.model)
        # for debug in tokenizer.debug_decisions(s):
        #     log.debug(format_debug_decision(debug))
        return self._tokenizer.span_tokenize(s)
332456
def get_language_stemmer(language):
    
    from lunr.languages import SUPPORTED_LANGUAGES
    from nltk.stem.snowball import SnowballStemmer

    return SnowballStemmer(SUPPORTED_LANGUAGES[language])
1303372
function attacher(options) {
    var Parser = this.Parser;
    var proto = Parser.prototype;

    /*
     * Add a tokenizer to the `Parser`.
     */
    proto.blockTokenizers.blockquote = tokenizeAnnotatedBlockquote;

    return transformer;
}
1047099
def create_stemmer(self, isDev=False):
        

        words = self.get_words(isDev)
        dictionary = ArrayDictionary(words)
        stemmer = Stemmer(dictionary)

        resultCache = ArrayCache()
        cachedStemmer = CachedStemmer(resultCache, stemmer)

        return cachedStemmer
478019
def _tokenize(self, text, token_class=None):
        
        token_class = token_class or Token
        tokens = {}

        for i, match in enumerate(self.regex.finditer(text)):
            value = match.group(0)

            try:
                token = tokens[value]
            except KeyError:
                type = match.lastgroup
                token = token_class(value, type=type)
                tokens[value] = token

            yield token