Generate ptokens for a given text
 ptokens: Tokens with position

 Example:

   >> analyzer = FreelingClient::Analyzer.new
   >> analyzer.ptoken(:morfo, "Este texto está en español.")

 Arguments:
   cmd: (Symbol)
   text: (String)
512722
def ptokens(cmd, text)
      Enumerator.new do |yielder|
        pos = 0
        tokens(cmd, text).each do |token|
          ne_text = token['form'].dup

          ne_regexp = build_regexp(ne_text)
          token_pos = text.index(ne_regexp, pos)

          if token_pos && token_pos < (pos + 5)
            token.pos = token_pos
            yielder << token

            pos = token_pos + ne_text.length
          else
            pos = pos + ne_text.length
          end
        end
      end
    end
661874
def tokenize(self, text):
        '''
        
        '''
        start = -1
        tokens = []
        for i, character in enumerate(text):
            if character == ' ' or character == '\t':
                if start >= 0:
                    word = text[start:i]
                    tokens.append({
                        'word': word,
                        'original_text': word,
                        'char_begin': start,
                        'char_end': i})
                    start = -1
            else:
                if start < 0:
                    start = i
        if start >= 0:
            tokens.append({
                'word': text[start:len(text)],
                'original_text': text[start:len(text)],
                'char_begin': start,
                'char_end': len(text)
            })
        return tokens
498052
public static Token[] makeTokens(String text, boolean tokenize) {
        if (text == null) {
            return null;
        }
        Token[] tokens;
        if (tokenize) {
            tokens = parse(text);
        } else {
            tokens = new Token[1];
            tokens[0] = new Token(text);
        }
        return tokens;
    }
420996
protected static List<Token> baseTokenize(String text) {
    String[] words = text.split(" ");
    List<Token> tokens = new LinkedList<Token>();
    for (String word : words) {
      tokens.add(new Token(word));
    }
    return tokens;
  }
555732
def process_text(self, t:str, tok:BaseTokenizer) -> List[str]:
        ""
        for rule in self.pre_rules: t = rule(t)
        toks = tok.tokenizer(t)
        for rule in self.post_rules: toks = rule(toks)
        return toks
906998
def t_TEXT(self, token):
        '
        text = token.lexer.lexmatch.group("text").decode("utf8")
        token.value = text
        return token
86340
def set_text(node, text, strategy):
    
    text = text.strip()
    textEquivs = node.get_TextEquiv()
    if not textEquivs:
        node.add_TextEquiv(TextEquivType(Unicode=text))
    #  elif strategy == 'index1':
    else:
        if len(textEquivs) > 1:
            index1 = [x for x in textEquivs if x.index == 1]
            if index1:
                index1[0].set_Unicode(text)
                return
        textEquivs[0].set_Unicode(text)
1173968
public function generateToken(string $source, string $target, string $text) : string
    {
        return sprintf('%d.%d', rand(10000, 99999), rand(10000, 99999));
    }
1221172
public List<String> tokenize(String text) {
		List<String> tokens = new ArrayList<String>();
		text = text.toLowerCase();
		char[] array = text.toCharArray();
		
		//convert all apostrophes to 0x27
		for(int i = 0; i < array.length; i++) {
			if(isApostrofe(array[i])) array[i] = 0x27;
		}
		
		int pos = 0;
		
		//term cycle
		while(pos < array.length) {
			//scan to the start of the term
			while(pos < array.length  && !Character.isLetterOrDigit(array[pos])) pos++;
			int start = pos;
			if(start == array.length) break;
			//scan to the end of the term
			while(pos < array.length  && isLetterOrDigitOrApostrofe(array[pos])) pos++;
			int newpos = pos;
			while(newpos > start && isApostrofe(array[newpos - 1])) newpos--;
			if(newpos > start) tokens.add(new String(array, start, newpos - start));
		}
		return tokens;
	}
840903
protected List<String> getTokens(String text){
		String[] tokens = text.toLowerCase().split("[\\s\\p{Punct}]");
		List<String> tokenList = new ArrayList<String>(tokens.length);
		for (int i = 0; i < tokens.length; i++){
			if (tokens[i].trim().length() < 1){
				continue;
			}
			tokenList.add(tokens[i].trim());
		}
		return tokenList;
	}
886761
function(context: ParserContext): ?Token {
    const delimiter = context.reader.peek();
    if (delimiter !== '"' && delimiter !== "'") return null;
    if (!isInsideOpenBracket(context)) return null;

    // read until the delimiter
    let stringValue = context.reader.read();
    while (!context.reader.isEOF()) {
      const next = context.reader.read();
      stringValue += next;

      if (next === delimiter) break;
    }

    return context.createToken("string", stringValue);
  }
1213035
def generate_token():
    
    length = 50
    stringset = string.ascii_letters + string.digits
    token = ''.join([stringset[i % len(stringset)] for i in [ord(x) for x in os.urandom(length)]])
    return token
223991
def token_generator(self, texts, **kwargs):
        
        for text_idx, text in enumerate(texts):
            if self.lower:
                text = text.lower()
            for char in text:
                yield text_idx, char
1123206
public static function generate( $text, $options ) {
		$tg = new self( $text, $options );
		$tg->tokenizer->beginStepping();
		while ( $tg->tokenizer->step() ) {
			foreach ( $tg->handler->tokens as $token ) {
				yield $token;
			}
			$tg->handler->tokens = [];
		}
		foreach ( $tg->handler->tokens as $token ) {
			yield $token;
		}
	}
214032
def pgettext(self, context, string, domain=None, **variables):
        """"""
        t = self.get_translations(domain)
        return t.upgettext(context, string) % variables
134586
def tokenize_list(self, text):
        
        return [self.get_record_token(record) for record in self.analyze(text)]
1035959
function createTokenizedTextProvider(pos, characterOptions, wordOptions) {
        var forwardIterator = createCharacterIterator(pos, false, null, characterOptions);
        var backwardIterator = createCharacterIterator(pos, true, null, characterOptions);
        var tokenizer = wordOptions.tokenizer;

        // Consumes a word and the whitespace beyond it
        function consumeWord(forward) {
            log.debug("consumeWord called, forward is " + forward);
            var pos, textChar;
            var newChars = [], it = forward ? forwardIterator : backwardIterator;

            var passedWordBoundary = false, insideWord = false;

            while ( (pos = it.next()) ) {
                textChar = pos.character;

                log.debug("Testing char '" + textChar + "'");

                if (allWhiteSpaceRegex.test(textChar)) {
                    if (insideWord) {
                        insideWord = false;
                        passedWordBoundary = true;
                    }
                } else {
                    log.debug("Got non-whitespace, passedWordBoundary is " + passedWordBoundary);
                    if (passedWordBoundary) {
                        it.rewind();
                        break;
                    } else {
                        insideWord = true;
                    }
                }
                newChars.push(pos);
            }

            log.debug("consumeWord done, pos is " + (pos ? pos.inspect() : "non-existent"));

            log.debug("consumeWord got new chars " + newChars.join(""));
            return newChars;
        }

        // Get initial word surrounding initial position and tokenize it
        var forwardChars = consumeWord(true);
        var backwardChars = consumeWord(false).reverse();
        var tokens = tokenize(backwardChars.concat(forwardChars), wordOptions, tokenizer);

        // Create initial token buffers
        var forwardTokensBuffer = forwardChars.length ?
            tokens.slice(arrayIndexOf(tokens, forwardChars[0].token)) : [];

        var backwardTokensBuffer = backwardChars.length ?
            tokens.slice(0, arrayIndexOf(tokens, backwardChars.pop().token) + 1) : [];

        function inspectBuffer(buffer) {
            var textPositions = ["[" + buffer.length + "]"];
            for (var i = 0; i < buffer.length; ++i) {
                textPositions.push("(word: " + buffer[i] + ", is word: " + buffer[i].isWord + ")");
            }
            return textPositions;
        }

        log.info("Initial word: ", inspectBuffer(forwardTokensBuffer) + "", " and ", inspectBuffer(backwardTokensBuffer) + "", forwardChars, backwardChars);

        return {
            nextEndToken: function() {
                var lastToken, forwardChars;

                // If we're down to the last token, consume character chunks until we have a word or run out of
                // characters to consume
                while ( forwardTokensBuffer.length == 1 &&
                    !(lastToken = forwardTokensBuffer[0]).isWord &&
                    (forwardChars = consumeWord(true)).length > 0) {

                    // Merge trailing non-word into next word and tokenize
                    forwardTokensBuffer = tokenize(lastToken.chars.concat(forwardChars), wordOptions, tokenizer);
                }

                return forwardTokensBuffer.shift();
            },

            previousStartToken: function() {
                var lastToken, backwardChars;

                // If we're down to the last token, consume character chunks until we have a word or run out of
                // characters to consume
                while ( backwardTokensBuffer.length == 1 &&
                    !(lastToken = backwardTokensBuffer[0]).isWord &&
                    (backwardChars = consumeWord(false)).length > 0) {

                    // Merge leading non-word into next word and tokenize
                    backwardTokensBuffer = tokenize(backwardChars.reverse().concat(lastToken.chars), wordOptions, tokenizer);
                }

                return backwardTokensBuffer.pop();
            },

            dispose: function() {
                forwardIterator.dispose();
                backwardIterator.dispose();
                forwardTokensBuffer = backwardTokensBuffer = null;
            }
        };
    }