Implements the tokenization algorithm. Recieves : source [string], nextToken [token]
5347
function (source, nextToken) {
      var tm = this,
          tokenizerRe,
          lastIndex,
          matches,
          textType;

      // Initialize for first run
      // if no template passed assume first pass
      // and use the main template.
      if (typeof source === "undefined") {
        source = tm.template;
      }

      // if no token passed, this is an open tag lookup
      if (typeof nextToken === "undefined") {
        tm._tokens = [];
        nextToken = tm.lex.tagOpen;
      }

      while (source) {
        // Use created token.
        tokenizerRe = nextToken.compiledRe;
        matches = tokenizerRe.exec(source);

        lastIndex = tokenizerRe.lastIndex;
        tokenizerRe.lastIndex = 0;

        // This guy matched. So let's push the tokens.
        if (matches) {

          // Flags to see if code or text
          if (nextToken === tm.lex.tagOpen) {
            // if the match was done using the tagOpen, then everything we matched is normal text.
            textType = "text";
          } else {
            // assume the match was a closing tag, so everything we have is code.
            textType = "code";
          }

          // if the token is a tagOpen, or tagClose, we need to push it with the text value matched.
          if (nextToken.hasText) {
            tm._tokens.push({
              type : textType,
              value : matches[1]
            });
          } else {
            // push the token name into the array to mark the open/close/print status on it.
            tm._tokens.push({
              type : nextToken.name
            });
          }

          // A print indicator just shows open prints, so we buffer the open print.
          if (nextToken === tm.lex.printIndicator) {
            tm._openPrints.push(tm._openBrackets);
          }

          // Closing an open print needs to count for open brackets, so we can separate the code blocks.
          if (nextToken === tm.lex.tagClose && tm._openPrints.length > 0) {
            tm._countBrackets(matches[1]);
          }

          // We matched, so let's keep going.
          // from where we left with our new next token.
          source    = source.substr(lastIndex);
          nextToken = tm._nextToken(nextToken);

        //No match:
        } else {
          // We couldn't match anything else. So: Either it's a syntax error,
          // or it's just text. If we were awaiting another token we fail, otherwise
          // assume it's just text.
          if (nextToken == tm.lex.tagOpen) {
            tm._tokens.push({
              type  : "text",
              value : source
            });

            source = "";

          } else {
            // If it wasn't optional. GO!
            if (nextToken.optional) {
              // So continue from where we left with our new next token.
              source    = source.substr(lastIndex);
              nextToken = tm._nextToken(nextToken);

            // so it was not a printing tag, so something went wrong.
            } else {
              throw "Syntax Error: Expecting " + nextToken.name + " ... Nowhere to be found. So I was like, what?";
            }
          }
        }
      }
    }
437414
def process_tokens(self, tokens):
        
        """
        for (tok_type, token, (start_row, _), _, _) in tokens:
            if tok_type == tokenize.STRING:
                self._check_string(token, start_row)
241772
def parse(source)
      buffer = ::Parser::Source::Buffer.new('(string)')
      buffer.source = source

      @parser.reset
      @parser.parse(buffer)
    end
137722
@Override
  public String[] synthesize(AnalyzedToken token, String posTag,
      boolean posTagRegExp) throws IOException {

    if (posTag != null && posTagRegExp) {
      String myPosTag = posTag;
      String det = "";
      if (posTag.endsWith(ADD_IND_DETERMINER)) {
        myPosTag = myPosTag.substring(0, myPosTag.indexOf(ADD_IND_DETERMINER) - "\\".length());
        det = aVsAnRule.suggestAorAn(token.getLemma());
        det = det.substring(0, det.indexOf(' ') + " ".length());
      } else if (posTag.endsWith(ADD_DETERMINER)) {
        myPosTag = myPosTag.substring(0, myPosTag.indexOf(ADD_DETERMINER) - "\\".length());
        det = "the ";
      }

      initPossibleTags();
      Pattern p = Pattern.compile(myPosTag);
      List<String> results = new ArrayList<>();

      for (String tag : possibleTags) {
        Matcher m = p.matcher(tag);
        if (m.matches()) {
          lookup(token.getLemma(), tag, results, det);
        }
      }
      return results.toArray(new String[results.size()]);
    }

    return synthesize(token, posTag);
  }
526157
Stream<Token> tokenize(String input) {
        List<Token> tokens = new ArrayList<Token>();
        int tokenPos = 0;
        while (!input.isEmpty()) {
            boolean matched = false;
            for (Token.Type tokenType : Token.Type.values()) {
                Matcher matcher = tokenType.pattern.matcher(input);
                if (matcher.find()) {
                    matched = true;
                    input = matcher.replaceFirst("");
                    if (tokenType != Token.Type.WHITESPACE) {
                        tokens.add(new Token(
                            tokenType,
                            matcher.group(),
                            tokenPos
                        ));
                    }
                    tokenPos += matcher.end();
                    break;
                }
            }
            if (!matched) {
                throw new LexerException(input);
            }
        }
        tokens.add(new Token(Token.Type.EOI, null, tokenPos));
        return new Stream<Token>(tokens.toArray(new Token[tokens.size()]));
    }
771013
public final void nextToken() {
        skipIgnoredToken();
        if (isVariableBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanVariable();
        } else if (isNCharBegin()) {
            currentToken = new Tokenizer(input, dictionary, ++offset).scanChars();
        } else if (isIdentifierBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanIdentifier();
        } else if (isHexDecimalBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanHexDecimal();
        } else if (isNumberBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanNumber();
        } else if (isSymbolBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanSymbol();
        } else if (isCharsBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanChars();
        } else if (isEnd()) {
            currentToken = new Token(Assist.END, "", offset);
        } else {
            throw new SQLParsingException(this, Assist.ERROR);
        }
        offset = currentToken.getEndPosition();
    }
753896
private void processIgnoredLeadingTokens(ParseTreeNode node, int position, int line, MemoEntry progress)
	    throws TreeException, ParserException {
	if (ignoredLeading) {
	    processIgnoredTokens(node, position, line, progress);
	}
    }
533436
function tokenize(source, options){
  var result = [];
  var tagStack = [];
  var lastTag = { children: result };
  var parseTag = false;
  var token;
  var state = TEXT;
  var pos = 0;
  var textStateEndPos = 0;
  var textEndPos;
  var bufferPos;
  var startPos;
  var m;

  /** @cut */ var attrMap;  // map to detect attribute duplicates
  /** @cut */ result.source_ = source;
  /** @cut */ result.warns = [];

  // skip starting whitespaces
  if (!options || options.trim !== false)
  {
    pos = textStateEndPos = source.match(/^\s*/)[0].length;
    source = source.trimRight();  // TODO: replace for basis.string.trimRight
  }

  while (pos < source.length || state != TEXT)
  {
    state.lastIndex = pos;
    startPos = pos;

    m = state.exec(source);

    if (!m || m.index !== pos)
    {
      // treat broken comment reference as comment content
      if (state == REFERENCE && token && token.type == TYPE_COMMENT)
      {
        state = COMMENT;
        continue;
      }

      if (parseTag)
        lastTag = tagStack.pop();

      if (token)
        lastTag.children.pop();

      if (token = lastTag.children.pop())
      {
        if (token.type == TYPE_TEXT && !token.refs)
          textStateEndPos -= token.value.length;
        else
          lastTag.children.push(token);
      }

      parseTag = false;
      state = TEXT;
      continue;
    }

    pos = state.lastIndex;

    //stat[state] = (stat[state] || 0) + 1;
    switch (state)
    {
      case TEXT:

        textEndPos = startPos + m[1].length;

        if (textStateEndPos != textEndPos)
        {
          var sourceText = textStateEndPos == startPos
            ? m[1]
            : source.substring(textStateEndPos, textEndPos);

          sourceText = sourceText.replace(/\s*(\r\n?|\n\r?)\s*/g, '');

          if (sourceText)
            lastTag.children.push({
              type: TYPE_TEXT,
              value: sourceText,
              range: {
                start_: textStateEndPos,
                end_: textEndPos
              }
            });
        }

        textStateEndPos = textEndPos;

        if (m[3])
        {
          lastTag.children.push({
            type: TYPE_TEXT,
            refs: ['l10n:' + m[3]],
            value: '{l10n:' + m[3] + '}',
            range: {
              start_: textEndPos,
              end_: pos
            }
          });
        }
        else if (m[2] == '{')
        {
          bufferPos = pos - 1;
          lastTag.children.push(token = {
            type: TYPE_TEXT,
            range: {
              start_: textEndPos,
              end_: textEndPos
            }
          });
          state = REFERENCE;
        }
        else if (m[4])
        {
          if (m[4] == '/')
          {
            token = null;
            state = CLOSE_TAG;
          }
          else //if (m[3] == '!--')
          {
            lastTag.children.push(token = {
              type: TYPE_COMMENT,
              range: {
                start_: textEndPos,
                end_: textEndPos
              }
            });

            if (m[5])
            {
              bufferPos = pos - m[5].length;
              state = REFERENCE;
            }
            else
            {
              bufferPos = pos;
              state = COMMENT;
            }
          }
        }
        else if (m[2]) // m[2] == '<' open tag
        {
          parseTag = true;
          tagStack.push(lastTag);

          lastTag.children.push(token = {
            type: TYPE_ELEMENT,
            attrs: [],
            children: [],
            range: {
              start_: textEndPos,
              end_: textEndPos
            }
          });
          lastTag = token;

          state = TAG_NAME;

          // create attribute map
          /** @cut */ attrMap = {};
        }

        break;

      case CLOSE_TAG:
        if (m[1] !== (lastTag.prefix ? lastTag.prefix + ':' : '') + lastTag.name)
        {
          lastTag.children.push({
            type: TYPE_TEXT,
            value: '</' + m[0],
            range: {
              start_: startPos - 2,
              end_: startPos + m[0].length
            }
          });
          /** @cut */ result.warns.push(['Wrong close tag: ' + source.substr(startPos - 2, m[0].length + 2), lastTag.children[lastTag.children.length - 1]]);
        }
        else
        {
          lastTag.range.end_ = startPos + m[0].length;
          lastTag = tagStack.pop();
        }

        state = TEXT;
        break;

      case TAG_NAME:
      case ATTRIBUTE_NAME_OR_END:
        if (m[2] == ':')
        {
          if (token.prefix)      // prefix was before, break tag parse
            state = BREAK_TAG_PARSE;
          else
            token.prefix = m[1];

          break;
        }

        if (m[1])
        {
          // store name (it may be null when check for attribute and end)
          token.name = m[1];
          token.range.end_ = startPos + m[1].length;

          // store attribute
          if (token.type == TYPE_ATTRIBUTE)
          {
            // detect attribute duplicates
            /** @cut */ var fullName = (token.prefix ? token.prefix + ':' : '') + token.name;
            /** @cut */ if (Object.prototype.hasOwnProperty.call(attrMap, fullName))
            /** @cut */   result.warns.push(['Duplicate attribute: ' + fullName, token]);
            /** @cut */ attrMap[fullName] = true;

            lastTag.attrs.push(token);
          }
        }

        if (m[2] == '{')
        {
          if (token.type == TYPE_ELEMENT)
            state = REFERENCE;
          else
            state = BREAK_TAG_PARSE;

          break;
        }

        if (m[3]) // end tag declaration
        {
          parseTag = false;
          lastTag.range.end_ = pos;

          if (m[3] == '/>' ||
              (!lastTag.prefix && SINGLETON_TAG.test(lastTag.name)))
          {
            /** @cut */ if (m[3] != '/>')
            /** @cut */   result.warns.push(['Tag <' + lastTag.name + '> doesn\'t closed explicit (use `/>` as tag ending)', lastTag]);

            lastTag = tagStack.pop();
          }
          else
          {
            // otherwise m[3] == '>'
            if (lastTag.prefix == 'b' && lastTag.name in TAG_IGNORE_CONTENT)
            {
              state = TAG_IGNORE_CONTENT[lastTag.name];
              break;
            }
          }

          state = TEXT;
          break;
        }

        if (m[2] == '=') // ATTRIBUTE_NAME_OR_END only
        {
          state = ATTRIBUTE_VALUE;
          break;
        }

        // m[2] == '\s+' next attr, state doesn't change
        token = {
          type: TYPE_ATTRIBUTE,
          range: {
            start_: pos,
            end_: pos
          }
        };
        state = ATTRIBUTE_NAME_OR_END;
        break;

      case COMMENT:
        token.value = source.substring(bufferPos, pos - 3);
        token.range.end_ = pos;
        state = TEXT;
        break;

      case REFERENCE:
        // add reference to token list name
        if (token.refs)
          token.refs.push(m[1]);
        else
          token.refs = [m[1]];

        // go next
        if (m[2] != '|') // m[2] == '}\s*'
        {
          if (token.type == TYPE_TEXT)
          {
            pos -= m[2].length - 1;
            token.value = source.substring(bufferPos, pos);
            token.range.end_ = pos;
            state = TEXT;
          }
          else if (token.type == TYPE_COMMENT)
          {
            state = COMMENT;
          }
          else if (token.type == TYPE_ATTRIBUTE && source[pos] == '=')
          {
            pos++;
            state = ATTRIBUTE_VALUE;
          }
          else // ATTRIBUTE || ELEMENT
          {
            token = {
              type: TYPE_ATTRIBUTE,
              range: {
                start_: pos,
                end_: pos
              }
            };
            state = ATTRIBUTE_NAME_OR_END;
          }
        }

        // continue to collect references
        break;

      case ATTRIBUTE_VALUE:
        token.value = m[1].replace(QUOTE_UNESCAPE, '"');
        token.range.end_ = pos;
        token.valueRange = {
          start_: startPos + 1,
          end_: startPos + 1 + m[1].length
        };

        token = {
          type: TYPE_ATTRIBUTE,
          range: {
            start_: pos,
            end_: pos
          }
        };
        state = ATTRIBUTE_NAME_OR_END;

        break;

      case TAG_IGNORE_CONTENT.text:
      case TAG_IGNORE_CONTENT.style:
        lastTag.children.push({
          type: TYPE_TEXT,
          value: m[1],
          range: {
            start_: startPos,
            end_: startPos + m[1].length
          }
        });

        lastTag = tagStack.pop();

        state = TEXT;
        break;

      default:
        throw SYNTAX_ERROR; // Must never to be here; bug in parser otherwise
    }

    if (state == TEXT)
      textStateEndPos = pos;
  }

  if (textStateEndPos != pos)
    lastTag.children.push({
      type: TYPE_TEXT,
      value: source.substring(textStateEndPos, pos),
      range: {
        start_: textStateEndPos,
        end_: pos
      }
    });

  // process attributes binding, join text tokens, normalize ranges and add location info
  postProcessing(result, options || {}, source);

  /** @cut */ if (lastTag.name)
  /** @cut */   result.warns.push(['No close tag for <' + (lastTag.prefix ? lastTag.prefix + ':' : '') + lastTag.name + '>', lastTag]);
  /** @cut */
  /** @cut */ if (!result.warns.length)
  /** @cut */   result.warns = false;

  result.templateTokens = true;

  return result;
}
359830
public static String[] tokenize(final String source, final char delimiter) {
		final Vector<String> v = tokenize2vector(source, delimiter);
		final String[] ret = new String[v.size()];
		for (int i = 0; i < v.size(); i++) {
			ret[i] = v.elementAt(i);
		}
		return ret;
	}
399808
def process_tokens(self, tokens):
        
        for tok_type, token, (start_row, start_col), _, _ in tokens:
            if tok_type == tokenize.STRING:
                # 'token' is the whole un-parsed token; we can look at the start
                # of it to see whether it's a raw or unicode string etc.
                self._process_string_token(token, start_row, start_col)
63015
def main(argv=None):
    ''''''

    t = CrfTokenizer()
    print t.tokenize("This is a sentence.")
    print t.tokenize("Buy???This...Now!!!")
    print t.tokenize("The <bold>only</bold> source.")
    print t.tokenize("The<bold>only</bold>source.")
    print t.tokenize("Big&gt;little.")
    print t.tokenize("Big & little.")
    print t.tokenize("blond&curly.")
    print t.tokenize("&brokenHtml")
    t.setGroupPunctuation(True)
    t.setRecognizeHtmlTags(True)
    t.setRecognizeHtmlEntities(True)
    print t.tokenize("Buy???This...Now!!!")
    print t.tokenize("The <bold>only</bold> source.")
    print t.tokenize("The<bold>only</bold>source.")
    print t.tokenize("Big&gt;little.")
    print t.tokenize("Big & little.")
    print t.tokenize("blond&curly.")
    print t.tokenize("&brokenHtml")
    t.setSkipHtmlTags(True)
    t.setSkipHtmlEntities(True)
    print t.tokenize("Buy???This...Now!!!")
    print t.tokenize("The <bold>only</bold> source.")
    print t.tokenize("The<bold>only</bold>source.")
    print t.tokenize("Big&gt;little.")
    print t.tokenize("Big & little.")
    print t.tokenize("blond&curly.")
    print t.tokenize("&brokenHtml")
    t.setTokenPrefix("X:")
    print t.tokenize("Tokenize with prefixes.")
    t.setTokenPrefix(None)
    print t.tokenize("No more  prefixes.")
    t.setRecognizePunctuation(False)
    print t.tokenize("This is a sentence.")
    print t.tokenize("Buy???This...Now!!!")
    print t.tokenize("The <bold>only</bold> source.")
    print t.tokenize("The<bold>only</bold>source.")
    print t.tokenize("Big&gt;little.")
    print t.tokenize("Big & little.")
    print t.tokenize("blond&curly.")
    print t.tokenize("&brokenHtml")
    print t.tokenize("A line break goes here\n\t \rand a new line starts")
    t.setRecognizeLinebreaks(True)
    print t.tokenize("A line break goes here\n\r \rand a new line starts")
1366123
public void train(StringWrapperIterator i0) 
	{
            SourcedStringWrapperIterator i = (SourcedStringWrapperIterator)i0;
            Set seenTokens = new HashSet();
            while (i.hasNext()) {
                BagOfSourcedTokens bag = asBagOfSourcedTokens(i.nextSourcedStringWrapper());
                seenTokens.clear();
                for (Iterator j=bag.tokenIterator(); j.hasNext(); ) {
                    totalTokenCount++;
                    Token tokj = (Token)j.next();
                    if (!seenTokens.contains(tokj)) {
                        seenTokens.add(tokj);
                        // increment documentFrequency counts
                        Integer df = (Integer)documentFrequency.get(tokj);
                        if (df==null) documentFrequency.put(tokj,ONE); 
                        else if (df==ONE) documentFrequency.put(tokj,TWO);
                        else if (df==TWO) documentFrequency.put(tokj,THREE);
                        else documentFrequency.put(tokj, new Integer(df.intValue()+1));
                    }
                }
                collectionSize++;
            }
	}
840200
def _generate_tokens(self, text):
    
    # This is technically an undocumented API for Python3, but allows us to use the same API as for
    # Python2. See http://stackoverflow.com/a/4952291/328565.
    for index, tok in enumerate(tokenize.generate_tokens(io.StringIO(text).readline)):
      tok_type, tok_str, start, end, line = tok
      yield Token(tok_type, tok_str, start, end, line, index,
                  self._line_numbers.line_to_offset(start[0], start[1]),
                  self._line_numbers.line_to_offset(end[0], end[1]))
882574
private void process(Iterator<String> tokens) {
        // NOTE: this method is intentionally private to ensure that the
        // IteratorFactory.tokenize() tokenization scheme is enforced on the
        // input data
        long numTokens = 0;
        while (tokens.hasNext()) {
            String token = tokens.next();
            if (doLowerCasing)
                token = token.toLowerCase();
            if (token.matches("[0-9]+"))
                token = "<NUM>";
            if (token.matches("[^\\w\\s;:\\(\\)\\[\\]'!/&?\",\\.<>]"))
                continue;

            Integer count = tokenToCount.get(token);
            tokenToCount.put(token, (count == null) ? 1 : 1 + count);
            numTokens++;
            if (numTokens % UPDATE_INTERVAL == 0 
                    && LOGGER.isLoggable(Level.FINE))
                LOGGER.fine("Processed " + numTokens + " tokens.  Currently " 
                            + tokenToCount.size() + " unique tokens");
        }
    }
1207287
function(source) {
    var proto = Lexer.prototype;
    var tokens = source.match(proto.regs.token);
    for (var i = 0, l = tokens.length; i < l; i ++) {
        if (proto.regs.strip.test(tokens[i])) {
            tokens.splice(i, 1);
        }
    }
    for (var n in proto) tokens[n] = proto[n];
    tokens.index = 0;
    return tokens;
}
1299759
@Override
    protected Source parseFrom( TokenStream tokens,
                                TypeSystem typeSystem ) {
        Position firstSourcePosition = tokens.nextPosition();
        Source source = super.parseFrom(tokens, typeSystem);
        if (tokens.matches(',') && source instanceof NamedSelector) {
            NamedSelector selector = (NamedSelector)source;
            JoinableSources joinedSources = new JoinableSources(selector, firstSourcePosition);
            while (tokens.canConsume(',')) {
                // This is a JCR-SQL-style JOIN ...
                Position nextSourcePosition = tokens.nextPosition();
                NamedSelector nextSource = parseNamedSelector(tokens, typeSystem);
                joinedSources.add(nextSource, nextSourcePosition);
            }
            source = joinedSources;
        }
        return source;
    }
88598
def new_from
      source_tokens = [ :STRING ]
      from_tok = Tree::From.new(basic_options)

      # read in all the derived columns - search sources:
      while peek_token and source_tokens.include? peek_token.token
        src_token = read_token
        # check if the provider is registered in the given list:
        if provider_exists? src_token.lexeme.downcase
          from_tok.sources[src_token.lexeme] = @providers[src_token.lexeme.downcase.to_sym]
        else raise UnregisteredProviderError, "#{src_token.lexeme} is not a registered provider"
        end
      end

      if from_tok.sources.count == 0
        raise SyntaxError, "No search sources given (#{error_location})"
      end

      from_tok
    end