Construct a tokenizer.  This creates both `tokenizeInline` and `tokenizeBlock`.
12535
function factory(type) {
  return tokenize

  // Tokenizer for a bound `type`.
  function tokenize(value, location) {
    var self = this
    var offset = self.offset
    var tokens = []
    var methods = self[type + 'Methods']
    var tokenizers = self[type + 'Tokenizers']
    var line = location.line
    var column = location.column
    var index
    var length
    var method
    var name
    var matched
    var valueLength

    // Trim white space only lines.
    if (!value) {
      return tokens
    }

    // Expose on `eat`.
    eat.now = now
    eat.file = self.file

    // Sync initial offset.
    updatePosition('')

    // Iterate over `value`, and iterate over all tokenizers.  When one eats
    // something, re-iterate with the remaining value.  If no tokenizer eats,
    // something failed (should not happen) and an exception is thrown.
    while (value) {
      index = -1
      length = methods.length
      matched = false

      while (++index < length) {
        name = methods[index]
        method = tokenizers[name]

        if (
          method &&
          /* istanbul ignore next */ (!method.onlyAtStart || self.atStart) &&
          (!method.notInList || !self.inList) &&
          (!method.notInBlock || !self.inBlock) &&
          (!method.notInLink || !self.inLink)
        ) {
          valueLength = value.length

          method.apply(self, [eat, value])

          matched = valueLength !== value.length

          if (matched) {
            break
          }
        }
      }

      /* istanbul ignore if */
      if (!matched) {
        self.file.fail(new Error('Infinite loop'), eat.now())
      }
    }

    self.eof = now()

    return tokens

    // Update line, column, and offset based on `value`.
    function updatePosition(subvalue) {
      var lastIndex = -1
      var index = subvalue.indexOf('\n')

      while (index !== -1) {
        line++
        lastIndex = index
        index = subvalue.indexOf('\n', index + 1)
      }

      if (lastIndex === -1) {
        column += subvalue.length
      } else {
        column = subvalue.length - lastIndex
      }

      if (line in offset) {
        if (lastIndex !== -1) {
          column += offset[line]
        } else if (column <= offset[line]) {
          column = offset[line] + 1
        }
      }
    }

    // Get offset.  Called before the first character is eaten to retrieve the
    // range’s offsets.
    function getOffset() {
      var indentation = []
      var pos = line + 1

      // Done.  Called when the last character is eaten to retrieve the range’s
      // offsets.
      return function() {
        var last = line + 1

        while (pos < last) {
          indentation.push((offset[pos] || 0) + 1)

          pos++
        }

        return indentation
      }
    }

    // Get the current position.
    function now() {
      var pos = {line: line, column: column}

      pos.offset = self.toOffset(pos)

      return pos
    }

    // Store position information for a node.
    function Position(start) {
      this.start = start
      this.end = now()
    }

    // Throw when a value is incorrectly eaten.  This shouldn’t happen but will
    // throw on new, incorrect rules.
    function validateEat(subvalue) {
      /* istanbul ignore if */
      if (value.substring(0, subvalue.length) !== subvalue) {
        // Capture stack-trace.
        self.file.fail(
          new Error(
            'Incorrectly eaten value: please report this warning on https://git.io/vg5Ft'
          ),
          now()
        )
      }
    }

    // Mark position and patch `node.position`.
    function position() {
      var before = now()

      return update

      // Add the position to a node.
      function update(node, indent) {
        var prev = node.position
        var start = prev ? prev.start : before
        var combined = []
        var n = prev && prev.end.line
        var l = before.line

        node.position = new Position(start)

        // If there was already a `position`, this node was merged.  Fixing
        // `start` wasn’t hard, but the indent is different.  Especially
        // because some information, the indent between `n` and `l` wasn’t
        // tracked.  Luckily, that space is (should be?) empty, so we can
        // safely check for it now.
        if (prev && indent && prev.indent) {
          combined = prev.indent

          if (n < l) {
            while (++n < l) {
              combined.push((offset[n] || 0) + 1)
            }

            combined.push(before.column)
          }

          indent = combined.concat(indent)
        }

        node.position.indent = indent || []

        return node
      }
    }

    // Add `node` to `parent`s children or to `tokens`.  Performs merges where
    // possible.
    function add(node, parent) {
      var children = parent ? parent.children : tokens
      var prev = children[children.length - 1]
      var fn

      if (
        prev &&
        node.type === prev.type &&
        (node.type === 'text' || node.type === 'blockquote') &&
        mergeable(prev) &&
        mergeable(node)
      ) {
        fn = node.type === 'text' ? mergeText : mergeBlockquote
        node = fn.call(self, prev, node)
      }

      if (node !== prev) {
        children.push(node)
      }

      if (self.atStart && tokens.length !== 0) {
        self.exitStart()
      }

      return node
    }

    // Remove `subvalue` from `value`.  `subvalue` must be at the start of
    // `value`.
    function eat(subvalue) {
      var indent = getOffset()
      var pos = position()
      var current = now()

      validateEat(subvalue)

      apply.reset = reset
      reset.test = test
      apply.test = test

      value = value.substring(subvalue.length)

      updatePosition(subvalue)

      indent = indent()

      return apply

      // Add the given arguments, add `position` to the returned node, and
      // return the node.
      function apply(node, parent) {
        return pos(add(pos(node), parent), indent)
      }

      // Functions just like apply, but resets the content: the line and
      // column are reversed, and the eaten value is re-added.   This is
      // useful for nodes with a single type of content, such as lists and
      // tables.  See `apply` above for what parameters are expected.
      function reset() {
        var node = apply.apply(null, arguments)

        line = current.line
        column = current.column
        value = subvalue + value

        return node
      }

      // Test the position, after eating, and reverse to a not-eaten state.
      function test() {
        var result = pos({})

        line = current.line
        column = current.column
        value = subvalue + value

        return result.position
      }
    }
  }
}
1087489
function processIdentifiers(tokenizer, scope) {
  let params = [];

  if (tokenizer.token.match(types.IDENTIFIER)) {
    const keys = [tokenizer.token.value];

    tokenizer.skipToken();

    while (tokenizer.token.match(types.DOT, types.OPEN_SQUARE_BRACKET)) {
      const token = tokenizer.token;

      tokenizer.skipToken();

      const isValidKey = token.match(types.OPEN_SQUARE_BRACKET) ?
        tokenizer.token.match(types.CONSTANT) :
        (
          (tokenizer.token.match(types.CONSTANT) && (typeof tokenizer.token.value === 'number')) ||
          tokenizer.token.match(types.IDENTIFIER)
        );

      if (!isValidKey) {
        throw new SyntaxError('Unexpected object key notation');
      }

      keys.push(tokenizer.token.value);
      tokenizer.skipToken();

      if (token.match(types.OPEN_SQUARE_BRACKET)) {
        if (!tokenizer.token.match(types.CLOSE_SQUARE_BRACKET)) {
          throw new SyntaxError('Closing square bracket expected');
        }

        tokenizer.skipToken();
      }
    }

    const identifier = keys.join('.');

    if (!tokenizer.token.match(types.OPEN_PARENTHESES)) {
      scope.identifiers.push(identifier);

      return createIdentifierNode(identifier);
    }

    params = [];
    tokenizer.skipToken();
    if (!tokenizer.token.match(types.CLOSE_PARENTHESES)) {
      params.push(processLogicalOr(tokenizer, scope));

      while (tokenizer.token.match(types.DELIMITER)) {
        params.push(processLogicalOr(tokenizer.skipToken(), scope));
      }
    }

    if (!tokenizer.token.match(types.CLOSE_PARENTHESES)) {
      throw new SyntaxError('Unexpected end of expression');
    }

    tokenizer.skipToken();
    return createOperatorNode(identifier, params);
  }

  return processConstants(tokenizer, scope);
}
981307
def new_text_block(self, **kwargs):
        """"""
        proto = {'content': '', 'type': self.markdown}
        proto.update(**kwargs)
        return proto
993419
def build_tokens_line(self):
        """"""
        logical = []
        comments = []
        length = 0
        prev_row = prev_col = mapping = None
        for token_type, text, start, end, line in self.tokens:
            if token_type in SKIP_TOKENS:
                continue
            if not mapping:
                mapping = [(0, start)]
            if token_type == tokenize.COMMENT:
                comments.append(text)
                continue
            if token_type == tokenize.STRING:
                text = mute_string(text)
            if prev_row:
                (start_row, start_col) = start
                if prev_row != start_row:    # different row
                    prev_text = self.lines[prev_row - 1][prev_col - 1]
                    if prev_text == ',' or (prev_text not in '{[(' and
                                            text not in '}])'):
                        text = ' ' + text
                elif prev_col != start_col:  # different column
                    text = line[prev_col:start_col] + text
            logical.append(text)
            length += len(text)
            mapping.append((length, end))
            (prev_row, prev_col) = end
        self.logical_line = ''.join(logical)
        self.noqa = comments and noqa(''.join(comments))
        return mapping
715002
def iter_token_lines(tokenlist):
    
    line = []
    for token, c in explode_tokens(tokenlist):
        line.append((token, c))

        if c == '\n':
            yield line
            line = []

    yield line
87829
private static Block block(final String comment, final int nameStart) {
      int attrIndex = comment.indexOf('{');
      if(attrIndex != -1) {
         return new Block(comment.substring(nameStart, attrIndex).trim(),
                 comment.substring(attrIndex).trim());
      } else {
         return new Block(comment.substring(nameStart).trim());
      }
   }
404939
def parse_block(self, block, lineno, indent):
        
        tree = self.driver.parse_tokens(self.wrap_toks(block, lineno, indent))
        tree.future_features = frozenset()
        return tree
1290334
function ParserInline() {
  var i;

  /**
   * ParserInline#ruler -> Ruler
   *
   * [[Ruler]] instance. Keep configuration of inline rules.
   **/
  this.ruler = new Ruler();

  for (i = 0; i < _rules.length; i++) {
    this.ruler.push(_rules[i][0], _rules[i][1]);
  }

  /**
   * ParserInline#ruler2 -> Ruler
   *
   * [[Ruler]] instance. Second ruler used for post-processing
   * (e.g. in emphasis-like rules).
   **/
  this.ruler2 = new Ruler();

  for (i = 0; i < _rules2.length; i++) {
    this.ruler2.push(_rules2[i][0], _rules2[i][1]);
  }
}
114753
def getblock(lines):
    """"""
    try:
        tokenize.tokenize(ListReader(lines).readline, BlockFinder().tokeneater)
    except EndOfBlock, eob:
        return lines[:eob.args[0]]
    # Fooling the indent/dedent logic implies a one-line definition
    return lines[:1]
1321558
def span_tokenize(self, s):
        
        if self._tokenizer is None:
            self._tokenizer = load_model(self.model)
        # for debug in tokenizer.debug_decisions(s):
        #     log.debug(format_debug_decision(debug))
        return self._tokenizer.span_tokenize(s)
1169106
def create_tokenizer(self, name, config=dict()):
        
        if name not in self.factories:
            raise KeyError(Errors.E002.format(name=name))
        factory = self.factories[name]
        return factory(self, **config)
698149
private Block wrapInMacroMarker(MacroBlock macroBlockToWrap, List<Block> newBlocks)
    {
        return new MacroMarkerBlock(macroBlockToWrap.getId(), macroBlockToWrap.getParameters(),
            macroBlockToWrap.getContent(), newBlocks, macroBlockToWrap.isInline());
    }
659824
function ( tokenizer ) {
	var start, string, escaped, unescaped, next;

	start = tokenizer.pos;

	string = '';

	escaped = getEscapedChars( tokenizer );
	if ( escaped ) {
		string += escaped;
	}

	unescaped = getUnescapedDoubleQuotedChars( tokenizer );
	if ( unescaped ) {
		string += unescaped;
	}

	if ( !string ) {
		return '';
	}

	next = getDoubleQuotedString( tokenizer );
	while ( next !== '' ) {
		string += next;
	}

	return string;
}
197048
def tokenize(self, s):
        
        javabridge.call(self.jobject, "tokenize", "(Ljava/lang/String;)V", s)
        return TokenIterator(self)
272356
function defineFix (token, actualIndent, expectedIndent) {
    if (token.type === 'Block' && token.loc.start.line !== token.loc.end.line) {
      // Fix indentation in multiline block comments.
      const lines = sourceCode.getText(token).match(LINES)
      const firstLine = lines.shift()
      if (lines.every(l => BLOCK_COMMENT_PREFIX.test(l))) {
        return fixer => {
          const range = [token.range[0] - actualIndent, token.range[1]]
          const indent = options.indentChar.repeat(expectedIndent)

          return fixer.replaceTextRange(
            range,
            `${indent}${firstLine}${lines.map(l => l.replace(BLOCK_COMMENT_PREFIX, `${indent} *`)).join('')}`
          )
        }
      }
    }

    return fixer => {
      const range = [token.range[0] - actualIndent, token.range[0]]
      const indent = options.indentChar.repeat(expectedIndent)
      return fixer.replaceTextRange(range, indent)
    }
  }
681857
def tokenize(stream, separator):
        
        for value in stream:
            for token in value.split(separator):
                if token:
                    yield token.strip()
430003
function tokenize() {
		var ch = walker.ch;
	
		if (ch === " " || ch === "\t") {
			return white();
		}

		if (ch === '/') {
			return comment();
		} 

		if (ch === '"' || ch === "'") {
			return str();
		}
		
		if (ch === '(') {
			return brace();
		}
	
		if (ch === '-' || ch === '.' || isDigit(ch)) { // tricky - char: minus (-1px) or dash (-moz-stuff)
			return num();
		}
	
		if (isNameChar(ch)) {
			return identifier();
		}

		if (isOp(ch)) {
			return op();
		}

		if (ch === '\r') {
			if (walker.peek() === '\n') {
				ch += walker.nextChar();
			}

			tokener(ch, 'line');
			walker.nextChar();
			return;
		}
		
		if (ch === '\n') {
			tokener(ch, 'line');
			walker.nextChar();
			return;
		}
		
		raiseError("Unrecognized character '" + ch + "'");
	}