Make a token that has no content
356292
def _make_marker_token(self, type_):
        """"""
        tok = Token(type_,
                    '',
                    self.line,
                    self.line_num,
                    self.start,
                    self.start)
        return tok
550443
def parse_block_scalar_empty_line(indent_token_class, content_token_class):
        """"""
        def callback(lexer, match, context):
            text = match.group()
            if (context.block_scalar_indent is None or
                    len(text) <= context.block_scalar_indent):
                if text:
                    yield match.start(), indent_token_class, text
            else:
                indentation = text[:context.block_scalar_indent]
                content = text[context.block_scalar_indent:]
                yield match.start(), indent_token_class, indentation
                yield (match.start()+context.block_scalar_indent,
                       content_token_class, content)
            context.pos = match.end()
        return callback
568862
function createEmptyContent(opts: Options): Block {
    return Block.create({
        type: opts.typeContent,
        nodes: [Text.create()]
    });
}
820765
def emptyArray(key, add_label=None):
    """"""

    result = {
        'key': key,
        'startEmpty': True
    }
    if add_label is not None:
        result['add'] = add_label
        result['style'] = {'add': 'btn-success'}
    return result
506378
function stripToken(val, exp) {
    exp = exp || constants_2.TOKEN_ANY_EXP;
    return val.trim().replace(exp, '');
}
218224
function process_raw(token) {
	var token = escapeRegex(token);
	return {
		type: "raw",
		piece: token + "/",
		last: token + "/?"
	};
}
354990
function() {
    var captures;
    if (captures = /^-*[_a-zA-Z$][-\w\d$]*/.exec(this.str)) {
      this.skip(captures);
      return new Token('ident', new nodes.Ident(captures[0]));
    }
  }
963409
def something(TokenClass):
    """"""
    def callback(lexer, match, context):
        text = match.group()
        if not text:
            return
        yield match.start(), TokenClass, text
        context.pos = match.end()
    return callback
886761
function(context: ParserContext): ?Token {
    const delimiter = context.reader.peek();
    if (delimiter !== '"' && delimiter !== "'") return null;
    if (!isInsideOpenBracket(context)) return null;

    // read until the delimiter
    let stringValue = context.reader.read();
    while (!context.reader.isEOF()) {
      const next = context.reader.read();
      stringValue += next;

      if (next === delimiter) break;
    }

    return context.createToken("string", stringValue);
  }
771994
def generate_tokens(self, text):
        """"""
        if text != self.last_text:
            string_io = io.StringIO(text)
            self.last_tokens = list(
                tokenize.generate_tokens(string_io.readline)
            )
            self.last_text = text
        return self.last_tokens
955808
private void skipEmptyLines() {
		int tmp = index;
		do {
			tmp = skipLineSpace(tmp);
			if (tmp < tokens.size() && tokens.get(tmp).kind != Token.Kind.NewLine) {
				return; // done
			} else if (tmp >= tokens.size()) {
				index = tmp;
				return; // end-of-file reached
			}
			// otherwise, skip newline and continue
			tmp = tmp + 1;
			index = tmp;
		} while (true);
		// deadcode
	}
1072412
private void append(Token token, String content) {
    if (content.isEmpty()) {
      throw new IllegalStateException(
          String.format(
              "shouldn't append empty content: %s @ %s",
              SoyFileParserConstants.tokenImage[token.kind], Tokens.createSrcLoc(fileName, token)));
    }
    // add a new offset if:
    // - this is the first token
    // - the previous token introduced a discontinuity (due to a special token, or whitespace
    //   joining)
    // - this token doesn't directly abut the previous token (this happens when there is a comment)
    boolean addOffset = false;
    if (offsets.isEmpty()) {
      addOffset = true;
    } else if (discontinuityReason != Reason.NONE) {
      addOffset = true;
    } else {
      // are the two tokens not adjacent? We don't actually record comments in the AST or token
      // stream so this is kind of a guess, but all known cases are due to comments.
      if (offsets.endLine() == token.beginLine) {
        if (offsets.endColumn() + 1 != token.beginColumn) {
          addOffset = true;
          discontinuityReason = Reason.COMMENT;
        }
      } else if (offsets.endLine() + 1 == token.beginLine && token.beginColumn != 1) {
        addOffset = true;
        discontinuityReason = Reason.COMMENT;
      }
    }
    if (addOffset) {
      offsets.add(buffer.length(), token.beginLine, token.beginColumn, discontinuityReason);
      discontinuityReason = Reason.NONE;
    }
    offsets.setEndLocation(token.endLine, token.endColumn);
    buffer.append(content);
  }
1174044
function formatReportedHTMLToken(token) {
            const valueLines = sourceCode.getText(token).split("\n")
            const value = valueLines[0]
            return value
        }
45353
function getHeaderContent(tokens, type, markup) {
  if (tokens[1].children.length === 0) {
    tokens[1].children.push(getEmptyText());
  } else if (tokens[1].children[0].type !== 'text') {
    tokens[1].children.unshift(getEmptyText());
  }
  const content = changeText(tokens[1].children, markup);
  return {
    type,
    content,
    length: getTokensLength(content)
  };
}
395010
function tokenize(content, rules = parseOrder) {
    let data = []
    if(content) {
        let lines = content.split(/(\r\n|\n)/)
        for (let idx = 0; idx < lines.length; idx++) {
            for (let i = 0; i < lines[idx].length;) {
                let chunk = lines[idx].slice(i)
                let match = rules.some(item => {
                    if (item.pattern.test(chunk)) {
                        let matchObject = chunk.match(item.pattern)
                        i += matchObject[0].length
                        data.push([ item.label, matchObject[0] ])
                        return true
                    }
                })
                if (!match) {
                    data.push([ UNKNOWN, chunk[i] ])
                    i++
                }
            }
        }
    }
    return data
}
302459
function tokenize(text) {
  const rawTokens = text.split(/[\s\/]+/mg);
  const tokens = [];
  rawTokens.forEach(token => {
    // Strip off unwanted trivial characters
    token = token
        .trim()
        .replace(/^[_\-"'`({[<$*)}\]>.]+/, '')
        .replace(/[_\-"'`({[<$*)}\]>.]+$/, '');
    // Ignore tokens that contain weird characters
    if (/^[\w.\-]+$/.test(token)) {
      tokens.push(token.toLowerCase());
      const ngTokenMatch = /^[nN]g([A-Z]\w*)/.exec(token);
      if (ngTokenMatch) {
        tokens.push(ngTokenMatch[1].toLowerCase());
      }
    }
  });
  return tokens;
}
462914
def parse_block_scalar_empty_line(IndentTokenClass, ContentTokenClass):
    """"""
    def callback(lexer, match, context):
        text = match.group()
        if (context.block_scalar_indent is None or
                len(text) <= context.block_scalar_indent):
            if text:
                yield match.start(), IndentTokenClass, text
        else:
            indentation = text[:context.block_scalar_indent]
            content = text[context.block_scalar_indent:]
            yield match.start(), IndentTokenClass, indentation
            yield (match.start()+context.block_scalar_indent,
                    ContentTokenClass, content)
        context.pos = match.end()
    return callback