/*
Parse the line for tokens to highlight
517610
private void checkForTokens( String content, int startOffset, int endOffset )
  {
    while( startOffset <= endOffset )
    {
      // skip the delimiters to find the start of a new token
      while( isDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        if( isError( startOffset ) )
        {
          setCharacterAttributes( startOffset, 1, _error, true );
        }
        else if( isWarning( startOffset ) )
        {
          setCharacterAttributes( startOffset, 1, _warning, true );
        }

        if( startOffset < endOffset )
        {
          startOffset++;
        }
        else
        {
          return;
        }
      }
      // Extract and process the entire token
      if( isQuoteDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        startOffset = getQuoteToken( content, startOffset, endOffset );
      }
      else if( isCharQuoteDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        startOffset = getQuoteToken( content, startOffset, endOffset );
      }
      else
      {
        startOffset = getOtherToken( content, startOffset, endOffset );
      }
    }
  }
967777
function getTokens(css) {
    // Parse string, character by character:
    for (pos = 0; pos < css.length; col++, pos++) {
      c = css.charAt(pos);
      cn = css.charAt(pos + 1);

      // If we meet `/*`, it's a start of a multiline comment.
      // Parse following characters as a multiline comment:
      if (c === '/' && cn === '*') {
        parseMLComment(css);
      }

      // If we meet `//` and it is not a part of url:
      else if (!urlMode && c === '/' && cn === '/') {
        // If we're currently inside a block, treat `//` as a start
        // of identifier. Else treat `//` as a start of a single-line
        // comment:
        parseSLComment(css);
      }

      // If current character is a double or single quote, it's a start
      // of a string:
      else if (c === '"' || c === "'") {
        parseString(css, c);
      }

      // If current character is a space:
      else if (c === ' ') {
        parseSpaces(css);
      }

      // If current character is `=`, it must be combined with next `=`
      else if (c === '=' && cn === '=') {
        parseEquality(css);
      }

      // If we meet `!=`, this must be inequality
      else if (c === '!' && cn === '=') {
        parseInequality(css);
      }

      // If current character is a punctuation mark:
      else if (c in Punctuation) {
        // Check for CRLF here or just LF
        if (c === '\r' && cn === '\n' || c === '\n') {
          // If \r we know the next character is \n due to statement above
          // so we push a CRLF token type to the token list and importantly
          // skip the next character so as not to double count newlines or
          // columns etc
          if (c === '\r') {
            pushToken(TokenType.Newline, '\r\n', col);
            pos++; // If CRLF skip the next character and push crlf token
          } else if (c === '\n') {
            // If just a LF newline and not part of CRLF newline we can just
            // push punctuation as usual
            pushToken(Punctuation[c], c, col);
          }

          ln++; // Go to next line
          col = 0; // Reset the column count
        } else if (c !== '\r' && c !== '\n') {
          // Handle all other punctuation and add to list of tokens
          pushToken(Punctuation[c], c, col);
        }// Go to next line
        if (c === ')') urlMode = false; // Exit url mode
        else if (c === '\t' && tabSize > 1) col += (tabSize - 1);
      }

      // If current character is a decimal digit:
      else if (isDecimalDigit(c)) {
        parseDecimalNumber(css);
      }

      // If current character is anything else:
      else {
        parseIdentifier(css);
      }
    }

    return tokens;
  }
96962
function isLastTokenInCurrentLine(token) {
            const tokenAfter = sourceCode.getTokenAfter(token);

            return !(tokenAfter && astUtils.isTokenOnSameLine(token, tokenAfter));
        }
1151255
function Parser (options) {
  this.tokens = []
  this.token = null
  this.count = -1
  this.indent = []
  this.options = options || defaults
  this.options.renderer = this.options.renderer || new Renderer() // jshint ignore:line
  this.renderer = this.options.renderer
  this.renderer.options = this.options

  this.anchorMode = [
    MODE.GHOST,
    MODE.BITBUCKET,
    MODE.GITLAB,
    MODE.GITHUB,
    MODE.PANDOC,
    MODE.UNIFIED,
    MODE.MARKDOWNIT,
    MODE.MARKED
  ].map(k => this.options[k] && k).filter(k => k)[0]
  if (this.anchorMode === MODE.UNIFIED) {
    this.anchorMode = MODE.GITHUB
  } this._anchors = new Anchor(this.anchorMode)
}
820798
function highlight(text, lang, options){

    function makeSafe(str){
      return str
        .replace(/\r\n|\n\r|\r/g, '\n')
        .replace(/&/g, '&amp;')
        .replace(/</g, '&lt;');
    }

    function normalize(text){
      text = text
        // cut first empty lines
        .replace(/^(?:\s*[\n]+)+?([ \t]*)/, '$1')
        .trimRight();

      // fix empty strings
      text = text.replace(/\n[ \t]+\n/g, '\n\n');

      // normalize text offset
      var minOffset = 1000;
      var lines = text.split(/\n+/);
      var startLine = Number(text.match(/^function/) != null); // fix for function.toString()

      for (var i = startLine; i < lines.length; i++)
      {
        var m = lines[i].match(/^\s*/);
        if (m[0].length < minOffset)
          minOffset = m[0].length;
        if (minOffset == 0)
          break;
      }

      if (minOffset > 0)
        text = text.replace(new RegExp('(^|\\n) {' + minOffset + '}', 'g'), '$1');

      return text;
    }

    function defaultWrapper(line, idx){
      return (
        '<div class="line ' + (idx % 2 ? 'odd' : 'even') + lineClass + '">' +
          '<span class="lineContent">' +
            (!options.noLineNumber
              ? '<input class="lineNumber" value="' + lead(idx + 1, numberWidth) + '" type="none" unselectable="on" readonly="readonly" tabindex="-1" />' +
                '<span class="over"></span>'
              : ''
            ) +
            line + '\r\n' +
          '</span>' +
        '</div>'
      );
    }

    //  MAIN PART

    if (!options)
      options = {};

    // prepare text
    var rangeStart = -1;
    var rangeEnd = -1;
    var rangeName = '';

    if (options.range)
    {
      var left = makeSafe(text.substr(0, options.range[0]));
      var range = makeSafe(text.substring(options.range[0], options.range[1]));
      var right = makeSafe(text.substr(options.range[1]));

      rangeStart = left.length;
      rangeEnd = rangeStart + range.length;
      rangeName = options.range[2] || 'range';

      text = left + range + right;
    }
    else
    {
      text = makeSafe(text);
    }

    if (!options.keepFormat)
      text = normalize(text || '');

    var parser = LANG_PARSER[lang] || LANG_PARSER.text;
    var html = parser(text, rangeStart, rangeEnd, rangeName);

    var lines = html.split('\n');
    var numberWidth = String(lines.length).length;
    var lineClass = (options.noLineNumber ? '' : ' hasLineNumber');
    var offsetRx = new RegExp('^(<span class="' + rangeName + '">)?([ \\t]+)');

    lines = lines
      .map(function(line){
        return line.replace(offsetRx, function(m, rangeSpan, spaces){
          return (rangeSpan || '') + repeat('\xA0', spaces.replace(/\t/g, '  ').length);
        });
      })
      .map(options.wrapper || defaultWrapper);

    return options.lines ? lines : lines.join('');
  }
239507
function handleEscaping(tokens, token, index) {
	if (index === 0) {
		return [token];
	}

	const previousToken = tokens[index - 1];

	if (!previousToken.endsWith('\\')) {
		return [...tokens, token];
	}

	return [...tokens.slice(0, index - 1), `${previousToken.slice(0, -1)} ${token}`];
}
930570
function _parseLeafToken(token) {
  const [, name, remainder] = token.match(LEAF_REGEX);

  return {
    name: name.toLowerCase(),
    params: _parseParams(remainder),
  };
}
820649
def get_line_indent(token):
    """"""
    start = token.start_mark.buffer.rfind('\n', 0,
                                          token.start_mark.pointer) + 1
    content = start
    while token.start_mark.buffer[content] == ' ':
        content += 1
    return content - start
862303
function Parser(options) {
    this.tokens = [];
    this.token = null;
    this.options = options || marked.defaults;
    this.options.renderer = this.options.renderer || new Renderer();
    this.renderer = this.options.renderer;
    this.renderer.options = this.options;
  }
555417
function highlight() {
    $('pre code').each(function (i, block) {
      hljs.highlightBlock(block);
    });
    $('pre code[highlight-lines]').each(function (i, block) {
      if (block.innerHTML === "") return;
      var lines = block.innerHTML.split('\n');

      queryString = block.getAttribute('highlight-lines');
      if (!queryString) return;

      var ranges = queryString.split(',');
      for (var j = 0, range; range = ranges[j++];) {
        var found = range.match(/^(\d+)\-(\d+)?$/);
        if (found) {
          // consider region as `{startlinenumber}-{endlinenumber}`, in which {endlinenumber} is optional
          var start = +found[1];
          var end = +found[2];
          if (isNaN(end) || end > lines.length) {
            end = lines.length;
          }
        } else {
          // consider region as a sigine line number
          if (isNaN(range)) continue;
          var start = +range;
          var end = start;
        }
        if (start <= 0 || end <= 0 || start > end || start > lines.length) {
          // skip current region if invalid
          continue;
        }
        lines[start - 1] = '<span class="line-highlight">' + lines[start - 1];
        lines[end - 1] = lines[end - 1] + '</span>';
      }

      block.innerHTML = lines.join('\n');
    });
  }
1272927
function highlightTokens(defs: Object, text: string) {
  return text.replace(jsTokens, function(...args) {
    const type = getTokenType(args);
    const colorize = defs[type];
    if (colorize) {
      return args[0]
        .split(NEWLINE)
        .map(str => colorize(str))
        .join("\n");
    } else {
      return args[0];
    }
  });
}
46386
function tokenizer(input) {
  const topLevelTokens = getTopLevelTokens();

  const length = input.length;

  const output = [];

  let cursor = 0;
  let lastBreak = 0;

  while (cursor < length) {
    const slice = input.slice(cursor);
    const found = matchPattern(topLevelTokens, slice, false);

    if (found && input[cursor - 1] === '\\') {
      const text = input.slice(lastBreak, cursor - 1);
      if (text) {
        output.push(new Text(text));
      }

      const escapedText = found[1][0];
      output.push(new Text(escapedText));

      cursor += escapedText.length;
      lastBreak = cursor;
    } else if (found) {
      const [Tok, matches] = found;

      const text = input.slice(lastBreak, cursor);
      if (text) {
        output.push(new Text(text));
      }
      output.push(new Tok(...matches));

      cursor += matches[0].length;
      lastBreak = cursor;
    } else {
      cursor += 1;
    }
  }
  const text = input.slice(lastBreak, cursor);
  if (text) {
    output.push(new Text(text));
  }

  // if there are more closes than opens
  // intelligently remove extra ones
  return removeExtraCloses(output);
}
753896
private void processIgnoredLeadingTokens(ParseTreeNode node, int position, int line, MemoEntry progress)
	    throws TreeException, ParserException {
	if (ignoredLeading) {
	    processIgnoredTokens(node, position, line, progress);
	}
    }
823966
def parse_line(line):
  
  columns = line.split()
  token = columns.pop(0)
  values = [float(column) for column in columns]
  return token, values
895971
function getActualLineIndentText(line) {
            let actualText = actualLineIndentTexts.get(line)
            if (actualText === undefined) {
                const lineText = getLineText(line)
                const index = lineText.search(/\S/u)
                if (index >= 0) {
                    actualText = lineText.slice(0, index)
                } else {
                    actualText = lineText
                }
                actualLineIndentTexts.set(line, actualText)
            }
            return actualText
        }
709290
function parse(lines) {
  var i, line, key, parts, values;

  for(i = 0;i < lines.length;i++) {
    line = lines[i];
    this.lineNumber++;

    // ignore comments and whitespace lines
    if(COMMENT.test(line) || WHITESPACE.test(line)) {
      continue;
    // got something to parse
    }else{

      // strip empty entires, leading whitespace is allowed
      parts = line.split(/\s+/).filter(function(p) {
        return p;
      })

      // get the config key
      key = parts[0];

      // remaining parts to parse as values
      values = parts.slice(1);
      try {
        this.conf.add(key, values, this.lineNumber);
      }catch(e) {
        throw new ConfigError(this.lineNumber, line, this.file, e);
      }
    }
  }
}
837066
private void handleToken(String token) throws ParseException
    {
        currentToken = token;

        if (skipParsing)
        {
            cmd.addArg(token);
        }
        else if ("--".equals(token))
        {
            skipParsing = true;
        }
        else if (currentOption != null && currentOption.acceptsArg() && isArgument(token))
        {
            currentOption.addValueForProcessing(Util.stripLeadingAndTrailingQuotes(token));
        }
        else if (token.startsWith("--"))
        {
            handleLongOption(token);
        }
        else if (token.startsWith("-") && !"-".equals(token))
        {
            handleShortAndLongOption(token);
        }
        else
        {
            handleUnknownToken(token);
        }

        if (currentOption != null && !currentOption.acceptsArg())
        {
            currentOption = null;
        }
    }